Texture Cache: "Texture Groups" and "Texture Dependencies" (#2001)

* Initial implementation (3d tex mips broken)

This works rather well for most games, just need to fix 3d texture mips.

* Cleanup

* Address feedback

* Copy Dependencies and various other fixes

* Fix layer/level offset for copy from view<->view.

* Remove dirty flag from dependency

The dirty flag behaviour is not needed - DeferredCopy is all we need.

* Fix tracking mip slices.

* Propagate granularity (fix astral chain)

* Address Feedback pt 1

* Save slice sizes as part of SizeInfo

* Fix nits

* Fix disposing multiple dependencies causing a crash

This list is obviously modified when removing dependencies, so create a copy of it.

Ryujinx.Graphics.Gpu/Image/Texture.cs

@@ -50,6 +50,11 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </summary>
         public TextureScaleMode ScaleMode { get; private set; }
 
+        /// <summary>
+        /// Group that this texture belongs to. Manages read/write memory tracking.
+        /// </summary>
+        public TextureGroup Group { get; private set; }
+
         /// <summary>
         /// Set when a texture has been modified by the Host GPU since it was last flushed.
         /// </summary>
@@ -63,10 +68,11 @@ namespace Ryujinx.Graphics.Gpu.Image
 
         private int _depth;
         private int _layers;
-        private int _firstLayer;
-        private int _firstLevel;
+        public int FirstLayer { get; private set; }
+        public int FirstLevel { get; private set; }
 
         private bool _hasData;
+        private bool _dirty = true;
         private int _updateCount;
         private byte[] _currentData;
 
@@ -99,12 +105,20 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </summary>
         public MultiRange Range { get; private set; }
 
+        /// <summary>
+        /// Layer size in bytes.
+        /// </summary>
+        public int LayerSize => _sizeInfo.LayerSize;
+
         /// <summary>
         /// Texture size in bytes.
         /// </summary>
         public ulong Size => (ulong)_sizeInfo.TotalSize;
 
-        private GpuRegionHandle _memoryTracking;
+        /// <summary>
+        /// Whether or not the texture belongs is a view.
+        /// </summary>
+        public bool IsView => _viewStorage != this;
 
         private int _referenceCount;
 
@@ -131,8 +145,8 @@ namespace Ryujinx.Graphics.Gpu.Image
         {
             InitializeTexture(context, info, sizeInfo, range);
 
-            _firstLayer = firstLayer;
-            _firstLevel = firstLevel;
+            FirstLayer = firstLayer;
+            FirstLevel = firstLevel;
 
             ScaleFactor = scaleFactor;
             ScaleMode = scaleMode;
@@ -186,8 +200,6 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <param name="withData">True if the texture is to be initialized with data</param>
         public void InitializeData(bool isView, bool withData = false)
         {
-            _memoryTracking = _context.PhysicalMemory.BeginTracking(Range);
-
             if (withData)
             {
                 Debug.Assert(!isView);
@@ -203,12 +215,13 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
             else
             {
-                // Don't update this texture the next time we synchronize.
-                ConsumeModified();
                 _hasData = true;
 
                 if (!isView)
                 {
+                    // Don't update this texture the next time we synchronize.
+                    ConsumeModified();
+
                     if (ScaleMode == TextureScaleMode.Scaled)
                     {
                         // Don't need to start at 1x as there is no data to scale, just go straight to the target scale.
@@ -221,6 +234,18 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
         }
 
+        /// <summary>
+        /// Initialize a new texture group with this texture as storage.
+        /// </summary>
+        /// <param name="hasLayerViews">True if the texture will have layer views</param>
+        /// <param name="hasMipViews">True if the texture will have mip views</param>
+        public void InitializeGroup(bool hasLayerViews, bool hasMipViews)
+        {
+            Group = new TextureGroup(_context, this);
+
+            Group.Initialize(ref _sizeInfo, hasLayerViews, hasMipViews);
+        }
+
         /// <summary>
         /// Create a texture view from this texture.
         /// A texture view is defined as a child texture, from a sub-range of their parent texture.
@@ -240,8 +265,8 @@ namespace Ryujinx.Graphics.Gpu.Image
                 info,
                 sizeInfo,
                 range,
-                _firstLayer + firstLayer,
-                _firstLevel + firstLevel,
+                FirstLayer + firstLayer,
+                FirstLevel + firstLevel,
                 ScaleFactor,
                 ScaleMode);
 
@@ -259,11 +284,26 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <param name="texture">The child texture</param>
         private void AddView(Texture texture)
         {
-            DisableMemoryTracking();
+            IncrementReferenceCount();
 
             _views.Add(texture);
 
             texture._viewStorage = this;
+
+            Group.UpdateViews(_views);
+
+            if (texture.Group != null && texture.Group != Group)
+            {
+                if (texture.Group.Storage == texture)
+                {
+                    // This texture's group is no longer used.
+                    Group.Inherit(texture.Group);
+
+                    texture.Group.Dispose();
+                }
+            }
+
+            texture.Group = Group;
         }
 
         /// <summary>
@@ -276,7 +316,27 @@ namespace Ryujinx.Graphics.Gpu.Image
 
             texture._viewStorage = texture;
 
-            DeleteIfNotUsed();
+            DecrementReferenceCount();
+        }
+
+        /// <summary>
+        /// Create a copy dependency to a texture that is view compatible with this one.
+        /// When either texture is modified, the texture data will be copied to the other to keep them in sync.
+        /// This is essentially an emulated view, useful for handling multiple view parents or format incompatibility.
+        /// This also forces a copy on creation, to or from the given texture to get them in sync immediately.
+        /// </summary>
+        /// <param name="contained">The view compatible texture to create a dependency to</param>
+        /// <param name="layer">The base layer of the given texture relative to this one</param>
+        /// <param name="level">The base level of the given texture relative to this one</param>
+        /// <param name="copyTo">True if this texture is first copied to the given one, false for the opposite direction</param>
+        public void CreateCopyDependency(Texture contained, int layer, int level, bool copyTo)
+        {
+            if (contained.Group == Group)
+            {
+                return;
+            }
+
+            Group.CreateCopyDependency(contained, FirstLayer + layer, FirstLevel + level, copyTo);
         }
 
         /// <summary>
@@ -294,12 +354,12 @@ namespace Ryujinx.Graphics.Gpu.Image
             int blockWidth = Info.FormatInfo.BlockWidth;
             int blockHeight = Info.FormatInfo.BlockHeight;
 
-            width  <<= _firstLevel;
-            height <<= _firstLevel;
+            width  <<= FirstLevel;
+            height <<= FirstLevel;
 
             if (Target == Target.Texture3D)
             {
-                depthOrLayers <<= _firstLevel;
+                depthOrLayers <<= FirstLevel;
             }
             else
             {
@@ -310,14 +370,14 @@ namespace Ryujinx.Graphics.Gpu.Image
 
             foreach (Texture view in _viewStorage._views)
             {
-                int viewWidth  = Math.Max(1, width  >> view._firstLevel);
-                int viewHeight = Math.Max(1, height >> view._firstLevel);
+                int viewWidth  = Math.Max(1, width  >> view.FirstLevel);
+                int viewHeight = Math.Max(1, height >> view.FirstLevel);
 
                 int viewDepthOrLayers;
 
                 if (view.Info.Target == Target.Texture3D)
                 {
-                    viewDepthOrLayers = Math.Max(1, depthOrLayers >> view._firstLevel);
+                    viewDepthOrLayers = Math.Max(1, depthOrLayers >> view.FirstLevel);
                 }
                 else
                 {
@@ -328,16 +388,6 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
         }
 
-        /// <summary>
-        /// Disables memory tracking on this texture. Currently used for view containers, as we assume their views are covering all memory regions.
-        /// Textures with disabled memory tracking also cannot flush in most circumstances.
-        /// </summary>
-        public void DisableMemoryTracking()
-        {
-            _memoryTracking?.Dispose();
-            _memoryTracking = null;
-        }
-
         /// <summary>
         /// Recreates the texture storage (or view, in the case of child textures) of this texture.
         /// This allows recreating the texture with a new size.
@@ -393,7 +443,7 @@ namespace Ryujinx.Graphics.Gpu.Image
 
             if (_viewStorage != this)
             {
-                ReplaceStorage(_viewStorage.HostTexture.CreateView(createInfo, _firstLayer, _firstLevel));
+                ReplaceStorage(_viewStorage.HostTexture.CreateView(createInfo, FirstLayer, FirstLevel));
             }
             else
             {
@@ -495,7 +545,7 @@ namespace Ryujinx.Graphics.Gpu.Image
                     view.ScaleFactor = scale;
 
                     TextureCreateInfo viewCreateInfo = TextureManager.GetCreateInfo(view.Info, _context.Capabilities, scale);
-                    ITexture newView = HostTexture.CreateView(viewCreateInfo, view._firstLayer - _firstLayer, view._firstLevel - _firstLevel);
+                    ITexture newView = HostTexture.CreateView(viewCreateInfo, view.FirstLayer - FirstLayer, view.FirstLevel - FirstLevel);
 
                     view.ReplaceStorage(newView);
                     view.ScaleMode = newScaleMode;
@@ -517,17 +567,10 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// Checks if the memory for this texture was modified, and returns true if it was.
         /// The modified flags are consumed as a result.
         /// </summary>
-        /// <remarks>
-        /// If there is no memory tracking for this texture, it will always report as modified.
-        /// </remarks>
         /// <returns>True if the texture was modified, false otherwise.</returns>
         public bool ConsumeModified()
         {
-            bool wasDirty = _memoryTracking?.Dirty ?? true;
-
-            _memoryTracking?.Reprotect();
-
-            return wasDirty;
+            return Group.ConsumeDirty(this);
         }
 
         /// <summary>
@@ -544,17 +587,42 @@ namespace Ryujinx.Graphics.Gpu.Image
                 return;
             }
 
-            if (_hasData)
+            if (!_dirty)
             {
-                if (_memoryTracking?.Dirty != true)
-                {
-                    return;
-                }
-
-                BlacklistScale();
+                return;
             }
 
-            _memoryTracking?.Reprotect();
+            _dirty = false;
+
+            if (_hasData)
+            {
+                Group.SynchronizeMemory(this);
+            }
+            else
+            {
+                Group.ConsumeDirty(this);
+                SynchronizeFull();
+            }
+        }
+
+        /// <summary>
+        /// Signal that this texture is dirty, indicating that the texture group must be checked.
+        /// </summary>
+        public void SignalGroupDirty()
+        {
+            _dirty = true;
+        }
+
+        /// <summary>
+        /// Fully synchronizes guest and host memory. 
+        /// This will replace the entire texture with the data present in guest memory.
+        /// </summary>
+        public void SynchronizeFull()
+        {
+            if (_hasData)
+            {
+                BlacklistScale();
+            }
 
             ReadOnlySpan<byte> data = _context.PhysicalMemory.GetSpan(Range);
 
@@ -596,7 +664,7 @@ namespace Ryujinx.Graphics.Gpu.Image
         {
             BlacklistScale();
 
-            _memoryTracking?.Reprotect();
+            Group.ConsumeDirty(this);
 
             IsModified = false;
 
@@ -605,18 +673,46 @@ namespace Ryujinx.Graphics.Gpu.Image
             _hasData = true;
         }
 
+        /// <summary>
+        /// Uploads new texture data to the host GPU for a specific layer/level.
+        /// </summary>
+        /// <param name="data">New data</param>
+        /// <param name="layer">Target layer</param>
+        /// <param name="level">Target level</param>
+        public void SetData(ReadOnlySpan<byte> data, int layer, int level)
+        {
+            BlacklistScale();
+
+            HostTexture.SetData(data, layer, level);
+
+            _currentData = null;
+
+            _hasData = true;
+        }
+
         /// <summary>
         /// Converts texture data to a format and layout that is supported by the host GPU.
         /// </summary>
         /// <param name="data">Data to be converted</param>
         /// <returns>Converted data</returns>
-        private ReadOnlySpan<byte> ConvertToHostCompatibleFormat(ReadOnlySpan<byte> data)
+        public ReadOnlySpan<byte> ConvertToHostCompatibleFormat(ReadOnlySpan<byte> data, int level = 0, bool single = false)
         {
+            int width = Info.Width;
+            int height = Info.Height;
+
+            int depth = single ? 1 : _depth;
+            int layers = single ? 1 : _layers;
+            int levels = single ? 1 : Info.Levels;
+
+            width = Math.Max(width >> level, 1);
+            height = Math.Max(height >> level, 1);
+            depth = Math.Max(depth >> level, 1);
+
             if (Info.IsLinear)
             {
                 data = LayoutConverter.ConvertLinearStridedToLinear(
-                    Info.Width,
-                    Info.Height,
+                    width,
+                    height,
                     Info.FormatInfo.BlockWidth,
                     Info.FormatInfo.BlockHeight,
                     Info.Stride,
@@ -626,11 +722,11 @@ namespace Ryujinx.Graphics.Gpu.Image
             else
             {
                 data = LayoutConverter.ConvertBlockLinearToLinear(
-                    Info.Width,
-                    Info.Height,
-                    _depth,
-                    Info.Levels,
-                    _layers,
+                    width,
+                    height,
+                    depth,
+                    levels,
+                    layers,
                     Info.FormatInfo.BlockWidth,
                     Info.FormatInfo.BlockHeight,
                     Info.FormatInfo.BytesPerPixel,
@@ -650,11 +746,11 @@ namespace Ryujinx.Graphics.Gpu.Image
                     data.ToArray(),
                     Info.FormatInfo.BlockWidth,
                     Info.FormatInfo.BlockHeight,
-                    Info.Width,
-                    Info.Height,
-                    _depth,
-                    Info.Levels,
-                    _layers,
+                    width,
+                    height,
+                    depth,
+                    levels,
+                    layers,
                     out Span<byte> decoded))
                 {
                     string texInfo = $"{Info.Target} {Info.FormatInfo.Format} {Info.Width}x{Info.Height}x{Info.DepthOrLayers} levels {Info.Levels}";
@@ -666,11 +762,11 @@ namespace Ryujinx.Graphics.Gpu.Image
             }
             else if (Target == Target.Texture3D && Info.FormatInfo.Format.IsBc4())
             {
-                data = BCnDecoder.DecodeBC4(data, Info.Width, Info.Height, _depth, Info.Levels, _layers, Info.FormatInfo.Format == Format.Bc4Snorm);
+                data = BCnDecoder.DecodeBC4(data, width, height, depth, levels, layers, Info.FormatInfo.Format == Format.Bc4Snorm);
             }
             else if (Target == Target.Texture3D && Info.FormatInfo.Format.IsBc5())
             {
-                data = BCnDecoder.DecodeBC5(data, Info.Width, Info.Height, _depth, Info.Levels, _layers, Info.FormatInfo.Format == Format.Bc5Snorm);
+                data = BCnDecoder.DecodeBC5(data, width, height, depth, levels, layers, Info.FormatInfo.Format == Format.Bc5Snorm);
             }
 
             return data;
@@ -710,7 +806,7 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </summary>
         public void ExternalFlush(ulong address, ulong size)
         {
-            if (!IsModified || _memoryTracking == null)
+            if (!IsModified)
             {
                 return;
             }
@@ -869,7 +965,7 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <param name="firstLayer">Texture view initial layer on this texture</param>
         /// <param name="firstLevel">Texture view first mipmap level on this texture</param>
         /// <returns>The level of compatiblilty a view with the given parameters created from this texture has</returns>
-        public TextureViewCompatibility IsViewCompatible(TextureInfo info, MultiRange range, out int firstLayer, out int firstLevel)
+        public TextureViewCompatibility IsViewCompatible(TextureInfo info, MultiRange range, int layerSize, out int firstLayer, out int firstLevel)
         {
             int offset = Range.FindOffset(range);
 
@@ -892,15 +988,17 @@ namespace Ryujinx.Graphics.Gpu.Image
                 return TextureViewCompatibility.Incompatible;
             }
 
-            if (!TextureCompatibility.ViewFormatCompatible(Info, info))
+            if (info.GetSlices() > 1 && LayerSize != layerSize)
             {
                 return TextureViewCompatibility.Incompatible;
             }
 
             TextureViewCompatibility result = TextureViewCompatibility.Full;
 
+            result = TextureCompatibility.PropagateViewCompatibility(result, TextureCompatibility.ViewFormatCompatible(Info, info));
             result = TextureCompatibility.PropagateViewCompatibility(result, TextureCompatibility.ViewSizeMatches(Info, info, firstLevel));
             result = TextureCompatibility.PropagateViewCompatibility(result, TextureCompatibility.ViewTargetCompatible(Info, info));
+            result = TextureCompatibility.PropagateViewCompatibility(result, TextureCompatibility.ViewSubImagesInBounds(Info, info, firstLayer, firstLevel));
 
             return (Info.SamplesInX == info.SamplesInX &&
                     Info.SamplesInY == info.SamplesInY) ? result : TextureViewCompatibility.Incompatible;
@@ -1003,14 +1101,37 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// <param name="firstLevel">The first level of the view</param>
         public void ReplaceView(Texture parent, TextureInfo info, ITexture hostTexture, int firstLayer, int firstLevel)
         {
+            IncrementReferenceCount();
             parent._viewStorage.SynchronizeMemory();
+
+            // If this texture has views, they must be given to the new parent.
+            if (_views.Count > 0)
+            {
+                Texture[] viewCopy = _views.ToArray();
+
+                foreach (Texture view in viewCopy)
+                {
+                    TextureCreateInfo createInfo = TextureManager.GetCreateInfo(view.Info, _context.Capabilities, ScaleFactor);
+
+                    ITexture newView = parent.HostTexture.CreateView(createInfo, view.FirstLayer + firstLayer, view.FirstLevel + firstLevel); 
+
+                    view.ReplaceView(parent, view.Info, newView, view.FirstLayer + firstLayer, view.FirstLevel + firstLevel);
+                }
+            }
+
             ReplaceStorage(hostTexture);
 
-            _firstLayer = parent._firstLayer + firstLayer;
-            _firstLevel = parent._firstLevel + firstLevel;
+            if (_viewStorage != this)
+            {
+                _viewStorage.RemoveView(this);
+            }
+
+            FirstLayer = parent.FirstLayer + firstLayer;
+            FirstLevel = parent.FirstLevel + firstLevel;
             parent._viewStorage.AddView(this);
 
             SetInfo(info);
+            DecrementReferenceCount();
         }
 
         /// <summary>
@@ -1031,14 +1152,28 @@ namespace Ryujinx.Graphics.Gpu.Image
         /// </summary>
         public void SignalModified()
         {
-            IsModified = true;
-
-            if (_viewStorage != this)
+            bool wasModified = IsModified;
+            if (!wasModified || Group.HasCopyDependencies)
             {
-                _viewStorage.SignalModified();
+                IsModified = true;
+                Group.SignalModified(this, !wasModified);
             }
+        }
 
-            _memoryTracking?.RegisterAction(ExternalFlush);
+        /// <summary>
+        /// Signals that a texture has been bound, or has been unbound.
+        /// During this time, lazy copies will not clear the dirty flag.
+        /// </summary>
+        /// <param name="bound">True if the texture has been bound, false if it has been unbound</param>
+        public void SignalModifying(bool bound)
+        {
+            bool wasModified = IsModified;
+
+            if (!wasModified || Group.HasCopyDependencies)
+            {
+                IsModified = true;
+                Group.SignalModifying(this, bound, !wasModified);
+            }
         }
 
         /// <summary>
@@ -1066,7 +1201,7 @@ namespace Ryujinx.Graphics.Gpu.Image
 
             foreach (Texture view in _views)
             {
-                if (texture.IsViewCompatible(view.Info, view.Range, out _, out _) != TextureViewCompatibility.Incompatible)
+                if (texture.IsViewCompatible(view.Info, view.Range, view.LayerSize, out _, out _) != TextureViewCompatibility.Incompatible)
                 {
                     return true;
                 }
@@ -1148,10 +1283,6 @@ namespace Ryujinx.Graphics.Gpu.Image
         public void Unmapped()
         {
             IsModified = false; // We shouldn't flush this texture, as its memory is no longer mapped.
-
-            var tracking = _memoryTracking;
-            tracking?.Reprotect();
-            tracking?.RegisterAction(null);
         }
 
         /// <summary>
@@ -1162,7 +1293,11 @@ namespace Ryujinx.Graphics.Gpu.Image
             DisposeTextures();
 
             Disposed?.Invoke(this);
-            _memoryTracking?.Dispose();
+
+            if (Group.Storage == this)
+            {
+                Group.Dispose();
+            }
         }
     }
 }