Mirrors/Ryujinx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement support for multi-range buffers using Vulkan sparse mappings (#5427)

Browse source 
* Pass MultiRange to BufferManager

* Implement support for multi-range buffers using Vulkan sparse mappings

* Use multi-range for remaining buffers, delete old methods

* Assume that more buffers are contiguous

* Dispose multi-range buffers after they are removed from the list

* Properly init BufferBounds for constant and storage buffers

* Do not try reading zero bytes data from an unmapped address on the shader cache + PR feedback

* Fix misaligned sparse buffer offsets

* Null check can be simplified

* PR feedback
This commit is contained in:
gdkchan 2023-12-04 16:30:19 -03:00 • committed by GitHub
parent 0531c16326
commit 1df6c07f78
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
33 changed files with 1241 additions and 233 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
View file

@ -43,6 +43,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
public int UnmappedSequence { get; private set; }
/// <summary>
/// Indicates if the buffer can be used in a sparse buffer mapping.
/// </summary>
public bool SparseCompatible { get; }
/// <summary>
/// Ranges of the buffer that have been modified on the GPU.
/// Ranges defined here cannot be updated from CPU until a CPU waiting sync point is reached.
@ -77,15 +82,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="physicalMemory">Physical memory where the buffer is mapped</param>
/// <param name="address">Start address of the buffer</param>
/// <param name="size">Size of the buffer in bytes</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="baseBuffers">Buffers which this buffer contains, and will inherit tracking handles from</param>
public Buffer(GpuContext context, PhysicalMemory physicalMemory, ulong address, ulong size, IEnumerable<Buffer> baseBuffers = null)
public Buffer(
GpuContext context,
PhysicalMemory physicalMemory,
ulong address,
ulong size,
bool sparseCompatible,
IEnumerable<Buffer> baseBuffers = null)
{
_context = context;
_physicalMemory = physicalMemory;
Address = address;
Size = size;
SparseCompatible = sparseCompatible;
Handle = context.Renderer.CreateBuffer((int)size, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
BufferAccess access = sparseCompatible ? BufferAccess.SparseCompatible : BufferAccess.Default;
Handle = context.Renderer.CreateBuffer((int)size, access, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
_useGranular = size > GranularBufferThreshold;

23
src/Ryujinx.Graphics.Gpu/Memory/BufferBounds.cs
View file

@ -1,4 +1,5 @@
using Ryujinx.Graphics.Shader;
using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory
{
@ -8,30 +9,28 @@ namespace Ryujinx.Graphics.Gpu.Memory
readonly struct BufferBounds
{
/// <summary>
/// Region virtual address.
/// Physical memory ranges where the buffer is mapped.
/// </summary>
public ulong Address { get; }
/// <summary>
/// Region size in bytes.
/// </summary>
public ulong Size { get; }
public MultiRange Range { get; }
/// <summary>
/// Buffer usage flags.
/// </summary>
public BufferUsageFlags Flags { get; }
/// <summary>
/// Indicates that the backing memory for the buffer does not exist.
/// </summary>
public bool IsUnmapped => Range.IsUnmapped;
/// <summary>
/// Creates a new buffer region.
/// </summary>
/// <param name="address">Region address</param>
/// <param name="size">Region size</param>
/// <param name="range">Physical memory ranges where the buffer is mapped</param>
/// <param name="flags">Buffer usage flags</param>
public BufferBounds(ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None)
public BufferBounds(MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
{
Address = address;
Size = size;
Range = range;
Flags = flags;
}
}

575
src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
View file

@ -11,12 +11,24 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
class BufferCache : IDisposable
{
private const int OverlapsBufferInitialCapacity = 10;
private const int OverlapsBufferMaxCapacity = 10000;
/// <summary>
/// Initial size for the array holding overlaps.
/// </summary>
public const int OverlapsBufferInitialCapacity = 10;
/// <summary>
/// Maximum size that an array holding overlaps may have after trimming.
/// </summary>
public const int OverlapsBufferMaxCapacity = 10000;
private const ulong BufferAlignmentSize = 0x1000;
private const ulong BufferAlignmentMask = BufferAlignmentSize - 1;
/// <summary>
/// Alignment required for sparse buffer mappings.
/// </summary>
public const ulong SparseBufferAlignmentSize = 0x10000;
private const ulong MaxDynamicGrowthSize = 0x100000;
private readonly GpuContext _context;
@ -27,6 +39,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// Must lock for any access from other threads.
/// </remarks>
private readonly RangeList<Buffer> _buffers;
private readonly MultiRangeList<MultiRangeBuffer> _multiRangeBuffers;
private Buffer[] _bufferOverlaps;
@ -47,6 +60,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_physicalMemory = physicalMemory;
_buffers = new RangeList<Buffer>();
_multiRangeBuffers = new MultiRangeList<MultiRangeBuffer>();
_bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity];
@ -66,45 +80,100 @@ namespace Ryujinx.Graphics.Gpu.Memory
Buffer[] overlaps = new Buffer[10];
int overlapCount;
ulong address = ((MemoryManager)sender).Translate(e.Address);
ulong size = e.Size;
MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size);
lock (_buffers)
for (int index = 0; index < range.Count; index++)
{
overlapCount = _buffers.FindOverlaps(address, size, ref overlaps);
}
MemoryRange subRange = range.GetSubRange(index);
for (int i = 0; i < overlapCount; i++)
{
overlaps[i].Unmapped(address, size);
lock (_buffers)
{
overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps);
}
for (int i = 0; i < overlapCount; i++)
{
overlaps[i].Unmapped(subRange.Address, subRange.Size);
}
}
}
/// <summary>
/// Performs address translation of the GPU virtual address, and creates a
/// new buffer, if needed, for the specified range.
/// new buffer, if needed, for the specified contiguous range.
/// </summary>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
/// <param name="size">Size in bytes of the buffer</param>
/// <returns>CPU virtual address of the buffer, after address translation</returns>
public ulong TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
/// <returns>Contiguous physical range of the buffer, after address translation</returns>
public MultiRange TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
{
if (gpuVa == 0)
{
return 0;
return new MultiRange(MemoryManager.PteUnmapped, size);
}
ulong address = memoryManager.Translate(gpuVa);
if (address == MemoryManager.PteUnmapped)
if (address != MemoryManager.PteUnmapped)
{
return 0;
CreateBuffer(address, size);
}
CreateBuffer(address, size);
return new MultiRange(address, size);
}
return address;
/// <summary>
/// Performs address translation of the GPU virtual address, and creates
/// new buffers, if needed, for the specified range.
/// </summary>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
/// <param name="size">Size in bytes of the buffer</param>
/// <returns>Physical ranges of the buffer, after address translation</returns>
public MultiRange TranslateAndCreateMultiBuffers(MemoryManager memoryManager, ulong gpuVa, ulong size)
{
if (gpuVa == 0)
{
return new MultiRange(MemoryManager.PteUnmapped, size);
}
bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
// Fast path not taken for non-contiguous ranges,
// since multi-range buffers are not coalesced, so a buffer that covers
// the entire cached range might not actually exist.
if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) &&
range.Count == 1)
{
return range;
}
CreateBuffer(range);
return range;
}
/// <summary>
/// Creates a new buffer for the specified range, if it does not yet exist.
/// This can be used to ensure the existance of a buffer.
/// </summary>
/// <param name="range">Physical ranges of memory where the buffer data is located</param>
public void CreateBuffer(MultiRange range)
{
if (range.Count > 1)
{
CreateMultiRangeBuffer(range);
}
else
{
MemoryRange subRange = range.GetSubRange(0);
if (subRange.Address != MemoryManager.PteUnmapped)
{
CreateBuffer(subRange.Address, subRange.Size);
}
}
}
/// <summary>
@ -118,7 +187,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size;
ulong alignedAddress = address & ~BufferAlignmentMask;
ulong alignedEndAddress = (endAddress + BufferAlignmentMask) & ~BufferAlignmentMask;
// The buffer must have the size of at least one page.
@ -130,6 +198,108 @@ namespace Ryujinx.Graphics.Gpu.Memory
CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress);
}
/// <summary>
/// Creates a new buffer for the specified range, if it does not yet exist.
/// This can be used to ensure the existance of a buffer.
/// </summary>
/// <param name="address">Address of the buffer in memory</param>
/// <param name="size">Size of the buffer in bytes</param>
/// <param name="alignment">Alignment of the start address of the buffer in bytes</param>
public void CreateBuffer(ulong address, ulong size, ulong alignment)
{
ulong alignmentMask = alignment - 1;
ulong pageAlignmentMask = BufferAlignmentMask;
ulong endAddress = address + size;
ulong alignedAddress = address & ~alignmentMask;
ulong alignedEndAddress = (endAddress + pageAlignmentMask) & ~pageAlignmentMask;
// The buffer must have the size of at least one page.
if (alignedEndAddress == alignedAddress)
{
alignedEndAddress += pageAlignmentMask;
}
CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress, alignment);
}
/// <summary>
/// Creates a buffer for a memory region composed of multiple physical ranges,
/// if it does not exist yet.
/// </summary>
/// <param name="range">Physical ranges of memory</param>
private void CreateMultiRangeBuffer(MultiRange range)
{
// Ensure all non-contiguous buffer we might use are sparse aligned.
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
if (subRange.Address != MemoryManager.PteUnmapped)
{
CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
}
}
// Create sparse buffer.
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
for (int index = 0; index < overlapCount; index++)
{
if (overlaps[index].Range.Contains(range))
{
return;
}
}
for (int index = 0; index < overlapCount; index++)
{
if (range.Contains(overlaps[index].Range))
{
_multiRangeBuffers.Remove(overlaps[index]);
overlaps[index].Dispose();
}
}
BufferRange[] storages = new BufferRange[range.Count];
MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
ulong alignmentMask = SparseBufferAlignmentSize - 1;
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
if (subRange.Address != MemoryManager.PteUnmapped)
{
ulong endAddress = subRange.Address + subRange.Size;
ulong alignedAddress = subRange.Address & ~alignmentMask;
ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
ulong alignedSize = alignedEndAddress - alignedAddress;
Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
storages[i] = bufferRange;
alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
}
else
{
ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
}
}
MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
_multiRangeBuffers.Add(multiRangeBuffer);
}
/// <summary>
/// Performs address translation of the GPU virtual address, and attempts to force
/// the buffer in the region as dirty.
@ -150,7 +320,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
result.EndGpuAddress < gpuVa + size ||
result.UnmappedSequence != result.Buffer.UnmappedSequence)
{
ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
MultiRange range = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
ulong address = range.GetSubRange(0).Address;
result = new BufferCacheEntry(address, gpuVa, GetBuffer(address, size));
_dirtyCache[gpuVa] = result;
@ -184,7 +355,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
result.EndGpuAddress < alignedEndGpuVa ||
result.UnmappedSequence != result.Buffer.UnmappedSequence)
{
ulong address = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
MultiRange range = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
ulong address = range.GetSubRange(0).Address;
result = new BufferCacheEntry(address, alignedGpuVa, GetBuffer(address, size));
_modifiedCache[alignedGpuVa] = result;
@ -204,7 +376,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the buffer</param>
private void CreateBufferAligned(ulong address, ulong size)
{
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
Buffer[] overlaps = _bufferOverlaps;
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
if (overlapsCount != 0)
{
@ -215,9 +388,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
// old buffer(s) to the new buffer.
ulong endAddress = address + size;
Buffer overlap0 = overlaps[0];
if (_bufferOverlaps[0].Address > address || _bufferOverlaps[0].EndAddress < endAddress)
if (overlap0.Address > address || overlap0.EndAddress < endAddress)
{
bool anySparseCompatible = false;
// Check if the following conditions are met:
// - We have a single overlap.
// - The overlap starts at or before the requested range. That is, the overlap happens at the end.
@ -228,23 +404,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the
// range crosses a page, and after alignment, ends having a size of 2 pages.
if (overlapsCount == 1 &&
address >= _bufferOverlaps[0].Address &&
endAddress - _bufferOverlaps[0].EndAddress <= BufferAlignmentSize * 2)
address >= overlap0.Address &&
endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2)
{
// Try to grow the buffer by 1.5x of its current size.
// This improves performance in the cases where the buffer is resized often by small amounts.
ulong existingSize = _bufferOverlaps[0].Size;
ulong existingSize = overlap0.Size;
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
size = Math.Max(size, growthSize);
endAddress = address + size;
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
}
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = _bufferOverlaps[index];
Buffer buffer = overlaps[index];
anySparseCompatible |= buffer.SparseCompatible;
address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress);
@ -257,35 +435,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong newSize = endAddress - address;
Buffer newBuffer = new(_context, _physicalMemory, address, newSize, _bufferOverlaps.Take(overlapsCount));
lock (_buffers)
{
_buffers.Add(newBuffer);
}
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = _bufferOverlaps[index];
int dstOffset = (int)(buffer.Address - newBuffer.Address);
buffer.CopyTo(newBuffer, dstOffset);
newBuffer.InheritModifiedRanges(buffer);
buffer.DecrementReferenceCount();
}
newBuffer.SynchronizeMemory(address, newSize);
// Existing buffers were modified, we need to rebind everything.
NotifyBuffersModified?.Invoke();
CreateBufferAligned(address, newSize, anySparseCompatible, overlaps, overlapsCount);
}
}
else
{
// No overlap, just create a new buffer.
Buffer buffer = new(_context, _physicalMemory, address, size);
Buffer buffer = new(_context, _physicalMemory, address, size, sparseCompatible: false);
lock (_buffers)
{
@ -296,6 +452,151 @@ namespace Ryujinx.Graphics.Gpu.Memory
ShrinkOverlapsBufferIfNeeded();
}
/// <summary>
/// Creates a new buffer for the specified range, if needed.
/// If a buffer where this range can be fully contained already exists,
/// then the creation of a new buffer is not necessary.
/// </summary>
/// <param name="address">Address of the buffer in guest memory</param>
/// <param name="size">Size in bytes of the buffer</param>
/// <param name="alignment">Alignment of the start address of the buffer</param>
private void CreateBufferAligned(ulong address, ulong size, ulong alignment)
{
Buffer[] overlaps = _bufferOverlaps;
int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
bool sparseAligned = alignment >= SparseBufferAlignmentSize;
if (overlapsCount != 0)
{
// If the buffer already exists, make sure if covers the entire range,
// and make sure it is properly aligned, otherwise sparse mapping may fail.
ulong endAddress = address + size;
Buffer overlap0 = overlaps[0];
if (overlap0.Address > address ||
overlap0.EndAddress < endAddress ||
(overlap0.Address & (alignment - 1)) != 0 ||
(!overlap0.SparseCompatible && sparseAligned))
{
// We need to make sure the new buffer is properly aligned.
// However, after the range is aligned, it is possible that it
// overlaps more buffers, so try again after each extension
// and ensure we cover all overlaps.
int oldOverlapsCount;
do
{
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = overlaps[index];
address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress);
}
address &= ~(alignment - 1);
oldOverlapsCount = overlapsCount;
overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps);
}
while (oldOverlapsCount != overlapsCount);
lock (_buffers)
{
for (int index = 0; index < overlapsCount; index++)
{
_buffers.Remove(overlaps[index]);
}
}
ulong newSize = endAddress - address;
CreateBufferAligned(address, newSize, sparseAligned, overlaps, overlapsCount);
}
}
else
{
// No overlap, just create a new buffer.
Buffer buffer = new(_context, _physicalMemory, address, size, sparseAligned);
lock (_buffers)
{
_buffers.Add(buffer);
}
}
ShrinkOverlapsBufferIfNeeded();
}
/// <summary>
/// Creates a new buffer for the specified range, if needed.
/// If a buffer where this range can be fully contained already exists,
/// then the creation of a new buffer is not necessary.
/// </summary>
/// <param name="address">Address of the buffer in guest memory</param>
/// <param name="size">Size in bytes of the buffer</param>
/// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="overlaps">Buffers overlapping the range</param>
/// <param name="overlapsCount">Total of overlaps</param>
private void CreateBufferAligned(ulong address, ulong size, bool sparseCompatible, Buffer[] overlaps, int overlapsCount)
{
Buffer newBuffer = new Buffer(_context, _physicalMemory, address, size, sparseCompatible, overlaps.Take(overlapsCount));
lock (_buffers)
{
_buffers.Add(newBuffer);
}
for (int index = 0; index < overlapsCount; index++)
{
Buffer buffer = overlaps[index];
int dstOffset = (int)(buffer.Address - newBuffer.Address);
buffer.CopyTo(newBuffer, dstOffset);
newBuffer.InheritModifiedRanges(buffer);
buffer.DecrementReferenceCount();
}
newBuffer.SynchronizeMemory(address, size);
// Existing buffers were modified, we need to rebind everything.
NotifyBuffersModified?.Invoke();
RecreateMultiRangeBuffers(address, size);
}
/// <summary>
/// Recreates all the multi-range buffers that overlaps a given physical memory range.
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size of the range in bytes</param>
private void RecreateMultiRangeBuffers(ulong address, ulong size)
{
if ((address & (SparseBufferAlignmentSize - 1)) != 0 || (size & (SparseBufferAlignmentSize - 1)) != 0)
{
return;
}
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
int overlapCount = _multiRangeBuffers.FindOverlaps(address, size, ref overlaps);
for (int index = 0; index < overlapCount; index++)
{
_multiRangeBuffers.Remove(overlaps[index]);
overlaps[index].Dispose();
}
for (int index = 0; index < overlapCount; index++)
{
CreateMultiRangeBuffer(overlaps[index].Range);
}
}
/// <summary>
/// Resizes the temporary buffer used for range list intersection results, if it has grown too much.
/// </summary>
@ -319,9 +620,63 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the copy</param>
public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
{
ulong srcAddress = TranslateAndCreateBuffer(memoryManager, srcVa, size);
ulong dstAddress = TranslateAndCreateBuffer(memoryManager, dstVa, size);
MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size);
MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size);
if (srcRange.Count == 1 && dstRange.Count == 1)
{
CopyBufferSingleRange(memoryManager, srcRange.GetSubRange(0).Address, dstRange.GetSubRange(0).Address, size);
}
else
{
ulong copiedSize = 0;
ulong srcOffset = 0;
ulong dstOffset = 0;
int srcRangeIndex = 0;
int dstRangeIndex = 0;
while (copiedSize < size)
{
if (srcRange.GetSubRange(srcRangeIndex).Size == srcOffset)
{
srcRangeIndex++;
srcOffset = 0;
}
if (dstRange.GetSubRange(dstRangeIndex).Size == dstOffset)
{
dstRangeIndex++;
dstOffset = 0;
}
MemoryRange srcSubRange = srcRange.GetSubRange(srcRangeIndex);
MemoryRange dstSubRange = dstRange.GetSubRange(dstRangeIndex);
ulong srcSize = srcSubRange.Size - srcOffset;
ulong dstSize = dstSubRange.Size - dstOffset;
ulong copySize = Math.Min(srcSize, dstSize);
CopyBufferSingleRange(memoryManager, srcSubRange.Address + srcOffset, dstSubRange.Address + dstOffset, copySize);
srcOffset += copySize;
dstOffset += copySize;
copiedSize += copySize;
}
}
}
/// <summary>
/// Copy a buffer data from a given address to another.
/// </summary>
/// <remarks>
/// This does a GPU side copy.
/// </remarks>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="srcAddress">Physical address of the copy source</param>
/// <param name="dstAddress">Physical address of the copy destination</param>
/// <param name="size">Size in bytes of the copy</param>
private void CopyBufferSingleRange(MemoryManager memoryManager, ulong srcAddress, ulong dstAddress, ulong size)
{
Buffer srcBuffer = GetBuffer(srcAddress, size);
Buffer dstBuffer = GetBuffer(dstAddress, size);
@ -360,39 +715,98 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="value">Value to be written into the buffer</param>
public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
{
ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size);
Buffer buffer = GetBuffer(address, size);
for (int index = 0; index < range.Count; index++)
{
MemoryRange subRange = range.GetSubRange(index);
Buffer buffer = GetBuffer(subRange.Address, subRange.Size);
int offset = (int)(address - buffer.Address);
int offset = (int)(subRange.Address - buffer.Address);
_context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)size, value);
_context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
memoryManager.Physical.FillTrackedResource(address, size, value, ResourceKind.Buffer);
memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
}
}
/// <summary>
/// Gets a buffer sub-range from a start address til a page boundary after the given size.
/// Gets a buffer sub-range starting at a given memory address, aligned to the next page boundary.
/// </summary>
/// <param name="address">Start address of the memory range</param>
/// <param name="size">Size in bytes of the memory range</param>
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
/// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range starting at the given memory address</returns>
public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false)
public BufferRange GetBufferRangeAligned(MultiRange range, bool write = false)
{
return GetBuffer(address, size, write).GetRangeAligned(address, size, write);
if (range.Count > 1)
{
return GetBuffer(range, write).GetRange(range);
}
else
{
MemoryRange subRange = range.GetSubRange(0);
return GetBuffer(subRange.Address, subRange.Size, write).GetRangeAligned(subRange.Address, subRange.Size, write);
}
}
/// <summary>
/// Gets a buffer sub-range for a given memory range.
/// </summary>
/// <param name="address">Start address of the memory range</param>
/// <param name="size">Size in bytes of the memory range</param>
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
/// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range for the given range</returns>
public BufferRange GetBufferRange(ulong address, ulong size, bool write = false)
public BufferRange GetBufferRange(MultiRange range, bool write = false)
{
return GetBuffer(address, size, write).GetRange(address, size, write);
if (range.Count > 1)
{
return GetBuffer(range, write).GetRange(range);
}
else
{
MemoryRange subRange = range.GetSubRange(0);
return GetBuffer(subRange.Address, subRange.Size, write).GetRange(subRange.Address, subRange.Size, write);
}
}
/// <summary>
/// Gets a buffer for a given memory range.
/// A buffer overlapping with the specified range is assumed to already exist on the cache.
/// </summary>
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
/// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer where the range is fully contained</returns>
private MultiRangeBuffer GetBuffer(MultiRange range, bool write = false)
{
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
subBuffer.SynchronizeMemory(subRange.Address, subRange.Size);
if (write)
{
subBuffer.SignalModified(subRange.Address, subRange.Size);
}
}
MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
MultiRangeBuffer buffer = null;
for (int i = 0; i < overlapCount; i++)
{
if (overlaps[i].Range.Contains(range))
{
buffer = overlaps[i];
break;
}
}
return buffer;
}
/// <summary>
@ -426,12 +840,33 @@ namespace Ryujinx.Graphics.Gpu.Memory
return buffer;
}
/// <summary>
/// Performs guest to host memory synchronization of a given memory range.
/// </summary>
/// <param name="range">Physical regions of memory where the buffer is mapped</param>
public void SynchronizeBufferRange(MultiRange range)
{
if (range.Count == 1)
{
MemoryRange subRange = range.GetSubRange(0);
SynchronizeBufferRange(subRange.Address, subRange.Size);
}
else
{
for (int index = 0; index < range.Count; index++)
{
MemoryRange subRange = range.GetSubRange(index);
SynchronizeBufferRange(subRange.Address, subRange.Size);
}
}
}
/// <summary>
/// Performs guest to host memory synchronization of a given memory range.
/// </summary>
/// <param name="address">Start address of the memory range</param>
/// <param name="size">Size in bytes of the memory range</param>
public void SynchronizeBufferRange(ulong address, ulong size)
private void SynchronizeBufferRange(ulong address, ulong size)
{
if (size != 0)
{
@ -491,7 +926,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <summary>
/// Disposes all buffers in the cache.
/// It's an error to use the buffer manager after disposal.
/// It's an error to use the buffer cache after disposal.
/// </summary>
public void Dispose()
{

136
src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
View file

@ -3,6 +3,7 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Gpu.Shader;
using Ryujinx.Graphics.Shader;
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
@ -62,18 +63,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
Bindings = new BufferDescriptor[count];
Buffers = new BufferBounds[count];
Unaligned = new bool[count];
Buffers.AsSpan().Fill(new BufferBounds(new MultiRange(MemoryManager.PteUnmapped, 0UL)));
}
/// <summary>
/// Sets the region of a buffer at a given slot.
/// </summary>
/// <param name="index">Buffer slot</param>
/// <param name="address">Region virtual address</param>
/// <param name="size">Region size in bytes</param>
/// <param name="range">Physical memory regions where the buffer is mapped</param>
/// <param name="flags">Buffer usage flags</param>
public void SetBounds(int index, ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None)
public void SetBounds(int index, MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
{
Buffers[index] = new BufferBounds(address, size, flags);
Buffers[index] = new BufferBounds(range, flags);
}
/// <summary>
@ -120,6 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_context = context;
_channel = channel;
_indexBuffer.Range = new MultiRange(MemoryManager.PteUnmapped, 0UL);
_vertexBuffers = new VertexBuffer[Constants.TotalVertexBuffers];
_transformFeedbackBuffers = new BufferBounds[Constants.TotalTransformFeedbackBuffers];
@ -150,10 +153,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="type">Type of each index buffer element</param>
public void SetIndexBuffer(ulong gpuVa, ulong size, IndexType type)
{
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_indexBuffer.Address = address;
_indexBuffer.Size = size;
_indexBuffer.Range = range;
_indexBuffer.Type = type;
_indexBufferDirty = true;
@ -181,16 +183,15 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="divisor">Vertex divisor of the buffer, for instanced draws</param>
public void SetVertexBuffer(int index, ulong gpuVa, ulong size, int stride, int divisor)
{
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_vertexBuffers[index].Address = address;
_vertexBuffers[index].Size = size;
_vertexBuffers[index].Range = range;
_vertexBuffers[index].Stride = stride;
_vertexBuffers[index].Divisor = divisor;
_vertexBuffersDirty = true;
if (address != 0)
if (!range.IsUnmapped)
{
_vertexBuffersEnableMask |= 1u << index;
}
@ -209,9 +210,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the transform feedback buffer</param>
public void SetTransformFeedbackBuffer(int index, ulong gpuVa, ulong size)
{
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
_transformFeedbackBuffers[index] = new BufferBounds(address, size);
_transformFeedbackBuffers[index] = new BufferBounds(range);
_transformFeedbackBuffersDirty = true;
}
@ -256,9 +257,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
_cpStorageBuffers.SetBounds(index, address, size, flags);
_cpStorageBuffers.SetBounds(index, range, flags);
}
/// <summary>
@ -280,15 +281,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
if (buffers.Buffers[index].Address != address ||
buffers.Buffers[index].Size != size)
if (!buffers.Buffers[index].Range.Equals(range))
{
_gpStorageBuffersDirty = true;
}
buffers.SetBounds(index, address, size, flags);
buffers.SetBounds(index, range, flags);
}
/// <summary>
@ -300,9 +300,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the storage buffer</param>
public void SetComputeUniformBuffer(int index, ulong gpuVa, ulong size)
{
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_cpUniformBuffers.SetBounds(index, address, size);
_cpUniformBuffers.SetBounds(index, range);
}
/// <summary>
@ -315,9 +315,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the storage buffer</param>
public void SetGraphicsUniformBuffer(int stage, int index, ulong gpuVa, ulong size)
{
ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_gpUniformBuffers[stage].SetBounds(index, address, size);
_gpUniformBuffers[stage].SetBounds(index, range);
_gpUniformBuffersDirty = true;
}
@ -379,7 +379,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
for (int i = 0; i < _cpUniformBuffers.Buffers.Length; i++)
{
if (_cpUniformBuffers.Buffers[i].Address != 0)
if (!_cpUniformBuffers.Buffers[i].IsUnmapped)
{
mask |= 1u << i;
}
@ -399,7 +399,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
for (int i = 0; i < _gpUniformBuffers[stage].Buffers.Length; i++)
{
if (_gpUniformBuffers[stage].Buffers[i].Address != 0)
if (!_gpUniformBuffers[stage].Buffers[i].IsUnmapped)
{
mask |= 1u << i;
}
@ -415,7 +415,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
public ulong GetComputeUniformBufferAddress(int index)
{
return _cpUniformBuffers.Buffers[index].Address;
return _cpUniformBuffers.Buffers[index].Range.GetSubRange(0).Address;
}
/// <summary>
@ -426,7 +426,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
public ulong GetGraphicsUniformBufferAddress(int stage, int index)
{
return _gpUniformBuffers[stage].Buffers[index].Address;
return _gpUniformBuffers[stage].Buffers[index].Range.GetSubRange(0).Address;
}
/// <summary>
@ -477,7 +477,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
foreach (var binding in _bufferTextures)
{
var isStore = binding.BindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore);
var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Address, binding.Size, isStore);
var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Range, isStore);
binding.Texture.SetStorage(range);
// The texture must be rebound to use the new storage if it was updated.
@ -511,16 +511,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
_indexBufferDirty = false;
if (_indexBuffer.Address != 0)
if (!_indexBuffer.Range.IsUnmapped)
{
BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Address, _indexBuffer.Size);
BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Range);
_context.Renderer.Pipeline.SetIndexBuffer(buffer, _indexBuffer.Type);
}
}
else if (_indexBuffer.Address != 0)
else if (!_indexBuffer.Range.IsUnmapped)
{
bufferCache.SynchronizeBufferRange(_indexBuffer.Address, _indexBuffer.Size);
bufferCache.SynchronizeBufferRange(_indexBuffer.Range);
}
}
else if (_rebind)
@ -540,12 +540,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
VertexBuffer vb = _vertexBuffers[index];
if (vb.Address == 0)
if (vb.Range.IsUnmapped)
{
continue;
}
BufferRange buffer = bufferCache.GetBufferRange(vb.Address, vb.Size);
BufferRange buffer = bufferCache.GetBufferRange(vb.Range);
vertexBuffers[index] = new VertexBufferDescriptor(buffer, vb.Stride, vb.Divisor);
}
@ -558,12 +558,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
VertexBuffer vb = _vertexBuffers[index];
if (vb.Address == 0)
if (vb.Range.IsUnmapped)
{
continue;
}
bufferCache.SynchronizeBufferRange(vb.Address, vb.Size);
bufferCache.SynchronizeBufferRange(vb.Range);
}
}
@ -579,13 +579,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.Address == 0)
if (tfb.IsUnmapped)
{
tfbs[index] = BufferRange.Empty;
continue;
}
tfbs[index] = bufferCache.GetBufferRange(tfb.Address, tfb.Size, write: true);
tfbs[index] = bufferCache.GetBufferRange(tfb.Range, write: true);
}
_context.Renderer.Pipeline.SetTransformFeedbackBuffers(tfbs);
@ -600,21 +600,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.Address == 0)
if (tfb.IsUnmapped)
{
buffers[index] = new BufferAssignment(index, BufferRange.Empty);
}
else
{
ulong endAddress = tfb.Address + tfb.Size;
ulong address = BitUtils.AlignDown(tfb.Address, (ulong)alignment);
ulong size = endAddress - address;
MultiRange range = tfb.Range;
ulong address0 = range.GetSubRange(0).Address;
ulong address = BitUtils.AlignDown(address0, (ulong)alignment);
int tfeOffset = ((int)tfb.Address & (alignment - 1)) / 4;
if (range.Count == 1)
{
range = new MultiRange(address, range.GetSubRange(0).Size + (address0 - address));
}
else
{
MemoryRange[] subRanges = new MemoryRange[range.Count];
subRanges[0] = new MemoryRange(address, range.GetSubRange(0).Size + (address0 - address));
for (int i = 1; i < range.Count; i++)
{
subRanges[i] = range.GetSubRange(i);
}
range = new MultiRange(subRanges);
}
int tfeOffset = ((int)address0 & (alignment - 1)) / 4;
_context.SupportBufferUpdater.SetTfeOffset(index, tfeOffset);
buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(address, size, write: true));
buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(range, write: true));
}
}
@ -627,12 +645,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.Address == 0)
if (tfb.IsUnmapped)
{
continue;
}
bufferCache.SynchronizeBufferRange(tfb.Address, tfb.Size);
bufferCache.SynchronizeBufferRange(tfb.Range);
}
}
@ -688,12 +706,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
if (bounds.Address != 0)
if (!bounds.IsUnmapped)
{
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite)
: bufferCache.GetBufferRange(bounds.Address, bounds.Size);
? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
: bufferCache.GetBufferRange(bounds.Range);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
}
@ -725,12 +743,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
if (bounds.Address != 0)
if (!bounds.IsUnmapped)
{
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite)
: bufferCache.GetBufferRange(bounds.Address, bounds.Size);
? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
: bufferCache.GetBufferRange(bounds.Range);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
}
@ -778,12 +796,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[binding.Slot];
if (bounds.Address == 0)
if (bounds.IsUnmapped)
{
continue;
}
_channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Address, bounds.Size);
_channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Range);
}
}
}
@ -793,23 +811,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
/// <param name="stage">Shader stage accessing the texture</param>
/// <param name="texture">Buffer texture</param>
/// <param name="address">Address of the buffer in memory</param>
/// <param name="size">Size of the buffer in bytes</param>
/// <param name="range">Physical ranges of memory where the buffer texture data is located</param>
/// <param name="bindingInfo">Binding info for the buffer texture</param>
/// <param name="format">Format of the buffer texture</param>
/// <param name="isImage">Whether the binding is for an image or a sampler</param>
public void SetBufferTextureStorage(
ShaderStage stage,
ITexture texture,
ulong address,
ulong size,
MultiRange range,
TextureBindingInfo bindingInfo,
Format format,
bool isImage)
{
_channel.MemoryManager.Physical.BufferCache.CreateBuffer(address, size);
_channel.MemoryManager.Physical.BufferCache.CreateBuffer(range);
_bufferTextures.Add(new BufferTextureBinding(stage, texture, address, size, bindingInfo, format, isImage));
_bufferTextures.Add(new BufferTextureBinding(stage, texture, range, bindingInfo, format, isImage));
}
/// <summary>

19
src/Ryujinx.Graphics.Gpu/Memory/BufferTextureBinding.cs
View file

@ -1,6 +1,7 @@
using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Shader;
using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory
{
@ -20,14 +21,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
public ITexture Texture { get; }
/// <summary>
/// The base address of the buffer binding.
/// Physical ranges of memory where the buffer texture data is located.
/// </summary>
public ulong Address { get; }
/// <summary>
/// The size of the buffer binding in bytes.
/// </summary>
public ulong Size { get; }
public MultiRange Range { get; }
/// <summary>
/// The image or sampler binding info for the buffer texture.
@ -49,24 +45,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
/// <param name="stage">Shader stage accessing the texture</param>
/// <param name="texture">Buffer texture</param>
/// <param name="address">Base address</param>
/// <param name="size">Size in bytes</param>
/// <param name="range">Physical ranges of memory where the buffer texture data is located</param>
/// <param name="bindingInfo">Binding info</param>
/// <param name="format">Binding format</param>
/// <param name="isImage">Whether the binding is for an image or a sampler</param>
public BufferTextureBinding(
ShaderStage stage,
ITexture texture,
ulong address,
ulong size,
MultiRange range,
TextureBindingInfo bindingInfo,
Format format,
bool isImage)
{
Stage = stage;
Texture = texture;
Address = address;
Size = size;
Range = range;
BindingInfo = bindingInfo;
Format = format;
IsImage = isImage;

5
src/Ryujinx.Graphics.Gpu/Memory/IndexBuffer.cs
View file

@ -1,4 +1,5 @@
using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory
{
@ -7,9 +8,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
struct IndexBuffer
{
public ulong Address;
public ulong Size;
public MultiRange Range;
public IndexType Type;
}
}

12
src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
View file

@ -39,6 +39,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
internal PhysicalMemory Physical { get; }
/// <summary>
/// Virtual buffer cache.
/// </summary>
internal VirtualBufferCache VirtualBufferCache { get; }
/// <summary>
/// Cache of GPU counters.
/// </summary>
@ -51,10 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
internal MemoryManager(PhysicalMemory physicalMemory)
{
Physical = physicalMemory;
VirtualBufferCache = new VirtualBufferCache(this);
CounterCache = new CounterCache();
_pageTable = new ulong[PtLvl0Size][];
MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler;
MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
}
@ -508,6 +515,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
regionSize += Math.Min(endVa - va, PageSize);
}
if (regions.Count == 0)
{
return new MultiRange(regionStart, regionSize);
}
regions.Add(new MemoryRange(regionStart, regionSize));
return new MultiRange(regions.ToArray());

60
src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs Normal file
View file

@ -0,0 +1,60 @@
using Ryujinx.Graphics.GAL;
using Ryujinx.Memory.Range;
using System;
namespace Ryujinx.Graphics.Gpu.Memory
{
/// <summary>
/// Buffer, used to store vertex and index data, uniform and storage buffers, and others.
/// </summary>
class MultiRangeBuffer : IMultiRangeItem, IDisposable
{
private readonly GpuContext _context;
/// <summary>
/// Host buffer handle.
/// </summary>
public BufferHandle Handle { get; }
/// <summary>
/// Range of memory where the data is located.
/// </summary>
public MultiRange Range { get; }
/// <summary>
/// Creates a new instance of the buffer.
/// </summary>
/// <param name="context">GPU context that the buffer belongs to</param>
/// <param name="range">Range of memory where the data is mapped</param>
/// <param name="storages">Backing memory for the buffers</param>
public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages)
{
_context = context;
Range = range;
Handle = context.Renderer.CreateBufferSparse(storages);
}
/// <summary>
/// Gets a sub-range from the buffer.
/// </summary>
/// <remarks>
/// This can be used to bind and use sub-ranges of the buffer on the host API.
/// </remarks>
/// <param name="range">Range of memory where the data is mapped</param>
/// <returns>The buffer sub-range</returns>
public BufferRange GetRange(MultiRange range)
{
int offset = Range.FindOffset(range);
return new BufferRange(Handle, offset, (int)range.GetSize());
}
/// <summary>
/// Disposes the host buffer.
/// </summary>
public void Dispose()
{
_context.Renderer.DeleteBuffer(Handle);
}
}
}

5
src/Ryujinx.Graphics.Gpu/Memory/VertexBuffer.cs
View file

@ -1,3 +1,5 @@
using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory
{
/// <summary>
@ -5,8 +7,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
struct VertexBuffer
{
public ulong Address;
public ulong Size;
public MultiRange Range;
public int Stride;
public int Divisor;
}

238
src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs Normal file
View file

@ -0,0 +1,238 @@
using Ryujinx.Memory.Range;
using System;
using System.Collections.Concurrent;
using System.Threading;
namespace Ryujinx.Graphics.Gpu.Memory
{
/// <summary>
/// Virtual buffer cache.
/// </summary>
class VirtualBufferCache
{
private readonly MemoryManager _memoryManager;
/// <summary>
/// Represents a GPU virtual memory range.
/// </summary>
private readonly struct VirtualRange : IRange
{
/// <summary>
/// GPU virtual address where the range starts.
/// </summary>
public ulong Address { get; }
/// <summary>
/// Size of the range in bytes.
/// </summary>
public ulong Size { get; }
/// <summary>
/// GPU virtual address where the range ends.
/// </summary>
public ulong EndAddress => Address + Size;
/// <summary>
/// Physical regions where the GPU virtual region is mapped.
/// </summary>
public MultiRange Range { get; }
/// <summary>
/// Creates a new virtual memory range.
/// </summary>
/// <param name="address">GPU virtual address where the range starts</param>
/// <param name="size">Size of the range in bytes</param>
/// <param name="range">Physical regions where the GPU virtual region is mapped</param>
public VirtualRange(ulong address, ulong size, MultiRange range)
{
Address = address;
Size = size;
Range = range;
}
/// <summary>
/// Checks if a given range overlaps with the buffer.
/// </summary>
/// <param name="address">Start address of the range</param>
/// <param name="size">Size in bytes of the range</param>
/// <returns>True if the range overlaps, false otherwise</returns>
public bool OverlapsWith(ulong address, ulong size)
{
return Address < address + size && address < EndAddress;
}
}
private readonly RangeList<VirtualRange> _virtualRanges;
private VirtualRange[] _virtualRangeOverlaps;
private readonly ConcurrentQueue<VirtualRange> _deferredUnmaps;
private int _hasDeferredUnmaps;
/// <summary>
/// Creates a new instance of the virtual buffer cache.
/// </summary>
/// <param name="memoryManager">Memory manager that the virtual buffer cache belongs to</param>
public VirtualBufferCache(MemoryManager memoryManager)
{
_memoryManager = memoryManager;
_virtualRanges = new RangeList<VirtualRange>();
_virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity];
_deferredUnmaps = new ConcurrentQueue<VirtualRange>();
}
/// <summary>
/// Handles removal of buffers written to a memory region being unmapped.
/// </summary>
/// <param name="sender">Sender object</param>
/// <param name="e">Event arguments</param>
public void MemoryUnmappedHandler(object sender, UnmapEventArgs e)
{
void EnqueueUnmap()
{
_deferredUnmaps.Enqueue(new VirtualRange(e.Address, e.Size, default));
Interlocked.Exchange(ref _hasDeferredUnmaps, 1);
}
e.AddRemapAction(EnqueueUnmap);
}
/// <summary>
/// Tries to get a existing, cached physical range for the specified virtual region.
/// If no cached range is found, a new one is created and added.
/// </summary>
/// <param name="gpuVa">GPU virtual address to get the physical range from</param>
/// <param name="size">Size in bytes of the region</param>
/// <param name="supportsSparse">Indicates host support for sparse buffer mapping of non-contiguous ranges</param>
/// <param name="range">Physical range for the specified GPU virtual region</param>
/// <returns>True if the range already existed, false if a new one was created and added</returns>
public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range)
{
VirtualRange[] overlaps = _virtualRangeOverlaps;
int overlapsCount;
if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0)
{
while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange))
{
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps);
for (int index = 0; index < overlapsCount; index++)
{
_virtualRanges.Remove(overlaps[index]);
}
}
}
bool found = false;
ulong originalVa = gpuVa;
overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps);
if (overlapsCount != 0)
{
// The virtual range already exists. We just need to check if our range fits inside
// the existing one, and if not, we must extend the existing one.
ulong endAddress = gpuVa + size;
VirtualRange overlap0 = overlaps[0];
if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress)
{
for (int index = 0; index < overlapsCount; index++)
{
VirtualRange virtualRange = overlaps[index];
gpuVa = Math.Min(gpuVa, virtualRange.Address);
endAddress = Math.Max(endAddress, virtualRange.EndAddress);
_virtualRanges.Remove(virtualRange);
}
ulong newSize = endAddress - gpuVa;
MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize);
_virtualRanges.Add(new(gpuVa, newSize, newRange));
range = newRange.Slice(originalVa - gpuVa, size);
}
else
{
found = true;
range = overlap0.Range.Slice(gpuVa - overlap0.Address, size);
}
}
else
{
// No overlap, just create a new virtual range.
range = _memoryManager.GetPhysicalRegions(gpuVa, size);
VirtualRange virtualRange = new(gpuVa, size, range);
_virtualRanges.Add(virtualRange);
}
ShrinkOverlapsBufferIfNeeded();
// If the the range is not properly aligned for sparse mapping,
// or if the host does not support sparse mapping, let's just
// force it to a single range.
// This might cause issues in some applications that uses sparse
// mappings.
if (!IsSparseAligned(range) || !supportsSparse)
{
range = new MultiRange(range.GetSubRange(0).Address, size);
}
return found;
}
/// <summary>
/// Checks if the physical memory ranges are valid for sparse mapping,
/// which requires all sub-ranges to be 64KB aligned.
/// </summary>
/// <param name="range">Range to check</param>
/// <returns>True if the range is valid for sparse mapping, false otherwise</returns>
private static bool IsSparseAligned(MultiRange range)
{
if (range.Count == 1)
{
return (range.GetSubRange(0).Address & (BufferCache.SparseBufferAlignmentSize - 1)) == 0;
}
for (int i = 0; i < range.Count; i++)
{
MemoryRange subRange = range.GetSubRange(i);
// Check if address is aligned. The address of the first sub-range can
// be misaligned as it is at the start.
if (i > 0 &&
subRange.Address != MemoryManager.PteUnmapped &&
(subRange.Address & (BufferCache.SparseBufferAlignmentSize - 1)) != 0)
{
return false;
}
// Check if the size is aligned. The size of the last sub-range can
// be misaligned as it is at the end.
if (i < range.Count - 1 && (subRange.Size & (BufferCache.SparseBufferAlignmentSize - 1)) != 0)
{
return false;
}
}
return true;
}
/// <summary>
/// Resizes the temporary buffer used for range list intersection results, if it has grown too much.
/// </summary>
private void ShrinkOverlapsBufferIfNeeded()
{
if (_virtualRangeOverlaps.Length > BufferCache.OverlapsBufferMaxCapacity)
{
Array.Resize(ref _virtualRangeOverlaps, BufferCache.OverlapsBufferMaxCapacity);
}
}
}
}