Move solution and projects to src
This commit is contained in:
parent
cd124bda58
commit
cee7121058
3466 changed files with 55 additions and 55 deletions
470
src/Ryujinx.Cpu/AddressSpace.cs
Normal file
470
src/Ryujinx.Cpu/AddressSpace.cs
Normal file
|
@ -0,0 +1,470 @@
|
|||
using Ryujinx.Common;
|
||||
using Ryujinx.Common.Collections;
|
||||
using Ryujinx.Memory;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
class AddressSpace : IDisposable
|
||||
{
|
||||
private const ulong PageSize = 0x1000;
|
||||
|
||||
private const int DefaultBlockAlignment = 1 << 20;
|
||||
|
||||
private enum MappingType : byte
|
||||
{
|
||||
None,
|
||||
Private,
|
||||
Shared
|
||||
}
|
||||
|
||||
private class Mapping : IntrusiveRedBlackTreeNode<Mapping>, IComparable<Mapping>
|
||||
{
|
||||
public ulong Address { get; private set; }
|
||||
public ulong Size { get; private set; }
|
||||
public ulong EndAddress => Address + Size;
|
||||
public MappingType Type { get; private set; }
|
||||
|
||||
public Mapping(ulong address, ulong size, MappingType type)
|
||||
{
|
||||
Address = address;
|
||||
Size = size;
|
||||
Type = type;
|
||||
}
|
||||
|
||||
public Mapping Split(ulong splitAddress)
|
||||
{
|
||||
ulong leftSize = splitAddress - Address;
|
||||
ulong rightSize = EndAddress - splitAddress;
|
||||
|
||||
Mapping left = new Mapping(Address, leftSize, Type);
|
||||
|
||||
Address = splitAddress;
|
||||
Size = rightSize;
|
||||
|
||||
return left;
|
||||
}
|
||||
|
||||
public void UpdateState(MappingType newType)
|
||||
{
|
||||
Type = newType;
|
||||
}
|
||||
|
||||
public void Extend(ulong sizeDelta)
|
||||
{
|
||||
Size += sizeDelta;
|
||||
}
|
||||
|
||||
public int CompareTo(Mapping other)
|
||||
{
|
||||
if (Address < other.Address)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
else if (Address <= other.EndAddress - 1UL)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class PrivateMapping : IntrusiveRedBlackTreeNode<PrivateMapping>, IComparable<PrivateMapping>
|
||||
{
|
||||
public ulong Address { get; private set; }
|
||||
public ulong Size { get; private set; }
|
||||
public ulong EndAddress => Address + Size;
|
||||
public PrivateMemoryAllocation PrivateAllocation { get; private set; }
|
||||
|
||||
public PrivateMapping(ulong address, ulong size, PrivateMemoryAllocation privateAllocation)
|
||||
{
|
||||
Address = address;
|
||||
Size = size;
|
||||
PrivateAllocation = privateAllocation;
|
||||
}
|
||||
|
||||
public PrivateMapping Split(ulong splitAddress)
|
||||
{
|
||||
ulong leftSize = splitAddress - Address;
|
||||
ulong rightSize = EndAddress - splitAddress;
|
||||
|
||||
(var leftAllocation, PrivateAllocation) = PrivateAllocation.Split(leftSize);
|
||||
|
||||
PrivateMapping left = new PrivateMapping(Address, leftSize, leftAllocation);
|
||||
|
||||
Address = splitAddress;
|
||||
Size = rightSize;
|
||||
|
||||
return left;
|
||||
}
|
||||
|
||||
public void Map(MemoryBlock baseBlock, MemoryBlock mirrorBlock, PrivateMemoryAllocation newAllocation)
|
||||
{
|
||||
baseBlock.MapView(newAllocation.Memory, newAllocation.Offset, Address, Size);
|
||||
mirrorBlock.MapView(newAllocation.Memory, newAllocation.Offset, Address, Size);
|
||||
PrivateAllocation = newAllocation;
|
||||
}
|
||||
|
||||
public void Unmap(MemoryBlock baseBlock, MemoryBlock mirrorBlock)
|
||||
{
|
||||
if (PrivateAllocation.IsValid)
|
||||
{
|
||||
baseBlock.UnmapView(PrivateAllocation.Memory, Address, Size);
|
||||
mirrorBlock.UnmapView(PrivateAllocation.Memory, Address, Size);
|
||||
PrivateAllocation.Dispose();
|
||||
}
|
||||
|
||||
PrivateAllocation = default;
|
||||
}
|
||||
|
||||
public void Extend(ulong sizeDelta)
|
||||
{
|
||||
Size += sizeDelta;
|
||||
}
|
||||
|
||||
public int CompareTo(PrivateMapping other)
|
||||
{
|
||||
if (Address < other.Address)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
else if (Address <= other.EndAddress - 1UL)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private readonly MemoryBlock _backingMemory;
|
||||
private readonly PrivateMemoryAllocator _privateMemoryAllocator;
|
||||
private readonly IntrusiveRedBlackTree<Mapping> _mappingTree;
|
||||
private readonly IntrusiveRedBlackTree<PrivateMapping> _privateTree;
|
||||
|
||||
private readonly object _treeLock;
|
||||
|
||||
private readonly bool _supports4KBPages;
|
||||
|
||||
public MemoryBlock Base { get; }
|
||||
public MemoryBlock Mirror { get; }
|
||||
|
||||
public AddressSpace(MemoryBlock backingMemory, ulong asSize, bool supports4KBPages)
|
||||
{
|
||||
if (!supports4KBPages)
|
||||
{
|
||||
_privateMemoryAllocator = new PrivateMemoryAllocator(DefaultBlockAlignment, MemoryAllocationFlags.Mirrorable | MemoryAllocationFlags.NoMap);
|
||||
_mappingTree = new IntrusiveRedBlackTree<Mapping>();
|
||||
_privateTree = new IntrusiveRedBlackTree<PrivateMapping>();
|
||||
_treeLock = new object();
|
||||
|
||||
_mappingTree.Add(new Mapping(0UL, asSize, MappingType.None));
|
||||
_privateTree.Add(new PrivateMapping(0UL, asSize, default));
|
||||
}
|
||||
|
||||
_backingMemory = backingMemory;
|
||||
_supports4KBPages = supports4KBPages;
|
||||
|
||||
MemoryAllocationFlags asFlags = MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible;
|
||||
|
||||
Base = new MemoryBlock(asSize, asFlags);
|
||||
Mirror = new MemoryBlock(asSize, asFlags);
|
||||
}
|
||||
|
||||
public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
|
||||
{
|
||||
if (_supports4KBPages)
|
||||
{
|
||||
Base.MapView(_backingMemory, pa, va, size);
|
||||
Mirror.MapView(_backingMemory, pa, va, size);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_treeLock)
|
||||
{
|
||||
ulong alignment = MemoryBlock.GetPageSize();
|
||||
bool isAligned = ((va | pa | size) & (alignment - 1)) == 0;
|
||||
|
||||
if (flags.HasFlag(MemoryMapFlags.Private) && !isAligned)
|
||||
{
|
||||
Update(va, pa, size, MappingType.Private);
|
||||
}
|
||||
else
|
||||
{
|
||||
// The update method assumes that shared mappings are already aligned.
|
||||
|
||||
if (!flags.HasFlag(MemoryMapFlags.Private))
|
||||
{
|
||||
if ((va & (alignment - 1)) != (pa & (alignment - 1)))
|
||||
{
|
||||
throw new InvalidMemoryRegionException($"Virtual address 0x{va:X} and physical address 0x{pa:X} are misaligned and can't be aligned.");
|
||||
}
|
||||
|
||||
ulong endAddress = va + size;
|
||||
va = BitUtils.AlignDown(va, alignment);
|
||||
pa = BitUtils.AlignDown(pa, alignment);
|
||||
size = BitUtils.AlignUp(endAddress, alignment) - va;
|
||||
}
|
||||
|
||||
Update(va, pa, size, MappingType.Shared);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Unmap(ulong va, ulong size)
|
||||
{
|
||||
if (_supports4KBPages)
|
||||
{
|
||||
Base.UnmapView(_backingMemory, va, size);
|
||||
Mirror.UnmapView(_backingMemory, va, size);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
lock (_treeLock)
|
||||
{
|
||||
Update(va, 0UL, size, MappingType.None);
|
||||
}
|
||||
}
|
||||
|
||||
private void Update(ulong va, ulong pa, ulong size, MappingType type)
|
||||
{
|
||||
Mapping map = _mappingTree.GetNode(new Mapping(va, 1UL, MappingType.None));
|
||||
|
||||
Update(map, va, pa, size, type);
|
||||
}
|
||||
|
||||
private Mapping Update(Mapping map, ulong va, ulong pa, ulong size, MappingType type)
|
||||
{
|
||||
ulong endAddress = va + size;
|
||||
|
||||
for (; map != null; map = map.Successor)
|
||||
{
|
||||
if (map.Address < va)
|
||||
{
|
||||
_mappingTree.Add(map.Split(va));
|
||||
}
|
||||
|
||||
if (map.EndAddress > endAddress)
|
||||
{
|
||||
Mapping newMap = map.Split(endAddress);
|
||||
_mappingTree.Add(newMap);
|
||||
map = newMap;
|
||||
}
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case MappingType.None:
|
||||
if (map.Type == MappingType.Shared)
|
||||
{
|
||||
ulong startOffset = map.Address - va;
|
||||
ulong mapVa = va + startOffset;
|
||||
ulong mapSize = Math.Min(size - startOffset, map.Size);
|
||||
ulong mapEndAddress = mapVa + mapSize;
|
||||
ulong alignment = MemoryBlock.GetPageSize();
|
||||
|
||||
mapVa = BitUtils.AlignDown(mapVa, alignment);
|
||||
mapEndAddress = BitUtils.AlignUp(mapEndAddress, alignment);
|
||||
|
||||
mapSize = mapEndAddress - mapVa;
|
||||
|
||||
Base.UnmapView(_backingMemory, mapVa, mapSize);
|
||||
Mirror.UnmapView(_backingMemory, mapVa, mapSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
UnmapPrivate(va, size);
|
||||
}
|
||||
break;
|
||||
case MappingType.Private:
|
||||
if (map.Type == MappingType.Shared)
|
||||
{
|
||||
throw new InvalidMemoryRegionException($"Private mapping request at 0x{va:X} with size 0x{size:X} overlaps shared mapping at 0x{map.Address:X} with size 0x{map.Size:X}.");
|
||||
}
|
||||
else
|
||||
{
|
||||
MapPrivate(va, size);
|
||||
}
|
||||
break;
|
||||
case MappingType.Shared:
|
||||
if (map.Type != MappingType.None)
|
||||
{
|
||||
throw new InvalidMemoryRegionException($"Shared mapping request at 0x{va:X} with size 0x{size:X} overlaps mapping at 0x{map.Address:X} with size 0x{map.Size:X}.");
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong startOffset = map.Address - va;
|
||||
ulong mapPa = pa + startOffset;
|
||||
ulong mapVa = va + startOffset;
|
||||
ulong mapSize = Math.Min(size - startOffset, map.Size);
|
||||
|
||||
Base.MapView(_backingMemory, mapPa, mapVa, mapSize);
|
||||
Mirror.MapView(_backingMemory, mapPa, mapVa, mapSize);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
map.UpdateState(type);
|
||||
map = TryCoalesce(map);
|
||||
|
||||
if (map.EndAddress >= endAddress)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
private Mapping TryCoalesce(Mapping map)
|
||||
{
|
||||
Mapping previousMap = map.Predecessor;
|
||||
Mapping nextMap = map.Successor;
|
||||
|
||||
if (previousMap != null && CanCoalesce(previousMap, map))
|
||||
{
|
||||
previousMap.Extend(map.Size);
|
||||
_mappingTree.Remove(map);
|
||||
map = previousMap;
|
||||
}
|
||||
|
||||
if (nextMap != null && CanCoalesce(map, nextMap))
|
||||
{
|
||||
map.Extend(nextMap.Size);
|
||||
_mappingTree.Remove(nextMap);
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
private static bool CanCoalesce(Mapping left, Mapping right)
|
||||
{
|
||||
return left.Type == right.Type;
|
||||
}
|
||||
|
||||
private void MapPrivate(ulong va, ulong size)
|
||||
{
|
||||
ulong endAddress = va + size;
|
||||
|
||||
ulong alignment = MemoryBlock.GetPageSize();
|
||||
|
||||
// Expand the range outwards based on page size to ensure that at least the requested region is mapped.
|
||||
ulong vaAligned = BitUtils.AlignDown(va, alignment);
|
||||
ulong endAddressAligned = BitUtils.AlignUp(endAddress, alignment);
|
||||
|
||||
ulong sizeAligned = endAddressAligned - vaAligned;
|
||||
|
||||
PrivateMapping map = _privateTree.GetNode(new PrivateMapping(va, 1UL, default));
|
||||
|
||||
for (; map != null; map = map.Successor)
|
||||
{
|
||||
if (!map.PrivateAllocation.IsValid)
|
||||
{
|
||||
if (map.Address < vaAligned)
|
||||
{
|
||||
_privateTree.Add(map.Split(vaAligned));
|
||||
}
|
||||
|
||||
if (map.EndAddress > endAddressAligned)
|
||||
{
|
||||
PrivateMapping newMap = map.Split(endAddressAligned);
|
||||
_privateTree.Add(newMap);
|
||||
map = newMap;
|
||||
}
|
||||
|
||||
map.Map(Base, Mirror, _privateMemoryAllocator.Allocate(map.Size, MemoryBlock.GetPageSize()));
|
||||
}
|
||||
|
||||
if (map.EndAddress >= endAddressAligned)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void UnmapPrivate(ulong va, ulong size)
|
||||
{
|
||||
ulong endAddress = va + size;
|
||||
|
||||
ulong alignment = MemoryBlock.GetPageSize();
|
||||
|
||||
// Shrink the range inwards based on page size to ensure we won't unmap memory that might be still in use.
|
||||
ulong vaAligned = BitUtils.AlignUp(va, alignment);
|
||||
ulong endAddressAligned = BitUtils.AlignDown(endAddress, alignment);
|
||||
|
||||
if (endAddressAligned <= vaAligned)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ulong alignedSize = endAddressAligned - vaAligned;
|
||||
|
||||
PrivateMapping map = _privateTree.GetNode(new PrivateMapping(va, 1UL, default));
|
||||
|
||||
for (; map != null; map = map.Successor)
|
||||
{
|
||||
if (map.PrivateAllocation.IsValid)
|
||||
{
|
||||
if (map.Address < vaAligned)
|
||||
{
|
||||
_privateTree.Add(map.Split(vaAligned));
|
||||
}
|
||||
|
||||
if (map.EndAddress > endAddressAligned)
|
||||
{
|
||||
PrivateMapping newMap = map.Split(endAddressAligned);
|
||||
_privateTree.Add(newMap);
|
||||
map = newMap;
|
||||
}
|
||||
|
||||
map.Unmap(Base, Mirror);
|
||||
map = TryCoalesce(map);
|
||||
}
|
||||
|
||||
if (map.EndAddress >= endAddressAligned)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private PrivateMapping TryCoalesce(PrivateMapping map)
|
||||
{
|
||||
PrivateMapping previousMap = map.Predecessor;
|
||||
PrivateMapping nextMap = map.Successor;
|
||||
|
||||
if (previousMap != null && CanCoalesce(previousMap, map))
|
||||
{
|
||||
previousMap.Extend(map.Size);
|
||||
_privateTree.Remove(map);
|
||||
map = previousMap;
|
||||
}
|
||||
|
||||
if (nextMap != null && CanCoalesce(map, nextMap))
|
||||
{
|
||||
map.Extend(nextMap.Size);
|
||||
_privateTree.Remove(nextMap);
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
private static bool CanCoalesce(PrivateMapping left, PrivateMapping right)
|
||||
{
|
||||
return !left.PrivateAllocation.IsValid && !right.PrivateAllocation.IsValid;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_privateMemoryAllocator?.Dispose();
|
||||
Base.Dispose();
|
||||
Mirror.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
27
src/Ryujinx.Cpu/AppleHv/Arm/ApFlags.cs
Normal file
27
src/Ryujinx.Cpu/AppleHv/Arm/ApFlags.cs
Normal file
|
@ -0,0 +1,27 @@
|
|||
namespace Ryujinx.Cpu.AppleHv.Arm
|
||||
{
|
||||
enum ApFlags : ulong
|
||||
{
|
||||
ApShift = 6,
|
||||
PxnShift = 53,
|
||||
UxnShift = 54,
|
||||
|
||||
UserExecuteKernelReadWriteExecute = (0UL << (int)ApShift),
|
||||
UserReadWriteExecuteKernelReadWrite = (1UL << (int)ApShift),
|
||||
UserExecuteKernelReadExecute = (2UL << (int)ApShift),
|
||||
UserReadExecuteKernelReadExecute = (3UL << (int)ApShift),
|
||||
|
||||
UserExecuteKernelReadWrite = (1UL << (int)PxnShift) | (0UL << (int)ApShift),
|
||||
UserExecuteKernelRead = (1UL << (int)PxnShift) | (2UL << (int)ApShift),
|
||||
UserReadExecuteKernelRead = (1UL << (int)PxnShift) | (3UL << (int)ApShift),
|
||||
|
||||
UserNoneKernelReadWriteExecute = (1UL << (int)UxnShift) | (0UL << (int)ApShift),
|
||||
UserReadWriteKernelReadWrite = (1UL << (int)UxnShift) | (1UL << (int)ApShift),
|
||||
UserNoneKernelReadExecute = (1UL << (int)UxnShift) | (2UL << (int)ApShift),
|
||||
UserReadKernelReadExecute = (1UL << (int)UxnShift) | (3UL << (int)ApShift),
|
||||
|
||||
UserNoneKernelReadWrite = (1UL << (int)PxnShift) | (1UL << (int)UxnShift) | (0UL << (int)ApShift),
|
||||
UserNoneKernelRead = (1UL << (int)PxnShift) | (1UL << (int)UxnShift) | (2UL << (int)ApShift),
|
||||
UserReadKernelRead = (1UL << (int)PxnShift) | (1UL << (int)UxnShift) | (3UL << (int)ApShift)
|
||||
}
|
||||
}
|
47
src/Ryujinx.Cpu/AppleHv/Arm/ExceptionClass.cs
Normal file
47
src/Ryujinx.Cpu/AppleHv/Arm/ExceptionClass.cs
Normal file
|
@ -0,0 +1,47 @@
|
|||
namespace Ryujinx.Cpu.AppleHv.Arm
|
||||
{
|
||||
enum ExceptionClass
|
||||
{
|
||||
Unknown = 0b000000,
|
||||
TrappedWfeWfiWfetWfit = 0b000001,
|
||||
TrappedMcrMrcCp15 = 0b000011,
|
||||
TrappedMcrrMrrcCp15 = 0b000100,
|
||||
TrappedMcrMrcCp14 = 0b000101,
|
||||
TrappedLdcStc = 0b000110,
|
||||
TrappedSveFpSimd = 0b000111,
|
||||
TrappedVmrs = 0b001000,
|
||||
TrappedPAuth = 0b001001,
|
||||
TrappedLd64bSt64bSt64bvSt64bv0 = 0b001010,
|
||||
TrappedMrrcCp14 = 0b001100,
|
||||
IllegalExecutionState = 0b001110,
|
||||
SvcAarch32 = 0b010001,
|
||||
HvcAarch32 = 0b010010,
|
||||
SmcAarch32 = 0b010011,
|
||||
SvcAarch64 = 0b010101,
|
||||
HvcAarch64 = 0b010110,
|
||||
SmcAarch64 = 0b010111,
|
||||
TrappedMsrMrsSystem = 0b011000,
|
||||
TrappedSve = 0b011001,
|
||||
TrappedEretEretaaEretab = 0b011010,
|
||||
PointerAuthenticationFailure = 0b011100,
|
||||
ImplementationDefinedEl3 = 0b011111,
|
||||
InstructionAbortLowerEl = 0b100000,
|
||||
InstructionAbortSameEl = 0b100001,
|
||||
PcAlignmentFault = 0b100010,
|
||||
DataAbortLowerEl = 0b100100,
|
||||
DataAbortSameEl = 0b100101,
|
||||
SpAlignmentFault = 0b100110,
|
||||
TrappedFpExceptionAarch32 = 0b101000,
|
||||
TrappedFpExceptionAarch64 = 0b101100,
|
||||
SErrorInterrupt = 0b101111,
|
||||
BreakpointLowerEl = 0b110000,
|
||||
BreakpointSameEl = 0b110001,
|
||||
SoftwareStepLowerEl = 0b110010,
|
||||
SoftwareStepSameEl = 0b110011,
|
||||
WatchpointLowerEl = 0b110100,
|
||||
WatchpointSameEl = 0b110101,
|
||||
BkptAarch32 = 0b111000,
|
||||
VectorCatchAarch32 = 0b111010,
|
||||
BrkAarch64 = 0b111100
|
||||
}
|
||||
}
|
17
src/Ryujinx.Cpu/AppleHv/DummyDiskCacheLoadState.cs
Normal file
17
src/Ryujinx.Cpu/AppleHv/DummyDiskCacheLoadState.cs
Normal file
|
@ -0,0 +1,17 @@
|
|||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
public class DummyDiskCacheLoadState : IDiskCacheLoadState
|
||||
{
|
||||
#pragma warning disable CS0067
|
||||
/// <inheritdoc/>
|
||||
public event Action<LoadState, int, int> StateChanged;
|
||||
#pragma warning restore CS0067
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Cancel()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
129
src/Ryujinx.Cpu/AppleHv/HvAddressSpace.cs
Normal file
129
src/Ryujinx.Cpu/AppleHv/HvAddressSpace.cs
Normal file
|
@ -0,0 +1,129 @@
|
|||
using Ryujinx.Cpu.AppleHv.Arm;
|
||||
using Ryujinx.Memory;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
class HvAddressSpace : IDisposable
|
||||
{
|
||||
private const ulong KernelRegionBase = unchecked((ulong)-(1L << 39));
|
||||
private const ulong KernelRegionCodeOffset = 0UL;
|
||||
private const ulong KernelRegionCodeSize = 0x2000UL;
|
||||
private const ulong KernelRegionTlbiEretOffset = KernelRegionCodeOffset + 0x1000UL;
|
||||
private const ulong KernelRegionEretOffset = KernelRegionTlbiEretOffset + 4UL;
|
||||
|
||||
public const ulong KernelRegionEretAddress = KernelRegionBase + KernelRegionEretOffset;
|
||||
public const ulong KernelRegionTlbiEretAddress = KernelRegionBase + KernelRegionTlbiEretOffset;
|
||||
|
||||
private const ulong AllocationGranule = 1UL << 14;
|
||||
|
||||
private readonly ulong _asBase;
|
||||
private readonly ulong _asSize;
|
||||
private readonly ulong _backingSize;
|
||||
|
||||
private readonly HvAddressSpaceRange _userRange;
|
||||
private readonly HvAddressSpaceRange _kernelRange;
|
||||
|
||||
private MemoryBlock _kernelCodeBlock;
|
||||
|
||||
public HvAddressSpace(MemoryBlock backingMemory, ulong asSize)
|
||||
{
|
||||
(_asBase, var ipaAllocator) = HvVm.CreateAddressSpace(backingMemory);
|
||||
_asSize = asSize;
|
||||
_backingSize = backingMemory.Size;
|
||||
|
||||
_userRange = new HvAddressSpaceRange(ipaAllocator);
|
||||
_kernelRange = new HvAddressSpaceRange(ipaAllocator);
|
||||
|
||||
_kernelCodeBlock = new MemoryBlock(AllocationGranule);
|
||||
|
||||
InitializeKernelCode(ipaAllocator);
|
||||
}
|
||||
|
||||
private void InitializeKernelCode(HvIpaAllocator ipaAllocator)
|
||||
{
|
||||
// Write exception handlers.
|
||||
for (ulong offset = 0; offset < 0x800; offset += 0x80)
|
||||
{
|
||||
// Offsets:
|
||||
// 0x0: Synchronous
|
||||
// 0x80: IRQ
|
||||
// 0x100: FIQ
|
||||
// 0x180: SError
|
||||
_kernelCodeBlock.Write(KernelRegionCodeOffset + offset, 0xD41FFFE2u); // HVC #0xFFFF
|
||||
_kernelCodeBlock.Write(KernelRegionCodeOffset + offset + 4, 0xD69F03E0u); // ERET
|
||||
}
|
||||
|
||||
_kernelCodeBlock.Write(KernelRegionTlbiEretOffset, 0xD508831Fu); // TLBI VMALLE1IS
|
||||
_kernelCodeBlock.Write(KernelRegionEretOffset, 0xD69F03E0u); // ERET
|
||||
|
||||
ulong kernelCodePa = ipaAllocator.Allocate(AllocationGranule);
|
||||
HvApi.hv_vm_map((ulong)_kernelCodeBlock.Pointer, kernelCodePa, AllocationGranule, hv_memory_flags_t.HV_MEMORY_READ | hv_memory_flags_t.HV_MEMORY_EXEC).ThrowOnError();
|
||||
|
||||
_kernelRange.Map(KernelRegionCodeOffset, kernelCodePa, KernelRegionCodeSize, ApFlags.UserNoneKernelReadExecute);
|
||||
}
|
||||
|
||||
public void InitializeMmu(ulong vcpu)
|
||||
{
|
||||
HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_VBAR_EL1, KernelRegionBase + KernelRegionCodeOffset);
|
||||
|
||||
HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_TTBR0_EL1, _userRange.GetIpaBase());
|
||||
HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_TTBR1_EL1, _kernelRange.GetIpaBase());
|
||||
HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_MAIR_EL1, 0xffUL);
|
||||
HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_TCR_EL1, 0x00000011B5193519UL);
|
||||
HvApi.hv_vcpu_set_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_SCTLR_EL1, 0x0000000034D5D925UL);
|
||||
}
|
||||
|
||||
public bool GetAndClearUserTlbInvalidationPending()
|
||||
{
|
||||
return _userRange.GetAndClearTlbInvalidationPending();
|
||||
}
|
||||
|
||||
public void MapUser(ulong va, ulong pa, ulong size, MemoryPermission permission)
|
||||
{
|
||||
pa += _asBase;
|
||||
|
||||
lock (_userRange)
|
||||
{
|
||||
_userRange.Map(va, pa, size, GetApFlags(permission));
|
||||
}
|
||||
}
|
||||
|
||||
public void UnmapUser(ulong va, ulong size)
|
||||
{
|
||||
lock (_userRange)
|
||||
{
|
||||
_userRange.Unmap(va, size);
|
||||
}
|
||||
}
|
||||
|
||||
public void ReprotectUser(ulong va, ulong size, MemoryPermission permission)
|
||||
{
|
||||
lock (_userRange)
|
||||
{
|
||||
_userRange.Reprotect(va, size, GetApFlags(permission));
|
||||
}
|
||||
}
|
||||
|
||||
private static ApFlags GetApFlags(MemoryPermission permission)
|
||||
{
|
||||
return permission switch
|
||||
{
|
||||
MemoryPermission.None => ApFlags.UserNoneKernelRead,
|
||||
MemoryPermission.Execute => ApFlags.UserExecuteKernelRead,
|
||||
MemoryPermission.Read => ApFlags.UserReadKernelRead,
|
||||
MemoryPermission.ReadAndWrite => ApFlags.UserReadWriteKernelReadWrite,
|
||||
MemoryPermission.ReadAndExecute => ApFlags.UserReadExecuteKernelRead,
|
||||
MemoryPermission.ReadWriteExecute => ApFlags.UserReadWriteExecuteKernelReadWrite,
|
||||
_ => throw new ArgumentException($"Permission \"{permission}\" is invalid.")
|
||||
};
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_userRange.Dispose();
|
||||
_kernelRange.Dispose();
|
||||
HvVm.DestroyAddressSpace(_asBase, _backingSize);
|
||||
}
|
||||
}
|
||||
}
|
370
src/Ryujinx.Cpu/AppleHv/HvAddressSpaceRange.cs
Normal file
370
src/Ryujinx.Cpu/AppleHv/HvAddressSpaceRange.cs
Normal file
|
@ -0,0 +1,370 @@
|
|||
using Ryujinx.Cpu.AppleHv.Arm;
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
class HvAddressSpaceRange : IDisposable
|
||||
{
|
||||
private const ulong AllocationGranule = 1UL << 14;
|
||||
|
||||
private const ulong AttributesMask = (0x3ffUL << 2) | (0x3fffUL << 50);
|
||||
|
||||
private const ulong BaseAttributes = (1UL << 10) | (3UL << 8); // Access flag set, inner shareable.
|
||||
|
||||
private const int LevelBits = 9;
|
||||
private const int LevelCount = 1 << LevelBits;
|
||||
private const int LevelMask = LevelCount - 1;
|
||||
private const int PageBits = 12;
|
||||
private const int PageSize = 1 << PageBits;
|
||||
private const int PageMask = PageSize - 1;
|
||||
private const int AllLevelsMask = PageMask | (LevelMask << PageBits) | (LevelMask << (PageBits + LevelBits));
|
||||
|
||||
private class PtLevel
|
||||
{
|
||||
public ulong Address => Allocation.Ipa + Allocation.Offset;
|
||||
public int EntriesCount;
|
||||
public readonly HvMemoryBlockAllocation Allocation;
|
||||
public readonly PtLevel[] Next;
|
||||
|
||||
public PtLevel(HvMemoryBlockAllocator blockAllocator, int count, bool hasNext)
|
||||
{
|
||||
ulong size = (ulong)count * sizeof(ulong);
|
||||
Allocation = blockAllocator.Allocate(size, PageSize);
|
||||
|
||||
AsSpan().Fill(0UL);
|
||||
|
||||
if (hasNext)
|
||||
{
|
||||
Next = new PtLevel[count];
|
||||
}
|
||||
}
|
||||
|
||||
public unsafe Span<ulong> AsSpan()
|
||||
{
|
||||
return MemoryMarshal.Cast<byte, ulong>(Allocation.Memory.GetSpan(Allocation.Offset, (int)Allocation.Size));
|
||||
}
|
||||
}
|
||||
|
||||
private PtLevel _level0;
|
||||
|
||||
private int _tlbInvalidationPending;
|
||||
|
||||
private readonly HvIpaAllocator _ipaAllocator;
|
||||
private readonly HvMemoryBlockAllocator _blockAllocator;
|
||||
|
||||
public HvAddressSpaceRange(HvIpaAllocator ipaAllocator)
|
||||
{
|
||||
_ipaAllocator = ipaAllocator;
|
||||
_blockAllocator = new HvMemoryBlockAllocator(ipaAllocator, (int)AllocationGranule);
|
||||
}
|
||||
|
||||
public ulong GetIpaBase()
|
||||
{
|
||||
return EnsureLevel0().Address;
|
||||
}
|
||||
|
||||
public bool GetAndClearTlbInvalidationPending()
|
||||
{
|
||||
return Interlocked.Exchange(ref _tlbInvalidationPending, 0) != 0;
|
||||
}
|
||||
|
||||
public void Map(ulong va, ulong pa, ulong size, ApFlags accessPermission)
|
||||
{
|
||||
MapImpl(va, pa, size, (ulong)accessPermission | BaseAttributes);
|
||||
}
|
||||
|
||||
public void Unmap(ulong va, ulong size)
|
||||
{
|
||||
UnmapImpl(EnsureLevel0(), 0, va, size);
|
||||
Interlocked.Exchange(ref _tlbInvalidationPending, 1);
|
||||
}
|
||||
|
||||
public void Reprotect(ulong va, ulong size, ApFlags accessPermission)
|
||||
{
|
||||
UpdateAttributes(va, size, (ulong)accessPermission | BaseAttributes);
|
||||
}
|
||||
|
||||
private void MapImpl(ulong va, ulong pa, ulong size, ulong attr)
|
||||
{
|
||||
PtLevel level0 = EnsureLevel0();
|
||||
|
||||
ulong endVa = va + size;
|
||||
|
||||
while (va < endVa)
|
||||
{
|
||||
(ulong mapSize, int depth) = GetMapSizeAndDepth(va, pa, endVa);
|
||||
|
||||
PtLevel currentLevel = level0;
|
||||
|
||||
for (int i = 0; i < depth; i++)
|
||||
{
|
||||
int l = (int)(va >> (PageBits + (2 - i) * LevelBits)) & LevelMask;
|
||||
EnsureTable(currentLevel, l, i == 0);
|
||||
currentLevel = currentLevel.Next[l];
|
||||
}
|
||||
|
||||
(ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth);
|
||||
|
||||
for (ulong i = 0; i < mapSize; i += blockSize)
|
||||
{
|
||||
if ((va >> blockShift) << blockShift != va ||
|
||||
(pa >> blockShift) << blockShift != pa)
|
||||
{
|
||||
Debug.Fail($"Block size 0x{blockSize:X} (log2: {blockShift}) is invalid for VA 0x{va:X} or PA 0x{pa:X}.");
|
||||
}
|
||||
|
||||
WriteBlock(currentLevel, (int)(va >> blockShift) & LevelMask, depth, pa, attr);
|
||||
|
||||
va += blockSize;
|
||||
pa += blockSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void UnmapImpl(PtLevel level, int depth, ulong va, ulong size)
|
||||
{
|
||||
ulong endVa = (va + size + PageMask) & ~((ulong)PageMask);
|
||||
va &= ~((ulong)PageMask);
|
||||
|
||||
(ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth);
|
||||
|
||||
while (va < endVa)
|
||||
{
|
||||
ulong nextEntryVa = GetNextAddress(va, blockSize);
|
||||
ulong chunckSize = Math.Min(endVa - va, nextEntryVa - va);
|
||||
|
||||
int l = (int)(va >> (PageBits + (2 - depth) * LevelBits)) & LevelMask;
|
||||
|
||||
PtLevel nextTable = level.Next != null ? level.Next[l] : null;
|
||||
|
||||
if (nextTable != null)
|
||||
{
|
||||
// Entry is a table, visit it and update attributes as required.
|
||||
UnmapImpl(nextTable, depth + 1, va, chunckSize);
|
||||
}
|
||||
else if (chunckSize != blockSize)
|
||||
{
|
||||
// Entry is a block but is not aligned, we need to turn it into a table.
|
||||
ref ulong pte = ref level.AsSpan()[l];
|
||||
nextTable = CreateTable(pte, depth + 1);
|
||||
level.Next[l] = nextTable;
|
||||
|
||||
// Now that we have a table, we can handle it like the first case.
|
||||
UnmapImpl(nextTable, depth + 1, va, chunckSize);
|
||||
|
||||
// Update PTE to point to the new table.
|
||||
pte = (nextTable.Address & ~(ulong)PageMask) | 3UL;
|
||||
}
|
||||
|
||||
// If entry is a block, or if entry is a table but it is empty, we can remove it.
|
||||
if (nextTable == null || nextTable.EntriesCount == 0)
|
||||
{
|
||||
// Entry is a block and is fully aligned, so we can just set it to 0.
|
||||
if (nextTable != null)
|
||||
{
|
||||
nextTable.Allocation.Dispose();
|
||||
level.Next[l] = null;
|
||||
}
|
||||
|
||||
level.AsSpan()[l] = 0UL;
|
||||
level.EntriesCount--;
|
||||
ValidateEntriesCount(level.EntriesCount);
|
||||
}
|
||||
|
||||
va += chunckSize;
|
||||
}
|
||||
}
|
||||
|
||||
private void UpdateAttributes(ulong va, ulong size, ulong newAttr)
|
||||
{
|
||||
UpdateAttributes(EnsureLevel0(), 0, va, size, newAttr);
|
||||
|
||||
Interlocked.Exchange(ref _tlbInvalidationPending, 1);
|
||||
}
|
||||
|
||||
private void UpdateAttributes(PtLevel level, int depth, ulong va, ulong size, ulong newAttr)
|
||||
{
|
||||
ulong endVa = (va + size + PageSize - 1) & ~((ulong)PageSize - 1);
|
||||
va &= ~((ulong)PageSize - 1);
|
||||
|
||||
(ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth);
|
||||
|
||||
while (va < endVa)
|
||||
{
|
||||
ulong nextEntryVa = GetNextAddress(va, blockSize);
|
||||
ulong chunckSize = Math.Min(endVa - va, nextEntryVa - va);
|
||||
|
||||
int l = (int)(va >> (PageBits + (2 - depth) * LevelBits)) & LevelMask;
|
||||
|
||||
ref ulong pte = ref level.AsSpan()[l];
|
||||
|
||||
// First check if the region is mapped.
|
||||
if ((pte & 3) != 0)
|
||||
{
|
||||
PtLevel nextTable = level.Next != null ? level.Next[l] : null;
|
||||
|
||||
if (nextTable != null)
|
||||
{
|
||||
// Entry is a table, visit it and update attributes as required.
|
||||
UpdateAttributes(nextTable, depth + 1, va, chunckSize, newAttr);
|
||||
}
|
||||
else if (chunckSize != blockSize)
|
||||
{
|
||||
// Entry is a block but is not aligned, we need to turn it into a table.
|
||||
nextTable = CreateTable(pte, depth + 1);
|
||||
level.Next[l] = nextTable;
|
||||
|
||||
// Now that we have a table, we can handle it like the first case.
|
||||
UpdateAttributes(nextTable, depth + 1, va, chunckSize, newAttr);
|
||||
|
||||
// Update PTE to point to the new table.
|
||||
pte = (nextTable.Address & ~(ulong)PageMask) | 3UL;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Entry is a block and is fully aligned, so we can just update the attributes.
|
||||
// Update PTE with the new attributes.
|
||||
pte = (pte & ~AttributesMask) | newAttr;
|
||||
}
|
||||
}
|
||||
|
||||
va += chunckSize;
|
||||
}
|
||||
}
|
||||
|
||||
private PtLevel CreateTable(ulong pte, int depth)
|
||||
{
|
||||
pte &= ~3UL;
|
||||
pte |= (depth == 2 ? 3UL : 1UL);
|
||||
|
||||
PtLevel level = new PtLevel(_blockAllocator, LevelCount, depth < 2);
|
||||
Span<ulong> currentLevel = level.AsSpan();
|
||||
|
||||
(ulong blockSize, int blockShift) = GetBlockSizeAndShift(depth);
|
||||
|
||||
// Fill in the blocks.
|
||||
for (int i = 0; i < LevelCount; i++)
|
||||
{
|
||||
ulong offset = (ulong)i << blockShift;
|
||||
currentLevel[i] = pte + offset;
|
||||
}
|
||||
|
||||
level.EntriesCount = LevelCount;
|
||||
|
||||
return level;
|
||||
}
|
||||
|
||||
private static (ulong, int) GetBlockSizeAndShift(int depth)
|
||||
{
|
||||
int blockShift = PageBits + (2 - depth) * LevelBits;
|
||||
ulong blockSize = 1UL << blockShift;
|
||||
|
||||
return (blockSize, blockShift);
|
||||
}
|
||||
|
||||
private static (ulong, int) GetMapSizeAndDepth(ulong va, ulong pa, ulong endVa)
|
||||
{
|
||||
// Both virtual and physical addresses must be aligned to the block size.
|
||||
ulong combinedAddress = va | pa;
|
||||
|
||||
ulong l0Alignment = 1UL << (PageBits + LevelBits * 2);
|
||||
ulong l1Alignment = 1UL << (PageBits + LevelBits);
|
||||
|
||||
if ((combinedAddress & (l0Alignment - 1)) == 0 && AlignDown(endVa, l0Alignment) > va)
|
||||
{
|
||||
return (AlignDown(endVa, l0Alignment) - va, 0);
|
||||
}
|
||||
else if ((combinedAddress & (l1Alignment - 1)) == 0 && AlignDown(endVa, l1Alignment) > va)
|
||||
{
|
||||
ulong nextOrderVa = GetNextAddress(va, l0Alignment);
|
||||
|
||||
if (nextOrderVa <= endVa)
|
||||
{
|
||||
return (nextOrderVa - va, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
return (AlignDown(endVa, l1Alignment) - va, 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong nextOrderVa = GetNextAddress(va, l1Alignment);
|
||||
|
||||
if (nextOrderVa <= endVa)
|
||||
{
|
||||
return (nextOrderVa - va, 2);
|
||||
}
|
||||
else
|
||||
{
|
||||
return (endVa - va, 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static ulong AlignDown(ulong va, ulong alignment)
|
||||
{
|
||||
return va & ~(alignment - 1);
|
||||
}
|
||||
|
||||
private static ulong GetNextAddress(ulong va, ulong alignment)
|
||||
{
|
||||
return (va + alignment) & ~(alignment - 1);
|
||||
}
|
||||
|
||||
private PtLevel EnsureLevel0()
|
||||
{
|
||||
PtLevel level0 = _level0;
|
||||
|
||||
if (level0 == null)
|
||||
{
|
||||
level0 = new PtLevel(_blockAllocator, LevelCount, true);
|
||||
_level0 = level0;
|
||||
}
|
||||
|
||||
return level0;
|
||||
}
|
||||
|
||||
private void EnsureTable(PtLevel level, int index, bool hasNext)
|
||||
{
|
||||
Span<ulong> currentTable = level.AsSpan();
|
||||
|
||||
if ((currentTable[index] & 1) == 0)
|
||||
{
|
||||
PtLevel nextLevel = new PtLevel(_blockAllocator, LevelCount, hasNext);
|
||||
|
||||
currentTable[index] = (nextLevel.Address & ~(ulong)PageMask) | 3UL;
|
||||
level.Next[index] = nextLevel;
|
||||
level.EntriesCount++;
|
||||
ValidateEntriesCount(level.EntriesCount);
|
||||
}
|
||||
else if (level.Next[index] == null)
|
||||
{
|
||||
Debug.Fail($"Index {index} is block, expected a table.");
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteBlock(PtLevel level, int index, int depth, ulong pa, ulong attr)
|
||||
{
|
||||
Span<ulong> currentTable = level.AsSpan();
|
||||
|
||||
currentTable[index] = (pa & ~((ulong)AllLevelsMask >> (depth * LevelBits))) | (depth == 2 ? 3UL : 1UL) | attr;
|
||||
|
||||
level.EntriesCount++;
|
||||
ValidateEntriesCount(level.EntriesCount);
|
||||
}
|
||||
|
||||
private static void ValidateEntriesCount(int count)
|
||||
{
|
||||
Debug.Assert(count >= 0 && count <= LevelCount, $"Entries count {count} is invalid.");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_blockAllocator.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
320
src/Ryujinx.Cpu/AppleHv/HvApi.cs
Normal file
320
src/Ryujinx.Cpu/AppleHv/HvApi.cs
Normal file
|
@ -0,0 +1,320 @@
|
|||
using ARMeilleure.State;
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
struct hv_vcpu_exit_exception_t
|
||||
{
|
||||
#pragma warning disable CS0649
|
||||
public ulong syndrome;
|
||||
public ulong virtual_address;
|
||||
public ulong physical_address;
|
||||
#pragma warning restore CS0649
|
||||
}
|
||||
|
||||
struct hv_vcpu_exit_t
|
||||
{
|
||||
#pragma warning disable CS0649
|
||||
public uint reason;
|
||||
public hv_vcpu_exit_exception_t exception;
|
||||
#pragma warning restore CS0649
|
||||
}
|
||||
|
||||
enum hv_reg_t : uint
|
||||
{
|
||||
HV_REG_X0,
|
||||
HV_REG_X1,
|
||||
HV_REG_X2,
|
||||
HV_REG_X3,
|
||||
HV_REG_X4,
|
||||
HV_REG_X5,
|
||||
HV_REG_X6,
|
||||
HV_REG_X7,
|
||||
HV_REG_X8,
|
||||
HV_REG_X9,
|
||||
HV_REG_X10,
|
||||
HV_REG_X11,
|
||||
HV_REG_X12,
|
||||
HV_REG_X13,
|
||||
HV_REG_X14,
|
||||
HV_REG_X15,
|
||||
HV_REG_X16,
|
||||
HV_REG_X17,
|
||||
HV_REG_X18,
|
||||
HV_REG_X19,
|
||||
HV_REG_X20,
|
||||
HV_REG_X21,
|
||||
HV_REG_X22,
|
||||
HV_REG_X23,
|
||||
HV_REG_X24,
|
||||
HV_REG_X25,
|
||||
HV_REG_X26,
|
||||
HV_REG_X27,
|
||||
HV_REG_X28,
|
||||
HV_REG_X29,
|
||||
HV_REG_FP = HV_REG_X29,
|
||||
HV_REG_X30,
|
||||
HV_REG_LR = HV_REG_X30,
|
||||
HV_REG_PC,
|
||||
HV_REG_FPCR,
|
||||
HV_REG_FPSR,
|
||||
HV_REG_CPSR,
|
||||
}
|
||||
|
||||
enum hv_simd_fp_reg_t : uint
|
||||
{
|
||||
HV_SIMD_FP_REG_Q0,
|
||||
HV_SIMD_FP_REG_Q1,
|
||||
HV_SIMD_FP_REG_Q2,
|
||||
HV_SIMD_FP_REG_Q3,
|
||||
HV_SIMD_FP_REG_Q4,
|
||||
HV_SIMD_FP_REG_Q5,
|
||||
HV_SIMD_FP_REG_Q6,
|
||||
HV_SIMD_FP_REG_Q7,
|
||||
HV_SIMD_FP_REG_Q8,
|
||||
HV_SIMD_FP_REG_Q9,
|
||||
HV_SIMD_FP_REG_Q10,
|
||||
HV_SIMD_FP_REG_Q11,
|
||||
HV_SIMD_FP_REG_Q12,
|
||||
HV_SIMD_FP_REG_Q13,
|
||||
HV_SIMD_FP_REG_Q14,
|
||||
HV_SIMD_FP_REG_Q15,
|
||||
HV_SIMD_FP_REG_Q16,
|
||||
HV_SIMD_FP_REG_Q17,
|
||||
HV_SIMD_FP_REG_Q18,
|
||||
HV_SIMD_FP_REG_Q19,
|
||||
HV_SIMD_FP_REG_Q20,
|
||||
HV_SIMD_FP_REG_Q21,
|
||||
HV_SIMD_FP_REG_Q22,
|
||||
HV_SIMD_FP_REG_Q23,
|
||||
HV_SIMD_FP_REG_Q24,
|
||||
HV_SIMD_FP_REG_Q25,
|
||||
HV_SIMD_FP_REG_Q26,
|
||||
HV_SIMD_FP_REG_Q27,
|
||||
HV_SIMD_FP_REG_Q28,
|
||||
HV_SIMD_FP_REG_Q29,
|
||||
HV_SIMD_FP_REG_Q30,
|
||||
HV_SIMD_FP_REG_Q31,
|
||||
}
|
||||
|
||||
enum hv_sys_reg_t : ushort
|
||||
{
|
||||
HV_SYS_REG_DBGBVR0_EL1 = 0x8004,
|
||||
HV_SYS_REG_DBGBCR0_EL1 = 0x8005,
|
||||
HV_SYS_REG_DBGWVR0_EL1 = 0x8006,
|
||||
HV_SYS_REG_DBGWCR0_EL1 = 0x8007,
|
||||
HV_SYS_REG_DBGBVR1_EL1 = 0x800c,
|
||||
HV_SYS_REG_DBGBCR1_EL1 = 0x800d,
|
||||
HV_SYS_REG_DBGWVR1_EL1 = 0x800e,
|
||||
HV_SYS_REG_DBGWCR1_EL1 = 0x800f,
|
||||
HV_SYS_REG_MDCCINT_EL1 = 0x8010,
|
||||
HV_SYS_REG_MDSCR_EL1 = 0x8012,
|
||||
HV_SYS_REG_DBGBVR2_EL1 = 0x8014,
|
||||
HV_SYS_REG_DBGBCR2_EL1 = 0x8015,
|
||||
HV_SYS_REG_DBGWVR2_EL1 = 0x8016,
|
||||
HV_SYS_REG_DBGWCR2_EL1 = 0x8017,
|
||||
HV_SYS_REG_DBGBVR3_EL1 = 0x801c,
|
||||
HV_SYS_REG_DBGBCR3_EL1 = 0x801d,
|
||||
HV_SYS_REG_DBGWVR3_EL1 = 0x801e,
|
||||
HV_SYS_REG_DBGWCR3_EL1 = 0x801f,
|
||||
HV_SYS_REG_DBGBVR4_EL1 = 0x8024,
|
||||
HV_SYS_REG_DBGBCR4_EL1 = 0x8025,
|
||||
HV_SYS_REG_DBGWVR4_EL1 = 0x8026,
|
||||
HV_SYS_REG_DBGWCR4_EL1 = 0x8027,
|
||||
HV_SYS_REG_DBGBVR5_EL1 = 0x802c,
|
||||
HV_SYS_REG_DBGBCR5_EL1 = 0x802d,
|
||||
HV_SYS_REG_DBGWVR5_EL1 = 0x802e,
|
||||
HV_SYS_REG_DBGWCR5_EL1 = 0x802f,
|
||||
HV_SYS_REG_DBGBVR6_EL1 = 0x8034,
|
||||
HV_SYS_REG_DBGBCR6_EL1 = 0x8035,
|
||||
HV_SYS_REG_DBGWVR6_EL1 = 0x8036,
|
||||
HV_SYS_REG_DBGWCR6_EL1 = 0x8037,
|
||||
HV_SYS_REG_DBGBVR7_EL1 = 0x803c,
|
||||
HV_SYS_REG_DBGBCR7_EL1 = 0x803d,
|
||||
HV_SYS_REG_DBGWVR7_EL1 = 0x803e,
|
||||
HV_SYS_REG_DBGWCR7_EL1 = 0x803f,
|
||||
HV_SYS_REG_DBGBVR8_EL1 = 0x8044,
|
||||
HV_SYS_REG_DBGBCR8_EL1 = 0x8045,
|
||||
HV_SYS_REG_DBGWVR8_EL1 = 0x8046,
|
||||
HV_SYS_REG_DBGWCR8_EL1 = 0x8047,
|
||||
HV_SYS_REG_DBGBVR9_EL1 = 0x804c,
|
||||
HV_SYS_REG_DBGBCR9_EL1 = 0x804d,
|
||||
HV_SYS_REG_DBGWVR9_EL1 = 0x804e,
|
||||
HV_SYS_REG_DBGWCR9_EL1 = 0x804f,
|
||||
HV_SYS_REG_DBGBVR10_EL1 = 0x8054,
|
||||
HV_SYS_REG_DBGBCR10_EL1 = 0x8055,
|
||||
HV_SYS_REG_DBGWVR10_EL1 = 0x8056,
|
||||
HV_SYS_REG_DBGWCR10_EL1 = 0x8057,
|
||||
HV_SYS_REG_DBGBVR11_EL1 = 0x805c,
|
||||
HV_SYS_REG_DBGBCR11_EL1 = 0x805d,
|
||||
HV_SYS_REG_DBGWVR11_EL1 = 0x805e,
|
||||
HV_SYS_REG_DBGWCR11_EL1 = 0x805f,
|
||||
HV_SYS_REG_DBGBVR12_EL1 = 0x8064,
|
||||
HV_SYS_REG_DBGBCR12_EL1 = 0x8065,
|
||||
HV_SYS_REG_DBGWVR12_EL1 = 0x8066,
|
||||
HV_SYS_REG_DBGWCR12_EL1 = 0x8067,
|
||||
HV_SYS_REG_DBGBVR13_EL1 = 0x806c,
|
||||
HV_SYS_REG_DBGBCR13_EL1 = 0x806d,
|
||||
HV_SYS_REG_DBGWVR13_EL1 = 0x806e,
|
||||
HV_SYS_REG_DBGWCR13_EL1 = 0x806f,
|
||||
HV_SYS_REG_DBGBVR14_EL1 = 0x8074,
|
||||
HV_SYS_REG_DBGBCR14_EL1 = 0x8075,
|
||||
HV_SYS_REG_DBGWVR14_EL1 = 0x8076,
|
||||
HV_SYS_REG_DBGWCR14_EL1 = 0x8077,
|
||||
HV_SYS_REG_DBGBVR15_EL1 = 0x807c,
|
||||
HV_SYS_REG_DBGBCR15_EL1 = 0x807d,
|
||||
HV_SYS_REG_DBGWVR15_EL1 = 0x807e,
|
||||
HV_SYS_REG_DBGWCR15_EL1 = 0x807f,
|
||||
HV_SYS_REG_MIDR_EL1 = 0xc000,
|
||||
HV_SYS_REG_MPIDR_EL1 = 0xc005,
|
||||
HV_SYS_REG_ID_AA64PFR0_EL1 = 0xc020,
|
||||
HV_SYS_REG_ID_AA64PFR1_EL1 = 0xc021,
|
||||
HV_SYS_REG_ID_AA64DFR0_EL1 = 0xc028,
|
||||
HV_SYS_REG_ID_AA64DFR1_EL1 = 0xc029,
|
||||
HV_SYS_REG_ID_AA64ISAR0_EL1 = 0xc030,
|
||||
HV_SYS_REG_ID_AA64ISAR1_EL1 = 0xc031,
|
||||
HV_SYS_REG_ID_AA64MMFR0_EL1 = 0xc038,
|
||||
HV_SYS_REG_ID_AA64MMFR1_EL1 = 0xc039,
|
||||
HV_SYS_REG_ID_AA64MMFR2_EL1 = 0xc03a,
|
||||
HV_SYS_REG_SCTLR_EL1 = 0xc080,
|
||||
HV_SYS_REG_CPACR_EL1 = 0xc082,
|
||||
HV_SYS_REG_TTBR0_EL1 = 0xc100,
|
||||
HV_SYS_REG_TTBR1_EL1 = 0xc101,
|
||||
HV_SYS_REG_TCR_EL1 = 0xc102,
|
||||
HV_SYS_REG_APIAKEYLO_EL1 = 0xc108,
|
||||
HV_SYS_REG_APIAKEYHI_EL1 = 0xc109,
|
||||
HV_SYS_REG_APIBKEYLO_EL1 = 0xc10a,
|
||||
HV_SYS_REG_APIBKEYHI_EL1 = 0xc10b,
|
||||
HV_SYS_REG_APDAKEYLO_EL1 = 0xc110,
|
||||
HV_SYS_REG_APDAKEYHI_EL1 = 0xc111,
|
||||
HV_SYS_REG_APDBKEYLO_EL1 = 0xc112,
|
||||
HV_SYS_REG_APDBKEYHI_EL1 = 0xc113,
|
||||
HV_SYS_REG_APGAKEYLO_EL1 = 0xc118,
|
||||
HV_SYS_REG_APGAKEYHI_EL1 = 0xc119,
|
||||
HV_SYS_REG_SPSR_EL1 = 0xc200,
|
||||
HV_SYS_REG_ELR_EL1 = 0xc201,
|
||||
HV_SYS_REG_SP_EL0 = 0xc208,
|
||||
HV_SYS_REG_AFSR0_EL1 = 0xc288,
|
||||
HV_SYS_REG_AFSR1_EL1 = 0xc289,
|
||||
HV_SYS_REG_ESR_EL1 = 0xc290,
|
||||
HV_SYS_REG_FAR_EL1 = 0xc300,
|
||||
HV_SYS_REG_PAR_EL1 = 0xc3a0,
|
||||
HV_SYS_REG_MAIR_EL1 = 0xc510,
|
||||
HV_SYS_REG_AMAIR_EL1 = 0xc518,
|
||||
HV_SYS_REG_VBAR_EL1 = 0xc600,
|
||||
HV_SYS_REG_CONTEXTIDR_EL1 = 0xc681,
|
||||
HV_SYS_REG_TPIDR_EL1 = 0xc684,
|
||||
HV_SYS_REG_CNTKCTL_EL1 = 0xc708,
|
||||
HV_SYS_REG_CSSELR_EL1 = 0xd000,
|
||||
HV_SYS_REG_TPIDR_EL0 = 0xde82,
|
||||
HV_SYS_REG_TPIDRRO_EL0 = 0xde83,
|
||||
HV_SYS_REG_CNTV_CTL_EL0 = 0xdf19,
|
||||
HV_SYS_REG_CNTV_CVAL_EL0 = 0xdf1a,
|
||||
HV_SYS_REG_SP_EL1 = 0xe208,
|
||||
}
|
||||
|
||||
enum hv_memory_flags_t : ulong
|
||||
{
|
||||
HV_MEMORY_READ = 1UL << 0,
|
||||
HV_MEMORY_WRITE = 1UL << 1,
|
||||
HV_MEMORY_EXEC = 1UL << 2
|
||||
}
|
||||
|
||||
enum hv_result_t : uint
|
||||
{
|
||||
HV_SUCCESS = 0,
|
||||
HV_ERROR = 0xfae94001,
|
||||
HV_BUSY = 0xfae94002,
|
||||
HV_BAD_ARGUMENT = 0xfae94003,
|
||||
HV_NO_RESOURCES = 0xfae94005,
|
||||
HV_NO_DEVICE = 0xfae94006,
|
||||
HV_DENIED = 0xfae94007,
|
||||
HV_UNSUPPORTED = 0xfae9400f
|
||||
}
|
||||
|
||||
enum hv_interrupt_type_t : uint
|
||||
{
|
||||
HV_INTERRUPT_TYPE_IRQ,
|
||||
HV_INTERRUPT_TYPE_FIQ
|
||||
}
|
||||
|
||||
struct hv_simd_fp_uchar16_t
|
||||
{
|
||||
public ulong Low;
|
||||
public ulong High;
|
||||
}
|
||||
|
||||
static class HvResultExtensions
|
||||
{
|
||||
public static void ThrowOnError(this hv_result_t result)
|
||||
{
|
||||
if (result != hv_result_t.HV_SUCCESS)
|
||||
{
|
||||
throw new Exception($"Unexpected result \"{result}\".");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static partial class HvApi
|
||||
{
|
||||
public const string LibraryName = "/System/Library/Frameworks/Hypervisor.framework/Hypervisor";
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vm_get_max_vcpu_count(out uint max_vcpu_count);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vm_create(IntPtr config);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vm_destroy();
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vm_map(ulong addr, ulong ipa, ulong size, hv_memory_flags_t flags);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vm_unmap(ulong ipa, ulong size);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vm_protect(ulong ipa, ulong size, hv_memory_flags_t flags);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public unsafe static partial hv_result_t hv_vcpu_create(out ulong vcpu, ref hv_vcpu_exit_t* exit, IntPtr config);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public unsafe static partial hv_result_t hv_vcpu_destroy(ulong vcpu);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_run(ulong vcpu);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpus_exit(ref ulong vcpus, uint vcpu_count);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_set_vtimer_mask(ulong vcpu, [MarshalAs(UnmanagedType.Bool)] bool vtimer_is_masked);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_get_reg(ulong vcpu, hv_reg_t reg, out ulong value);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_set_reg(ulong vcpu, hv_reg_t reg, ulong value);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_get_simd_fp_reg(ulong vcpu, hv_simd_fp_reg_t reg, out hv_simd_fp_uchar16_t value);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_set_simd_fp_reg(ulong vcpu, hv_simd_fp_reg_t reg, hv_simd_fp_uchar16_t value); // DO NOT USE DIRECTLY!
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_get_sys_reg(ulong vcpu, hv_sys_reg_t reg, out ulong value);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_set_sys_reg(ulong vcpu, hv_sys_reg_t reg, ulong value);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_get_pending_interrupt(ulong vcpu, hv_interrupt_type_t type, [MarshalAs(UnmanagedType.Bool)] out bool pending);
|
||||
|
||||
[LibraryImport(LibraryName, SetLastError = true)]
|
||||
public static partial hv_result_t hv_vcpu_set_pending_interrupt(ulong vcpu, hv_interrupt_type_t type, [MarshalAs(UnmanagedType.Bool)] bool pending);
|
||||
}
|
||||
}
|
47
src/Ryujinx.Cpu/AppleHv/HvCpuContext.cs
Normal file
47
src/Ryujinx.Cpu/AppleHv/HvCpuContext.cs
Normal file
|
@ -0,0 +1,47 @@
|
|||
using ARMeilleure.Memory;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
class HvCpuContext : ICpuContext
|
||||
{
|
||||
private readonly ITickSource _tickSource;
|
||||
private readonly HvMemoryManager _memoryManager;
|
||||
|
||||
public HvCpuContext(ITickSource tickSource, IMemoryManager memory, bool for64Bit)
|
||||
{
|
||||
_tickSource = tickSource;
|
||||
_memoryManager = (HvMemoryManager)memory;
|
||||
}
|
||||
|
||||
private void UnmapHandler(ulong address, ulong size)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks)
|
||||
{
|
||||
return new HvExecutionContext(_tickSource, exceptionCallbacks);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Execute(IExecutionContext context, ulong address)
|
||||
{
|
||||
((HvExecutionContext)context).Execute(_memoryManager, address);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void InvalidateCacheRegion(ulong address, ulong size)
|
||||
{
|
||||
}
|
||||
|
||||
public IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled)
|
||||
{
|
||||
return new DummyDiskCacheLoadState();
|
||||
}
|
||||
|
||||
public void PrepareCodeRange(ulong address, ulong size)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
20
src/Ryujinx.Cpu/AppleHv/HvEngine.cs
Normal file
20
src/Ryujinx.Cpu/AppleHv/HvEngine.cs
Normal file
|
@ -0,0 +1,20 @@
|
|||
using ARMeilleure.Memory;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
public class HvEngine : ICpuEngine
|
||||
{
|
||||
private readonly ITickSource _tickSource;
|
||||
|
||||
public HvEngine(ITickSource tickSource)
|
||||
{
|
||||
_tickSource = tickSource;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ICpuContext CreateCpuContext(IMemoryManager memoryManager, bool for64Bit)
|
||||
{
|
||||
return new HvCpuContext(_tickSource, memoryManager, for64Bit);
|
||||
}
|
||||
}
|
||||
}
|
284
src/Ryujinx.Cpu/AppleHv/HvExecutionContext.cs
Normal file
284
src/Ryujinx.Cpu/AppleHv/HvExecutionContext.cs
Normal file
|
@ -0,0 +1,284 @@
|
|||
using ARMeilleure.State;
|
||||
using Ryujinx.Cpu.AppleHv.Arm;
|
||||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
class HvExecutionContext : IExecutionContext
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public ulong Pc => _impl.ElrEl1;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public long TpidrEl0
|
||||
{
|
||||
get => _impl.TpidrEl0;
|
||||
set => _impl.TpidrEl0 = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public long TpidrroEl0
|
||||
{
|
||||
get => _impl.TpidrroEl0;
|
||||
set => _impl.TpidrroEl0 = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public uint Pstate
|
||||
{
|
||||
get => _impl.Pstate;
|
||||
set => _impl.Pstate = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public uint Fpcr
|
||||
{
|
||||
get => _impl.Fpcr;
|
||||
set => _impl.Fpcr = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public uint Fpsr
|
||||
{
|
||||
get => _impl.Fpsr;
|
||||
set => _impl.Fpsr = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool IsAarch32
|
||||
{
|
||||
get => false;
|
||||
set
|
||||
{
|
||||
if (value)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool Running { get; private set; }
|
||||
|
||||
private readonly ICounter _counter;
|
||||
private readonly IHvExecutionContext _shadowContext;
|
||||
private IHvExecutionContext _impl;
|
||||
|
||||
private readonly ExceptionCallbacks _exceptionCallbacks;
|
||||
|
||||
public HvExecutionContext(ICounter counter, ExceptionCallbacks exceptionCallbacks)
|
||||
{
|
||||
_counter = counter;
|
||||
_shadowContext = new HvExecutionContextShadow();
|
||||
_impl = _shadowContext;
|
||||
_exceptionCallbacks = exceptionCallbacks;
|
||||
Running = true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ulong GetX(int index) => _impl.GetX(index);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SetX(int index, ulong value) => _impl.SetX(index, value);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public V128 GetV(int index) => _impl.GetV(index);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SetV(int index, V128 value) => _impl.SetV(index, value);
|
||||
|
||||
private void InterruptHandler()
|
||||
{
|
||||
_exceptionCallbacks.InterruptCallback?.Invoke(this);
|
||||
}
|
||||
|
||||
private void BreakHandler(ulong address, int imm)
|
||||
{
|
||||
_exceptionCallbacks.BreakCallback?.Invoke(this, address, imm);
|
||||
}
|
||||
|
||||
private void SupervisorCallHandler(ulong address, int imm)
|
||||
{
|
||||
_exceptionCallbacks.SupervisorCallback?.Invoke(this, address, imm);
|
||||
}
|
||||
|
||||
private void UndefinedHandler(ulong address, int opCode)
|
||||
{
|
||||
_exceptionCallbacks.UndefinedCallback?.Invoke(this, address, opCode);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void RequestInterrupt()
|
||||
{
|
||||
_impl.RequestInterrupt();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void StopRunning()
|
||||
{
|
||||
Running = false;
|
||||
RequestInterrupt();
|
||||
}
|
||||
|
||||
public unsafe void Execute(HvMemoryManager memoryManager, ulong address)
|
||||
{
|
||||
HvVcpu vcpu = HvVcpuPool.Instance.Create(memoryManager.AddressSpace, _shadowContext, SwapContext);
|
||||
|
||||
HvApi.hv_vcpu_set_reg(vcpu.Handle, hv_reg_t.HV_REG_PC, address).ThrowOnError();
|
||||
|
||||
while (Running)
|
||||
{
|
||||
HvApi.hv_vcpu_run(vcpu.Handle).ThrowOnError();
|
||||
|
||||
uint reason = vcpu.ExitInfo->reason;
|
||||
|
||||
if (reason == 1)
|
||||
{
|
||||
uint hvEsr = (uint)vcpu.ExitInfo->exception.syndrome;
|
||||
ExceptionClass hvEc = (ExceptionClass)(hvEsr >> 26);
|
||||
|
||||
if (hvEc != ExceptionClass.HvcAarch64)
|
||||
{
|
||||
throw new Exception($"Unhandled exception from guest kernel with ESR 0x{hvEsr:X} ({hvEc}).");
|
||||
}
|
||||
|
||||
address = SynchronousException(memoryManager, ref vcpu);
|
||||
HvApi.hv_vcpu_set_reg(vcpu.Handle, hv_reg_t.HV_REG_PC, address).ThrowOnError();
|
||||
}
|
||||
else if (reason == 0)
|
||||
{
|
||||
if (_impl.GetAndClearInterruptRequested())
|
||||
{
|
||||
ReturnToPool(vcpu);
|
||||
InterruptHandler();
|
||||
vcpu = RentFromPool(memoryManager.AddressSpace, vcpu);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception($"Unhandled exit reason {reason}.");
|
||||
}
|
||||
}
|
||||
|
||||
HvVcpuPool.Instance.Destroy(vcpu, SwapContext);
|
||||
}
|
||||
|
||||
private ulong SynchronousException(HvMemoryManager memoryManager, ref HvVcpu vcpu)
|
||||
{
|
||||
ulong vcpuHandle = vcpu.Handle;
|
||||
|
||||
HvApi.hv_vcpu_get_sys_reg(vcpuHandle, hv_sys_reg_t.HV_SYS_REG_ELR_EL1, out ulong elr).ThrowOnError();
|
||||
HvApi.hv_vcpu_get_sys_reg(vcpuHandle, hv_sys_reg_t.HV_SYS_REG_ESR_EL1, out ulong esr).ThrowOnError();
|
||||
|
||||
ExceptionClass ec = (ExceptionClass)((uint)esr >> 26);
|
||||
|
||||
switch (ec)
|
||||
{
|
||||
case ExceptionClass.DataAbortLowerEl:
|
||||
DataAbort(memoryManager.Tracking, vcpuHandle, (uint)esr);
|
||||
break;
|
||||
case ExceptionClass.TrappedMsrMrsSystem:
|
||||
InstructionTrap((uint)esr);
|
||||
HvApi.hv_vcpu_set_sys_reg(vcpuHandle, hv_sys_reg_t.HV_SYS_REG_ELR_EL1, elr + 4UL).ThrowOnError();
|
||||
break;
|
||||
case ExceptionClass.SvcAarch64:
|
||||
ReturnToPool(vcpu);
|
||||
ushort id = (ushort)esr;
|
||||
SupervisorCallHandler(elr - 4UL, id);
|
||||
vcpu = RentFromPool(memoryManager.AddressSpace, vcpu);
|
||||
break;
|
||||
default:
|
||||
throw new Exception($"Unhandled guest exception {ec}.");
|
||||
}
|
||||
|
||||
// Make sure we will continue running at EL0.
|
||||
if (memoryManager.AddressSpace.GetAndClearUserTlbInvalidationPending())
|
||||
{
|
||||
// TODO: Invalidate only the range that was modified?
|
||||
return HvAddressSpace.KernelRegionTlbiEretAddress;
|
||||
}
|
||||
else
|
||||
{
|
||||
return HvAddressSpace.KernelRegionEretAddress;
|
||||
}
|
||||
}
|
||||
|
||||
private void DataAbort(MemoryTracking tracking, ulong vcpu, uint esr)
|
||||
{
|
||||
bool write = (esr & (1u << 6)) != 0;
|
||||
bool farValid = (esr & (1u << 10)) == 0;
|
||||
int accessSizeLog2 = (int)((esr >> 22) & 3);
|
||||
|
||||
if (farValid)
|
||||
{
|
||||
HvApi.hv_vcpu_get_sys_reg(vcpu, hv_sys_reg_t.HV_SYS_REG_FAR_EL1, out ulong far).ThrowOnError();
|
||||
|
||||
ulong size = 1UL << accessSizeLog2;
|
||||
|
||||
if (!tracking.VirtualMemoryEvent(far, size, write))
|
||||
{
|
||||
string rw = write ? "write" : "read";
|
||||
throw new Exception($"Unhandled invalid memory access at VA 0x{far:X} with size 0x{size:X} ({rw}).");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception($"Unhandled invalid memory access at unknown VA with ESR 0x{esr:X}.");
|
||||
}
|
||||
}
|
||||
|
||||
private void InstructionTrap(uint esr)
|
||||
{
|
||||
bool read = (esr & 1) != 0;
|
||||
uint rt = (esr >> 5) & 0x1f;
|
||||
|
||||
if (read)
|
||||
{
|
||||
// Op0 Op2 Op1 CRn 00000 CRm
|
||||
switch ((esr >> 1) & 0x1ffe0f)
|
||||
{
|
||||
case 0b11_000_011_1110_00000_0000: // CNTFRQ_EL0
|
||||
WriteRt(rt, _counter.Frequency);
|
||||
break;
|
||||
case 0b11_001_011_1110_00000_0000: // CNTPCT_EL0
|
||||
WriteRt(rt, _counter.Counter);
|
||||
break;
|
||||
default:
|
||||
throw new Exception($"Unhandled system register read with ESR 0x{esr:X}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception($"Unhandled system register write with ESR 0x{esr:X}");
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteRt(uint rt, ulong value)
|
||||
{
|
||||
if (rt < 31)
|
||||
{
|
||||
SetX((int)rt, value);
|
||||
}
|
||||
}
|
||||
|
||||
private void ReturnToPool(HvVcpu vcpu)
|
||||
{
|
||||
HvVcpuPool.Instance.Return(vcpu, SwapContext);
|
||||
}
|
||||
|
||||
private HvVcpu RentFromPool(HvAddressSpace addressSpace, HvVcpu vcpu)
|
||||
{
|
||||
return HvVcpuPool.Instance.Rent(addressSpace, _shadowContext, vcpu, SwapContext);
|
||||
}
|
||||
|
||||
private void SwapContext(IHvExecutionContext newContext)
|
||||
{
|
||||
_impl = newContext;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
59
src/Ryujinx.Cpu/AppleHv/HvExecutionContextShadow.cs
Normal file
59
src/Ryujinx.Cpu/AppleHv/HvExecutionContextShadow.cs
Normal file
|
@ -0,0 +1,59 @@
|
|||
using ARMeilleure.State;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
unsafe class HvExecutionContextShadow : IHvExecutionContext
|
||||
{
|
||||
public ulong Pc { get; set; }
|
||||
public ulong ElrEl1 { get; set; }
|
||||
public ulong EsrEl1 { get; set; }
|
||||
|
||||
public long TpidrEl0 { get; set; }
|
||||
public long TpidrroEl0 { get; set; }
|
||||
|
||||
public uint Pstate { get; set; }
|
||||
|
||||
public uint Fpcr { get; set; }
|
||||
public uint Fpsr { get; set; }
|
||||
|
||||
public bool IsAarch32 { get; set; }
|
||||
|
||||
private readonly ulong[] _x;
|
||||
private readonly V128[] _v;
|
||||
|
||||
public HvExecutionContextShadow()
|
||||
{
|
||||
_x = new ulong[32];
|
||||
_v = new V128[32];
|
||||
}
|
||||
|
||||
public ulong GetX(int index)
|
||||
{
|
||||
return _x[index];
|
||||
}
|
||||
|
||||
public void SetX(int index, ulong value)
|
||||
{
|
||||
_x[index] = value;
|
||||
}
|
||||
|
||||
public V128 GetV(int index)
|
||||
{
|
||||
return _v[index];
|
||||
}
|
||||
|
||||
public void SetV(int index, V128 value)
|
||||
{
|
||||
_v[index] = value;
|
||||
}
|
||||
|
||||
public void RequestInterrupt()
|
||||
{
|
||||
}
|
||||
|
||||
public bool GetAndClearInterruptRequested()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
196
src/Ryujinx.Cpu/AppleHv/HvExecutionContextVcpu.cs
Normal file
196
src/Ryujinx.Cpu/AppleHv/HvExecutionContextVcpu.cs
Normal file
|
@ -0,0 +1,196 @@
|
|||
using ARMeilleure.State;
|
||||
using Ryujinx.Memory;
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
class HvExecutionContextVcpu : IHvExecutionContext
|
||||
{
|
||||
private static MemoryBlock _setSimdFpRegFuncMem;
|
||||
private delegate hv_result_t SetSimdFpReg(ulong vcpu, hv_simd_fp_reg_t reg, in V128 value, IntPtr funcPtr);
|
||||
private static SetSimdFpReg _setSimdFpReg;
|
||||
private static IntPtr _setSimdFpRegNativePtr;
|
||||
|
||||
static HvExecutionContextVcpu()
|
||||
{
|
||||
// .NET does not support passing vectors by value, so we need to pass a pointer and use a native
|
||||
// function to load the value into a vector register.
|
||||
_setSimdFpRegFuncMem = new MemoryBlock(MemoryBlock.GetPageSize());
|
||||
_setSimdFpRegFuncMem.Write(0, 0x3DC00040u); // LDR Q0, [X2]
|
||||
_setSimdFpRegFuncMem.Write(4, 0xD61F0060u); // BR X3
|
||||
_setSimdFpRegFuncMem.Reprotect(0, _setSimdFpRegFuncMem.Size, MemoryPermission.ReadAndExecute);
|
||||
|
||||
_setSimdFpReg = Marshal.GetDelegateForFunctionPointer<SetSimdFpReg>(_setSimdFpRegFuncMem.Pointer);
|
||||
|
||||
if (NativeLibrary.TryLoad(HvApi.LibraryName, out IntPtr hvLibHandle))
|
||||
{
|
||||
_setSimdFpRegNativePtr = NativeLibrary.GetExport(hvLibHandle, nameof(HvApi.hv_vcpu_set_simd_fp_reg));
|
||||
}
|
||||
}
|
||||
|
||||
public ulong Pc
|
||||
{
|
||||
get
|
||||
{
|
||||
HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_PC, out ulong pc).ThrowOnError();
|
||||
return pc;
|
||||
}
|
||||
set
|
||||
{
|
||||
HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_PC, value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
public ulong ElrEl1
|
||||
{
|
||||
get
|
||||
{
|
||||
HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_ELR_EL1, out ulong elr).ThrowOnError();
|
||||
return elr;
|
||||
}
|
||||
set
|
||||
{
|
||||
HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_ELR_EL1, value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
public ulong EsrEl1
|
||||
{
|
||||
get
|
||||
{
|
||||
HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_ESR_EL1, out ulong esr).ThrowOnError();
|
||||
return esr;
|
||||
}
|
||||
set
|
||||
{
|
||||
HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_ESR_EL1, value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
public long TpidrEl0
|
||||
{
|
||||
get
|
||||
{
|
||||
HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_TPIDR_EL0, out ulong tpidrEl0).ThrowOnError();
|
||||
return (long)tpidrEl0;
|
||||
}
|
||||
set
|
||||
{
|
||||
HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_TPIDR_EL0, (ulong)value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
public long TpidrroEl0
|
||||
{
|
||||
get
|
||||
{
|
||||
HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_TPIDRRO_EL0, out ulong tpidrroEl0).ThrowOnError();
|
||||
return (long)tpidrroEl0;
|
||||
}
|
||||
set
|
||||
{
|
||||
HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_TPIDRRO_EL0, (ulong)value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
public uint Pstate
|
||||
{
|
||||
get
|
||||
{
|
||||
HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_CPSR, out ulong cpsr).ThrowOnError();
|
||||
return (uint)cpsr;
|
||||
}
|
||||
set
|
||||
{
|
||||
HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_CPSR, (ulong)value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
public uint Fpcr
|
||||
{
|
||||
get
|
||||
{
|
||||
HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_FPCR, out ulong fpcr).ThrowOnError();
|
||||
return (uint)fpcr;
|
||||
}
|
||||
set
|
||||
{
|
||||
HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_FPCR, (ulong)value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
public uint Fpsr
|
||||
{
|
||||
get
|
||||
{
|
||||
HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_FPSR, out ulong fpsr).ThrowOnError();
|
||||
return (uint)fpsr;
|
||||
}
|
||||
set
|
||||
{
|
||||
HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_FPSR, (ulong)value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
private ulong _vcpu;
|
||||
private int _interruptRequested;
|
||||
|
||||
public HvExecutionContextVcpu(ulong vcpu)
|
||||
{
|
||||
_vcpu = vcpu;
|
||||
}
|
||||
|
||||
public ulong GetX(int index)
|
||||
{
|
||||
if (index == 31)
|
||||
{
|
||||
HvApi.hv_vcpu_get_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_SP_EL0, out ulong value).ThrowOnError();
|
||||
return value;
|
||||
}
|
||||
else
|
||||
{
|
||||
HvApi.hv_vcpu_get_reg(_vcpu, hv_reg_t.HV_REG_X0 + (uint)index, out ulong value).ThrowOnError();
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
public void SetX(int index, ulong value)
|
||||
{
|
||||
if (index == 31)
|
||||
{
|
||||
HvApi.hv_vcpu_set_sys_reg(_vcpu, hv_sys_reg_t.HV_SYS_REG_SP_EL0, value).ThrowOnError();
|
||||
}
|
||||
else
|
||||
{
|
||||
HvApi.hv_vcpu_set_reg(_vcpu, hv_reg_t.HV_REG_X0 + (uint)index, value).ThrowOnError();
|
||||
}
|
||||
}
|
||||
|
||||
public V128 GetV(int index)
|
||||
{
|
||||
HvApi.hv_vcpu_get_simd_fp_reg(_vcpu, hv_simd_fp_reg_t.HV_SIMD_FP_REG_Q0 + (uint)index, out hv_simd_fp_uchar16_t value).ThrowOnError();
|
||||
return new V128(value.Low, value.High);
|
||||
}
|
||||
|
||||
public void SetV(int index, V128 value)
|
||||
{
|
||||
_setSimdFpReg(_vcpu, hv_simd_fp_reg_t.HV_SIMD_FP_REG_Q0 + (uint)index, value, _setSimdFpRegNativePtr).ThrowOnError();
|
||||
}
|
||||
|
||||
public void RequestInterrupt()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _interruptRequested, 1) == 0)
|
||||
{
|
||||
ulong vcpu = _vcpu;
|
||||
HvApi.hv_vcpus_exit(ref vcpu, 1);
|
||||
}
|
||||
}
|
||||
|
||||
public bool GetAndClearInterruptRequested()
|
||||
{
|
||||
return Interlocked.Exchange(ref _interruptRequested, 0) != 0;
|
||||
}
|
||||
}
|
||||
}
|
34
src/Ryujinx.Cpu/AppleHv/HvIpaAllocator.cs
Normal file
34
src/Ryujinx.Cpu/AppleHv/HvIpaAllocator.cs
Normal file
|
@ -0,0 +1,34 @@
|
|||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
class HvIpaAllocator
|
||||
{
|
||||
private const ulong AllocationGranule = 1UL << 14;
|
||||
private const ulong IpaRegionSize = 1UL << 35;
|
||||
|
||||
private readonly PrivateMemoryAllocator.Block _block;
|
||||
|
||||
public HvIpaAllocator()
|
||||
{
|
||||
_block = new PrivateMemoryAllocator.Block(null, IpaRegionSize);
|
||||
}
|
||||
|
||||
public ulong Allocate(ulong size, ulong alignment = AllocationGranule)
|
||||
{
|
||||
ulong offset = _block.Allocate(size, alignment);
|
||||
|
||||
if (offset == PrivateMemoryAllocator.InvalidOffset)
|
||||
{
|
||||
throw new InvalidOperationException($"No enough free IPA memory to allocate 0x{size:X} bytes with alignment 0x{alignment:X}.");
|
||||
}
|
||||
|
||||
return offset;
|
||||
}
|
||||
|
||||
public void Free(ulong offset, ulong size)
|
||||
{
|
||||
_block.Free(offset, size);
|
||||
}
|
||||
}
|
||||
}
|
34
src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocation.cs
Normal file
34
src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocation.cs
Normal file
|
@ -0,0 +1,34 @@
|
|||
using Ryujinx.Memory;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
struct HvMemoryBlockAllocation : IDisposable
|
||||
{
|
||||
private readonly HvMemoryBlockAllocator _owner;
|
||||
private readonly HvMemoryBlockAllocator.Block _block;
|
||||
|
||||
public bool IsValid => _owner != null;
|
||||
public MemoryBlock Memory => _block.Memory;
|
||||
public ulong Ipa => _block.Ipa;
|
||||
public ulong Offset { get; }
|
||||
public ulong Size { get; }
|
||||
|
||||
public HvMemoryBlockAllocation(
|
||||
HvMemoryBlockAllocator owner,
|
||||
HvMemoryBlockAllocator.Block block,
|
||||
ulong offset,
|
||||
ulong size)
|
||||
{
|
||||
_owner = owner;
|
||||
_block = block;
|
||||
Offset = offset;
|
||||
Size = size;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_owner.Free(_block, Offset, Size);
|
||||
}
|
||||
}
|
||||
}
|
59
src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs
Normal file
59
src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs
Normal file
|
@ -0,0 +1,59 @@
|
|||
using Ryujinx.Memory;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
class HvMemoryBlockAllocator : PrivateMemoryAllocatorImpl<HvMemoryBlockAllocator.Block>
|
||||
{
|
||||
private const ulong InvalidOffset = ulong.MaxValue;
|
||||
|
||||
public class Block : PrivateMemoryAllocator.Block
|
||||
{
|
||||
private readonly HvIpaAllocator _ipaAllocator;
|
||||
public ulong Ipa { get; }
|
||||
|
||||
public Block(HvIpaAllocator ipaAllocator, MemoryBlock memory, ulong size) : base(memory, size)
|
||||
{
|
||||
_ipaAllocator = ipaAllocator;
|
||||
|
||||
lock (ipaAllocator)
|
||||
{
|
||||
Ipa = ipaAllocator.Allocate(size);
|
||||
}
|
||||
|
||||
HvApi.hv_vm_map((ulong)Memory.Pointer, Ipa, size, hv_memory_flags_t.HV_MEMORY_READ | hv_memory_flags_t.HV_MEMORY_WRITE).ThrowOnError();
|
||||
}
|
||||
|
||||
public override void Destroy()
|
||||
{
|
||||
HvApi.hv_vm_unmap(Ipa, Size).ThrowOnError();
|
||||
|
||||
lock (_ipaAllocator)
|
||||
{
|
||||
_ipaAllocator.Free(Ipa, Size);
|
||||
}
|
||||
|
||||
base.Destroy();
|
||||
}
|
||||
}
|
||||
|
||||
private readonly HvIpaAllocator _ipaAllocator;
|
||||
|
||||
public HvMemoryBlockAllocator(HvIpaAllocator ipaAllocator, int blockAlignment) : base(blockAlignment, MemoryAllocationFlags.None)
|
||||
{
|
||||
_ipaAllocator = ipaAllocator;
|
||||
}
|
||||
|
||||
public unsafe HvMemoryBlockAllocation Allocate(ulong size, ulong alignment)
|
||||
{
|
||||
var allocation = Allocate(size, alignment, CreateBlock);
|
||||
|
||||
return new HvMemoryBlockAllocation(this, allocation.Block, allocation.Offset, allocation.Size);
|
||||
}
|
||||
|
||||
private Block CreateBlock(MemoryBlock memory, ulong size)
|
||||
{
|
||||
return new Block(_ipaAllocator, memory, size);
|
||||
}
|
||||
}
|
||||
}
|
947
src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs
Normal file
947
src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs
Normal file
|
@ -0,0 +1,947 @@
|
|||
using ARMeilleure.Memory;
|
||||
using Ryujinx.Cpu.Tracking;
|
||||
using Ryujinx.Memory;
|
||||
using Ryujinx.Memory.Range;
|
||||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a CPU memory manager which maps guest virtual memory directly onto the Hypervisor page table.
|
||||
/// </summary>
|
||||
public class HvMemoryManager : MemoryManagerBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
|
||||
{
|
||||
public const int PageBits = 12;
|
||||
public const int PageSize = 1 << PageBits;
|
||||
public const int PageMask = PageSize - 1;
|
||||
|
||||
public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry.
|
||||
public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set.
|
||||
|
||||
private enum HostMappedPtBits : ulong
|
||||
{
|
||||
Unmapped = 0,
|
||||
Mapped,
|
||||
WriteTracked,
|
||||
ReadWriteTracked,
|
||||
|
||||
MappedReplicated = 0x5555555555555555,
|
||||
WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa,
|
||||
ReadWriteTrackedReplicated = ulong.MaxValue
|
||||
}
|
||||
|
||||
private readonly InvalidAccessHandler _invalidAccessHandler;
|
||||
|
||||
private readonly ulong _addressSpaceSize;
|
||||
|
||||
private readonly HvAddressSpace _addressSpace;
|
||||
|
||||
internal HvAddressSpace AddressSpace => _addressSpace;
|
||||
|
||||
private readonly MemoryBlock _backingMemory;
|
||||
private readonly PageTable<ulong> _pageTable;
|
||||
|
||||
private readonly ulong[] _pageBitmap;
|
||||
|
||||
public bool Supports4KBPages => true;
|
||||
|
||||
public int AddressSpaceBits { get; }
|
||||
|
||||
public IntPtr PageTablePointer => IntPtr.Zero;
|
||||
|
||||
public MemoryManagerType Type => MemoryManagerType.SoftwarePageTable;
|
||||
|
||||
public MemoryTracking Tracking { get; }
|
||||
|
||||
public event Action<ulong, ulong> UnmapEvent;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the Hypervisor memory manager.
|
||||
/// </summary>
|
||||
/// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
|
||||
/// <param name="addressSpaceSize">Size of the address space</param>
|
||||
/// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
|
||||
public HvMemoryManager(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler = null)
|
||||
{
|
||||
_backingMemory = backingMemory;
|
||||
_pageTable = new PageTable<ulong>();
|
||||
_invalidAccessHandler = invalidAccessHandler;
|
||||
_addressSpaceSize = addressSpaceSize;
|
||||
|
||||
ulong asSize = PageSize;
|
||||
int asBits = PageBits;
|
||||
|
||||
while (asSize < addressSpaceSize)
|
||||
{
|
||||
asSize <<= 1;
|
||||
asBits++;
|
||||
}
|
||||
|
||||
_addressSpace = new HvAddressSpace(backingMemory, asSize);
|
||||
|
||||
AddressSpaceBits = asBits;
|
||||
|
||||
_pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))];
|
||||
Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the virtual address is part of the addressable space.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address</param>
|
||||
/// <returns>True if the virtual address is part of the addressable space</returns>
|
||||
private bool ValidateAddress(ulong va)
|
||||
{
|
||||
return va < _addressSpaceSize;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the combination of virtual address and size is part of the addressable space.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
/// <returns>True if the combination of virtual address and size is part of the addressable space</returns>
|
||||
private bool ValidateAddressAndSize(ulong va, ulong size)
|
||||
{
|
||||
ulong endVa = va + size;
|
||||
return endVa >= va && endVa >= size && endVa <= _addressSpaceSize;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures the combination of virtual address and size is part of the addressable space.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
/// <exception cref="InvalidMemoryRegionException">Throw when the memory region specified outside the addressable space</exception>
|
||||
private void AssertValidAddressAndSize(ulong va, ulong size)
|
||||
{
|
||||
if (!ValidateAddressAndSize(va, size))
|
||||
{
|
||||
throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures the combination of virtual address and size is part of the addressable space and fully mapped.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
private void AssertMapped(ulong va, ulong size)
|
||||
{
|
||||
if (!ValidateAddressAndSize(va, size) || !IsRangeMappedImpl(va, size))
|
||||
{
|
||||
throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
PtMap(va, pa, size);
|
||||
_addressSpace.MapUser(va, pa, size, MemoryPermission.ReadWriteExecute);
|
||||
AddMapping(va, size);
|
||||
|
||||
Tracking.Map(va, size);
|
||||
}
|
||||
|
||||
private void PtMap(ulong va, ulong pa, ulong size)
|
||||
{
|
||||
while (size != 0)
|
||||
{
|
||||
_pageTable.Map(va, pa);
|
||||
|
||||
va += PageSize;
|
||||
pa += PageSize;
|
||||
size -= PageSize;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void MapForeign(ulong va, nuint hostPointer, ulong size)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Unmap(ulong va, ulong size)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
UnmapEvent?.Invoke(va, size);
|
||||
Tracking.Unmap(va, size);
|
||||
|
||||
RemoveMapping(va, size);
|
||||
_addressSpace.UnmapUser(va, size);
|
||||
PtUnmap(va, size);
|
||||
}
|
||||
|
||||
private void PtUnmap(ulong va, ulong size)
|
||||
{
|
||||
while (size != 0)
|
||||
{
|
||||
_pageTable.Unmap(va);
|
||||
|
||||
va += PageSize;
|
||||
size -= PageSize;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public T Read<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
return MemoryMarshal.Cast<byte, T>(GetSpan(va, Unsafe.SizeOf<T>()))[0];
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public T ReadTracked<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
try
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), false);
|
||||
|
||||
return Read<T>(va);
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Read(ulong va, Span<byte> data)
|
||||
{
|
||||
ReadImpl(va, data);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Write<T>(ulong va, T value) where T : unmanaged
|
||||
{
|
||||
Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Write(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
SignalMemoryTracking(va, (ulong)data.Length, true);
|
||||
|
||||
WriteImpl(va, data);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
WriteImpl(va, data);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
SignalMemoryTracking(va, (ulong)data.Length, false);
|
||||
|
||||
if (IsContiguousAndMapped(va, data.Length))
|
||||
{
|
||||
var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length);
|
||||
|
||||
bool changed = !data.SequenceEqual(target);
|
||||
|
||||
if (changed)
|
||||
{
|
||||
data.CopyTo(target);
|
||||
}
|
||||
|
||||
return changed;
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteImpl(va, data);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private void WriteImpl(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
try
|
||||
{
|
||||
AssertValidAddressAndSize(va, (ulong)data.Length);
|
||||
|
||||
if (IsContiguousAndMapped(va, data.Length))
|
||||
{
|
||||
data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length));
|
||||
}
|
||||
else
|
||||
{
|
||||
int offset = 0, size;
|
||||
|
||||
if ((va & PageMask) != 0)
|
||||
{
|
||||
ulong pa = GetPhysicalAddressChecked(va);
|
||||
|
||||
size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
|
||||
|
||||
data.Slice(0, size).CopyTo(_backingMemory.GetSpan(pa, size));
|
||||
|
||||
offset += size;
|
||||
}
|
||||
|
||||
for (; offset < data.Length; offset += size)
|
||||
{
|
||||
ulong pa = GetPhysicalAddressChecked(va + (ulong)offset);
|
||||
|
||||
size = Math.Min(data.Length - offset, PageSize);
|
||||
|
||||
data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return ReadOnlySpan<byte>.Empty;
|
||||
}
|
||||
|
||||
if (tracked)
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)size, false);
|
||||
}
|
||||
|
||||
if (IsContiguousAndMapped(va, size))
|
||||
{
|
||||
return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size);
|
||||
}
|
||||
else
|
||||
{
|
||||
Span<byte> data = new byte[size];
|
||||
|
||||
ReadImpl(va, data);
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return new WritableRegion(null, va, Memory<byte>.Empty);
|
||||
}
|
||||
|
||||
if (tracked)
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)size, true);
|
||||
}
|
||||
|
||||
if (IsContiguousAndMapped(va, size))
|
||||
{
|
||||
return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size));
|
||||
}
|
||||
else
|
||||
{
|
||||
Memory<byte> memory = new byte[size];
|
||||
|
||||
ReadImpl(va, memory.Span);
|
||||
|
||||
return new WritableRegion(this, va, memory);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ref T GetRef<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
if (!IsContiguous(va, Unsafe.SizeOf<T>()))
|
||||
{
|
||||
ThrowMemoryNotContiguous();
|
||||
}
|
||||
|
||||
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
|
||||
|
||||
return ref _backingMemory.GetRef<T>(GetPhysicalAddressChecked(va));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMapped(ulong va)
|
||||
{
|
||||
return ValidateAddress(va) && IsMappedImpl(va);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsMappedImpl(ulong va)
|
||||
{
|
||||
ulong page = va >> PageBits;
|
||||
|
||||
int bit = (int)((page & 31) << 1);
|
||||
|
||||
int pageIndex = (int)(page >> PageToPteShift);
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex];
|
||||
|
||||
ulong pte = Volatile.Read(ref pageRef);
|
||||
|
||||
return ((pte >> bit) & 3) != 0;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool IsRangeMapped(ulong va, ulong size)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
return IsRangeMappedImpl(va, size);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex)
|
||||
{
|
||||
startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1);
|
||||
endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1));
|
||||
|
||||
pageIndex = (int)(pageStart >> PageToPteShift);
|
||||
pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift);
|
||||
}
|
||||
|
||||
private bool IsRangeMappedImpl(ulong va, ulong size)
|
||||
{
|
||||
int pages = GetPagesCount(va, size, out _);
|
||||
|
||||
if (pages == 1)
|
||||
{
|
||||
return IsMappedImpl(va);
|
||||
}
|
||||
|
||||
ulong pageStart = va >> PageBits;
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
// Check if either bit in each 2 bit page entry is set.
|
||||
// OR the block with itself shifted down by 1, and check the first bit of each entry.
|
||||
|
||||
ulong mask = BlockMappedMask & startMask;
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask &= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
ulong pte = Volatile.Read(ref pageRef);
|
||||
|
||||
pte |= pte >> 1;
|
||||
if ((pte & mask) != mask)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
mask = BlockMappedMask;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsContiguous(ulong va, int size)
|
||||
{
|
||||
if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
|
||||
for (int page = 0; page < pages - 1; page++)
|
||||
{
|
||||
if (!ValidateAddress(va + PageSize))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
va += PageSize;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return Enumerable.Empty<HostMemoryRange>();
|
||||
}
|
||||
|
||||
var guestRegions = GetPhysicalRegionsImpl(va, size);
|
||||
if (guestRegions == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var regions = new HostMemoryRange[guestRegions.Count];
|
||||
|
||||
for (int i = 0; i < regions.Length; i++)
|
||||
{
|
||||
var guestRegion = guestRegions[i];
|
||||
IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size);
|
||||
regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size);
|
||||
}
|
||||
|
||||
return regions;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return Enumerable.Empty<MemoryRange>();
|
||||
}
|
||||
|
||||
return GetPhysicalRegionsImpl(va, size);
|
||||
}
|
||||
|
||||
private List<MemoryRange> GetPhysicalRegionsImpl(ulong va, ulong size)
|
||||
{
|
||||
if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
|
||||
var regions = new List<MemoryRange>();
|
||||
|
||||
ulong regionStart = GetPhysicalAddressInternal(va);
|
||||
ulong regionSize = PageSize;
|
||||
|
||||
for (int page = 0; page < pages - 1; page++)
|
||||
{
|
||||
if (!ValidateAddress(va + PageSize))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ulong newPa = GetPhysicalAddressInternal(va + PageSize);
|
||||
|
||||
if (GetPhysicalAddressInternal(va) + PageSize != newPa)
|
||||
{
|
||||
regions.Add(new MemoryRange(regionStart, regionSize));
|
||||
regionStart = newPa;
|
||||
regionSize = 0;
|
||||
}
|
||||
|
||||
va += PageSize;
|
||||
regionSize += PageSize;
|
||||
}
|
||||
|
||||
regions.Add(new MemoryRange(regionStart, regionSize));
|
||||
|
||||
return regions;
|
||||
}
|
||||
|
||||
private void ReadImpl(ulong va, Span<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
AssertValidAddressAndSize(va, (ulong)data.Length);
|
||||
|
||||
int offset = 0, size;
|
||||
|
||||
if ((va & PageMask) != 0)
|
||||
{
|
||||
ulong pa = GetPhysicalAddressChecked(va);
|
||||
|
||||
size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
|
||||
|
||||
_backingMemory.GetSpan(pa, size).CopyTo(data.Slice(0, size));
|
||||
|
||||
offset += size;
|
||||
}
|
||||
|
||||
for (; offset < data.Length; offset += size)
|
||||
{
|
||||
ulong pa = GetPhysicalAddressChecked(va + (ulong)offset);
|
||||
|
||||
size = Math.Min(data.Length - offset, PageSize);
|
||||
|
||||
_backingMemory.GetSpan(pa, size).CopyTo(data.Slice(offset, size));
|
||||
}
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
/// <remarks>
|
||||
/// This function also validates that the given range is both valid and mapped, and will throw if it is not.
|
||||
/// </remarks>
|
||||
public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
if (precise)
|
||||
{
|
||||
Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Software table, used for managed memory tracking.
|
||||
|
||||
int pages = GetPagesCount(va, size, out _);
|
||||
ulong pageStart = va >> PageBits;
|
||||
|
||||
if (pages == 1)
|
||||
{
|
||||
ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked);
|
||||
|
||||
int bit = (int)((pageStart & 31) << 1);
|
||||
|
||||
int pageIndex = (int)(pageStart >> PageToPteShift);
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex];
|
||||
|
||||
ulong pte = Volatile.Read(ref pageRef);
|
||||
ulong state = ((pte >> bit) & 3);
|
||||
|
||||
if (state >= tag)
|
||||
{
|
||||
Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
|
||||
return;
|
||||
}
|
||||
else if (state == 0)
|
||||
{
|
||||
ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
ulong mask = startMask;
|
||||
|
||||
ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated;
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask &= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
|
||||
ulong pte = Volatile.Read(ref pageRef);
|
||||
ulong mappedMask = mask & BlockMappedMask;
|
||||
|
||||
ulong mappedPte = pte | (pte >> 1);
|
||||
if ((mappedPte & mappedMask) != mappedMask)
|
||||
{
|
||||
ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
|
||||
pte &= mask;
|
||||
if ((pte & anyTrackingTag) != 0) // Search for any tracking.
|
||||
{
|
||||
// Writes trigger any tracking.
|
||||
// Only trigger tracking from reads if both bits are set on any page.
|
||||
if (write || (pte & (pte >> 1) & BlockMappedMask) != 0)
|
||||
{
|
||||
Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
mask = ulong.MaxValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the number of pages in a virtual address range.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range</param>
|
||||
/// <param name="startVa">The virtual address of the beginning of the first page</param>
|
||||
/// <remarks>This function does not differentiate between allocated and unallocated pages.</remarks>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int GetPagesCount(ulong va, ulong size, out ulong startVa)
|
||||
{
|
||||
// WARNING: Always check if ulong does not overflow during the operations.
|
||||
startVa = va & ~(ulong)PageMask;
|
||||
ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
|
||||
|
||||
return (int)(vaSpan / PageSize);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
|
||||
{
|
||||
// Protection is inverted on software pages, since the default value is 0.
|
||||
protection = (~protection) & MemoryPermission.ReadAndWrite;
|
||||
|
||||
int pages = GetPagesCount(va, size, out va);
|
||||
ulong pageStart = va >> PageBits;
|
||||
|
||||
if (pages == 1)
|
||||
{
|
||||
ulong protTag = protection switch
|
||||
{
|
||||
MemoryPermission.None => (ulong)HostMappedPtBits.Mapped,
|
||||
MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked,
|
||||
_ => (ulong)HostMappedPtBits.ReadWriteTracked,
|
||||
};
|
||||
|
||||
int bit = (int)((pageStart & 31) << 1);
|
||||
|
||||
ulong tagMask = 3UL << bit;
|
||||
ulong invTagMask = ~tagMask;
|
||||
|
||||
ulong tag = protTag << bit;
|
||||
|
||||
int pageIndex = (int)(pageStart >> PageToPteShift);
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex];
|
||||
|
||||
ulong pte;
|
||||
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
}
|
||||
while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
ulong mask = startMask;
|
||||
|
||||
ulong protTag = protection switch
|
||||
{
|
||||
MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated,
|
||||
MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated,
|
||||
_ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated,
|
||||
};
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask &= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
|
||||
ulong pte;
|
||||
ulong mappedMask;
|
||||
|
||||
// Change the protection of all 2 bit entries that are mapped.
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
|
||||
mappedMask = pte | (pte >> 1);
|
||||
mappedMask |= (mappedMask & BlockMappedMask) << 1;
|
||||
mappedMask &= mask; // Only update mapped pages within the given range.
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte);
|
||||
|
||||
mask = ulong.MaxValue;
|
||||
}
|
||||
}
|
||||
|
||||
protection = protection switch
|
||||
{
|
||||
MemoryPermission.None => MemoryPermission.ReadAndWrite,
|
||||
MemoryPermission.Write => MemoryPermission.Read,
|
||||
_ => MemoryPermission.None
|
||||
};
|
||||
|
||||
_addressSpace.ReprotectUser(va, size, protection);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuRegionHandle BeginTracking(ulong address, ulong size, int id)
|
||||
{
|
||||
return new CpuRegionHandle(Tracking.BeginTracking(address, size, id));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id)
|
||||
{
|
||||
return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity, id));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuSmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
|
||||
{
|
||||
return new CpuSmartMultiRegionHandle(Tracking.BeginSmartGranularTracking(address, size, granularity, id));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds the given address mapping to the page table.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual memory address</param>
|
||||
/// <param name="size">Size to be mapped</param>
|
||||
private void AddMapping(ulong va, ulong size)
|
||||
{
|
||||
int pages = GetPagesCount(va, size, out _);
|
||||
ulong pageStart = va >> PageBits;
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
ulong mask = startMask;
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask &= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
|
||||
ulong pte;
|
||||
ulong mappedMask;
|
||||
|
||||
// Map all 2-bit entries that are unmapped.
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
|
||||
mappedMask = pte | (pte >> 1);
|
||||
mappedMask |= (mappedMask & BlockMappedMask) << 1;
|
||||
mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged.
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte);
|
||||
|
||||
mask = ulong.MaxValue;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the given address mapping from the page table.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual memory address</param>
|
||||
/// <param name="size">Size to be unmapped</param>
|
||||
private void RemoveMapping(ulong va, ulong size)
|
||||
{
|
||||
int pages = GetPagesCount(va, size, out _);
|
||||
ulong pageStart = va >> PageBits;
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
startMask = ~startMask;
|
||||
endMask = ~endMask;
|
||||
|
||||
ulong mask = startMask;
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask |= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
ulong pte;
|
||||
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte);
|
||||
|
||||
mask = 0;
|
||||
}
|
||||
}
|
||||
|
||||
private ulong GetPhysicalAddressChecked(ulong va)
|
||||
{
|
||||
if (!IsMapped(va))
|
||||
{
|
||||
ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}");
|
||||
}
|
||||
|
||||
return GetPhysicalAddressInternal(va);
|
||||
}
|
||||
|
||||
private ulong GetPhysicalAddressInternal(ulong va)
|
||||
{
|
||||
return _pageTable.Read(va) + (va & PageMask);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes of resources used by the memory manager.
|
||||
/// </summary>
|
||||
protected override void Destroy()
|
||||
{
|
||||
_addressSpace.Dispose();
|
||||
}
|
||||
|
||||
private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message);
|
||||
}
|
||||
}
|
25
src/Ryujinx.Cpu/AppleHv/HvVcpu.cs
Normal file
25
src/Ryujinx.Cpu/AppleHv/HvVcpu.cs
Normal file
|
@ -0,0 +1,25 @@
|
|||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
unsafe class HvVcpu
|
||||
{
|
||||
public readonly ulong Handle;
|
||||
public readonly hv_vcpu_exit_t* ExitInfo;
|
||||
public readonly IHvExecutionContext ShadowContext;
|
||||
public readonly IHvExecutionContext NativeContext;
|
||||
public readonly bool IsEphemeral;
|
||||
|
||||
public HvVcpu(
|
||||
ulong handle,
|
||||
hv_vcpu_exit_t* exitInfo,
|
||||
IHvExecutionContext shadowContext,
|
||||
IHvExecutionContext nativeContext,
|
||||
bool isEphemeral)
|
||||
{
|
||||
Handle = handle;
|
||||
ExitInfo = exitInfo;
|
||||
ShadowContext = shadowContext;
|
||||
NativeContext = nativeContext;
|
||||
IsEphemeral = isEphemeral;
|
||||
}
|
||||
}
|
||||
}
|
103
src/Ryujinx.Cpu/AppleHv/HvVcpuPool.cs
Normal file
103
src/Ryujinx.Cpu/AppleHv/HvVcpuPool.cs
Normal file
|
@ -0,0 +1,103 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
class HvVcpuPool
|
||||
{
|
||||
// Since there's a limit on the number of VCPUs we can create,
|
||||
// and we assign one VCPU per guest thread, we need to ensure
|
||||
// there are enough VCPUs available for at least the maximum number of active guest threads.
|
||||
// To do that, we always destroy and re-create VCPUs that are above a given limit.
|
||||
// Those VCPUs are called "ephemeral" here because they are not kept for long.
|
||||
//
|
||||
// In the future, we might want to consider a smarter approach that only makes
|
||||
// VCPUs for threads that are not running frequently "ephemeral", but this is
|
||||
// complicated because VCPUs can only be destroyed by the same thread that created them.
|
||||
|
||||
private const int MaxActiveVcpus = 4;
|
||||
|
||||
public static readonly HvVcpuPool Instance = new HvVcpuPool();
|
||||
|
||||
private int _totalVcpus;
|
||||
private int _maxVcpus;
|
||||
|
||||
public HvVcpuPool()
|
||||
{
|
||||
HvApi.hv_vm_get_max_vcpu_count(out uint maxVcpuCount).ThrowOnError();
|
||||
_maxVcpus = (int)maxVcpuCount;
|
||||
}
|
||||
|
||||
public HvVcpu Create(HvAddressSpace addressSpace, IHvExecutionContext shadowContext, Action<IHvExecutionContext> swapContext)
|
||||
{
|
||||
HvVcpu vcpu = CreateNew(addressSpace, shadowContext);
|
||||
vcpu.NativeContext.Load(shadowContext);
|
||||
swapContext(vcpu.NativeContext);
|
||||
return vcpu;
|
||||
}
|
||||
|
||||
public void Destroy(HvVcpu vcpu, Action<IHvExecutionContext> swapContext)
|
||||
{
|
||||
vcpu.ShadowContext.Load(vcpu.NativeContext);
|
||||
swapContext(vcpu.ShadowContext);
|
||||
DestroyVcpu(vcpu);
|
||||
}
|
||||
|
||||
public void Return(HvVcpu vcpu, Action<IHvExecutionContext> swapContext)
|
||||
{
|
||||
if (vcpu.IsEphemeral)
|
||||
{
|
||||
Destroy(vcpu, swapContext);
|
||||
}
|
||||
}
|
||||
|
||||
public HvVcpu Rent(HvAddressSpace addressSpace, IHvExecutionContext shadowContext, HvVcpu vcpu, Action<IHvExecutionContext> swapContext)
|
||||
{
|
||||
if (vcpu.IsEphemeral)
|
||||
{
|
||||
return Create(addressSpace, shadowContext, swapContext);
|
||||
}
|
||||
else
|
||||
{
|
||||
return vcpu;
|
||||
}
|
||||
}
|
||||
|
||||
private unsafe HvVcpu CreateNew(HvAddressSpace addressSpace, IHvExecutionContext shadowContext)
|
||||
{
|
||||
int newCount = IncrementVcpuCount();
|
||||
bool isEphemeral = newCount > _maxVcpus - MaxActiveVcpus;
|
||||
|
||||
// Create VCPU.
|
||||
hv_vcpu_exit_t* exitInfo = null;
|
||||
HvApi.hv_vcpu_create(out ulong vcpuHandle, ref exitInfo, IntPtr.Zero).ThrowOnError();
|
||||
|
||||
// Enable FP and SIMD instructions.
|
||||
HvApi.hv_vcpu_set_sys_reg(vcpuHandle, hv_sys_reg_t.HV_SYS_REG_CPACR_EL1, 0b11 << 20).ThrowOnError();
|
||||
|
||||
addressSpace.InitializeMmu(vcpuHandle);
|
||||
|
||||
HvExecutionContextVcpu nativeContext = new HvExecutionContextVcpu(vcpuHandle);
|
||||
|
||||
HvVcpu vcpu = new HvVcpu(vcpuHandle, exitInfo, shadowContext, nativeContext, isEphemeral);
|
||||
|
||||
return vcpu;
|
||||
}
|
||||
|
||||
private void DestroyVcpu(HvVcpu vcpu)
|
||||
{
|
||||
HvApi.hv_vcpu_destroy(vcpu.Handle).ThrowOnError();
|
||||
DecrementVcpuCount();
|
||||
}
|
||||
|
||||
private int IncrementVcpuCount()
|
||||
{
|
||||
return Interlocked.Increment(ref _totalVcpus);
|
||||
}
|
||||
|
||||
private void DecrementVcpuCount()
|
||||
{
|
||||
Interlocked.Decrement(ref _totalVcpus);
|
||||
}
|
||||
}
|
||||
}
|
68
src/Ryujinx.Cpu/AppleHv/HvVm.cs
Normal file
68
src/Ryujinx.Cpu/AppleHv/HvVm.cs
Normal file
|
@ -0,0 +1,68 @@
|
|||
using Ryujinx.Memory;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
static class HvVm
|
||||
{
|
||||
// This alignment allows us to use larger blocks on the page table.
|
||||
private const ulong AsIpaAlignment = 1UL << 30;
|
||||
|
||||
private static int _addressSpaces;
|
||||
private static HvIpaAllocator _ipaAllocator;
|
||||
private static object _lock = new object();
|
||||
|
||||
public static (ulong, HvIpaAllocator) CreateAddressSpace(MemoryBlock block)
|
||||
{
|
||||
HvIpaAllocator ipaAllocator;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (++_addressSpaces == 1)
|
||||
{
|
||||
HvApi.hv_vm_create(IntPtr.Zero).ThrowOnError();
|
||||
_ipaAllocator = ipaAllocator = new HvIpaAllocator();
|
||||
}
|
||||
else
|
||||
{
|
||||
ipaAllocator = _ipaAllocator;
|
||||
}
|
||||
}
|
||||
|
||||
ulong baseAddress;
|
||||
|
||||
lock (ipaAllocator)
|
||||
{
|
||||
baseAddress = ipaAllocator.Allocate(block.Size, AsIpaAlignment);
|
||||
}
|
||||
|
||||
var rwx = hv_memory_flags_t.HV_MEMORY_READ | hv_memory_flags_t.HV_MEMORY_WRITE | hv_memory_flags_t.HV_MEMORY_EXEC;
|
||||
|
||||
HvApi.hv_vm_map((ulong)block.Pointer, baseAddress, block.Size, rwx).ThrowOnError();
|
||||
|
||||
return (baseAddress, ipaAllocator);
|
||||
}
|
||||
|
||||
public static void DestroyAddressSpace(ulong address, ulong size)
|
||||
{
|
||||
HvApi.hv_vm_unmap(address, size);
|
||||
|
||||
HvIpaAllocator ipaAllocator;
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
if (--_addressSpaces == 0)
|
||||
{
|
||||
HvApi.hv_vm_destroy().ThrowOnError();
|
||||
}
|
||||
|
||||
ipaAllocator = _ipaAllocator;
|
||||
}
|
||||
|
||||
lock (ipaAllocator)
|
||||
{
|
||||
ipaAllocator.Free(address, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
46
src/Ryujinx.Cpu/AppleHv/IHvExecutionContext.cs
Normal file
46
src/Ryujinx.Cpu/AppleHv/IHvExecutionContext.cs
Normal file
|
@ -0,0 +1,46 @@
|
|||
using ARMeilleure.State;
|
||||
|
||||
namespace Ryujinx.Cpu.AppleHv
|
||||
{
|
||||
public interface IHvExecutionContext
|
||||
{
|
||||
ulong Pc { get; set; }
|
||||
ulong ElrEl1 { get; set; }
|
||||
ulong EsrEl1 { get; set; }
|
||||
|
||||
long TpidrEl0 { get; set; }
|
||||
long TpidrroEl0 { get; set; }
|
||||
|
||||
uint Pstate { get; set; }
|
||||
|
||||
uint Fpcr { get; set; }
|
||||
uint Fpsr { get; set; }
|
||||
|
||||
ulong GetX(int index);
|
||||
void SetX(int index, ulong value);
|
||||
|
||||
V128 GetV(int index);
|
||||
void SetV(int index, V128 value);
|
||||
|
||||
public void Load(IHvExecutionContext context)
|
||||
{
|
||||
Pc = context.Pc;
|
||||
ElrEl1 = context.ElrEl1;
|
||||
EsrEl1 = context.EsrEl1;
|
||||
TpidrEl0 = context.TpidrEl0;
|
||||
TpidrroEl0 = context.TpidrroEl0;
|
||||
Pstate = context.Pstate;
|
||||
Fpcr = context.Fpcr;
|
||||
Fpsr = context.Fpsr;
|
||||
|
||||
for (int i = 0; i < 32; i++)
|
||||
{
|
||||
SetX(i, context.GetX(i));
|
||||
SetV(i, context.GetV(i));
|
||||
}
|
||||
}
|
||||
|
||||
void RequestInterrupt();
|
||||
bool GetAndClearInterruptRequested();
|
||||
}
|
||||
}
|
64
src/Ryujinx.Cpu/ExceptionCallbacks.cs
Normal file
64
src/Ryujinx.Cpu/ExceptionCallbacks.cs
Normal file
|
@ -0,0 +1,64 @@
|
|||
namespace Ryujinx.Cpu
|
||||
{
|
||||
/// <summary>
|
||||
/// Exception callback without any additional arguments.
|
||||
/// </summary>
|
||||
/// <param name="context">Context for the thread where the exception was triggered</param>
|
||||
public delegate void ExceptionCallbackNoArgs(IExecutionContext context);
|
||||
|
||||
/// <summary>
|
||||
/// Exception callback.
|
||||
/// </summary>
|
||||
/// <param name="context">Context for the thread where the exception was triggered</param>
|
||||
/// <param name="address">Address of the instruction that caused the exception</param>
|
||||
/// <param name="imm">Immediate value of the instruction that caused the exception, or for undefined instruction, the instruction itself</param>
|
||||
public delegate void ExceptionCallback(IExecutionContext context, ulong address, int imm);
|
||||
|
||||
/// <summary>
|
||||
/// Stores handlers for the various CPU exceptions.
|
||||
/// </summary>
|
||||
public readonly struct ExceptionCallbacks
|
||||
{
|
||||
/// <summary>
|
||||
/// Handler for CPU interrupts triggered using <see cref="IExecutionContext.RequestInterrupt"/>.
|
||||
/// </summary>
|
||||
public readonly ExceptionCallbackNoArgs InterruptCallback;
|
||||
|
||||
/// <summary>
|
||||
/// Handler for CPU software interrupts caused by the Arm BRK instruction.
|
||||
/// </summary>
|
||||
public readonly ExceptionCallback BreakCallback;
|
||||
|
||||
/// <summary>
|
||||
/// Handler for CPU software interrupts caused by the Arm SVC instruction.
|
||||
/// </summary>
|
||||
public readonly ExceptionCallback SupervisorCallback;
|
||||
|
||||
/// <summary>
|
||||
/// Handler for CPU software interrupts caused by any undefined Arm instruction.
|
||||
/// </summary>
|
||||
public readonly ExceptionCallback UndefinedCallback;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new exception callbacks structure.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// All handlers are optional, and if null, the CPU will just continue executing as if nothing happened.
|
||||
/// </remarks>
|
||||
/// <param name="interruptCallback">Handler for CPU interrupts triggered using <see cref="IExecutionContext.RequestInterrupt"/></param>
|
||||
/// <param name="breakCallback">Handler for CPU software interrupts caused by the Arm BRK instruction</param>
|
||||
/// <param name="supervisorCallback">Handler for CPU software interrupts caused by the Arm SVC instruction</param>
|
||||
/// <param name="undefinedCallback">Handler for CPU software interrupts caused by any undefined Arm instruction</param>
|
||||
public ExceptionCallbacks(
|
||||
ExceptionCallbackNoArgs interruptCallback = null,
|
||||
ExceptionCallback breakCallback = null,
|
||||
ExceptionCallback supervisorCallback = null,
|
||||
ExceptionCallback undefinedCallback = null)
|
||||
{
|
||||
InterruptCallback = interruptCallback;
|
||||
BreakCallback = breakCallback;
|
||||
SupervisorCallback = supervisorCallback;
|
||||
UndefinedCallback = undefinedCallback;
|
||||
}
|
||||
}
|
||||
}
|
61
src/Ryujinx.Cpu/ICpuContext.cs
Normal file
61
src/Ryujinx.Cpu/ICpuContext.cs
Normal file
|
@ -0,0 +1,61 @@
|
|||
namespace Ryujinx.Cpu
|
||||
{
|
||||
/// <summary>
|
||||
/// CPU context interface.
|
||||
/// </summary>
|
||||
public interface ICpuContext
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a new execution context that will store thread CPU register state when executing guest code.
|
||||
/// </summary>
|
||||
/// <param name="exceptionCallbacks">Optional functions to be called when the CPU receives an interrupt</param>
|
||||
/// <returns>Execution context</returns>
|
||||
IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks);
|
||||
|
||||
/// <summary>
|
||||
/// Starts executing code at a specified entry point address.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This function only returns when the execution is stopped, by calling <see cref="IExecutionContext.StopRunning"/>.
|
||||
/// </remarks>
|
||||
/// <param name="context">Execution context to be used for this run</param>
|
||||
/// <param name="address">Entry point address</param>
|
||||
void Execute(IExecutionContext context, ulong address);
|
||||
|
||||
/// <summary>
|
||||
/// Invalidates the instruction cache for a given memory region.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This should be called if code is modified to make the CPU emulator aware of the modifications,
|
||||
/// otherwise it might run stale code which will lead to errors and crashes.
|
||||
/// Calling this function is not necessary if the code memory was modified by guest code,
|
||||
/// as the expectation is that it will do it on its own using the appropriate cache invalidation instructions,
|
||||
/// except on Arm32 where those instructions can't be used in unprivileged mode.
|
||||
/// </remarks>
|
||||
/// <param name="address">Address of the region to be invalidated</param>
|
||||
/// <param name="size">Size of the region to be invalidated</param>
|
||||
void InvalidateCacheRegion(ulong address, ulong size);
|
||||
|
||||
/// <summary>
|
||||
/// Loads cached code from disk for a given application.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// If the execution engine is recompiling guest code, this can be used to load cached code from disk.
|
||||
/// </remarks>
|
||||
/// <param name="titleIdText">Title ID of the application in padded hex form</param>
|
||||
/// <param name="displayVersion">Version of the application</param>
|
||||
/// <param name="enabled">True if the cache should be loaded from disk if it exists, false otherwise</param>
|
||||
/// <returns>Disk cache load progress reporter and manager</returns>
|
||||
IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled);
|
||||
|
||||
/// <summary>
|
||||
/// Indicates that code has been loaded into guest memory, and that it might be executed in the future.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Some execution engines might use this information to cache recompiled code on disk or to ensure it can be executed.
|
||||
/// </remarks>
|
||||
/// <param name="address">CPU virtual address where the code starts</param>
|
||||
/// <param name="size">Size of the code range in bytes</param>
|
||||
void PrepareCodeRange(ulong address, ulong size);
|
||||
}
|
||||
}
|
18
src/Ryujinx.Cpu/ICpuEngine.cs
Normal file
18
src/Ryujinx.Cpu/ICpuEngine.cs
Normal file
|
@ -0,0 +1,18 @@
|
|||
using ARMeilleure.Memory;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
/// <summary>
|
||||
/// CPU execution engine interface.
|
||||
/// </summary>
|
||||
public interface ICpuEngine
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a new CPU context that can be used to run code for multiple threads sharing an address space.
|
||||
/// </summary>
|
||||
/// <param name="memoryManager">Memory manager for the address space of the context</param>
|
||||
/// <param name="for64Bit">Indicates if the context will be used to run 64-bit or 32-bit Arm code</param>
|
||||
/// <returns>CPU context</returns>
|
||||
ICpuContext CreateCpuContext(IMemoryManager memoryManager, bool for64Bit);
|
||||
}
|
||||
}
|
20
src/Ryujinx.Cpu/IDiskCacheState.cs
Normal file
20
src/Ryujinx.Cpu/IDiskCacheState.cs
Normal file
|
@ -0,0 +1,20 @@
|
|||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
/// <summary>
|
||||
/// Disk cache load state report and management interface.
|
||||
/// </summary>
|
||||
public interface IDiskCacheLoadState
|
||||
{
|
||||
/// <summary>
|
||||
/// Event used to report the cache load progress.
|
||||
/// </summary>
|
||||
event Action<LoadState, int, int> StateChanged;
|
||||
|
||||
/// <summary>
|
||||
/// Cancels the disk cache load process.
|
||||
/// </summary>
|
||||
void Cancel();
|
||||
}
|
||||
}
|
112
src/Ryujinx.Cpu/IExecutionContext.cs
Normal file
112
src/Ryujinx.Cpu/IExecutionContext.cs
Normal file
|
@ -0,0 +1,112 @@
|
|||
using ARMeilleure.State;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
/// <summary>
|
||||
/// CPU register state interface.
|
||||
/// </summary>
|
||||
public interface IExecutionContext : IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Current Program Counter.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// In some implementations, this value might not be accurate and might not point to the last instruction executed.
|
||||
/// </remarks>
|
||||
ulong Pc { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Thread ID Register (EL0).
|
||||
/// </summary>
|
||||
long TpidrEl0 { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Thread ID Register (read-only) (EL0).
|
||||
/// </summary>
|
||||
long TpidrroEl0 { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Processor State Register.
|
||||
/// </summary>
|
||||
uint Pstate { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Floating-point Control Register.
|
||||
/// </summary>
|
||||
uint Fpcr { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Floating-point Status Register.
|
||||
/// </summary>
|
||||
uint Fpsr { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Indicates whenever the CPU is running 64-bit (AArch64 mode) or 32-bit (AArch32 mode) code.
|
||||
/// </summary>
|
||||
bool IsAarch32 { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Indicates whenever the CPU is still running code.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Even if this is false, the guest code might be still exiting.
|
||||
/// One must not assume that the code is no longer running from this property alone.
|
||||
/// </remarks>
|
||||
bool Running { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of a general purpose register.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The special <paramref name="index"/> of 31 can be used to access the SP (Stack Pointer) register.
|
||||
/// </remarks>
|
||||
/// <param name="index">Index of the register, in the range 0-31 (inclusive)</param>
|
||||
/// <returns>The register value</returns>
|
||||
ulong GetX(int index);
|
||||
|
||||
/// <summary>
|
||||
/// Sets the value of a general purpose register.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The special <paramref name="index"/> of 31 can be used to access the SP (Stack Pointer) register.
|
||||
/// </remarks>
|
||||
/// <param name="index">Index of the register, in the range 0-31 (inclusive)</param>
|
||||
/// <param name="value">Value to be set</param>
|
||||
void SetX(int index, ulong value);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the value of a FP/SIMD register.
|
||||
/// </summary>
|
||||
/// <param name="index">Index of the register, in the range 0-31 (inclusive)</param>
|
||||
/// <returns>The register value</returns>
|
||||
V128 GetV(int index);
|
||||
|
||||
/// <summary>
|
||||
/// Sets the value of a FP/SIMD register.
|
||||
/// </summary>
|
||||
/// <param name="index">Index of the register, in the range 0-31 (inclusive)</param>
|
||||
/// <param name="value">Value to be set</param>
|
||||
void SetV(int index, V128 value);
|
||||
|
||||
/// <summary>
|
||||
/// Requests the thread to stop running temporarily and call <see cref="ExceptionCallbacks.InterruptCallback"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The thread might not pause immediately.
|
||||
/// One must not assume that guest code is no longer being executed by the thread after calling this function.
|
||||
/// </remarks>
|
||||
void RequestInterrupt();
|
||||
|
||||
/// <summary>
|
||||
/// Requests the thread to stop running guest code and return as soon as possible.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The thread might not stop immediately.
|
||||
/// One must not assume that guest code is no longer being executed by the thread after calling this function.
|
||||
/// After a thread has been stopped, it can't be restarted with the same <see cref="IExecutionContext"/>.
|
||||
/// If you only need to pause the thread temporarily, use <see cref="RequestInterrupt"/> instead.
|
||||
/// </remarks>
|
||||
void StopRunning();
|
||||
}
|
||||
}
|
31
src/Ryujinx.Cpu/ITickSource.cs
Normal file
31
src/Ryujinx.Cpu/ITickSource.cs
Normal file
|
@ -0,0 +1,31 @@
|
|||
using ARMeilleure.State;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
/// <summary>
|
||||
/// Tick source interface.
|
||||
/// </summary>
|
||||
public interface ITickSource : ICounter
|
||||
{
|
||||
/// <summary>
|
||||
/// Time elapsed since the counter was created.
|
||||
/// </summary>
|
||||
TimeSpan ElapsedTime { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Time elapsed since the counter was created, in seconds.
|
||||
/// </summary>
|
||||
double ElapsedSeconds { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Stops counting.
|
||||
/// </summary>
|
||||
void Suspend();
|
||||
|
||||
/// <summary>
|
||||
/// Resumes counting after a call to <see cref="Suspend"/>.
|
||||
/// </summary>
|
||||
void Resume();
|
||||
}
|
||||
}
|
56
src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs
Normal file
56
src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs
Normal file
|
@ -0,0 +1,56 @@
|
|||
using Ryujinx.Cpu.Tracking;
|
||||
using Ryujinx.Memory;
|
||||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
public interface IVirtualMemoryManagerTracked : IVirtualMemoryManager
|
||||
{
|
||||
/// <summary>
|
||||
/// Reads data from CPU mapped memory, with read tracking
|
||||
/// </summary>
|
||||
/// <typeparam name="T">Type of the data being read</typeparam>
|
||||
/// <param name="va">Virtual address of the data in memory</param>
|
||||
/// <returns>The data</returns>
|
||||
T ReadTracked<T>(ulong va) where T : unmanaged;
|
||||
|
||||
/// <summary>
|
||||
/// Writes data to CPU mapped memory, without write tracking.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address to write the data into</param>
|
||||
/// <param name="data">Data to be written</param>
|
||||
void WriteUntracked(ulong va, ReadOnlySpan<byte> data);
|
||||
|
||||
/// <summary>
|
||||
/// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with.
|
||||
/// </summary>
|
||||
/// <param name="address">CPU virtual address of the region</param>
|
||||
/// <param name="size">Size of the region</param>
|
||||
/// <param name="id">Handle ID</param>
|
||||
/// <returns>The memory tracking handle</returns>
|
||||
CpuRegionHandle BeginTracking(ulong address, ulong size, int id);
|
||||
|
||||
/// <summary>
|
||||
/// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.
|
||||
/// </summary>
|
||||
/// <param name="address">CPU virtual address of the region</param>
|
||||
/// <param name="size">Size of the region</param>
|
||||
/// <param name="handles">Handles to inherit state from or reuse. When none are present, provide null</param>
|
||||
/// <param name="granularity">Desired granularity of write tracking</param>
|
||||
/// <param name="id">Handle ID</param>
|
||||
/// <returns>The memory tracking handle</returns>
|
||||
CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id);
|
||||
|
||||
/// <summary>
|
||||
/// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.
|
||||
/// </summary>
|
||||
/// <param name="address">CPU virtual address of the region</param>
|
||||
/// <param name="size">Size of the region</param>
|
||||
/// <param name="granularity">Desired granularity of write tracking</param>
|
||||
/// <param name="id">Handle ID</param>
|
||||
/// <returns>The memory tracking handle</returns>
|
||||
CpuSmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id);
|
||||
}
|
||||
}
|
53
src/Ryujinx.Cpu/Jit/JitCpuContext.cs
Normal file
53
src/Ryujinx.Cpu/Jit/JitCpuContext.cs
Normal file
|
@ -0,0 +1,53 @@
|
|||
using ARMeilleure.Memory;
|
||||
using ARMeilleure.Translation;
|
||||
|
||||
namespace Ryujinx.Cpu.Jit
|
||||
{
|
||||
class JitCpuContext : ICpuContext
|
||||
{
|
||||
private readonly ITickSource _tickSource;
|
||||
private readonly Translator _translator;
|
||||
|
||||
public JitCpuContext(ITickSource tickSource, IMemoryManager memory, bool for64Bit)
|
||||
{
|
||||
_tickSource = tickSource;
|
||||
_translator = new Translator(new JitMemoryAllocator(), memory, for64Bit);
|
||||
memory.UnmapEvent += UnmapHandler;
|
||||
}
|
||||
|
||||
private void UnmapHandler(ulong address, ulong size)
|
||||
{
|
||||
_translator.InvalidateJitCacheRegion(address, size);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IExecutionContext CreateExecutionContext(ExceptionCallbacks exceptionCallbacks)
|
||||
{
|
||||
return new JitExecutionContext(new JitMemoryAllocator(), _tickSource, exceptionCallbacks);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Execute(IExecutionContext context, ulong address)
|
||||
{
|
||||
_translator.Execute(((JitExecutionContext)context).Impl, address);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void InvalidateCacheRegion(ulong address, ulong size)
|
||||
{
|
||||
_translator.InvalidateJitCacheRegion(address, size);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IDiskCacheLoadState LoadDiskCache(string titleIdText, string displayVersion, bool enabled)
|
||||
{
|
||||
return new JitDiskCacheLoadState(_translator.LoadDiskCache(titleIdText, displayVersion, enabled));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void PrepareCodeRange(ulong address, ulong size)
|
||||
{
|
||||
_translator.PrepareCodeRange(address, size);
|
||||
}
|
||||
}
|
||||
}
|
38
src/Ryujinx.Cpu/Jit/JitDiskCacheLoadState.cs
Normal file
38
src/Ryujinx.Cpu/Jit/JitDiskCacheLoadState.cs
Normal file
|
@ -0,0 +1,38 @@
|
|||
using ARMeilleure.Translation.PTC;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.Jit
|
||||
{
|
||||
public class JitDiskCacheLoadState : IDiskCacheLoadState
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public event Action<LoadState, int, int> StateChanged;
|
||||
|
||||
private readonly IPtcLoadState _loadState;
|
||||
|
||||
public JitDiskCacheLoadState(IPtcLoadState loadState)
|
||||
{
|
||||
loadState.PtcStateChanged += LoadStateChanged;
|
||||
_loadState = loadState;
|
||||
}
|
||||
|
||||
private void LoadStateChanged(PtcLoadingState newState, int current, int total)
|
||||
{
|
||||
LoadState state = newState switch
|
||||
{
|
||||
PtcLoadingState.Start => LoadState.Unloaded,
|
||||
PtcLoadingState.Loading => LoadState.Loading,
|
||||
PtcLoadingState.Loaded => LoadState.Loaded,
|
||||
_ => throw new ArgumentException($"Invalid load state \"{newState}\".")
|
||||
};
|
||||
|
||||
StateChanged?.Invoke(state, current, total);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Cancel()
|
||||
{
|
||||
_loadState.Continue();
|
||||
}
|
||||
}
|
||||
}
|
20
src/Ryujinx.Cpu/Jit/JitEngine.cs
Normal file
20
src/Ryujinx.Cpu/Jit/JitEngine.cs
Normal file
|
@ -0,0 +1,20 @@
|
|||
using ARMeilleure.Memory;
|
||||
|
||||
namespace Ryujinx.Cpu.Jit
|
||||
{
|
||||
public class JitEngine : ICpuEngine
|
||||
{
|
||||
private readonly ITickSource _tickSource;
|
||||
|
||||
public JitEngine(ITickSource tickSource)
|
||||
{
|
||||
_tickSource = tickSource;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ICpuContext CreateCpuContext(IMemoryManager memoryManager, bool for64Bit)
|
||||
{
|
||||
return new JitCpuContext(_tickSource, memoryManager, for64Bit);
|
||||
}
|
||||
}
|
||||
}
|
123
src/Ryujinx.Cpu/Jit/JitExecutionContext.cs
Normal file
123
src/Ryujinx.Cpu/Jit/JitExecutionContext.cs
Normal file
|
@ -0,0 +1,123 @@
|
|||
using ARMeilleure.Memory;
|
||||
using ARMeilleure.State;
|
||||
|
||||
namespace Ryujinx.Cpu.Jit
|
||||
{
|
||||
class JitExecutionContext : IExecutionContext
|
||||
{
|
||||
private readonly ExecutionContext _impl;
|
||||
internal ExecutionContext Impl => _impl;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ulong Pc => _impl.Pc;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public long TpidrEl0
|
||||
{
|
||||
get => _impl.TpidrEl0;
|
||||
set => _impl.TpidrEl0 = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public long TpidrroEl0
|
||||
{
|
||||
get => _impl.TpidrroEl0;
|
||||
set => _impl.TpidrroEl0 = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public uint Pstate
|
||||
{
|
||||
get => _impl.Pstate;
|
||||
set => _impl.Pstate = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public uint Fpcr
|
||||
{
|
||||
get => (uint)_impl.Fpcr;
|
||||
set => _impl.Fpcr = (FPCR)value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public uint Fpsr
|
||||
{
|
||||
get => (uint)_impl.Fpsr;
|
||||
set => _impl.Fpsr = (FPSR)value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool IsAarch32
|
||||
{
|
||||
get => _impl.IsAarch32;
|
||||
set => _impl.IsAarch32 = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool Running => _impl.Running;
|
||||
|
||||
private readonly ExceptionCallbacks _exceptionCallbacks;
|
||||
|
||||
public JitExecutionContext(IJitMemoryAllocator allocator, ICounter counter, ExceptionCallbacks exceptionCallbacks)
|
||||
{
|
||||
_impl = new ExecutionContext(
|
||||
allocator,
|
||||
counter,
|
||||
InterruptHandler,
|
||||
BreakHandler,
|
||||
SupervisorCallHandler,
|
||||
UndefinedHandler);
|
||||
|
||||
_exceptionCallbacks = exceptionCallbacks;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ulong GetX(int index) => _impl.GetX(index);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SetX(int index, ulong value) => _impl.SetX(index, value);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public V128 GetV(int index) => _impl.GetV(index);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SetV(int index, V128 value) => _impl.SetV(index, value);
|
||||
|
||||
private void InterruptHandler(ExecutionContext context)
|
||||
{
|
||||
_exceptionCallbacks.InterruptCallback?.Invoke(this);
|
||||
}
|
||||
|
||||
private void BreakHandler(ExecutionContext context, ulong address, int imm)
|
||||
{
|
||||
_exceptionCallbacks.BreakCallback?.Invoke(this, address, imm);
|
||||
}
|
||||
|
||||
private void SupervisorCallHandler(ExecutionContext context, ulong address, int imm)
|
||||
{
|
||||
_exceptionCallbacks.SupervisorCallback?.Invoke(this, address, imm);
|
||||
}
|
||||
|
||||
private void UndefinedHandler(ExecutionContext context, ulong address, int opCode)
|
||||
{
|
||||
_exceptionCallbacks.UndefinedCallback?.Invoke(this, address, opCode);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void RequestInterrupt()
|
||||
{
|
||||
_impl.RequestInterrupt();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void StopRunning()
|
||||
{
|
||||
_impl.StopRunning();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_impl.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
13
src/Ryujinx.Cpu/Jit/JitMemoryAllocator.cs
Normal file
13
src/Ryujinx.Cpu/Jit/JitMemoryAllocator.cs
Normal file
|
@ -0,0 +1,13 @@
|
|||
using ARMeilleure.Memory;
|
||||
using Ryujinx.Memory;
|
||||
|
||||
namespace Ryujinx.Cpu.Jit
|
||||
{
|
||||
public class JitMemoryAllocator : IJitMemoryAllocator
|
||||
{
|
||||
public IJitMemoryBlock Allocate(ulong size) => new JitMemoryBlock(size, MemoryAllocationFlags.None);
|
||||
public IJitMemoryBlock Reserve(ulong size) => new JitMemoryBlock(size, MemoryAllocationFlags.Reserve | MemoryAllocationFlags.Jit);
|
||||
|
||||
public ulong GetPageSize() => MemoryBlock.GetPageSize();
|
||||
}
|
||||
}
|
24
src/Ryujinx.Cpu/Jit/JitMemoryBlock.cs
Normal file
24
src/Ryujinx.Cpu/Jit/JitMemoryBlock.cs
Normal file
|
@ -0,0 +1,24 @@
|
|||
using ARMeilleure.Memory;
|
||||
using Ryujinx.Memory;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.Jit
|
||||
{
|
||||
public class JitMemoryBlock : IJitMemoryBlock
|
||||
{
|
||||
private readonly MemoryBlock _impl;
|
||||
|
||||
public IntPtr Pointer => _impl.Pointer;
|
||||
|
||||
public JitMemoryBlock(ulong size, MemoryAllocationFlags flags)
|
||||
{
|
||||
_impl = new MemoryBlock(size, flags);
|
||||
}
|
||||
|
||||
public bool Commit(ulong offset, ulong size) => _impl.Commit(offset, size);
|
||||
public void MapAsRx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadAndExecute);
|
||||
public void MapAsRwx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadWriteExecute);
|
||||
|
||||
public void Dispose() => _impl.Dispose();
|
||||
}
|
||||
}
|
704
src/Ryujinx.Cpu/Jit/MemoryManager.cs
Normal file
704
src/Ryujinx.Cpu/Jit/MemoryManager.cs
Normal file
|
@ -0,0 +1,704 @@
|
|||
using ARMeilleure.Memory;
|
||||
using Ryujinx.Cpu.Tracking;
|
||||
using Ryujinx.Memory;
|
||||
using Ryujinx.Memory.Range;
|
||||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Cpu.Jit
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a CPU memory manager.
|
||||
/// </summary>
|
||||
public sealed class MemoryManager : MemoryManagerBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
|
||||
{
|
||||
public const int PageBits = 12;
|
||||
public const int PageSize = 1 << PageBits;
|
||||
public const int PageMask = PageSize - 1;
|
||||
|
||||
private const int PteSize = 8;
|
||||
|
||||
private const int PointerTagBit = 62;
|
||||
|
||||
private readonly MemoryBlock _backingMemory;
|
||||
private readonly InvalidAccessHandler _invalidAccessHandler;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool Supports4KBPages => true;
|
||||
|
||||
/// <summary>
|
||||
/// Address space width in bits.
|
||||
/// </summary>
|
||||
public int AddressSpaceBits { get; }
|
||||
|
||||
private readonly ulong _addressSpaceSize;
|
||||
|
||||
private readonly MemoryBlock _pageTable;
|
||||
|
||||
/// <summary>
|
||||
/// Page table base pointer.
|
||||
/// </summary>
|
||||
public IntPtr PageTablePointer => _pageTable.Pointer;
|
||||
|
||||
public MemoryManagerType Type => MemoryManagerType.SoftwarePageTable;
|
||||
|
||||
public MemoryTracking Tracking { get; }
|
||||
|
||||
public event Action<ulong, ulong> UnmapEvent;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the memory manager.
|
||||
/// </summary>
|
||||
/// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
|
||||
/// <param name="addressSpaceSize">Size of the address space</param>
|
||||
/// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
|
||||
public MemoryManager(MemoryBlock backingMemory, ulong addressSpaceSize, InvalidAccessHandler invalidAccessHandler = null)
|
||||
{
|
||||
_backingMemory = backingMemory;
|
||||
_invalidAccessHandler = invalidAccessHandler;
|
||||
|
||||
ulong asSize = PageSize;
|
||||
int asBits = PageBits;
|
||||
|
||||
while (asSize < addressSpaceSize)
|
||||
{
|
||||
asSize <<= 1;
|
||||
asBits++;
|
||||
}
|
||||
|
||||
AddressSpaceBits = asBits;
|
||||
_addressSpaceSize = asSize;
|
||||
_pageTable = new MemoryBlock((asSize / PageSize) * PteSize);
|
||||
|
||||
Tracking = new MemoryTracking(this, PageSize);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
ulong remainingSize = size;
|
||||
ulong oVa = va;
|
||||
ulong oPa = pa;
|
||||
while (remainingSize != 0)
|
||||
{
|
||||
_pageTable.Write((va / PageSize) * PteSize, PaToPte(pa));
|
||||
|
||||
va += PageSize;
|
||||
pa += PageSize;
|
||||
remainingSize -= PageSize;
|
||||
}
|
||||
|
||||
Tracking.Map(oVa, size);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void MapForeign(ulong va, nuint hostPointer, ulong size)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Unmap(ulong va, ulong size)
|
||||
{
|
||||
// If size is 0, there's nothing to unmap, just exit early.
|
||||
if (size == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
UnmapEvent?.Invoke(va, size);
|
||||
Tracking.Unmap(va, size);
|
||||
|
||||
ulong remainingSize = size;
|
||||
while (remainingSize != 0)
|
||||
{
|
||||
_pageTable.Write((va / PageSize) * PteSize, 0UL);
|
||||
|
||||
va += PageSize;
|
||||
remainingSize -= PageSize;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public T Read<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
return MemoryMarshal.Cast<byte, T>(GetSpan(va, Unsafe.SizeOf<T>()))[0];
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public T ReadTracked<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
try
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), false);
|
||||
|
||||
return Read<T>(va);
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Read(ulong va, Span<byte> data)
|
||||
{
|
||||
ReadImpl(va, data);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Write<T>(ulong va, T value) where T : unmanaged
|
||||
{
|
||||
Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Write(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
SignalMemoryTracking(va, (ulong)data.Length, true);
|
||||
|
||||
WriteImpl(va, data);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
WriteImpl(va, data);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
SignalMemoryTracking(va, (ulong)data.Length, false);
|
||||
|
||||
if (IsContiguousAndMapped(va, data.Length))
|
||||
{
|
||||
var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length);
|
||||
|
||||
bool changed = !data.SequenceEqual(target);
|
||||
|
||||
if (changed)
|
||||
{
|
||||
data.CopyTo(target);
|
||||
}
|
||||
|
||||
return changed;
|
||||
}
|
||||
else
|
||||
{
|
||||
WriteImpl(va, data);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes data to CPU mapped memory.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address to write the data into</param>
|
||||
/// <param name="data">Data to be written</param>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void WriteImpl(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
try
|
||||
{
|
||||
AssertValidAddressAndSize(va, (ulong)data.Length);
|
||||
|
||||
if (IsContiguousAndMapped(va, data.Length))
|
||||
{
|
||||
data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length));
|
||||
}
|
||||
else
|
||||
{
|
||||
int offset = 0, size;
|
||||
|
||||
if ((va & PageMask) != 0)
|
||||
{
|
||||
ulong pa = GetPhysicalAddressInternal(va);
|
||||
|
||||
size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
|
||||
|
||||
data.Slice(0, size).CopyTo(_backingMemory.GetSpan(pa, size));
|
||||
|
||||
offset += size;
|
||||
}
|
||||
|
||||
for (; offset < data.Length; offset += size)
|
||||
{
|
||||
ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
|
||||
|
||||
size = Math.Min(data.Length - offset, PageSize);
|
||||
|
||||
data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return ReadOnlySpan<byte>.Empty;
|
||||
}
|
||||
|
||||
if (tracked)
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)size, false);
|
||||
}
|
||||
|
||||
if (IsContiguousAndMapped(va, size))
|
||||
{
|
||||
return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size);
|
||||
}
|
||||
else
|
||||
{
|
||||
Span<byte> data = new byte[size];
|
||||
|
||||
ReadImpl(va, data);
|
||||
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public unsafe WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return new WritableRegion(null, va, Memory<byte>.Empty);
|
||||
}
|
||||
|
||||
if (IsContiguousAndMapped(va, size))
|
||||
{
|
||||
if (tracked)
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)size, true);
|
||||
}
|
||||
|
||||
return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size));
|
||||
}
|
||||
else
|
||||
{
|
||||
Memory<byte> memory = new byte[size];
|
||||
|
||||
GetSpan(va, size).CopyTo(memory.Span);
|
||||
|
||||
return new WritableRegion(this, va, memory, tracked);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ref T GetRef<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
if (!IsContiguous(va, Unsafe.SizeOf<T>()))
|
||||
{
|
||||
ThrowMemoryNotContiguous();
|
||||
}
|
||||
|
||||
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
|
||||
|
||||
return ref _backingMemory.GetRef<T>(GetPhysicalAddressInternal(va));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the number of pages in a virtual address range.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range</param>
|
||||
/// <param name="startVa">The virtual address of the beginning of the first page</param>
|
||||
/// <remarks>This function does not differentiate between allocated and unallocated pages.</remarks>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int GetPagesCount(ulong va, uint size, out ulong startVa)
|
||||
{
|
||||
// WARNING: Always check if ulong does not overflow during the operations.
|
||||
startVa = va & ~(ulong)PageMask;
|
||||
ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
|
||||
|
||||
return (int)(vaSpan / PageSize);
|
||||
}
|
||||
|
||||
private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va);
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsContiguous(ulong va, int size)
|
||||
{
|
||||
if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
|
||||
for (int page = 0; page < pages - 1; page++)
|
||||
{
|
||||
if (!ValidateAddress(va + PageSize))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
va += PageSize;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return Enumerable.Empty<HostMemoryRange>();
|
||||
}
|
||||
|
||||
var guestRegions = GetPhysicalRegionsImpl(va, size);
|
||||
if (guestRegions == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var regions = new HostMemoryRange[guestRegions.Count];
|
||||
|
||||
for (int i = 0; i < regions.Length; i++)
|
||||
{
|
||||
var guestRegion = guestRegions[i];
|
||||
IntPtr pointer = _backingMemory.GetPointer(guestRegion.Address, guestRegion.Size);
|
||||
regions[i] = new HostMemoryRange((nuint)(ulong)pointer, guestRegion.Size);
|
||||
}
|
||||
|
||||
return regions;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
|
||||
{
|
||||
if (size == 0)
|
||||
{
|
||||
return Enumerable.Empty<MemoryRange>();
|
||||
}
|
||||
|
||||
return GetPhysicalRegionsImpl(va, size);
|
||||
}
|
||||
|
||||
private List<MemoryRange> GetPhysicalRegionsImpl(ulong va, ulong size)
|
||||
{
|
||||
if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
|
||||
var regions = new List<MemoryRange>();
|
||||
|
||||
ulong regionStart = GetPhysicalAddressInternal(va);
|
||||
ulong regionSize = PageSize;
|
||||
|
||||
for (int page = 0; page < pages - 1; page++)
|
||||
{
|
||||
if (!ValidateAddress(va + PageSize))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ulong newPa = GetPhysicalAddressInternal(va + PageSize);
|
||||
|
||||
if (GetPhysicalAddressInternal(va) + PageSize != newPa)
|
||||
{
|
||||
regions.Add(new MemoryRange(regionStart, regionSize));
|
||||
regionStart = newPa;
|
||||
regionSize = 0;
|
||||
}
|
||||
|
||||
va += PageSize;
|
||||
regionSize += PageSize;
|
||||
}
|
||||
|
||||
regions.Add(new MemoryRange(regionStart, regionSize));
|
||||
|
||||
return regions;
|
||||
}
|
||||
|
||||
private void ReadImpl(ulong va, Span<byte> data)
|
||||
{
|
||||
if (data.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
AssertValidAddressAndSize(va, (ulong)data.Length);
|
||||
|
||||
int offset = 0, size;
|
||||
|
||||
if ((va & PageMask) != 0)
|
||||
{
|
||||
ulong pa = GetPhysicalAddressInternal(va);
|
||||
|
||||
size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
|
||||
|
||||
_backingMemory.GetSpan(pa, size).CopyTo(data.Slice(0, size));
|
||||
|
||||
offset += size;
|
||||
}
|
||||
|
||||
for (; offset < data.Length; offset += size)
|
||||
{
|
||||
ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
|
||||
|
||||
size = Math.Min(data.Length - offset, PageSize);
|
||||
|
||||
_backingMemory.GetSpan(pa, size).CopyTo(data.Slice(offset, size));
|
||||
}
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool IsRangeMapped(ulong va, ulong size)
|
||||
{
|
||||
if (size == 0UL)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!ValidateAddressAndSize(va, size))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
|
||||
for (int page = 0; page < pages; page++)
|
||||
{
|
||||
if (!IsMapped(va))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
va += PageSize;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMapped(ulong va)
|
||||
{
|
||||
if (!ValidateAddress(va))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return _pageTable.Read<ulong>((va / PageSize) * PteSize) != 0;
|
||||
}
|
||||
|
||||
private bool ValidateAddress(ulong va)
|
||||
{
|
||||
return va < _addressSpaceSize;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the combination of virtual address and size is part of the addressable space.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
/// <returns>True if the combination of virtual address and size is part of the addressable space</returns>
|
||||
private bool ValidateAddressAndSize(ulong va, ulong size)
|
||||
{
|
||||
ulong endVa = va + size;
|
||||
return endVa >= va && endVa >= size && endVa <= _addressSpaceSize;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures the combination of virtual address and size is part of the addressable space.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
/// <exception cref="InvalidMemoryRegionException">Throw when the memory region specified outside the addressable space</exception>
|
||||
private void AssertValidAddressAndSize(ulong va, ulong size)
|
||||
{
|
||||
if (!ValidateAddressAndSize(va, size))
|
||||
{
|
||||
throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
}
|
||||
|
||||
private ulong GetPhysicalAddress(ulong va)
|
||||
{
|
||||
// We return -1L if the virtual address is invalid or unmapped.
|
||||
if (!ValidateAddress(va) || !IsMapped(va))
|
||||
{
|
||||
return ulong.MaxValue;
|
||||
}
|
||||
|
||||
return GetPhysicalAddressInternal(va);
|
||||
}
|
||||
|
||||
private ulong GetPhysicalAddressInternal(ulong va)
|
||||
{
|
||||
return PteToPa(_pageTable.Read<ulong>((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
// Protection is inverted on software pages, since the default value is 0.
|
||||
protection = (~protection) & MemoryPermission.ReadAndWrite;
|
||||
|
||||
long tag = protection switch
|
||||
{
|
||||
MemoryPermission.None => 0L,
|
||||
MemoryPermission.Write => 2L << PointerTagBit,
|
||||
_ => 3L << PointerTagBit
|
||||
};
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
ulong pageStart = va >> PageBits;
|
||||
long invTagMask = ~(0xffffL << 48);
|
||||
|
||||
for (int page = 0; page < pages; page++)
|
||||
{
|
||||
ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize);
|
||||
|
||||
long pte;
|
||||
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
}
|
||||
while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
|
||||
|
||||
pageStart++;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuRegionHandle BeginTracking(ulong address, ulong size, int id)
|
||||
{
|
||||
return new CpuRegionHandle(Tracking.BeginTracking(address, size, id));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id)
|
||||
{
|
||||
return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity, id));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuSmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
|
||||
{
|
||||
return new CpuSmartMultiRegionHandle(Tracking.BeginSmartGranularTracking(address, size, granularity, id));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
if (precise)
|
||||
{
|
||||
Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
|
||||
return;
|
||||
}
|
||||
|
||||
// We emulate guard pages for software memory access. This makes for an easy transition to
|
||||
// tracking using host guard pages in future, but also supporting platforms where this is not possible.
|
||||
|
||||
// Write tag includes read protection, since we don't have any read actions that aren't performed before write too.
|
||||
long tag = (write ? 3L : 1L) << PointerTagBit;
|
||||
|
||||
int pages = GetPagesCount(va, (uint)size, out _);
|
||||
ulong pageStart = va >> PageBits;
|
||||
|
||||
for (int page = 0; page < pages; page++)
|
||||
{
|
||||
ref long pageRef = ref _pageTable.GetRef<long>(pageStart * PteSize);
|
||||
|
||||
long pte;
|
||||
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
|
||||
if ((pte & tag) != 0)
|
||||
{
|
||||
Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
|
||||
break;
|
||||
}
|
||||
|
||||
pageStart++;
|
||||
}
|
||||
}
|
||||
|
||||
private ulong PaToPte(ulong pa)
|
||||
{
|
||||
return (ulong)_backingMemory.GetPointer(pa, PageSize);
|
||||
}
|
||||
|
||||
private ulong PteToPa(ulong pte)
|
||||
{
|
||||
return (ulong)((long)pte - _backingMemory.Pointer.ToInt64());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes of resources used by the memory manager.
|
||||
/// </summary>
|
||||
protected override void Destroy() => _pageTable.Dispose();
|
||||
|
||||
private void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message);
|
||||
}
|
||||
}
|
817
src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
Normal file
817
src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
Normal file
|
@ -0,0 +1,817 @@
|
|||
using ARMeilleure.Memory;
|
||||
using Ryujinx.Cpu.Tracking;
|
||||
using Ryujinx.Memory;
|
||||
using Ryujinx.Memory.Range;
|
||||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Cpu.Jit
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region.
|
||||
/// </summary>
|
||||
public sealed class MemoryManagerHostMapped : MemoryManagerBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
|
||||
{
|
||||
public const int PageBits = 12;
|
||||
public const int PageSize = 1 << PageBits;
|
||||
public const int PageMask = PageSize - 1;
|
||||
|
||||
public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry.
|
||||
public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set.
|
||||
|
||||
private enum HostMappedPtBits : ulong
|
||||
{
|
||||
Unmapped = 0,
|
||||
Mapped,
|
||||
WriteTracked,
|
||||
ReadWriteTracked,
|
||||
|
||||
MappedReplicated = 0x5555555555555555,
|
||||
WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa,
|
||||
ReadWriteTrackedReplicated = ulong.MaxValue
|
||||
}
|
||||
|
||||
private readonly InvalidAccessHandler _invalidAccessHandler;
|
||||
private readonly bool _unsafeMode;
|
||||
|
||||
private readonly AddressSpace _addressSpace;
|
||||
private readonly ulong _addressSpaceSize;
|
||||
|
||||
private readonly PageTable<ulong> _pageTable;
|
||||
|
||||
private readonly MemoryEhMeilleure _memoryEh;
|
||||
|
||||
private readonly ulong[] _pageBitmap;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool Supports4KBPages => MemoryBlock.GetPageSize() == PageSize;
|
||||
|
||||
public int AddressSpaceBits { get; }
|
||||
|
||||
public IntPtr PageTablePointer => _addressSpace.Base.Pointer;
|
||||
|
||||
public MemoryManagerType Type => _unsafeMode ? MemoryManagerType.HostMappedUnsafe : MemoryManagerType.HostMapped;
|
||||
|
||||
public MemoryTracking Tracking { get; }
|
||||
|
||||
public event Action<ulong, ulong> UnmapEvent;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the host mapped memory manager.
|
||||
/// </summary>
|
||||
/// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
|
||||
/// <param name="addressSpaceSize">Size of the address space</param>
|
||||
/// <param name="unsafeMode">True if unmanaged access should not be masked (unsafe), false otherwise.</param>
|
||||
/// <param name="invalidAccessHandler">Optional function to handle invalid memory accesses</param>
|
||||
public MemoryManagerHostMapped(MemoryBlock backingMemory, ulong addressSpaceSize, bool unsafeMode, InvalidAccessHandler invalidAccessHandler = null)
|
||||
{
|
||||
_pageTable = new PageTable<ulong>();
|
||||
_invalidAccessHandler = invalidAccessHandler;
|
||||
_unsafeMode = unsafeMode;
|
||||
_addressSpaceSize = addressSpaceSize;
|
||||
|
||||
ulong asSize = PageSize;
|
||||
int asBits = PageBits;
|
||||
|
||||
while (asSize < addressSpaceSize)
|
||||
{
|
||||
asSize <<= 1;
|
||||
asBits++;
|
||||
}
|
||||
|
||||
AddressSpaceBits = asBits;
|
||||
|
||||
_pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))];
|
||||
|
||||
_addressSpace = new AddressSpace(backingMemory, asSize, Supports4KBPages);
|
||||
|
||||
Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler);
|
||||
_memoryEh = new MemoryEhMeilleure(_addressSpace.Base, _addressSpace.Mirror, Tracking);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the virtual address is part of the addressable space.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address</param>
|
||||
/// <returns>True if the virtual address is part of the addressable space</returns>
|
||||
private bool ValidateAddress(ulong va)
|
||||
{
|
||||
return va < _addressSpaceSize;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the combination of virtual address and size is part of the addressable space.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
/// <returns>True if the combination of virtual address and size is part of the addressable space</returns>
|
||||
private bool ValidateAddressAndSize(ulong va, ulong size)
|
||||
{
|
||||
ulong endVa = va + size;
|
||||
return endVa >= va && endVa >= size && endVa <= _addressSpaceSize;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures the combination of virtual address and size is part of the addressable space.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
/// <exception cref="InvalidMemoryRegionException">Throw when the memory region specified outside the addressable space</exception>
|
||||
private void AssertValidAddressAndSize(ulong va, ulong size)
|
||||
{
|
||||
if (!ValidateAddressAndSize(va, size))
|
||||
{
|
||||
throw new InvalidMemoryRegionException($"va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ensures the combination of virtual address and size is part of the addressable space and fully mapped.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range in bytes</param>
|
||||
private void AssertMapped(ulong va, ulong size)
|
||||
{
|
||||
if (!ValidateAddressAndSize(va, size) || !IsRangeMappedImpl(va, size))
|
||||
{
|
||||
throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
_addressSpace.Map(va, pa, size, flags);
|
||||
AddMapping(va, size);
|
||||
PtMap(va, pa, size);
|
||||
|
||||
Tracking.Map(va, size);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void MapForeign(ulong va, nuint hostPointer, ulong size)
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Unmap(ulong va, ulong size)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
UnmapEvent?.Invoke(va, size);
|
||||
Tracking.Unmap(va, size);
|
||||
|
||||
RemoveMapping(va, size);
|
||||
PtUnmap(va, size);
|
||||
_addressSpace.Unmap(va, size);
|
||||
}
|
||||
|
||||
private void PtMap(ulong va, ulong pa, ulong size)
|
||||
{
|
||||
while (size != 0)
|
||||
{
|
||||
_pageTable.Map(va, pa);
|
||||
|
||||
va += PageSize;
|
||||
pa += PageSize;
|
||||
size -= PageSize;
|
||||
}
|
||||
}
|
||||
|
||||
private void PtUnmap(ulong va, ulong size)
|
||||
{
|
||||
while (size != 0)
|
||||
{
|
||||
_pageTable.Unmap(va);
|
||||
|
||||
va += PageSize;
|
||||
size -= PageSize;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public T Read<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
try
|
||||
{
|
||||
AssertMapped(va, (ulong)Unsafe.SizeOf<T>());
|
||||
|
||||
return _addressSpace.Mirror.Read<T>(va);
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public T ReadTracked<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
try
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), false);
|
||||
|
||||
return Read<T>(va);
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
return default;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Read(ulong va, Span<byte> data)
|
||||
{
|
||||
try
|
||||
{
|
||||
AssertMapped(va, (ulong)data.Length);
|
||||
|
||||
_addressSpace.Mirror.Read(va, data);
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Write<T>(ulong va, T value) where T : unmanaged
|
||||
{
|
||||
try
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), write: true);
|
||||
|
||||
_addressSpace.Mirror.Write(va, value);
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Write(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
try
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)data.Length, write: true);
|
||||
|
||||
_addressSpace.Mirror.Write(va, data);
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void WriteUntracked(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
try
|
||||
{
|
||||
AssertMapped(va, (ulong)data.Length);
|
||||
|
||||
_addressSpace.Mirror.Write(va, data);
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan<byte> data)
|
||||
{
|
||||
try
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)data.Length, false);
|
||||
|
||||
Span<byte> target = _addressSpace.Mirror.GetSpan(va, data.Length);
|
||||
bool changed = !data.SequenceEqual(target);
|
||||
|
||||
if (changed)
|
||||
{
|
||||
data.CopyTo(target);
|
||||
}
|
||||
|
||||
return changed;
|
||||
}
|
||||
catch (InvalidMemoryRegionException)
|
||||
{
|
||||
if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
|
||||
{
|
||||
throw;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ReadOnlySpan<byte> GetSpan(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
if (tracked)
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)size, write: false);
|
||||
}
|
||||
else
|
||||
{
|
||||
AssertMapped(va, (ulong)size);
|
||||
}
|
||||
|
||||
return _addressSpace.Mirror.GetSpan(va, size);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
|
||||
{
|
||||
if (tracked)
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)size, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
AssertMapped(va, (ulong)size);
|
||||
}
|
||||
|
||||
return _addressSpace.Mirror.GetWritableRegion(va, size);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ref T GetRef<T>(ulong va) where T : unmanaged
|
||||
{
|
||||
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf<T>(), true);
|
||||
|
||||
return ref _addressSpace.Mirror.GetRef<T>(va);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
public bool IsMapped(ulong va)
|
||||
{
|
||||
return ValidateAddress(va) && IsMappedImpl(va);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private bool IsMappedImpl(ulong va)
|
||||
{
|
||||
ulong page = va >> PageBits;
|
||||
|
||||
int bit = (int)((page & 31) << 1);
|
||||
|
||||
int pageIndex = (int)(page >> PageToPteShift);
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex];
|
||||
|
||||
ulong pte = Volatile.Read(ref pageRef);
|
||||
|
||||
return ((pte >> bit) & 3) != 0;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public bool IsRangeMapped(ulong va, ulong size)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
return IsRangeMappedImpl(va, size);
|
||||
}
|
||||
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex)
|
||||
{
|
||||
startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1);
|
||||
endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1));
|
||||
|
||||
pageIndex = (int)(pageStart >> PageToPteShift);
|
||||
pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift);
|
||||
}
|
||||
|
||||
private bool IsRangeMappedImpl(ulong va, ulong size)
|
||||
{
|
||||
int pages = GetPagesCount(va, size, out _);
|
||||
|
||||
if (pages == 1)
|
||||
{
|
||||
return IsMappedImpl(va);
|
||||
}
|
||||
|
||||
ulong pageStart = va >> PageBits;
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
// Check if either bit in each 2 bit page entry is set.
|
||||
// OR the block with itself shifted down by 1, and check the first bit of each entry.
|
||||
|
||||
ulong mask = BlockMappedMask & startMask;
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask &= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
ulong pte = Volatile.Read(ref pageRef);
|
||||
|
||||
pte |= pte >> 1;
|
||||
if ((pte & mask) != mask)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
mask = BlockMappedMask;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IEnumerable<HostMemoryRange> GetHostRegions(ulong va, ulong size)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
return Enumerable.Repeat(new HostMemoryRange((nuint)(ulong)_addressSpace.Mirror.GetPointer(va, size), size), 1);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IEnumerable<MemoryRange> GetPhysicalRegions(ulong va, ulong size)
|
||||
{
|
||||
int pages = GetPagesCount(va, (uint)size, out va);
|
||||
|
||||
var regions = new List<MemoryRange>();
|
||||
|
||||
ulong regionStart = GetPhysicalAddressChecked(va);
|
||||
ulong regionSize = PageSize;
|
||||
|
||||
for (int page = 0; page < pages - 1; page++)
|
||||
{
|
||||
if (!ValidateAddress(va + PageSize))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
ulong newPa = GetPhysicalAddressChecked(va + PageSize);
|
||||
|
||||
if (GetPhysicalAddressChecked(va) + PageSize != newPa)
|
||||
{
|
||||
regions.Add(new MemoryRange(regionStart, regionSize));
|
||||
regionStart = newPa;
|
||||
regionSize = 0;
|
||||
}
|
||||
|
||||
va += PageSize;
|
||||
regionSize += PageSize;
|
||||
}
|
||||
|
||||
regions.Add(new MemoryRange(regionStart, regionSize));
|
||||
|
||||
return regions;
|
||||
}
|
||||
|
||||
private ulong GetPhysicalAddressChecked(ulong va)
|
||||
{
|
||||
if (!IsMapped(va))
|
||||
{
|
||||
ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}");
|
||||
}
|
||||
|
||||
return GetPhysicalAddressInternal(va);
|
||||
}
|
||||
|
||||
private ulong GetPhysicalAddressInternal(ulong va)
|
||||
{
|
||||
return _pageTable.Read(va) + (va & PageMask);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
/// <remarks>
|
||||
/// This function also validates that the given range is both valid and mapped, and will throw if it is not.
|
||||
/// </remarks>
|
||||
public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
|
||||
{
|
||||
AssertValidAddressAndSize(va, size);
|
||||
|
||||
if (precise)
|
||||
{
|
||||
Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
|
||||
return;
|
||||
}
|
||||
|
||||
// Software table, used for managed memory tracking.
|
||||
|
||||
int pages = GetPagesCount(va, size, out _);
|
||||
ulong pageStart = va >> PageBits;
|
||||
|
||||
if (pages == 1)
|
||||
{
|
||||
ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked);
|
||||
|
||||
int bit = (int)((pageStart & 31) << 1);
|
||||
|
||||
int pageIndex = (int)(pageStart >> PageToPteShift);
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex];
|
||||
|
||||
ulong pte = Volatile.Read(ref pageRef);
|
||||
ulong state = ((pte >> bit) & 3);
|
||||
|
||||
if (state >= tag)
|
||||
{
|
||||
Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
|
||||
return;
|
||||
}
|
||||
else if (state == 0)
|
||||
{
|
||||
ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
ulong mask = startMask;
|
||||
|
||||
ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated;
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask &= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
|
||||
ulong pte = Volatile.Read(ref pageRef);
|
||||
ulong mappedMask = mask & BlockMappedMask;
|
||||
|
||||
ulong mappedPte = pte | (pte >> 1);
|
||||
if ((mappedPte & mappedMask) != mappedMask)
|
||||
{
|
||||
ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
|
||||
}
|
||||
|
||||
pte &= mask;
|
||||
if ((pte & anyTrackingTag) != 0) // Search for any tracking.
|
||||
{
|
||||
// Writes trigger any tracking.
|
||||
// Only trigger tracking from reads if both bits are set on any page.
|
||||
if (write || (pte & (pte >> 1) & BlockMappedMask) != 0)
|
||||
{
|
||||
Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
mask = ulong.MaxValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Computes the number of pages in a virtual address range.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual address of the range</param>
|
||||
/// <param name="size">Size of the range</param>
|
||||
/// <param name="startVa">The virtual address of the beginning of the first page</param>
|
||||
/// <remarks>This function does not differentiate between allocated and unallocated pages.</remarks>
|
||||
[MethodImpl(MethodImplOptions.AggressiveInlining)]
|
||||
private int GetPagesCount(ulong va, ulong size, out ulong startVa)
|
||||
{
|
||||
// WARNING: Always check if ulong does not overflow during the operations.
|
||||
startVa = va & ~(ulong)PageMask;
|
||||
ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
|
||||
|
||||
return (int)(vaSpan / PageSize);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
|
||||
{
|
||||
// Protection is inverted on software pages, since the default value is 0.
|
||||
protection = (~protection) & MemoryPermission.ReadAndWrite;
|
||||
|
||||
int pages = GetPagesCount(va, size, out va);
|
||||
ulong pageStart = va >> PageBits;
|
||||
|
||||
if (pages == 1)
|
||||
{
|
||||
ulong protTag = protection switch
|
||||
{
|
||||
MemoryPermission.None => (ulong)HostMappedPtBits.Mapped,
|
||||
MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked,
|
||||
_ => (ulong)HostMappedPtBits.ReadWriteTracked,
|
||||
};
|
||||
|
||||
int bit = (int)((pageStart & 31) << 1);
|
||||
|
||||
ulong tagMask = 3UL << bit;
|
||||
ulong invTagMask = ~tagMask;
|
||||
|
||||
ulong tag = protTag << bit;
|
||||
|
||||
int pageIndex = (int)(pageStart >> PageToPteShift);
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex];
|
||||
|
||||
ulong pte;
|
||||
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
}
|
||||
while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
|
||||
}
|
||||
else
|
||||
{
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
ulong mask = startMask;
|
||||
|
||||
ulong protTag = protection switch
|
||||
{
|
||||
MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated,
|
||||
MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated,
|
||||
_ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated,
|
||||
};
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask &= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
|
||||
ulong pte;
|
||||
ulong mappedMask;
|
||||
|
||||
// Change the protection of all 2 bit entries that are mapped.
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
|
||||
mappedMask = pte | (pte >> 1);
|
||||
mappedMask |= (mappedMask & BlockMappedMask) << 1;
|
||||
mappedMask &= mask; // Only update mapped pages within the given range.
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte);
|
||||
|
||||
mask = ulong.MaxValue;
|
||||
}
|
||||
}
|
||||
|
||||
protection = protection switch
|
||||
{
|
||||
MemoryPermission.None => MemoryPermission.ReadAndWrite,
|
||||
MemoryPermission.Write => MemoryPermission.Read,
|
||||
_ => MemoryPermission.None
|
||||
};
|
||||
|
||||
_addressSpace.Base.Reprotect(va, size, protection, false);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuRegionHandle BeginTracking(ulong address, ulong size, int id)
|
||||
{
|
||||
return new CpuRegionHandle(Tracking.BeginTracking(address, size, id));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuMultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable<IRegionHandle> handles, ulong granularity, int id)
|
||||
{
|
||||
return new CpuMultiRegionHandle(Tracking.BeginGranularTracking(address, size, handles, granularity, id));
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public CpuSmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
|
||||
{
|
||||
return new CpuSmartMultiRegionHandle(Tracking.BeginSmartGranularTracking(address, size, granularity, id));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds the given address mapping to the page table.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual memory address</param>
|
||||
/// <param name="size">Size to be mapped</param>
|
||||
private void AddMapping(ulong va, ulong size)
|
||||
{
|
||||
int pages = GetPagesCount(va, size, out _);
|
||||
ulong pageStart = va >> PageBits;
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
ulong mask = startMask;
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask &= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
|
||||
ulong pte;
|
||||
ulong mappedMask;
|
||||
|
||||
// Map all 2-bit entries that are unmapped.
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
|
||||
mappedMask = pte | (pte >> 1);
|
||||
mappedMask |= (mappedMask & BlockMappedMask) << 1;
|
||||
mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged.
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte);
|
||||
|
||||
mask = ulong.MaxValue;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the given address mapping from the page table.
|
||||
/// </summary>
|
||||
/// <param name="va">Virtual memory address</param>
|
||||
/// <param name="size">Size to be unmapped</param>
|
||||
private void RemoveMapping(ulong va, ulong size)
|
||||
{
|
||||
int pages = GetPagesCount(va, size, out _);
|
||||
ulong pageStart = va >> PageBits;
|
||||
ulong pageEnd = pageStart + (ulong)pages;
|
||||
|
||||
GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
|
||||
|
||||
startMask = ~startMask;
|
||||
endMask = ~endMask;
|
||||
|
||||
ulong mask = startMask;
|
||||
|
||||
while (pageIndex <= pageEndIndex)
|
||||
{
|
||||
if (pageIndex == pageEndIndex)
|
||||
{
|
||||
mask |= endMask;
|
||||
}
|
||||
|
||||
ref ulong pageRef = ref _pageBitmap[pageIndex++];
|
||||
ulong pte;
|
||||
|
||||
do
|
||||
{
|
||||
pte = Volatile.Read(ref pageRef);
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte);
|
||||
|
||||
mask = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes of resources used by the memory manager.
|
||||
/// </summary>
|
||||
protected override void Destroy()
|
||||
{
|
||||
_addressSpace.Dispose();
|
||||
_memoryEh.Dispose();
|
||||
}
|
||||
|
||||
private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message);
|
||||
}
|
||||
}
|
12
src/Ryujinx.Cpu/LoadState.cs
Normal file
12
src/Ryujinx.Cpu/LoadState.cs
Normal file
|
@ -0,0 +1,12 @@
|
|||
namespace Ryujinx.Cpu
|
||||
{
|
||||
/// <summary>
|
||||
/// Load state.
|
||||
/// </summary>
|
||||
public enum LoadState
|
||||
{
|
||||
Unloaded,
|
||||
Loading,
|
||||
Loaded
|
||||
}
|
||||
}
|
62
src/Ryujinx.Cpu/MemoryEhMeilleure.cs
Normal file
62
src/Ryujinx.Cpu/MemoryEhMeilleure.cs
Normal file
|
@ -0,0 +1,62 @@
|
|||
using ARMeilleure.Signal;
|
||||
using Ryujinx.Memory;
|
||||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
public class MemoryEhMeilleure : IDisposable
|
||||
{
|
||||
private delegate bool TrackingEventDelegate(ulong address, ulong size, bool write);
|
||||
|
||||
private readonly MemoryTracking _tracking;
|
||||
private readonly TrackingEventDelegate _trackingEvent;
|
||||
|
||||
private readonly ulong _baseAddress;
|
||||
private readonly ulong _mirrorAddress;
|
||||
|
||||
public MemoryEhMeilleure(MemoryBlock addressSpace, MemoryBlock addressSpaceMirror, MemoryTracking tracking)
|
||||
{
|
||||
_tracking = tracking;
|
||||
|
||||
_baseAddress = (ulong)addressSpace.Pointer;
|
||||
ulong endAddress = _baseAddress + addressSpace.Size;
|
||||
|
||||
_trackingEvent = new TrackingEventDelegate(tracking.VirtualMemoryEvent);
|
||||
bool added = NativeSignalHandler.AddTrackedRegion((nuint)_baseAddress, (nuint)endAddress, Marshal.GetFunctionPointerForDelegate(_trackingEvent));
|
||||
|
||||
if (!added)
|
||||
{
|
||||
throw new InvalidOperationException("Number of allowed tracked regions exceeded.");
|
||||
}
|
||||
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
// Add a tracking event with no signal handler for the mirror on Windows.
|
||||
// The native handler has its own code to check for the partial overlap race when regions are protected by accident,
|
||||
// and when there is no signal handler present.
|
||||
|
||||
_mirrorAddress = (ulong)addressSpaceMirror.Pointer;
|
||||
ulong endAddressMirror = _mirrorAddress + addressSpace.Size;
|
||||
|
||||
bool addedMirror = NativeSignalHandler.AddTrackedRegion((nuint)_mirrorAddress, (nuint)endAddressMirror, IntPtr.Zero);
|
||||
|
||||
if (!addedMirror)
|
||||
{
|
||||
throw new InvalidOperationException("Number of allowed tracked regions exceeded.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
NativeSignalHandler.RemoveTrackedRegion((nuint)_baseAddress);
|
||||
|
||||
if (_mirrorAddress != 0)
|
||||
{
|
||||
NativeSignalHandler.RemoveTrackedRegion((nuint)_mirrorAddress);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
63
src/Ryujinx.Cpu/MemoryHelper.cs
Normal file
63
src/Ryujinx.Cpu/MemoryHelper.cs
Normal file
|
@ -0,0 +1,63 @@
|
|||
using Microsoft.IO;
|
||||
using Ryujinx.Common.Memory;
|
||||
using Ryujinx.Memory;
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
public static class MemoryHelper
|
||||
{
|
||||
public static void FillWithZeros(IVirtualMemoryManager memory, ulong position, int size)
|
||||
{
|
||||
int size8 = size & ~(8 - 1);
|
||||
|
||||
for (int offs = 0; offs < size8; offs += 8)
|
||||
{
|
||||
memory.Write<long>(position + (ulong)offs, 0);
|
||||
}
|
||||
|
||||
for (int offs = size8; offs < (size - size8); offs++)
|
||||
{
|
||||
memory.Write<byte>(position + (ulong)offs, 0);
|
||||
}
|
||||
}
|
||||
|
||||
public unsafe static T Read<T>(IVirtualMemoryManager memory, ulong position) where T : unmanaged
|
||||
{
|
||||
return MemoryMarshal.Cast<byte, T>(memory.GetSpan(position, Unsafe.SizeOf<T>()))[0];
|
||||
}
|
||||
|
||||
public unsafe static ulong Write<T>(IVirtualMemoryManager memory, ulong position, T value) where T : unmanaged
|
||||
{
|
||||
ReadOnlySpan<byte> data = MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateReadOnlySpan(ref value, 1));
|
||||
|
||||
memory.Write(position, data);
|
||||
|
||||
return (ulong)data.Length;
|
||||
}
|
||||
|
||||
public static string ReadAsciiString(IVirtualMemoryManager memory, ulong position, long maxSize = -1)
|
||||
{
|
||||
using (RecyclableMemoryStream ms = MemoryStreamManager.Shared.GetStream())
|
||||
{
|
||||
for (long offs = 0; offs < maxSize || maxSize == -1; offs++)
|
||||
{
|
||||
byte value = memory.Read<byte>(position + (ulong)offs);
|
||||
|
||||
if (value == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
ms.WriteByte(value);
|
||||
}
|
||||
|
||||
return Encoding.ASCII.GetString(ms.GetReadOnlySequence());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
32
src/Ryujinx.Cpu/MemoryManagerBase.cs
Normal file
32
src/Ryujinx.Cpu/MemoryManagerBase.cs
Normal file
|
@ -0,0 +1,32 @@
|
|||
using Ryujinx.Memory;
|
||||
using System.Diagnostics;
|
||||
using System.Threading;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
public abstract class MemoryManagerBase : IRefCounted
|
||||
{
|
||||
private int _referenceCount;
|
||||
|
||||
public void IncrementReferenceCount()
|
||||
{
|
||||
int newRefCount = Interlocked.Increment(ref _referenceCount);
|
||||
|
||||
Debug.Assert(newRefCount >= 1);
|
||||
}
|
||||
|
||||
public void DecrementReferenceCount()
|
||||
{
|
||||
int newRefCount = Interlocked.Decrement(ref _referenceCount);
|
||||
|
||||
Debug.Assert(newRefCount >= 0);
|
||||
|
||||
if (newRefCount == 0)
|
||||
{
|
||||
Destroy();
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract void Destroy();
|
||||
}
|
||||
}
|
41
src/Ryujinx.Cpu/PrivateMemoryAllocation.cs
Normal file
41
src/Ryujinx.Cpu/PrivateMemoryAllocation.cs
Normal file
|
@ -0,0 +1,41 @@
|
|||
using Ryujinx.Memory;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
struct PrivateMemoryAllocation : IDisposable
|
||||
{
|
||||
private readonly PrivateMemoryAllocator _owner;
|
||||
private readonly PrivateMemoryAllocator.Block _block;
|
||||
|
||||
public bool IsValid => _owner != null;
|
||||
public MemoryBlock Memory => _block?.Memory;
|
||||
public ulong Offset { get; }
|
||||
public ulong Size { get; }
|
||||
|
||||
public PrivateMemoryAllocation(
|
||||
PrivateMemoryAllocator owner,
|
||||
PrivateMemoryAllocator.Block block,
|
||||
ulong offset,
|
||||
ulong size)
|
||||
{
|
||||
_owner = owner;
|
||||
_block = block;
|
||||
Offset = offset;
|
||||
Size = size;
|
||||
}
|
||||
|
||||
public (PrivateMemoryAllocation, PrivateMemoryAllocation) Split(ulong splitOffset)
|
||||
{
|
||||
PrivateMemoryAllocation left = new PrivateMemoryAllocation(_owner, _block, Offset, splitOffset);
|
||||
PrivateMemoryAllocation right = new PrivateMemoryAllocation(_owner, _block, Offset + splitOffset, Size - splitOffset);
|
||||
|
||||
return (left, right);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_owner.Free(_block, Offset, Size);
|
||||
}
|
||||
}
|
||||
}
|
268
src/Ryujinx.Cpu/PrivateMemoryAllocator.cs
Normal file
268
src/Ryujinx.Cpu/PrivateMemoryAllocator.cs
Normal file
|
@ -0,0 +1,268 @@
|
|||
using Ryujinx.Common;
|
||||
using Ryujinx.Memory;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
class PrivateMemoryAllocator : PrivateMemoryAllocatorImpl<PrivateMemoryAllocator.Block>
|
||||
{
|
||||
public const ulong InvalidOffset = ulong.MaxValue;
|
||||
|
||||
public class Block : IComparable<Block>
|
||||
{
|
||||
public MemoryBlock Memory { get; private set; }
|
||||
public ulong Size { get; }
|
||||
|
||||
private struct Range : IComparable<Range>
|
||||
{
|
||||
public ulong Offset { get; }
|
||||
public ulong Size { get; }
|
||||
|
||||
public Range(ulong offset, ulong size)
|
||||
{
|
||||
Offset = offset;
|
||||
Size = size;
|
||||
}
|
||||
|
||||
public int CompareTo(Range other)
|
||||
{
|
||||
return Offset.CompareTo(other.Offset);
|
||||
}
|
||||
}
|
||||
|
||||
private readonly List<Range> _freeRanges;
|
||||
|
||||
public Block(MemoryBlock memory, ulong size)
|
||||
{
|
||||
Memory = memory;
|
||||
Size = size;
|
||||
_freeRanges = new List<Range>
|
||||
{
|
||||
new Range(0, size)
|
||||
};
|
||||
}
|
||||
|
||||
public ulong Allocate(ulong size, ulong alignment)
|
||||
{
|
||||
for (int i = 0; i < _freeRanges.Count; i++)
|
||||
{
|
||||
var range = _freeRanges[i];
|
||||
|
||||
ulong alignedOffset = BitUtils.AlignUp(range.Offset, alignment);
|
||||
ulong sizeDelta = alignedOffset - range.Offset;
|
||||
ulong usableSize = range.Size - sizeDelta;
|
||||
|
||||
if (sizeDelta < range.Size && usableSize >= size)
|
||||
{
|
||||
_freeRanges.RemoveAt(i);
|
||||
|
||||
if (sizeDelta != 0)
|
||||
{
|
||||
InsertFreeRange(range.Offset, sizeDelta);
|
||||
}
|
||||
|
||||
ulong endOffset = range.Offset + range.Size;
|
||||
ulong remainingSize = endOffset - (alignedOffset + size);
|
||||
if (remainingSize != 0)
|
||||
{
|
||||
InsertFreeRange(endOffset - remainingSize, remainingSize);
|
||||
}
|
||||
|
||||
return alignedOffset;
|
||||
}
|
||||
}
|
||||
|
||||
return InvalidOffset;
|
||||
}
|
||||
|
||||
public void Free(ulong offset, ulong size)
|
||||
{
|
||||
InsertFreeRangeComingled(offset, size);
|
||||
}
|
||||
|
||||
private void InsertFreeRange(ulong offset, ulong size)
|
||||
{
|
||||
var range = new Range(offset, size);
|
||||
int index = _freeRanges.BinarySearch(range);
|
||||
if (index < 0)
|
||||
{
|
||||
index = ~index;
|
||||
}
|
||||
|
||||
_freeRanges.Insert(index, range);
|
||||
}
|
||||
|
||||
private void InsertFreeRangeComingled(ulong offset, ulong size)
|
||||
{
|
||||
ulong endOffset = offset + size;
|
||||
var range = new Range(offset, size);
|
||||
int index = _freeRanges.BinarySearch(range);
|
||||
if (index < 0)
|
||||
{
|
||||
index = ~index;
|
||||
}
|
||||
|
||||
if (index < _freeRanges.Count && _freeRanges[index].Offset == endOffset)
|
||||
{
|
||||
endOffset = _freeRanges[index].Offset + _freeRanges[index].Size;
|
||||
_freeRanges.RemoveAt(index);
|
||||
}
|
||||
|
||||
if (index > 0 && _freeRanges[index - 1].Offset + _freeRanges[index - 1].Size == offset)
|
||||
{
|
||||
offset = _freeRanges[index - 1].Offset;
|
||||
_freeRanges.RemoveAt(--index);
|
||||
}
|
||||
|
||||
range = new Range(offset, endOffset - offset);
|
||||
|
||||
_freeRanges.Insert(index, range);
|
||||
}
|
||||
|
||||
public bool IsTotallyFree()
|
||||
{
|
||||
if (_freeRanges.Count == 1 && _freeRanges[0].Size == Size)
|
||||
{
|
||||
Debug.Assert(_freeRanges[0].Offset == 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public int CompareTo(Block other)
|
||||
{
|
||||
return Size.CompareTo(other.Size);
|
||||
}
|
||||
|
||||
public virtual void Destroy()
|
||||
{
|
||||
Memory.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
public PrivateMemoryAllocator(int blockAlignment, MemoryAllocationFlags allocationFlags) : base(blockAlignment, allocationFlags)
|
||||
{
|
||||
}
|
||||
|
||||
public PrivateMemoryAllocation Allocate(ulong size, ulong alignment)
|
||||
{
|
||||
var allocation = Allocate(size, alignment, CreateBlock);
|
||||
|
||||
return new PrivateMemoryAllocation(this, allocation.Block, allocation.Offset, allocation.Size);
|
||||
}
|
||||
|
||||
private Block CreateBlock(MemoryBlock memory, ulong size)
|
||||
{
|
||||
return new Block(memory, size);
|
||||
}
|
||||
}
|
||||
|
||||
class PrivateMemoryAllocatorImpl<T> : IDisposable where T : PrivateMemoryAllocator.Block
|
||||
{
|
||||
private const ulong InvalidOffset = ulong.MaxValue;
|
||||
|
||||
public struct Allocation
|
||||
{
|
||||
public T Block { get; }
|
||||
public ulong Offset { get; }
|
||||
public ulong Size { get; }
|
||||
|
||||
public Allocation(T block, ulong offset, ulong size)
|
||||
{
|
||||
Block = block;
|
||||
Offset = offset;
|
||||
Size = size;
|
||||
}
|
||||
}
|
||||
|
||||
private readonly List<T> _blocks;
|
||||
|
||||
private readonly int _blockAlignment;
|
||||
private readonly MemoryAllocationFlags _allocationFlags;
|
||||
|
||||
public PrivateMemoryAllocatorImpl(int blockAlignment, MemoryAllocationFlags allocationFlags)
|
||||
{
|
||||
_blocks = new List<T>();
|
||||
_blockAlignment = blockAlignment;
|
||||
_allocationFlags = allocationFlags;
|
||||
}
|
||||
|
||||
protected Allocation Allocate(ulong size, ulong alignment, Func<MemoryBlock, ulong, T> createBlock)
|
||||
{
|
||||
// Ensure we have a sane alignment value.
|
||||
if ((ulong)(int)alignment != alignment || (int)alignment <= 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(alignment), $"Invalid alignment 0x{alignment:X}.");
|
||||
}
|
||||
|
||||
for (int i = 0; i < _blocks.Count; i++)
|
||||
{
|
||||
var block = _blocks[i];
|
||||
|
||||
if (block.Size >= size)
|
||||
{
|
||||
ulong offset = block.Allocate(size, alignment);
|
||||
if (offset != InvalidOffset)
|
||||
{
|
||||
return new Allocation(block, offset, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ulong blockAlignedSize = BitUtils.AlignUp(size, (ulong)_blockAlignment);
|
||||
|
||||
var memory = new MemoryBlock(blockAlignedSize, _allocationFlags);
|
||||
var newBlock = createBlock(memory, blockAlignedSize);
|
||||
|
||||
InsertBlock(newBlock);
|
||||
|
||||
ulong newBlockOffset = newBlock.Allocate(size, alignment);
|
||||
Debug.Assert(newBlockOffset != InvalidOffset);
|
||||
|
||||
return new Allocation(newBlock, newBlockOffset, size);
|
||||
}
|
||||
|
||||
public void Free(PrivateMemoryAllocator.Block block, ulong offset, ulong size)
|
||||
{
|
||||
block.Free(offset, size);
|
||||
|
||||
if (block.IsTotallyFree())
|
||||
{
|
||||
for (int i = 0; i < _blocks.Count; i++)
|
||||
{
|
||||
if (_blocks[i] == block)
|
||||
{
|
||||
_blocks.RemoveAt(i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
block.Destroy();
|
||||
}
|
||||
}
|
||||
|
||||
private void InsertBlock(T block)
|
||||
{
|
||||
int index = _blocks.BinarySearch(block);
|
||||
if (index < 0)
|
||||
{
|
||||
index = ~index;
|
||||
}
|
||||
|
||||
_blocks.Insert(index, block);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
for (int i = 0; i < _blocks.Count; i++)
|
||||
{
|
||||
_blocks[i].Destroy();
|
||||
}
|
||||
|
||||
_blocks.Clear();
|
||||
}
|
||||
}
|
||||
}
|
13
src/Ryujinx.Cpu/Ryujinx.Cpu.csproj
Normal file
13
src/Ryujinx.Cpu/Ryujinx.Cpu.csproj
Normal file
|
@ -0,0 +1,13 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net7.0</TargetFramework>
|
||||
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\ARMeilleure\ARMeilleure.csproj" />
|
||||
<ProjectReference Include="..\Ryujinx.Memory\Ryujinx.Memory.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
45
src/Ryujinx.Cpu/TickSource.cs
Normal file
45
src/Ryujinx.Cpu/TickSource.cs
Normal file
|
@ -0,0 +1,45 @@
|
|||
using System;
|
||||
using System.Diagnostics;
|
||||
|
||||
namespace Ryujinx.Cpu
|
||||
{
|
||||
public class TickSource : ITickSource
|
||||
{
|
||||
private static Stopwatch _tickCounter;
|
||||
|
||||
private static double _hostTickFreq;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ulong Frequency { get; }
|
||||
|
||||
/// <inheritdoc/>
|
||||
public ulong Counter => (ulong)(ElapsedSeconds * Frequency);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public TimeSpan ElapsedTime => _tickCounter.Elapsed;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public double ElapsedSeconds => _tickCounter.ElapsedTicks * _hostTickFreq;
|
||||
|
||||
public TickSource(ulong frequency)
|
||||
{
|
||||
Frequency = frequency;
|
||||
_hostTickFreq = 1.0 / Stopwatch.Frequency;
|
||||
|
||||
_tickCounter = new Stopwatch();
|
||||
_tickCounter.Start();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Suspend()
|
||||
{
|
||||
_tickCounter.Stop();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Resume()
|
||||
{
|
||||
_tickCounter.Start();
|
||||
}
|
||||
}
|
||||
}
|
28
src/Ryujinx.Cpu/Tracking/CpuMultiRegionHandle.cs
Normal file
28
src/Ryujinx.Cpu/Tracking/CpuMultiRegionHandle.cs
Normal file
|
@ -0,0 +1,28 @@
|
|||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace Ryujinx.Cpu.Tracking
|
||||
{
|
||||
public class CpuMultiRegionHandle : IMultiRegionHandle
|
||||
{
|
||||
private readonly MultiRegionHandle _impl;
|
||||
|
||||
public bool Dirty => _impl.Dirty;
|
||||
|
||||
internal CpuMultiRegionHandle(MultiRegionHandle impl)
|
||||
{
|
||||
_impl = impl;
|
||||
}
|
||||
|
||||
public void Dispose() => _impl.Dispose();
|
||||
public void ForceDirty(ulong address, ulong size) => _impl.ForceDirty(address, size);
|
||||
public IEnumerable<IRegionHandle> GetHandles() => _impl.GetHandles();
|
||||
public void QueryModified(Action<ulong, ulong> modifiedAction) => _impl.QueryModified(modifiedAction);
|
||||
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction) => _impl.QueryModified(address, size, modifiedAction);
|
||||
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber) => _impl.QueryModified(address, size, modifiedAction, sequenceNumber);
|
||||
public void RegisterAction(ulong address, ulong size, RegionSignal action) => _impl.RegisterAction(address, size, action);
|
||||
public void RegisterPreciseAction(ulong address, ulong size, PreciseRegionSignal action) => _impl.RegisterPreciseAction(address, size, action);
|
||||
public void SignalWrite() => _impl.SignalWrite();
|
||||
}
|
||||
}
|
37
src/Ryujinx.Cpu/Tracking/CpuRegionHandle.cs
Normal file
37
src/Ryujinx.Cpu/Tracking/CpuRegionHandle.cs
Normal file
|
@ -0,0 +1,37 @@
|
|||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.Tracking
|
||||
{
|
||||
public class CpuRegionHandle : IRegionHandle
|
||||
{
|
||||
private readonly RegionHandle _impl;
|
||||
|
||||
public bool Dirty => _impl.Dirty;
|
||||
public bool Unmapped => _impl.Unmapped;
|
||||
public ulong Address => _impl.Address;
|
||||
public ulong Size => _impl.Size;
|
||||
public ulong EndAddress => _impl.EndAddress;
|
||||
|
||||
internal CpuRegionHandle(RegionHandle impl)
|
||||
{
|
||||
_impl = impl;
|
||||
}
|
||||
|
||||
public void Dispose() => _impl.Dispose();
|
||||
public bool DirtyOrVolatile() => _impl.DirtyOrVolatile();
|
||||
public void ForceDirty() => _impl.ForceDirty();
|
||||
public IRegionHandle GetHandle() => _impl;
|
||||
public void RegisterAction(RegionSignal action) => _impl.RegisterAction(action);
|
||||
public void RegisterPreciseAction(PreciseRegionSignal action) => _impl.RegisterPreciseAction(action);
|
||||
public void RegisterDirtyEvent(Action action) => _impl.RegisterDirtyEvent(action);
|
||||
public void Reprotect(bool asDirty = false) => _impl.Reprotect(asDirty);
|
||||
|
||||
public bool OverlapsWith(ulong address, ulong size) => _impl.OverlapsWith(address, size);
|
||||
|
||||
public bool RangeEquals(CpuRegionHandle other)
|
||||
{
|
||||
return _impl.RealAddress == other._impl.RealAddress && _impl.RealSize == other._impl.RealSize;
|
||||
}
|
||||
}
|
||||
}
|
26
src/Ryujinx.Cpu/Tracking/CpuSmartMultiRegionHandle.cs
Normal file
26
src/Ryujinx.Cpu/Tracking/CpuSmartMultiRegionHandle.cs
Normal file
|
@ -0,0 +1,26 @@
|
|||
using Ryujinx.Memory.Tracking;
|
||||
using System;
|
||||
|
||||
namespace Ryujinx.Cpu.Tracking
|
||||
{
|
||||
public class CpuSmartMultiRegionHandle : IMultiRegionHandle
|
||||
{
|
||||
private readonly SmartMultiRegionHandle _impl;
|
||||
|
||||
public bool Dirty => _impl.Dirty;
|
||||
|
||||
internal CpuSmartMultiRegionHandle(SmartMultiRegionHandle impl)
|
||||
{
|
||||
_impl = impl;
|
||||
}
|
||||
|
||||
public void Dispose() => _impl.Dispose();
|
||||
public void ForceDirty(ulong address, ulong size) => _impl.ForceDirty(address, size);
|
||||
public void RegisterAction(RegionSignal action) => _impl.RegisterAction(action);
|
||||
public void RegisterPreciseAction(PreciseRegionSignal action) => _impl.RegisterPreciseAction(action);
|
||||
public void QueryModified(Action<ulong, ulong> modifiedAction) => _impl.QueryModified(modifiedAction);
|
||||
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction) => _impl.QueryModified(address, size, modifiedAction);
|
||||
public void QueryModified(ulong address, ulong size, Action<ulong, ulong> modifiedAction, int sequenceNumber) => _impl.QueryModified(address, size, modifiedAction, sequenceNumber);
|
||||
public void SignalWrite() => _impl.SignalWrite();
|
||||
}
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue