Mirrors/Ryujinx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement a new physical memory manager and replace DeviceMemory (#856)

Browse source 
* Implement a new physical memory manager and replace DeviceMemory

* Proper generic constraints

* Fix debug build

* Add memory tests

* New CPU memory manager and general code cleanup

* Remove host memory management from CPU project, use Ryujinx.Memory instead

* Fix tests

* Document exceptions on MemoryBlock

* Fix leak on unix memory allocation

* Proper disposal of some objects on tests

* Fix JitCache not being set as initialized

* GetRef without checks for 8-bits and 16-bits CAS

* Add MemoryBlock destructor

* Throw in separate method to improve codegen

* Address PR feedback

* QueryModified improvements

* Fix memory write tracking not marking all pages as modified in some cases

* Simplify MarkRegionAsModified

* Remove XML doc for ghost param

* Add back optimization to avoid useless buffer updates

* Add Ryujinx.Cpu project, move MemoryManager there and remove MemoryBlockWrapper

* Some nits

* Do not perform address translation when size is 0

* Address PR feedback and format NativeInterface class

* Remove ghost parameter description

* Update Ryujinx.Cpu to .NET Core 3.1

* Address PR feedback

* Fix build

* Return a well defined value for GetPhysicalAddress with invalid VA, and do not return unmapped ranges as modified

* Typo
This commit is contained in:
gdkchan 2020-05-03 19:54:50 -03:00 • committed by GitHub
parent 1758424208
commit f77694e4f7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
126 changed files with 2176 additions and 2092 deletions
Download patch file Download diff file Expand all files Collapse all files

53
ARMeilleure/CodeGen/X86/HardwareCapabilities.cs
View file

@ -1,52 +1,23 @@
using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.Translation;
using System.Runtime.Intrinsics.X86;
namespace ARMeilleure.CodeGen.X86
{
static class HardwareCapabilities
{
private delegate ulong GetFeatureInfo();
private static ulong _featureInfo;
public static bool SupportsSse3 => (_featureInfo & (1UL << 0)) != 0;
public static bool SupportsPclmulqdq => (_featureInfo & (1UL << 1)) != 0;
public static bool SupportsSsse3 => (_featureInfo & (1UL << 9)) != 0;
public static bool SupportsFma => (_featureInfo & (1UL << 12)) != 0;
public static bool SupportsCx16 => (_featureInfo & (1UL << 13)) != 0;
public static bool SupportsSse41 => (_featureInfo & (1UL << 19)) != 0;
public static bool SupportsSse42 => (_featureInfo & (1UL << 20)) != 0;
public static bool SupportsPopcnt => (_featureInfo & (1UL << 23)) != 0;
public static bool SupportsAesni => (_featureInfo & (1UL << 25)) != 0;
public static bool SupportsAvx => (_featureInfo & (1UL << 28)) != 0;
public static bool SupportsF16c => (_featureInfo & (1UL << 29)) != 0;
public static bool SupportsSse => (_featureInfo & (1UL << 32 + 25)) != 0;
public static bool SupportsSse2 => (_featureInfo & (1UL << 32 + 26)) != 0;
public static bool SupportsSse => Sse.IsSupported;
public static bool SupportsSse2 => Sse2.IsSupported;
public static bool SupportsSse3 => Sse3.IsSupported;
public static bool SupportsSsse3 => Ssse3.IsSupported;
public static bool SupportsSse41 => Sse41.IsSupported;
public static bool SupportsSse42 => Sse42.IsSupported;
public static bool SupportsPclmulqdq => Pclmulqdq.IsSupported;
public static bool SupportsFma => Fma.IsSupported;
public static bool SupportsPopcnt => Popcnt.IsSupported;
public static bool SupportsAesni => Aes.IsSupported;
public static bool SupportsAvx => Avx.IsSupported;
public static bool ForceLegacySse { get; set; }
public static bool SupportsVexEncoding => SupportsAvx && !ForceLegacySse;
static HardwareCapabilities()
{
EmitterContext context = new EmitterContext();
Operand featureInfo = context.CpuId();
context.Return(featureInfo);
ControlFlowGraph cfg = context.GetControlFlowGraph();
OperandType[] argTypes = new OperandType[0];
GetFeatureInfo getFeatureInfo = Compiler.Compile<GetFeatureInfo>(
cfg,
argTypes,
OperandType.I64,
CompilerOptions.HighCq);
_featureInfo = getFeatureInfo();
}
}
}

18
ARMeilleure/Decoders/Decoder.cs
View file

@ -17,7 +17,7 @@ namespace ARMeilleure.Decoders
// For lower code quality translation, we set a lower limit since we're blocking execution.
private const int MaxInstsPerFunctionLowCq = 500;
public static Block[] DecodeBasicBlock(MemoryManager memory, ulong address, ExecutionMode mode)
public static Block[] DecodeBasicBlock(IMemoryManager memory, ulong address, ExecutionMode mode)
{
Block block = new Block(address);
@ -26,7 +26,7 @@ namespace ARMeilleure.Decoders
return new Block[] { block };
}
public static Block[] DecodeFunction(MemoryManager memory, ulong address, ExecutionMode mode, bool highCq)
public static Block[] DecodeFunction(IMemoryManager memory, ulong address, ExecutionMode mode, bool highCq)
{
List<Block> blocks = new List<Block>();
@ -42,7 +42,7 @@ namespace ARMeilleure.Decoders
{
if (!visited.TryGetValue(blkAddress, out Block block))
{
if (opsCount > instructionLimit || !memory.IsMapped((long)blkAddress))
if (opsCount > instructionLimit || !memory.IsMapped(blkAddress))
{
return null;
}
@ -176,10 +176,10 @@ namespace ARMeilleure.Decoders
}
private static void FillBlock(
MemoryManager memory,
ExecutionMode mode,
Block block,
ulong limitAddress)
IMemoryManager memory,
ExecutionMode mode,
Block block,
ulong limitAddress)
{
ulong address = block.Address;
@ -302,9 +302,9 @@ namespace ARMeilleure.Decoders
opCode.Instruction.Name == InstName.Und;
}
public static OpCode DecodeOpCode(MemoryManager memory, ulong address, ExecutionMode mode)
public static OpCode DecodeOpCode(IMemoryManager memory, ulong address, ExecutionMode mode)
{
int opCode = memory.ReadInt32((long)address);
int opCode = memory.Read<int>(address);
InstDescriptor inst;

32
ARMeilleure/Instructions/InstEmitMemoryHelper.cs
View file

@ -1,6 +1,5 @@
using ARMeilleure.Decoders;
using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.Memory;
using ARMeilleure.Translation;
using System;
@ -11,6 +10,9 @@ namespace ARMeilleure.Instructions
{
static class InstEmitMemoryHelper
{
private const int PageBits = 12;
private const int PageMask = (1 << PageBits) - 1;
private enum Extension
{
Zx,
@ -318,28 +320,32 @@ namespace ARMeilleure.Instructions
private static Operand EmitAddressCheck(ArmEmitterContext context, Operand address, int size)
{
long addressCheckMask = ~(context.Memory.AddressSpaceSize - 1);
ulong addressCheckMask = ~((1UL << context.Memory.AddressSpaceBits) - 1);
addressCheckMask |= (1u << size) - 1;
return context.BitwiseAnd(address, Const(address.Type, addressCheckMask));
return context.BitwiseAnd(address, Const(address.Type, (long)addressCheckMask));
}
private static Operand EmitPtPointerLoad(ArmEmitterContext context, Operand address, Operand lblFallbackPath)
private static Operand EmitPtPointerLoad(ArmEmitterContext context, Operand address, Operand lblSlowPath)
{
Operand pte = Const(context.Memory.PageTable.ToInt64());
int ptLevelBits = context.Memory.AddressSpaceBits - 12; // 12 = Number of page bits.
int ptLevelSize = 1 << ptLevelBits;
int ptLevelMask = ptLevelSize - 1;
int bit = MemoryManager.PageBits;
Operand pte = Const(context.Memory.PageTablePointer.ToInt64());
int bit = PageBits;
do
{
Operand addrPart = context.ShiftRightUI(address, Const(bit));
bit += context.Memory.PtLevelBits;
bit += ptLevelBits;
if (bit < context.Memory.AddressSpaceBits)
{
addrPart = context.BitwiseAnd(addrPart, Const(addrPart.Type, context.Memory.PtLevelMask));
addrPart = context.BitwiseAnd(addrPart, Const(addrPart.Type, ptLevelMask));
}
Operand pteOffset = context.ShiftLeft(addrPart, Const(3));
@ -355,20 +361,16 @@ namespace ARMeilleure.Instructions
}
while (bit < context.Memory.AddressSpaceBits);
Operand hasFlagSet = context.BitwiseAnd(pte, Const((long)MemoryManager.PteFlagsMask));
context.BranchIfTrue(lblSlowPath, context.ICompareLess(pte, Const(0L)));
context.BranchIfTrue(lblFallbackPath, hasFlagSet);
Operand pageOffset = context.BitwiseAnd(address, Const(address.Type, MemoryManager.PageMask));
Operand pageOffset = context.BitwiseAnd(address, Const(address.Type, PageMask));
if (pageOffset.Type == OperandType.I32)
{
pageOffset = context.ZeroExtend32(OperandType.I64, pageOffset);
}
Operand physAddr = context.Add(pte, pageOffset);
return physAddr;
return context.Add(pte, pageOffset);
}
private static void EmitReadIntFallback(ArmEmitterContext context, Operand address, int rt, int size)

143
ARMeilleure/Instructions/NativeInterface.cs
View file

@ -3,6 +3,7 @@ using ARMeilleure.State;
using ARMeilleure.Translation;
using System;
using System.Runtime.InteropServices;
using System.Threading;
namespace ARMeilleure.Instructions
{
@ -12,18 +13,18 @@ namespace ARMeilleure.Instructions
private class ThreadContext
{
public ExecutionContext Context { get; }
public MemoryManager Memory { get; }
public Translator Translator { get; }
public State.ExecutionContext Context { get; }
public IMemoryManager Memory { get; }
public Translator Translator { get; }
public ulong ExclusiveAddress { get; set; }
public ulong ExclusiveValueLow { get; set; }
public ulong ExclusiveAddress { get; set; }
public ulong ExclusiveValueLow { get; set; }
public ulong ExclusiveValueHigh { get; set; }
public ThreadContext(ExecutionContext context, MemoryManager memory, Translator translator)
public ThreadContext(State.ExecutionContext context, IMemoryManager memory, Translator translator)
{
Context = context;
Memory = memory;
Context = context;
Memory = memory;
Translator = translator;
ExclusiveAddress = ulong.MaxValue;
@ -33,7 +34,7 @@ namespace ARMeilleure.Instructions
[ThreadStatic]
private static ThreadContext _context;
public static void RegisterThread(ExecutionContext context, MemoryManager memory, Translator translator)
public static void RegisterThread(State.ExecutionContext context, IMemoryManager memory, Translator translator)
{
_context = new ThreadContext(context, memory, translator);
}
@ -70,7 +71,7 @@ namespace ARMeilleure.Instructions
Statistics.ResumeTimer();
}
#region "System registers"
#region "System registers"
public static ulong GetCtrEl0()
{
return (ulong)GetContext().CtrEl0;
@ -93,7 +94,8 @@ namespace ARMeilleure.Instructions
public static uint GetFpscr()
{
ExecutionContext context = GetContext();
var context = GetContext();
uint result = (uint)(context.Fpsr & FPSR.A32Mask) | (uint)(context.Fpcr & FPCR.A32Mask);
result |= context.GetFPstateFlag(FPState.NFlag) ? (1u << 31) : 0;
@ -146,7 +148,7 @@ namespace ARMeilleure.Instructions
public static void SetFpscr(uint value)
{
ExecutionContext context = GetContext();
var context = GetContext();
context.SetFPstateFlag(FPState.NFlag, (value & (1u << 31)) != 0);
context.SetFPstateFlag(FPState.ZFlag, (value & (1u << 30)) != 0);
@ -171,37 +173,37 @@ namespace ARMeilleure.Instructions
#region "Read"
public static byte ReadByte(ulong address)
{
return GetMemoryManager().ReadByte((long)address);
return GetMemoryManager().Read<byte>(address);
}
public static ushort ReadUInt16(ulong address)
{
return GetMemoryManager().ReadUInt16((long)address);
return GetMemoryManager().Read<ushort>(address);
}
public static uint ReadUInt32(ulong address)
{
return GetMemoryManager().ReadUInt32((long)address);
return GetMemoryManager().Read<uint>(address);
}
public static ulong ReadUInt64(ulong address)
{
return GetMemoryManager().ReadUInt64((long)address);
return GetMemoryManager().Read<ulong>(address);
}
public static V128 ReadVector128(ulong address)
{
return GetMemoryManager().ReadVector128((long)address);
return GetMemoryManager().Read<V128>(address);
}
#endregion
#endregion
#region "Read exclusive"
#region "Read exclusive"
public static byte ReadByteExclusive(ulong address)
{
byte value = _context.Memory.ReadByte((long)address);
byte value = _context.Memory.Read<byte>(address);
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value;
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value;
_context.ExclusiveValueHigh = 0;
return value;
@ -209,10 +211,10 @@ namespace ARMeilleure.Instructions
public static ushort ReadUInt16Exclusive(ulong address)
{
ushort value = _context.Memory.ReadUInt16((long)address);
ushort value = _context.Memory.Read<ushort>(address);
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value;
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value;
_context.ExclusiveValueHigh = 0;
return value;
@ -220,10 +222,10 @@ namespace ARMeilleure.Instructions
public static uint ReadUInt32Exclusive(ulong address)
{
uint value = _context.Memory.ReadUInt32((long)address);
uint value = _context.Memory.Read<uint>(address);
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value;
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value;
_context.ExclusiveValueHigh = 0;
return value;
@ -231,10 +233,10 @@ namespace ARMeilleure.Instructions
public static ulong ReadUInt64Exclusive(ulong address)
{
ulong value = _context.Memory.ReadUInt64((long)address);
ulong value = _context.Memory.Read<ulong>(address);
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value;
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value;
_context.ExclusiveValueHigh = 0;
return value;
@ -242,54 +244,60 @@ namespace ARMeilleure.Instructions
public static V128 ReadVector128Exclusive(ulong address)
{
V128 value = _context.Memory.AtomicLoadInt128((long)address);
V128 value = MemoryManagerPal.AtomicLoad128(ref _context.Memory.GetRef<V128>(address));
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value.Extract<ulong>(0);
_context.ExclusiveAddress = GetMaskedExclusiveAddress(address);
_context.ExclusiveValueLow = value.Extract<ulong>(0);
_context.ExclusiveValueHigh = value.Extract<ulong>(1);
return value;
}
#endregion
#endregion
#region "Write"
#region "Write"
public static void WriteByte(ulong address, byte value)
{
GetMemoryManager().WriteByte((long)address, value);
GetMemoryManager().Write(address, value);
}
public static void WriteUInt16(ulong address, ushort value)
{
GetMemoryManager().WriteUInt16((long)address, value);
GetMemoryManager().Write(address, value);
}
public static void WriteUInt32(ulong address, uint value)
{
GetMemoryManager().WriteUInt32((long)address, value);
GetMemoryManager().Write(address, value);
}
public static void WriteUInt64(ulong address, ulong value)
{
GetMemoryManager().WriteUInt64((long)address, value);
GetMemoryManager().Write(address, value);
}
public static void WriteVector128(ulong address, V128 value)
{
GetMemoryManager().WriteVector128((long)address, value);
GetMemoryManager().Write(address, value);
}
#endregion
#endregion
#region "Write exclusive"
#region "Write exclusive"
public static int WriteByteExclusive(ulong address, byte value)
{
bool success = _context.ExclusiveAddress == GetMaskedExclusiveAddress(address);
if (success)
{
success = _context.Memory.AtomicCompareExchangeByte(
(long)address,
(byte)_context.ExclusiveValueLow,
(byte)value);
ref int valueRef = ref _context.Memory.GetRefNoChecks<int>(address);
int currentValue = valueRef;
byte expected = (byte)_context.ExclusiveValueLow;
int expected32 = (currentValue & ~byte.MaxValue) | expected;
int desired32 = (currentValue & ~byte.MaxValue) | value;
success = Interlocked.CompareExchange(ref valueRef, desired32, expected32) == expected32;
if (success)
{
@ -306,10 +314,16 @@ namespace ARMeilleure.Instructions
if (success)
{
success = _context.Memory.AtomicCompareExchangeInt16(
(long)address,
(short)_context.ExclusiveValueLow,
(short)value);
ref int valueRef = ref _context.Memory.GetRefNoChecks<int>(address);
int currentValue = valueRef;
ushort expected = (ushort)_context.ExclusiveValueLow;
int expected32 = (currentValue & ~ushort.MaxValue) | expected;
int desired32 = (currentValue & ~ushort.MaxValue) | value;
success = Interlocked.CompareExchange(ref valueRef, desired32, expected32) == expected32;
if (success)
{
@ -326,10 +340,9 @@ namespace ARMeilleure.Instructions
if (success)
{
success = _context.Memory.AtomicCompareExchangeInt32(
(long)address,
(int)_context.ExclusiveValueLow,
(int)value);
ref int valueRef = ref _context.Memory.GetRef<int>(address);
success = Interlocked.CompareExchange(ref valueRef, (int)value, (int)_context.ExclusiveValueLow) == (int)_context.ExclusiveValueLow;
if (success)
{
@ -346,10 +359,9 @@ namespace ARMeilleure.Instructions
if (success)
{
success = _context.Memory.AtomicCompareExchangeInt64(
(long)address,
(long)_context.ExclusiveValueLow,
(long)value);
ref long valueRef = ref _context.Memory.GetRef<long>(address);
success = Interlocked.CompareExchange(ref valueRef, (long)value, (long)_context.ExclusiveValueLow) == (long)_context.ExclusiveValueLow;
if (success)
{
@ -368,7 +380,9 @@ namespace ARMeilleure.Instructions
{
V128 expected = new V128(_context.ExclusiveValueLow, _context.ExclusiveValueHigh);
success = _context.Memory.AtomicCompareExchangeInt128((long)address, expected, value);
ref V128 location = ref _context.Memory.GetRef<V128>(address);
success = MemoryManagerPal.CompareAndSwap128(ref location, expected, value) == expected;
if (success)
{
@ -378,7 +392,7 @@ namespace ARMeilleure.Instructions
return success ? 0 : 1;
}
#endregion
#endregion
private static ulong GetMaskedExclusiveAddress(ulong address)
{
@ -412,7 +426,8 @@ namespace ARMeilleure.Instructions
{
Statistics.PauseTimer();
ExecutionContext context = GetContext();
var context = GetContext();
context.CheckInterrupt();
Statistics.ResumeTimer();
@ -420,12 +435,12 @@ namespace ARMeilleure.Instructions
return context.Running;
}
public static ExecutionContext GetContext()
public static State.ExecutionContext GetContext()
{
return _context.Context;
}
public static MemoryManager GetMemoryManager()
public static IMemoryManager GetMemoryManager()
{
return _context.Memory;
}

8
ARMeilleure/Memory/IJitMemoryAllocator.cs Normal file
View file

@ -0,0 +1,8 @@
namespace ARMeilleure.Memory
{
public interface IJitMemoryAllocator
{
IJitMemoryBlock Allocate(ulong size);
IJitMemoryBlock Reserve(ulong size);
}
}

14
ARMeilleure/Memory/IJitMemoryBlock.cs Normal file
View file

@ -0,0 +1,14 @@
using System;
namespace ARMeilleure.Memory
{
public interface IJitMemoryBlock : IDisposable
{
IntPtr Pointer { get; }
bool Commit(ulong offset, ulong size);
void MapAsRx(ulong offset, ulong size);
void MapAsRwx(ulong offset, ulong size);
}
}

19
ARMeilleure/Memory/IMemoryManager.cs Normal file
View file

@ -0,0 +1,19 @@
using System;
namespace ARMeilleure.Memory
{
public interface IMemoryManager
{
int AddressSpaceBits { get; }
IntPtr PageTablePointer { get; }
T Read<T>(ulong va) where T : unmanaged;
void Write<T>(ulong va, T value) where T : unmanaged;
ref T GetRef<T>(ulong va) where T : unmanaged;
ref T GetRefNoChecks<T>(ulong va) where T : unmanaged;
bool IsMapped(ulong va);
}
}

71
ARMeilleure/Memory/MemoryHelper.cs
View file

@ -1,71 +0,0 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
namespace ARMeilleure.Memory
{
public static class MemoryHelper
{
public static void FillWithZeros(MemoryManager memory, long position, int size)
{
int size8 = size & ~(8 - 1);
for (int offs = 0; offs < size8; offs += 8)
{
memory.WriteInt64(position + offs, 0);
}
for (int offs = size8; offs < (size - size8); offs++)
{
memory.WriteByte(position + offs, 0);
}
}
public unsafe static T Read<T>(MemoryManager memory, long position) where T : struct
{
long size = Marshal.SizeOf<T>();
byte[] data = memory.ReadBytes(position, size);
fixed (byte* ptr = data)
{
return Marshal.PtrToStructure<T>((IntPtr)ptr);
}
}
public unsafe static void Write<T>(MemoryManager memory, long position, T value) where T : struct
{
long size = Marshal.SizeOf<T>();
byte[] data = new byte[size];
fixed (byte* ptr = data)
{
Marshal.StructureToPtr<T>(value, (IntPtr)ptr, false);
}
memory.WriteBytes(position, data);
}
public static string ReadAsciiString(MemoryManager memory, long position, long maxSize = -1)
{
using (MemoryStream ms = new MemoryStream())
{
for (long offs = 0; offs < maxSize || maxSize == -1; offs++)
{
byte value = (byte)memory.ReadByte(position + offs);
if (value == 0)
{
break;
}
ms.WriteByte(value);
}
return Encoding.ASCII.GetString(ms.ToArray());
}
}
}
}

128
ARMeilleure/Memory/MemoryManagement.cs
View file

@ -1,128 +0,0 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace ARMeilleure.Memory
{
public static class MemoryManagement
{
public static IntPtr Allocate(ulong size)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
IntPtr sizeNint = new IntPtr((long)size);
return MemoryManagementWindows.Allocate(sizeNint);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return MemoryManagementUnix.Allocate(size);
}
else
{
throw new PlatformNotSupportedException();
}
}
public static IntPtr AllocateWriteTracked(ulong size)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
IntPtr sizeNint = new IntPtr((long)size);
return MemoryManagementWindows.AllocateWriteTracked(sizeNint);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return MemoryManagementUnix.Allocate(size);
}
else
{
throw new PlatformNotSupportedException();
}
}
public static bool Commit(IntPtr address, ulong size)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
IntPtr sizeNint = new IntPtr((long)size);
return MemoryManagementWindows.Commit(address, sizeNint);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return MemoryManagementUnix.Commit(address, size);
}
else
{
throw new PlatformNotSupportedException();
}
}
public static void Reprotect(IntPtr address, ulong size, MemoryProtection permission)
{
bool result;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
IntPtr sizeNint = new IntPtr((long)size);
result = MemoryManagementWindows.Reprotect(address, sizeNint, permission);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
result = MemoryManagementUnix.Reprotect(address, size, permission);
}
else
{
throw new PlatformNotSupportedException();
}
if (!result)
{
throw new MemoryProtectionException(permission);
}
}
public static IntPtr Reserve(ulong size)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
IntPtr sizeNint = new IntPtr((long)size);
return MemoryManagementWindows.Reserve(sizeNint);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return MemoryManagementUnix.Reserve(size);
}
else
{
throw new PlatformNotSupportedException();
}
}
public static bool Free(IntPtr address)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return MemoryManagementWindows.Free(address);
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
return MemoryManagementUnix.Free(address);
}
else
{
throw new PlatformNotSupportedException();
}
}
}
}

94
ARMeilleure/Memory/MemoryManagementUnix.cs
View file

@ -1,94 +0,0 @@
using Mono.Unix.Native;
using System;
namespace ARMeilleure.Memory
{
static class MemoryManagementUnix
{
public static IntPtr Allocate(ulong size)
{
ulong pageSize = (ulong)Syscall.sysconf(SysconfName._SC_PAGESIZE);
const MmapProts prot = MmapProts.PROT_READ | MmapProts.PROT_WRITE;
const MmapFlags flags = MmapFlags.MAP_PRIVATE | MmapFlags.MAP_ANONYMOUS;
IntPtr ptr = Syscall.mmap(IntPtr.Zero, size + pageSize, prot, flags, -1, 0);
if (ptr == IntPtr.Zero)
{
throw new OutOfMemoryException();
}
unsafe
{
ptr = new IntPtr(ptr.ToInt64() + (long)pageSize);
*((ulong*)ptr - 1) = size;
}
return ptr;
}
public static bool Commit(IntPtr address, ulong size)
{
return Syscall.mprotect(address, size, MmapProts.PROT_READ | MmapProts.PROT_WRITE) == 0;
}
public static bool Reprotect(IntPtr address, ulong size, Memory.MemoryProtection protection)
{
MmapProts prot = GetProtection(protection);
return Syscall.mprotect(address, size, prot) == 0;
}
public static IntPtr Reserve(ulong size)
{
ulong pageSize = (ulong)Syscall.sysconf(SysconfName._SC_PAGESIZE);
const MmapProts prot = MmapProts.PROT_NONE;
const MmapFlags flags = MmapFlags.MAP_PRIVATE | MmapFlags.MAP_ANONYMOUS;
IntPtr ptr = Syscall.mmap(IntPtr.Zero, size + pageSize, prot, flags, -1, 0);
if (ptr == IntPtr.Zero)
{
throw new OutOfMemoryException();
}
return ptr;
}
private static MmapProts GetProtection(Memory.MemoryProtection protection)
{
switch (protection)
{
case Memory.MemoryProtection.None: return MmapProts.PROT_NONE;
case Memory.MemoryProtection.Read: return MmapProts.PROT_READ;
case Memory.MemoryProtection.ReadAndWrite: return MmapProts.PROT_READ | MmapProts.PROT_WRITE;
case Memory.MemoryProtection.ReadAndExecute: return MmapProts.PROT_READ | MmapProts.PROT_EXEC;
case Memory.MemoryProtection.ReadWriteExecute: return MmapProts.PROT_READ | MmapProts.PROT_WRITE | MmapProts.PROT_EXEC;
case Memory.MemoryProtection.Execute: return MmapProts.PROT_EXEC;
default: throw new ArgumentException($"Invalid permission \"{protection}\".");
}
}
public static bool Free(IntPtr address)
{
ulong pageSize = (ulong)Syscall.sysconf(SysconfName._SC_PAGESIZE);
ulong size;
unsafe
{
size = *((ulong*)address - 1);
address = new IntPtr(address.ToInt64() - (long)pageSize);
}
return Syscall.munmap(address, size + pageSize) == 0;
}
}
}

142
ARMeilleure/Memory/MemoryManagementWindows.cs
View file

@ -1,142 +0,0 @@
using System;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace ARMeilleure.Memory
{
static class MemoryManagementWindows
{
[Flags]
private enum AllocationType : uint
{
Commit = 0x1000,
Reserve = 0x2000,
Decommit = 0x4000,
Release = 0x8000,
Reset = 0x80000,
Physical = 0x400000,
TopDown = 0x100000,
WriteWatch = 0x200000,
LargePages = 0x20000000
}
[Flags]
private enum MemoryProtection : uint
{
NoAccess = 0x01,
ReadOnly = 0x02,
ReadWrite = 0x04,
WriteCopy = 0x08,
Execute = 0x10,
ExecuteRead = 0x20,
ExecuteReadWrite = 0x40,
ExecuteWriteCopy = 0x80,
GuardModifierflag = 0x100,
NoCacheModifierflag = 0x200,
WriteCombineModifierflag = 0x400
}
[DllImport("kernel32.dll")]
private static extern IntPtr VirtualAlloc(
IntPtr lpAddress,
IntPtr dwSize,
AllocationType flAllocationType,
MemoryProtection flProtect);
[DllImport("kernel32.dll")]
private static extern bool VirtualProtect(
IntPtr lpAddress,
IntPtr dwSize,
MemoryProtection flNewProtect,
out MemoryProtection lpflOldProtect);
[DllImport("kernel32.dll")]
private static extern bool VirtualFree(
IntPtr lpAddress,
IntPtr dwSize,
AllocationType dwFreeType);
public static IntPtr Allocate(IntPtr size)
{
const AllocationType flags =
AllocationType.Reserve |
AllocationType.Commit;
IntPtr ptr = VirtualAlloc(IntPtr.Zero, size, flags, MemoryProtection.ReadWrite);
if (ptr == IntPtr.Zero)
{
throw new OutOfMemoryException();
}
return ptr;
}
public static IntPtr AllocateWriteTracked(IntPtr size)
{
const AllocationType flags =
AllocationType.Reserve |
AllocationType.Commit |
AllocationType.WriteWatch;
IntPtr ptr = VirtualAlloc(IntPtr.Zero, size, flags, MemoryProtection.ReadWrite);
if (ptr == IntPtr.Zero)
{
throw new OutOfMemoryException();
}
return ptr;
}
public static bool Commit(IntPtr location, IntPtr size)
{
const AllocationType flags = AllocationType.Commit;
IntPtr ptr = VirtualAlloc(location, size, flags, MemoryProtection.ReadWrite);
return ptr != IntPtr.Zero;
}
public static bool Reprotect(IntPtr address, IntPtr size, Memory.MemoryProtection protection)
{
MemoryProtection prot = GetProtection(protection);
return VirtualProtect(address, size, prot, out _);
}
public static IntPtr Reserve(IntPtr size)
{
const AllocationType flags = AllocationType.Reserve;
IntPtr ptr = VirtualAlloc(IntPtr.Zero, size, flags, MemoryProtection.ReadWrite);
if (ptr == IntPtr.Zero)
{
throw new OutOfMemoryException();
}
return ptr;
}
private static MemoryProtection GetProtection(Memory.MemoryProtection protection)
{
switch (protection)
{
case Memory.MemoryProtection.None: return MemoryProtection.NoAccess;
case Memory.MemoryProtection.Read: return MemoryProtection.ReadOnly;
case Memory.MemoryProtection.ReadAndWrite: return MemoryProtection.ReadWrite;
case Memory.MemoryProtection.ReadAndExecute: return MemoryProtection.ExecuteRead;
case Memory.MemoryProtection.ReadWriteExecute: return MemoryProtection.ExecuteReadWrite;
case Memory.MemoryProtection.Execute: return MemoryProtection.Execute;
default: throw new ArgumentException($"Invalid permission \"{protection}\".");
}
}
public static bool Free(IntPtr address)
{
return VirtualFree(address, IntPtr.Zero, AllocationType.Release);
}
}
}

738
ARMeilleure/Memory/MemoryManager.cs
View file

@ -1,738 +0,0 @@
using ARMeilleure.State;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using static ARMeilleure.Memory.MemoryManagement;
namespace ARMeilleure.Memory
{
public unsafe class MemoryManager
{
public const int PageBits = 12;
public const int PageSize = 1 << PageBits;
public const int PageMask = PageSize - 1;
internal const long PteFlagsMask = 7;
public IntPtr Ram { get; private set; }
private byte* _ramPtr;
private IntPtr _pageTable;
internal IntPtr PageTable => _pageTable;
internal int PtLevelBits { get; }
internal int PtLevelSize { get; }
internal int PtLevelMask { get; }
public int AddressSpaceBits { get; }
public long AddressSpaceSize { get; }
public MemoryManager(
IntPtr ram,
int addressSpaceBits = 48,
bool useFlatPageTable = false)
{
Ram = ram;
_ramPtr = (byte*)ram;
AddressSpaceBits = addressSpaceBits;
AddressSpaceSize = 1L << addressSpaceBits;
// When flat page table is requested, we use a single
// array for the mappings of the entire address space.
// This has better performance, but also high memory usage.
// The multi level page table uses 9 bits per level, so
// the memory usage is lower, but the performance is also
// lower, since each address translation requires multiple reads.
if (useFlatPageTable)
{
PtLevelBits = addressSpaceBits - PageBits;
}
else
{
PtLevelBits = 9;
}
PtLevelSize = 1 << PtLevelBits;
PtLevelMask = PtLevelSize - 1;
_pageTable = Allocate((ulong)(PtLevelSize * IntPtr.Size));
}
public void Map(long va, long pa, long size)
{
SetPtEntries(va, _ramPtr + pa, size);
}
public void Unmap(long position, long size)
{
SetPtEntries(position, null, size);
}
public bool IsMapped(long position)
{
return Translate(position) != IntPtr.Zero;
}
public long GetPhysicalAddress(long virtualAddress)
{
byte* ptr = (byte*)Translate(virtualAddress);
return (long)(ptr - _ramPtr);
}
private IntPtr Translate(long position)
{
if (!IsValidPosition(position))
{
return IntPtr.Zero;
}
byte* ptr = GetPtEntry(position);
ulong ptrUlong = (ulong)ptr;
if ((ptrUlong & PteFlagsMask) != 0)
{
ptrUlong &= ~(ulong)PteFlagsMask;
ptr = (byte*)ptrUlong;
}
if (ptr == null)
{
return IntPtr.Zero;
}
return new IntPtr(ptr + (position & PageMask));
}
private IntPtr TranslateWrite(long position)
{
if (!IsValidPosition(position))
{
return IntPtr.Zero;
}
byte* ptr = GetPtEntry(position);
ulong ptrUlong = (ulong)ptr;
if ((ptrUlong & PteFlagsMask) != 0)
{
ClearPtEntryFlag(position, PteFlagsMask);
ptrUlong &= ~(ulong)PteFlagsMask;
ptr = (byte*)ptrUlong;
}
return new IntPtr(ptr + (position & PageMask));
}
private byte* GetPtEntry(long position)
{
return *(byte**)GetPtPtr(position);
}
private void SetPtEntries(long va, byte* ptr, long size)
{
long endPosition = (va + size + PageMask) & ~PageMask;
while ((ulong)va < (ulong)endPosition)
{
SetPtEntry(va, ptr);
va += PageSize;
if (ptr != null)
{
ptr += PageSize;
}
}
}
private void SetPtEntry(long position, byte* ptr)
{
*(byte**)GetPtPtr(position) = ptr;
}
private void SetPtEntryFlag(long position, long flag)
{
ModifyPtEntryFlag(position, flag, setFlag: true);
}
private void ClearPtEntryFlag(long position, long flag)
{
ModifyPtEntryFlag(position, flag, setFlag: false);
}
private void ModifyPtEntryFlag(long position, long flag, bool setFlag)
{
IntPtr* pt = (IntPtr*)_pageTable;
while (true)
{
IntPtr* ptPtr = GetPtPtr(position);
IntPtr old = *ptPtr;
long modified = old.ToInt64();
if (setFlag)
{
modified |= flag;
}
else
{
modified &= ~flag;
}
IntPtr origValue = Interlocked.CompareExchange(ref *ptPtr, new IntPtr(modified), old);
if (origValue == old)
{
break;
}
}
}
private IntPtr* GetPtPtr(long position)
{
if (!IsValidPosition(position))
{
throw new ArgumentOutOfRangeException(nameof(position));
}
IntPtr nextPtr = _pageTable;
IntPtr* ptePtr = null;
int bit = PageBits;
while (true)
{
long index = (position >> bit) & PtLevelMask;
ptePtr = &((IntPtr*)nextPtr)[index];
bit += PtLevelBits;
if (bit >= AddressSpaceBits)
{
break;
}
nextPtr = *ptePtr;
if (nextPtr == IntPtr.Zero)
{
// Entry does not yet exist, allocate a new one.
IntPtr newPtr = Allocate((ulong)(PtLevelSize * IntPtr.Size));
// Try to swap the current pointer (should be zero), with the allocated one.
nextPtr = Interlocked.CompareExchange(ref *ptePtr, newPtr, IntPtr.Zero);
// If the old pointer is not null, then another thread already has set it.
if (nextPtr != IntPtr.Zero)
{
Free(newPtr);
}
else
{
nextPtr = newPtr;
}
}
}
return ptePtr;
}
public unsafe (ulong, ulong)[] GetModifiedRanges(ulong address, ulong size, int id)
{
ulong idMask = 1UL << id;
List<(ulong, ulong)> ranges = new List<(ulong, ulong)>();
ulong endAddress = (address + size + PageMask) & ~(ulong)PageMask;
address &= ~(ulong)PageMask;
ulong currAddr = address;
ulong currSize = 0;
while (address < endAddress)
{
// If the address is invalid, we stop and consider all the remaining memory
// as not modified (since the address is invalid, we can't check, and technically
// the memory doesn't exist).
if (!IsValidPosition((long)address))
{
break;
}
byte* ptr = ((byte**)_pageTable)[address >> PageBits];
ulong ptrUlong = (ulong)ptr;
if ((ptrUlong & idMask) == 0)
{
// Modified.
currSize += PageSize;
SetPtEntryFlag((long)address, (long)idMask);
}
else
{
if (currSize != 0)
{
ranges.Add((currAddr, currSize));
}
currAddr = address + PageSize;
currSize = 0;
}
address += PageSize;
}
if (currSize != 0)
{
ranges.Add((currAddr, currSize));
}
return ranges.ToArray();
}
private bool IsContiguous(long position, long size)
{
long endPos = position + size;
position &= ~PageMask;
long expectedPa = GetPhysicalAddress(position);
while ((ulong)position < (ulong)endPos)
{
long pa = GetPhysicalAddress(position);
if (pa != expectedPa)
{
return false;
}
position += PageSize;
expectedPa += PageSize;
}
return true;
}
public bool IsValidPosition(long position)
{
return (ulong)position < (ulong)AddressSpaceSize;
}
internal V128 AtomicLoadInt128(long position)
{
if ((position & 0xf) != 0)
{
AbortWithAlignmentFault(position);
}
IntPtr ptr = TranslateWrite(position);
return MemoryManagerPal.AtomicLoad128(ptr);
}
internal bool AtomicCompareExchangeByte(long position, byte expected, byte desired)
{
int* ptr = (int*)Translate(position);
int currentValue = *ptr;
int expected32 = (currentValue & ~byte.MaxValue) | expected;
int desired32 = (currentValue & ~byte.MaxValue) | desired;
return Interlocked.CompareExchange(ref *ptr, desired32, expected32) == expected32;
}
internal bool AtomicCompareExchangeInt16(long position, short expected, short desired)
{
if ((position & 1) != 0)
{
AbortWithAlignmentFault(position);
}
int* ptr = (int*)Translate(position);
int currentValue = *ptr;
int expected32 = (currentValue & ~ushort.MaxValue) | (ushort)expected;
int desired32 = (currentValue & ~ushort.MaxValue) | (ushort)desired;
return Interlocked.CompareExchange(ref *ptr, desired32, expected32) == expected32;
}
public bool AtomicCompareExchangeInt32(long position, int expected, int desired)
{
if ((position & 3) != 0)
{
AbortWithAlignmentFault(position);
}
int* ptr = (int*)TranslateWrite(position);
return Interlocked.CompareExchange(ref *ptr, desired, expected) == expected;
}
internal bool AtomicCompareExchangeInt64(long position, long expected, long desired)
{
if ((position & 7) != 0)
{
AbortWithAlignmentFault(position);
}
long* ptr = (long*)TranslateWrite(position);
return Interlocked.CompareExchange(ref *ptr, desired, expected) == expected;
}
internal bool AtomicCompareExchangeInt128(long position, V128 expected, V128 desired)
{
if ((position & 0xf) != 0)
{
AbortWithAlignmentFault(position);
}
IntPtr ptr = TranslateWrite(position);
return MemoryManagerPal.CompareAndSwap128(ptr, expected, desired) == expected;
}
public int AtomicIncrementInt32(long position)
{
if ((position & 3) != 0)
{
AbortWithAlignmentFault(position);
}
int* ptr = (int*)TranslateWrite(position);
return Interlocked.Increment(ref *ptr);
}
public int AtomicDecrementInt32(long position)
{
if ((position & 3) != 0)
{
AbortWithAlignmentFault(position);
}
int* ptr = (int*)TranslateWrite(position);
return Interlocked.Decrement(ref *ptr);
}
private void AbortWithAlignmentFault(long position)
{
// TODO: Abort mode and exception support on the CPU.
throw new InvalidOperationException($"Tried to compare exchange a misaligned address 0x{position:X16}.");
}
public sbyte ReadSByte(long position)
{
return (sbyte)ReadByte(position);
}
public short ReadInt16(long position)
{
return (short)ReadUInt16(position);
}
public int ReadInt32(long position)
{
return (int)ReadUInt32(position);
}
public long ReadInt64(long position)
{
return (long)ReadUInt64(position);
}
public byte ReadByte(long position)
{
return *((byte*)Translate(position));
}
public ushort ReadUInt16(long position)
{
if ((position & 1) == 0)
{
return *((ushort*)Translate(position));
}
else
{
return (ushort)(ReadByte(position + 0) << 0 |
ReadByte(position + 1) << 8);
}
}
public uint ReadUInt32(long position)
{
if ((position & 3) == 0)
{
return *((uint*)Translate(position));
}
else
{
return (uint)(ReadUInt16(position + 0) << 0 |
ReadUInt16(position + 2) << 16);
}
}
public ulong ReadUInt64(long position)
{
if ((position & 7) == 0)
{
return *((ulong*)Translate(position));
}
else
{
return (ulong)ReadUInt32(position + 0) << 0 |
(ulong)ReadUInt32(position + 4) << 32;
}
}
public V128 ReadVector128(long position)
{
return new V128(ReadUInt64(position), ReadUInt64(position + 8));
}
public byte[] ReadBytes(long position, long size)
{
long endAddr = position + size;
if ((ulong)size > int.MaxValue)
{
throw new ArgumentOutOfRangeException(nameof(size));
}
if ((ulong)endAddr < (ulong)position)
{
throw new ArgumentOutOfRangeException(nameof(position));
}
byte[] data = new byte[size];
int offset = 0;
while ((ulong)position < (ulong)endAddr)
{
long pageLimit = (position + PageSize) & ~(long)PageMask;
if ((ulong)pageLimit > (ulong)endAddr)
{
pageLimit = endAddr;
}
int copySize = (int)(pageLimit - position);
Marshal.Copy(Translate(position), data, offset, copySize);
position += copySize;
offset += copySize;
}
return data;
}
public ReadOnlySpan<byte> GetSpan(ulong address, ulong size)
{
if (IsContiguous(address, size))
{
return new ReadOnlySpan<byte>((void*)Translate((long)address), (int)size);
}
else
{
return ReadBytes((long)address, (long)size);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool IsContiguous(ulong address, ulong size)
{
if (!IsValidPosition((long)address))
{
return false;
}
ulong endVa = (address + size + PageMask) & ~(ulong)PageMask;
address &= ~(ulong)PageMask;
int pages = (int)((endVa - address) / PageSize);
for (int page = 0; page < pages - 1; page++)
{
if (!IsValidPosition((long)address + PageSize))
{
return false;
}
if (GetPtEntry((long)address) + PageSize != GetPtEntry((long)address + PageSize))
{
return false;
}
address += PageSize;
}
return true;
}
public void WriteSByte(long position, sbyte value)
{
WriteByte(position, (byte)value);
}
public void WriteInt16(long position, short value)
{
WriteUInt16(position, (ushort)value);
}
public void WriteInt32(long position, int value)
{
WriteUInt32(position, (uint)value);
}
public void WriteInt64(long position, long value)
{
WriteUInt64(position, (ulong)value);
}
public void WriteByte(long position, byte value)
{
*((byte*)TranslateWrite(position)) = value;
}
public void WriteUInt16(long position, ushort value)
{
if ((position & 1) == 0)
{
*((ushort*)TranslateWrite(position)) = value;
}
else
{
WriteByte(position + 0, (byte)(value >> 0));
WriteByte(position + 1, (byte)(value >> 8));
}
}
public void WriteUInt32(long position, uint value)
{
if ((position & 3) == 0)
{
*((uint*)TranslateWrite(position)) = value;
}
else
{
WriteUInt16(position + 0, (ushort)(value >> 0));
WriteUInt16(position + 2, (ushort)(value >> 16));
}
}
public void WriteUInt64(long position, ulong value)
{
if ((position & 7) == 0)
{
*((ulong*)TranslateWrite(position)) = value;
}
else
{
WriteUInt32(position + 0, (uint)(value >> 0));
WriteUInt32(position + 4, (uint)(value >> 32));
}
}
public void WriteVector128(long position, V128 value)
{
WriteUInt64(position + 0, value.Extract<ulong>(0));
WriteUInt64(position + 8, value.Extract<ulong>(1));
}
public void WriteBytes(long position, byte[] data)
{
long endAddr = position + data.Length;
if ((ulong)endAddr < (ulong)position)
{
throw new ArgumentOutOfRangeException(nameof(position));
}
int offset = 0;
while ((ulong)position < (ulong)endAddr)
{
long pageLimit = (position + PageSize) & ~(long)PageMask;
if ((ulong)pageLimit > (ulong)endAddr)
{
pageLimit = endAddr;
}
int copySize = (int)(pageLimit - position);
Marshal.Copy(data, offset, TranslateWrite(position), copySize);
position += copySize;
offset += copySize;
}
}
public void Dispose()
{
Dispose(true);
}
protected virtual void Dispose(bool disposing)
{
IntPtr ptr = Interlocked.Exchange(ref _pageTable, IntPtr.Zero);
if (ptr != IntPtr.Zero)
{
FreePageTableEntry(ptr, PageBits);
}
}
private void FreePageTableEntry(IntPtr ptr, int levelBitEnd)
{
levelBitEnd += PtLevelBits;
if (levelBitEnd >= AddressSpaceBits)
{
Free(ptr);
return;
}
for (int index = 0; index < PtLevelSize; index++)
{
IntPtr ptePtr = ((IntPtr*)ptr)[index];
if (ptePtr != IntPtr.Zero)
{
FreePageTableEntry(ptePtr, levelBitEnd);
}
}
Free(ptr);
}
}
}

11
ARMeilleure/Memory/MemoryManagerPal.cs
View file

@ -1,13 +1,12 @@
using ARMeilleure.IntermediateRepresentation;
using ARMeilleure.State;
using ARMeilleure.Translation;
using System;
namespace ARMeilleure.Memory
{
static class MemoryManagerPal
{
private delegate V128 CompareExchange128(IntPtr address, V128 expected, V128 desired);
private delegate V128 CompareExchange128(ref V128 location, V128 expected, V128 desired);
private static CompareExchange128 _compareExchange128;
@ -18,14 +17,14 @@ namespace ARMeilleure.Memory
_lock = new object();
}
public static V128 AtomicLoad128(IntPtr address)
public static V128 AtomicLoad128(ref V128 location)
{
return GetCompareAndSwap128()(address, V128.Zero, V128.Zero);
return GetCompareAndSwap128()(ref location, V128.Zero, V128.Zero);
}
public static V128 CompareAndSwap128(IntPtr address, V128 expected, V128 desired)
public static V128 CompareAndSwap128(ref V128 location, V128 expected, V128 desired)
{
return GetCompareAndSwap128()(address, expected, desired);
return GetCompareAndSwap128()(ref location, expected, desired);
}
private static CompareExchange128 GetCompareAndSwap128()

17
ARMeilleure/Memory/MemoryProtection.cs
View file

@ -1,17 +0,0 @@
using System;
namespace ARMeilleure.Memory
{
[Flags]
public enum MemoryProtection
{
None = 0,
Read = 1 << 0,
Write = 1 << 1,
Execute = 1 << 2,
ReadAndWrite = Read | Write,
ReadAndExecute = Read | Execute,
ReadWriteExecute = Read | Write | Execute
}
}

9
ARMeilleure/Memory/MemoryProtectionException.cs
View file

@ -1,9 +0,0 @@
using System;
namespace ARMeilleure.Memory
{
class MemoryProtectionException : Exception
{
public MemoryProtectionException(MemoryProtection protection) : base($"Failed to set memory protection to \"{protection}\".") { }
}
}

16
ARMeilleure/Memory/ReservedRegion.cs
View file

@ -1,6 +1,4 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace ARMeilleure.Memory
{
@ -8,20 +6,22 @@ namespace ARMeilleure.Memory
{
private const int DefaultGranularity = 65536; // Mapping granularity in Windows.
public IntPtr Pointer { get; }
public IJitMemoryBlock Block { get; }
private ulong _maxSize;
private ulong _sizeGranularity;
public IntPtr Pointer => Block.Pointer;
private readonly ulong _maxSize;
private readonly ulong _sizeGranularity;
private ulong _currentSize;
public ReservedRegion(ulong maxSize, ulong granularity = 0)
public ReservedRegion(IJitMemoryAllocator allocator, ulong maxSize, ulong granularity = 0)
{
if (granularity == 0)
{
granularity = DefaultGranularity;
}
Pointer = MemoryManagement.Reserve(maxSize);
Block = allocator.Reserve(maxSize);
_maxSize = maxSize;
_sizeGranularity = granularity;
_currentSize = 0;
@ -43,7 +43,7 @@ namespace ARMeilleure.Memory
{
ulong overflowBytes = desiredSize - _currentSize;
ulong moreToCommit = (((_sizeGranularity - 1) + overflowBytes) / _sizeGranularity) * _sizeGranularity; // Round up.
MemoryManagement.Commit(new IntPtr((long)Pointer + (long)_currentSize), moreToCommit);
Block.Commit(_currentSize, moreToCommit);
_currentSize += moreToCommit;
}
}

5
ARMeilleure/State/ExecutionContext.cs
View file

@ -1,3 +1,4 @@
using ARMeilleure.Memory;
using System;
using System.Diagnostics;
@ -73,9 +74,9 @@ namespace ARMeilleure.State
_tickCounter.Start();
}
public ExecutionContext()
public ExecutionContext(IJitMemoryAllocator allocator)
{
_nativeContext = new NativeContext();
_nativeContext = new NativeContext(allocator);
Running = true;

10
ARMeilleure/State/NativeContext.cs
View file

@ -17,11 +17,13 @@ namespace ARMeilleure.State
RegisterConsts.FlagsCount * FlagSize +
RegisterConsts.FpFlagsCount * FlagSize + ExtraSize;
public IntPtr BasePtr { get; }
private readonly IJitMemoryBlock _block;
public NativeContext()
public IntPtr BasePtr => _block.Pointer;
public NativeContext(IJitMemoryAllocator allocator)
{
BasePtr = MemoryManagement.Allocate(TotalSize);
_block = allocator.Allocate(TotalSize);
}
public ulong GetX(int index)
@ -193,7 +195,7 @@ namespace ARMeilleure.State
public void Dispose()
{
MemoryManagement.Free(BasePtr);
_block.Dispose();
}
}
}

4
ARMeilleure/Translation/ArmEmitterContext.cs
View file

@ -37,7 +37,7 @@ namespace ARMeilleure.Translation
public OpCode CurrOp { get; set; }
public MemoryManager Memory { get; }
public IMemoryManager Memory { get; }
public Aarch32Mode Mode { get; }
@ -47,7 +47,7 @@ namespace ARMeilleure.Translation
public bool HighCq { get; }
public ArmEmitterContext(MemoryManager memory, JumpTable jumpTable, long baseAddress, bool highCq, Aarch32Mode mode)
public ArmEmitterContext(IMemoryManager memory, JumpTable jumpTable, long baseAddress, bool highCq, Aarch32Mode mode)
{
Memory = memory;
JumpTable = jumpTable;

4
ARMeilleure/Translation/Compiler.cs
View file

@ -11,14 +11,14 @@ namespace ARMeilleure.Translation
{
public static T Compile<T>(ControlFlowGraph cfg, OperandType[] argTypes, OperandType retType, CompilerOptions options)
{
CompiledFunction func = CompileAndGetCf(cfg, argTypes, retType, options);
CompiledFunction func = Compile(cfg, argTypes, retType, options);
IntPtr codePtr = JitCache.Map(func);
return Marshal.GetDelegateForFunctionPointer<T>(codePtr);
}
public static CompiledFunction CompileAndGetCf(ControlFlowGraph cfg, OperandType[] argTypes, OperandType retType, CompilerOptions options)
public static CompiledFunction Compile(ControlFlowGraph cfg, OperandType[] argTypes, OperandType retType, CompilerOptions options)
{
Logger.StartPass(PassName.Dominance);

19
ARMeilleure/Translation/DirectCallStubs.cs
View file

@ -17,14 +17,9 @@ namespace ARMeilleure.Translation
private static GuestFunction _indirectCallStub;
private static GuestFunction _indirectTailCallStub;
private static object _lock;
private static readonly object _lock = new object();
private static bool _initialized;
static DirectCallStubs()
{
_lock = new object();
}
public static void InitializeStubs()
{
if (_initialized) return;
@ -85,11 +80,7 @@ namespace ARMeilleure.Translation
OperandType.I64
};
return Compiler.Compile<GuestFunction>(
cfg,
argTypes,
OperandType.I64,
CompilerOptions.HighCq);
return Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, CompilerOptions.HighCq);
}
/// <summary>
@ -121,11 +112,7 @@ namespace ARMeilleure.Translation
OperandType.I64
};
return Compiler.Compile<GuestFunction>(
cfg,
argTypes,
OperandType.I64,
CompilerOptions.HighCq);
return Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, CompilerOptions.HighCq);
}
}
}

51
ARMeilleure/Translation/JitCache.cs
View file

@ -2,6 +2,7 @@ using ARMeilleure.CodeGen;
using ARMeilleure.Memory;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.InteropServices;
namespace ARMeilleure.Translation
@ -12,35 +13,33 @@ namespace ARMeilleure.Translation
private const int PageMask = PageSize - 1;
private const int CodeAlignment = 4; // Bytes
private const int CacheSize = 2047 * 1024 * 1024;
private static ReservedRegion _jitRegion;
private static IntPtr _basePointer => _jitRegion.Pointer;
private static int _offset;
private static readonly List<JitCacheEntry> _cacheEntries = new List<JitCacheEntry>();
private static List<JitCacheEntry> _cacheEntries;
private static readonly object _lock = new object();
private static bool _initialized;
private static object _lock;
static JitCache()
public static void Initialize(IJitMemoryAllocator allocator)
{
_jitRegion = new ReservedRegion(CacheSize);
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
if (_initialized) return;
lock (_lock)
{
_jitRegion.ExpandIfNeeded(PageSize);
JitUnwindWindows.InstallFunctionTableHandler(_basePointer, CacheSize);
if (_initialized) return;
_jitRegion = new ReservedRegion(allocator, CacheSize);
// The first page is used for the table based SEH structs.
_offset = PageSize;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
_jitRegion.ExpandIfNeeded(PageSize);
JitUnwindWindows.InstallFunctionTableHandler(_jitRegion.Pointer, CacheSize);
// The first page is used for the table based SEH structs.
_offset = PageSize;
}
_initialized = true;
}
_cacheEntries = new List<JitCacheEntry>();
_lock = new object();
}
public static IntPtr Map(CompiledFunction func)
@ -49,9 +48,11 @@ namespace ARMeilleure.Translation
lock (_lock)
{
Debug.Assert(_initialized);
int funcOffset = Allocate(code.Length);
IntPtr funcPtr = _basePointer + funcOffset;
IntPtr funcPtr = _jitRegion.Pointer + funcOffset;
Marshal.Copy(code, 0, funcPtr, code.Length);
@ -77,18 +78,14 @@ namespace ARMeilleure.Translation
if (fullPagesSize != 0)
{
IntPtr funcPtr = _basePointer + pageStart;
MemoryManagement.Reprotect(funcPtr, (ulong)fullPagesSize, MemoryProtection.ReadAndExecute);
_jitRegion.Block.MapAsRx((ulong)pageStart, (ulong)fullPagesSize);
}
int remaining = endOffs - pageEnd;
if (remaining != 0)
{
IntPtr funcPtr = _basePointer + pageEnd;
MemoryManagement.Reprotect(funcPtr, (ulong)remaining, MemoryProtection.ReadWriteExecute);
_jitRegion.Block.MapAsRwx((ulong)pageEnd, (ulong)remaining);
}
}
@ -132,7 +129,7 @@ namespace ARMeilleure.Translation
}
}
entry = default(JitCacheEntry);
entry = default;
return false;
}

22
ARMeilleure/Translation/JumpTable.cs
View file

@ -9,13 +9,6 @@ namespace ARMeilleure.Translation
{
class JumpTable
{
public static JumpTable Instance { get; }
static JumpTable()
{
Instance = new JumpTable();
}
// The jump table is a block of (guestAddress, hostAddress) function mappings.
// Each entry corresponds to one branch in a JIT compiled function. The entries are
// reserved specifically for each call.
@ -60,23 +53,23 @@ namespace ARMeilleure.Translation
public IntPtr JumpPointer => _jumpRegion.Pointer;
public IntPtr DynamicPointer => _dynamicRegion.Pointer;
public JumpTable()
public JumpTable(IJitMemoryAllocator allocator)
{
_jumpRegion = new ReservedRegion(JumpTableByteSize);
_dynamicRegion = new ReservedRegion(DynamicTableByteSize);
_jumpRegion = new ReservedRegion(allocator, JumpTableByteSize);
_dynamicRegion = new ReservedRegion(allocator, DynamicTableByteSize);
_targets = new ConcurrentDictionary<ulong, TranslatedFunction>();
_dependants = new ConcurrentDictionary<ulong, LinkedList<int>>();
}
public void RegisterFunction(ulong address, TranslatedFunction func) {
public void RegisterFunction(ulong address, TranslatedFunction func)
{
address &= ~3UL;
_targets.AddOrUpdate(address, func, (key, oldFunc) => func);
long funcPtr = func.GetPointer().ToInt64();
// Update all jump table entries that target this address.
LinkedList<int> myDependants;
if (_dependants.TryGetValue(address, out myDependants))
if (_dependants.TryGetValue(address, out LinkedList<int> myDependants))
{
lock (myDependants)
{
@ -125,8 +118,7 @@ namespace ARMeilleure.Translation
// Is the address we have already registered? If so, put the function address in the jump table.
// If not, it will point to the direct call stub.
long value = (long)DirectCallStubs.DirectCallStub(isJump);
TranslatedFunction func;
if (_targets.TryGetValue((ulong)address, out func))
if (_targets.TryGetValue((ulong)address, out TranslatedFunction func))
{
value = func.GetPointer().ToInt64();
}

28
ARMeilleure/Translation/Translator.cs
View file

@ -9,6 +9,7 @@ using System.Collections.Concurrent;
using System.Threading;
using static ARMeilleure.IntermediateRepresentation.OperandHelper;
using static ARMeilleure.IntermediateRepresentation.OperationHelper;
namespace ARMeilleure.Translation
{
@ -18,30 +19,31 @@ namespace ARMeilleure.Translation
private const bool AlwaysTranslateFunctions = true; // If false, only translates a single block for lowCq.
private MemoryManager _memory;
private readonly IMemoryManager _memory;
private ConcurrentDictionary<ulong, TranslatedFunction> _funcs;
private readonly ConcurrentDictionary<ulong, TranslatedFunction> _funcs;
private JumpTable _jumpTable;
private readonly JumpTable _jumpTable;
private PriorityQueue<RejitRequest> _backgroundQueue;
private readonly PriorityQueue<RejitRequest> _backgroundQueue;
private AutoResetEvent _backgroundTranslatorEvent;
private readonly AutoResetEvent _backgroundTranslatorEvent;
private volatile int _threadCount;
public Translator(MemoryManager memory)
public Translator(IJitMemoryAllocator allocator, IMemoryManager memory)
{
_memory = memory;
_funcs = new ConcurrentDictionary<ulong, TranslatedFunction>();
_jumpTable = JumpTable.Instance;
_jumpTable = new JumpTable(allocator);
_backgroundQueue = new PriorityQueue<RejitRequest>(2);
_backgroundTranslatorEvent = new AutoResetEvent(false);
JitCache.Initialize(allocator);
DirectCallStubs.InitializeStubs();
}
@ -146,8 +148,8 @@ namespace ARMeilleure.Translation
{
ArmEmitterContext context = new ArmEmitterContext(_memory, _jumpTable, (long)address, highCq, Aarch32Mode.User);
OperandHelper.PrepareOperandPool(highCq);
OperationHelper.PrepareOperationPool(highCq);
PrepareOperandPool(highCq);
PrepareOperationPool(highCq);
Logger.StartPass(PassName.Decoding);
@ -178,14 +180,12 @@ namespace ARMeilleure.Translation
OperandType[] argTypes = new OperandType[] { OperandType.I64 };
CompilerOptions options = highCq
? CompilerOptions.HighCq
: CompilerOptions.None;
CompilerOptions options = highCq ? CompilerOptions.HighCq : CompilerOptions.None;
GuestFunction func = Compiler.Compile<GuestFunction>(cfg, argTypes, OperandType.I64, options);
OperandHelper.ResetOperandPool(highCq);
OperationHelper.ResetOperationPool(highCq);
ResetOperandPool(highCq);
ResetOperationPool(highCq);
return new TranslatedFunction(func, rejit: !highCq);
}