Add a new JIT compiler for CPU code (#693)

* Start of the ARMeilleure project

* Refactoring around the old IRAdapter, now renamed to PreAllocator

* Optimize the LowestBitSet method

* Add CLZ support and fix CLS implementation

* Add missing Equals and GetHashCode overrides on some structs, misc small tweaks

* Implement the ByteSwap IR instruction, and some refactoring on the assembler

* Implement the DivideUI IR instruction and fix 64-bits IDIV

* Correct constant operand type on CSINC

* Move division instructions implementation to InstEmitDiv

* Fix destination type for the ConditionalSelect IR instruction

* Implement UMULH and SMULH, with new IR instructions

* Fix some issues with shift instructions

* Fix constant types for BFM instructions

* Fix up new tests using the new V128 struct

* Update tests

* Move DIV tests to a separate file

* Add support for calls, and some instructions that depends on them

* Start adding support for SIMD & FP types, along with some of the related ARM instructions

* Fix some typos and the divide instruction with FP operands

* Fix wrong method call on Clz_V

* Implement ARM FP & SIMD move instructions, Saddlv_V, and misc. fixes

* Implement SIMD logical instructions and more misc. fixes

* Fix PSRAD x86 instruction encoding, TRN, UABD and UABDL implementations

* Implement float conversion instruction, merge in LDj3SNuD fixes, and some other misc. fixes

* Implement SIMD shift instruction and fix Dup_V

* Add SCVTF and UCVTF (vector, fixed-point) variants to the opcode table

* Fix check with tolerance on tester

* Implement FP & SIMD comparison instructions, and some fixes

* Update FCVT (Scalar) encoding on the table to support the Half-float variants

* Support passing V128 structs, some cleanup on the register allocator, merge LDj3SNuD fixes

* Use old memory access methods, made a start on SIMD memory insts support, some fixes

* Fix float constant passed to functions, save and restore non-volatile XMM registers, other fixes

* Fix arguments count with struct return values, other fixes

* More instructions

* Misc. fixes and integrate LDj3SNuD fixes

* Update tests

* Add a faster linear scan allocator, unwinding support on windows, and other changes

* Update Ryujinx.HLE

* Update Ryujinx.Graphics

* Fix V128 return pointer passing, RCX is clobbered

* Update Ryujinx.Tests

* Update ITimeZoneService

* Stop using GetFunctionPointer as that can't be called from native code, misc. fixes and tweaks

* Use generic GetFunctionPointerForDelegate method and other tweaks

* Some refactoring on the code generator, assert on invalid operations and use a separate enum for intrinsics

* Remove some unused code on the assembler

* Fix REX.W prefix regression on float conversion instructions, add some sort of profiler

* Add hardware capability detection

* Fix regression on Sha1h and revert Fcm** changes

* Add SSE2-only paths on vector extract and insert, some refactoring on the pre-allocator

* Fix silly mistake introduced on last commit on CpuId

* Generate inline stack probes when the stack allocation is too large

* Initial support for the System-V ABI

* Support multiple destination operands

* Fix SSE2 VectorInsert8 path, and other fixes

* Change placement of XMM callee save and restore code to match other compilers

* Rename Dest to Destination and Inst to Instruction

* Fix a regression related to calls and the V128 type

* Add an extra space on comments to match code style

* Some refactoring

* Fix vector insert FP32 SSE2 path

* Port over the ARM32 instructions

* Avoid memory protection races on JIT Cache

* Another fix on VectorInsert FP32 (thanks to LDj3SNuD

* Float operands don't need to use the same register when VEX is supported

* Add a new register allocator, higher quality code for hot code (tier up), and other tweaks

* Some nits, small improvements on the pre allocator

* CpuThreadState is gone

* Allow changing CPU emulators with a config entry

* Add runtime identifiers on the ARMeilleure project

* Allow switching between CPUs through a config entry (pt. 2)

* Change win10-x64 to win-x64 on projects

* Update the Ryujinx project to use ARMeilleure

* Ensure that the selected register is valid on the hybrid allocator

* Allow exiting on returns to 0 (should fix test regression)

* Remove register assignments for most used variables on the hybrid allocator

* Do not use fixed registers as spill temp

* Add missing namespace and remove unneeded using

* Address PR feedback

* Fix types, etc

* Enable AssumeStrictAbiCompliance by default

* Ensure that Spill and Fill don't load or store any more than necessary

ARMeilleure/Memory/IMemory.cs

@@ -0,0 +1,37 @@
+namespace ARMeilleure.Memory
+{
+    public interface IMemory
+    {
+        sbyte ReadSByte(long position);
+
+        short ReadInt16(long position);
+
+        int ReadInt32(long position);
+
+        long ReadInt64(long position);
+
+        byte ReadByte(long position);
+
+        ushort ReadUInt16(long position);
+
+        uint ReadUInt32(long position);
+
+        ulong ReadUInt64(long position);
+
+        void WriteSByte(long position, sbyte value);
+
+        void WriteInt16(long position, short value);
+
+        void WriteInt32(long position, int value);
+
+        void WriteInt64(long position, long value);
+
+        void WriteByte(long position, byte value);
+
+        void WriteUInt16(long position, ushort value);
+
+        void WriteUInt32(long position, uint value);
+
+        void WriteUInt64(long position, ulong value);
+    }
+}

ARMeilleure/Memory/IMemoryManager.cs

@@ -0,0 +1,40 @@
+using ARMeilleure.State;
+using System;
+
+namespace ARMeilleure.Memory
+{
+    public interface IMemoryManager : IMemory, IDisposable
+    {
+        void Map(long va, long pa, long size);
+
+        void Unmap(long position, long size);
+
+        bool IsMapped(long position);
+
+        long GetPhysicalAddress(long virtualAddress);
+
+        bool IsRegionModified(long position, long size);
+
+        bool TryGetHostAddress(long position, long size, out IntPtr ptr);
+
+        bool IsValidPosition(long position);
+
+        bool AtomicCompareExchangeInt32(long position, int expected, int desired);
+
+        int AtomicIncrementInt32(long position);
+
+        int AtomicDecrementInt32(long position);
+
+        byte[] ReadBytes(long position, long size);
+
+        void ReadBytes(long position, byte[] data, int startIndex, int size);
+
+        void WriteVector128(long position, V128 value);
+
+        void WriteBytes(long position, byte[] data);
+
+        void WriteBytes(long position, byte[] data, int startIndex, int size);
+
+        void CopyBytes(long src, long dst, long size);
+    }
+}

ARMeilleure/Memory/MemoryHelper.cs

@@ -0,0 +1,71 @@
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace ARMeilleure.Memory
+{
+    public static class MemoryHelper
+    {
+        public static void FillWithZeros(IMemoryManager memory, long position, int size)
+        {
+            int size8 = size & ~(8 - 1);
+
+            for (int offs = 0; offs < size8; offs += 8)
+            {
+                memory.WriteInt64(position + offs, 0);
+            }
+
+            for (int offs = size8; offs < (size - size8); offs++)
+            {
+                memory.WriteByte(position + offs, 0);
+            }
+        }
+
+        public unsafe static T Read<T>(IMemoryManager memory, long position) where T : struct
+        {
+            long size = Marshal.SizeOf<T>();
+
+            byte[] data = memory.ReadBytes(position, size);
+
+            fixed (byte* ptr = data)
+            {
+                return Marshal.PtrToStructure<T>((IntPtr)ptr);
+            }
+        }
+
+        public unsafe static void Write<T>(IMemoryManager memory, long position, T value) where T : struct
+        {
+            long size = Marshal.SizeOf<T>();
+
+            byte[] data = new byte[size];
+
+            fixed (byte* ptr = data)
+            {
+                Marshal.StructureToPtr<T>(value, (IntPtr)ptr, false);
+            }
+
+            memory.WriteBytes(position, data);
+        }
+
+        public static string ReadAsciiString(IMemoryManager memory, long position, long maxSize = -1)
+        {
+            using (MemoryStream ms = new MemoryStream())
+            {
+                for (long offs = 0; offs < maxSize || maxSize == -1; offs++)
+                {
+                    byte value = (byte)memory.ReadByte(position + offs);
+
+                    if (value == 0)
+                    {
+                        break;
+                    }
+
+                    ms.WriteByte(value);
+                }
+
+                return Encoding.ASCII.GetString(ms.ToArray());
+            }
+        }
+    }
+}

ARMeilleure/Memory/MemoryManagement.cs

@@ -0,0 +1,114 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Memory
+{
+    public static class MemoryManagement
+    {
+        public static bool HasWriteWatchSupport => RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
+
+        public static IntPtr Allocate(ulong size)
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                IntPtr sizeNint = new IntPtr((long)size);
+
+                return MemoryManagementWindows.Allocate(sizeNint);
+            }
+            else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
+                     RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                return MemoryManagementUnix.Allocate(size);
+            }
+            else
+            {
+                throw new PlatformNotSupportedException();
+            }
+        }
+
+        public static IntPtr AllocateWriteTracked(ulong size)
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                IntPtr sizeNint = new IntPtr((long)size);
+
+                return MemoryManagementWindows.AllocateWriteTracked(sizeNint);
+            }
+            else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
+                     RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                return MemoryManagementUnix.Allocate(size);
+            }
+            else
+            {
+                throw new PlatformNotSupportedException();
+            }
+        }
+
+        public static void Reprotect(IntPtr address, ulong size, MemoryProtection permission)
+        {
+            bool result;
+
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                IntPtr sizeNint = new IntPtr((long)size);
+
+                result = MemoryManagementWindows.Reprotect(address, sizeNint, permission);
+            }
+            else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
+                     RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                result = MemoryManagementUnix.Reprotect(address, size, permission);
+            }
+            else
+            {
+                throw new PlatformNotSupportedException();
+            }
+
+            if (!result)
+            {
+                throw new MemoryProtectionException(permission);
+            }
+        }
+
+        public static bool Free(IntPtr address)
+        {
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                return MemoryManagementWindows.Free(address);
+            }
+            else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ||
+                     RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
+            {
+                return MemoryManagementUnix.Free(address);
+            }
+            else
+            {
+                throw new PlatformNotSupportedException();
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static bool GetModifiedPages(
+            IntPtr    address,
+            IntPtr    size,
+            IntPtr[]  addresses,
+            out ulong count)
+        {
+            // This is only supported on windows, but returning
+            // false (failed) is also valid for platforms without
+            // write tracking support on the OS.
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                return MemoryManagementWindows.GetModifiedPages(address, size, addresses, out count);
+            }
+            else
+            {
+                count = 0;
+
+                return false;
+            }
+        }
+    }
+}

ARMeilleure/Memory/MemoryManagementUnix.cs

@@ -0,0 +1,71 @@
+using Mono.Unix.Native;
+using System;
+
+namespace ARMeilleure.Memory
+{
+    static class MemoryManagementUnix
+    {
+        public static IntPtr Allocate(ulong size)
+        {
+            ulong pageSize = (ulong)Syscall.sysconf(SysconfName._SC_PAGESIZE);
+
+            const MmapProts prot = MmapProts.PROT_READ | MmapProts.PROT_WRITE;
+
+            const MmapFlags flags = MmapFlags.MAP_PRIVATE | MmapFlags.MAP_ANONYMOUS;
+
+            IntPtr ptr = Syscall.mmap(IntPtr.Zero, size + pageSize, prot, flags, -1, 0);
+
+            if (ptr == IntPtr.Zero)
+            {
+                throw new OutOfMemoryException();
+            }
+
+            unsafe
+            {
+                ptr = new IntPtr(ptr.ToInt64() + (long)pageSize);
+
+                *((ulong*)ptr - 1) = size;
+            }
+
+            return ptr;
+        }
+
+        public static bool Reprotect(IntPtr address, ulong size, Memory.MemoryProtection protection)
+        {
+            MmapProts prot = GetProtection(protection);
+
+            return Syscall.mprotect(address, size, prot) == 0;
+        }
+
+        private static MmapProts GetProtection(Memory.MemoryProtection protection)
+        {
+            switch (protection)
+            {
+                case Memory.MemoryProtection.None:             return MmapProts.PROT_NONE;
+                case Memory.MemoryProtection.Read:             return MmapProts.PROT_READ;
+                case Memory.MemoryProtection.ReadAndWrite:     return MmapProts.PROT_READ | MmapProts.PROT_WRITE;
+                case Memory.MemoryProtection.ReadAndExecute:   return MmapProts.PROT_READ | MmapProts.PROT_EXEC;
+                case Memory.MemoryProtection.ReadWriteExecute: return MmapProts.PROT_READ | MmapProts.PROT_WRITE | MmapProts.PROT_EXEC;
+                case Memory.MemoryProtection.Execute:          return MmapProts.PROT_EXEC;
+
+                default: throw new ArgumentException($"Invalid permission \"{protection}\".");
+            }
+        }
+
+        public static bool Free(IntPtr address)
+        {
+            ulong pageSize = (ulong)Syscall.sysconf(SysconfName._SC_PAGESIZE);
+
+            ulong size;
+
+            unsafe
+            {
+                size = *((ulong*)address - 1);
+
+                address = new IntPtr(address.ToInt64() - (long)pageSize);
+            }
+
+            return Syscall.munmap(address, size + pageSize) == 0;
+        }
+    }
+}

ARMeilleure/Memory/MemoryManagementWindows.cs

@@ -0,0 +1,156 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Memory
+{
+    static class MemoryManagementWindows
+    {
+        [Flags]
+        private enum AllocationType : uint
+        {
+            Commit     = 0x1000,
+            Reserve    = 0x2000,
+            Decommit   = 0x4000,
+            Release    = 0x8000,
+            Reset      = 0x80000,
+            Physical   = 0x400000,
+            TopDown    = 0x100000,
+            WriteWatch = 0x200000,
+            LargePages = 0x20000000
+        }
+
+        [Flags]
+        private enum MemoryProtection : uint
+        {
+            NoAccess                 = 0x01,
+            ReadOnly                 = 0x02,
+            ReadWrite                = 0x04,
+            WriteCopy                = 0x08,
+            Execute                  = 0x10,
+            ExecuteRead              = 0x20,
+            ExecuteReadWrite         = 0x40,
+            ExecuteWriteCopy         = 0x80,
+            GuardModifierflag        = 0x100,
+            NoCacheModifierflag      = 0x200,
+            WriteCombineModifierflag = 0x400
+        }
+
+        private enum WriteWatchFlags : uint
+        {
+            None  = 0,
+            Reset = 1
+        }
+
+        [DllImport("kernel32.dll")]
+        private static extern IntPtr VirtualAlloc(
+            IntPtr           lpAddress,
+            IntPtr           dwSize,
+            AllocationType   flAllocationType,
+            MemoryProtection flProtect);
+
+        [DllImport("kernel32.dll")]
+        private static extern bool VirtualProtect(
+            IntPtr               lpAddress,
+            IntPtr               dwSize,
+            MemoryProtection     flNewProtect,
+            out MemoryProtection lpflOldProtect);
+
+        [DllImport("kernel32.dll")]
+        private static extern bool VirtualFree(
+            IntPtr         lpAddress,
+            IntPtr         dwSize,
+            AllocationType dwFreeType);
+
+        [DllImport("kernel32.dll")]
+        private static extern int GetWriteWatch(
+            WriteWatchFlags dwFlags,
+            IntPtr          lpBaseAddress,
+            IntPtr          dwRegionSize,
+            IntPtr[]        lpAddresses,
+            ref ulong       lpdwCount,
+            out uint        lpdwGranularity);
+
+        public static IntPtr Allocate(IntPtr size)
+        {
+            const AllocationType flags =
+                AllocationType.Reserve |
+                AllocationType.Commit;
+
+            IntPtr ptr = VirtualAlloc(IntPtr.Zero, size, flags, MemoryProtection.ReadWrite);
+
+            if (ptr == IntPtr.Zero)
+            {
+                throw new OutOfMemoryException();
+            }
+
+            return ptr;
+        }
+
+        public static IntPtr AllocateWriteTracked(IntPtr size)
+        {
+            const AllocationType flags =
+                AllocationType.Reserve |
+                AllocationType.Commit  |
+                AllocationType.WriteWatch;
+
+            IntPtr ptr = VirtualAlloc(IntPtr.Zero, size, flags, MemoryProtection.ReadWrite);
+
+            if (ptr == IntPtr.Zero)
+            {
+                throw new OutOfMemoryException();
+            }
+
+            return ptr;
+        }
+
+        public static bool Reprotect(IntPtr address, IntPtr size, Memory.MemoryProtection protection)
+        {
+            MemoryProtection prot = GetProtection(protection);
+
+            return VirtualProtect(address, size, prot, out _);
+        }
+
+        private static MemoryProtection GetProtection(Memory.MemoryProtection protection)
+        {
+            switch (protection)
+            {
+                case Memory.MemoryProtection.None:             return MemoryProtection.NoAccess;
+                case Memory.MemoryProtection.Read:             return MemoryProtection.ReadOnly;
+                case Memory.MemoryProtection.ReadAndWrite:     return MemoryProtection.ReadWrite;
+                case Memory.MemoryProtection.ReadAndExecute:   return MemoryProtection.ExecuteRead;
+                case Memory.MemoryProtection.ReadWriteExecute: return MemoryProtection.ExecuteReadWrite;
+                case Memory.MemoryProtection.Execute:          return MemoryProtection.Execute;
+
+                default: throw new ArgumentException($"Invalid permission \"{protection}\".");
+            }
+        }
+
+        public static bool Free(IntPtr address)
+        {
+            return VirtualFree(address, IntPtr.Zero, AllocationType.Release);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static bool GetModifiedPages(
+            IntPtr    address,
+            IntPtr    size,
+            IntPtr[]  addresses,
+            out ulong count)
+        {
+            ulong pagesCount = (ulong)addresses.Length;
+
+            int result = GetWriteWatch(
+                WriteWatchFlags.Reset,
+                address,
+                size,
+                addresses,
+                ref pagesCount,
+                out uint granularity);
+
+            count = pagesCount;
+
+            return result == 0;
+        }
+    }
+}

ARMeilleure/Memory/MemoryManager.cs

@@ -0,0 +1,835 @@
+using ARMeilleure.State;
+using System;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+using static ARMeilleure.Memory.MemoryManagement;
+
+namespace ARMeilleure.Memory
+{
+    public unsafe class MemoryManager : IMemoryManager
+    {
+        public const int PageBits = 12;
+        public const int PageSize = 1 << PageBits;
+        public const int PageMask = PageSize - 1;
+
+        private const long PteFlagNotModified = 1;
+
+        internal const long PteFlagsMask = 7;
+
+        public IntPtr Ram { get; private set; }
+
+        private byte* _ramPtr;
+
+        private IntPtr _pageTable;
+
+        internal IntPtr PageTable => _pageTable;
+
+        internal int PtLevelBits { get; }
+        internal int PtLevelSize { get; }
+        internal int PtLevelMask { get; }
+
+        public bool HasWriteWatchSupport => MemoryManagement.HasWriteWatchSupport;
+
+        public int  AddressSpaceBits { get; }
+        public long AddressSpaceSize { get; }
+
+        public MemoryManager(
+            IntPtr ram,
+            int    addressSpaceBits = 48,
+            bool   useFlatPageTable = false)
+        {
+            Ram = ram;
+
+            _ramPtr = (byte*)ram;
+
+            AddressSpaceBits = addressSpaceBits;
+            AddressSpaceSize = 1L << addressSpaceBits;
+
+            // When flat page table is requested, we use a single
+            // array for the mappings of the entire address space.
+            // This has better performance, but also high memory usage.
+            // The multi level page table uses 9 bits per level, so
+            // the memory usage is lower, but the performance is also
+            // lower, since each address translation requires multiple reads.
+            if (useFlatPageTable)
+            {
+                PtLevelBits = addressSpaceBits - PageBits;
+            }
+            else
+            {
+                PtLevelBits = 9;
+            }
+
+            PtLevelSize = 1 << PtLevelBits;
+            PtLevelMask = PtLevelSize - 1;
+
+            _pageTable = Allocate((ulong)(PtLevelSize * IntPtr.Size));
+        }
+
+        public void Map(long va, long pa, long size)
+        {
+            SetPtEntries(va, _ramPtr + pa, size);
+        }
+
+        public void Unmap(long position, long size)
+        {
+            SetPtEntries(position, null, size);
+        }
+
+        public bool IsMapped(long position)
+        {
+            return Translate(position) != IntPtr.Zero;
+        }
+
+        public long GetPhysicalAddress(long virtualAddress)
+        {
+            byte* ptr = (byte*)Translate(virtualAddress);
+
+            return (long)(ptr - _ramPtr);
+        }
+
+        private IntPtr Translate(long position)
+        {
+            if (!IsValidPosition(position))
+            {
+                return IntPtr.Zero;
+            }
+
+            byte* ptr = GetPtEntry(position);
+
+            ulong ptrUlong = (ulong)ptr;
+
+            if ((ptrUlong & PteFlagsMask) != 0)
+            {
+                ptrUlong &= ~(ulong)PteFlagsMask;
+
+                ptr = (byte*)ptrUlong;
+            }
+
+            return new IntPtr(ptr + (position & PageMask));
+        }
+
+        private IntPtr TranslateWrite(long position)
+        {
+            if (!IsValidPosition(position))
+            {
+                return IntPtr.Zero;
+            }
+
+            byte* ptr = GetPtEntry(position);
+
+            ulong ptrUlong = (ulong)ptr;
+
+            if ((ptrUlong & PteFlagsMask) != 0)
+            {
+                if ((ptrUlong & PteFlagNotModified) != 0)
+                {
+                    ClearPtEntryFlag(position, PteFlagNotModified);
+                }
+
+                ptrUlong &= ~(ulong)PteFlagsMask;
+
+                ptr = (byte*)ptrUlong;
+            }
+
+            return new IntPtr(ptr + (position & PageMask));
+        }
+
+        private byte* GetPtEntry(long position)
+        {
+            return *(byte**)GetPtPtr(position);
+        }
+
+        private void SetPtEntries(long va, byte* ptr, long size)
+        {
+            long endPosition = (va + size + PageMask) & ~PageMask;
+
+            while ((ulong)va < (ulong)endPosition)
+            {
+                SetPtEntry(va, ptr);
+
+                va += PageSize;
+
+                if (ptr != null)
+                {
+                    ptr += PageSize;
+                }
+            }
+        }
+
+        private void SetPtEntry(long position, byte* ptr)
+        {
+            *(byte**)GetPtPtr(position) = ptr;
+        }
+
+        private void SetPtEntryFlag(long position, long flag)
+        {
+            ModifyPtEntryFlag(position, flag, setFlag: true);
+        }
+
+        private void ClearPtEntryFlag(long position, long flag)
+        {
+            ModifyPtEntryFlag(position, flag, setFlag: false);
+        }
+
+        private void ModifyPtEntryFlag(long position, long flag, bool setFlag)
+        {
+            IntPtr* pt = (IntPtr*)_pageTable;
+
+            while (true)
+            {
+                IntPtr* ptPtr = GetPtPtr(position);
+
+                IntPtr old = *ptPtr;
+
+                long modified = old.ToInt64();
+
+                if (setFlag)
+                {
+                    modified |= flag;
+                }
+                else
+                {
+                    modified &= ~flag;
+                }
+
+                IntPtr origValue = Interlocked.CompareExchange(ref *ptPtr, new IntPtr(modified), old);
+
+                if (origValue == old)
+                {
+                    break;
+                }
+            }
+        }
+
+        private IntPtr* GetPtPtr(long position)
+        {
+            if (!IsValidPosition(position))
+            {
+                throw new ArgumentOutOfRangeException(nameof(position));
+            }
+
+            IntPtr nextPtr = _pageTable;
+
+            IntPtr* ptePtr = null;
+
+            int bit = PageBits;
+
+            while (true)
+            {
+                long index = (position >> bit) & PtLevelMask;
+
+                ptePtr = &((IntPtr*)nextPtr)[index];
+
+                bit += PtLevelBits;
+
+                if (bit >= AddressSpaceBits)
+                {
+                    break;
+                }
+
+                nextPtr = *ptePtr;
+
+                if (nextPtr == IntPtr.Zero)
+                {
+                    // Entry does not yet exist, allocate a new one.
+                    IntPtr newPtr = Allocate((ulong)(PtLevelSize * IntPtr.Size));
+
+                    // Try to swap the current pointer (should be zero), with the allocated one.
+                    nextPtr = Interlocked.CompareExchange(ref *ptePtr, newPtr, IntPtr.Zero);
+
+                    // If the old pointer is not null, then another thread already has set it.
+                    if (nextPtr != IntPtr.Zero)
+                    {
+                        Free(newPtr);
+                    }
+                    else
+                    {
+                        nextPtr = newPtr;
+                    }
+                }
+            }
+
+            return ptePtr;
+        }
+
+        public bool IsRegionModified(long position, long size)
+        {
+            if (!HasWriteWatchSupport)
+            {
+                return IsRegionModifiedFallback(position, size);
+            }
+
+            IntPtr address = Translate(position);
+
+            IntPtr baseAddr     = address;
+            IntPtr expectedAddr = address;
+
+            long pendingPages = 0;
+
+            long pages = size / PageSize;
+
+            bool modified = false;
+
+            bool IsAnyPageModified()
+            {
+                IntPtr pendingSize = new IntPtr(pendingPages * PageSize);
+
+                IntPtr[] addresses = new IntPtr[pendingPages];
+
+                bool result = GetModifiedPages(baseAddr, pendingSize, addresses, out ulong count);
+
+                if (result)
+                {
+                    return count != 0;
+                }
+                else
+                {
+                    return true;
+                }
+            }
+
+            while (pages-- > 0)
+            {
+                if (address != expectedAddr)
+                {
+                    modified |= IsAnyPageModified();
+
+                    baseAddr = address;
+
+                    pendingPages = 0;
+                }
+
+                expectedAddr = address + PageSize;
+
+                pendingPages++;
+
+                if (pages == 0)
+                {
+                    break;
+                }
+
+                position += PageSize;
+
+                address = Translate(position);
+            }
+
+            if (pendingPages != 0)
+            {
+                modified |= IsAnyPageModified();
+            }
+
+            return modified;
+        }
+
+        private unsafe bool IsRegionModifiedFallback(long position, long size)
+        {
+            long endAddr = (position + size + PageMask) & ~PageMask;
+
+            bool modified = false;
+
+            while ((ulong)position < (ulong)endAddr)
+            {
+                if (IsValidPosition(position))
+                {
+                    byte* ptr = ((byte**)_pageTable)[position >> PageBits];
+
+                    ulong ptrUlong = (ulong)ptr;
+
+                    if ((ptrUlong & PteFlagNotModified) == 0)
+                    {
+                        modified = true;
+
+                        SetPtEntryFlag(position, PteFlagNotModified);
+                    }
+                }
+                else
+                {
+                    modified = true;
+                }
+
+                position += PageSize;
+            }
+
+            return modified;
+        }
+
+        public bool TryGetHostAddress(long position, long size, out IntPtr ptr)
+        {
+            if (IsContiguous(position, size))
+            {
+                ptr = (IntPtr)Translate(position);
+
+                return true;
+            }
+
+            ptr = IntPtr.Zero;
+
+            return false;
+        }
+
+        private bool IsContiguous(long position, long size)
+        {
+            long endPos = position + size;
+
+            position &= ~PageMask;
+
+            long expectedPa = GetPhysicalAddress(position);
+
+            while ((ulong)position < (ulong)endPos)
+            {
+                long pa = GetPhysicalAddress(position);
+
+                if (pa != expectedPa)
+                {
+                    return false;
+                }
+
+                position   += PageSize;
+                expectedPa += PageSize;
+            }
+
+            return true;
+        }
+
+        public bool IsValidPosition(long position)
+        {
+            return (ulong)position < (ulong)AddressSpaceSize;
+        }
+
+        internal V128 AtomicLoadInt128(long position)
+        {
+            if ((position & 0xf) != 0)
+            {
+                AbortWithAlignmentFault(position);
+            }
+
+            IntPtr ptr = TranslateWrite(position);
+
+            return MemoryManagerPal.AtomicLoad128(ptr);
+        }
+
+        internal bool AtomicCompareExchangeByte(long position, byte expected, byte desired)
+        {
+            int* ptr = (int*)Translate(position);
+
+            int currentValue = *ptr;
+
+            int expected32 = (currentValue & ~byte.MaxValue) | expected;
+            int desired32  = (currentValue & ~byte.MaxValue) | desired;
+
+            return Interlocked.CompareExchange(ref *ptr, desired32, expected32) == expected32;
+        }
+
+        internal bool AtomicCompareExchangeInt16(long position, short expected, short desired)
+        {
+            if ((position & 1) != 0)
+            {
+                AbortWithAlignmentFault(position);
+            }
+
+            int* ptr = (int*)Translate(position);
+
+            int currentValue = *ptr;
+
+            int expected32 = (currentValue & ~ushort.MaxValue) | (ushort)expected;
+            int desired32  = (currentValue & ~ushort.MaxValue) | (ushort)desired;
+
+            return Interlocked.CompareExchange(ref *ptr, desired32, expected32) == expected32;
+        }
+
+        public bool AtomicCompareExchangeInt32(long position, int expected, int desired)
+        {
+            if ((position & 3) != 0)
+            {
+                AbortWithAlignmentFault(position);
+            }
+
+            int* ptr = (int*)TranslateWrite(position);
+
+            return Interlocked.CompareExchange(ref *ptr, desired, expected) == expected;
+        }
+
+        internal bool AtomicCompareExchangeInt64(long position, long expected, long desired)
+        {
+            if ((position & 7) != 0)
+            {
+                AbortWithAlignmentFault(position);
+            }
+
+            long* ptr = (long*)TranslateWrite(position);
+
+            return Interlocked.CompareExchange(ref *ptr, desired, expected) == expected;
+        }
+
+        internal bool AtomicCompareExchangeInt128(long position, V128 expected, V128 desired)
+        {
+            if ((position & 0xf) != 0)
+            {
+                AbortWithAlignmentFault(position);
+            }
+
+            IntPtr ptr = TranslateWrite(position);
+
+            return MemoryManagerPal.CompareAndSwap128(ptr, expected, desired) == expected;
+        }
+
+        public int AtomicIncrementInt32(long position)
+        {
+            if ((position & 3) != 0)
+            {
+                AbortWithAlignmentFault(position);
+            }
+
+            int* ptr = (int*)TranslateWrite(position);
+
+            return Interlocked.Increment(ref *ptr);
+        }
+
+        public int AtomicDecrementInt32(long position)
+        {
+            if ((position & 3) != 0)
+            {
+                AbortWithAlignmentFault(position);
+            }
+
+            int* ptr = (int*)TranslateWrite(position);
+
+            return Interlocked.Decrement(ref *ptr);
+        }
+
+        private void AbortWithAlignmentFault(long position)
+        {
+            // TODO: Abort mode and exception support on the CPU.
+            throw new InvalidOperationException($"Tried to compare exchange a misaligned address 0x{position:X16}.");
+        }
+
+        public sbyte ReadSByte(long position)
+        {
+            return (sbyte)ReadByte(position);
+        }
+
+        public short ReadInt16(long position)
+        {
+            return (short)ReadUInt16(position);
+        }
+
+        public int ReadInt32(long position)
+        {
+            return (int)ReadUInt32(position);
+        }
+
+        public long ReadInt64(long position)
+        {
+            return (long)ReadUInt64(position);
+        }
+
+        public byte ReadByte(long position)
+        {
+            return *((byte*)Translate(position));
+        }
+
+        public ushort ReadUInt16(long position)
+        {
+            if ((position & 1) == 0)
+            {
+                return *((ushort*)Translate(position));
+            }
+            else
+            {
+                return (ushort)(ReadByte(position + 0) << 0 |
+                                ReadByte(position + 1) << 8);
+            }
+        }
+
+        public uint ReadUInt32(long position)
+        {
+            if ((position & 3) == 0)
+            {
+                return *((uint*)Translate(position));
+            }
+            else
+            {
+                return (uint)(ReadUInt16(position + 0) << 0 |
+                              ReadUInt16(position + 2) << 16);
+            }
+        }
+
+        public ulong ReadUInt64(long position)
+        {
+            if ((position & 7) == 0)
+            {
+                return *((ulong*)Translate(position));
+            }
+            else
+            {
+                return (ulong)ReadUInt32(position + 0) << 0 |
+                       (ulong)ReadUInt32(position + 4) << 32;
+            }
+        }
+
+        public V128 ReadVector128(long position)
+        {
+            return new V128(ReadUInt64(position), ReadUInt64(position + 8));
+        }
+
+        public byte[] ReadBytes(long position, long size)
+        {
+            long endAddr = position + size;
+
+            if ((ulong)size > int.MaxValue)
+            {
+                throw new ArgumentOutOfRangeException(nameof(size));
+            }
+
+            if ((ulong)endAddr < (ulong)position)
+            {
+                throw new ArgumentOutOfRangeException(nameof(position));
+            }
+
+            byte[] data = new byte[size];
+
+            int offset = 0;
+
+            while ((ulong)position < (ulong)endAddr)
+            {
+                long pageLimit = (position + PageSize) & ~(long)PageMask;
+
+                if ((ulong)pageLimit > (ulong)endAddr)
+                {
+                    pageLimit = endAddr;
+                }
+
+                int copySize = (int)(pageLimit - position);
+
+                Marshal.Copy(Translate(position), data, offset, copySize);
+
+                position += copySize;
+                offset   += copySize;
+            }
+
+            return data;
+        }
+
+        public void ReadBytes(long position, byte[] data, int startIndex, int size)
+        {
+            // Note: This will be moved later.
+            long endAddr = position + size;
+
+            if ((ulong)size > int.MaxValue)
+            {
+                throw new ArgumentOutOfRangeException(nameof(size));
+            }
+
+            if ((ulong)endAddr < (ulong)position)
+            {
+                throw new ArgumentOutOfRangeException(nameof(position));
+            }
+
+            int offset = startIndex;
+
+            while ((ulong)position < (ulong)endAddr)
+            {
+                long pageLimit = (position + PageSize) & ~(long)PageMask;
+
+                if ((ulong)pageLimit > (ulong)endAddr)
+                {
+                    pageLimit = endAddr;
+                }
+
+                int copySize = (int)(pageLimit - position);
+
+                Marshal.Copy(Translate(position), data, offset, copySize);
+
+                position += copySize;
+                offset   += copySize;
+            }
+        }
+
+        public void WriteSByte(long position, sbyte value)
+        {
+            WriteByte(position, (byte)value);
+        }
+
+        public void WriteInt16(long position, short value)
+        {
+            WriteUInt16(position, (ushort)value);
+        }
+
+        public void WriteInt32(long position, int value)
+        {
+            WriteUInt32(position, (uint)value);
+        }
+
+        public void WriteInt64(long position, long value)
+        {
+            WriteUInt64(position, (ulong)value);
+        }
+
+        public void WriteByte(long position, byte value)
+        {
+            *((byte*)TranslateWrite(position)) = value;
+        }
+
+        public void WriteUInt16(long position, ushort value)
+        {
+            if ((position & 1) == 0)
+            {
+                *((ushort*)TranslateWrite(position)) = value;
+            }
+            else
+            {
+                WriteByte(position + 0, (byte)(value >> 0));
+                WriteByte(position + 1, (byte)(value >> 8));
+            }
+        }
+
+        public void WriteUInt32(long position, uint value)
+        {
+            if ((position & 3) == 0)
+            {
+                *((uint*)TranslateWrite(position)) = value;
+            }
+            else
+            {
+                WriteUInt16(position + 0, (ushort)(value >> 0));
+                WriteUInt16(position + 2, (ushort)(value >> 16));
+            }
+        }
+
+        public void WriteUInt64(long position, ulong value)
+        {
+            if ((position & 7) == 0)
+            {
+                *((ulong*)TranslateWrite(position)) = value;
+            }
+            else
+            {
+                WriteUInt32(position + 0, (uint)(value >> 0));
+                WriteUInt32(position + 4, (uint)(value >> 32));
+            }
+        }
+
+        public void WriteVector128(long position, V128 value)
+        {
+            WriteUInt64(position + 0, value.GetUInt64(0));
+            WriteUInt64(position + 8, value.GetUInt64(1));
+        }
+
+        public void WriteBytes(long position, byte[] data)
+        {
+            long endAddr = position + data.Length;
+
+            if ((ulong)endAddr < (ulong)position)
+            {
+                throw new ArgumentOutOfRangeException(nameof(position));
+            }
+
+            int offset = 0;
+
+            while ((ulong)position < (ulong)endAddr)
+            {
+                long pageLimit = (position + PageSize) & ~(long)PageMask;
+
+                if ((ulong)pageLimit > (ulong)endAddr)
+                {
+                    pageLimit = endAddr;
+                }
+
+                int copySize = (int)(pageLimit - position);
+
+                Marshal.Copy(data, offset, TranslateWrite(position), copySize);
+
+                position += copySize;
+                offset   += copySize;
+            }
+        }
+
+        public void WriteBytes(long position, byte[] data, int startIndex, int size)
+        {
+            // Note: This will be moved later.
+            long endAddr = position + size;
+
+            if ((ulong)endAddr < (ulong)position)
+            {
+                throw new ArgumentOutOfRangeException(nameof(position));
+            }
+
+            int offset = startIndex;
+
+            while ((ulong)position < (ulong)endAddr)
+            {
+                long pageLimit = (position + PageSize) & ~(long)PageMask;
+
+                if ((ulong)pageLimit > (ulong)endAddr)
+                {
+                    pageLimit = endAddr;
+                }
+
+                int copySize = (int)(pageLimit - position);
+
+                Marshal.Copy(data, offset, Translate(position), copySize);
+
+                position += copySize;
+                offset   += copySize;
+            }
+        }
+
+        public void CopyBytes(long src, long dst, long size)
+        {
+            // Note: This will be moved later.
+            if (IsContiguous(src, size) &&
+                IsContiguous(dst, size))
+            {
+                byte* srcPtr = (byte*)Translate(src);
+                byte* dstPtr = (byte*)Translate(dst);
+
+                Buffer.MemoryCopy(srcPtr, dstPtr, size, size);
+            }
+            else
+            {
+                WriteBytes(dst, ReadBytes(src, size));
+            }
+        }
+
+        public void Dispose()
+        {
+            Dispose(true);
+        }
+
+        protected virtual void Dispose(bool disposing)
+        {
+            IntPtr ptr = Interlocked.Exchange(ref _pageTable, IntPtr.Zero);
+
+            if (ptr != IntPtr.Zero)
+            {
+                FreePageTableEntry(ptr, PageBits);
+            }
+        }
+
+        private void FreePageTableEntry(IntPtr ptr, int levelBitEnd)
+        {
+            levelBitEnd += PtLevelBits;
+
+            if (levelBitEnd >= AddressSpaceBits)
+            {
+                Free(ptr);
+
+                return;
+            }
+
+            for (int index = 0; index < PtLevelSize; index++)
+            {
+                IntPtr ptePtr = ((IntPtr*)ptr)[index];
+
+                if (ptePtr != IntPtr.Zero)
+                {
+                    FreePageTableEntry(ptePtr, levelBitEnd);
+                }
+            }
+
+            Free(ptr);
+        }
+    }
+}

ARMeilleure/Memory/MemoryManagerPal.cs

@@ -0,0 +1,77 @@
+using ARMeilleure.IntermediateRepresentation;
+using ARMeilleure.State;
+using ARMeilleure.Translation;
+using System;
+
+namespace ARMeilleure.Memory
+{
+    static class MemoryManagerPal
+    {
+        private delegate V128 CompareExchange128(IntPtr address, V128 expected, V128 desired);
+
+        private static CompareExchange128 _compareExchange128;
+
+        private static object _lock;
+
+        static MemoryManagerPal()
+        {
+            _lock = new object();
+        }
+
+        public static V128 AtomicLoad128(IntPtr address)
+        {
+            return GetCompareAndSwap128()(address, V128.Zero, V128.Zero);
+        }
+
+        public static V128 CompareAndSwap128(IntPtr address, V128 expected, V128 desired)
+        {
+            return GetCompareAndSwap128()(address, expected, desired);
+        }
+
+        private static CompareExchange128 GetCompareAndSwap128()
+        {
+            if (_compareExchange128 == null)
+            {
+                GenerateCompareAndSwap128();
+            }
+
+            return _compareExchange128;
+        }
+
+        private static void GenerateCompareAndSwap128()
+        {
+            lock (_lock)
+            {
+                if (_compareExchange128 != null)
+                {
+                    return;
+                }
+
+                EmitterContext context = new EmitterContext();
+
+                Operand address  = context.LoadArgument(OperandType.I64,  0);
+                Operand expected = context.LoadArgument(OperandType.V128, 1);
+                Operand desired  = context.LoadArgument(OperandType.V128, 2);
+
+                Operand result = context.CompareAndSwap128(address, expected, desired);
+
+                context.Return(result);
+
+                ControlFlowGraph cfg = context.GetControlFlowGraph();
+
+                OperandType[] argTypes = new OperandType[]
+                {
+                    OperandType.I64,
+                    OperandType.V128,
+                    OperandType.V128
+                };
+
+                _compareExchange128 = Compiler.Compile<CompareExchange128>(
+                    cfg,
+                    argTypes,
+                    OperandType.V128,
+                    CompilerOptions.HighCq);
+            }
+        }
+    }
+}

ARMeilleure/Memory/MemoryProtection.cs

@@ -0,0 +1,17 @@
+using System;
+
+namespace ARMeilleure.Memory
+{
+    [Flags]
+    public enum MemoryProtection
+    {
+        None    = 0,
+        Read    = 1 << 0,
+        Write   = 1 << 1,
+        Execute = 1 << 2,
+
+        ReadAndWrite     = Read | Write,
+        ReadAndExecute   = Read | Execute,
+        ReadWriteExecute = Read | Write | Execute
+    }
+}

ARMeilleure/Memory/MemoryProtectionException.cs

@@ -0,0 +1,9 @@
+using System;
+
+namespace ARMeilleure.Memory
+{
+    class MemoryProtectionException : Exception
+    {
+        public MemoryProtectionException(MemoryProtection protection) :  base($"Failed to set memory protection to \"{protection}\".") { }
+    }
+}