Merge pull request #12074 from GPUCode/yuwu-on-the-metal

Implement Native Code Execution (NCE)

src/common/CMakeLists.txt

@@ -52,6 +52,7 @@ add_library(common STATIC
     fiber.cpp
     fiber.h
     fixed_point.h
+    free_region_manager.h
     fs/file.cpp
     fs/file.h
     fs/fs.cpp
@@ -166,6 +167,13 @@ if (WIN32)
   target_link_libraries(common PRIVATE ntdll)
 endif()
 
+if (NOT WIN32)
+  target_sources(common PRIVATE
+    signal_chain.cpp
+    signal_chain.h
+  )
+endif()
+
 if(ANDROID)
     target_sources(common
         PRIVATE
@@ -200,7 +208,7 @@ if(ARCHITECTURE_x86_64)
     target_link_libraries(common PRIVATE xbyak::xbyak)
 endif()
 
-if (ARCHITECTURE_arm64 AND (ANDROID OR LINUX))
+if (HAS_NCE)
     target_sources(common
         PRIVATE
             arm64/native_clock.cpp

src/common/free_region_manager.h

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include <mutex>
+#include <boost/icl/interval_set.hpp>
+
+namespace Common {
+
+class FreeRegionManager {
+public:
+    explicit FreeRegionManager() = default;
+    ~FreeRegionManager() = default;
+
+    void SetAddressSpace(void* start, size_t size) {
+        this->FreeBlock(start, size);
+    }
+
+    std::pair<void*, size_t> FreeBlock(void* block_ptr, size_t size) {
+        std::scoped_lock lk(m_mutex);
+
+        // Check to see if we are adjacent to any regions.
+        auto start_address = reinterpret_cast<uintptr_t>(block_ptr);
+        auto end_address = start_address + size;
+        auto it = m_free_regions.find({start_address - 1, end_address + 1});
+
+        // If we are, join with them, ensuring we stay in bounds.
+        if (it != m_free_regions.end()) {
+            start_address = std::min(start_address, it->lower());
+            end_address = std::max(end_address, it->upper());
+        }
+
+        // Free the relevant region.
+        m_free_regions.insert({start_address, end_address});
+
+        // Return the adjusted pointers.
+        block_ptr = reinterpret_cast<void*>(start_address);
+        size = end_address - start_address;
+        return {block_ptr, size};
+    }
+
+    void AllocateBlock(void* block_ptr, size_t size) {
+        std::scoped_lock lk(m_mutex);
+
+        auto address = reinterpret_cast<uintptr_t>(block_ptr);
+        m_free_regions.subtract({address, address + size});
+    }
+
+private:
+    std::mutex m_mutex;
+    boost::icl::interval_set<uintptr_t> m_free_regions;
+};
+
+} // namespace Common

src/common/host_memory.cpp

@@ -21,15 +21,18 @@
 #include <boost/icl/interval_set.hpp>
 #include <fcntl.h>
 #include <sys/mman.h>
+#include <sys/random.h>
 #include <unistd.h>
 #include "common/scope_exit.h"
 
 #endif // ^^^ Linux ^^^
 
 #include <mutex>
+#include <random>
 
 #include "common/alignment.h"
 #include "common/assert.h"
+#include "common/free_region_manager.h"
 #include "common/host_memory.h"
 #include "common/logging/log.h"
 
@@ -141,7 +144,7 @@ public:
         Release();
     }
 
-    void Map(size_t virtual_offset, size_t host_offset, size_t length) {
+    void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) {
         std::unique_lock lock{placeholder_mutex};
         if (!IsNiechePlaceholder(virtual_offset, length)) {
             Split(virtual_offset, length);
@@ -160,7 +163,7 @@ public:
         }
     }
 
-    void Protect(size_t virtual_offset, size_t length, bool read, bool write) {
+    void Protect(size_t virtual_offset, size_t length, bool read, bool write, bool execute) {
         DWORD new_flags{};
         if (read && write) {
             new_flags = PAGE_READWRITE;
@@ -186,6 +189,11 @@ public:
         }
     }
 
+    void EnableDirectMappedAddress() {
+        // TODO
+        UNREACHABLE();
+    }
+
     const size_t backing_size; ///< Size of the backing memory in bytes
     const size_t virtual_size; ///< Size of the virtual address placeholder in bytes
 
@@ -353,6 +361,55 @@ private:
 
 #elif defined(__linux__) || defined(__FreeBSD__) // ^^^ Windows ^^^ vvv Linux vvv
 
+#ifdef ARCHITECTURE_arm64
+
+static void* ChooseVirtualBase(size_t virtual_size) {
+    constexpr uintptr_t Map39BitSize = (1ULL << 39);
+    constexpr uintptr_t Map36BitSize = (1ULL << 36);
+
+    // This is not a cryptographic application, we just want something random.
+    std::mt19937_64 rng;
+
+    // We want to ensure we are allocating at an address aligned to the L2 block size.
+    // For Qualcomm devices, we must also allocate memory above 36 bits.
+    const size_t lower = Map36BitSize / HugePageSize;
+    const size_t upper = (Map39BitSize - virtual_size) / HugePageSize;
+    const size_t range = upper - lower;
+
+    // Try up to 64 times to allocate memory at random addresses in the range.
+    for (int i = 0; i < 64; i++) {
+        // Calculate a possible location.
+        uintptr_t hint_address = ((rng() % range) + lower) * HugePageSize;
+
+        // Try to map.
+        // Note: we may be able to take advantage of MAP_FIXED_NOREPLACE here.
+        void* map_pointer =
+            mmap(reinterpret_cast<void*>(hint_address), virtual_size, PROT_READ | PROT_WRITE,
+                 MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+
+        // If we successfully mapped, we're done.
+        if (reinterpret_cast<uintptr_t>(map_pointer) == hint_address) {
+            return map_pointer;
+        }
+
+        // Unmap if necessary, and try again.
+        if (map_pointer != MAP_FAILED) {
+            munmap(map_pointer, virtual_size);
+        }
+    }
+
+    return MAP_FAILED;
+}
+
+#else
+
+static void* ChooseVirtualBase(size_t virtual_size) {
+    return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE,
+                MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+}
+
+#endif
+
 class HostMemory::Impl {
 public:
     explicit Impl(size_t backing_size_, size_t virtual_size_)
@@ -415,8 +472,7 @@ public:
             }
         }
 #else
-        virtual_base = static_cast<u8*>(mmap(nullptr, virtual_size, PROT_NONE,
-                                             MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0));
+        virtual_base = virtual_map_base = static_cast<u8*>(ChooseVirtualBase(virtual_size));
         if (virtual_base == MAP_FAILED) {
             LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno));
             throw std::bad_alloc{};
@@ -424,7 +480,7 @@ public:
         madvise(virtual_base, virtual_size, MADV_HUGEPAGE);
 #endif
 
-        placeholders.add({0, virtual_size});
+        free_manager.SetAddressSpace(virtual_base, virtual_size);
         good = true;
     }
 
@@ -432,14 +488,29 @@ public:
         Release();
     }
 
-    void Map(size_t virtual_offset, size_t host_offset, size_t length) {
-        {
-            std::scoped_lock lock{placeholder_mutex};
-            placeholders.subtract({virtual_offset, virtual_offset + length});
-        }
+    void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) {
+        // Intersect the range with our address space.
+        AdjustMap(&virtual_offset, &length);
 
-        void* ret = mmap(virtual_base + virtual_offset, length, PROT_READ | PROT_WRITE,
-                         MAP_SHARED | MAP_FIXED, fd, host_offset);
+        // We are removing a placeholder.
+        free_manager.AllocateBlock(virtual_base + virtual_offset, length);
+
+        // Deduce mapping protection flags.
+        int flags = PROT_NONE;
+        if (True(perms & MemoryPermission::Read)) {
+            flags |= PROT_READ;
+        }
+        if (True(perms & MemoryPermission::Write)) {
+            flags |= PROT_WRITE;
+        }
+#ifdef ARCHITECTURE_arm64
+        if (True(perms & MemoryPermission::Execute)) {
+            flags |= PROT_EXEC;
+        }
+#endif
+
+        void* ret = mmap(virtual_base + virtual_offset, length, flags, MAP_SHARED | MAP_FIXED, fd,
+                         host_offset);
         ASSERT_MSG(ret != MAP_FAILED, "mmap failed: {}", strerror(errno));
     }
 
@@ -447,47 +518,54 @@ public:
         // The method name is wrong. We're still talking about the virtual range.
         // We don't want to unmap, we want to reserve this memory.
 
-        {
-            std::scoped_lock lock{placeholder_mutex};
-            auto it = placeholders.find({virtual_offset - 1, virtual_offset + length + 1});
+        // Intersect the range with our address space.
+        AdjustMap(&virtual_offset, &length);
 
-            if (it != placeholders.end()) {
-                size_t prev_upper = virtual_offset + length;
-                virtual_offset = std::min(virtual_offset, it->lower());
-                length = std::max(it->upper(), prev_upper) - virtual_offset;
-            }
+        // Merge with any adjacent placeholder mappings.
+        auto [merged_pointer, merged_size] =
+            free_manager.FreeBlock(virtual_base + virtual_offset, length);
 
-            placeholders.add({virtual_offset, virtual_offset + length});
-        }
-
-        void* ret = mmap(virtual_base + virtual_offset, length, PROT_NONE,
+        void* ret = mmap(merged_pointer, merged_size, PROT_NONE,
                          MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
         ASSERT_MSG(ret != MAP_FAILED, "mmap failed: {}", strerror(errno));
     }
 
-    void Protect(size_t virtual_offset, size_t length, bool read, bool write) {
-        int flags = 0;
+    void Protect(size_t virtual_offset, size_t length, bool read, bool write, bool execute) {
+        // Intersect the range with our address space.
+        AdjustMap(&virtual_offset, &length);
+
+        int flags = PROT_NONE;
         if (read) {
             flags |= PROT_READ;
         }
         if (write) {
             flags |= PROT_WRITE;
         }
+#ifdef HAS_NCE
+        if (execute) {
+            flags |= PROT_EXEC;
+        }
+#endif
         int ret = mprotect(virtual_base + virtual_offset, length, flags);
         ASSERT_MSG(ret == 0, "mprotect failed: {}", strerror(errno));
     }
 
+    void EnableDirectMappedAddress() {
+        virtual_base = nullptr;
+    }
+
     const size_t backing_size; ///< Size of the backing memory in bytes
     const size_t virtual_size; ///< Size of the virtual address placeholder in bytes
 
     u8* backing_base{reinterpret_cast<u8*>(MAP_FAILED)};
     u8* virtual_base{reinterpret_cast<u8*>(MAP_FAILED)};
+    u8* virtual_map_base{reinterpret_cast<u8*>(MAP_FAILED)};
 
 private:
     /// Release all resources in the object
     void Release() {
-        if (virtual_base != MAP_FAILED) {
-            int ret = munmap(virtual_base, virtual_size);
+        if (virtual_map_base != MAP_FAILED) {
+            int ret = munmap(virtual_map_base, virtual_size);
             ASSERT_MSG(ret == 0, "munmap failed: {}", strerror(errno));
         }
 
@@ -502,10 +580,29 @@ private:
         }
     }
 
-    int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create
+    void AdjustMap(size_t* virtual_offset, size_t* length) {
+        if (virtual_base != nullptr) {
+            return;
+        }
 
-    boost::icl::interval_set<size_t> placeholders; ///< Mapped placeholders
-    std::mutex placeholder_mutex;                  ///< Mutex for placeholders
+        // If we are direct mapped, we want to make sure we are operating on a region
+        // that is in range of our virtual mapping.
+        size_t intended_start = *virtual_offset;
+        size_t intended_end = intended_start + *length;
+        size_t address_space_start = reinterpret_cast<size_t>(virtual_map_base);
+        size_t address_space_end = address_space_start + virtual_size;
+
+        if (address_space_start > intended_end || intended_start > address_space_end) {
+            *virtual_offset = 0;
+            *length = 0;
+        } else {
+            *virtual_offset = std::max(intended_start, address_space_start);
+            *length = std::min(intended_end, address_space_end) - *virtual_offset;
+        }
+    }
+
+    int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create
+    FreeRegionManager free_manager{};
 };
 
 #else // ^^^ Linux ^^^ vvv Generic vvv
@@ -518,11 +615,13 @@ public:
         throw std::bad_alloc{};
     }
 
-    void Map(size_t virtual_offset, size_t host_offset, size_t length) {}
+    void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perm) {}
 
     void Unmap(size_t virtual_offset, size_t length) {}
 
-    void Protect(size_t virtual_offset, size_t length, bool read, bool write) {}
+    void Protect(size_t virtual_offset, size_t length, bool read, bool write, bool execute) {}
+
+    void EnableDirectMappedAddress() {}
 
     u8* backing_base{nullptr};
     u8* virtual_base{nullptr};
@@ -535,15 +634,16 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_)
     try {
         // Try to allocate a fastmem arena.
         // The implementation will fail with std::bad_alloc on errors.
-        impl = std::make_unique<HostMemory::Impl>(AlignUp(backing_size, PageAlignment),
-                                                  AlignUp(virtual_size, PageAlignment) +
-                                                      3 * HugePageSize);
+        impl =
+            std::make_unique<HostMemory::Impl>(AlignUp(backing_size, PageAlignment),
+                                               AlignUp(virtual_size, PageAlignment) + HugePageSize);
         backing_base = impl->backing_base;
         virtual_base = impl->virtual_base;
 
         if (virtual_base) {
-            virtual_base += 2 * HugePageSize - 1;
-            virtual_base -= reinterpret_cast<size_t>(virtual_base) & (HugePageSize - 1);
+            // Ensure the virtual base is aligned to the L2 block size.
+            virtual_base = reinterpret_cast<u8*>(
+                Common::AlignUp(reinterpret_cast<uintptr_t>(virtual_base), HugePageSize));
             virtual_base_offset = virtual_base - impl->virtual_base;
         }
 
@@ -562,7 +662,8 @@ HostMemory::HostMemory(HostMemory&&) noexcept = default;
 
 HostMemory& HostMemory::operator=(HostMemory&&) noexcept = default;
 
-void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length) {
+void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length,
+                     MemoryPermission perms) {
     ASSERT(virtual_offset % PageAlignment == 0);
     ASSERT(host_offset % PageAlignment == 0);
     ASSERT(length % PageAlignment == 0);
@@ -571,7 +672,7 @@ void HostMemory::Map(size_t virtual_offset, size_t host_offset, size_t length) {
     if (length == 0 || !virtual_base || !impl) {
         return;
     }
-    impl->Map(virtual_offset + virtual_base_offset, host_offset, length);
+    impl->Map(virtual_offset + virtual_base_offset, host_offset, length, perms);
 }
 
 void HostMemory::Unmap(size_t virtual_offset, size_t length) {
@@ -584,14 +685,22 @@ void HostMemory::Unmap(size_t virtual_offset, size_t length) {
     impl->Unmap(virtual_offset + virtual_base_offset, length);
 }
 
-void HostMemory::Protect(size_t virtual_offset, size_t length, bool read, bool write) {
+void HostMemory::Protect(size_t virtual_offset, size_t length, bool read, bool write,
+                         bool execute) {
     ASSERT(virtual_offset % PageAlignment == 0);
     ASSERT(length % PageAlignment == 0);
     ASSERT(virtual_offset + length <= virtual_size);
     if (length == 0 || !virtual_base || !impl) {
         return;
     }
-    impl->Protect(virtual_offset + virtual_base_offset, length, read, write);
+    impl->Protect(virtual_offset + virtual_base_offset, length, read, write, execute);
+}
+
+void HostMemory::EnableDirectMappedAddress() {
+    if (impl) {
+        impl->EnableDirectMappedAddress();
+        virtual_size += reinterpret_cast<uintptr_t>(virtual_base);
+    }
 }
 
 } // namespace Common

src/common/host_memory.h

@@ -4,11 +4,20 @@
 #pragma once
 
 #include <memory>
+#include "common/common_funcs.h"
 #include "common/common_types.h"
 #include "common/virtual_buffer.h"
 
 namespace Common {
 
+enum class MemoryPermission : u32 {
+    Read = 1 << 0,
+    Write = 1 << 1,
+    ReadWrite = Read | Write,
+    Execute = 1 << 2,
+};
+DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission)
+
 /**
  * A low level linear memory buffer, which supports multiple mappings
  * Its purpose is to rebuild a given sparse memory layout, including mirrors.
@@ -31,11 +40,13 @@ public:
     HostMemory(HostMemory&& other) noexcept;
     HostMemory& operator=(HostMemory&& other) noexcept;
 
-    void Map(size_t virtual_offset, size_t host_offset, size_t length);
+    void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms);
 
     void Unmap(size_t virtual_offset, size_t length);
 
-    void Protect(size_t virtual_offset, size_t length, bool read, bool write);
+    void Protect(size_t virtual_offset, size_t length, bool read, bool write, bool execute = false);
+
+    void EnableDirectMappedAddress();
 
     [[nodiscard]] u8* BackingBasePointer() noexcept {
         return backing_base;

src/common/settings.cpp

@@ -41,6 +41,7 @@ SWITCHABLE(AspectRatio, true);
 SWITCHABLE(AstcDecodeMode, true);
 SWITCHABLE(AstcRecompression, true);
 SWITCHABLE(AudioMode, true);
+SWITCHABLE(CpuBackend, true);
 SWITCHABLE(CpuAccuracy, true);
 SWITCHABLE(FullscreenMode, true);
 SWITCHABLE(GpuAccuracy, true);
@@ -155,6 +156,22 @@ bool IsFastmemEnabled() {
     return true;
 }
 
+static bool is_nce_enabled = false;
+
+void SetNceEnabled(bool is_39bit) {
+    const bool is_nce_selected = values.cpu_backend.GetValue() == CpuBackend::Nce;
+    is_nce_enabled = IsFastmemEnabled() && is_nce_selected && is_39bit;
+    if (is_nce_selected && !is_nce_enabled) {
+        LOG_WARNING(
+            Common,
+            "Program does not utilize 39-bit address space, unable to natively execute code");
+    }
+}
+
+bool IsNceEnabled() {
+    return is_nce_enabled;
+}
+
 bool IsDockedMode() {
     return values.use_docked_mode.GetValue() == Settings::ConsoleMode::Docked;
 }

src/common/settings.h

@@ -63,6 +63,7 @@ SWITCHABLE(AspectRatio, true);
 SWITCHABLE(AstcDecodeMode, true);
 SWITCHABLE(AstcRecompression, true);
 SWITCHABLE(AudioMode, true);
+SWITCHABLE(CpuBackend, true);
 SWITCHABLE(CpuAccuracy, true);
 SWITCHABLE(FullscreenMode, true);
 SWITCHABLE(GpuAccuracy, true);
@@ -179,6 +180,14 @@ struct Values {
                                              &use_speed_limit};
 
     // Cpu
+    SwitchableSetting<CpuBackend, true> cpu_backend{
+        linkage,         CpuBackend::Dynarmic, CpuBackend::Dynarmic,
+#ifdef HAS_NCE
+        CpuBackend::Nce,
+#else
+                                                    CpuBackend::Dynarmic,
+#endif
+        "cpu_backend",   Category::Cpu};
     SwitchableSetting<CpuAccuracy, true> cpu_accuracy{linkage,           CpuAccuracy::Auto,
                                                       CpuAccuracy::Auto, CpuAccuracy::Paranoid,
                                                       "cpu_accuracy",    Category::Cpu};
@@ -569,6 +578,8 @@ bool IsGPULevelExtreme();
 bool IsGPULevelHigh();
 
 bool IsFastmemEnabled();
+void SetNceEnabled(bool is_64bit);
+bool IsNceEnabled();
 
 bool IsDockedMode();
 

src/common/settings_enums.h

@@ -129,6 +129,8 @@ ENUM(ShaderBackend, Glsl, Glasm, SpirV);
 
 ENUM(GpuAccuracy, Normal, High, Extreme);
 
+ENUM(CpuBackend, Dynarmic, Nce);
+
 ENUM(CpuAccuracy, Auto, Accurate, Unsafe, Paranoid);
 
 ENUM(MemoryLayout, Memory_4Gb, Memory_6Gb, Memory_8Gb);

src/common/signal_chain.cpp

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <dlfcn.h>
+
+#include "common/assert.h"
+#include "common/dynamic_library.h"
+#include "common/scope_exit.h"
+#include "common/signal_chain.h"
+
+namespace Common {
+
+template <typename T>
+T* LookupLibcSymbol(const char* name) {
+#if defined(__BIONIC__)
+    Common::DynamicLibrary provider("libc.so");
+    if (!provider.IsOpen()) {
+        UNREACHABLE_MSG("Failed to open libc!");
+    }
+#else
+    // For other operating environments, we assume the symbol is not overridden.
+    const char* base = nullptr;
+    Common::DynamicLibrary provider(base);
+#endif
+
+    void* sym = provider.GetSymbolAddress(name);
+    if (sym == nullptr) {
+        sym = dlsym(RTLD_DEFAULT, name);
+    }
+    if (sym == nullptr) {
+        UNREACHABLE_MSG("Unable to find symbol {}!", name);
+    }
+
+    return reinterpret_cast<T*>(sym);
+}
+
+int SigAction(int signum, const struct sigaction* act, struct sigaction* oldact) {
+    static auto libc_sigaction = LookupLibcSymbol<decltype(sigaction)>("sigaction");
+    return libc_sigaction(signum, act, oldact);
+}
+
+} // namespace Common

src/common/signal_chain.h

@@ -0,0 +1,19 @@
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#ifndef _WIN32
+
+#include <signal.h>
+
+namespace Common {
+
+// Android's ART overrides sigaction with its own wrapper. This is problematic for SIGSEGV
+// in particular, because ART's handler accesses tpidr_el0, which conflicts with NCE.
+// This extracts the libc symbol and calls it directly.
+int SigAction(int signum, const struct sigaction* act, struct sigaction* oldact);
+
+} // namespace Common
+
+#endif

src/common/wall_clock.cpp

@@ -10,7 +10,7 @@
 #include "common/x64/rdtsc.h"
 #endif
 
-#if defined(ARCHITECTURE_arm64) && defined(__linux__)
+#ifdef HAS_NCE
 #include "common/arm64/native_clock.h"
 #endif
 
@@ -68,7 +68,7 @@ std::unique_ptr<WallClock> CreateOptimalClock() {
         // - Is not more precise than 1 GHz (1ns resolution)
         return std::make_unique<StandardWallClock>();
     }
-#elif defined(ARCHITECTURE_arm64) && defined(__linux__)
+#elif defined(HAS_NCE)
     return std::make_unique<Arm64::NativeClock>();
 #else
     return std::make_unique<StandardWallClock>();