From 4d769d9c7ebf14d8e993cb72c565f1d8209c92c0 Mon Sep 17 00:00:00 2001
From: Stephen Miller <56742918+StevenMiller123@users.noreply.github.com>
Date: Fri, 16 May 2025 09:41:22 -0500
Subject: [PATCH] Proper error handling for MapMemory errors (#2938)

* Properly handle ENOMEM error return in MapMemory

Needed for Assassin's Creed Unity to behave properly.

* Change error message

If I left the message as-is, we'd probably see inexperienced people claiming the assert means your device needs more memory, which is completely false.

* Clang

You know you're doing something right when Clang complains.

* Attempt to handle MemoryMapFlags::NoOverwrite

Based on my interpretation of red_prig's descriptions. These changes are untested, as I'm not able to test right now.

* Fix flag description

* Move overwrite check to while condition
---
 src/core/memory.cpp | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 13290336c..a08f8b0e9 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -376,19 +376,22 @@ int MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, size_t size, M
         // To account for this, unmap any reserved areas within this mapping range first.
         auto unmap_addr = mapped_addr;
         auto unmap_size = size;
-        while (!vma.IsMapped() && unmap_addr < mapped_addr + size && remaining_size < size) {
+        // If flag NoOverwrite is provided, don't overwrite mapped VMAs.
+        // When it isn't provided, VMAs can be overwritten regardless of if they're mapped.
+        while ((False(flags & MemoryMapFlags::NoOverwrite) || !vma.IsMapped()) &&
+               unmap_addr < mapped_addr + size && remaining_size < size) {
             auto unmapped = UnmapBytesFromEntry(unmap_addr, vma, unmap_size);
             unmap_addr += unmapped;
             unmap_size -= unmapped;
             vma = FindVMA(unmap_addr)->second;
         }
 
-        // This should return SCE_KERNEL_ERROR_ENOMEM but rarely happens.
         vma = FindVMA(mapped_addr)->second;
         remaining_size = vma.base + vma.size - mapped_addr;
-        ASSERT_MSG(!vma.IsMapped() && remaining_size >= size,
-                   "Memory region {:#x} to {:#x} isn't free enough to map region {:#x} to {:#x}",
-                   vma.base, vma.base + vma.size, virtual_addr, virtual_addr + size);
+        if (vma.IsMapped() || remaining_size < size) {
+            LOG_ERROR(Kernel_Vmm, "Unable to map {:#x} bytes at address {:#x}", size, mapped_addr);
+            return ORBIS_KERNEL_ERROR_ENOMEM;
+        }
     }
 
     // Find the first free area starting with provided virtual address.