From 66c4dedbc8a425ef5e444038c739d494c202507d Mon Sep 17 00:00:00 2001
From: Stephen Miller <millerste004@gmail.com>
Date: Thu, 1 May 2025 23:23:15 -0500
Subject: [PATCH] Place an upper bound on returns from SearchFree

Right now, this upper bound is set based on the limitations of our GPU buffer cache and page table.
Someone with more experience in that area of code should probably fix that at some point.
---
 src/core/libraries/kernel/memory.cpp |  3 +-
 src/core/memory.cpp                  | 51 +++++++++++++++++++++++-----
 2 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/src/core/libraries/kernel/memory.cpp b/src/core/libraries/kernel/memory.cpp
index 4bffe2e34..566c2daca 100644
--- a/src/core/libraries/kernel/memory.cpp
+++ b/src/core/libraries/kernel/memory.cpp
@@ -152,9 +152,8 @@ s32 PS4_SYSV_ABI sceKernelReserveVirtualRange(void** addr, u64 len, int flags, u
     auto* memory = Core::Memory::Instance();
     const VAddr in_addr = reinterpret_cast<VAddr>(*addr);
     const auto map_flags = static_cast<Core::MemoryMapFlags>(flags);
-    memory->Reserve(addr, in_addr, len, map_flags, alignment);
 
-    return ORBIS_OK;
+    return memory->Reserve(addr, in_addr, len, map_flags, alignment);
 }
 
 int PS4_SYSV_ABI sceKernelMapNamedDirectMemory(void** addr, u64 len, int prot, int flags,
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index c04f9d0f2..9f2312078 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -222,6 +222,10 @@ int MemoryManager::PoolReserve(void** out_addr, VAddr virtual_addr, size_t size,
     // Find the first free area starting with provided virtual address.
     if (False(flags & MemoryMapFlags::Fixed)) {
         mapped_addr = SearchFree(mapped_addr, size, alignment);
+        if (mapped_addr == -1) {
+            // No suitable memory areas to map to
+            return ORBIS_KERNEL_ERROR_ENOMEM;
+        }
     }
 
     // Add virtual memory area
@@ -260,6 +264,10 @@ int MemoryManager::Reserve(void** out_addr, VAddr virtual_addr, size_t size, Mem
     // Find the first free area starting with provided virtual address.
     if (False(flags & MemoryMapFlags::Fixed)) {
         mapped_addr = SearchFree(mapped_addr, size, alignment);
+        if (mapped_addr == -1) {
+            // No suitable memory areas to map to
+            return ORBIS_KERNEL_ERROR_ENOMEM;
+        }
     }
 
     // Add virtual memory area
@@ -334,6 +342,10 @@ int MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, size_t size, M
     // Find the first free area starting with provided virtual address.
     if (False(flags & MemoryMapFlags::Fixed)) {
         mapped_addr = SearchFree(mapped_addr, size, alignment);
+        if (mapped_addr == -1) {
+            // No suitable memory areas to map to
+            return ORBIS_KERNEL_ERROR_ENOMEM;
+        }
     }
 
     // Perform the mapping.
@@ -366,6 +378,10 @@ int MemoryManager::MapFile(void** out_addr, VAddr virtual_addr, size_t size, Mem
     // Find first free area to map the file.
     if (False(flags & MemoryMapFlags::Fixed)) {
         mapped_addr = SearchFree(mapped_addr, size_aligned, 1);
+        if (mapped_addr == -1) {
+            // No suitable memory areas to map to
+            return ORBIS_KERNEL_ERROR_ENOMEM;
+        }
     }
 
     if (True(flags & MemoryMapFlags::Fixed)) {
@@ -701,6 +717,12 @@ VAddr MemoryManager::SearchFree(VAddr virtual_addr, size_t size, u32 alignment)
         virtual_addr = min_search_address;
     }
 
+    // If the requested address is beyond the maximum our code can handle, return an error.
+    auto max_search_address = 0x10000000000;
+    if (virtual_addr >= max_search_address) {
+        return -1;
+    }
+
     auto it = FindVMA(virtual_addr);
     ASSERT_MSG(it != vma_map.end(), "Specified mapping address was not found!");
 
@@ -708,24 +730,37 @@ VAddr MemoryManager::SearchFree(VAddr virtual_addr, size_t size, u32 alignment)
     if (it->second.IsFree() && it->second.Contains(virtual_addr, size)) {
         return virtual_addr;
     }
+
     // Search for the first free VMA that fits our mapping.
-    const auto is_suitable = [&] {
+    while (it != vma_map.end()) {
         if (!it->second.IsFree()) {
-            return false;
+            it++;
+            continue;
         }
+
         const auto& vma = it->second;
         virtual_addr = Common::AlignUp(vma.base, alignment);
         // Sometimes the alignment itself might be larger than the VMA.
         if (virtual_addr > vma.base + vma.size) {
-            return false;
+            it++;
+            continue;
         }
+
+        // Make sure the address is within our defined bounds
+        if (virtual_addr >= max_search_address) {
+            return -1;
+        }
+
+        // If there's enough space in the VMA, return the address.
         const size_t remaining_size = vma.base + vma.size - virtual_addr;
-        return remaining_size >= size;
-    };
-    while (!is_suitable()) {
-        ++it;
+        if (remaining_size >= size) {
+            return virtual_addr;
+        }
+        it++;
     }
-    return virtual_addr;
+
+    // Couldn't find a suitable VMA, return an error.
+    return -1;
 }
 
 MemoryManager::VMAHandle MemoryManager::CarveVMA(VAddr virtual_addr, size_t size) {