diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index ab59219b2..bf9d1cabd 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -262,10 +262,7 @@ int MemoryManager::PoolCommit(VAddr virtual_addr, size_t size, MemoryProt prot)
     void* out_addr = impl.Map(mapped_addr, size, alignment, -1, false);
     TRACK_ALLOC(out_addr, size, "VMEM");
 
-    if (prot >= MemoryProt::GpuRead) {
-        // PS4s only map to GPU memory when the protection includes GPU access.
-        // If the address to map to is too high, PS4s throw a page fault and crash.
-        ASSERT_MSG(IsValidGpuMapping(mapped_addr, size), "Invalid address for GPU mapping");
+    if (IsValidGpuMapping(mapped_addr, size)) {
         rasterizer->MapMemory(mapped_addr, size);
     }
 
@@ -345,19 +342,15 @@ s32 MemoryManager::MapMemory(void** out_addr, VAddr virtual_addr, u64 size, Memo
         MergeAdjacent(vma_map, new_vma_handle);
     }
 
-    if (prot >= MemoryProt::GpuRead) {
-        // PS4s only map to GPU memory when the protection includes GPU access.
-        // If the address to map to is too high, PS4s throw a page fault and crash.
-        ASSERT_MSG(IsValidGpuMapping(mapped_addr, size), "Invalid address for GPU mapping");
-        rasterizer->MapMemory(mapped_addr, size);
-    }
-
     if (type == VMAType::Reserved || type == VMAType::PoolReserved) {
         // For Reserved/PoolReserved mappings, we don't perform any address space allocations.
         // Just set out_addr to mapped_addr instead.
         *out_addr = std::bit_cast<void*>(mapped_addr);
     } else {
-        // Type is either Direct, Flexible, or Code, these need to be mapped in our address space.
+        // If this is not a reservation, then map to GPU and address space
+        if (IsValidGpuMapping(mapped_addr, size)) {
+            rasterizer->MapMemory(mapped_addr, size);
+        }
         *out_addr = impl.Map(mapped_addr, size, alignment, phys_addr, is_exec);
     }
 
@@ -429,7 +422,6 @@ s32 MemoryManager::PoolDecommit(VAddr virtual_addr, size_t size) {
     const bool is_exec = vma_base.is_exec;
     const auto start_in_vma = virtual_addr - vma_base_addr;
     const auto type = vma_base.type;
-    const auto prot = vma_base.prot;
 
     if (type != VMAType::PoolReserved && type != VMAType::Pooled) {
         LOG_ERROR(Kernel_Vmm, "Attempting to decommit non-pooled memory!");
@@ -437,15 +429,15 @@ s32 MemoryManager::PoolDecommit(VAddr virtual_addr, size_t size) {
     }
 
     if (type == VMAType::Pooled) {
+        // We always map PoolCommitted memory to GPU, so unmap when decomitting.
+        if (IsValidGpuMapping(virtual_addr, size)) {
+            rasterizer->UnmapMemory(virtual_addr, size);
+        }
+
         // Track how much pooled memory is decommitted
         pool_budget += size;
     }
 
-    if (prot >= MemoryProt::GpuRead) {
-        // If this mapping has GPU access, unmap from GPU.
-        rasterizer->UnmapMemory(virtual_addr, size);
-    }
-
     // Mark region as free and attempt to coalesce it with neighbours.
     const auto new_it = CarveVMA(virtual_addr, size);
     auto& vma = new_it->second;
@@ -486,15 +478,11 @@ u64 MemoryManager::UnmapBytesFromEntry(VAddr virtual_addr, VirtualMemoryArea vma
     if (type == VMAType::Free) {
         return adjusted_size;
     }
+
     if (type == VMAType::Flexible) {
         flexible_usage -= adjusted_size;
     }
 
-    if (prot >= MemoryProt::GpuRead) {
-        // If this mapping has GPU access, unmap from GPU.
-        rasterizer->UnmapMemory(virtual_addr, size);
-    }
-
     // Mark region as free and attempt to coalesce it with neighbours.
     const auto new_it = CarveVMA(virtual_addr, adjusted_size);
     auto& vma = new_it->second;
@@ -507,6 +495,11 @@ u64 MemoryManager::UnmapBytesFromEntry(VAddr virtual_addr, VirtualMemoryArea vma
     auto& post_merge_vma = post_merge_it->second;
     bool readonly_file = post_merge_vma.prot == MemoryProt::CpuRead && type == VMAType::File;
     if (type != VMAType::Reserved && type != VMAType::PoolReserved) {
+        // If this mapping has GPU access, unmap from GPU.
+        if (IsValidGpuMapping(virtual_addr, size)) {
+            rasterizer->UnmapMemory(virtual_addr, size);
+        }
+
         // Unmap the memory region.
         impl.Unmap(vma_base_addr, vma_base_size, start_in_vma, start_in_vma + adjusted_size,
                    phys_base, is_exec, has_backing, readonly_file);
@@ -576,18 +569,6 @@ s64 MemoryManager::ProtectBytes(VAddr addr, VirtualMemoryArea vma_base, size_t s
         return ORBIS_KERNEL_ERROR_EINVAL;
     }
 
-    if (vma_base.prot < MemoryProt::GpuRead && prot >= MemoryProt::GpuRead) {
-        // New protection will give the GPU access to this VMA, perform a rasterizer map
-        ASSERT_MSG(IsValidGpuMapping(addr, size), "Invalid address for GPU mapping");
-        rasterizer->MapMemory(addr, size);
-    }
-
-    if (vma_base.prot >= MemoryProt::GpuRead && prot < MemoryProt::GpuRead) {
-        // New protection will remove the GPU's access to this VMA, perform a rasterizer unmap
-        ASSERT_MSG(IsValidGpuMapping(addr, size), "Invalid address for GPU unmap");
-        rasterizer->UnmapMemory(addr, size);
-    }
-
     // Change protection
     vma_base.prot = prot;