diff --git a/src/shader_recompiler/info.h b/src/shader_recompiler/info.h
index 5d159275b..72977b711 100644
--- a/src/shader_recompiler/info.h
+++ b/src/shader_recompiler/info.h
@@ -58,7 +58,6 @@ struct BufferResource {
     BufferType buffer_type;
     u8 instance_attrib{};
     bool is_written{};
-    bool is_read{};
     bool is_formatted{};
 
     bool IsSpecial() const noexcept {
diff --git a/src/shader_recompiler/ir/passes/resource_tracking_pass.cpp b/src/shader_recompiler/ir/passes/resource_tracking_pass.cpp
index d5d140c93..f3972769c 100644
--- a/src/shader_recompiler/ir/passes/resource_tracking_pass.cpp
+++ b/src/shader_recompiler/ir/passes/resource_tracking_pass.cpp
@@ -209,7 +209,6 @@ public:
         auto& buffer = buffer_resources[index];
         buffer.used_types |= desc.used_types;
         buffer.is_written |= desc.is_written;
-        buffer.is_read |= desc.is_read;
         buffer.is_formatted |= desc.is_formatted;
         return index;
     }
@@ -380,7 +379,6 @@ s32 TryHandleInlineCbuf(IR::Inst& inst, Info& info, Descriptors& descriptors,
         .used_types = BufferDataType(inst, cbuf.GetNumberFmt()),
         .inline_cbuf = cbuf,
         .buffer_type = BufferType::Guest,
-        .is_read = true,
     });
 }
 
@@ -392,13 +390,11 @@ void PatchBufferSharp(IR::Block& block, IR::Inst& inst, Info& info, Descriptors&
         IR::Inst* producer = handle->Arg(0).InstRecursive();
         SharpLocation sharp;
         std::tie(sharp, buffer) = TrackSharp<AmdGpu::Buffer>(producer, info);
-        const bool is_written = IsBufferStore(inst);
         binding = descriptors.Add(BufferResource{
             .sharp_idx = sharp,
             .used_types = BufferDataType(inst, buffer.GetNumberFmt()),
             .buffer_type = BufferType::Guest,
-            .is_written = is_written,
-            .is_read = !is_written,
+            .is_written = IsBufferStore(inst),
             .is_formatted = inst.GetOpcode() == IR::Opcode::LoadBufferFormatF32 ||
                             inst.GetOpcode() == IR::Opcode::StoreBufferFormatF32,
         });
@@ -425,12 +421,11 @@ void PatchImageSharp(IR::Block& block, IR::Inst& inst, Info& info, Descriptors&
     // Read image sharp.
     const auto tsharp = TrackSharp(tsharp_handle, info);
     const auto inst_info = inst.Flags<IR::TextureInstInfo>();
-    const bool is_atomic = IsImageAtomicInstruction(inst);
-    const bool is_written = inst.GetOpcode() == IR::Opcode::ImageWrite || is_atomic;
+    const bool is_written = inst.GetOpcode() == IR::Opcode::ImageWrite;
     const ImageResource image_res = {
         .sharp_idx = tsharp,
         .is_depth = bool(inst_info.is_depth),
-        .is_atomic = is_atomic,
+        .is_atomic = IsImageAtomicInstruction(inst),
         .is_array = bool(inst_info.is_array),
         .is_written = is_written,
         .is_r128 = bool(inst_info.is_r128),
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 514de1743..e4e026485 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -511,8 +511,7 @@ bool Rasterizer::IsComputeMetaClear(const Pipeline* pipeline) {
     // will need its full emulation anyways.
     for (const auto& desc : info.buffers) {
         const VAddr address = desc.GetSharp(info).base_address;
-        if (!desc.IsSpecial() && desc.is_written && !desc.is_read &&
-            texture_cache.ClearMeta(address)) {
+        if (!desc.IsSpecial() && desc.is_written && texture_cache.ClearMeta(address)) {
             // Assume all slices were updates
             LOG_TRACE(Render_Vulkan, "Metadata update skipped");
             return true;
diff --git a/src/video_core/texture_cache/image.h b/src/video_core/texture_cache/image.h
index 2dbaff053..31b67e021 100644
--- a/src/video_core/texture_cache/image.h
+++ b/src/video_core/texture_cache/image.h
@@ -27,9 +27,10 @@ enum ImageFlagBits : u32 {
     CpuDirty = 1 << 1,      ///< Contents have been modified from the CPU
     GpuDirty = 1 << 2, ///< Contents have been modified from the GPU (valid data in buffer cache)
     Dirty = MaybeCpuDirty | CpuDirty | GpuDirty,
-    GpuModified = 1 << 3, ///< Contents have been modified from the GPU
-    Registered = 1 << 6,  ///< True when the image is registered
-    Picked = 1 << 7,      ///< Temporary flag to mark the image as picked
+    GpuModified = 1 << 3,    ///< Contents have been modified from the GPU
+    Registered = 1 << 6,     ///< True when the image is registered
+    Picked = 1 << 7,         ///< Temporary flag to mark the image as picked
+    MetaRegistered = 1 << 8, ///< True when metadata for this surface is known and registered
 };
 DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits)
 
diff --git a/src/video_core/texture_cache/texture_cache.cpp b/src/video_core/texture_cache/texture_cache.cpp
index 41c776bfd..a50601af6 100644
--- a/src/video_core/texture_cache/texture_cache.cpp
+++ b/src/video_core/texture_cache/texture_cache.cpp
@@ -451,16 +451,20 @@ ImageView& TextureCache::FindRenderTarget(BaseDesc& desc) {
     UpdateImage(image_id);
 
     // Register meta data for this color buffer
-    if (desc.info.meta_info.cmask_addr) {
-        surface_metas.emplace(desc.info.meta_info.cmask_addr,
-                              MetaDataInfo{.type = MetaDataInfo::Type::CMask});
-        image.info.meta_info.cmask_addr = desc.info.meta_info.cmask_addr;
-    }
+    if (!(image.flags & ImageFlagBits::MetaRegistered)) {
+        if (desc.info.meta_info.cmask_addr) {
+            surface_metas.emplace(desc.info.meta_info.cmask_addr,
+                                  MetaDataInfo{.type = MetaDataInfo::Type::CMask});
+            image.info.meta_info.cmask_addr = desc.info.meta_info.cmask_addr;
+            image.flags |= ImageFlagBits::MetaRegistered;
+        }
 
-    if (desc.info.meta_info.fmask_addr) {
-        surface_metas.emplace(desc.info.meta_info.fmask_addr,
-                              MetaDataInfo{.type = MetaDataInfo::Type::FMask});
-        image.info.meta_info.fmask_addr = desc.info.meta_info.fmask_addr;
+        if (desc.info.meta_info.fmask_addr) {
+            surface_metas.emplace(desc.info.meta_info.fmask_addr,
+                                  MetaDataInfo{.type = MetaDataInfo::Type::FMask});
+            image.info.meta_info.fmask_addr = desc.info.meta_info.fmask_addr;
+            image.flags |= ImageFlagBits::MetaRegistered;
+        }
     }
 
     return RegisterImageView(image_id, desc.view_info);
@@ -475,11 +479,15 @@ ImageView& TextureCache::FindDepthTarget(BaseDesc& desc) {
     UpdateImage(image_id);
 
     // Register meta data for this depth buffer
-    if (desc.info.meta_info.htile_addr) {
-        surface_metas.emplace(desc.info.meta_info.htile_addr,
-                              MetaDataInfo{.type = MetaDataInfo::Type::HTile,
-                                           .clear_mask = image.info.meta_info.htile_clear_mask});
-        image.info.meta_info.htile_addr = desc.info.meta_info.htile_addr;
+    if (!(image.flags & ImageFlagBits::MetaRegistered)) {
+        if (desc.info.meta_info.htile_addr) {
+            surface_metas.emplace(
+                desc.info.meta_info.htile_addr,
+                MetaDataInfo{.type = MetaDataInfo::Type::HTile,
+                             .clear_mask = image.info.meta_info.htile_clear_mask});
+            image.info.meta_info.htile_addr = desc.info.meta_info.htile_addr;
+            image.flags |= ImageFlagBits::MetaRegistered;
+        }
     }
 
     // If there is a stencil attachment, link depth and stencil.