Merge branch 'main' into input-fix-real

src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp

@@ -19,8 +19,7 @@ Id SharedAtomicU32(EmitContext& ctx, Id offset, Id value,
     const Id shift_id{ctx.ConstU32(2U)};
     const Id index{ctx.OpShiftRightLogical(ctx.U32[1], offset, shift_id)};
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 4u)};
-    const Id pointer{
+    const Id pointer{ctx.EmitSharedMemoryAccess(ctx.shared_u32, ctx.shared_memory_u32, index)};
-        ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, ctx.u32_zero_value, index)};
     const auto [scope, semantics]{AtomicArgs(ctx)};
     return AccessBoundsCheck<32>(ctx, index, ctx.ConstU32(num_elements), [&] {
         return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics, value);
@@ -32,8 +31,7 @@ Id SharedAtomicU32IncDec(EmitContext& ctx, Id offset,
     const Id shift_id{ctx.ConstU32(2U)};
     const Id index{ctx.OpShiftRightLogical(ctx.U32[1], offset, shift_id)};
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 4u)};
-    const Id pointer{
+    const Id pointer{ctx.EmitSharedMemoryAccess(ctx.shared_u32, ctx.shared_memory_u32, index)};
-        ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, ctx.u32_zero_value, index)};
     const auto [scope, semantics]{AtomicArgs(ctx)};
     return AccessBoundsCheck<32>(ctx, index, ctx.ConstU32(num_elements), [&] {
         return (ctx.*atomic_func)(ctx.U32[1], pointer, scope, semantics);
@@ -45,8 +43,7 @@ Id SharedAtomicU64(EmitContext& ctx, Id offset, Id value,
     const Id shift_id{ctx.ConstU32(3U)};
     const Id index{ctx.OpShiftRightLogical(ctx.U32[1], offset, shift_id)};
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 8u)};
-    const Id pointer{
+    const Id pointer{ctx.EmitSharedMemoryAccess(ctx.shared_u64, ctx.shared_memory_u64, index)};
-        ctx.OpAccessChain(ctx.shared_u64, ctx.shared_memory_u64, ctx.u32_zero_value, index)};
     const auto [scope, semantics]{AtomicArgs(ctx)};
     return AccessBoundsCheck<64>(ctx, index, ctx.ConstU32(num_elements), [&] {
         return (ctx.*atomic_func)(ctx.U64, pointer, scope, semantics, value);

src/shader_recompiler/backend/spirv/emit_spirv_shared_memory.cpp

@@ -14,8 +14,7 @@ Id EmitLoadSharedU16(EmitContext& ctx, Id offset) {
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 2u)};
 
     return AccessBoundsCheck<16>(ctx, index, ctx.ConstU32(num_elements), [&] {
-        const Id pointer =
+        const Id pointer = ctx.EmitSharedMemoryAccess(ctx.shared_u16, ctx.shared_memory_u16, index);
-            ctx.OpAccessChain(ctx.shared_u16, ctx.shared_memory_u16, ctx.u32_zero_value, index);
         return ctx.OpLoad(ctx.U16, pointer);
     });
 }
@@ -26,8 +25,7 @@ Id EmitLoadSharedU32(EmitContext& ctx, Id offset) {
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 4u)};
 
     return AccessBoundsCheck<32>(ctx, index, ctx.ConstU32(num_elements), [&] {
-        const Id pointer =
+        const Id pointer = ctx.EmitSharedMemoryAccess(ctx.shared_u32, ctx.shared_memory_u32, index);
-            ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, ctx.u32_zero_value, index);
         return ctx.OpLoad(ctx.U32[1], pointer);
     });
 }
@@ -38,8 +36,7 @@ Id EmitLoadSharedU64(EmitContext& ctx, Id offset) {
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 8u)};
 
     return AccessBoundsCheck<64>(ctx, index, ctx.ConstU32(num_elements), [&] {
-        const Id pointer{
+        const Id pointer = ctx.EmitSharedMemoryAccess(ctx.shared_u64, ctx.shared_memory_u64, index);
-            ctx.OpAccessChain(ctx.shared_u64, ctx.shared_memory_u64, ctx.u32_zero_value, index)};
         return ctx.OpLoad(ctx.U64, pointer);
     });
 }
@@ -50,8 +47,7 @@ void EmitWriteSharedU16(EmitContext& ctx, Id offset, Id value) {
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 2u)};
 
     AccessBoundsCheck<16>(ctx, index, ctx.ConstU32(num_elements), [&] {
-        const Id pointer =
+        const Id pointer = ctx.EmitSharedMemoryAccess(ctx.shared_u16, ctx.shared_memory_u16, index);
-            ctx.OpAccessChain(ctx.shared_u16, ctx.shared_memory_u16, ctx.u32_zero_value, index);
         ctx.OpStore(pointer, value);
         return Id{0};
     });
@@ -63,8 +59,7 @@ void EmitWriteSharedU32(EmitContext& ctx, Id offset, Id value) {
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 4u)};
 
     AccessBoundsCheck<32>(ctx, index, ctx.ConstU32(num_elements), [&] {
-        const Id pointer =
+        const Id pointer = ctx.EmitSharedMemoryAccess(ctx.shared_u32, ctx.shared_memory_u32, index);
-            ctx.OpAccessChain(ctx.shared_u32, ctx.shared_memory_u32, ctx.u32_zero_value, index);
         ctx.OpStore(pointer, value);
         return Id{0};
     });
@@ -76,8 +71,7 @@ void EmitWriteSharedU64(EmitContext& ctx, Id offset, Id value) {
     const u32 num_elements{Common::DivCeil(ctx.runtime_info.cs_info.shared_memory_size, 8u)};
 
     AccessBoundsCheck<64>(ctx, index, ctx.ConstU32(num_elements), [&] {
-        const Id pointer{
+        const Id pointer = ctx.EmitSharedMemoryAccess(ctx.shared_u64, ctx.shared_memory_u64, index);
-            ctx.OpAccessChain(ctx.shared_u64, ctx.shared_memory_u64, ctx.u32_zero_value, index)};
         ctx.OpStore(pointer, value);
         return Id{0};
     });

src/shader_recompiler/backend/spirv/spirv_emit_context.cpp

@@ -972,7 +972,12 @@ void EmitContext::DefineImagesAndSamplers() {
         const Id id{AddGlobalVariable(sampler_pointer_type, spv::StorageClass::UniformConstant)};
         Decorate(id, spv::Decoration::Binding, binding.unified++);
         Decorate(id, spv::Decoration::DescriptorSet, 0U);
-        Name(id, fmt::format("{}_{}{}", stage, "samp", samp_desc.sharp_idx));
+        auto sharp_desc = std::holds_alternative<u32>(samp_desc.sampler)
+                              ? fmt::format("sgpr:{}", std::get<u32>(samp_desc.sampler))
+                              : fmt::format("inline:{:#x}:{:#x}",
+                                            std::get<AmdGpu::Sampler>(samp_desc.sampler).raw0,
+                                            std::get<AmdGpu::Sampler>(samp_desc.sampler).raw1);
+        Name(id, fmt::format("{}_{}{}", stage, "samp", sharp_desc));
         samplers.push_back(id);
         interfaces.push_back(id);
     }
@@ -995,19 +1000,26 @@ void EmitContext::DefineSharedMemory() {
 
         const u32 num_elements{Common::DivCeil(shared_memory_size, element_size)};
         const Id array_type{TypeArray(element_type, ConstU32(num_elements))};
-        Decorate(array_type, spv::Decoration::ArrayStride, element_size);
 
+        const auto mem_type = [&] {
+            if (num_types > 1) {
                 const Id struct_type{TypeStruct(array_type)};
+                Decorate(struct_type, spv::Decoration::Block);
                 MemberDecorate(struct_type, 0u, spv::Decoration::Offset, 0u);
+                return struct_type;
+            } else {
+                return array_type;
+            }
+        }();
 
-        const Id pointer = TypePointer(spv::StorageClass::Workgroup, struct_type);
+        const Id pointer = TypePointer(spv::StorageClass::Workgroup, mem_type);
         const Id element_pointer = TypePointer(spv::StorageClass::Workgroup, element_type);
         const Id variable = AddGlobalVariable(pointer, spv::StorageClass::Workgroup);
         Name(variable, name);
         interfaces.push_back(variable);
 
         if (num_types > 1) {
-            Decorate(struct_type, spv::Decoration::Block);
+            Decorate(array_type, spv::Decoration::ArrayStride, element_size);
             Decorate(variable, spv::Decoration::Aliased);
         }
 

src/shader_recompiler/backend/spirv/spirv_emit_context.h

@@ -203,6 +203,14 @@ public:
         return final_result;
     }
 
+    Id EmitSharedMemoryAccess(const Id result_type, const Id shared_mem, const Id index) {
+        if (std::popcount(static_cast<u32>(info.shared_types)) > 1) {
+            return OpAccessChain(result_type, shared_mem, u32_zero_value, index);
+        }
+        // Workgroup layout struct omitted.
+        return OpAccessChain(result_type, shared_mem, index);
+    }
+
     Info& info;
     const RuntimeInfo& runtime_info;
     const Profile& profile;

src/shader_recompiler/frontend/translate/vector_memory.cpp

@@ -531,8 +531,10 @@ IR::Value EmitImageSample(IR::IREmitter& ir, const GcnInst& inst, const IR::Scal
     // Load first dword of T# and S#. We will use them as the handle that will guide resource
     // tracking pass where to read the sharps. This will later also get patched to the SPIRV texture
     // binding index.
-    const IR::Value handle =
+    const IR::Value handle = ir.GetScalarReg(tsharp_reg);
-        ir.CompositeConstruct(ir.GetScalarReg(tsharp_reg), ir.GetScalarReg(sampler_reg));
+    const IR::Value inline_sampler =
+        ir.CompositeConstruct(ir.GetScalarReg(sampler_reg), ir.GetScalarReg(sampler_reg + 1),
+                              ir.GetScalarReg(sampler_reg + 2), ir.GetScalarReg(sampler_reg + 3));
 
     // Determine how many address registers need to be passed.
     // The image type is unknown, so add all 4 possible base registers and resolve later.
@@ -568,7 +570,8 @@ IR::Value EmitImageSample(IR::IREmitter& ir, const GcnInst& inst, const IR::Scal
     const IR::Value address4 = get_addr_reg(12);
 
     // Issue the placeholder IR instruction.
-    IR::Value texel = ir.ImageSampleRaw(handle, address1, address2, address3, address4, info);
+    IR::Value texel =
+        ir.ImageSampleRaw(handle, address1, address2, address3, address4, inline_sampler, info);
     if (info.is_depth && !gather) {
         // For non-gather depth sampling, only return a single value.
         texel = ir.CompositeExtract(texel, 0);

src/shader_recompiler/info.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include <span>
+#include <variant>
 #include <vector>
 #include <boost/container/small_vector.hpp>
 #include <boost/container/static_vector.hpp>
@@ -91,11 +92,15 @@ struct ImageResource {
 using ImageResourceList = boost::container::small_vector<ImageResource, NumImages>;
 
 struct SamplerResource {
-    u32 sharp_idx;
+    std::variant<u32, AmdGpu::Sampler> sampler;
-    AmdGpu::Sampler inline_sampler{};
     u32 associated_image : 4;
     u32 disable_aniso : 1;
 
+    SamplerResource(u32 sharp_idx, u32 associated_image_, bool disable_aniso_)
+        : sampler{sharp_idx}, associated_image{associated_image_}, disable_aniso{disable_aniso_} {}
+    SamplerResource(AmdGpu::Sampler sampler_)
+        : sampler{sampler_}, associated_image{0}, disable_aniso(0) {}
+
     constexpr AmdGpu::Sampler GetSharp(const Info& info) const noexcept;
 };
 using SamplerResourceList = boost::container::small_vector<SamplerResource, NumSamplers>;
@@ -318,7 +323,9 @@ constexpr AmdGpu::Image ImageResource::GetSharp(const Info& info) const noexcept
 }
 
 constexpr AmdGpu::Sampler SamplerResource::GetSharp(const Info& info) const noexcept {
-    return inline_sampler ? inline_sampler : info.ReadUdSharp<AmdGpu::Sampler>(sharp_idx);
+    return std::holds_alternative<AmdGpu::Sampler>(sampler)
+               ? std::get<AmdGpu::Sampler>(sampler)
+               : info.ReadUdSharp<AmdGpu::Sampler>(std::get<u32>(sampler));
 }
 
 constexpr AmdGpu::Image FMaskResource::GetSharp(const Info& info) const noexcept {

src/shader_recompiler/ir/ir_emitter.cpp

@@ -1964,9 +1964,9 @@ Value IREmitter::ImageAtomicExchange(const Value& handle, const Value& coords, c
 
 Value IREmitter::ImageSampleRaw(const Value& handle, const Value& address1, const Value& address2,
                                 const Value& address3, const Value& address4,
-                                TextureInstInfo info) {
+                                const Value& inline_sampler, TextureInstInfo info) {
-    return Inst(Opcode::ImageSampleRaw, Flags{info}, handle, address1, address2, address3,
+    return Inst(Opcode::ImageSampleRaw, Flags{info}, handle, address1, address2, address3, address4,
-                address4);
+                inline_sampler);
 }
 
 Value IREmitter::ImageSampleImplicitLod(const Value& handle, const Value& coords, const F32& bias,

src/shader_recompiler/ir/ir_emitter.h

@@ -349,7 +349,8 @@ public:
 
     [[nodiscard]] Value ImageSampleRaw(const Value& handle, const Value& address1,
                                        const Value& address2, const Value& address3,
-                                       const Value& address4, TextureInstInfo info);
+                                       const Value& address4, const Value& inline_sampler,
+                                       TextureInstInfo info);
 
     [[nodiscard]] Value ImageSampleImplicitLod(const Value& handle, const Value& body,
                                                const F32& bias, const Value& offset,

src/shader_recompiler/ir/opcodes.inc

@@ -412,7 +412,7 @@ OPCODE(ConvertU8U32,                                        U8,             U32,
 OPCODE(ConvertU32U8,                                        U32,            U8,                                                                             )
 
 // Image operations
-OPCODE(ImageSampleRaw,                                      F32x4,          Opaque,         F32x4,          F32x4,          F32x4,          F32,            )
+OPCODE(ImageSampleRaw,                                      F32x4,          Opaque,         F32x4,          F32x4,          F32x4,          F32,            Opaque,      )
 OPCODE(ImageSampleImplicitLod,                              F32x4,          Opaque,         F32x4,          F32,            Opaque,                         )
 OPCODE(ImageSampleExplicitLod,                              F32x4,          Opaque,         Opaque,         F32,            Opaque,                         )
 OPCODE(ImageSampleDrefImplicitLod,                          F32x4,          Opaque,         Opaque,         F32,            F32,            Opaque,         )

src/shader_recompiler/ir/passes/resource_tracking_pass.cpp

@@ -168,7 +168,7 @@ public:
 
     u32 Add(const SamplerResource& desc) {
         const u32 index{Add(sampler_resources, desc, [this, &desc](const auto& existing) {
-            return desc.sharp_idx == existing.sharp_idx;
+            return desc.sampler == existing.sampler;
         })};
         return index;
     }
@@ -351,8 +351,7 @@ void PatchBufferSharp(IR::Block& block, IR::Inst& inst, Info& info, Descriptors&
 void PatchImageSharp(IR::Block& block, IR::Inst& inst, Info& info, Descriptors& descriptors) {
     const auto pred = [](const IR::Inst* inst) -> std::optional<const IR::Inst*> {
         const auto opcode = inst->GetOpcode();
-        if (opcode == IR::Opcode::CompositeConstructU32x2 || // IMAGE_SAMPLE (image+sampler)
+        if (opcode == IR::Opcode::ReadConst || // IMAGE_LOAD (image only)
-            opcode == IR::Opcode::ReadConst ||               // IMAGE_LOAD (image only)
             opcode == IR::Opcode::GetUserData) {
             return inst;
         }
@@ -360,9 +359,7 @@ void PatchImageSharp(IR::Block& block, IR::Inst& inst, Info& info, Descriptors&
     };
     const auto result = IR::BreadthFirstSearch(&inst, pred);
     ASSERT_MSG(result, "Unable to find image sharp source");
-    const IR::Inst* producer = result.value();
+    const IR::Inst* tsharp_handle = result.value();
-    const bool has_sampler = producer->GetOpcode() == IR::Opcode::CompositeConstructU32x2;
-    const auto tsharp_handle = has_sampler ? producer->Arg(0).InstRecursive() : producer;
 
     // Read image sharp.
     const auto tsharp = TrackSharp(tsharp_handle, info);
@@ -427,29 +424,32 @@ void PatchImageSharp(IR::Block& block, IR::Inst& inst, Info& info, Descriptors&
 
     if (inst.GetOpcode() == IR::Opcode::ImageSampleRaw) {
         // Read sampler sharp.
-        const auto [sampler_binding, sampler] = [&] -> std::pair<u32, AmdGpu::Sampler> {
+        const auto sampler_binding = [&] -> u32 {
-            ASSERT(producer->GetOpcode() == IR::Opcode::CompositeConstructU32x2);
+            const auto sampler = inst.Arg(5).InstRecursive();
-            const IR::Value& handle = producer->Arg(1);
+            ASSERT(sampler && sampler->GetOpcode() == IR::Opcode::CompositeConstructU32x4);
+            const auto handle = sampler->Arg(0);
             // Inline sampler resource.
             if (handle.IsImmediate()) {
-                LOG_WARNING(Render_Vulkan, "Inline sampler detected");
+                LOG_DEBUG(Render_Vulkan, "Inline sampler detected");
-                const auto inline_sampler = AmdGpu::Sampler{.raw0 = handle.U32()};
+                const auto [s1, s2, s3, s4] =
-                const auto binding = descriptors.Add(SamplerResource{
+                    std::tuple{sampler->Arg(0), sampler->Arg(1), sampler->Arg(2), sampler->Arg(3)};
-                    .sharp_idx = std::numeric_limits<u32>::max(),
+                ASSERT(s1.IsImmediate() && s2.IsImmediate() && s3.IsImmediate() &&
-                    .inline_sampler = inline_sampler,
+                       s4.IsImmediate());
-                });
+                const auto inline_sampler = AmdGpu::Sampler{
-                return {binding, inline_sampler};
+                    .raw0 = u64(s2.U32()) << 32 | u64(s1.U32()),
-            }
+                    .raw1 = u64(s4.U32()) << 32 | u64(s3.U32()),
+                };
+                const auto binding = descriptors.Add(SamplerResource{inline_sampler});
+                return binding;
+            } else {
                 // Normal sampler resource.
                 const auto ssharp_handle = handle.InstRecursive();
                 const auto& [ssharp_ud, disable_aniso] = TryDisableAnisoLod0(ssharp_handle);
                 const auto ssharp = TrackSharp(ssharp_ud, info);
-            const auto binding = descriptors.Add(SamplerResource{
+                const auto binding =
-                .sharp_idx = ssharp,
+                    descriptors.Add(SamplerResource{ssharp, image_binding, disable_aniso});
-                .associated_image = image_binding,
+                return binding;
-                .disable_aniso = disable_aniso,
+            }
-            });
-            return {binding, info.ReadUdSharp<AmdGpu::Sampler>(ssharp)};
         }();
         // Patch image and sampler handle.
         inst.SetArg(0, ir.Imm32(image_binding | sampler_binding << 16));