video_out: HDR support (#2381)

* Initial HDR support

* fix for crashes when debug tools used

src/common/config.cpp

@@ -31,6 +31,7 @@ std::filesystem::path find_fs_path_or(const basic_value<TC>& v, const K& ky,
 
 namespace Config {
 
+static bool isHDRAllowed = false;
 static bool isNeo = false;
 static bool isFullscreen = false;
 static std::string fullscreenMode = "borderless";
@@ -101,6 +102,10 @@ static bool showBackgroundImage = true;
 // Language
 u32 m_language = 1; // english
 
+bool allowHDR() {
+    return isHDRAllowed;
+}
+
 bool GetUseUnifiedInputConfig() {
     return useUnifiedInputConfig;
 }
@@ -651,6 +656,7 @@ void load(const std::filesystem::path& path) {
     if (data.contains("General")) {
         const toml::value& general = data.at("General");
 
+        isHDRAllowed = toml::find_or<bool>(general, "allowHDR", false);
         isNeo = toml::find_or<bool>(general, "isPS4Pro", false);
         isFullscreen = toml::find_or<bool>(general, "Fullscreen", false);
         fullscreenMode = toml::find_or<std::string>(general, "FullscreenMode", "borderless");
@@ -786,6 +792,7 @@ void save(const std::filesystem::path& path) {
         fmt::print("Saving new configuration file {}\n", fmt::UTF(path.u8string()));
     }
 
+    data["General"]["allowHDR"] = isHDRAllowed;
     data["General"]["isPS4Pro"] = isNeo;
     data["General"]["Fullscreen"] = isFullscreen;
     data["General"]["FullscreenMode"] = fullscreenMode;
@@ -894,6 +901,7 @@ void saveMainWindow(const std::filesystem::path& path) {
 }
 
 void setDefaultValues() {
+    isHDRAllowed = false;
     isNeo = false;
     isFullscreen = false;
     isTrophyPopupDisabled = false;

src/common/config.h

@@ -51,6 +51,7 @@ void SetUseUnifiedInputConfig(bool use);
 u32 getScreenWidth();
 u32 getScreenHeight();
 s32 getGpuId();
+bool allowHDR();
 
 bool debugDump();
 bool collectShadersForDebug();

src/core/libraries/videoout/driver.h

@@ -18,7 +18,6 @@ struct Frame;
 namespace Libraries::VideoOut {
 
 struct VideoOutPort {
-    bool is_open = false;
     SceVideoOutResolutionStatus resolution;
     std::array<VideoOutBuffer, MaxDisplayBuffers> buffer_slots;
     std::array<u64, MaxDisplayBuffers> buffer_labels; // should be contiguous in memory
@@ -33,6 +32,8 @@ struct VideoOutPort {
     std::condition_variable vo_cv;
     std::condition_variable vblank_cv;
     int flip_rate = 0;
+    bool is_open = false;
+    bool is_mode_changing = false; // Used to prevent flip during mode change
 
     s32 FindFreeGroup() const {
         s32 index = 0;

src/core/libraries/videoout/video_out.cpp

@@ -3,6 +3,7 @@
 
 #include "common/assert.h"
 #include "common/config.h"
+#include "common/elf_info.h"
 #include "common/logging/log.h"
 #include "core/libraries/libs.h"
 #include "core/libraries/system/userservice.h"
@@ -315,6 +316,12 @@ s32 sceVideoOutSubmitEopFlip(s32 handle, u32 buf_id, u32 mode, u32 arg, void** u
 s32 PS4_SYSV_ABI sceVideoOutGetDeviceCapabilityInfo(
     s32 handle, SceVideoOutDeviceCapabilityInfo* pDeviceCapabilityInfo) {
     pDeviceCapabilityInfo->capability = 0;
+    if (presenter->IsHDRSupported()) {
+        auto& game_info = Common::ElfInfo::Instance();
+        if (game_info.GetPSFAttributes().support_hdr) {
+            pDeviceCapabilityInfo->capability |= ORBIS_VIDEO_OUT_DEVICE_CAPABILITY_BT2020_PQ;
+        }
+    }
     return ORBIS_OK;
 }
 
@@ -352,6 +359,49 @@ s32 PS4_SYSV_ABI sceVideoOutAdjustColor(s32 handle, const SceVideoOutColorSettin
     return ORBIS_OK;
 }
 
+struct Mode {
+    u32 size;
+    u8 encoding;
+    u8 range;
+    u8 colorimetry;
+    u8 depth;
+    u64 refresh_rate;
+    u64 resolution;
+    u8 reserved[8];
+};
+
+void PS4_SYSV_ABI sceVideoOutModeSetAny_(Mode* mode, u32 size) {
+    std::memset(mode, 0xff, size);
+    mode->size = size;
+}
+
+s32 PS4_SYSV_ABI sceVideoOutConfigureOutputMode_(s32 handle, u32 reserved, const Mode* mode,
+                                                 const void* options, u32 size_mode,
+                                                 u32 size_options) {
+    auto* port = driver->GetPort(handle);
+    if (!port) {
+        return ORBIS_VIDEO_OUT_ERROR_INVALID_HANDLE;
+    }
+
+    if (reserved != 0) {
+        return ORBIS_VIDEO_OUT_ERROR_INVALID_VALUE;
+    }
+
+    if (mode->colorimetry != OrbisVideoOutColorimetry::Any) {
+        auto& game_info = Common::ElfInfo::Instance();
+        if (mode->colorimetry == OrbisVideoOutColorimetry::Bt2020PQ &&
+            game_info.GetPSFAttributes().support_hdr) {
+            port->is_mode_changing = true;
+            presenter->SetHDR(true);
+            port->is_mode_changing = false;
+        } else {
+            return ORBIS_VIDEO_OUT_ERROR_INVALID_VALUE;
+        }
+    }
+
+    return ORBIS_OK;
+}
+
 void RegisterLib(Core::Loader::SymbolsResolver* sym) {
     driver = std::make_unique<VideoOutDriver>(Config::getScreenWidth(), Config::getScreenHeight());
 
@@ -390,6 +440,10 @@ void RegisterLib(Core::Loader::SymbolsResolver* sym) {
                  sceVideoOutAdjustColor);
     LIB_FUNCTION("-Ozn0F1AFRg", "libSceVideoOut", 1, "libSceVideoOut", 0, 0,
                  sceVideoOutDeleteFlipEvent);
+    LIB_FUNCTION("pjkDsgxli6c", "libSceVideoOut", 1, "libSceVideoOut", 0, 0,
+                 sceVideoOutModeSetAny_);
+    LIB_FUNCTION("N1bEoJ4SRw4", "libSceVideoOut", 1, "libSceVideoOut", 0, 0,
+                 sceVideoOutConfigureOutputMode_);
 
     // openOrbis appears to have libSceVideoOut_v1 module libSceVideoOut_v1.1
     LIB_FUNCTION("Up36PTk687E", "libSceVideoOut", 1, "libSceVideoOut", 1, 1, sceVideoOutOpen);

src/core/libraries/videoout/video_out.h

@@ -40,6 +40,13 @@ constexpr int SCE_VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_NONE = 0;
 constexpr int SCE_VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_VR = 7;
 constexpr int SCE_VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_STRICT_COLORIMETRY = 8;
 
+constexpr int ORBIS_VIDEO_OUT_DEVICE_CAPABILITY_BT2020_PQ = 0x80;
+
+enum OrbisVideoOutColorimetry : u8 {
+    Bt2020PQ = 12,
+    Any = 0xFF,
+};
+
 enum class OrbisVideoOutEventId : s16 {
     Flip = 0,
     Vblank = 1,

src/imgui/renderer/imgui_core.cpp

@@ -118,6 +118,10 @@ void OnResize() {
     Sdl::OnResize();
 }
 
+void OnSurfaceFormatChange(vk::Format surface_format) {
+    Vulkan::OnSurfaceFormatChange(surface_format);
+}
+
 void Shutdown(const vk::Device& device) {
     auto result = device.waitIdle();
     if (result != vk::Result::eSuccess) {

src/imgui/renderer/imgui_core.h

@@ -22,6 +22,8 @@ void Initialize(const Vulkan::Instance& instance, const Frontend::WindowSDL& win
 
 void OnResize();
 
+void OnSurfaceFormatChange(vk::Format surface_format);
+
 void Shutdown(const vk::Device& device);
 
 bool ProcessEvent(SDL_Event* event);

src/imgui/renderer/imgui_impl_vulkan.cpp

@@ -1265,4 +1265,20 @@ void Shutdown() {
     IM_DELETE(bd);
 }
 
+void OnSurfaceFormatChange(vk::Format surface_format) {
+    VkData* bd = GetBackendData();
+    const InitInfo& v = bd->init_info;
+    auto& pl_format = const_cast<vk::Format&>(
+        bd->init_info.pipeline_rendering_create_info.pColorAttachmentFormats[0]);
+    if (pl_format != surface_format) {
+        pl_format = surface_format;
+        if (bd->pipeline) {
+            v.device.destroyPipeline(bd->pipeline, v.allocator);
+            bd->pipeline = VK_NULL_HANDLE;
+            CreatePipeline(v.device, v.allocator, v.pipeline_cache, nullptr, &bd->pipeline,
+                           v.subpass);
+        }
+    }
+}
+
 } // namespace ImGui::Vulkan

src/imgui/renderer/imgui_impl_vulkan.h

@@ -67,5 +67,6 @@ void RenderDrawData(ImDrawData& draw_data, vk::CommandBuffer command_buffer,
                     vk::Pipeline pipeline = VK_NULL_HANDLE);
 
 void SetBlendEnabled(bool enabled);
+void OnSurfaceFormatChange(vk::Format surface_format);
 
 } // namespace ImGui::Vulkan

src/video_core/host_shaders/post_process.frag

@@ -10,16 +10,23 @@ layout (binding = 0) uniform sampler2D texSampler;
 
 layout(push_constant) uniform settings {
     float gamma;
+    bool hdr;
 } pp;
 
 const float cutoff = 0.0031308, a = 1.055, b = 0.055, d = 12.92;
-vec3 gamma(vec3 rgb)
+vec3 gamma(vec3 rgb) {
-{
+    return mix(
-    return mix(a * pow(rgb, vec3(1.0 / (2.4 + 1.0 - pp.gamma))) - b, d * rgb / pp.gamma, lessThan(rgb, vec3(cutoff)));
+        a * pow(rgb, vec3(1.0 / (2.4 + 1.0 - pp.gamma))) - b,
+        d * rgb / pp.gamma,
+        lessThan(rgb, vec3(cutoff))
+    );
 }
 
-void main()
+void main() {
-{
     vec4 color_linear = texture(texSampler, uv);
+    if (pp.hdr) {
+        color = color_linear;
+    } else {
         color = vec4(gamma(color_linear.rgb), color_linear.a);
     }
+}

src/video_core/renderer_vulkan/vk_platform.cpp

@@ -160,6 +160,10 @@ std::vector<const char*> GetInstanceExtensions(Frontend::WindowSystemType window
         extensions.push_back(VK_KHR_SURFACE_EXTENSION_NAME);
     }
 
+    if (Config::allowHDR()) {
+        extensions.push_back(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME);
+    }
+
     if (enable_debug_utils) {
         extensions.push_back(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
     }

src/video_core/renderer_vulkan/vk_presenter.cpp

@@ -397,6 +397,7 @@ void Presenter::RecreateFrame(Frame* frame, u32 width, u32 height) {
     frame->height = height;
 
     frame->imgui_texture = ImGui::Vulkan::AddTexture(view, vk::ImageLayout::eShaderReadOnlyOptimal);
+    frame->is_hdr = swapchain.GetHDR();
 }
 
 Frame* Presenter::PrepareLastFrame() {
@@ -562,7 +563,8 @@ Frame* Presenter::PrepareFrameInternal(VideoCore::ImageId image_id, bool is_eop)
     if (image_id != VideoCore::NULL_IMAGE_ID) {
         const auto& image = texture_cache.GetImage(image_id);
         const auto extent = image.info.size;
-        if (frame->width != extent.width || frame->height != extent.height) {
+        if (frame->width != extent.width || frame->height != extent.height ||
+            frame->is_hdr != swapchain.GetHDR()) {
             RecreateFrame(frame, extent.width, extent.height);
         }
     }
@@ -913,7 +915,7 @@ Frame* Presenter::GetRenderFrame() {
     }
 
     // Initialize default frame image
-    if (frame->width == 0 || frame->height == 0) {
+    if (frame->width == 0 || frame->height == 0 || frame->is_hdr != swapchain.GetHDR()) {
         RecreateFrame(frame, 1920, 1080);
     }
 

src/video_core/renderer_vulkan/vk_presenter.h

@@ -31,6 +31,7 @@ struct Frame {
     vk::Fence present_done;
     vk::Semaphore ready_semaphore;
     u64 ready_tick;
+    bool is_hdr{false};
 
     ImTextureID imgui_texture;
 };
@@ -46,6 +47,7 @@ class Rasterizer;
 class Presenter {
     struct PostProcessSettings {
         float gamma = 1.0f;
+        bool hdr = false;
     };
 
 public:
@@ -102,6 +104,18 @@ public:
         return *rasterizer.get();
     }
 
+    bool IsHDRSupported() const {
+        return swapchain.HasHDR();
+    }
+
+    void SetHDR(bool enable) {
+        if (!IsHDRSupported()) {
+            return;
+        }
+        swapchain.SetHDR(enable);
+        pp_settings.hdr = enable;
+    }
+
 private:
     void CreatePostProcessPipeline();
     Frame* PrepareFrameInternal(VideoCore::ImageId image_id, bool is_eop = true);

src/video_core/renderer_vulkan/vk_swapchain.cpp

@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <limits>
 #include "common/assert.h"
+#include "common/config.h"
 #include "common/logging/log.h"
 #include "imgui/renderer/imgui_core.h"
 #include "sdl_window.h"
@@ -12,8 +13,13 @@
 
 namespace Vulkan {
 
-Swapchain::Swapchain(const Instance& instance_, const Frontend::WindowSDL& window)
+static constexpr vk::SurfaceFormatKHR SURFACE_FORMAT_HDR = {
-    : instance{instance_}, surface{CreateSurface(instance.GetInstance(), window)} {
+    .format = vk::Format::eA2B10G10R10UnormPack32,
+    .colorSpace = vk::ColorSpaceKHR::eHdr10St2084EXT,
+};
+
+Swapchain::Swapchain(const Instance& instance_, const Frontend::WindowSDL& window_)
+    : instance{instance_}, window{window_}, surface{CreateSurface(instance.GetInstance(), window)} {
     FindPresentFormat();
 
     Create(window.GetWidth(), window.GetHeight());
@@ -57,11 +63,12 @@ void Swapchain::Create(u32 width_, u32 height_) {
     const u32 queue_family_indices_count = exclusive ? 1u : 2u;
     const vk::SharingMode sharing_mode =
         exclusive ? vk::SharingMode::eExclusive : vk::SharingMode::eConcurrent;
+    const auto format = needs_hdr ? SURFACE_FORMAT_HDR : surface_format;
     const vk::SwapchainCreateInfoKHR swapchain_info = {
         .surface = surface,
         .minImageCount = image_count,
-        .imageFormat = surface_format.format,
+        .imageFormat = format.format,
-        .imageColorSpace = surface_format.colorSpace,
+        .imageColorSpace = format.colorSpace,
         .imageExtent = extent,
         .imageArrayLayers = 1,
         .imageUsage = vk::ImageUsageFlagBits::eColorAttachment |
@@ -86,10 +93,28 @@ void Swapchain::Create(u32 width_, u32 height_) {
 }
 
 void Swapchain::Recreate(u32 width_, u32 height_) {
-    LOG_DEBUG(Render_Vulkan, "Recreate the swapchain: width={} height={}", width_, height_);
+    LOG_DEBUG(Render_Vulkan, "Recreate the swapchain: width={} height={} HDR={}", width_, height_,
+              needs_hdr);
     Create(width_, height_);
 }
 
+void Swapchain::SetHDR(bool hdr) {
+    if (needs_hdr == hdr) {
+        return;
+    }
+
+    auto result = instance.GetDevice().waitIdle();
+    if (result != vk::Result::eSuccess) {
+        LOG_WARNING(ImGui, "Failed to wait for Vulkan device idle on mode change: {}",
+                    vk::to_string(result));
+    }
+
+    needs_hdr = hdr;
+    Recreate(width, height);
+    ImGui::Core::OnSurfaceFormatChange(needs_hdr ? SURFACE_FORMAT_HDR.format
+                                                 : surface_format.format);
+}
+
 bool Swapchain::AcquireNextImage() {
     vk::Device device = instance.GetDevice();
     vk::Result result =
@@ -144,6 +169,16 @@ void Swapchain::FindPresentFormat() {
     ASSERT_MSG(formats_result == vk::Result::eSuccess, "Failed to query surface formats: {}",
                vk::to_string(formats_result));
 
+    // Check if the device supports HDR formats. Here we care of Rec.2020 PQ only as it is expected
+    // game output. Other variants as e.g. linear Rec.2020 will require additional color space
+    // rotation
+    supports_hdr =
+        std::find_if(formats.begin(), formats.end(), [](const vk::SurfaceFormatKHR& format) {
+            return format == SURFACE_FORMAT_HDR;
+        }) != formats.end();
+    // Also make sure that user allowed us to use HDR
+    supports_hdr &= Config::allowHDR();
+
     // If there is a single undefined surface format, the device doesn't care, so we'll just use
     // RGBA sRGB.
     if (formats[0].format == vk::Format::eUndefined) {
@@ -262,7 +297,7 @@ void Swapchain::SetupImages() {
         auto [im_view_result, im_view] = device.createImageView(vk::ImageViewCreateInfo{
             .image = images[i],
             .viewType = vk::ImageViewType::e2D,
-            .format = surface_format.format,
+            .format = needs_hdr ? SURFACE_FORMAT_HDR.format : surface_format.format,
             .subresourceRange =
                 {
                     .aspectMask = vk::ImageAspectFlagBits::eColor,

src/video_core/renderer_vulkan/vk_swapchain.h

@@ -82,6 +82,16 @@ public:
         return present_ready[image_index];
     }
 
+    bool HasHDR() const {
+        return supports_hdr;
+    }
+
+    void SetHDR(bool hdr);
+
+    bool GetHDR() const {
+        return needs_hdr;
+    }
+
 private:
     /// Selects the best available swapchain image format
     void FindPresentFormat();
@@ -100,6 +110,7 @@ private:
 
 private:
     const Instance& instance;
+    const Frontend::WindowSDL& window;
     vk::SwapchainKHR swapchain{};
     vk::SurfaceKHR surface{};
     vk::SurfaceFormatKHR surface_format;
@@ -117,6 +128,8 @@ private:
     u32 image_index = 0;
     u32 frame_index = 0;
     bool needs_recreation = true;
+    bool needs_hdr = false;    // The game requested HDR swapchain
+    bool supports_hdr = false; // SC supports HDR output
 };
 
 } // namespace Vulkan

src/video_core/texture_cache/image_info.cpp

@@ -22,6 +22,7 @@ static vk::Format ConvertPixelFormat(const VideoOutFormat format) {
         return vk::Format::eR8G8B8A8Srgb;
     case VideoOutFormat::A2R10G10B10:
     case VideoOutFormat::A2R10G10B10Srgb:
+    case VideoOutFormat::A2R10G10B10Bt2020Pq:
         return vk::Format::eA2R10G10B10UnormPack32;
     default:
         break;