video_out: HDR support (#2381)

* Initial HDR support

* fix for crashes when debug tools used

src/core/libraries/videoout/driver.h

@@ -18,7 +18,6 @@ struct Frame;
 namespace Libraries::VideoOut {
 
 struct VideoOutPort {
-    bool is_open = false;
     SceVideoOutResolutionStatus resolution;
     std::array<VideoOutBuffer, MaxDisplayBuffers> buffer_slots;
     std::array<u64, MaxDisplayBuffers> buffer_labels; // should be contiguous in memory
@@ -33,6 +32,8 @@ struct VideoOutPort {
     std::condition_variable vo_cv;
     std::condition_variable vblank_cv;
     int flip_rate = 0;
+    bool is_open = false;
+    bool is_mode_changing = false; // Used to prevent flip during mode change
 
     s32 FindFreeGroup() const {
         s32 index = 0;

src/core/libraries/videoout/video_out.cpp

@@ -3,6 +3,7 @@
 
 #include "common/assert.h"
 #include "common/config.h"
+#include "common/elf_info.h"
 #include "common/logging/log.h"
 #include "core/libraries/libs.h"
 #include "core/libraries/system/userservice.h"
@@ -315,6 +316,12 @@ s32 sceVideoOutSubmitEopFlip(s32 handle, u32 buf_id, u32 mode, u32 arg, void** u
 s32 PS4_SYSV_ABI sceVideoOutGetDeviceCapabilityInfo(
     s32 handle, SceVideoOutDeviceCapabilityInfo* pDeviceCapabilityInfo) {
     pDeviceCapabilityInfo->capability = 0;
+    if (presenter->IsHDRSupported()) {
+        auto& game_info = Common::ElfInfo::Instance();
+        if (game_info.GetPSFAttributes().support_hdr) {
+            pDeviceCapabilityInfo->capability |= ORBIS_VIDEO_OUT_DEVICE_CAPABILITY_BT2020_PQ;
+        }
+    }
     return ORBIS_OK;
 }
 
@@ -352,6 +359,49 @@ s32 PS4_SYSV_ABI sceVideoOutAdjustColor(s32 handle, const SceVideoOutColorSettin
     return ORBIS_OK;
 }
 
+struct Mode {
+    u32 size;
+    u8 encoding;
+    u8 range;
+    u8 colorimetry;
+    u8 depth;
+    u64 refresh_rate;
+    u64 resolution;
+    u8 reserved[8];
+};
+
+void PS4_SYSV_ABI sceVideoOutModeSetAny_(Mode* mode, u32 size) {
+    std::memset(mode, 0xff, size);
+    mode->size = size;
+}
+
+s32 PS4_SYSV_ABI sceVideoOutConfigureOutputMode_(s32 handle, u32 reserved, const Mode* mode,
+                                                 const void* options, u32 size_mode,
+                                                 u32 size_options) {
+    auto* port = driver->GetPort(handle);
+    if (!port) {
+        return ORBIS_VIDEO_OUT_ERROR_INVALID_HANDLE;
+    }
+
+    if (reserved != 0) {
+        return ORBIS_VIDEO_OUT_ERROR_INVALID_VALUE;
+    }
+
+    if (mode->colorimetry != OrbisVideoOutColorimetry::Any) {
+        auto& game_info = Common::ElfInfo::Instance();
+        if (mode->colorimetry == OrbisVideoOutColorimetry::Bt2020PQ &&
+            game_info.GetPSFAttributes().support_hdr) {
+            port->is_mode_changing = true;
+            presenter->SetHDR(true);
+            port->is_mode_changing = false;
+        } else {
+            return ORBIS_VIDEO_OUT_ERROR_INVALID_VALUE;
+        }
+    }
+
+    return ORBIS_OK;
+}
+
 void RegisterLib(Core::Loader::SymbolsResolver* sym) {
     driver = std::make_unique<VideoOutDriver>(Config::getScreenWidth(), Config::getScreenHeight());
 
@@ -390,6 +440,10 @@ void RegisterLib(Core::Loader::SymbolsResolver* sym) {
                  sceVideoOutAdjustColor);
     LIB_FUNCTION("-Ozn0F1AFRg", "libSceVideoOut", 1, "libSceVideoOut", 0, 0,
                  sceVideoOutDeleteFlipEvent);
+    LIB_FUNCTION("pjkDsgxli6c", "libSceVideoOut", 1, "libSceVideoOut", 0, 0,
+                 sceVideoOutModeSetAny_);
+    LIB_FUNCTION("N1bEoJ4SRw4", "libSceVideoOut", 1, "libSceVideoOut", 0, 0,
+                 sceVideoOutConfigureOutputMode_);
 
     // openOrbis appears to have libSceVideoOut_v1 module libSceVideoOut_v1.1
     LIB_FUNCTION("Up36PTk687E", "libSceVideoOut", 1, "libSceVideoOut", 1, 1, sceVideoOutOpen);

src/core/libraries/videoout/video_out.h

@@ -40,6 +40,13 @@ constexpr int SCE_VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_NONE = 0;
 constexpr int SCE_VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_VR = 7;
 constexpr int SCE_VIDEO_OUT_BUFFER_ATTRIBUTE_OPTION_STRICT_COLORIMETRY = 8;
 
+constexpr int ORBIS_VIDEO_OUT_DEVICE_CAPABILITY_BT2020_PQ = 0x80;
+
+enum OrbisVideoOutColorimetry : u8 {
+    Bt2020PQ = 12,
+    Any = 0xFF,
+};
+
 enum class OrbisVideoOutEventId : s16 {
     Flip = 0,
     Vblank = 1,