debug: Add renderdoc capture hotkey

src/core/CMakeLists.txt

@@ -864,6 +864,8 @@ add_library(core STATIC
     telemetry_session.h
     tools/freezer.cpp
     tools/freezer.h
+    tools/renderdoc.cpp
+    tools/renderdoc.h
 )
 
 if (MSVC)
@@ -879,6 +881,7 @@ else()
         -Werror=conversion
 
         -Wno-sign-conversion
+        -Wno-cast-function-type
 
         $<$<CXX_COMPILER_ID:Clang>:-fsized-deallocation>
     )
@@ -887,7 +890,7 @@ endif()
 create_target_directory_groups(core)
 
 target_link_libraries(core PUBLIC common PRIVATE audio_core network video_core nx_tzdb)
-target_link_libraries(core PUBLIC Boost::headers PRIVATE fmt::fmt nlohmann_json::nlohmann_json mbedtls Opus::opus)
+target_link_libraries(core PUBLIC Boost::headers PRIVATE fmt::fmt nlohmann_json::nlohmann_json mbedtls Opus::opus renderdoc)
 if (MINGW)
     target_link_libraries(core PRIVATE ${MSWSOCK_LIBRARY})
 endif()

src/core/core.cpp

@@ -51,6 +51,7 @@
 #include "core/reporter.h"
 #include "core/telemetry_session.h"
 #include "core/tools/freezer.h"
+#include "core/tools/renderdoc.h"
 #include "network/network.h"
 #include "video_core/host1x/host1x.h"
 #include "video_core/renderer_base.h"
@@ -281,6 +282,10 @@ struct System::Impl {
         microprofile_cpu[2] = MICROPROFILE_TOKEN(ARM_CPU2);
         microprofile_cpu[3] = MICROPROFILE_TOKEN(ARM_CPU3);
 
+        if (Settings::values.enable_renderdoc_hotkey) {
+            renderdoc_api = std::make_unique<Tools::RenderdocAPI>();
+        }
+
         LOG_DEBUG(Core, "Initialized OK");
 
         return SystemResultStatus::Success;
@@ -521,6 +526,8 @@ struct System::Impl {
     std::unique_ptr<Tools::Freezer> memory_freezer;
     std::array<u8, 0x20> build_id{};
 
+    std::unique_ptr<Tools::RenderdocAPI> renderdoc_api;
+
     /// Frontend applets
     Service::AM::Applets::AppletManager applet_manager;
 
@@ -1024,6 +1031,10 @@ const Network::RoomNetwork& System::GetRoomNetwork() const {
     return impl->room_network;
 }
 
+Tools::RenderdocAPI& System::GetRenderdocAPI() {
+    return *impl->renderdoc_api;
+}
+
 void System::RunServer(std::unique_ptr<Service::ServerManager>&& server_manager) {
     return impl->kernel.RunServer(std::move(server_manager));
 }

src/core/core.h

@@ -102,6 +102,10 @@ namespace Network {
 class RoomNetwork;
 }
 
+namespace Tools {
+class RenderdocAPI;
+}
+
 namespace Core {
 
 class ARM_Interface;
@@ -413,6 +417,8 @@ public:
     /// Gets an immutable reference to the Room Network.
     [[nodiscard]] const Network::RoomNetwork& GetRoomNetwork() const;
 
+    [[nodiscard]] Tools::RenderdocAPI& GetRenderdocAPI();
+
     void SetExitLocked(bool locked);
     bool GetExitLocked() const;
 

src/core/tools/renderdoc.cpp

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <renderdoc_app.h>
+
+#include "common/assert.h"
+#include "common/dynamic_library.h"
+#include "core/tools/renderdoc.h"
+
+#ifdef WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+namespace Tools {
+
+RenderdocAPI::RenderdocAPI() {
+#ifdef WIN32
+    if (HMODULE mod = GetModuleHandleA("renderdoc.dll")) {
+        const auto RENDERDOC_GetAPI =
+            reinterpret_cast<pRENDERDOC_GetAPI>(GetProcAddress(mod, "RENDERDOC_GetAPI"));
+        const s32 ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, (void**)&rdoc_api);
+        ASSERT(ret == 1);
+    }
+#else
+#ifdef ANDROID
+    static constexpr const char RENDERDOC_LIB[] = "libVkLayer_GLES_RenderDoc.so";
+#else
+    static constexpr const char RENDERDOC_LIB[] = "librenderdoc.so";
+#endif
+    if (void* mod = dlopen(RENDERDOC_LIB, RTLD_NOW | RTLD_NOLOAD)) {
+        const auto RENDERDOC_GetAPI =
+            reinterpret_cast<pRENDERDOC_GetAPI>(dlsym(mod, "RENDERDOC_GetAPI"));
+        const s32 ret = RENDERDOC_GetAPI(eRENDERDOC_API_Version_1_6_0, (void**)&rdoc_api);
+        ASSERT(ret == 1);
+    }
+#endif
+}
+
+RenderdocAPI::~RenderdocAPI() = default;
+
+void RenderdocAPI::ToggleCapture() {
+    if (!rdoc_api) [[unlikely]] {
+        return;
+    }
+    if (!is_capturing) {
+        rdoc_api->StartFrameCapture(NULL, NULL);
+    } else {
+        rdoc_api->EndFrameCapture(NULL, NULL);
+    }
+    is_capturing = !is_capturing;
+}
+
+} // namespace Tools

src/core/tools/renderdoc.h

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+struct RENDERDOC_API_1_6_0;
+
+namespace Tools {
+
+class RenderdocAPI {
+public:
+    explicit RenderdocAPI();
+    ~RenderdocAPI();
+
+    void ToggleCapture();
+
+private:
+    RENDERDOC_API_1_6_0* rdoc_api{};
+    bool is_capturing{false};
+};
+
+} // namespace Tools