kernel: Rewrite pthread emulation (#1440)

* libkernel: Cleanup some function places

* kernel: Refactor thread functions

* kernel: It builds

* kernel: Fix a bunch of bugs, kernel thread heap

* kernel: File cleanup pt1

* File cleanup pt2

* File cleanup pt3

* File cleanup pt4

* kernel: Add missing funcs

* kernel: Add basic exceptions for linux

* gnmdriver: Add workload functions

* kernel: Fix new pthreads code on macOS. (#1441)

* kernel: Downgrade edeadlk to log

* gnmdriver: Add sceGnmSubmitCommandBuffersForWorkload

* exception: Add context register population for macOS. (#1444)

* kernel: Pthread rewrite touchups for Windows

* kernel: Multiplatform thread implementation

* mutex: Remove spamming log

* pthread_spec: Make assert into a log

* pthread_spec: Zero initialize array

* Attempt to fix non-Windows builds

* hotfix: change incorrect NID for scePthreadAttrSetaffinity

* scePthreadAttrSetaffinity implementation

* Attempt to fix Linux

* windows: Address a bunch of address space problems

* address_space: Fix unmap of region surrounded by placeholders

* libs: Reduce logging

* pthread: Implement condvar with waitable atomics and sleepqueue

* sleepq: Separate and make faster

* time: Remove delay execution

* Causes high cpu usage in Tohou Luna Nights

* kernel: Cleanup files again

* pthread: Add missing include

* semaphore: Use binary_semaphore instead of condvar

* Seems more reliable

* libraries/sysmodule: log module on `sceSysmoduleIsLoaded`

* libraries/kernel: implement `scePthreadSetPrio`

---------

Co-authored-by: squidbus <175574877+squidbus@users.noreply.github.com>
Co-authored-by: Daniel R. <47796739+polybiusproxy@users.noreply.github.com>

src/core/libraries/avplayer/avplayer.cpp

@@ -1,18 +1,14 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include "avplayer.h"
-
-#include "avplayer_impl.h"
 #include "common/logging/log.h"
+#include "core/libraries/avplayer/avplayer.h"
+#include "core/libraries/avplayer/avplayer_impl.h"
 #include "core/libraries/error_codes.h"
-#include "core/libraries/kernel/thread_management.h"
 #include "core/libraries/libs.h"
 
 namespace Libraries::AvPlayer {
 
-using namespace Kernel;
-
 s32 PS4_SYSV_ABI sceAvPlayerAddSource(SceAvPlayerHandle handle, const char* filename) {
     LOG_TRACE(Lib_AvPlayer, "filename = {}", filename);
     if (handle == nullptr) {
@@ -309,7 +305,7 @@ void RegisterlibSceAvPlayer(Core::Loader::SymbolsResolver* sym) {
     LIB_FUNCTION("XC9wM+xULz8", "libSceAvPlayer", 1, "libSceAvPlayer", 1, 0, sceAvPlayerJumpToTime);
     LIB_FUNCTION("9y5v+fGN4Wk", "libSceAvPlayer", 1, "libSceAvPlayer", 1, 0, sceAvPlayerPause);
     LIB_FUNCTION("HD1YKVU26-M", "libSceAvPlayer", 1, "libSceAvPlayer", 1, 0, sceAvPlayerPostInit);
-    LIB_FUNCTION("agig-iDRrTE", "libSceAvPlayer", 1, "libSceAvPlayer", 1, 0, sceAvPlayerPrintf);
+    // LIB_FUNCTION("agig-iDRrTE", "libSceAvPlayer", 1, "libSceAvPlayer", 1, 0, sceAvPlayerPrintf);
     LIB_FUNCTION("w5moABNwnRY", "libSceAvPlayer", 1, "libSceAvPlayer", 1, 0, sceAvPlayerResume);
     LIB_FUNCTION("k-q+xOxdc3E", "libSceAvPlayer", 1, "libSceAvPlayer", 1, 0,
                  sceAvPlayerSetAvSyncMode);

src/core/libraries/avplayer/avplayer.h

@@ -5,8 +5,8 @@
 
 #include "common/types.h"
 
-#include <stdarg.h> // va_list
-#include <stddef.h> // size_t
+#include <cstdarg> // va_list
+#include <cstddef> // size_t
 
 namespace Core::Loader {
 class SymbolsResolver;

src/core/libraries/avplayer/avplayer_common.cpp

@@ -1,24 +1,16 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include "avplayer.h"
-#include "avplayer_common.h"
+#include <algorithm> // std::equal
+#include <cctype>    // std::tolower
 
-#include <algorithm>   // std::equal
-#include <cctype>      // std::tolower
-#include <string_view> // std::string_view
+#include "core/libraries/avplayer/avplayer.h"
+#include "core/libraries/avplayer/avplayer_common.h"
 
 namespace Libraries::AvPlayer {
 
-using namespace Kernel;
-
-static bool ichar_equals(char a, char b) {
-    return std::tolower(static_cast<unsigned char>(a)) ==
-           std::tolower(static_cast<unsigned char>(b));
-}
-
 static bool iequals(std::string_view l, std::string_view r) {
-    return std::ranges::equal(l, r, ichar_equals);
+    return std::ranges::equal(l, r, [](u8 a, u8 b) { return std::tolower(a) == std::tolower(b); });
 }
 
 SceAvPlayerSourceType GetSourceType(std::string_view path) {

src/core/libraries/avplayer/avplayer_common.h

@@ -3,16 +3,14 @@
 
 #pragma once
 
-#include "avplayer.h"
-
-#include "common/assert.h"
-#include "common/logging/log.h"
-#include "core/libraries/kernel/thread_management.h"
-
+#include <mutex>
 #include <optional>
+#include <string_view>
 #include <utility>
 #include <queue>
 
+#include "core/libraries/avplayer/avplayer.h"
+
 #define AVPLAYER_IS_ERROR(x) ((x) < 0)
 
 namespace Libraries::AvPlayer {

src/core/libraries/avplayer/avplayer_file_streamer.cpp

@@ -1,20 +1,16 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include "avplayer_file_streamer.h"
-
-#include "avplayer_common.h"
-
+#include <algorithm> // std::max, std::min
 #include <magic_enum.hpp>
+#include "core/libraries/avplayer/avplayer_file_streamer.h"
 
 extern "C" {
 #include <libavformat/avformat.h>
 #include <libavformat/avio.h>
 }
 
-#include <algorithm> // std::max, std::min
-
-#define AVPLAYER_AVIO_BUFFER_SIZE 4096
+constexpr u32 AVPLAYER_AVIO_BUFFER_SIZE = 4096;
 
 namespace Libraries::AvPlayer {
 

src/core/libraries/avplayer/avplayer_file_streamer.h

@@ -3,11 +3,9 @@
 
 #pragma once
 
-#include "avplayer.h"
-#include "avplayer_data_streamer.h"
-
 #include <string_view>
-#include <vector>
+#include "core/libraries/avplayer/avplayer.h"
+#include "core/libraries/avplayer/avplayer_data_streamer.h"
 
 struct AVIOContext;
 

src/core/libraries/avplayer/avplayer_impl.cpp

@@ -1,17 +1,10 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include "avplayer_common.h"
-#include "avplayer_file_streamer.h"
-#include "avplayer_impl.h"
-
-#include "common/logging/log.h"
-#include "common/singleton.h"
+#include "core/libraries/avplayer/avplayer_common.h"
+#include "core/libraries/avplayer/avplayer_impl.h"
 #include "core/libraries/error_codes.h"
-#include "core/libraries/kernel/libkernel.h"
-#include "core/linker.h"
-
-using namespace Libraries::Kernel;
+#include "core/tls.h"
 
 namespace Libraries::AvPlayer {
 
@@ -19,32 +12,28 @@ void* PS4_SYSV_ABI AvPlayer::Allocate(void* handle, u32 alignment, u32 size) {
     const auto* const self = reinterpret_cast<AvPlayer*>(handle);
     const auto allocate = self->m_init_data_original.memory_replacement.allocate;
     const auto ptr = self->m_init_data_original.memory_replacement.object_ptr;
-    const auto* linker = Common::Singleton<Core::Linker>::Instance();
-    return linker->ExecuteGuest(allocate, ptr, alignment, size);
+    return Core::ExecuteGuest(allocate, ptr, alignment, size);
 }
 
 void PS4_SYSV_ABI AvPlayer::Deallocate(void* handle, void* memory) {
     const auto* const self = reinterpret_cast<AvPlayer*>(handle);
     const auto deallocate = self->m_init_data_original.memory_replacement.deallocate;
     const auto ptr = self->m_init_data_original.memory_replacement.object_ptr;
-    const auto* linker = Common::Singleton<Core::Linker>::Instance();
-    return linker->ExecuteGuest(deallocate, ptr, memory);
+    return Core::ExecuteGuest(deallocate, ptr, memory);
 }
 
 void* PS4_SYSV_ABI AvPlayer::AllocateTexture(void* handle, u32 alignment, u32 size) {
     const auto* const self = reinterpret_cast<AvPlayer*>(handle);
     const auto allocate = self->m_init_data_original.memory_replacement.allocate_texture;
     const auto ptr = self->m_init_data_original.memory_replacement.object_ptr;
-    const auto* linker = Common::Singleton<Core::Linker>::Instance();
-    return linker->ExecuteGuest(allocate, ptr, alignment, size);
+    return Core::ExecuteGuest(allocate, ptr, alignment, size);
 }
 
 void PS4_SYSV_ABI AvPlayer::DeallocateTexture(void* handle, void* memory) {
     const auto* const self = reinterpret_cast<AvPlayer*>(handle);
     const auto deallocate = self->m_init_data_original.memory_replacement.deallocate_texture;
     const auto ptr = self->m_init_data_original.memory_replacement.object_ptr;
-    const auto* linker = Common::Singleton<Core::Linker>::Instance();
-    return linker->ExecuteGuest(deallocate, ptr, memory);
+    return Core::ExecuteGuest(deallocate, ptr, memory);
 }
 
 int PS4_SYSV_ABI AvPlayer::OpenFile(void* handle, const char* filename) {
@@ -53,8 +42,7 @@ int PS4_SYSV_ABI AvPlayer::OpenFile(void* handle, const char* filename) {
 
     const auto open = self->m_init_data_original.file_replacement.open;
     const auto ptr = self->m_init_data_original.file_replacement.object_ptr;
-    const auto* linker = Common::Singleton<Core::Linker>::Instance();
-    return linker->ExecuteGuest(open, ptr, filename);
+    return Core::ExecuteGuest(open, ptr, filename);
 }
 
 int PS4_SYSV_ABI AvPlayer::CloseFile(void* handle) {
@@ -63,8 +51,7 @@ int PS4_SYSV_ABI AvPlayer::CloseFile(void* handle) {
 
     const auto close = self->m_init_data_original.file_replacement.close;
     const auto ptr = self->m_init_data_original.file_replacement.object_ptr;
-    const auto* linker = Common::Singleton<Core::Linker>::Instance();
-    return linker->ExecuteGuest(close, ptr);
+    return Core::ExecuteGuest(close, ptr);
 }
 
 int PS4_SYSV_ABI AvPlayer::ReadOffsetFile(void* handle, u8* buffer, u64 position, u32 length) {
@@ -73,8 +60,7 @@ int PS4_SYSV_ABI AvPlayer::ReadOffsetFile(void* handle, u8* buffer, u64 position
 
     const auto read_offset = self->m_init_data_original.file_replacement.readOffset;
     const auto ptr = self->m_init_data_original.file_replacement.object_ptr;
-    const auto* linker = Common::Singleton<Core::Linker>::Instance();
-    return linker->ExecuteGuest(read_offset, ptr, buffer, position, length);
+    return Core::ExecuteGuest(read_offset, ptr, buffer, position, length);
 }
 
 u64 PS4_SYSV_ABI AvPlayer::SizeFile(void* handle) {
@@ -83,8 +69,7 @@ u64 PS4_SYSV_ABI AvPlayer::SizeFile(void* handle) {
 
     const auto size = self->m_init_data_original.file_replacement.size;
     const auto ptr = self->m_init_data_original.file_replacement.object_ptr;
-    const auto* linker = Common::Singleton<Core::Linker>::Instance();
-    return linker->ExecuteGuest(size, ptr);
+    return Core::ExecuteGuest(size, ptr);
 }
 
 SceAvPlayerInitData AvPlayer::StubInitData(const SceAvPlayerInitData& data) {

src/core/libraries/avplayer/avplayer_impl.h

@@ -3,11 +3,8 @@
 
 #pragma once
 
-#include "avplayer.h"
-#include "avplayer_data_streamer.h"
-#include "avplayer_state.h"
-
-#include "core/libraries/kernel/thread_management.h"
+#include "core/libraries/avplayer/avplayer.h"
+#include "core/libraries/avplayer/avplayer_state.h"
 
 #include <mutex>
 
@@ -17,7 +14,6 @@ extern "C" {
 }
 
 #include <memory>
-#include <vector>
 
 namespace Libraries::AvPlayer {
 

src/core/libraries/avplayer/avplayer_source.cpp

@@ -1,16 +1,12 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include "avplayer_source.h"
-
-#include "avplayer_file_streamer.h"
-
 #include "common/alignment.h"
 #include "common/singleton.h"
 #include "common/thread.h"
-
 #include "core/file_sys/fs.h"
-#include "core/libraries/kernel/time_management.h"
+#include "core/libraries/avplayer/avplayer_file_streamer.h"
+#include "core/libraries/avplayer/avplayer_source.h"
 
 #include <magic_enum.hpp>
 
@@ -35,8 +31,6 @@ av_always_inline std::string av_err2string(int errnum) {
 
 namespace Libraries::AvPlayer {
 
-using namespace Kernel;
-
 AvPlayerSource::AvPlayerSource(AvPlayerStateCallback& state, bool use_vdec2)
     : m_state(state), m_use_vdec2(use_vdec2) {}
 
@@ -258,11 +252,9 @@ bool AvPlayerSource::Start() {
         LOG_ERROR(Lib_AvPlayer, "Could not start playback. NULL context.");
         return false;
     }
-    m_demuxer_thread = std::jthread([this](std::stop_token stop) { this->DemuxerThread(stop); });
-    m_video_decoder_thread =
-        std::jthread([this](std::stop_token stop) { this->VideoDecoderThread(stop); });
-    m_audio_decoder_thread =
-        std::jthread([this](std::stop_token stop) { this->AudioDecoderThread(stop); });
+    m_demuxer_thread.Run([this](std::stop_token stop) { this->DemuxerThread(stop); });
+    m_video_decoder_thread.Run([this](std::stop_token stop) { this->VideoDecoderThread(stop); });
+    m_audio_decoder_thread.Run([this](std::stop_token stop) { this->AudioDecoderThread(stop); });
     m_start_time = std::chrono::high_resolution_clock::now();
     return true;
 }
@@ -275,18 +267,10 @@ bool AvPlayerSource::Stop() {
         return false;
     }
 
-    m_video_decoder_thread.request_stop();
-    m_audio_decoder_thread.request_stop();
-    m_demuxer_thread.request_stop();
-    if (m_demuxer_thread.joinable()) {
-        m_demuxer_thread.join();
-    }
-    if (m_video_decoder_thread.joinable()) {
-        m_video_decoder_thread.join();
-    }
-    if (m_audio_decoder_thread.joinable()) {
-        m_audio_decoder_thread.join();
-    }
+    m_video_decoder_thread.Stop();
+    m_audio_decoder_thread.Stop();
+    m_demuxer_thread.Stop();
+
     if (m_current_audio_frame.has_value()) {
         m_audio_buffers.Push(std::move(m_current_audio_frame.value()));
         m_current_audio_frame.reset();
@@ -510,12 +494,8 @@ void AvPlayerSource::DemuxerThread(std::stop_token stop) {
     m_video_frames_cv.Notify();
     m_audio_frames_cv.Notify();
 
-    if (m_video_decoder_thread.joinable()) {
-        m_video_decoder_thread.join();
-    }
-    if (m_audio_decoder_thread.joinable()) {
-        m_audio_decoder_thread.join();
-    }
+    m_video_decoder_thread.Join();
+    m_audio_decoder_thread.Join();
     m_state.OnEOF();
 
     LOG_INFO(Lib_AvPlayer, "Demuxer Thread exited normally");
@@ -808,8 +788,8 @@ void AvPlayerSource::AudioDecoderThread(std::stop_token stop) {
 }
 
 bool AvPlayerSource::HasRunningThreads() const {
-    return m_demuxer_thread.joinable() || m_video_decoder_thread.joinable() ||
-           m_audio_decoder_thread.joinable();
+    return m_demuxer_thread.Joinable() || m_video_decoder_thread.Joinable() ||
+           m_audio_decoder_thread.Joinable();
 }
 
 } // namespace Libraries::AvPlayer

src/core/libraries/avplayer/avplayer_source.h

@@ -3,20 +3,18 @@
 
 #pragma once
 
-#include "avplayer.h"
-#include "avplayer_common.h"
-#include "avplayer_data_streamer.h"
-
-#include "common/polyfill_thread.h"
-#include "common/types.h"
-#include "core/libraries/kernel/thread_management.h"
-
 #include <atomic>
 #include <chrono>
 #include <condition_variable>
 #include <mutex>
 #include <optional>
-#include <string>
+#include <string_view>
+
+#include "common/assert.h"
+#include "core/libraries/avplayer/avplayer.h"
+#include "core/libraries/avplayer/avplayer_common.h"
+#include "core/libraries/avplayer/avplayer_data_streamer.h"
+#include "core/libraries/kernel/threads.h"
 
 struct AVCodecContext;
 struct AVFormatContext;
@@ -139,8 +137,6 @@ public:
     bool IsActive();
 
 private:
-    using ScePthread = Kernel::ScePthread;
-
     static void ReleaseAVPacket(AVPacket* packet);
     static void ReleaseAVFrame(AVFrame* frame);
     static void ReleaseAVCodecContext(AVCodecContext* context);
@@ -204,9 +200,9 @@ private:
     EventCV m_stop_cv{};
 
     std::mutex m_state_mutex{};
-    std::jthread m_demuxer_thread{};
-    std::jthread m_video_decoder_thread{};
-    std::jthread m_audio_decoder_thread{};
+    Kernel::Thread m_demuxer_thread{};
+    Kernel::Thread m_video_decoder_thread{};
+    Kernel::Thread m_audio_decoder_thread{};
 
     AVFormatContextPtr m_avformat_context{nullptr, &ReleaseAVFormatContext};
     AVCodecContextPtr m_video_codec_context{nullptr, &ReleaseAVCodecContext};

src/core/libraries/avplayer/avplayer_state.cpp

@@ -1,22 +1,17 @@
 // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project
 // SPDX-License-Identifier: GPL-2.0-or-later
 
-#include "avplayer_file_streamer.h"
-#include "avplayer_source.h"
-#include "avplayer_state.h"
-
-#include "common/singleton.h"
+#include "common/logging/log.h"
 #include "common/thread.h"
+#include "core/libraries/avplayer/avplayer_source.h"
+#include "core/libraries/avplayer/avplayer_state.h"
 #include "core/libraries/error_codes.h"
-#include "core/libraries/kernel/time_management.h"
-#include "core/linker.h"
+#include "core/tls.h"
 
 #include <magic_enum.hpp>
 
 namespace Libraries::AvPlayer {
 
-using namespace Kernel;
-
 void PS4_SYSV_ABI AvPlayerState::AutoPlayEventCallback(void* opaque, SceAvPlayerEvents event_id,
                                                        s32 source_id, void* event_data) {
     auto const self = reinterpret_cast<AvPlayerState*>(opaque);
@@ -96,8 +91,7 @@ void AvPlayerState::DefaultEventCallback(void* opaque, SceAvPlayerEvents event_i
     const auto callback = self->m_event_replacement.event_callback;
     const auto ptr = self->m_event_replacement.object_ptr;
     if (callback != nullptr) {
-        const auto* linker = Common::Singleton<Core::Linker>::Instance();
-        linker->ExecuteGuest(callback, ptr, event_id, 0, event_data);
+        Core::ExecuteGuest(callback, ptr, event_id, 0, event_data);
     }
 }
 
@@ -123,10 +117,7 @@ AvPlayerState::~AvPlayerState() {
         std::unique_lock lock(m_source_mutex);
         m_up_source.reset();
     }
-    if (m_controller_thread.joinable()) {
-        m_controller_thread.request_stop();
-        m_controller_thread.join();
-    }
+    m_controller_thread.Stop();
     m_event_queue.Clear();
 }
 
@@ -227,8 +218,7 @@ void AvPlayerState::WarningEvent(s32 id) {
 
 // Called inside GAME thread
 void AvPlayerState::StartControllerThread() {
-    m_controller_thread =
-        std::jthread([this](std::stop_token stop) { this->AvControllerThread(stop); });
+    m_controller_thread.Run([this](std::stop_token stop) { this->AvControllerThread(stop); });
 }
 
 // Called inside GAME thread

src/core/libraries/avplayer/avplayer_state.h

@@ -3,17 +3,14 @@
 
 #pragma once
 
-#include "avplayer.h"
-#include "avplayer_data_streamer.h"
-#include "avplayer_source.h"
-
-#include "common/polyfill_thread.h"
-#include "core/libraries/kernel/thread_management.h"
-
 #include <memory>
 #include <mutex>
 #include <shared_mutex>
 
+#include "core/libraries/avplayer/avplayer.h"
+#include "core/libraries/avplayer/avplayer_source.h"
+#include "core/libraries/kernel/threads.h"
+
 namespace Libraries::AvPlayer {
 
 class Stream;
@@ -83,7 +80,7 @@ private:
     std::shared_mutex m_source_mutex{};
     std::mutex m_state_machine_mutex{};
     std::mutex m_event_handler_mutex{};
-    std::jthread m_controller_thread{};
+    Kernel::Thread m_controller_thread{};
     AvPlayerQueue<AvPlayerEvent> m_event_queue{};
 };