From 2378ff44e0964e0a165c85ade9f1cbb12a3f311c Mon Sep 17 00:00:00 2001 From: IndecisiveTurtle <47210458+raphaelthegreat@users.noreply.github.com> Date: Sat, 16 Nov 2024 01:31:36 +0200 Subject: [PATCH] sleepq: Separate and make faster --- CMakeLists.txt | 2 + src/core/address_space.cpp | 2 +- src/core/libraries/kernel/threads/condvar.cpp | 93 ++++------------ src/core/libraries/kernel/threads/pthread.h | 15 +-- src/core/libraries/kernel/threads/sleepq.cpp | 101 ++++++++++++++++++ src/core/libraries/kernel/threads/sleepq.h | 38 +++++++ .../libraries/kernel/threads/thread_state.cpp | 1 + 7 files changed, 165 insertions(+), 87 deletions(-) create mode 100644 src/core/libraries/kernel/threads/sleepq.cpp create mode 100644 src/core/libraries/kernel/threads/sleepq.h diff --git a/CMakeLists.txt b/CMakeLists.txt index fd71b8a61..f4b052af1 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -219,6 +219,8 @@ set(KERNEL_LIB src/core/libraries/kernel/threads/condvar.cpp src/core/libraries/kernel/threads/pthread_spec.cpp src/core/libraries/kernel/threads/rwlock.cpp src/core/libraries/kernel/threads/semaphore.cpp + src/core/libraries/kernel/threads/sleepq.cpp + src/core/libraries/kernel/threads/sleepq.h src/core/libraries/kernel/threads/stack.cpp src/core/libraries/kernel/threads/tcb.cpp src/core/libraries/kernel/threads/pthread.h diff --git a/src/core/address_space.cpp b/src/core/address_space.cpp index 68d3b45fc..24f5e9f87 100644 --- a/src/core/address_space.cpp +++ b/src/core/address_space.cpp @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later -#include #include +#include #include "common/alignment.h" #include "common/arch.h" #include "common/assert.h" diff --git a/src/core/libraries/kernel/threads/condvar.cpp b/src/core/libraries/kernel/threads/condvar.cpp index 67c1ed12d..0bfd8f018 100644 --- a/src/core/libraries/kernel/threads/condvar.cpp +++ b/src/core/libraries/kernel/threads/condvar.cpp @@ -7,6 +7,7 @@ #include "core/libraries/error_codes.h" #include "core/libraries/kernel/kernel.h" #include "core/libraries/kernel/threads/pthread.h" +#include "core/libraries/kernel/threads/sleepq.h" #include "core/libraries/libs.h" namespace Libraries::Kernel { @@ -21,59 +22,6 @@ static constexpr PthreadCondAttr PhreadCondattrDefault = { .c_clockid = ClockId::Realtime, }; -static std::mutex sc_lock; -static std::unordered_map sc_table; - -void SleepqAdd(void* wchan, Pthread* td) { - auto [it, is_new] = sc_table.try_emplace(wchan, td->sleepqueue); - if (!is_new) { - it->second->sq_freeq.push_front(td->sleepqueue); - } - td->sleepqueue = nullptr; - td->wchan = wchan; - it->second->sq_blocked.push_back(td); -} - -int SleepqRemove(SleepQueue* sq, Pthread* td) { - std::erase(sq->sq_blocked, td); - if (sq->sq_blocked.empty()) { - td->sleepqueue = sq; - sc_table.erase(td->wchan); - td->wchan = nullptr; - return 0; - } else { - td->sleepqueue = sq->sq_freeq.front(); - sq->sq_freeq.pop_front(); - td->wchan = nullptr; - return 1; - } -} - -void SleepqDrop(SleepQueue* sq, void (*callback)(Pthread*, void*), void* arg) { - if (sq->sq_blocked.empty()) { - return; - } - - sc_table.erase(sq); - Pthread* td = sq->sq_blocked.front(); - sq->sq_blocked.pop_front(); - - callback(td, arg); - - td->sleepqueue = sq; - td->wchan = nullptr; - - auto sq2 = sq->sq_freeq.begin(); - for (Pthread* td : sq->sq_blocked) { - callback(td, arg); - td->sleepqueue = *sq2; - td->wchan = nullptr; - ++sq2; - } - sq->sq_blocked.clear(); - sq->sq_freeq.clear(); -} - static int CondInit(PthreadCondT* cond, const PthreadCondAttrT* cond_attr, const char* name) { auto* cvp = new PthreadCond{}; if (cvp == nullptr) { @@ -154,7 +102,7 @@ int PthreadCond::Wait(PthreadMutexT* mutex, const OrbisKernelTimespec* abstime, Pthread* curthread = g_curthread; ASSERT_MSG(curthread->wchan == nullptr, "Thread was already on queue."); // _thr_testcancel(curthread); - sc_lock.lock(); + SleepqLock(this); /* * set __has_user_waiters before unlocking mutex, this allows @@ -171,32 +119,32 @@ int PthreadCond::Wait(PthreadMutexT* mutex, const OrbisKernelTimespec* abstime, int error = 0; for (;;) { - curthread->wake_sema.try_acquire(); - sc_lock.unlock(); + void(curthread->wake_sema.try_acquire()); + SleepqUnlock(this); //_thr_cancel_enter2(curthread, 0); int error = curthread->Sleep(abstime, usec) ? 0 : POSIX_ETIMEDOUT; //_thr_cancel_leave(curthread, 0); - sc_lock.lock(); + SleepqLock(this); if (curthread->wchan == nullptr) { error = 0; break; } else if (curthread->ShouldCancel()) { - SleepQueue* sq = sc_table[this]; + SleepQueue* sq = SleepqLookup(this); has_user_waiters = SleepqRemove(sq, curthread); - sc_lock.unlock(); + SleepqUnlock(this); curthread->mutex_obj = nullptr; mp->CvLock(recurse); return 0; } else if (error == POSIX_ETIMEDOUT) { - SleepQueue* sq = sc_table[this]; + SleepQueue* sq = SleepqLookup(this); has_user_waiters = SleepqRemove(sq, curthread); break; } UNREACHABLE(); } - sc_lock.unlock(); + SleepqUnlock(this); curthread->mutex_obj = nullptr; mp->CvLock(recurse); return error; @@ -230,14 +178,13 @@ int PS4_SYSV_ABI posix_pthread_cond_reltimedwait_np(PthreadCondT* cond, PthreadM int PthreadCond::Signal() { Pthread* curthread = g_curthread; - sc_lock.lock(); - auto it = sc_table.find(this); - if (it == sc_table.end()) { - sc_lock.unlock(); + SleepqLock(this); + SleepQueue* sq = SleepqLookup(this); + if (sq == nullptr) { + SleepqUnlock(this); return 0; } - SleepQueue* sq = it->second; Pthread* td = sq->sq_blocked.front(); PthreadMutex* mp = td->mutex_obj; has_user_waiters = SleepqRemove(sq, td); @@ -253,7 +200,7 @@ int PthreadCond::Signal() { waddr = &td->wake_sema; } - sc_lock.unlock(); + SleepqUnlock(this); if (waddr != nullptr) { waddr->release(); } @@ -293,16 +240,16 @@ int PthreadCond::Broadcast() { } }; - sc_lock.lock(); - auto it = sc_table.find(this); - if (it == sc_table.end()) { - sc_lock.unlock(); + SleepqLock(this); + SleepQueue* sq = SleepqLookup(this); + if (sq == nullptr) { + SleepqUnlock(this); return 0; } - SleepqDrop(it->second, drop_cb, &ba); + SleepqDrop(sq, drop_cb, &ba); has_user_waiters = 0; - sc_lock.unlock(); + SleepqUnlock(this); for (int i = 0; i < ba.count; i++) { ba.waddrs[i]->release(); diff --git a/src/core/libraries/kernel/threads/pthread.h b/src/core/libraries/kernel/threads/pthread.h index b4364eb6e..eff8807a1 100644 --- a/src/core/libraries/kernel/threads/pthread.h +++ b/src/core/libraries/kernel/threads/pthread.h @@ -8,7 +8,6 @@ #include #include #include -#include #include "common/enum.h" #include "core/libraries/kernel/time.h" @@ -23,9 +22,6 @@ namespace Libraries::Kernel { struct Pthread; -using ListBaseHook = - boost::intrusive::list_base_hook>; - enum class PthreadMutexFlags : u32 { TypeMask = 0xff, Defered = 0x200, @@ -46,7 +42,7 @@ enum class PthreadMutexProt : u32 { Protect = 2, }; -struct PthreadMutex : public ListBaseHook { +struct PthreadMutex { std::timed_mutex m_lock; PthreadMutexFlags m_flags; Pthread* m_owner; @@ -240,14 +236,7 @@ using PthreadEntryFunc = void* PS4_SYSV_ABI (*)(void*); constexpr u32 TidTerminated = 1; -struct SleepQueue { - std::list sq_blocked; - std::forward_list sq_freeq; - std::list sq_hash; - std::forward_list sq_flink; - void* sq_wchan; - int sq_type; -}; +struct SleepQueue; struct SchedParam { int sched_priority; diff --git a/src/core/libraries/kernel/threads/sleepq.cpp b/src/core/libraries/kernel/threads/sleepq.cpp new file mode 100644 index 000000000..d998141d0 --- /dev/null +++ b/src/core/libraries/kernel/threads/sleepq.cpp @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include "common/spin_lock.h" +#include "core/libraries/kernel/threads/pthread.h" +#include "core/libraries/kernel/threads/sleepq.h" + +namespace Libraries::Kernel { + +static constexpr int HASHSHIFT = 9; +static constexpr int HASHSIZE = (1 << HASHSHIFT); +#define SC_HASH(wchan) \ + ((u32)((((uintptr_t)(wchan) >> 3) ^ ((uintptr_t)(wchan) >> (HASHSHIFT + 3))) & (HASHSIZE - 1))) +#define SC_LOOKUP(wc) &sc_table[SC_HASH(wc)] + +struct SleepQueueChain { + Common::SpinLock sc_lock; + SleepqList sc_queues; + int sc_type; +}; + +static std::array sc_table{}; + +void SleepqLock(void* wchan) { + SleepQueueChain* sc = SC_LOOKUP(wchan); + sc->sc_lock.lock(); +} + +void SleepqUnlock(void* wchan) { + SleepQueueChain* sc = SC_LOOKUP(wchan); + sc->sc_lock.unlock(); +} + +SleepQueue* SleepqLookup(void* wchan) { + SleepQueueChain* sc = SC_LOOKUP(wchan); + for (auto& sq : sc->sc_queues) { + if (sq.sq_wchan == wchan) { + return std::addressof(sq); + } + } + return nullptr; +} + +void SleepqAdd(void* wchan, Pthread* td) { + SleepQueue* sq = SleepqLookup(wchan); + if (sq != nullptr) { + sq->sq_freeq.push_front(*td->sleepqueue); + } else { + SleepQueueChain* sc = SC_LOOKUP(wchan); + sq = td->sleepqueue; + sc->sc_queues.push_front(*sq); + sq->sq_wchan = wchan; + /* sq->sq_type = type; */ + } + td->sleepqueue = NULL; + td->wchan = wchan; + sq->sq_blocked.push_front(td); +} + +int SleepqRemove(SleepQueue* sq, Pthread* td) { + std::erase(sq->sq_blocked, td); + if (sq->sq_blocked.empty()) { + td->sleepqueue = sq; + sq->unlink(); + td->wchan = nullptr; + return 0; + } else { + td->sleepqueue = std::addressof(sq->sq_freeq.front()); + sq->sq_freeq.pop_front(); + td->wchan = nullptr; + return 1; + } +} + +void SleepqDrop(SleepQueue* sq, void (*callback)(Pthread*, void*), void* arg) { + if (sq->sq_blocked.empty()) { + return; + } + + sq->unlink(); + Pthread* td = sq->sq_blocked.front(); + sq->sq_blocked.pop_front(); + + callback(td, arg); + + td->sleepqueue = sq; + td->wchan = nullptr; + + auto sq2 = sq->sq_freeq.begin(); + for (Pthread* td : sq->sq_blocked) { + callback(td, arg); + td->sleepqueue = std::addressof(*sq2); + td->wchan = nullptr; + ++sq2; + } + sq->sq_blocked.clear(); + sq->sq_freeq.clear(); +} + +} // namespace Libraries::Kernel \ No newline at end of file diff --git a/src/core/libraries/kernel/threads/sleepq.h b/src/core/libraries/kernel/threads/sleepq.h new file mode 100644 index 000000000..9274942e3 --- /dev/null +++ b/src/core/libraries/kernel/threads/sleepq.h @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright 2024 shadPS4 Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include +#include +#include "common/types.h" + +namespace Libraries::Kernel { + +struct Pthread; + +using ListBaseHook = + boost::intrusive::list_base_hook>; + +using SleepqList = boost::intrusive::list>; + +struct SleepQueue : public ListBaseHook { + std::list sq_blocked; + SleepqList sq_freeq; + void* sq_wchan; + int sq_type; +}; + +void SleepqLock(void* wchan); + +void SleepqUnlock(void* wchan); + +SleepQueue* SleepqLookup(void* wchan); + +void SleepqAdd(void* wchan, Pthread* td); + +int SleepqRemove(SleepQueue* sq, Pthread* td); + +void SleepqDrop(SleepQueue* sq, void (*callback)(Pthread*, void*), void* arg); + +} // namespace Libraries::Kernel \ No newline at end of file diff --git a/src/core/libraries/kernel/threads/thread_state.cpp b/src/core/libraries/kernel/threads/thread_state.cpp index ddc4d4608..e968c39ae 100644 --- a/src/core/libraries/kernel/threads/thread_state.cpp +++ b/src/core/libraries/kernel/threads/thread_state.cpp @@ -6,6 +6,7 @@ #include "common/scope_exit.h" #include "core/libraries/error_codes.h" #include "core/libraries/kernel/threads/pthread.h" +#include "core/libraries/kernel/threads/sleepq.h" #include "core/libraries/kernel/threads/thread_state.h" #include "core/memory.h" #include "core/tls.h"