Mirrors/yuzu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #3955 from FernandoS27/prometheus-2b

Browse source 
Remake Kernel Scheduling, CPU Management & Boot Management (Prometheus)
This commit is contained in:
bunnei 2020-06-28 12:37:50 -04:00 • committed by GitHub
commit b05795d704
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
116 changed files with 4014 additions and 2336 deletions
Download patch file Download diff file Expand all files Collapse all files

228
src/core/hle/kernel/address_arbiter.cpp
View file

@ -7,11 +7,15 @@
#include "common/assert.h"
#include "common/common_types.h"
#include "core/arm/exclusive_monitor.h"
#include "core/core.h"
#include "core/hle/kernel/address_arbiter.h"
#include "core/hle/kernel/errors.h"
#include "core/hle/kernel/handle_table.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/thread.h"
#include "core/hle/kernel/time_manager.h"
#include "core/hle/result.h"
#include "core/memory.h"
@ -20,6 +24,7 @@ namespace Kernel {
// Wake up num_to_wake (or all) threads in a vector.
void AddressArbiter::WakeThreads(const std::vector<std::shared_ptr<Thread>>& waiting_threads,
s32 num_to_wake) {
auto& time_manager = system.Kernel().TimeManager();
// Only process up to 'target' threads, unless 'target' is <= 0, in which case process
// them all.
std::size_t last = waiting_threads.size();
@ -29,12 +34,10 @@ void AddressArbiter::WakeThreads(const std::vector<std::shared_ptr<Thread>>& wai
// Signal the waiting threads.
for (std::size_t i = 0; i < last; i++) {
ASSERT(waiting_threads[i]->GetStatus() == ThreadStatus::WaitArb);
waiting_threads[i]->SetWaitSynchronizationResult(RESULT_SUCCESS);
waiting_threads[i]->SetSynchronizationResults(nullptr, RESULT_SUCCESS);
RemoveThread(waiting_threads[i]);
waiting_threads[i]->SetArbiterWaitAddress(0);
waiting_threads[i]->WaitForArbitration(false);
waiting_threads[i]->ResumeFromWait();
system.PrepareReschedule(waiting_threads[i]->GetProcessorID());
}
}
@ -56,6 +59,7 @@ ResultCode AddressArbiter::SignalToAddress(VAddr address, SignalType type, s32 v
}
ResultCode AddressArbiter::SignalToAddressOnly(VAddr address, s32 num_to_wake) {
SchedulerLock lock(system.Kernel());
const std::vector<std::shared_ptr<Thread>> waiting_threads =
GetThreadsWaitingOnAddress(address);
WakeThreads(waiting_threads, num_to_wake);
@ -64,6 +68,7 @@ ResultCode AddressArbiter::SignalToAddressOnly(VAddr address, s32 num_to_wake) {
ResultCode AddressArbiter::IncrementAndSignalToAddressIfEqual(VAddr address, s32 value,
s32 num_to_wake) {
SchedulerLock lock(system.Kernel());
auto& memory = system.Memory();
// Ensure that we can write to the address.
@ -71,16 +76,24 @@ ResultCode AddressArbiter::IncrementAndSignalToAddressIfEqual(VAddr address, s32
return ERR_INVALID_ADDRESS_STATE;
}
if (static_cast<s32>(memory.Read32(address)) != value) {
return ERR_INVALID_STATE;
}
const std::size_t current_core = system.CurrentCoreIndex();
auto& monitor = system.Monitor();
u32 current_value;
do {
current_value = monitor.ExclusiveRead32(current_core, address);
if (current_value != value) {
return ERR_INVALID_STATE;
}
current_value++;
} while (!monitor.ExclusiveWrite32(current_core, address, current_value));
memory.Write32(address, static_cast<u32>(value + 1));
return SignalToAddressOnly(address, num_to_wake);
}
ResultCode AddressArbiter::ModifyByWaitingCountAndSignalToAddressIfEqual(VAddr address, s32 value,
s32 num_to_wake) {
SchedulerLock lock(system.Kernel());
auto& memory = system.Memory();
// Ensure that we can write to the address.
@ -92,29 +105,33 @@ ResultCode AddressArbiter::ModifyByWaitingCountAndSignalToAddressIfEqual(VAddr a
const std::vector<std::shared_ptr<Thread>> waiting_threads =
GetThreadsWaitingOnAddress(address);
// Determine the modified value depending on the waiting count.
const std::size_t current_core = system.CurrentCoreIndex();
auto& monitor = system.Monitor();
s32 updated_value;
if (num_to_wake <= 0) {
if (waiting_threads.empty()) {
updated_value = value + 1;
} else {
updated_value = value - 1;
}
} else {
if (waiting_threads.empty()) {
updated_value = value + 1;
} else if (waiting_threads.size() <= static_cast<u32>(num_to_wake)) {
updated_value = value - 1;
} else {
updated_value = value;
}
}
do {
updated_value = monitor.ExclusiveRead32(current_core, address);
if (static_cast<s32>(memory.Read32(address)) != value) {
return ERR_INVALID_STATE;
}
if (updated_value != value) {
return ERR_INVALID_STATE;
}
// Determine the modified value depending on the waiting count.
if (num_to_wake <= 0) {
if (waiting_threads.empty()) {
updated_value = value + 1;
} else {
updated_value = value - 1;
}
} else {
if (waiting_threads.empty()) {
updated_value = value + 1;
} else if (waiting_threads.size() <= static_cast<u32>(num_to_wake)) {
updated_value = value - 1;
} else {
updated_value = value;
}
}
} while (!monitor.ExclusiveWrite32(current_core, address, updated_value));
memory.Write32(address, static_cast<u32>(updated_value));
WakeThreads(waiting_threads, num_to_wake);
return RESULT_SUCCESS;
}
@ -136,60 +153,127 @@ ResultCode AddressArbiter::WaitForAddress(VAddr address, ArbitrationType type, s
ResultCode AddressArbiter::WaitForAddressIfLessThan(VAddr address, s32 value, s64 timeout,
bool should_decrement) {
auto& memory = system.Memory();
auto& kernel = system.Kernel();
Thread* current_thread = system.CurrentScheduler().GetCurrentThread();
// Ensure that we can read the address.
if (!memory.IsValidVirtualAddress(address)) {
return ERR_INVALID_ADDRESS_STATE;
Handle event_handle = InvalidHandle;
{
SchedulerLockAndSleep lock(kernel, event_handle, current_thread, timeout);
if (current_thread->IsPendingTermination()) {
lock.CancelSleep();
return ERR_THREAD_TERMINATING;
}
// Ensure that we can read the address.
if (!memory.IsValidVirtualAddress(address)) {
lock.CancelSleep();
return ERR_INVALID_ADDRESS_STATE;
}
s32 current_value = static_cast<s32>(memory.Read32(address));
if (current_value >= value) {
lock.CancelSleep();
return ERR_INVALID_STATE;
}
current_thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT);
s32 decrement_value;
const std::size_t current_core = system.CurrentCoreIndex();
auto& monitor = system.Monitor();
do {
current_value = static_cast<s32>(monitor.ExclusiveRead32(current_core, address));
if (should_decrement) {
decrement_value = current_value - 1;
} else {
decrement_value = current_value;
}
} while (
!monitor.ExclusiveWrite32(current_core, address, static_cast<u32>(decrement_value)));
// Short-circuit without rescheduling, if timeout is zero.
if (timeout == 0) {
lock.CancelSleep();
return RESULT_TIMEOUT;
}
current_thread->SetArbiterWaitAddress(address);
InsertThread(SharedFrom(current_thread));
current_thread->SetStatus(ThreadStatus::WaitArb);
current_thread->WaitForArbitration(true);
}
const s32 cur_value = static_cast<s32>(memory.Read32(address));
if (cur_value >= value) {
return ERR_INVALID_STATE;
if (event_handle != InvalidHandle) {
auto& time_manager = kernel.TimeManager();
time_manager.UnscheduleTimeEvent(event_handle);
}
if (should_decrement) {
memory.Write32(address, static_cast<u32>(cur_value - 1));
{
SchedulerLock lock(kernel);
if (current_thread->IsWaitingForArbitration()) {
RemoveThread(SharedFrom(current_thread));
current_thread->WaitForArbitration(false);
}
}
// Short-circuit without rescheduling, if timeout is zero.
if (timeout == 0) {
return RESULT_TIMEOUT;
}
return WaitForAddressImpl(address, timeout);
return current_thread->GetSignalingResult();
}
ResultCode AddressArbiter::WaitForAddressIfEqual(VAddr address, s32 value, s64 timeout) {
auto& memory = system.Memory();
// Ensure that we can read the address.
if (!memory.IsValidVirtualAddress(address)) {
return ERR_INVALID_ADDRESS_STATE;
}
// Only wait for the address if equal.
if (static_cast<s32>(memory.Read32(address)) != value) {
return ERR_INVALID_STATE;
}
// Short-circuit without rescheduling if timeout is zero.
if (timeout == 0) {
return RESULT_TIMEOUT;
}
return WaitForAddressImpl(address, timeout);
}
ResultCode AddressArbiter::WaitForAddressImpl(VAddr address, s64 timeout) {
auto& kernel = system.Kernel();
Thread* current_thread = system.CurrentScheduler().GetCurrentThread();
current_thread->SetArbiterWaitAddress(address);
InsertThread(SharedFrom(current_thread));
current_thread->SetStatus(ThreadStatus::WaitArb);
current_thread->InvalidateWakeupCallback();
current_thread->WakeAfterDelay(timeout);
system.PrepareReschedule(current_thread->GetProcessorID());
return RESULT_TIMEOUT;
Handle event_handle = InvalidHandle;
{
SchedulerLockAndSleep lock(kernel, event_handle, current_thread, timeout);
if (current_thread->IsPendingTermination()) {
lock.CancelSleep();
return ERR_THREAD_TERMINATING;
}
// Ensure that we can read the address.
if (!memory.IsValidVirtualAddress(address)) {
lock.CancelSleep();
return ERR_INVALID_ADDRESS_STATE;
}
s32 current_value = static_cast<s32>(memory.Read32(address));
if (current_value != value) {
lock.CancelSleep();
return ERR_INVALID_STATE;
}
// Short-circuit without rescheduling, if timeout is zero.
if (timeout == 0) {
lock.CancelSleep();
return RESULT_TIMEOUT;
}
current_thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT);
current_thread->SetArbiterWaitAddress(address);
InsertThread(SharedFrom(current_thread));
current_thread->SetStatus(ThreadStatus::WaitArb);
current_thread->WaitForArbitration(true);
}
if (event_handle != InvalidHandle) {
auto& time_manager = kernel.TimeManager();
time_manager.UnscheduleTimeEvent(event_handle);
}
{
SchedulerLock lock(kernel);
if (current_thread->IsWaitingForArbitration()) {
RemoveThread(SharedFrom(current_thread));
current_thread->WaitForArbitration(false);
}
}
return current_thread->GetSignalingResult();
}
void AddressArbiter::HandleWakeupThread(std::shared_ptr<Thread> thread) {
@ -221,9 +305,9 @@ void AddressArbiter::RemoveThread(std::shared_ptr<Thread> thread) {
const auto iter = std::find_if(thread_list.cbegin(), thread_list.cend(),
[&thread](const auto& entry) { return thread == entry; });
ASSERT(iter != thread_list.cend());
thread_list.erase(iter);
if (iter != thread_list.cend()) {
thread_list.erase(iter);
}
}
std::vector<std::shared_ptr<Thread>> AddressArbiter::GetThreadsWaitingOnAddress(

3
src/core/hle/kernel/address_arbiter.h
View file

@ -73,9 +73,6 @@ private:
/// Waits on an address if the value passed is equal to the argument value.
ResultCode WaitForAddressIfEqual(VAddr address, s32 value, s64 timeout);
// Waits on the given address with a timeout in nanoseconds
ResultCode WaitForAddressImpl(VAddr address, s64 timeout);
/// Wake up num_to_wake (or all) threads in a vector.
void WakeThreads(const std::vector<std::shared_ptr<Thread>>& waiting_threads, s32 num_to_wake);

2
src/core/hle/kernel/client_port.cpp
View file

@ -34,7 +34,7 @@ ResultVal<std::shared_ptr<ClientSession>> ClientPort::Connect() {
}
// Wake the threads waiting on the ServerPort
server_port->WakeupAllWaitingThreads();
server_port->Signal();
return MakeResult(std::move(client));
}

1
src/core/hle/kernel/errors.h
View file

@ -12,6 +12,7 @@ namespace Kernel {
constexpr ResultCode ERR_MAX_CONNECTIONS_REACHED{ErrorModule::Kernel, 7};
constexpr ResultCode ERR_INVALID_CAPABILITY_DESCRIPTOR{ErrorModule::Kernel, 14};
constexpr ResultCode ERR_THREAD_TERMINATING{ErrorModule::Kernel, 59};
constexpr ResultCode ERR_INVALID_SIZE{ErrorModule::Kernel, 101};
constexpr ResultCode ERR_INVALID_ADDRESS{ErrorModule::Kernel, 102};
constexpr ResultCode ERR_OUT_OF_RESOURCES{ErrorModule::Kernel, 103};

40
src/core/hle/kernel/hle_ipc.cpp
View file

@ -14,14 +14,17 @@
#include "common/common_types.h"
#include "common/logging/log.h"
#include "core/hle/ipc_helpers.h"
#include "core/hle/kernel/errors.h"
#include "core/hle/kernel/handle_table.h"
#include "core/hle/kernel/hle_ipc.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/kernel/object.h"
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/readable_event.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/server_session.h"
#include "core/hle/kernel/thread.h"
#include "core/hle/kernel/time_manager.h"
#include "core/hle/kernel/writable_event.h"
#include "core/memory.h"
@ -46,15 +49,6 @@ std::shared_ptr<WritableEvent> HLERequestContext::SleepClientThread(
const std::string& reason, u64 timeout, WakeupCallback&& callback,
std::shared_ptr<WritableEvent> writable_event) {
// Put the client thread to sleep until the wait event is signaled or the timeout expires.
thread->SetWakeupCallback(
[context = *this, callback](ThreadWakeupReason reason, std::shared_ptr<Thread> thread,
std::shared_ptr<SynchronizationObject> object,
std::size_t index) mutable -> bool {
ASSERT(thread->GetStatus() == ThreadStatus::WaitHLEEvent);
callback(thread, context, reason);
context.WriteToOutgoingCommandBuffer(*thread);
return true;
});
if (!writable_event) {
// Create event if not provided
@ -62,14 +56,26 @@ std::shared_ptr<WritableEvent> HLERequestContext::SleepClientThread(
writable_event = pair.writable;
}
const auto readable_event{writable_event->GetReadableEvent()};
writable_event->Clear();
thread->SetStatus(ThreadStatus::WaitHLEEvent);
thread->SetSynchronizationObjects({readable_event});
readable_event->AddWaitingThread(thread);
if (timeout > 0) {
thread->WakeAfterDelay(timeout);
{
Handle event_handle = InvalidHandle;
SchedulerLockAndSleep lock(kernel, event_handle, thread.get(), timeout);
thread->SetHLECallback(
[context = *this, callback](std::shared_ptr<Thread> thread) mutable -> bool {
ThreadWakeupReason reason = thread->GetSignalingResult() == RESULT_TIMEOUT
? ThreadWakeupReason::Timeout
: ThreadWakeupReason::Signal;
callback(thread, context, reason);
context.WriteToOutgoingCommandBuffer(*thread);
return true;
});
const auto readable_event{writable_event->GetReadableEvent()};
writable_event->Clear();
thread->SetHLESyncObject(readable_event.get());
thread->SetStatus(ThreadStatus::WaitHLEEvent);
thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT);
readable_event->AddWaitingThread(thread);
lock.Release();
thread->SetHLETimeEvent(event_handle);
}
is_thread_waiting = true;

255
src/core/hle/kernel/kernel.cpp
View file

@ -2,6 +2,7 @@
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include <array>
#include <atomic>
#include <bitset>
#include <functional>
@ -13,11 +14,15 @@
#include "common/assert.h"
#include "common/logging/log.h"
#include "common/microprofile.h"
#include "common/thread.h"
#include "core/arm/arm_interface.h"
#include "core/arm/cpu_interrupt_handler.h"
#include "core/arm/exclusive_monitor.h"
#include "core/core.h"
#include "core/core_timing.h"
#include "core/core_timing_util.h"
#include "core/cpu_manager.h"
#include "core/device_memory.h"
#include "core/hardware_properties.h"
#include "core/hle/kernel/client_port.h"
@ -39,85 +44,28 @@
#include "core/hle/result.h"
#include "core/memory.h"
MICROPROFILE_DEFINE(Kernel_SVC, "Kernel", "SVC", MP_RGB(70, 200, 70));
namespace Kernel {
/**
* Callback that will wake up the thread it was scheduled for
* @param thread_handle The handle of the thread that's been awoken
* @param cycles_late The number of CPU cycles that have passed since the desired wakeup time
*/
static void ThreadWakeupCallback(u64 thread_handle, [[maybe_unused]] s64 cycles_late) {
const auto proper_handle = static_cast<Handle>(thread_handle);
const auto& system = Core::System::GetInstance();
// Lock the global kernel mutex when we enter the kernel HLE.
std::lock_guard lock{HLE::g_hle_lock};
std::shared_ptr<Thread> thread =
system.Kernel().RetrieveThreadFromGlobalHandleTable(proper_handle);
if (thread == nullptr) {
LOG_CRITICAL(Kernel, "Callback fired for invalid thread {:08X}", proper_handle);
return;
}
bool resume = true;
if (thread->GetStatus() == ThreadStatus::WaitSynch ||
thread->GetStatus() == ThreadStatus::WaitHLEEvent) {
// Remove the thread from each of its waiting objects' waitlists
for (const auto& object : thread->GetSynchronizationObjects()) {
object->RemoveWaitingThread(thread);
}
thread->ClearSynchronizationObjects();
// Invoke the wakeup callback before clearing the wait objects
if (thread->HasWakeupCallback()) {
resume = thread->InvokeWakeupCallback(ThreadWakeupReason::Timeout, thread, nullptr, 0);
}
} else if (thread->GetStatus() == ThreadStatus::WaitMutex ||
thread->GetStatus() == ThreadStatus::WaitCondVar) {
thread->SetMutexWaitAddress(0);
thread->SetWaitHandle(0);
if (thread->GetStatus() == ThreadStatus::WaitCondVar) {
thread->GetOwnerProcess()->RemoveConditionVariableThread(thread);
thread->SetCondVarWaitAddress(0);
}
auto* const lock_owner = thread->GetLockOwner();
// Threads waking up by timeout from WaitProcessWideKey do not perform priority inheritance
// and don't have a lock owner unless SignalProcessWideKey was called first and the thread
// wasn't awakened due to the mutex already being acquired.
if (lock_owner != nullptr) {
lock_owner->RemoveMutexWaiter(thread);
}
}
if (thread->GetStatus() == ThreadStatus::WaitArb) {
auto& address_arbiter = thread->GetOwnerProcess()->GetAddressArbiter();
address_arbiter.HandleWakeupThread(thread);
}
if (resume) {
if (thread->GetStatus() == ThreadStatus::WaitCondVar ||
thread->GetStatus() == ThreadStatus::WaitArb) {
thread->SetWaitSynchronizationResult(RESULT_TIMEOUT);
}
thread->ResumeFromWait();
}
}
struct KernelCore::Impl {
explicit Impl(Core::System& system, KernelCore& kernel)
: global_scheduler{kernel}, synchronization{system}, time_manager{system}, system{system} {}
void SetMulticore(bool is_multicore) {
this->is_multicore = is_multicore;
}
void Initialize(KernelCore& kernel) {
Shutdown();
RegisterHostThread();
InitializePhysicalCores();
InitializeSystemResourceLimit(kernel);
InitializeMemoryLayout();
InitializeThreads();
InitializePreemption();
InitializePreemption(kernel);
InitializeSchedulers();
InitializeSuspendThreads();
}
void Shutdown() {
@ -126,13 +74,26 @@ struct KernelCore::Impl {
next_user_process_id = Process::ProcessIDMin;
next_thread_id = 1;
for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
if (suspend_threads[i]) {
suspend_threads[i].reset();
}
}
for (std::size_t i = 0; i < cores.size(); i++) {
cores[i].Shutdown();
schedulers[i].reset();
}
cores.clear();
registered_core_threads.reset();
process_list.clear();
current_process = nullptr;
system_resource_limit = nullptr;
global_handle_table.Clear();
thread_wakeup_event_type = nullptr;
preemption_event = nullptr;
global_scheduler.Shutdown();
@ -145,13 +106,21 @@ struct KernelCore::Impl {
cores.clear();
exclusive_monitor.reset();
host_thread_ids.clear();
}
void InitializePhysicalCores() {
exclusive_monitor =
Core::MakeExclusiveMonitor(system.Memory(), Core::Hardware::NUM_CPU_CORES);
for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
cores.emplace_back(system, i, *exclusive_monitor);
schedulers[i] = std::make_unique<Kernel::Scheduler>(system, i);
cores.emplace_back(system, i, *schedulers[i], interrupts[i]);
}
}
void InitializeSchedulers() {
for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
cores[i].Scheduler().Initialize();
}
}
@ -173,15 +142,13 @@ struct KernelCore::Impl {
}
}
void InitializeThreads() {
thread_wakeup_event_type =
Core::Timing::CreateEvent("ThreadWakeupCallback", ThreadWakeupCallback);
}
void InitializePreemption() {
preemption_event =
Core::Timing::CreateEvent("PreemptionCallback", [this](u64 userdata, s64 cycles_late) {
global_scheduler.PreemptThreads();
void InitializePreemption(KernelCore& kernel) {
preemption_event = Core::Timing::CreateEvent(
"PreemptionCallback", [this, &kernel](u64 userdata, s64 cycles_late) {
{
SchedulerLock lock(kernel);
global_scheduler.PreemptThreads();
}
s64 time_interval = Core::Timing::msToCycles(std::chrono::milliseconds(10));
system.CoreTiming().ScheduleEvent(time_interval, preemption_event);
});
@ -190,6 +157,20 @@ struct KernelCore::Impl {
system.CoreTiming().ScheduleEvent(time_interval, preemption_event);
}
void InitializeSuspendThreads() {
for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
std::string name = "Suspend Thread Id:" + std::to_string(i);
std::function<void(void*)> init_func =
system.GetCpuManager().GetSuspendThreadStartFunc();
void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
ThreadType type =
static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_SUSPEND);
auto thread_res = Thread::Create(system, type, name, 0, 0, 0, static_cast<u32>(i), 0,
nullptr, std::move(init_func), init_func_parameter);
suspend_threads[i] = std::move(thread_res).Unwrap();
}
}
void MakeCurrentProcess(Process* process) {
current_process = process;
@ -197,15 +178,17 @@ struct KernelCore::Impl {
return;
}
for (auto& core : cores) {
core.SetIs64Bit(process->Is64BitProcess());
u32 core_id = GetCurrentHostThreadID();
if (core_id < Core::Hardware::NUM_CPU_CORES) {
system.Memory().SetCurrentPageTable(*process, core_id);
}
system.Memory().SetCurrentPageTable(*process);
}
void RegisterCoreThread(std::size_t core_id) {
std::unique_lock lock{register_thread_mutex};
if (!is_multicore) {
single_core_thread_id = std::this_thread::get_id();
}
const std::thread::id this_id = std::this_thread::get_id();
const auto it = host_thread_ids.find(this_id);
ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
@ -219,12 +202,19 @@ struct KernelCore::Impl {
std::unique_lock lock{register_thread_mutex};
const std::thread::id this_id = std::this_thread::get_id();
const auto it = host_thread_ids.find(this_id);
ASSERT(it == host_thread_ids.end());
if (it != host_thread_ids.end()) {
return;
}
host_thread_ids[this_id] = registered_thread_ids++;
}
u32 GetCurrentHostThreadID() const {
const std::thread::id this_id = std::this_thread::get_id();
if (!is_multicore) {
if (single_core_thread_id == this_id) {
return static_cast<u32>(system.GetCpuManager().CurrentCore());
}
}
const auto it = host_thread_ids.find(this_id);
if (it == host_thread_ids.end()) {
return Core::INVALID_HOST_THREAD_ID;
@ -240,7 +230,7 @@ struct KernelCore::Impl {
}
const Kernel::Scheduler& sched = cores[result.host_handle].Scheduler();
const Kernel::Thread* current = sched.GetCurrentThread();
if (current != nullptr) {
if (current != nullptr && !current->IsPhantomMode()) {
result.guest_handle = current->GetGlobalHandle();
} else {
result.guest_handle = InvalidHandle;
@ -313,7 +303,6 @@ struct KernelCore::Impl {
std::shared_ptr<ResourceLimit> system_resource_limit;
std::shared_ptr<Core::Timing::EventType> thread_wakeup_event_type;
std::shared_ptr<Core::Timing::EventType> preemption_event;
// This is the kernel's handle table or supervisor handle table which
@ -343,6 +332,15 @@ struct KernelCore::Impl {
std::shared_ptr<Kernel::SharedMemory> irs_shared_mem;
std::shared_ptr<Kernel::SharedMemory> time_shared_mem;
std::array<std::shared_ptr<Thread>, Core::Hardware::NUM_CPU_CORES> suspend_threads{};
std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES> interrupts{};
std::array<std::unique_ptr<Kernel::Scheduler>, Core::Hardware::NUM_CPU_CORES> schedulers{};
bool is_multicore{};
std::thread::id single_core_thread_id{};
std::array<u64, Core::Hardware::NUM_CPU_CORES> svc_ticks{};
// System context
Core::System& system;
};
@ -352,6 +350,10 @@ KernelCore::~KernelCore() {
Shutdown();
}
void KernelCore::SetMulticore(bool is_multicore) {
impl->SetMulticore(is_multicore);
}
void KernelCore::Initialize() {
impl->Initialize(*this);
}
@ -397,11 +399,11 @@ const Kernel::GlobalScheduler& KernelCore::GlobalScheduler() const {
}
Kernel::Scheduler& KernelCore::Scheduler(std::size_t id) {
return impl->cores[id].Scheduler();
return *impl->schedulers[id];
}
const Kernel::Scheduler& KernelCore::Scheduler(std::size_t id) const {
return impl->cores[id].Scheduler();
return *impl->schedulers[id];
}
Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) {
@ -412,6 +414,39 @@ const Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) const {
return impl->cores[id];
}
Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() {
u32 core_id = impl->GetCurrentHostThreadID();
ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
return impl->cores[core_id];
}
const Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() const {
u32 core_id = impl->GetCurrentHostThreadID();
ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
return impl->cores[core_id];
}
Kernel::Scheduler& KernelCore::CurrentScheduler() {
u32 core_id = impl->GetCurrentHostThreadID();
ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
return *impl->schedulers[core_id];
}
const Kernel::Scheduler& KernelCore::CurrentScheduler() const {
u32 core_id = impl->GetCurrentHostThreadID();
ASSERT(core_id < Core::Hardware::NUM_CPU_CORES);
return *impl->schedulers[core_id];
}
std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& KernelCore::Interrupts() {
return impl->interrupts;
}
const std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& KernelCore::Interrupts()
const {
return impl->interrupts;
}
Kernel::Synchronization& KernelCore::Synchronization() {
return impl->synchronization;
}
@ -437,15 +472,17 @@ const Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() const {
}
void KernelCore::InvalidateAllInstructionCaches() {
for (std::size_t i = 0; i < impl->global_scheduler.CpuCoresCount(); i++) {
PhysicalCore(i).ArmInterface().ClearInstructionCache();
auto& threads = GlobalScheduler().GetThreadList();
for (auto& thread : threads) {
if (!thread->IsHLEThread()) {
auto& arm_interface = thread->ArmInterface();
arm_interface.ClearInstructionCache();
}
}
}
void KernelCore::PrepareReschedule(std::size_t id) {
if (id < impl->global_scheduler.CpuCoresCount()) {
impl->cores[id].Stop();
}
// TODO: Reimplement, this
}
void KernelCore::AddNamedPort(std::string name, std::shared_ptr<ClientPort> port) {
@ -481,10 +518,6 @@ u64 KernelCore::CreateNewUserProcessID() {
return impl->next_user_process_id++;
}
const std::shared_ptr<Core::Timing::EventType>& KernelCore::ThreadWakeupCallbackEventType() const {
return impl->thread_wakeup_event_type;
}
Kernel::HandleTable& KernelCore::GlobalHandleTable() {
return impl->global_handle_table;
}
@ -557,4 +590,34 @@ const Kernel::SharedMemory& KernelCore::GetTimeSharedMem() const {
return *impl->time_shared_mem;
}
void KernelCore::Suspend(bool in_suspention) {
const bool should_suspend = exception_exited || in_suspention;
{
SchedulerLock lock(*this);
ThreadStatus status = should_suspend ? ThreadStatus::Ready : ThreadStatus::WaitSleep;
for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
impl->suspend_threads[i]->SetStatus(status);
}
}
}
bool KernelCore::IsMulticore() const {
return impl->is_multicore;
}
void KernelCore::ExceptionalExit() {
exception_exited = true;
Suspend(true);
}
void KernelCore::EnterSVCProfile() {
std::size_t core = impl->GetCurrentHostThreadID();
impl->svc_ticks[core] = MicroProfileEnter(MICROPROFILE_TOKEN(Kernel_SVC));
}
void KernelCore::ExitSVCProfile() {
std::size_t core = impl->GetCurrentHostThreadID();
MicroProfileLeave(MICROPROFILE_TOKEN(Kernel_SVC), impl->svc_ticks[core]);
}
} // namespace Kernel

39
src/core/hle/kernel/kernel.h
View file

@ -4,15 +4,17 @@
#pragma once
#include <array>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "core/hardware_properties.h"
#include "core/hle/kernel/memory/memory_types.h"
#include "core/hle/kernel/object.h"
namespace Core {
struct EmuThreadHandle;
class CPUInterruptHandler;
class ExclusiveMonitor;
class System;
} // namespace Core
@ -65,6 +67,9 @@ public:
KernelCore(KernelCore&&) = delete;
KernelCore& operator=(KernelCore&&) = delete;
/// Sets if emulation is multicore or single core, must be set before Initialize
void SetMulticore(bool is_multicore);
/// Resets the kernel to a clean slate for use.
void Initialize();
@ -110,6 +115,18 @@ public:
/// Gets the an instance of the respective physical CPU core.
const Kernel::PhysicalCore& PhysicalCore(std::size_t id) const;
/// Gets the sole instance of the Scheduler at the current running core.
Kernel::Scheduler& CurrentScheduler();
/// Gets the sole instance of the Scheduler at the current running core.
const Kernel::Scheduler& CurrentScheduler() const;
/// Gets the an instance of the current physical CPU core.
Kernel::PhysicalCore& CurrentPhysicalCore();
/// Gets the an instance of the current physical CPU core.
const Kernel::PhysicalCore& CurrentPhysicalCore() const;
/// Gets the an instance of the Synchronization Interface.
Kernel::Synchronization& Synchronization();
@ -129,6 +146,10 @@ public:
const Core::ExclusiveMonitor& GetExclusiveMonitor() const;
std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& Interrupts();
const std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& Interrupts() const;
void InvalidateAllInstructionCaches();
/// Adds a port to the named port table
@ -191,6 +212,18 @@ public:
/// Gets the shared memory object for Time services.
const Kernel::SharedMemory& GetTimeSharedMem() const;
/// Suspend/unsuspend the OS.
void Suspend(bool in_suspention);
/// Exceptional exit the OS.
void ExceptionalExit();
bool IsMulticore() const;
void EnterSVCProfile();
void ExitSVCProfile();
private:
friend class Object;
friend class Process;
@ -208,9 +241,6 @@ private:
/// Creates a new thread ID, incrementing the internal thread ID counter.
u64 CreateNewThreadID();
/// Retrieves the event type used for thread wakeup callbacks.
const std::shared_ptr<Core::Timing::EventType>& ThreadWakeupCallbackEventType() const;
/// Provides a reference to the global handle table.
Kernel::HandleTable& GlobalHandleTable();
@ -219,6 +249,7 @@ private:
struct Impl;
std::unique_ptr<Impl> impl;
bool exception_exited{};
};
} // namespace Kernel

132
src/core/hle/kernel/mutex.cpp
View file

@ -34,8 +34,6 @@ static std::pair<std::shared_ptr<Thread>, u32> GetHighestPriorityMutexWaitingThr
if (thread->GetMutexWaitAddress() != mutex_addr)
continue;
ASSERT(thread->GetStatus() == ThreadStatus::WaitMutex);
++num_waiters;
if (highest_priority_thread == nullptr ||
thread->GetPriority() < highest_priority_thread->GetPriority()) {
@ -49,6 +47,7 @@ static std::pair<std::shared_ptr<Thread>, u32> GetHighestPriorityMutexWaitingThr
/// Update the mutex owner field of all threads waiting on the mutex to point to the new owner.
static void TransferMutexOwnership(VAddr mutex_addr, std::shared_ptr<Thread> current_thread,
std::shared_ptr<Thread> new_owner) {
current_thread->RemoveMutexWaiter(new_owner);
const auto threads = current_thread->GetMutexWaitingThreads();
for (const auto& thread : threads) {
if (thread->GetMutexWaitAddress() != mutex_addr)
@ -72,85 +71,100 @@ ResultCode Mutex::TryAcquire(VAddr address, Handle holding_thread_handle,
return ERR_INVALID_ADDRESS;
}
const auto& handle_table = system.Kernel().CurrentProcess()->GetHandleTable();
auto& kernel = system.Kernel();
std::shared_ptr<Thread> current_thread =
SharedFrom(system.CurrentScheduler().GetCurrentThread());
std::shared_ptr<Thread> holding_thread = handle_table.Get<Thread>(holding_thread_handle);
std::shared_ptr<Thread> requesting_thread = handle_table.Get<Thread>(requesting_thread_handle);
SharedFrom(kernel.CurrentScheduler().GetCurrentThread());
{
SchedulerLock lock(kernel);
// The mutex address must be 4-byte aligned
if ((address % sizeof(u32)) != 0) {
return ERR_INVALID_ADDRESS;
}
// TODO(Subv): It is currently unknown if it is possible to lock a mutex in behalf of another
// thread.
ASSERT(requesting_thread == current_thread);
const auto& handle_table = kernel.CurrentProcess()->GetHandleTable();
std::shared_ptr<Thread> holding_thread = handle_table.Get<Thread>(holding_thread_handle);
std::shared_ptr<Thread> requesting_thread =
handle_table.Get<Thread>(requesting_thread_handle);
const u32 addr_value = system.Memory().Read32(address);
// TODO(Subv): It is currently unknown if it is possible to lock a mutex in behalf of
// another thread.
ASSERT(requesting_thread == current_thread);
// If the mutex isn't being held, just return success.
if (addr_value != (holding_thread_handle | Mutex::MutexHasWaitersFlag)) {
return RESULT_SUCCESS;
current_thread->SetSynchronizationResults(nullptr, RESULT_SUCCESS);
const u32 addr_value = system.Memory().Read32(address);
// If the mutex isn't being held, just return success.
if (addr_value != (holding_thread_handle | Mutex::MutexHasWaitersFlag)) {
return RESULT_SUCCESS;
}
if (holding_thread == nullptr) {
return ERR_INVALID_HANDLE;
}
// Wait until the mutex is released
current_thread->SetMutexWaitAddress(address);
current_thread->SetWaitHandle(requesting_thread_handle);
current_thread->SetStatus(ThreadStatus::WaitMutex);
// Update the lock holder thread's priority to prevent priority inversion.
holding_thread->AddMutexWaiter(current_thread);
}
if (holding_thread == nullptr) {
LOG_ERROR(Kernel, "Holding thread does not exist! thread_handle={:08X}",
holding_thread_handle);
return ERR_INVALID_HANDLE;
{
SchedulerLock lock(kernel);
auto* owner = current_thread->GetLockOwner();
if (owner != nullptr) {
owner->RemoveMutexWaiter(current_thread);
}
}
// Wait until the mutex is released
current_thread->SetMutexWaitAddress(address);
current_thread->SetWaitHandle(requesting_thread_handle);
current_thread->SetStatus(ThreadStatus::WaitMutex);
current_thread->InvalidateWakeupCallback();
// Update the lock holder thread's priority to prevent priority inversion.
holding_thread->AddMutexWaiter(current_thread);
system.PrepareReschedule();
return RESULT_SUCCESS;
return current_thread->GetSignalingResult();
}
ResultCode Mutex::Release(VAddr address) {
std::pair<ResultCode, std::shared_ptr<Thread>> Mutex::Unlock(std::shared_ptr<Thread> owner,
VAddr address) {
// The mutex address must be 4-byte aligned
if ((address % sizeof(u32)) != 0) {
LOG_ERROR(Kernel, "Address is not 4-byte aligned! address={:016X}", address);
return ERR_INVALID_ADDRESS;
return {ERR_INVALID_ADDRESS, nullptr};
}
std::shared_ptr<Thread> current_thread =
SharedFrom(system.CurrentScheduler().GetCurrentThread());
auto [thread, num_waiters] = GetHighestPriorityMutexWaitingThread(current_thread, address);
// There are no more threads waiting for the mutex, release it completely.
if (thread == nullptr) {
auto [new_owner, num_waiters] = GetHighestPriorityMutexWaitingThread(owner, address);
if (new_owner == nullptr) {
system.Memory().Write32(address, 0);
return RESULT_SUCCESS;
return {RESULT_SUCCESS, nullptr};
}
// Transfer the ownership of the mutex from the previous owner to the new one.
TransferMutexOwnership(address, current_thread, thread);
u32 mutex_value = thread->GetWaitHandle();
TransferMutexOwnership(address, owner, new_owner);
u32 mutex_value = new_owner->GetWaitHandle();
if (num_waiters >= 2) {
// Notify the guest that there are still some threads waiting for the mutex
mutex_value |= Mutex::MutexHasWaitersFlag;
}
new_owner->SetSynchronizationResults(nullptr, RESULT_SUCCESS);
new_owner->SetLockOwner(nullptr);
new_owner->ResumeFromWait();
// Grant the mutex to the next waiting thread and resume it.
system.Memory().Write32(address, mutex_value);
ASSERT(thread->GetStatus() == ThreadStatus::WaitMutex);
thread->ResumeFromWait();
thread->SetLockOwner(nullptr);
thread->SetCondVarWaitAddress(0);
thread->SetMutexWaitAddress(0);
thread->SetWaitHandle(0);
thread->SetWaitSynchronizationResult(RESULT_SUCCESS);
system.PrepareReschedule();
return RESULT_SUCCESS;
return {RESULT_SUCCESS, new_owner};
}
ResultCode Mutex::Release(VAddr address) {
auto& kernel = system.Kernel();
SchedulerLock lock(kernel);
std::shared_ptr<Thread> current_thread =
SharedFrom(kernel.CurrentScheduler().GetCurrentThread());
auto [result, new_owner] = Unlock(current_thread, address);
if (result != RESULT_SUCCESS && new_owner != nullptr) {
new_owner->SetSynchronizationResults(nullptr, result);
}
return result;
}
} // namespace Kernel

4
src/core/hle/kernel/mutex.h
View file

@ -28,6 +28,10 @@ public:
ResultCode TryAcquire(VAddr address, Handle holding_thread_handle,
Handle requesting_thread_handle);
/// Unlocks a mutex for owner at address
std::pair<ResultCode, std::shared_ptr<Thread>> Unlock(std::shared_ptr<Thread> owner,
VAddr address);
/// Releases the mutex at the specified address.
ResultCode Release(VAddr address);

58
src/core/hle/kernel/physical_core.cpp
View file

@ -2,12 +2,15 @@
// Licensed under GPLv2 or any later version
// Refer to the license.txt file included.
#include "common/assert.h"
#include "common/logging/log.h"
#include "common/spin_lock.h"
#include "core/arm/arm_interface.h"
#ifdef ARCHITECTURE_x86_64
#include "core/arm/dynarmic/arm_dynarmic_32.h"
#include "core/arm/dynarmic/arm_dynarmic_64.h"
#endif
#include "core/arm/cpu_interrupt_handler.h"
#include "core/arm/exclusive_monitor.h"
#include "core/arm/unicorn/arm_unicorn.h"
#include "core/core.h"
@ -17,50 +20,37 @@
namespace Kernel {
PhysicalCore::PhysicalCore(Core::System& system, std::size_t id,
Core::ExclusiveMonitor& exclusive_monitor)
: core_index{id} {
#ifdef ARCHITECTURE_x86_64
arm_interface_32 =
std::make_unique<Core::ARM_Dynarmic_32>(system, exclusive_monitor, core_index);
arm_interface_64 =
std::make_unique<Core::ARM_Dynarmic_64>(system, exclusive_monitor, core_index);
PhysicalCore::PhysicalCore(Core::System& system, std::size_t id, Kernel::Scheduler& scheduler,
Core::CPUInterruptHandler& interrupt_handler)
: interrupt_handler{interrupt_handler}, core_index{id}, scheduler{scheduler} {
#else
using Core::ARM_Unicorn;
arm_interface_32 = std::make_unique<ARM_Unicorn>(system, ARM_Unicorn::Arch::AArch32);
arm_interface_64 = std::make_unique<ARM_Unicorn>(system, ARM_Unicorn::Arch::AArch64);
LOG_WARNING(Core, "CPU JIT requested, but Dynarmic not available");
#endif
scheduler = std::make_unique<Kernel::Scheduler>(system, core_index);
guard = std::make_unique<Common::SpinLock>();
}
PhysicalCore::~PhysicalCore() = default;
void PhysicalCore::Run() {
arm_interface->Run();
arm_interface->ClearExclusiveState();
}
void PhysicalCore::Step() {
arm_interface->Step();
}
void PhysicalCore::Stop() {
arm_interface->PrepareReschedule();
void PhysicalCore::Idle() {
interrupt_handler.AwaitInterrupt();
}
void PhysicalCore::Shutdown() {
scheduler->Shutdown();
scheduler.Shutdown();
}
void PhysicalCore::SetIs64Bit(bool is_64_bit) {
if (is_64_bit) {
arm_interface = arm_interface_64.get();
} else {
arm_interface = arm_interface_32.get();
}
bool PhysicalCore::IsInterrupted() const {
return interrupt_handler.IsInterrupted();
}
void PhysicalCore::Interrupt() {
guard->lock();
interrupt_handler.SetInterrupt(true);
guard->unlock();
}
void PhysicalCore::ClearInterrupt() {
guard->lock();
interrupt_handler.SetInterrupt(false);
guard->unlock();
}
} // namespace Kernel

44
src/core/hle/kernel/physical_core.h
View file

@ -7,12 +7,17 @@
#include <cstddef>
#include <memory>
namespace Common {
class SpinLock;
}
namespace Kernel {
class Scheduler;
} // namespace Kernel
namespace Core {
class ARM_Interface;
class CPUInterruptHandler;
class ExclusiveMonitor;
class System;
} // namespace Core
@ -21,7 +26,8 @@ namespace Kernel {
class PhysicalCore {
public:
PhysicalCore(Core::System& system, std::size_t id, Core::ExclusiveMonitor& exclusive_monitor);
PhysicalCore(Core::System& system, std::size_t id, Kernel::Scheduler& scheduler,
Core::CPUInterruptHandler& interrupt_handler);
~PhysicalCore();
PhysicalCore(const PhysicalCore&) = delete;
@ -30,24 +36,19 @@ public:
PhysicalCore(PhysicalCore&&) = default;
PhysicalCore& operator=(PhysicalCore&&) = default;
/// Execute current jit state
void Run();
/// Execute a single instruction in current jit.
void Step();
/// Stop JIT execution/exit
void Stop();
void Idle();
/// Interrupt this physical core.
void Interrupt();
/// Clear this core's interrupt
void ClearInterrupt();
/// Check if this core is interrupted
bool IsInterrupted() const;
// Shutdown this physical core.
void Shutdown();
Core::ARM_Interface& ArmInterface() {
return *arm_interface;
}
const Core::ARM_Interface& ArmInterface() const {
return *arm_interface;
}
bool IsMainCore() const {
return core_index == 0;
}
@ -61,21 +62,18 @@ public:
}
Kernel::Scheduler& Scheduler() {
return *scheduler;
return scheduler;
}
const Kernel::Scheduler& Scheduler() const {
return *scheduler;
return scheduler;
}
void SetIs64Bit(bool is_64_bit);
private:
Core::CPUInterruptHandler& interrupt_handler;
std::size_t core_index;
std::unique_ptr<Core::ARM_Interface> arm_interface_32;
std::unique_ptr<Core::ARM_Interface> arm_interface_64;
std::unique_ptr<Kernel::Scheduler> scheduler;
Core::ARM_Interface* arm_interface{};
Kernel::Scheduler& scheduler;
std::unique_ptr<Common::SpinLock> guard;
};
} // namespace Kernel

23
src/core/hle/kernel/process.cpp
View file

@ -22,6 +22,7 @@
#include "core/hle/kernel/resource_limit.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/thread.h"
#include "core/hle/lock.h"
#include "core/memory.h"
#include "core/settings.h"
@ -30,14 +31,15 @@ namespace {
/**
* Sets up the primary application thread
*
* @param system The system instance to create the main thread under.
* @param owner_process The parent process for the main thread
* @param kernel The kernel instance to create the main thread under.
* @param priority The priority to give the main thread
*/
void SetupMainThread(Process& owner_process, KernelCore& kernel, u32 priority, VAddr stack_top) {
void SetupMainThread(Core::System& system, Process& owner_process, u32 priority, VAddr stack_top) {
const VAddr entry_point = owner_process.PageTable().GetCodeRegionStart();
auto thread_res = Thread::Create(kernel, "main", entry_point, priority, 0,
owner_process.GetIdealCore(), stack_top, owner_process);
ThreadType type = THREADTYPE_USER;
auto thread_res = Thread::Create(system, type, "main", entry_point, priority, 0,
owner_process.GetIdealCore(), stack_top, &owner_process);
std::shared_ptr<Thread> thread = std::move(thread_res).Unwrap();
@ -48,8 +50,12 @@ void SetupMainThread(Process& owner_process, KernelCore& kernel, u32 priority, V
thread->GetContext32().cpu_registers[1] = thread_handle;
thread->GetContext64().cpu_registers[1] = thread_handle;
auto& kernel = system.Kernel();
// Threads by default are dormant, wake up the main thread so it runs when the scheduler fires
thread->ResumeFromWait();
{
SchedulerLock lock{kernel};
thread->SetStatus(ThreadStatus::Ready);
}
}
} // Anonymous namespace
@ -182,7 +188,6 @@ void Process::RemoveConditionVariableThread(std::shared_ptr<Thread> thread) {
}
++it;
}
UNREACHABLE();
}
std::vector<std::shared_ptr<Thread>> Process::GetConditionVariableThreads(
@ -207,6 +212,7 @@ void Process::UnregisterThread(const Thread* thread) {
}
ResultCode Process::ClearSignalState() {
SchedulerLock lock(system.Kernel());
if (status == ProcessStatus::Exited) {
LOG_ERROR(Kernel, "called on a terminated process instance.");
return ERR_INVALID_STATE;
@ -294,7 +300,7 @@ void Process::Run(s32 main_thread_priority, u64 stack_size) {
ChangeStatus(ProcessStatus::Running);
SetupMainThread(*this, kernel, main_thread_priority, main_thread_stack_top);
SetupMainThread(system, *this, main_thread_priority, main_thread_stack_top);
resource_limit->Reserve(ResourceType::Threads, 1);
resource_limit->Reserve(ResourceType::PhysicalMemory, main_thread_stack_size);
}
@ -340,6 +346,7 @@ static auto FindTLSPageWithAvailableSlots(std::vector<TLSPage>& tls_pages) {
}
VAddr Process::CreateTLSRegion() {
SchedulerLock lock(system.Kernel());
if (auto tls_page_iter{FindTLSPageWithAvailableSlots(tls_pages)};
tls_page_iter != tls_pages.cend()) {
return *tls_page_iter->ReserveSlot();
@ -370,6 +377,7 @@ VAddr Process::CreateTLSRegion() {
}
void Process::FreeTLSRegion(VAddr tls_address) {
SchedulerLock lock(system.Kernel());
const VAddr aligned_address = Common::AlignDown(tls_address, Core::Memory::PAGE_SIZE);
auto iter =
std::find_if(tls_pages.begin(), tls_pages.end(), [aligned_address](const auto& page) {
@ -384,6 +392,7 @@ void Process::FreeTLSRegion(VAddr tls_address) {
}
void Process::LoadModule(CodeSet code_set, VAddr base_addr) {
std::lock_guard lock{HLE::g_hle_lock};
const auto ReprotectSegment = [&](const CodeSet::Segment& segment,
Memory::MemoryPermission permission) {
page_table->SetCodeMemoryPermission(segment.addr + base_addr, segment.size, permission);

3
src/core/hle/kernel/readable_event.cpp
View file

@ -6,8 +6,10 @@
#include "common/assert.h"
#include "common/logging/log.h"
#include "core/hle/kernel/errors.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/kernel/object.h"
#include "core/hle/kernel/readable_event.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/thread.h"
namespace Kernel {
@ -37,6 +39,7 @@ void ReadableEvent::Clear() {
}
ResultCode ReadableEvent::Reset() {
SchedulerLock lock(kernel);
if (!is_signaled) {
LOG_TRACE(Kernel, "Handle is not signaled! object_id={}, object_type={}, object_name={}",
GetObjectId(), GetTypeName(), GetName());

582
src/core/hle/kernel/scheduler.cpp
View file

@ -11,11 +11,15 @@
#include <utility>
#include "common/assert.h"
#include "common/bit_util.h"
#include "common/fiber.h"
#include "common/logging/log.h"
#include "core/arm/arm_interface.h"
#include "core/core.h"
#include "core/core_timing.h"
#include "core/cpu_manager.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/kernel/physical_core.h"
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/time_manager.h"
@ -27,103 +31,151 @@ GlobalScheduler::GlobalScheduler(KernelCore& kernel) : kernel{kernel} {}
GlobalScheduler::~GlobalScheduler() = default;
void GlobalScheduler::AddThread(std::shared_ptr<Thread> thread) {
global_list_guard.lock();
thread_list.push_back(std::move(thread));
global_list_guard.unlock();
}
void GlobalScheduler::RemoveThread(std::shared_ptr<Thread> thread) {
global_list_guard.lock();
thread_list.erase(std::remove(thread_list.begin(), thread_list.end(), thread),
thread_list.end());
global_list_guard.unlock();
}
void GlobalScheduler::UnloadThread(std::size_t core) {
Scheduler& sched = kernel.Scheduler(core);
sched.UnloadThread();
}
void GlobalScheduler::SelectThread(std::size_t core) {
u32 GlobalScheduler::SelectThreads() {
ASSERT(is_locked);
const auto update_thread = [](Thread* thread, Scheduler& sched) {
if (thread != sched.selected_thread.get()) {
sched.guard.lock();
if (thread != sched.selected_thread_set.get()) {
if (thread == nullptr) {
++sched.idle_selection_count;
}
sched.selected_thread = SharedFrom(thread);
sched.selected_thread_set = SharedFrom(thread);
}
sched.is_context_switch_pending = sched.selected_thread != sched.current_thread;
const bool reschedule_pending =
sched.is_context_switch_pending || (sched.selected_thread_set != sched.current_thread);
sched.is_context_switch_pending = reschedule_pending;
std::atomic_thread_fence(std::memory_order_seq_cst);
sched.guard.unlock();
return reschedule_pending;
};
Scheduler& sched = kernel.Scheduler(core);
Thread* current_thread = nullptr;
if (!is_reselection_pending.load()) {
return 0;
}
std::array<Thread*, Core::Hardware::NUM_CPU_CORES> top_threads{};
u32 idle_cores{};
// Step 1: Get top thread in schedule queue.
current_thread = scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front();
if (current_thread) {
update_thread(current_thread, sched);
return;
}
// Step 2: Try selecting a suggested thread.
Thread* winner = nullptr;
std::set<s32> sug_cores;
for (auto thread : suggested_queue[core]) {
s32 this_core = thread->GetProcessorID();
Thread* thread_on_core = nullptr;
if (this_core >= 0) {
thread_on_core = scheduled_queue[this_core].front();
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
Thread* top_thread =
scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front();
if (top_thread != nullptr) {
// TODO(Blinkhawk): Implement Thread Pinning
} else {
idle_cores |= (1ul << core);
}
if (this_core < 0 || thread != thread_on_core) {
winner = thread;
break;
}
sug_cores.insert(this_core);
top_threads[core] = top_thread;
}
// if we got a suggested thread, select it, else do a second pass.
if (winner && winner->GetPriority() > 2) {
if (winner->IsRunning()) {
UnloadThread(static_cast<u32>(winner->GetProcessorID()));
}
TransferToCore(winner->GetPriority(), static_cast<s32>(core), winner);
update_thread(winner, sched);
return;
}
// Step 3: Select a suggested thread from another core
for (auto& src_core : sug_cores) {
auto it = scheduled_queue[src_core].begin();
it++;
if (it != scheduled_queue[src_core].end()) {
Thread* thread_on_core = scheduled_queue[src_core].front();
Thread* to_change = *it;
if (thread_on_core->IsRunning() || to_change->IsRunning()) {
UnloadThread(static_cast<u32>(src_core));
while (idle_cores != 0) {
u32 core_id = Common::CountTrailingZeroes32(idle_cores);
if (!suggested_queue[core_id].empty()) {
std::array<s32, Core::Hardware::NUM_CPU_CORES> migration_candidates{};
std::size_t num_candidates = 0;
auto iter = suggested_queue[core_id].begin();
Thread* suggested = nullptr;
// Step 2: Try selecting a suggested thread.
while (iter != suggested_queue[core_id].end()) {
suggested = *iter;
iter++;
s32 suggested_core_id = suggested->GetProcessorID();
Thread* top_thread =
suggested_core_id >= 0 ? top_threads[suggested_core_id] : nullptr;
if (top_thread != suggested) {
if (top_thread != nullptr &&
top_thread->GetPriority() < THREADPRIO_MAX_CORE_MIGRATION) {
suggested = nullptr;
break;
// There's a too high thread to do core migration, cancel
}
TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id), suggested);
break;
}
suggested = nullptr;
migration_candidates[num_candidates++] = suggested_core_id;
}
TransferToCore(thread_on_core->GetPriority(), static_cast<s32>(core), thread_on_core);
current_thread = thread_on_core;
break;
// Step 3: Select a suggested thread from another core
if (suggested == nullptr) {
for (std::size_t i = 0; i < num_candidates; i++) {
s32 candidate_core = migration_candidates[i];
suggested = top_threads[candidate_core];
auto it = scheduled_queue[candidate_core].begin();
it++;
Thread* next = it != scheduled_queue[candidate_core].end() ? *it : nullptr;
if (next != nullptr) {
TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id),
suggested);
top_threads[candidate_core] = next;
break;
} else {
suggested = nullptr;
}
}
}
top_threads[core_id] = suggested;
}
idle_cores &= ~(1ul << core_id);
}
u32 cores_needing_context_switch{};
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
Scheduler& sched = kernel.Scheduler(core);
ASSERT(top_threads[core] == nullptr || top_threads[core]->GetProcessorID() == core);
if (update_thread(top_threads[core], sched)) {
cores_needing_context_switch |= (1ul << core);
}
}
update_thread(current_thread, sched);
return cores_needing_context_switch;
}
bool GlobalScheduler::YieldThread(Thread* yielding_thread) {
ASSERT(is_locked);
// Note: caller should use critical section, etc.
if (!yielding_thread->IsRunnable()) {
// Normally this case shouldn't happen except for SetThreadActivity.
is_reselection_pending.store(true, std::memory_order_release);
return false;
}
const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID());
const u32 priority = yielding_thread->GetPriority();
// Yield the thread
const Thread* const winner = scheduled_queue[core_id].front(priority);
ASSERT_MSG(yielding_thread == winner, "Thread yielding without being in front");
scheduled_queue[core_id].yield(priority);
Reschedule(priority, core_id, yielding_thread);
const Thread* const winner = scheduled_queue[core_id].front();
if (kernel.GetCurrentHostThreadID() != core_id) {
is_reselection_pending.store(true, std::memory_order_release);
}
return AskForReselectionOrMarkRedundant(yielding_thread, winner);
}
bool GlobalScheduler::YieldThreadAndBalanceLoad(Thread* yielding_thread) {
ASSERT(is_locked);
// Note: caller should check if !thread.IsSchedulerOperationRedundant and use critical section,
// etc.
if (!yielding_thread->IsRunnable()) {
// Normally this case shouldn't happen except for SetThreadActivity.
is_reselection_pending.store(true, std::memory_order_release);
return false;
}
const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID());
const u32 priority = yielding_thread->GetPriority();
// Yield the thread
ASSERT_MSG(yielding_thread == scheduled_queue[core_id].front(priority),
"Thread yielding without being in front");
scheduled_queue[core_id].yield(priority);
Reschedule(priority, core_id, yielding_thread);
std::array<Thread*, Core::Hardware::NUM_CPU_CORES> current_threads;
for (std::size_t i = 0; i < current_threads.size(); i++) {
@ -153,21 +205,28 @@ bool GlobalScheduler::YieldThreadAndBalanceLoad(Thread* yielding_thread) {
if (winner != nullptr) {
if (winner != yielding_thread) {
if (winner->IsRunning()) {
UnloadThread(static_cast<u32>(winner->GetProcessorID()));
}
TransferToCore(winner->GetPriority(), s32(core_id), winner);
}
} else {
winner = next_thread;
}
if (kernel.GetCurrentHostThreadID() != core_id) {
is_reselection_pending.store(true, std::memory_order_release);
}
return AskForReselectionOrMarkRedundant(yielding_thread, winner);
}
bool GlobalScheduler::YieldThreadAndWaitForLoadBalancing(Thread* yielding_thread) {
ASSERT(is_locked);
// Note: caller should check if !thread.IsSchedulerOperationRedundant and use critical section,
// etc.
if (!yielding_thread->IsRunnable()) {
// Normally this case shouldn't happen except for SetThreadActivity.
is_reselection_pending.store(true, std::memory_order_release);
return false;
}
Thread* winner = nullptr;
const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID());
@ -195,25 +254,31 @@ bool GlobalScheduler::YieldThreadAndWaitForLoadBalancing(Thread* yielding_thread
}
if (winner != nullptr) {
if (winner != yielding_thread) {
if (winner->IsRunning()) {
UnloadThread(static_cast<u32>(winner->GetProcessorID()));
}
TransferToCore(winner->GetPriority(), static_cast<s32>(core_id), winner);
}
} else {
winner = yielding_thread;
}
} else {
winner = scheduled_queue[core_id].front();
}
if (kernel.GetCurrentHostThreadID() != core_id) {
is_reselection_pending.store(true, std::memory_order_release);
}
return AskForReselectionOrMarkRedundant(yielding_thread, winner);
}
void GlobalScheduler::PreemptThreads() {
ASSERT(is_locked);
for (std::size_t core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) {
const u32 priority = preemption_priorities[core_id];
if (scheduled_queue[core_id].size(priority) > 0) {
scheduled_queue[core_id].front(priority)->IncrementYieldCount();
if (scheduled_queue[core_id].size(priority) > 1) {
scheduled_queue[core_id].front(priority)->IncrementYieldCount();
}
scheduled_queue[core_id].yield(priority);
if (scheduled_queue[core_id].size(priority) > 1) {
scheduled_queue[core_id].front(priority)->IncrementYieldCount();
@ -247,9 +312,6 @@ void GlobalScheduler::PreemptThreads() {
}
if (winner != nullptr) {
if (winner->IsRunning()) {
UnloadThread(static_cast<u32>(winner->GetProcessorID()));
}
TransferToCore(winner->GetPriority(), s32(core_id), winner);
current_thread =
winner->GetPriority() <= current_thread->GetPriority() ? winner : current_thread;
@ -280,9 +342,6 @@ void GlobalScheduler::PreemptThreads() {
}
if (winner != nullptr) {
if (winner->IsRunning()) {
UnloadThread(static_cast<u32>(winner->GetProcessorID()));
}
TransferToCore(winner->GetPriority(), s32(core_id), winner);
current_thread = winner;
}
@ -292,34 +351,65 @@ void GlobalScheduler::PreemptThreads() {
}
}
void GlobalScheduler::EnableInterruptAndSchedule(u32 cores_pending_reschedule,
Core::EmuThreadHandle global_thread) {
u32 current_core = global_thread.host_handle;
bool must_context_switch = global_thread.guest_handle != InvalidHandle &&
(current_core < Core::Hardware::NUM_CPU_CORES);
while (cores_pending_reschedule != 0) {
u32 core = Common::CountTrailingZeroes32(cores_pending_reschedule);
ASSERT(core < Core::Hardware::NUM_CPU_CORES);
if (!must_context_switch || core != current_core) {
auto& phys_core = kernel.PhysicalCore(core);
phys_core.Interrupt();
} else {
must_context_switch = true;
}
cores_pending_reschedule &= ~(1ul << core);
}
if (must_context_switch) {
auto& core_scheduler = kernel.CurrentScheduler();
kernel.ExitSVCProfile();
core_scheduler.TryDoContextSwitch();
kernel.EnterSVCProfile();
}
}
void GlobalScheduler::Suggest(u32 priority, std::size_t core, Thread* thread) {
ASSERT(is_locked);
suggested_queue[core].add(thread, priority);
}
void GlobalScheduler::Unsuggest(u32 priority, std::size_t core, Thread* thread) {
ASSERT(is_locked);
suggested_queue[core].remove(thread, priority);
}
void GlobalScheduler::Schedule(u32 priority, std::size_t core, Thread* thread) {
ASSERT(is_locked);
ASSERT_MSG(thread->GetProcessorID() == s32(core), "Thread must be assigned to this core.");
scheduled_queue[core].add(thread, priority);
}
void GlobalScheduler::SchedulePrepend(u32 priority, std::size_t core, Thread* thread) {
ASSERT(is_locked);
ASSERT_MSG(thread->GetProcessorID() == s32(core), "Thread must be assigned to this core.");
scheduled_queue[core].add(thread, priority, false);
}
void GlobalScheduler::Reschedule(u32 priority, std::size_t core, Thread* thread) {
ASSERT(is_locked);
scheduled_queue[core].remove(thread, priority);
scheduled_queue[core].add(thread, priority);
}
void GlobalScheduler::Unschedule(u32 priority, std::size_t core, Thread* thread) {
ASSERT(is_locked);
scheduled_queue[core].remove(thread, priority);
}
void GlobalScheduler::TransferToCore(u32 priority, s32 destination_core, Thread* thread) {
ASSERT(is_locked);
const bool schedulable = thread->GetPriority() < THREADPRIO_COUNT;
const s32 source_core = thread->GetProcessorID();
if (source_core == destination_core || !schedulable) {
@ -349,6 +439,108 @@ bool GlobalScheduler::AskForReselectionOrMarkRedundant(Thread* current_thread,
}
}
void GlobalScheduler::AdjustSchedulingOnStatus(Thread* thread, u32 old_flags) {
if (old_flags == thread->scheduling_state) {
return;
}
ASSERT(is_locked);
if (old_flags == static_cast<u32>(ThreadSchedStatus::Runnable)) {
// In this case the thread was running, now it's pausing/exitting
if (thread->processor_id >= 0) {
Unschedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (core != static_cast<u32>(thread->processor_id) &&
((thread->affinity_mask >> core) & 1) != 0) {
Unsuggest(thread->current_priority, core, thread);
}
}
} else if (thread->scheduling_state == static_cast<u32>(ThreadSchedStatus::Runnable)) {
// The thread is now set to running from being stopped
if (thread->processor_id >= 0) {
Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (core != static_cast<u32>(thread->processor_id) &&
((thread->affinity_mask >> core) & 1) != 0) {
Suggest(thread->current_priority, core, thread);
}
}
}
SetReselectionPending();
}
void GlobalScheduler::AdjustSchedulingOnPriority(Thread* thread, u32 old_priority) {
if (thread->scheduling_state != static_cast<u32>(ThreadSchedStatus::Runnable)) {
return;
}
ASSERT(is_locked);
if (thread->processor_id >= 0) {
Unschedule(old_priority, static_cast<u32>(thread->processor_id), thread);
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (core != static_cast<u32>(thread->processor_id) &&
((thread->affinity_mask >> core) & 1) != 0) {
Unsuggest(old_priority, core, thread);
}
}
if (thread->processor_id >= 0) {
if (thread == kernel.CurrentScheduler().GetCurrentThread()) {
SchedulePrepend(thread->current_priority, static_cast<u32>(thread->processor_id),
thread);
} else {
Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread);
}
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (core != static_cast<u32>(thread->processor_id) &&
((thread->affinity_mask >> core) & 1) != 0) {
Suggest(thread->current_priority, core, thread);
}
}
thread->IncrementYieldCount();
SetReselectionPending();
}
void GlobalScheduler::AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask,
s32 old_core) {
if (thread->scheduling_state != static_cast<u32>(ThreadSchedStatus::Runnable) ||
thread->current_priority >= THREADPRIO_COUNT) {
return;
}
ASSERT(is_locked);
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (((old_affinity_mask >> core) & 1) != 0) {
if (core == static_cast<u32>(old_core)) {
Unschedule(thread->current_priority, core, thread);
} else {
Unsuggest(thread->current_priority, core, thread);
}
}
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (((thread->affinity_mask >> core) & 1) != 0) {
if (core == static_cast<u32>(thread->processor_id)) {
Schedule(thread->current_priority, core, thread);
} else {
Suggest(thread->current_priority, core, thread);
}
}
}
thread->IncrementYieldCount();
SetReselectionPending();
}
void GlobalScheduler::Shutdown() {
for (std::size_t core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
scheduled_queue[core].clear();
@ -359,10 +551,12 @@ void GlobalScheduler::Shutdown() {
void GlobalScheduler::Lock() {
Core::EmuThreadHandle current_thread = kernel.GetCurrentEmuThreadID();
ASSERT(!current_thread.IsInvalid());
if (current_thread == current_owner) {
++scope_lock;
} else {
inner_lock.lock();
is_locked = true;
current_owner = current_thread;
ASSERT(current_owner != Core::EmuThreadHandle::InvalidHandle());
scope_lock = 1;
@ -374,17 +568,18 @@ void GlobalScheduler::Unlock() {
ASSERT(scope_lock > 0);
return;
}
for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
SelectThread(i);
}
u32 cores_pending_reschedule = SelectThreads();
Core::EmuThreadHandle leaving_thread = current_owner;
current_owner = Core::EmuThreadHandle::InvalidHandle();
scope_lock = 1;
is_locked = false;
inner_lock.unlock();
// TODO(Blinkhawk): Setup the interrupts and change context on current core.
EnableInterruptAndSchedule(cores_pending_reschedule, leaving_thread);
}
Scheduler::Scheduler(Core::System& system, std::size_t core_id)
: system{system}, core_id{core_id} {}
Scheduler::Scheduler(Core::System& system, std::size_t core_id) : system(system), core_id(core_id) {
switch_fiber = std::make_shared<Common::Fiber>(std::function<void(void*)>(OnSwitch), this);
}
Scheduler::~Scheduler() = default;
@ -393,56 +588,128 @@ bool Scheduler::HaveReadyThreads() const {
}
Thread* Scheduler::GetCurrentThread() const {
return current_thread.get();
if (current_thread) {
return current_thread.get();
}
return idle_thread.get();
}
Thread* Scheduler::GetSelectedThread() const {
return selected_thread.get();
}
void Scheduler::SelectThreads() {
system.GlobalScheduler().SelectThread(core_id);
}
u64 Scheduler::GetLastContextSwitchTicks() const {
return last_context_switch_time;
}
void Scheduler::TryDoContextSwitch() {
auto& phys_core = system.Kernel().CurrentPhysicalCore();
if (phys_core.IsInterrupted()) {
phys_core.ClearInterrupt();
}
guard.lock();
if (is_context_switch_pending) {
SwitchContext();
} else {
guard.unlock();
}
}
void Scheduler::UnloadThread() {
Thread* const previous_thread = GetCurrentThread();
Process* const previous_process = system.Kernel().CurrentProcess();
void Scheduler::OnThreadStart() {
SwitchContextStep2();
}
UpdateLastContextSwitchTime(previous_thread, previous_process);
// Save context for previous thread
if (previous_thread) {
system.ArmInterface(core_id).SaveContext(previous_thread->GetContext32());
system.ArmInterface(core_id).SaveContext(previous_thread->GetContext64());
// Save the TPIDR_EL0 system register in case it was modified.
previous_thread->SetTPIDR_EL0(system.ArmInterface(core_id).GetTPIDR_EL0());
if (previous_thread->GetStatus() == ThreadStatus::Running) {
// This is only the case when a reschedule is triggered without the current thread
// yielding execution (i.e. an event triggered, system core time-sliced, etc)
previous_thread->SetStatus(ThreadStatus::Ready);
void Scheduler::Unload() {
Thread* thread = current_thread.get();
if (thread) {
thread->SetContinuousOnSVC(false);
thread->last_running_ticks = system.CoreTiming().GetCPUTicks();
thread->SetIsRunning(false);
if (!thread->IsHLEThread() && !thread->HasExited()) {
Core::ARM_Interface& cpu_core = thread->ArmInterface();
cpu_core.SaveContext(thread->GetContext32());
cpu_core.SaveContext(thread->GetContext64());
// Save the TPIDR_EL0 system register in case it was modified.
thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0());
cpu_core.ClearExclusiveState();
}
previous_thread->SetIsRunning(false);
thread->context_guard.unlock();
}
current_thread = nullptr;
}
void Scheduler::Reload() {
Thread* thread = current_thread.get();
if (thread) {
ASSERT_MSG(thread->GetSchedulingStatus() == ThreadSchedStatus::Runnable,
"Thread must be runnable.");
// Cancel any outstanding wakeup events for this thread
thread->SetIsRunning(true);
thread->SetWasRunning(false);
thread->last_running_ticks = system.CoreTiming().GetCPUTicks();
auto* const thread_owner_process = thread->GetOwnerProcess();
if (thread_owner_process != nullptr) {
system.Kernel().MakeCurrentProcess(thread_owner_process);
}
if (!thread->IsHLEThread()) {
Core::ARM_Interface& cpu_core = thread->ArmInterface();
cpu_core.LoadContext(thread->GetContext32());
cpu_core.LoadContext(thread->GetContext64());
cpu_core.SetTlsAddress(thread->GetTLSAddress());
cpu_core.SetTPIDR_EL0(thread->GetTPIDR_EL0());
cpu_core.ChangeProcessorID(this->core_id);
cpu_core.ClearExclusiveState();
}
}
}
void Scheduler::SwitchContextStep2() {
Thread* previous_thread = current_thread_prev.get();
Thread* new_thread = selected_thread.get();
// Load context of new thread
Process* const previous_process =
previous_thread != nullptr ? previous_thread->GetOwnerProcess() : nullptr;
if (new_thread) {
ASSERT_MSG(new_thread->GetSchedulingStatus() == ThreadSchedStatus::Runnable,
"Thread must be runnable.");
// Cancel any outstanding wakeup events for this thread
new_thread->SetIsRunning(true);
new_thread->last_running_ticks = system.CoreTiming().GetCPUTicks();
new_thread->SetWasRunning(false);
auto* const thread_owner_process = current_thread->GetOwnerProcess();
if (thread_owner_process != nullptr) {
system.Kernel().MakeCurrentProcess(thread_owner_process);
}
if (!new_thread->IsHLEThread()) {
Core::ARM_Interface& cpu_core = new_thread->ArmInterface();
cpu_core.LoadContext(new_thread->GetContext32());
cpu_core.LoadContext(new_thread->GetContext64());
cpu_core.SetTlsAddress(new_thread->GetTLSAddress());
cpu_core.SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
cpu_core.ChangeProcessorID(this->core_id);
cpu_core.ClearExclusiveState();
}
}
TryDoContextSwitch();
}
void Scheduler::SwitchContext() {
Thread* const previous_thread = GetCurrentThread();
Thread* const new_thread = GetSelectedThread();
current_thread_prev = current_thread;
selected_thread = selected_thread_set;
Thread* previous_thread = current_thread_prev.get();
Thread* new_thread = selected_thread.get();
current_thread = selected_thread;
is_context_switch_pending = false;
if (new_thread == previous_thread) {
guard.unlock();
return;
}
@ -452,51 +719,75 @@ void Scheduler::SwitchContext() {
// Save context for previous thread
if (previous_thread) {
system.ArmInterface(core_id).SaveContext(previous_thread->GetContext32());
system.ArmInterface(core_id).SaveContext(previous_thread->GetContext64());
// Save the TPIDR_EL0 system register in case it was modified.
previous_thread->SetTPIDR_EL0(system.ArmInterface(core_id).GetTPIDR_EL0());
if (previous_thread->GetStatus() == ThreadStatus::Running) {
// This is only the case when a reschedule is triggered without the current thread
// yielding execution (i.e. an event triggered, system core time-sliced, etc)
previous_thread->SetStatus(ThreadStatus::Ready);
if (new_thread != nullptr && new_thread->IsSuspendThread()) {
previous_thread->SetWasRunning(true);
}
previous_thread->SetContinuousOnSVC(false);
previous_thread->last_running_ticks = system.CoreTiming().GetCPUTicks();
previous_thread->SetIsRunning(false);
if (!previous_thread->IsHLEThread() && !previous_thread->HasExited()) {
Core::ARM_Interface& cpu_core = previous_thread->ArmInterface();
cpu_core.SaveContext(previous_thread->GetContext32());
cpu_core.SaveContext(previous_thread->GetContext64());
// Save the TPIDR_EL0 system register in case it was modified.
previous_thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0());
cpu_core.ClearExclusiveState();
}
previous_thread->context_guard.unlock();
}
// Load context of new thread
if (new_thread) {
ASSERT_MSG(new_thread->GetProcessorID() == s32(this->core_id),
"Thread must be assigned to this core.");
ASSERT_MSG(new_thread->GetStatus() == ThreadStatus::Ready,
"Thread must be ready to become running.");
// Cancel any outstanding wakeup events for this thread
new_thread->CancelWakeupTimer();
current_thread = SharedFrom(new_thread);
new_thread->SetStatus(ThreadStatus::Running);
new_thread->SetIsRunning(true);
auto* const thread_owner_process = current_thread->GetOwnerProcess();
if (previous_process != thread_owner_process) {
system.Kernel().MakeCurrentProcess(thread_owner_process);
}
system.ArmInterface(core_id).LoadContext(new_thread->GetContext32());
system.ArmInterface(core_id).LoadContext(new_thread->GetContext64());
system.ArmInterface(core_id).SetTlsAddress(new_thread->GetTLSAddress());
system.ArmInterface(core_id).SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
std::shared_ptr<Common::Fiber>* old_context;
if (previous_thread != nullptr) {
old_context = &previous_thread->GetHostContext();
} else {
current_thread = nullptr;
// Note: We do not reset the current process and current page table when idling because
// technically we haven't changed processes, our threads are just paused.
old_context = &idle_thread->GetHostContext();
}
guard.unlock();
Common::Fiber::YieldTo(*old_context, switch_fiber);
/// When a thread wakes up, the scheduler may have changed to other in another core.
auto& next_scheduler = system.Kernel().CurrentScheduler();
next_scheduler.SwitchContextStep2();
}
void Scheduler::OnSwitch(void* this_scheduler) {
Scheduler* sched = static_cast<Scheduler*>(this_scheduler);
sched->SwitchToCurrent();
}
void Scheduler::SwitchToCurrent() {
while (true) {
guard.lock();
selected_thread = selected_thread_set;
current_thread = selected_thread;
is_context_switch_pending = false;
guard.unlock();
while (!is_context_switch_pending) {
if (current_thread != nullptr && !current_thread->IsHLEThread()) {
current_thread->context_guard.lock();
if (!current_thread->IsRunnable()) {
current_thread->context_guard.unlock();
break;
}
if (current_thread->GetProcessorID() != core_id) {
current_thread->context_guard.unlock();
break;
}
}
std::shared_ptr<Common::Fiber>* next_context;
if (current_thread != nullptr) {
next_context = &current_thread->GetHostContext();
} else {
next_context = &idle_thread->GetHostContext();
}
Common::Fiber::YieldTo(switch_fiber, *next_context);
}
}
}
void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) {
const u64 prev_switch_ticks = last_context_switch_time;
const u64 most_recent_switch_ticks = system.CoreTiming().GetTicks();
const u64 most_recent_switch_ticks = system.CoreTiming().GetCPUTicks();
const u64 update_ticks = most_recent_switch_ticks - prev_switch_ticks;
if (thread != nullptr) {
@ -510,6 +801,16 @@ void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) {
last_context_switch_time = most_recent_switch_ticks;
}
void Scheduler::Initialize() {
std::string name = "Idle Thread Id:" + std::to_string(core_id);
std::function<void(void*)> init_func = system.GetCpuManager().GetIdleThreadStartFunc();
void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
ThreadType type = static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_IDLE);
auto thread_res = Thread::Create(system, type, name, 0, 64, 0, static_cast<u32>(core_id), 0,
nullptr, std::move(init_func), init_func_parameter);
idle_thread = std::move(thread_res).Unwrap();
}
void Scheduler::Shutdown() {
current_thread = nullptr;
selected_thread = nullptr;
@ -538,4 +839,13 @@ SchedulerLockAndSleep::~SchedulerLockAndSleep() {
time_manager.ScheduleTimeEvent(event_handle, time_task, nanoseconds);
}
void SchedulerLockAndSleep::Release() {
if (sleep_cancelled) {
return;
}
auto& time_manager = kernel.TimeManager();
time_manager.ScheduleTimeEvent(event_handle, time_task, nanoseconds);
sleep_cancelled = true;
}
} // namespace Kernel

127
src/core/hle/kernel/scheduler.h
View file

@ -11,9 +11,14 @@
#include "common/common_types.h"
#include "common/multi_level_queue.h"
#include "common/spin_lock.h"
#include "core/hardware_properties.h"
#include "core/hle/kernel/thread.h"
namespace Common {
class Fiber;
}
namespace Core {
class ARM_Interface;
class System;
@ -41,41 +46,17 @@ public:
return thread_list;
}
/**
* Add a thread to the suggested queue of a cpu core. Suggested threads may be
* picked if no thread is scheduled to run on the core.
*/
void Suggest(u32 priority, std::size_t core, Thread* thread);
/// Notify the scheduler a thread's status has changed.
void AdjustSchedulingOnStatus(Thread* thread, u32 old_flags);
/// Notify the scheduler a thread's priority has changed.
void AdjustSchedulingOnPriority(Thread* thread, u32 old_priority);
/// Notify the scheduler a thread's core and/or affinity mask has changed.
void AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask, s32 old_core);
/**
* Remove a thread to the suggested queue of a cpu core. Suggested threads may be
* picked if no thread is scheduled to run on the core.
*/
void Unsuggest(u32 priority, std::size_t core, Thread* thread);
/**
* Add a thread to the scheduling queue of a cpu core. The thread is added at the
* back the queue in its priority level.
*/
void Schedule(u32 priority, std::size_t core, Thread* thread);
/**
* Add a thread to the scheduling queue of a cpu core. The thread is added at the
* front the queue in its priority level.
*/
void SchedulePrepend(u32 priority, std::size_t core, Thread* thread);
/// Reschedule an already scheduled thread based on a new priority
void Reschedule(u32 priority, std::size_t core, Thread* thread);
/// Unschedules a thread.
void Unschedule(u32 priority, std::size_t core, Thread* thread);
/// Selects a core and forces it to unload its current thread's context
void UnloadThread(std::size_t core);
/**
* Takes care of selecting the new scheduled thread in three steps:
* Takes care of selecting the new scheduled threads in three steps:
*
* 1. First a thread is selected from the top of the priority queue. If no thread
* is obtained then we move to step two, else we are done.
@ -85,8 +66,10 @@ public:
*
* 3. Third is no suggested thread is found, we do a second pass and pick a running
* thread in another core and swap it with its current thread.
*
* returns the cores needing scheduling.
*/
void SelectThread(std::size_t core);
u32 SelectThreads();
bool HaveReadyThreads(std::size_t core_id) const {
return !scheduled_queue[core_id].empty();
@ -149,6 +132,40 @@ private:
/// Unlocks the scheduler, reselects threads, interrupts cores for rescheduling
/// and reschedules current core if needed.
void Unlock();
void EnableInterruptAndSchedule(u32 cores_pending_reschedule,
Core::EmuThreadHandle global_thread);
/**
* Add a thread to the suggested queue of a cpu core. Suggested threads may be
* picked if no thread is scheduled to run on the core.
*/
void Suggest(u32 priority, std::size_t core, Thread* thread);
/**
* Remove a thread to the suggested queue of a cpu core. Suggested threads may be
* picked if no thread is scheduled to run on the core.
*/
void Unsuggest(u32 priority, std::size_t core, Thread* thread);
/**
* Add a thread to the scheduling queue of a cpu core. The thread is added at the
* back the queue in its priority level.
*/
void Schedule(u32 priority, std::size_t core, Thread* thread);
/**
* Add a thread to the scheduling queue of a cpu core. The thread is added at the
* front the queue in its priority level.
*/
void SchedulePrepend(u32 priority, std::size_t core, Thread* thread);
/// Reschedule an already scheduled thread based on a new priority
void Reschedule(u32 priority, std::size_t core, Thread* thread);
/// Unschedules a thread.
void Unschedule(u32 priority, std::size_t core, Thread* thread);
/**
* Transfers a thread into an specific core. If the destination_core is -1
* it will be unscheduled from its source code and added into its suggested
@ -170,10 +187,13 @@ private:
std::array<u32, Core::Hardware::NUM_CPU_CORES> preemption_priorities = {59, 59, 59, 62};
/// Scheduler lock mechanisms.
std::mutex inner_lock{}; // TODO(Blinkhawk): Replace for a SpinLock
bool is_locked{};
Common::SpinLock inner_lock{};
std::atomic<s64> scope_lock{};
Core::EmuThreadHandle current_owner{Core::EmuThreadHandle::InvalidHandle()};
Common::SpinLock global_list_guard{};
/// Lists all thread ids that aren't deleted/etc.
std::vector<std::shared_ptr<Thread>> thread_list;
KernelCore& kernel;
@ -190,11 +210,11 @@ public:
/// Reschedules to the next available thread (call after current thread is suspended)
void TryDoContextSwitch();
/// Unloads currently running thread
void UnloadThread();
/// Select the threads in top of the scheduling multilist.
void SelectThreads();
/// The next two are for SingleCore Only.
/// Unload current thread before preempting core.
void Unload();
/// Reload current thread after core preemption.
void Reload();
/// Gets the current running thread
Thread* GetCurrentThread() const;
@ -209,15 +229,30 @@ public:
return is_context_switch_pending;
}
void Initialize();
/// Shutdowns the scheduler.
void Shutdown();
void OnThreadStart();
std::shared_ptr<Common::Fiber>& ControlContext() {
return switch_fiber;
}
const std::shared_ptr<Common::Fiber>& ControlContext() const {
return switch_fiber;
}
private:
friend class GlobalScheduler;
/// Switches the CPU's active thread context to that of the specified thread
void SwitchContext();
/// When a thread wakes up, it must run this through it's new scheduler
void SwitchContextStep2();
/**
* Called on every context switch to update the internal timestamp
* This also updates the running time ticks for the given thread and
@ -231,14 +266,24 @@ private:
*/
void UpdateLastContextSwitchTime(Thread* thread, Process* process);
static void OnSwitch(void* this_scheduler);
void SwitchToCurrent();
std::shared_ptr<Thread> current_thread = nullptr;
std::shared_ptr<Thread> selected_thread = nullptr;
std::shared_ptr<Thread> current_thread_prev = nullptr;
std::shared_ptr<Thread> selected_thread_set = nullptr;
std::shared_ptr<Thread> idle_thread = nullptr;
std::shared_ptr<Common::Fiber> switch_fiber = nullptr;
Core::System& system;
u64 last_context_switch_time = 0;
u64 idle_selection_count = 0;
const std::size_t core_id;
Common::SpinLock guard{};
bool is_context_switch_pending = false;
};
@ -261,6 +306,8 @@ public:
sleep_cancelled = true;
}
void Release();
private:
Handle& event_handle;
Thread* time_task;

16
src/core/hle/kernel/server_session.cpp
View file

@ -17,6 +17,7 @@
#include "core/hle/kernel/hle_ipc.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/server_session.h"
#include "core/hle/kernel/session.h"
#include "core/hle/kernel/thread.h"
@ -168,9 +169,12 @@ ResultCode ServerSession::CompleteSyncRequest() {
}
// Some service requests require the thread to block
if (!context.IsThreadWaiting()) {
context.GetThread().ResumeFromWait();
context.GetThread().SetWaitSynchronizationResult(result);
{
SchedulerLock lock(kernel);
if (!context.IsThreadWaiting()) {
context.GetThread().ResumeFromWait();
context.GetThread().SetSynchronizationResults(nullptr, result);
}
}
request_queue.Pop();
@ -180,8 +184,10 @@ ResultCode ServerSession::CompleteSyncRequest() {
ResultCode ServerSession::HandleSyncRequest(std::shared_ptr<Thread> thread,
Core::Memory::Memory& memory) {
Core::System::GetInstance().CoreTiming().ScheduleEvent(20000, request_event, {});
return QueueSyncRequest(std::move(thread), memory);
ResultCode result = QueueSyncRequest(std::move(thread), memory);
const u64 delay = kernel.IsMulticore() ? 0U : 20000U;
Core::System::GetInstance().CoreTiming().ScheduleEvent(delay, request_event, {});
return result;
}
} // namespace Kernel

464
src/core/hle/kernel/svc.cpp
View file

@ -10,14 +10,15 @@
#include "common/alignment.h"
#include "common/assert.h"
#include "common/fiber.h"
#include "common/logging/log.h"
#include "common/microprofile.h"
#include "common/string_util.h"
#include "core/arm/exclusive_monitor.h"
#include "core/core.h"
#include "core/core_manager.h"
#include "core/core_timing.h"
#include "core/core_timing_util.h"
#include "core/cpu_manager.h"
#include "core/hle/kernel/address_arbiter.h"
#include "core/hle/kernel/client_port.h"
#include "core/hle/kernel/client_session.h"
@ -27,6 +28,7 @@
#include "core/hle/kernel/memory/memory_block.h"
#include "core/hle/kernel/memory/page_table.h"
#include "core/hle/kernel/mutex.h"
#include "core/hle/kernel/physical_core.h"
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/readable_event.h"
#include "core/hle/kernel/resource_limit.h"
@ -37,6 +39,7 @@
#include "core/hle/kernel/svc_wrap.h"
#include "core/hle/kernel/synchronization.h"
#include "core/hle/kernel/thread.h"
#include "core/hle/kernel/time_manager.h"
#include "core/hle/kernel/transfer_memory.h"
#include "core/hle/kernel/writable_event.h"
#include "core/hle/lock.h"
@ -133,6 +136,7 @@ enum class ResourceLimitValueType {
ResultVal<s64> RetrieveResourceLimitValue(Core::System& system, Handle resource_limit,
u32 resource_type, ResourceLimitValueType value_type) {
std::lock_guard lock{HLE::g_hle_lock};
const auto type = static_cast<ResourceType>(resource_type);
if (!IsValidResourceType(type)) {
LOG_ERROR(Kernel_SVC, "Invalid resource limit type: '{}'", resource_type);
@ -160,6 +164,7 @@ ResultVal<s64> RetrieveResourceLimitValue(Core::System& system, Handle resource_
/// Set the process heap to a given Size. It can both extend and shrink the heap.
static ResultCode SetHeapSize(Core::System& system, VAddr* heap_addr, u64 heap_size) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_TRACE(Kernel_SVC, "called, heap_size=0x{:X}", heap_size);
// Size must be a multiple of 0x200000 (2MB) and be equal to or less than 8GB.
@ -190,6 +195,7 @@ static ResultCode SetHeapSize32(Core::System& system, u32* heap_addr, u32 heap_s
static ResultCode SetMemoryAttribute(Core::System& system, VAddr address, u64 size, u32 mask,
u32 attribute) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_DEBUG(Kernel_SVC,
"called, address=0x{:016X}, size=0x{:X}, mask=0x{:08X}, attribute=0x{:08X}", address,
size, mask, attribute);
@ -226,8 +232,15 @@ static ResultCode SetMemoryAttribute(Core::System& system, VAddr address, u64 si
static_cast<Memory::MemoryAttribute>(attribute));
}
static ResultCode SetMemoryAttribute32(Core::System& system, u32 address, u32 size, u32 mask,
u32 attribute) {
return SetMemoryAttribute(system, static_cast<VAddr>(address), static_cast<std::size_t>(size),
mask, attribute);
}
/// Maps a memory range into a different range.
static ResultCode MapMemory(Core::System& system, VAddr dst_addr, VAddr src_addr, u64 size) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_TRACE(Kernel_SVC, "called, dst_addr=0x{:X}, src_addr=0x{:X}, size=0x{:X}", dst_addr,
src_addr, size);
@ -241,8 +254,14 @@ static ResultCode MapMemory(Core::System& system, VAddr dst_addr, VAddr src_addr
return page_table.Map(dst_addr, src_addr, size);
}
static ResultCode MapMemory32(Core::System& system, u32 dst_addr, u32 src_addr, u32 size) {
return MapMemory(system, static_cast<VAddr>(dst_addr), static_cast<VAddr>(src_addr),
static_cast<std::size_t>(size));
}
/// Unmaps a region that was previously mapped with svcMapMemory
static ResultCode UnmapMemory(Core::System& system, VAddr dst_addr, VAddr src_addr, u64 size) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_TRACE(Kernel_SVC, "called, dst_addr=0x{:X}, src_addr=0x{:X}, size=0x{:X}", dst_addr,
src_addr, size);
@ -256,9 +275,15 @@ static ResultCode UnmapMemory(Core::System& system, VAddr dst_addr, VAddr src_ad
return page_table.Unmap(dst_addr, src_addr, size);
}
static ResultCode UnmapMemory32(Core::System& system, u32 dst_addr, u32 src_addr, u32 size) {
return UnmapMemory(system, static_cast<VAddr>(dst_addr), static_cast<VAddr>(src_addr),
static_cast<std::size_t>(size));
}
/// Connect to an OS service given the port name, returns the handle to the port to out
static ResultCode ConnectToNamedPort(Core::System& system, Handle* out_handle,
VAddr port_name_address) {
std::lock_guard lock{HLE::g_hle_lock};
auto& memory = system.Memory();
if (!memory.IsValidVirtualAddress(port_name_address)) {
@ -317,11 +342,30 @@ static ResultCode SendSyncRequest(Core::System& system, Handle handle) {
LOG_TRACE(Kernel_SVC, "called handle=0x{:08X}({})", handle, session->GetName());
auto thread = system.CurrentScheduler().GetCurrentThread();
thread->InvalidateWakeupCallback();
thread->SetStatus(ThreadStatus::WaitIPC);
system.PrepareReschedule(thread->GetProcessorID());
{
SchedulerLock lock(system.Kernel());
thread->InvalidateHLECallback();
thread->SetStatus(ThreadStatus::WaitIPC);
session->SendSyncRequest(SharedFrom(thread), system.Memory());
}
return session->SendSyncRequest(SharedFrom(thread), system.Memory());
if (thread->HasHLECallback()) {
Handle event_handle = thread->GetHLETimeEvent();
if (event_handle != InvalidHandle) {
auto& time_manager = system.Kernel().TimeManager();
time_manager.UnscheduleTimeEvent(event_handle);
}
{
SchedulerLock lock(system.Kernel());
auto* sync_object = thread->GetHLESyncObject();
sync_object->RemoveWaitingThread(SharedFrom(thread));
}
thread->InvokeHLECallback(SharedFrom(thread));
}
return thread->GetSignalingResult();
}
static ResultCode SendSyncRequest32(Core::System& system, Handle handle) {
@ -383,6 +427,15 @@ static ResultCode GetProcessId(Core::System& system, u64* process_id, Handle han
return ERR_INVALID_HANDLE;
}
static ResultCode GetProcessId32(Core::System& system, u32* process_id_low, u32* process_id_high,
Handle handle) {
u64 process_id{};
const auto result = GetProcessId(system, &process_id, handle);
*process_id_low = static_cast<u32>(process_id);
*process_id_high = static_cast<u32>(process_id >> 32);
return result;
}
/// Wait for the given handles to synchronize, timeout after the specified nanoseconds
static ResultCode WaitSynchronization(Core::System& system, Handle* index, VAddr handles_address,
u64 handle_count, s64 nano_seconds) {
@ -447,10 +500,13 @@ static ResultCode CancelSynchronization(Core::System& system, Handle thread_hand
}
thread->CancelWait();
system.PrepareReschedule(thread->GetProcessorID());
return RESULT_SUCCESS;
}
static ResultCode CancelSynchronization32(Core::System& system, Handle thread_handle) {
return CancelSynchronization(system, thread_handle);
}
/// Attempts to locks a mutex, creating it if it does not already exist
static ResultCode ArbitrateLock(Core::System& system, Handle holding_thread_handle,
VAddr mutex_addr, Handle requesting_thread_handle) {
@ -475,6 +531,12 @@ static ResultCode ArbitrateLock(Core::System& system, Handle holding_thread_hand
requesting_thread_handle);
}
static ResultCode ArbitrateLock32(Core::System& system, Handle holding_thread_handle,
u32 mutex_addr, Handle requesting_thread_handle) {
return ArbitrateLock(system, holding_thread_handle, static_cast<VAddr>(mutex_addr),
requesting_thread_handle);
}
/// Unlock a mutex
static ResultCode ArbitrateUnlock(Core::System& system, VAddr mutex_addr) {
LOG_TRACE(Kernel_SVC, "called mutex_addr=0x{:X}", mutex_addr);
@ -494,6 +556,10 @@ static ResultCode ArbitrateUnlock(Core::System& system, VAddr mutex_addr) {
return current_process->GetMutex().Release(mutex_addr);
}
static ResultCode ArbitrateUnlock32(Core::System& system, u32 mutex_addr) {
return ArbitrateUnlock(system, static_cast<VAddr>(mutex_addr));
}
enum class BreakType : u32 {
Panic = 0,
AssertionFailed = 1,
@ -594,6 +660,7 @@ static void Break(Core::System& system, u32 reason, u64 info1, u64 info2) {
info2, has_dumped_buffer ? std::make_optional(debug_buffer) : std::nullopt);
if (!break_reason.signal_debugger) {
SchedulerLock lock(system.Kernel());
LOG_CRITICAL(
Debug_Emulated,
"Emulated program broke execution! reason=0x{:016X}, info1=0x{:016X}, info2=0x{:016X}",
@ -605,14 +672,16 @@ static void Break(Core::System& system, u32 reason, u64 info1, u64 info2) {
const auto thread_processor_id = current_thread->GetProcessorID();
system.ArmInterface(static_cast<std::size_t>(thread_processor_id)).LogBacktrace();
system.Kernel().CurrentProcess()->PrepareForTermination();
// Kill the current thread
system.Kernel().ExceptionalExit();
current_thread->Stop();
system.PrepareReschedule();
}
}
static void Break32(Core::System& system, u32 reason, u32 info1, u32 info2) {
Break(system, reason, static_cast<u64>(info1), static_cast<u64>(info2));
}
/// Used to output a message on a debug hardware unit - does nothing on a retail unit
static void OutputDebugString([[maybe_unused]] Core::System& system, VAddr address, u64 len) {
if (len == 0) {
@ -627,6 +696,7 @@ static void OutputDebugString([[maybe_unused]] Core::System& system, VAddr addre
/// Gets system/memory information for the current process
static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 handle,
u64 info_sub_id) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_TRACE(Kernel_SVC, "called info_id=0x{:X}, info_sub_id=0x{:X}, handle=0x{:08X}", info_id,
info_sub_id, handle);
@ -863,9 +933,9 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha
if (same_thread && info_sub_id == 0xFFFFFFFFFFFFFFFF) {
const u64 thread_ticks = current_thread->GetTotalCPUTimeTicks();
out_ticks = thread_ticks + (core_timing.GetTicks() - prev_ctx_ticks);
out_ticks = thread_ticks + (core_timing.GetCPUTicks() - prev_ctx_ticks);
} else if (same_thread && info_sub_id == system.CurrentCoreIndex()) {
out_ticks = core_timing.GetTicks() - prev_ctx_ticks;
out_ticks = core_timing.GetCPUTicks() - prev_ctx_ticks;
}
*result = out_ticks;
@ -892,6 +962,7 @@ static ResultCode GetInfo32(Core::System& system, u32* result_low, u32* result_h
/// Maps memory at a desired address
static ResultCode MapPhysicalMemory(Core::System& system, VAddr addr, u64 size) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_DEBUG(Kernel_SVC, "called, addr=0x{:016X}, size=0x{:X}", addr, size);
if (!Common::Is4KBAligned(addr)) {
@ -939,8 +1010,13 @@ static ResultCode MapPhysicalMemory(Core::System& system, VAddr addr, u64 size)
return page_table.MapPhysicalMemory(addr, size);
}
static ResultCode MapPhysicalMemory32(Core::System& system, u32 addr, u32 size) {
return MapPhysicalMemory(system, static_cast<VAddr>(addr), static_cast<std::size_t>(size));
}
/// Unmaps memory previously mapped via MapPhysicalMemory
static ResultCode UnmapPhysicalMemory(Core::System& system, VAddr addr, u64 size) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_DEBUG(Kernel_SVC, "called, addr=0x{:016X}, size=0x{:X}", addr, size);
if (!Common::Is4KBAligned(addr)) {
@ -988,6 +1064,10 @@ static ResultCode UnmapPhysicalMemory(Core::System& system, VAddr addr, u64 size
return page_table.UnmapPhysicalMemory(addr, size);
}
static ResultCode UnmapPhysicalMemory32(Core::System& system, u32 addr, u32 size) {
return UnmapPhysicalMemory(system, static_cast<VAddr>(addr), static_cast<std::size_t>(size));
}
/// Sets the thread activity
static ResultCode SetThreadActivity(Core::System& system, Handle handle, u32 activity) {
LOG_DEBUG(Kernel_SVC, "called, handle=0x{:08X}, activity=0x{:08X}", handle, activity);
@ -1017,10 +1097,11 @@ static ResultCode SetThreadActivity(Core::System& system, Handle handle, u32 act
return ERR_BUSY;
}
thread->SetActivity(static_cast<ThreadActivity>(activity));
return thread->SetActivity(static_cast<ThreadActivity>(activity));
}
system.PrepareReschedule(thread->GetProcessorID());
return RESULT_SUCCESS;
static ResultCode SetThreadActivity32(Core::System& system, Handle handle, u32 activity) {
return SetThreadActivity(system, handle, activity);
}
/// Gets the thread context
@ -1064,6 +1145,10 @@ static ResultCode GetThreadContext(Core::System& system, VAddr thread_context, H
return RESULT_SUCCESS;
}
static ResultCode GetThreadContext32(Core::System& system, u32 thread_context, Handle handle) {
return GetThreadContext(system, static_cast<VAddr>(thread_context), handle);
}
/// Gets the priority for the specified thread
static ResultCode GetThreadPriority(Core::System& system, u32* priority, Handle handle) {
LOG_TRACE(Kernel_SVC, "called");
@ -1071,6 +1156,7 @@ static ResultCode GetThreadPriority(Core::System& system, u32* priority, Handle
const auto& handle_table = system.Kernel().CurrentProcess()->GetHandleTable();
const std::shared_ptr<Thread> thread = handle_table.Get<Thread>(handle);
if (!thread) {
*priority = 0;
LOG_ERROR(Kernel_SVC, "Thread handle does not exist, handle=0x{:08X}", handle);
return ERR_INVALID_HANDLE;
}
@ -1105,18 +1191,26 @@ static ResultCode SetThreadPriority(Core::System& system, Handle handle, u32 pri
thread->SetPriority(priority);
system.PrepareReschedule(thread->GetProcessorID());
return RESULT_SUCCESS;
}
static ResultCode SetThreadPriority32(Core::System& system, Handle handle, u32 priority) {
return SetThreadPriority(system, handle, priority);
}
/// Get which CPU core is executing the current thread
static u32 GetCurrentProcessorNumber(Core::System& system) {
LOG_TRACE(Kernel_SVC, "called");
return system.CurrentScheduler().GetCurrentThread()->GetProcessorID();
return static_cast<u32>(system.CurrentPhysicalCore().CoreIndex());
}
static u32 GetCurrentProcessorNumber32(Core::System& system) {
return GetCurrentProcessorNumber(system);
}
static ResultCode MapSharedMemory(Core::System& system, Handle shared_memory_handle, VAddr addr,
u64 size, u32 permissions) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_TRACE(Kernel_SVC,
"called, shared_memory_handle=0x{:X}, addr=0x{:X}, size=0x{:X}, permissions=0x{:08X}",
shared_memory_handle, addr, size, permissions);
@ -1187,9 +1281,16 @@ static ResultCode MapSharedMemory(Core::System& system, Handle shared_memory_han
return shared_memory->Map(*current_process, addr, size, permission_type);
}
static ResultCode MapSharedMemory32(Core::System& system, Handle shared_memory_handle, u32 addr,
u32 size, u32 permissions) {
return MapSharedMemory(system, shared_memory_handle, static_cast<VAddr>(addr),
static_cast<std::size_t>(size), permissions);
}
static ResultCode QueryProcessMemory(Core::System& system, VAddr memory_info_address,
VAddr page_info_address, Handle process_handle,
VAddr address) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_TRACE(Kernel_SVC, "called process=0x{:08X} address={:X}", process_handle, address);
const auto& handle_table = system.Kernel().CurrentProcess()->GetHandleTable();
std::shared_ptr<Process> process = handle_table.Get<Process>(process_handle);
@ -1372,6 +1473,7 @@ static ResultCode UnmapProcessCodeMemory(Core::System& system, Handle process_ha
/// Exits the current process
static void ExitProcess(Core::System& system) {
auto* current_process = system.Kernel().CurrentProcess();
UNIMPLEMENTED();
LOG_INFO(Kernel_SVC, "Process {} exiting", current_process->GetProcessID());
ASSERT_MSG(current_process->GetStatus() == ProcessStatus::Running,
@ -1381,8 +1483,10 @@ static void ExitProcess(Core::System& system) {
// Kill the current thread
system.CurrentScheduler().GetCurrentThread()->Stop();
}
system.PrepareReschedule();
static void ExitProcess32(Core::System& system) {
ExitProcess(system);
}
/// Creates a new thread
@ -1428,9 +1532,10 @@ static ResultCode CreateThread(Core::System& system, Handle* out_handle, VAddr e
ASSERT(kernel.CurrentProcess()->GetResourceLimit()->Reserve(ResourceType::Threads, 1));
ThreadType type = THREADTYPE_USER;
CASCADE_RESULT(std::shared_ptr<Thread> thread,
Thread::Create(kernel, "", entry_point, priority, arg, processor_id, stack_top,
*current_process));
Thread::Create(system, type, "", entry_point, priority, arg, processor_id,
stack_top, current_process));
const auto new_thread_handle = current_process->GetHandleTable().Create(thread);
if (new_thread_handle.Failed()) {
@ -1444,11 +1549,15 @@ static ResultCode CreateThread(Core::System& system, Handle* out_handle, VAddr e
thread->SetName(
fmt::format("thread[entry_point={:X}, handle={:X}]", entry_point, *new_thread_handle));
system.PrepareReschedule(thread->GetProcessorID());
return RESULT_SUCCESS;
}
static ResultCode CreateThread32(Core::System& system, Handle* out_handle, u32 priority,
u32 entry_point, u32 arg, u32 stack_top, s32 processor_id) {
return CreateThread(system, out_handle, static_cast<VAddr>(entry_point), static_cast<u64>(arg),
static_cast<VAddr>(stack_top), priority, processor_id);
}
/// Starts the thread for the provided handle
static ResultCode StartThread(Core::System& system, Handle thread_handle) {
LOG_DEBUG(Kernel_SVC, "called thread=0x{:08X}", thread_handle);
@ -1463,13 +1572,11 @@ static ResultCode StartThread(Core::System& system, Handle thread_handle) {
ASSERT(thread->GetStatus() == ThreadStatus::Dormant);
thread->ResumeFromWait();
return thread->Start();
}
if (thread->GetStatus() == ThreadStatus::Ready) {
system.PrepareReschedule(thread->GetProcessorID());
}
return RESULT_SUCCESS;
static ResultCode StartThread32(Core::System& system, Handle thread_handle) {
return StartThread(system, thread_handle);
}
/// Called when a thread exits
@ -1477,9 +1584,12 @@ static void ExitThread(Core::System& system) {
LOG_DEBUG(Kernel_SVC, "called, pc=0x{:08X}", system.CurrentArmInterface().GetPC());
auto* const current_thread = system.CurrentScheduler().GetCurrentThread();
current_thread->Stop();
system.GlobalScheduler().RemoveThread(SharedFrom(current_thread));
system.PrepareReschedule();
current_thread->Stop();
}
static void ExitThread32(Core::System& system) {
ExitThread(system);
}
/// Sleep the current thread
@ -1498,15 +1608,21 @@ static void SleepThread(Core::System& system, s64 nanoseconds) {
if (nanoseconds <= 0) {
switch (static_cast<SleepType>(nanoseconds)) {
case SleepType::YieldWithoutLoadBalancing:
is_redundant = current_thread->YieldSimple();
case SleepType::YieldWithoutLoadBalancing: {
auto pair = current_thread->YieldSimple();
is_redundant = pair.second;
break;
case SleepType::YieldWithLoadBalancing:
is_redundant = current_thread->YieldAndBalanceLoad();
}
case SleepType::YieldWithLoadBalancing: {
auto pair = current_thread->YieldAndBalanceLoad();
is_redundant = pair.second;
break;
case SleepType::YieldAndWaitForLoadBalancing:
is_redundant = current_thread->YieldAndWaitForLoadBalancing();
}
case SleepType::YieldAndWaitForLoadBalancing: {
auto pair = current_thread->YieldAndWaitForLoadBalancing();
is_redundant = pair.second;
break;
}
default:
UNREACHABLE_MSG("Unimplemented sleep yield type '{:016X}'!", nanoseconds);
}
@ -1514,13 +1630,18 @@ static void SleepThread(Core::System& system, s64 nanoseconds) {
current_thread->Sleep(nanoseconds);
}
if (is_redundant) {
// If it's redundant, the core is pretty much idle. Some games keep idling
// a core while it's doing nothing, we advance timing to avoid costly continuous
// calls.
system.CoreTiming().AddTicks(2000);
if (is_redundant && !system.Kernel().IsMulticore()) {
system.Kernel().ExitSVCProfile();
system.CoreTiming().AddTicks(1000U);
system.GetCpuManager().PreemptSingleCore();
system.Kernel().EnterSVCProfile();
}
system.PrepareReschedule(current_thread->GetProcessorID());
}
static void SleepThread32(Core::System& system, u32 nanoseconds_low, u32 nanoseconds_high) {
const s64 nanoseconds = static_cast<s64>(static_cast<u64>(nanoseconds_low) |
(static_cast<u64>(nanoseconds_high) << 32));
SleepThread(system, nanoseconds);
}
/// Wait process wide key atomic
@ -1547,31 +1668,69 @@ static ResultCode WaitProcessWideKeyAtomic(Core::System& system, VAddr mutex_add
}
ASSERT(condition_variable_addr == Common::AlignDown(condition_variable_addr, 4));
auto& kernel = system.Kernel();
Handle event_handle;
Thread* current_thread = system.CurrentScheduler().GetCurrentThread();
auto* const current_process = system.Kernel().CurrentProcess();
const auto& handle_table = current_process->GetHandleTable();
std::shared_ptr<Thread> thread = handle_table.Get<Thread>(thread_handle);
ASSERT(thread);
{
SchedulerLockAndSleep lock(kernel, event_handle, current_thread, nano_seconds);
const auto& handle_table = current_process->GetHandleTable();
std::shared_ptr<Thread> thread = handle_table.Get<Thread>(thread_handle);
ASSERT(thread);
const auto release_result = current_process->GetMutex().Release(mutex_addr);
if (release_result.IsError()) {
return release_result;
current_thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT);
if (thread->IsPendingTermination()) {
lock.CancelSleep();
return ERR_THREAD_TERMINATING;
}
const auto release_result = current_process->GetMutex().Release(mutex_addr);
if (release_result.IsError()) {
lock.CancelSleep();
return release_result;
}
if (nano_seconds == 0) {
lock.CancelSleep();
return RESULT_TIMEOUT;
}
current_thread->SetCondVarWaitAddress(condition_variable_addr);
current_thread->SetMutexWaitAddress(mutex_addr);
current_thread->SetWaitHandle(thread_handle);
current_thread->SetStatus(ThreadStatus::WaitCondVar);
current_process->InsertConditionVariableThread(SharedFrom(current_thread));
}
Thread* current_thread = system.CurrentScheduler().GetCurrentThread();
current_thread->SetCondVarWaitAddress(condition_variable_addr);
current_thread->SetMutexWaitAddress(mutex_addr);
current_thread->SetWaitHandle(thread_handle);
current_thread->SetStatus(ThreadStatus::WaitCondVar);
current_thread->InvalidateWakeupCallback();
current_process->InsertConditionVariableThread(SharedFrom(current_thread));
if (event_handle != InvalidHandle) {
auto& time_manager = kernel.TimeManager();
time_manager.UnscheduleTimeEvent(event_handle);
}
current_thread->WakeAfterDelay(nano_seconds);
{
SchedulerLock lock(kernel);
auto* owner = current_thread->GetLockOwner();
if (owner != nullptr) {
owner->RemoveMutexWaiter(SharedFrom(current_thread));
}
current_process->RemoveConditionVariableThread(SharedFrom(current_thread));
}
// Note: Deliberately don't attempt to inherit the lock owner's priority.
system.PrepareReschedule(current_thread->GetProcessorID());
return RESULT_SUCCESS;
return current_thread->GetSignalingResult();
}
static ResultCode WaitProcessWideKeyAtomic32(Core::System& system, u32 mutex_addr,
u32 condition_variable_addr, Handle thread_handle,
u32 nanoseconds_low, u32 nanoseconds_high) {
const s64 nanoseconds =
static_cast<s64>(nanoseconds_low | (static_cast<u64>(nanoseconds_high) << 32));
return WaitProcessWideKeyAtomic(system, static_cast<VAddr>(mutex_addr),
static_cast<VAddr>(condition_variable_addr), thread_handle,
nanoseconds);
}
/// Signal process wide key
@ -1582,7 +1741,9 @@ static void SignalProcessWideKey(Core::System& system, VAddr condition_variable_
ASSERT(condition_variable_addr == Common::AlignDown(condition_variable_addr, 4));
// Retrieve a list of all threads that are waiting for this condition variable.
auto* const current_process = system.Kernel().CurrentProcess();
auto& kernel = system.Kernel();
SchedulerLock lock(kernel);
auto* const current_process = kernel.CurrentProcess();
std::vector<std::shared_ptr<Thread>> waiting_threads =
current_process->GetConditionVariableThreads(condition_variable_addr);
@ -1591,7 +1752,7 @@ static void SignalProcessWideKey(Core::System& system, VAddr condition_variable_
std::size_t last = waiting_threads.size();
if (target > 0)
last = std::min(waiting_threads.size(), static_cast<std::size_t>(target));
auto& time_manager = kernel.TimeManager();
for (std::size_t index = 0; index < last; ++index) {
auto& thread = waiting_threads[index];
@ -1599,7 +1760,6 @@ static void SignalProcessWideKey(Core::System& system, VAddr condition_variable_
// liberate Cond Var Thread.
current_process->RemoveConditionVariableThread(thread);
thread->SetCondVarWaitAddress(0);
const std::size_t current_core = system.CurrentCoreIndex();
auto& monitor = system.Monitor();
@ -1610,10 +1770,8 @@ static void SignalProcessWideKey(Core::System& system, VAddr condition_variable_
u32 update_val = 0;
const VAddr mutex_address = thread->GetMutexWaitAddress();
do {
monitor.SetExclusive(current_core, mutex_address);
// If the mutex is not yet acquired, acquire it.
mutex_val = memory.Read32(mutex_address);
mutex_val = monitor.ExclusiveRead32(current_core, mutex_address);
if (mutex_val != 0) {
update_val = mutex_val | Mutex::MutexHasWaitersFlag;
@ -1621,33 +1779,28 @@ static void SignalProcessWideKey(Core::System& system, VAddr condition_variable_
update_val = thread->GetWaitHandle();
}
} while (!monitor.ExclusiveWrite32(current_core, mutex_address, update_val));
monitor.ClearExclusive();
if (mutex_val == 0) {
// We were able to acquire the mutex, resume this thread.
ASSERT(thread->GetStatus() == ThreadStatus::WaitCondVar);
thread->ResumeFromWait();
auto* const lock_owner = thread->GetLockOwner();
if (lock_owner != nullptr) {
lock_owner->RemoveMutexWaiter(thread);
}
thread->SetLockOwner(nullptr);
thread->SetMutexWaitAddress(0);
thread->SetWaitHandle(0);
thread->SetWaitSynchronizationResult(RESULT_SUCCESS);
system.PrepareReschedule(thread->GetProcessorID());
thread->SetSynchronizationResults(nullptr, RESULT_SUCCESS);
thread->ResumeFromWait();
} else {
// The mutex is already owned by some other thread, make this thread wait on it.
const Handle owner_handle = static_cast<Handle>(mutex_val & Mutex::MutexOwnerMask);
const auto& handle_table = system.Kernel().CurrentProcess()->GetHandleTable();
auto owner = handle_table.Get<Thread>(owner_handle);
ASSERT(owner);
ASSERT(thread->GetStatus() == ThreadStatus::WaitCondVar);
thread->InvalidateWakeupCallback();
thread->SetStatus(ThreadStatus::WaitMutex);
if (thread->GetStatus() == ThreadStatus::WaitCondVar) {
thread->SetStatus(ThreadStatus::WaitMutex);
}
owner->AddMutexWaiter(thread);
system.PrepareReschedule(thread->GetProcessorID());
}
}
}
@ -1678,12 +1831,15 @@ static ResultCode WaitForAddress(Core::System& system, VAddr address, u32 type,
auto& address_arbiter = system.Kernel().CurrentProcess()->GetAddressArbiter();
const ResultCode result =
address_arbiter.WaitForAddress(address, arbitration_type, value, timeout);
if (result == RESULT_SUCCESS) {
system.PrepareReschedule();
}
return result;
}
static ResultCode WaitForAddress32(Core::System& system, u32 address, u32 type, s32 value,
u32 timeout_low, u32 timeout_high) {
s64 timeout = static_cast<s64>(timeout_low | (static_cast<u64>(timeout_high) << 32));
return WaitForAddress(system, static_cast<VAddr>(address), type, value, timeout);
}
// Signals to an address (via Address Arbiter)
static ResultCode SignalToAddress(Core::System& system, VAddr address, u32 type, s32 value,
s32 num_to_wake) {
@ -1707,6 +1863,11 @@ static ResultCode SignalToAddress(Core::System& system, VAddr address, u32 type,
return address_arbiter.SignalToAddress(address, signal_type, value, num_to_wake);
}
static ResultCode SignalToAddress32(Core::System& system, u32 address, u32 type, s32 value,
s32 num_to_wake) {
return SignalToAddress(system, static_cast<VAddr>(address), type, value, num_to_wake);
}
static void KernelDebug([[maybe_unused]] Core::System& system,
[[maybe_unused]] u32 kernel_debug_type, [[maybe_unused]] u64 param1,
[[maybe_unused]] u64 param2, [[maybe_unused]] u64 param3) {
@ -1725,14 +1886,21 @@ static u64 GetSystemTick(Core::System& system) {
auto& core_timing = system.CoreTiming();
// Returns the value of cntpct_el0 (https://switchbrew.org/wiki/SVC#svcGetSystemTick)
const u64 result{Core::Timing::CpuCyclesToClockCycles(system.CoreTiming().GetTicks())};
const u64 result{system.CoreTiming().GetClockTicks()};
// Advance time to defeat dumb games that busy-wait for the frame to end.
core_timing.AddTicks(400);
if (!system.Kernel().IsMulticore()) {
core_timing.AddTicks(400U);
}
return result;
}
static void GetSystemTick32(Core::System& system, u32* time_low, u32* time_high) {
u64 time = GetSystemTick(system);
*time_low = static_cast<u32>(time);
*time_high = static_cast<u32>(time >> 32);
}
/// Close a handle
static ResultCode CloseHandle(Core::System& system, Handle handle) {
LOG_TRACE(Kernel_SVC, "Closing handle 0x{:08X}", handle);
@ -1765,9 +1933,14 @@ static ResultCode ResetSignal(Core::System& system, Handle handle) {
return ERR_INVALID_HANDLE;
}
static ResultCode ResetSignal32(Core::System& system, Handle handle) {
return ResetSignal(system, handle);
}
/// Creates a TransferMemory object
static ResultCode CreateTransferMemory(Core::System& system, Handle* handle, VAddr addr, u64 size,
u32 permissions) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_DEBUG(Kernel_SVC, "called addr=0x{:X}, size=0x{:X}, perms=0x{:08X}", addr, size,
permissions);
@ -1812,6 +1985,12 @@ static ResultCode CreateTransferMemory(Core::System& system, Handle* handle, VAd
return RESULT_SUCCESS;
}
static ResultCode CreateTransferMemory32(Core::System& system, Handle* handle, u32 addr, u32 size,
u32 permissions) {
return CreateTransferMemory(system, handle, static_cast<VAddr>(addr),
static_cast<std::size_t>(size), permissions);
}
static ResultCode GetThreadCoreMask(Core::System& system, Handle thread_handle, u32* core,
u64* mask) {
LOG_TRACE(Kernel_SVC, "called, handle=0x{:08X}", thread_handle);
@ -1821,6 +2000,8 @@ static ResultCode GetThreadCoreMask(Core::System& system, Handle thread_handle,
if (!thread) {
LOG_ERROR(Kernel_SVC, "Thread handle does not exist, thread_handle=0x{:08X}",
thread_handle);
*core = 0;
*mask = 0;
return ERR_INVALID_HANDLE;
}
@ -1830,6 +2011,15 @@ static ResultCode GetThreadCoreMask(Core::System& system, Handle thread_handle,
return RESULT_SUCCESS;
}
static ResultCode GetThreadCoreMask32(Core::System& system, Handle thread_handle, u32* core,
u32* mask_low, u32* mask_high) {
u64 mask{};
const auto result = GetThreadCoreMask(system, thread_handle, core, &mask);
*mask_high = static_cast<u32>(mask >> 32);
*mask_low = static_cast<u32>(mask);
return result;
}
static ResultCode SetThreadCoreMask(Core::System& system, Handle thread_handle, u32 core,
u64 affinity_mask) {
LOG_DEBUG(Kernel_SVC, "called, handle=0x{:08X}, core=0x{:X}, affinity_mask=0x{:016X}",
@ -1861,7 +2051,7 @@ static ResultCode SetThreadCoreMask(Core::System& system, Handle thread_handle,
return ERR_INVALID_COMBINATION;
}
if (core < Core::NUM_CPU_CORES) {
if (core < Core::Hardware::NUM_CPU_CORES) {
if ((affinity_mask & (1ULL << core)) == 0) {
LOG_ERROR(Kernel_SVC,
"Core is not enabled for the current mask, core={}, mask={:016X}", core,
@ -1883,11 +2073,14 @@ static ResultCode SetThreadCoreMask(Core::System& system, Handle thread_handle,
return ERR_INVALID_HANDLE;
}
system.PrepareReschedule(thread->GetProcessorID());
thread->ChangeCore(core, affinity_mask);
system.PrepareReschedule(thread->GetProcessorID());
return thread->SetCoreAndAffinityMask(core, affinity_mask);
}
return RESULT_SUCCESS;
static ResultCode SetThreadCoreMask32(Core::System& system, Handle thread_handle, u32 core,
u32 affinity_mask_low, u32 affinity_mask_high) {
const u64 affinity_mask =
static_cast<u64>(affinity_mask_low) | (static_cast<u64>(affinity_mask_high) << 32);
return SetThreadCoreMask(system, thread_handle, core, affinity_mask);
}
static ResultCode CreateEvent(Core::System& system, Handle* write_handle, Handle* read_handle) {
@ -1918,6 +2111,10 @@ static ResultCode CreateEvent(Core::System& system, Handle* write_handle, Handle
return RESULT_SUCCESS;
}
static ResultCode CreateEvent32(Core::System& system, Handle* write_handle, Handle* read_handle) {
return CreateEvent(system, write_handle, read_handle);
}
static ResultCode ClearEvent(Core::System& system, Handle handle) {
LOG_TRACE(Kernel_SVC, "called, event=0x{:08X}", handle);
@ -1939,6 +2136,10 @@ static ResultCode ClearEvent(Core::System& system, Handle handle) {
return ERR_INVALID_HANDLE;
}
static ResultCode ClearEvent32(Core::System& system, Handle handle) {
return ClearEvent(system, handle);
}
static ResultCode SignalEvent(Core::System& system, Handle handle) {
LOG_DEBUG(Kernel_SVC, "called. Handle=0x{:08X}", handle);
@ -1951,10 +2152,13 @@ static ResultCode SignalEvent(Core::System& system, Handle handle) {
}
writable_event->Signal();
system.PrepareReschedule();
return RESULT_SUCCESS;
}
static ResultCode SignalEvent32(Core::System& system, Handle handle) {
return SignalEvent(system, handle);
}
static ResultCode GetProcessInfo(Core::System& system, u64* out, Handle process_handle, u32 type) {
LOG_DEBUG(Kernel_SVC, "called, handle=0x{:08X}, type=0x{:X}", process_handle, type);
@ -1982,6 +2186,7 @@ static ResultCode GetProcessInfo(Core::System& system, u64* out, Handle process_
}
static ResultCode CreateResourceLimit(Core::System& system, Handle* out_handle) {
std::lock_guard lock{HLE::g_hle_lock};
LOG_DEBUG(Kernel_SVC, "called");
auto& kernel = system.Kernel();
@ -2139,6 +2344,15 @@ static ResultCode GetThreadList(Core::System& system, u32* out_num_threads, VAdd
return RESULT_SUCCESS;
}
static ResultCode FlushProcessDataCache32(Core::System& system, Handle handle, u32 address,
u32 size) {
// Note(Blinkhawk): For emulation purposes of the data cache this is mostly a nope
// as all emulation is done in the same cache level in host architecture, thus data cache
// does not need flushing.
LOG_DEBUG(Kernel_SVC, "called");
return RESULT_SUCCESS;
}
namespace {
struct FunctionDef {
using Func = void(Core::System&);
@ -2153,57 +2367,57 @@ static const FunctionDef SVC_Table_32[] = {
{0x00, nullptr, "Unknown"},
{0x01, SvcWrap32<SetHeapSize32>, "SetHeapSize32"},
{0x02, nullptr, "Unknown"},
{0x03, nullptr, "SetMemoryAttribute32"},
{0x04, nullptr, "MapMemory32"},
{0x05, nullptr, "UnmapMemory32"},
{0x03, SvcWrap32<SetMemoryAttribute32>, "SetMemoryAttribute32"},
{0x04, SvcWrap32<MapMemory32>, "MapMemory32"},
{0x05, SvcWrap32<UnmapMemory32>, "UnmapMemory32"},
{0x06, SvcWrap32<QueryMemory32>, "QueryMemory32"},
{0x07, nullptr, "ExitProcess32"},
{0x08, nullptr, "CreateThread32"},
{0x09, nullptr, "StartThread32"},
{0x0a, nullptr, "ExitThread32"},
{0x0b, nullptr, "SleepThread32"},
{0x07, SvcWrap32<ExitProcess32>, "ExitProcess32"},
{0x08, SvcWrap32<CreateThread32>, "CreateThread32"},
{0x09, SvcWrap32<StartThread32>, "StartThread32"},
{0x0a, SvcWrap32<ExitThread32>, "ExitThread32"},
{0x0b, SvcWrap32<SleepThread32>, "SleepThread32"},
{0x0c, SvcWrap32<GetThreadPriority32>, "GetThreadPriority32"},
{0x0d, nullptr, "SetThreadPriority32"},
{0x0e, nullptr, "GetThreadCoreMask32"},
{0x0f, nullptr, "SetThreadCoreMask32"},
{0x10, nullptr, "GetCurrentProcessorNumber32"},
{0x11, nullptr, "SignalEvent32"},
{0x12, nullptr, "ClearEvent32"},
{0x13, nullptr, "MapSharedMemory32"},
{0x0d, SvcWrap32<SetThreadPriority32>, "SetThreadPriority32"},
{0x0e, SvcWrap32<GetThreadCoreMask32>, "GetThreadCoreMask32"},
{0x0f, SvcWrap32<SetThreadCoreMask32>, "SetThreadCoreMask32"},
{0x10, SvcWrap32<GetCurrentProcessorNumber32>, "GetCurrentProcessorNumber32"},
{0x11, SvcWrap32<SignalEvent32>, "SignalEvent32"},
{0x12, SvcWrap32<ClearEvent32>, "ClearEvent32"},
{0x13, SvcWrap32<MapSharedMemory32>, "MapSharedMemory32"},
{0x14, nullptr, "UnmapSharedMemory32"},
{0x15, nullptr, "CreateTransferMemory32"},
{0x15, SvcWrap32<CreateTransferMemory32>, "CreateTransferMemory32"},
{0x16, SvcWrap32<CloseHandle32>, "CloseHandle32"},
{0x17, nullptr, "ResetSignal32"},
{0x17, SvcWrap32<ResetSignal32>, "ResetSignal32"},
{0x18, SvcWrap32<WaitSynchronization32>, "WaitSynchronization32"},
{0x19, nullptr, "CancelSynchronization32"},
{0x1a, nullptr, "ArbitrateLock32"},
{0x1b, nullptr, "ArbitrateUnlock32"},
{0x1c, nullptr, "WaitProcessWideKeyAtomic32"},
{0x19, SvcWrap32<CancelSynchronization32>, "CancelSynchronization32"},
{0x1a, SvcWrap32<ArbitrateLock32>, "ArbitrateLock32"},
{0x1b, SvcWrap32<ArbitrateUnlock32>, "ArbitrateUnlock32"},
{0x1c, SvcWrap32<WaitProcessWideKeyAtomic32>, "WaitProcessWideKeyAtomic32"},
{0x1d, SvcWrap32<SignalProcessWideKey32>, "SignalProcessWideKey32"},
{0x1e, nullptr, "GetSystemTick32"},
{0x1e, SvcWrap32<GetSystemTick32>, "GetSystemTick32"},
{0x1f, SvcWrap32<ConnectToNamedPort32>, "ConnectToNamedPort32"},
{0x20, nullptr, "Unknown"},
{0x21, SvcWrap32<SendSyncRequest32>, "SendSyncRequest32"},
{0x22, nullptr, "SendSyncRequestWithUserBuffer32"},
{0x23, nullptr, "Unknown"},
{0x24, nullptr, "GetProcessId32"},
{0x24, SvcWrap32<GetProcessId32>, "GetProcessId32"},
{0x25, SvcWrap32<GetThreadId32>, "GetThreadId32"},
{0x26, nullptr, "Break32"},
{0x26, SvcWrap32<Break32>, "Break32"},
{0x27, nullptr, "OutputDebugString32"},
{0x28, nullptr, "Unknown"},
{0x29, SvcWrap32<GetInfo32>, "GetInfo32"},
{0x2a, nullptr, "Unknown"},
{0x2b, nullptr, "Unknown"},
{0x2c, nullptr, "MapPhysicalMemory32"},
{0x2d, nullptr, "UnmapPhysicalMemory32"},
{0x2c, SvcWrap32<MapPhysicalMemory32>, "MapPhysicalMemory32"},
{0x2d, SvcWrap32<UnmapPhysicalMemory32>, "UnmapPhysicalMemory32"},
{0x2e, nullptr, "Unknown"},
{0x2f, nullptr, "Unknown"},
{0x30, nullptr, "Unknown"},
{0x31, nullptr, "Unknown"},
{0x32, nullptr, "SetThreadActivity32"},
{0x33, nullptr, "GetThreadContext32"},
{0x34, nullptr, "WaitForAddress32"},
{0x35, nullptr, "SignalToAddress32"},
{0x32, SvcWrap32<SetThreadActivity32>, "SetThreadActivity32"},
{0x33, SvcWrap32<GetThreadContext32>, "GetThreadContext32"},
{0x34, SvcWrap32<WaitForAddress32>, "WaitForAddress32"},
{0x35, SvcWrap32<SignalToAddress32>, "SignalToAddress32"},
{0x36, nullptr, "Unknown"},
{0x37, nullptr, "Unknown"},
{0x38, nullptr, "Unknown"},
@ -2219,7 +2433,7 @@ static const FunctionDef SVC_Table_32[] = {
{0x42, nullptr, "Unknown"},
{0x43, nullptr, "ReplyAndReceive32"},
{0x44, nullptr, "Unknown"},
{0x45, nullptr, "CreateEvent32"},
{0x45, SvcWrap32<CreateEvent32>, "CreateEvent32"},
{0x46, nullptr, "Unknown"},
{0x47, nullptr, "Unknown"},
{0x48, nullptr, "Unknown"},
@ -2245,7 +2459,7 @@ static const FunctionDef SVC_Table_32[] = {
{0x5c, nullptr, "Unknown"},
{0x5d, nullptr, "Unknown"},
{0x5e, nullptr, "Unknown"},
{0x5F, nullptr, "FlushProcessDataCache32"},
{0x5F, SvcWrap32<FlushProcessDataCache32>, "FlushProcessDataCache32"},
{0x60, nullptr, "Unknown"},
{0x61, nullptr, "Unknown"},
{0x62, nullptr, "Unknown"},
@ -2423,13 +2637,10 @@ static const FunctionDef* GetSVCInfo64(u32 func_num) {
return &SVC_Table_64[func_num];
}
MICROPROFILE_DEFINE(Kernel_SVC, "Kernel", "SVC", MP_RGB(70, 200, 70));
void Call(Core::System& system, u32 immediate) {
MICROPROFILE_SCOPE(Kernel_SVC);
// Lock the global kernel mutex when we enter the kernel HLE.
std::lock_guard lock{HLE::g_hle_lock};
system.ExitDynarmicProfile();
auto& kernel = system.Kernel();
kernel.EnterSVCProfile();
const FunctionDef* info = system.CurrentProcess()->Is64BitProcess() ? GetSVCInfo64(immediate)
: GetSVCInfo32(immediate);
@ -2442,6 +2653,9 @@ void Call(Core::System& system, u32 immediate) {
} else {
LOG_CRITICAL(Kernel_SVC, "Unknown SVC function 0x{:X}", immediate);
}
kernel.ExitSVCProfile();
system.EnterDynarmicProfile();
}
} // namespace Kernel::Svc

137
src/core/hle/kernel/svc_wrap.h
View file

@ -350,13 +350,50 @@ void SvcWrap64(Core::System& system) {
func(system, static_cast<u32>(Param(system, 0)), Param(system, 1), Param(system, 2));
}
// Used by QueryMemory32
// Used by QueryMemory32, ArbitrateLock32
template <ResultCode func(Core::System&, u32, u32, u32)>
void SvcWrap32(Core::System& system) {
FuncReturn32(system,
func(system, Param32(system, 0), Param32(system, 1), Param32(system, 2)).raw);
}
// Used by Break32
template <void func(Core::System&, u32, u32, u32)>
void SvcWrap32(Core::System& system) {
func(system, Param32(system, 0), Param32(system, 1), Param32(system, 2));
}
// Used by ExitProcess32, ExitThread32
template <void func(Core::System&)>
void SvcWrap32(Core::System& system) {
func(system);
}
// Used by GetCurrentProcessorNumber32
template <u32 func(Core::System&)>
void SvcWrap32(Core::System& system) {
FuncReturn32(system, func(system));
}
// Used by SleepThread32
template <void func(Core::System&, u32, u32)>
void SvcWrap32(Core::System& system) {
func(system, Param32(system, 0), Param32(system, 1));
}
// Used by CreateThread32
template <ResultCode func(Core::System&, Handle*, u32, u32, u32, u32, s32)>
void SvcWrap32(Core::System& system) {
Handle param_1 = 0;
const u32 retval = func(system, &param_1, Param32(system, 0), Param32(system, 1),
Param32(system, 2), Param32(system, 3), Param32(system, 4))
.raw;
system.CurrentArmInterface().SetReg(1, param_1);
FuncReturn(system, retval);
}
// Used by GetInfo32
template <ResultCode func(Core::System&, u32*, u32*, u32, u32, u32, u32)>
void SvcWrap32(Core::System& system) {
@ -393,18 +430,114 @@ void SvcWrap32(Core::System& system) {
FuncReturn(system, retval);
}
// Used by GetSystemTick32
template <void func(Core::System&, u32*, u32*)>
void SvcWrap32(Core::System& system) {
u32 param_1 = 0;
u32 param_2 = 0;
func(system, &param_1, &param_2);
system.CurrentArmInterface().SetReg(0, param_1);
system.CurrentArmInterface().SetReg(1, param_2);
}
// Used by CreateEvent32
template <ResultCode func(Core::System&, Handle*, Handle*)>
void SvcWrap32(Core::System& system) {
Handle param_1 = 0;
Handle param_2 = 0;
const u32 retval = func(system, &param_1, &param_2).raw;
system.CurrentArmInterface().SetReg(1, param_1);
system.CurrentArmInterface().SetReg(2, param_2);
FuncReturn(system, retval);
}
// Used by GetThreadId32
template <ResultCode func(Core::System&, Handle, u32*, u32*, u32*)>
void SvcWrap32(Core::System& system) {
u32 param_1 = 0;
u32 param_2 = 0;
u32 param_3 = 0;
const u32 retval = func(system, Param32(system, 2), &param_1, &param_2, &param_3).raw;
system.CurrentArmInterface().SetReg(1, param_1);
system.CurrentArmInterface().SetReg(2, param_2);
system.CurrentArmInterface().SetReg(3, param_3);
FuncReturn(system, retval);
}
// Used by SignalProcessWideKey32
template <void func(Core::System&, u32, s32)>
void SvcWrap32(Core::System& system) {
func(system, static_cast<u32>(Param(system, 0)), static_cast<s32>(Param(system, 1)));
}
// Used by SendSyncRequest32
// Used by SetThreadPriority32
template <ResultCode func(Core::System&, Handle, u32)>
void SvcWrap32(Core::System& system) {
const u32 retval =
func(system, static_cast<Handle>(Param(system, 0)), static_cast<u32>(Param(system, 1))).raw;
FuncReturn(system, retval);
}
// Used by SetThreadCoreMask32
template <ResultCode func(Core::System&, Handle, u32, u32, u32)>
void SvcWrap32(Core::System& system) {
const u32 retval =
func(system, static_cast<Handle>(Param(system, 0)), static_cast<u32>(Param(system, 1)),
static_cast<u32>(Param(system, 2)), static_cast<u32>(Param(system, 3)))
.raw;
FuncReturn(system, retval);
}
// Used by WaitProcessWideKeyAtomic32
template <ResultCode func(Core::System&, u32, u32, Handle, u32, u32)>
void SvcWrap32(Core::System& system) {
const u32 retval =
func(system, static_cast<u32>(Param(system, 0)), static_cast<u32>(Param(system, 1)),
static_cast<Handle>(Param(system, 2)), static_cast<u32>(Param(system, 3)),
static_cast<u32>(Param(system, 4)))
.raw;
FuncReturn(system, retval);
}
// Used by WaitForAddress32
template <ResultCode func(Core::System&, u32, u32, s32, u32, u32)>
void SvcWrap32(Core::System& system) {
const u32 retval = func(system, static_cast<u32>(Param(system, 0)),
static_cast<u32>(Param(system, 1)), static_cast<s32>(Param(system, 2)),
static_cast<u32>(Param(system, 3)), static_cast<u32>(Param(system, 4)))
.raw;
FuncReturn(system, retval);
}
// Used by SignalToAddress32
template <ResultCode func(Core::System&, u32, u32, s32, s32)>
void SvcWrap32(Core::System& system) {
const u32 retval =
func(system, static_cast<u32>(Param(system, 0)), static_cast<u32>(Param(system, 1)),
static_cast<s32>(Param(system, 2)), static_cast<s32>(Param(system, 3)))
.raw;
FuncReturn(system, retval);
}
// Used by SendSyncRequest32, ArbitrateUnlock32
template <ResultCode func(Core::System&, u32)>
void SvcWrap32(Core::System& system) {
FuncReturn(system, func(system, static_cast<u32>(Param(system, 0))).raw);
}
// Used by CreateTransferMemory32
template <ResultCode func(Core::System&, Handle*, u32, u32, u32)>
void SvcWrap32(Core::System& system) {
Handle handle = 0;
const u32 retval =
func(system, &handle, Param32(system, 1), Param32(system, 2), Param32(system, 3)).raw;
system.CurrentArmInterface().SetReg(1, handle);
FuncReturn(system, retval);
}
// Used by WaitSynchronization32
template <ResultCode func(Core::System&, u32, u32, s32, u32, Handle*)>
void SvcWrap32(Core::System& system) {

139
src/core/hle/kernel/synchronization.cpp
View file

@ -10,78 +10,107 @@
#include "core/hle/kernel/synchronization.h"
#include "core/hle/kernel/synchronization_object.h"
#include "core/hle/kernel/thread.h"
#include "core/hle/kernel/time_manager.h"
namespace Kernel {
/// Default thread wakeup callback for WaitSynchronization
static bool DefaultThreadWakeupCallback(ThreadWakeupReason reason, std::shared_ptr<Thread> thread,
std::shared_ptr<SynchronizationObject> object,
std::size_t index) {
ASSERT(thread->GetStatus() == ThreadStatus::WaitSynch);
if (reason == ThreadWakeupReason::Timeout) {
thread->SetWaitSynchronizationResult(RESULT_TIMEOUT);
return true;
}
ASSERT(reason == ThreadWakeupReason::Signal);
thread->SetWaitSynchronizationResult(RESULT_SUCCESS);
thread->SetWaitSynchronizationOutput(static_cast<u32>(index));
return true;
}
Synchronization::Synchronization(Core::System& system) : system{system} {}
void Synchronization::SignalObject(SynchronizationObject& obj) const {
auto& kernel = system.Kernel();
SchedulerLock lock(kernel);
auto& time_manager = kernel.TimeManager();
if (obj.IsSignaled()) {
obj.WakeupAllWaitingThreads();
for (auto thread : obj.GetWaitingThreads()) {
if (thread->GetSchedulingStatus() == ThreadSchedStatus::Paused) {
if (thread->GetStatus() != ThreadStatus::WaitHLEEvent) {
ASSERT(thread->GetStatus() == ThreadStatus::WaitSynch);
ASSERT(thread->IsWaitingSync());
}
thread->SetSynchronizationResults(&obj, RESULT_SUCCESS);
thread->ResumeFromWait();
}
}
obj.ClearWaitingThreads();
}
}
std::pair<ResultCode, Handle> Synchronization::WaitFor(
std::vector<std::shared_ptr<SynchronizationObject>>& sync_objects, s64 nano_seconds) {
auto& kernel = system.Kernel();
auto* const thread = system.CurrentScheduler().GetCurrentThread();
// Find the first object that is acquirable in the provided list of objects
const auto itr = std::find_if(sync_objects.begin(), sync_objects.end(),
[thread](const std::shared_ptr<SynchronizationObject>& object) {
return object->IsSignaled();
});
Handle event_handle = InvalidHandle;
{
SchedulerLockAndSleep lock(kernel, event_handle, thread, nano_seconds);
const auto itr =
std::find_if(sync_objects.begin(), sync_objects.end(),
[thread](const std::shared_ptr<SynchronizationObject>& object) {
return object->IsSignaled();
});
if (itr != sync_objects.end()) {
// We found a ready object, acquire it and set the result value
SynchronizationObject* object = itr->get();
object->Acquire(thread);
const u32 index = static_cast<s32>(std::distance(sync_objects.begin(), itr));
return {RESULT_SUCCESS, index};
if (itr != sync_objects.end()) {
// We found a ready object, acquire it and set the result value
SynchronizationObject* object = itr->get();
object->Acquire(thread);
const u32 index = static_cast<s32>(std::distance(sync_objects.begin(), itr));
lock.CancelSleep();
return {RESULT_SUCCESS, index};
}
if (nano_seconds == 0) {
lock.CancelSleep();
return {RESULT_TIMEOUT, InvalidHandle};
}
if (thread->IsPendingTermination()) {
lock.CancelSleep();
return {ERR_THREAD_TERMINATING, InvalidHandle};
}
if (thread->IsSyncCancelled()) {
thread->SetSyncCancelled(false);
lock.CancelSleep();
return {ERR_SYNCHRONIZATION_CANCELED, InvalidHandle};
}
for (auto& object : sync_objects) {
object->AddWaitingThread(SharedFrom(thread));
}
thread->SetSynchronizationObjects(&sync_objects);
thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT);
thread->SetStatus(ThreadStatus::WaitSynch);
thread->SetWaitingSync(true);
}
thread->SetWaitingSync(false);
if (event_handle != InvalidHandle) {
auto& time_manager = kernel.TimeManager();
time_manager.UnscheduleTimeEvent(event_handle);
}
// No objects were ready to be acquired, prepare to suspend the thread.
// If a timeout value of 0 was provided, just return the Timeout error code instead of
// suspending the thread.
if (nano_seconds == 0) {
return {RESULT_TIMEOUT, InvalidHandle};
{
SchedulerLock lock(kernel);
ResultCode signaling_result = thread->GetSignalingResult();
SynchronizationObject* signaling_object = thread->GetSignalingObject();
thread->SetSynchronizationObjects(nullptr);
auto shared_thread = SharedFrom(thread);
for (auto& obj : sync_objects) {
obj->RemoveWaitingThread(shared_thread);
}
if (signaling_object != nullptr) {
const auto itr = std::find_if(
sync_objects.begin(), sync_objects.end(),
[signaling_object](const std::shared_ptr<SynchronizationObject>& object) {
return object.get() == signaling_object;
});
ASSERT(itr != sync_objects.end());
signaling_object->Acquire(thread);
const u32 index = static_cast<s32>(std::distance(sync_objects.begin(), itr));
return {signaling_result, index};
}
return {signaling_result, -1};
}
if (thread->IsSyncCancelled()) {
thread->SetSyncCancelled(false);
return {ERR_SYNCHRONIZATION_CANCELED, InvalidHandle};
}
for (auto& object : sync_objects) {
object->AddWaitingThread(SharedFrom(thread));
}
thread->SetSynchronizationObjects(std::move(sync_objects));
thread->SetStatus(ThreadStatus::WaitSynch);
// Create an event to wake the thread up after the specified nanosecond delay has passed
thread->WakeAfterDelay(nano_seconds);
thread->SetWakeupCallback(DefaultThreadWakeupCallback);
system.PrepareReschedule(thread->GetProcessorID());
return {RESULT_TIMEOUT, InvalidHandle};
}
} // namespace Kernel

64
src/core/hle/kernel/synchronization_object.cpp
View file

@ -38,68 +38,8 @@ void SynchronizationObject::RemoveWaitingThread(std::shared_ptr<Thread> thread)
waiting_threads.erase(itr);
}
std::shared_ptr<Thread> SynchronizationObject::GetHighestPriorityReadyThread() const {
Thread* candidate = nullptr;
u32 candidate_priority = THREADPRIO_LOWEST + 1;
for (const auto& thread : waiting_threads) {
const ThreadStatus thread_status = thread->GetStatus();
// The list of waiting threads must not contain threads that are not waiting to be awakened.
ASSERT_MSG(thread_status == ThreadStatus::WaitSynch ||
thread_status == ThreadStatus::WaitHLEEvent,
"Inconsistent thread statuses in waiting_threads");
if (thread->GetPriority() >= candidate_priority)
continue;
if (ShouldWait(thread.get()))
continue;
candidate = thread.get();
candidate_priority = thread->GetPriority();
}
return SharedFrom(candidate);
}
void SynchronizationObject::WakeupWaitingThread(std::shared_ptr<Thread> thread) {
ASSERT(!ShouldWait(thread.get()));
if (!thread) {
return;
}
if (thread->IsSleepingOnWait()) {
for (const auto& object : thread->GetSynchronizationObjects()) {
ASSERT(!object->ShouldWait(thread.get()));
object->Acquire(thread.get());
}
} else {
Acquire(thread.get());
}
const std::size_t index = thread->GetSynchronizationObjectIndex(SharedFrom(this));
thread->ClearSynchronizationObjects();
thread->CancelWakeupTimer();
bool resume = true;
if (thread->HasWakeupCallback()) {
resume = thread->InvokeWakeupCallback(ThreadWakeupReason::Signal, thread, SharedFrom(this),
index);
}
if (resume) {
thread->ResumeFromWait();
kernel.PrepareReschedule(thread->GetProcessorID());
}
}
void SynchronizationObject::WakeupAllWaitingThreads() {
while (auto thread = GetHighestPriorityReadyThread()) {
WakeupWaitingThread(thread);
}
void SynchronizationObject::ClearWaitingThreads() {
waiting_threads.clear();
}
const std::vector<std::shared_ptr<Thread>>& SynchronizationObject::GetWaitingThreads() const {

18
src/core/hle/kernel/synchronization_object.h
View file

@ -12,6 +12,7 @@
namespace Kernel {
class KernelCore;
class Synchronization;
class Thread;
/// Class that represents a Kernel object that a thread can be waiting on
@ -49,24 +50,11 @@ public:
*/
void RemoveWaitingThread(std::shared_ptr<Thread> thread);
/**
* Wake up all threads waiting on this object that can be awoken, in priority order,
* and set the synchronization result and output of the thread.
*/
void WakeupAllWaitingThreads();
/**
* Wakes up a single thread waiting on this object.
* @param thread Thread that is waiting on this object to wakeup.
*/
void WakeupWaitingThread(std::shared_ptr<Thread> thread);
/// Obtains the highest priority thread that is ready to run from this object's waiting list.
std::shared_ptr<Thread> GetHighestPriorityReadyThread() const;
/// Get a const reference to the waiting threads list for debug use
const std::vector<std::shared_ptr<Thread>>& GetWaitingThreads() const;
void ClearWaitingThreads();
protected:
bool is_signaled{}; // Tells if this sync object is signalled;

424
src/core/hle/kernel/thread.cpp
View file

@ -9,12 +9,21 @@
#include "common/assert.h"
#include "common/common_types.h"
#include "common/fiber.h"
#include "common/logging/log.h"
#include "common/thread_queue_list.h"
#include "core/arm/arm_interface.h"
#ifdef ARCHITECTURE_x86_64
#include "core/arm/dynarmic/arm_dynarmic_32.h"
#include "core/arm/dynarmic/arm_dynarmic_64.h"
#endif
#include "core/arm/cpu_interrupt_handler.h"
#include "core/arm/exclusive_monitor.h"
#include "core/arm/unicorn/arm_unicorn.h"
#include "core/core.h"
#include "core/core_timing.h"
#include "core/core_timing_util.h"
#include "core/cpu_manager.h"
#include "core/hardware_properties.h"
#include "core/hle/kernel/errors.h"
#include "core/hle/kernel/handle_table.h"
@ -23,6 +32,7 @@
#include "core/hle/kernel/process.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/thread.h"
#include "core/hle/kernel/time_manager.h"
#include "core/hle/result.h"
#include "core/memory.h"
@ -44,46 +54,26 @@ Thread::Thread(KernelCore& kernel) : SynchronizationObject{kernel} {}
Thread::~Thread() = default;
void Thread::Stop() {
// Cancel any outstanding wakeup events for this thread
Core::System::GetInstance().CoreTiming().UnscheduleEvent(kernel.ThreadWakeupCallbackEventType(),
global_handle);
kernel.GlobalHandleTable().Close(global_handle);
global_handle = 0;
SetStatus(ThreadStatus::Dead);
Signal();
{
SchedulerLock lock(kernel);
SetStatus(ThreadStatus::Dead);
Signal();
kernel.GlobalHandleTable().Close(global_handle);
// Clean up any dangling references in objects that this thread was waiting for
for (auto& wait_object : wait_objects) {
wait_object->RemoveWaitingThread(SharedFrom(this));
if (owner_process) {
owner_process->UnregisterThread(this);
// Mark the TLS slot in the thread's page as free.
owner_process->FreeTLSRegion(tls_address);
}
arm_interface.reset();
has_exited = true;
}
wait_objects.clear();
owner_process->UnregisterThread(this);
// Mark the TLS slot in the thread's page as free.
owner_process->FreeTLSRegion(tls_address);
}
void Thread::WakeAfterDelay(s64 nanoseconds) {
// Don't schedule a wakeup if the thread wants to wait forever
if (nanoseconds == -1)
return;
// This function might be called from any thread so we have to be cautious and use the
// thread-safe version of ScheduleEvent.
const s64 cycles = Core::Timing::nsToCycles(std::chrono::nanoseconds{nanoseconds});
Core::System::GetInstance().CoreTiming().ScheduleEvent(
cycles, kernel.ThreadWakeupCallbackEventType(), global_handle);
}
void Thread::CancelWakeupTimer() {
Core::System::GetInstance().CoreTiming().UnscheduleEvent(kernel.ThreadWakeupCallbackEventType(),
global_handle);
global_handle = 0;
}
void Thread::ResumeFromWait() {
ASSERT_MSG(wait_objects.empty(), "Thread is waking up while waiting for objects");
SchedulerLock lock(kernel);
switch (status) {
case ThreadStatus::Paused:
case ThreadStatus::WaitSynch:
@ -99,7 +89,7 @@ void Thread::ResumeFromWait() {
case ThreadStatus::Ready:
// The thread's wakeup callback must have already been cleared when the thread was first
// awoken.
ASSERT(wakeup_callback == nullptr);
ASSERT(hle_callback == nullptr);
// If the thread is waiting on multiple wait objects, it might be awoken more than once
// before actually resuming. We can ignore subsequent wakeups if the thread status has
// already been set to ThreadStatus::Ready.
@ -115,24 +105,31 @@ void Thread::ResumeFromWait() {
return;
}
wakeup_callback = nullptr;
SetStatus(ThreadStatus::Ready);
}
if (activity == ThreadActivity::Paused) {
SetStatus(ThreadStatus::Paused);
return;
}
void Thread::OnWakeUp() {
SchedulerLock lock(kernel);
SetStatus(ThreadStatus::Ready);
}
ResultCode Thread::Start() {
SchedulerLock lock(kernel);
SetStatus(ThreadStatus::Ready);
return RESULT_SUCCESS;
}
void Thread::CancelWait() {
if (GetSchedulingStatus() != ThreadSchedStatus::Paused) {
SchedulerLock lock(kernel);
if (GetSchedulingStatus() != ThreadSchedStatus::Paused || !is_waiting_on_sync) {
is_sync_cancelled = true;
return;
}
// TODO(Blinkhawk): Implement cancel of server session
is_sync_cancelled = false;
SetWaitSynchronizationResult(ERR_SYNCHRONIZATION_CANCELED);
ResumeFromWait();
SetSynchronizationResults(nullptr, ERR_SYNCHRONIZATION_CANCELED);
SetStatus(ThreadStatus::Ready);
}
static void ResetThreadContext32(Core::ARM_Interface::ThreadContext32& context, u32 stack_top,
@ -153,12 +150,29 @@ static void ResetThreadContext64(Core::ARM_Interface::ThreadContext64& context,
context.fpcr = 0;
}
ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::string name,
VAddr entry_point, u32 priority, u64 arg,
s32 processor_id, VAddr stack_top,
Process& owner_process) {
std::shared_ptr<Common::Fiber>& Thread::GetHostContext() {
return host_context;
}
ResultVal<std::shared_ptr<Thread>> Thread::Create(Core::System& system, ThreadType type_flags,
std::string name, VAddr entry_point, u32 priority,
u64 arg, s32 processor_id, VAddr stack_top,
Process* owner_process) {
std::function<void(void*)> init_func = system.GetCpuManager().GetGuestThreadStartFunc();
void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater();
return Create(system, type_flags, name, entry_point, priority, arg, processor_id, stack_top,
owner_process, std::move(init_func), init_func_parameter);
}
ResultVal<std::shared_ptr<Thread>> Thread::Create(Core::System& system, ThreadType type_flags,
std::string name, VAddr entry_point, u32 priority,
u64 arg, s32 processor_id, VAddr stack_top,
Process* owner_process,
std::function<void(void*)>&& thread_start_func,
void* thread_start_parameter) {
auto& kernel = system.Kernel();
// Check if priority is in ranged. Lowest priority -> highest priority id.
if (priority > THREADPRIO_LOWEST) {
if (priority > THREADPRIO_LOWEST && ((type_flags & THREADTYPE_IDLE) == 0)) {
LOG_ERROR(Kernel_SVC, "Invalid thread priority: {}", priority);
return ERR_INVALID_THREAD_PRIORITY;
}
@ -168,11 +182,12 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin
return ERR_INVALID_PROCESSOR_ID;
}
auto& system = Core::System::GetInstance();
if (!system.Memory().IsValidVirtualAddress(owner_process, entry_point)) {
LOG_ERROR(Kernel_SVC, "(name={}): invalid entry {:016X}", name, entry_point);
// TODO (bunnei): Find the correct error code to use here
return RESULT_UNKNOWN;
if (owner_process) {
if (!system.Memory().IsValidVirtualAddress(*owner_process, entry_point)) {
LOG_ERROR(Kernel_SVC, "(name={}): invalid entry {:016X}", name, entry_point);
// TODO (bunnei): Find the correct error code to use here
return RESULT_UNKNOWN;
}
}
std::shared_ptr<Thread> thread = std::make_shared<Thread>(kernel);
@ -183,51 +198,82 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin
thread->stack_top = stack_top;
thread->tpidr_el0 = 0;
thread->nominal_priority = thread->current_priority = priority;
thread->last_running_ticks = system.CoreTiming().GetTicks();
thread->last_running_ticks = 0;
thread->processor_id = processor_id;
thread->ideal_core = processor_id;
thread->affinity_mask = 1ULL << processor_id;
thread->wait_objects.clear();
thread->wait_objects = nullptr;
thread->mutex_wait_address = 0;
thread->condvar_wait_address = 0;
thread->wait_handle = 0;
thread->name = std::move(name);
thread->global_handle = kernel.GlobalHandleTable().Create(thread).Unwrap();
thread->owner_process = &owner_process;
auto& scheduler = kernel.GlobalScheduler();
scheduler.AddThread(thread);
thread->tls_address = thread->owner_process->CreateTLSRegion();
thread->owner_process = owner_process;
thread->type = type_flags;
if ((type_flags & THREADTYPE_IDLE) == 0) {
auto& scheduler = kernel.GlobalScheduler();
scheduler.AddThread(thread);
}
if (owner_process) {
thread->tls_address = thread->owner_process->CreateTLSRegion();
thread->owner_process->RegisterThread(thread.get());
} else {
thread->tls_address = 0;
}
// TODO(peachum): move to ScheduleThread() when scheduler is added so selected core is used
// to initialize the context
thread->arm_interface.reset();
if ((type_flags & THREADTYPE_HLE) == 0) {
#ifdef ARCHITECTURE_x86_64
if (owner_process && !owner_process->Is64BitProcess()) {
thread->arm_interface = std::make_unique<Core::ARM_Dynarmic_32>(
system, kernel.Interrupts(), kernel.IsMulticore(), kernel.GetExclusiveMonitor(),
processor_id);
} else {
thread->arm_interface = std::make_unique<Core::ARM_Dynarmic_64>(
system, kernel.Interrupts(), kernel.IsMulticore(), kernel.GetExclusiveMonitor(),
processor_id);
}
thread->owner_process->RegisterThread(thread.get());
ResetThreadContext32(thread->context_32, static_cast<u32>(stack_top),
static_cast<u32>(entry_point), static_cast<u32>(arg));
ResetThreadContext64(thread->context_64, stack_top, entry_point, arg);
#else
if (owner_process && !owner_process->Is64BitProcess()) {
thread->arm_interface = std::make_shared<Core::ARM_Unicorn>(
system, kernel.Interrupts(), kernel.IsMulticore(), ARM_Unicorn::Arch::AArch32,
processor_id);
} else {
thread->arm_interface = std::make_shared<Core::ARM_Unicorn>(
system, kernel.Interrupts(), kernel.IsMulticore(), ARM_Unicorn::Arch::AArch64,
processor_id);
}
LOG_WARNING(Core, "CPU JIT requested, but Dynarmic not available");
#endif
ResetThreadContext32(thread->context_32, static_cast<u32>(stack_top),
static_cast<u32>(entry_point), static_cast<u32>(arg));
ResetThreadContext64(thread->context_64, stack_top, entry_point, arg);
}
thread->host_context =
std::make_shared<Common::Fiber>(std::move(thread_start_func), thread_start_parameter);
return MakeResult<std::shared_ptr<Thread>>(std::move(thread));
}
void Thread::SetPriority(u32 priority) {
SchedulerLock lock(kernel);
ASSERT_MSG(priority <= THREADPRIO_LOWEST && priority >= THREADPRIO_HIGHEST,
"Invalid priority value.");
nominal_priority = priority;
UpdatePriority();
}
void Thread::SetWaitSynchronizationResult(ResultCode result) {
context_32.cpu_registers[0] = result.raw;
context_64.cpu_registers[0] = result.raw;
}
void Thread::SetWaitSynchronizationOutput(s32 output) {
context_32.cpu_registers[1] = output;
context_64.cpu_registers[1] = output;
void Thread::SetSynchronizationResults(SynchronizationObject* object, ResultCode result) {
signaling_object = object;
signaling_result = result;
}
s32 Thread::GetSynchronizationObjectIndex(std::shared_ptr<SynchronizationObject> object) const {
ASSERT_MSG(!wait_objects.empty(), "Thread is not waiting for anything");
const auto match = std::find(wait_objects.rbegin(), wait_objects.rend(), object);
return static_cast<s32>(std::distance(match, wait_objects.rend()) - 1);
ASSERT_MSG(!wait_objects->empty(), "Thread is not waiting for anything");
const auto match = std::find(wait_objects->rbegin(), wait_objects->rend(), object);
return static_cast<s32>(std::distance(match, wait_objects->rend()) - 1);
}
VAddr Thread::GetCommandBufferAddress() const {
@ -236,6 +282,14 @@ VAddr Thread::GetCommandBufferAddress() const {
return GetTLSAddress() + command_header_offset;
}
Core::ARM_Interface& Thread::ArmInterface() {
return *arm_interface;
}
const Core::ARM_Interface& Thread::ArmInterface() const {
return *arm_interface;
}
void Thread::SetStatus(ThreadStatus new_status) {
if (new_status == status) {
return;
@ -257,10 +311,6 @@ void Thread::SetStatus(ThreadStatus new_status) {
break;
}
if (status == ThreadStatus::Running) {
last_running_ticks = Core::System::GetInstance().CoreTiming().GetTicks();
}
status = new_status;
}
@ -341,75 +391,116 @@ void Thread::UpdatePriority() {
lock_owner->UpdatePriority();
}
void Thread::ChangeCore(u32 core, u64 mask) {
SetCoreAndAffinityMask(core, mask);
}
bool Thread::AllSynchronizationObjectsReady() const {
return std::none_of(wait_objects.begin(), wait_objects.end(),
return std::none_of(wait_objects->begin(), wait_objects->end(),
[this](const std::shared_ptr<SynchronizationObject>& object) {
return object->ShouldWait(this);
});
}
bool Thread::InvokeWakeupCallback(ThreadWakeupReason reason, std::shared_ptr<Thread> thread,
std::shared_ptr<SynchronizationObject> object,
std::size_t index) {
ASSERT(wakeup_callback);
return wakeup_callback(reason, std::move(thread), std::move(object), index);
bool Thread::InvokeHLECallback(std::shared_ptr<Thread> thread) {
ASSERT(hle_callback);
return hle_callback(std::move(thread));
}
void Thread::SetActivity(ThreadActivity value) {
activity = value;
ResultCode Thread::SetActivity(ThreadActivity value) {
SchedulerLock lock(kernel);
auto sched_status = GetSchedulingStatus();
if (sched_status != ThreadSchedStatus::Runnable && sched_status != ThreadSchedStatus::Paused) {
return ERR_INVALID_STATE;
}
if (IsPendingTermination()) {
return RESULT_SUCCESS;
}
if (value == ThreadActivity::Paused) {
// Set status if not waiting
if (status == ThreadStatus::Ready || status == ThreadStatus::Running) {
SetStatus(ThreadStatus::Paused);
kernel.PrepareReschedule(processor_id);
if ((pausing_state & static_cast<u32>(ThreadSchedFlags::ThreadPauseFlag)) != 0) {
return ERR_INVALID_STATE;
}
} else if (status == ThreadStatus::Paused) {
// Ready to reschedule
ResumeFromWait();
AddSchedulingFlag(ThreadSchedFlags::ThreadPauseFlag);
} else {
if ((pausing_state & static_cast<u32>(ThreadSchedFlags::ThreadPauseFlag)) == 0) {
return ERR_INVALID_STATE;
}
RemoveSchedulingFlag(ThreadSchedFlags::ThreadPauseFlag);
}
return RESULT_SUCCESS;
}
void Thread::Sleep(s64 nanoseconds) {
// Sleep current thread and check for next thread to schedule
SetStatus(ThreadStatus::WaitSleep);
ResultCode Thread::Sleep(s64 nanoseconds) {
Handle event_handle{};
{
SchedulerLockAndSleep lock(kernel, event_handle, this, nanoseconds);
SetStatus(ThreadStatus::WaitSleep);
}
// Create an event to wake the thread up after the specified nanosecond delay has passed
WakeAfterDelay(nanoseconds);
if (event_handle != InvalidHandle) {
auto& time_manager = kernel.TimeManager();
time_manager.UnscheduleTimeEvent(event_handle);
}
return RESULT_SUCCESS;
}
bool Thread::YieldSimple() {
auto& scheduler = kernel.GlobalScheduler();
return scheduler.YieldThread(this);
std::pair<ResultCode, bool> Thread::YieldSimple() {
bool is_redundant = false;
{
SchedulerLock lock(kernel);
is_redundant = kernel.GlobalScheduler().YieldThread(this);
}
return {RESULT_SUCCESS, is_redundant};
}
bool Thread::YieldAndBalanceLoad() {
auto& scheduler = kernel.GlobalScheduler();
return scheduler.YieldThreadAndBalanceLoad(this);
std::pair<ResultCode, bool> Thread::YieldAndBalanceLoad() {
bool is_redundant = false;
{
SchedulerLock lock(kernel);
is_redundant = kernel.GlobalScheduler().YieldThreadAndBalanceLoad(this);
}
return {RESULT_SUCCESS, is_redundant};
}
bool Thread::YieldAndWaitForLoadBalancing() {
auto& scheduler = kernel.GlobalScheduler();
return scheduler.YieldThreadAndWaitForLoadBalancing(this);
std::pair<ResultCode, bool> Thread::YieldAndWaitForLoadBalancing() {
bool is_redundant = false;
{
SchedulerLock lock(kernel);
is_redundant = kernel.GlobalScheduler().YieldThreadAndWaitForLoadBalancing(this);
}
return {RESULT_SUCCESS, is_redundant};
}
void Thread::AddSchedulingFlag(ThreadSchedFlags flag) {
const u32 old_state = scheduling_state;
pausing_state |= static_cast<u32>(flag);
const u32 base_scheduling = static_cast<u32>(GetSchedulingStatus());
scheduling_state = base_scheduling | pausing_state;
kernel.GlobalScheduler().AdjustSchedulingOnStatus(this, old_state);
}
void Thread::RemoveSchedulingFlag(ThreadSchedFlags flag) {
const u32 old_state = scheduling_state;
pausing_state &= ~static_cast<u32>(flag);
const u32 base_scheduling = static_cast<u32>(GetSchedulingStatus());
scheduling_state = base_scheduling | pausing_state;
kernel.GlobalScheduler().AdjustSchedulingOnStatus(this, old_state);
}
void Thread::SetSchedulingStatus(ThreadSchedStatus new_status) {
const u32 old_flags = scheduling_state;
const u32 old_state = scheduling_state;
scheduling_state = (scheduling_state & static_cast<u32>(ThreadSchedMasks::HighMask)) |
static_cast<u32>(new_status);
AdjustSchedulingOnStatus(old_flags);
kernel.GlobalScheduler().AdjustSchedulingOnStatus(this, old_state);
}
void Thread::SetCurrentPriority(u32 new_priority) {
const u32 old_priority = std::exchange(current_priority, new_priority);
AdjustSchedulingOnPriority(old_priority);
kernel.GlobalScheduler().AdjustSchedulingOnPriority(this, old_priority);
}
ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) {
SchedulerLock lock(kernel);
const auto HighestSetCore = [](u64 mask, u32 max_cores) {
for (s32 core = static_cast<s32>(max_cores - 1); core >= 0; core--) {
if (((mask >> core) & 1) != 0) {
@ -443,111 +534,12 @@ ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) {
processor_id = ideal_core;
}
}
AdjustSchedulingOnAffinity(old_affinity_mask, old_core);
kernel.GlobalScheduler().AdjustSchedulingOnAffinity(this, old_affinity_mask, old_core);
}
}
return RESULT_SUCCESS;
}
void Thread::AdjustSchedulingOnStatus(u32 old_flags) {
if (old_flags == scheduling_state) {
return;
}
auto& scheduler = kernel.GlobalScheduler();
if (static_cast<ThreadSchedStatus>(old_flags & static_cast<u32>(ThreadSchedMasks::LowMask)) ==
ThreadSchedStatus::Runnable) {
// In this case the thread was running, now it's pausing/exitting
if (processor_id >= 0) {
scheduler.Unschedule(current_priority, static_cast<u32>(processor_id), this);
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
scheduler.Unsuggest(current_priority, core, this);
}
}
} else if (GetSchedulingStatus() == ThreadSchedStatus::Runnable) {
// The thread is now set to running from being stopped
if (processor_id >= 0) {
scheduler.Schedule(current_priority, static_cast<u32>(processor_id), this);
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
scheduler.Suggest(current_priority, core, this);
}
}
}
scheduler.SetReselectionPending();
}
void Thread::AdjustSchedulingOnPriority(u32 old_priority) {
if (GetSchedulingStatus() != ThreadSchedStatus::Runnable) {
return;
}
auto& scheduler = kernel.GlobalScheduler();
if (processor_id >= 0) {
scheduler.Unschedule(old_priority, static_cast<u32>(processor_id), this);
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
scheduler.Unsuggest(old_priority, core, this);
}
}
// Add thread to the new priority queues.
Thread* current_thread = GetCurrentThread();
if (processor_id >= 0) {
if (current_thread == this) {
scheduler.SchedulePrepend(current_priority, static_cast<u32>(processor_id), this);
} else {
scheduler.Schedule(current_priority, static_cast<u32>(processor_id), this);
}
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) {
scheduler.Suggest(current_priority, core, this);
}
}
scheduler.SetReselectionPending();
}
void Thread::AdjustSchedulingOnAffinity(u64 old_affinity_mask, s32 old_core) {
auto& scheduler = kernel.GlobalScheduler();
if (GetSchedulingStatus() != ThreadSchedStatus::Runnable ||
current_priority >= THREADPRIO_COUNT) {
return;
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (((old_affinity_mask >> core) & 1) != 0) {
if (core == static_cast<u32>(old_core)) {
scheduler.Unschedule(current_priority, core, this);
} else {
scheduler.Unsuggest(current_priority, core, this);
}
}
}
for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) {
if (((affinity_mask >> core) & 1) != 0) {
if (core == static_cast<u32>(processor_id)) {
scheduler.Schedule(current_priority, core, this);
} else {
scheduler.Suggest(current_priority, core, this);
}
}
}
scheduler.SetReselectionPending();
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/**

283
src/core/hle/kernel/thread.h
View file

@ -6,26 +6,47 @@
#include <functional>
#include <string>
#include <utility>
#include <vector>
#include "common/common_types.h"
#include "common/spin_lock.h"
#include "core/arm/arm_interface.h"
#include "core/hle/kernel/object.h"
#include "core/hle/kernel/synchronization_object.h"
#include "core/hle/result.h"
namespace Common {
class Fiber;
}
namespace Core {
class ARM_Interface;
class System;
} // namespace Core
namespace Kernel {
class GlobalScheduler;
class KernelCore;
class Process;
class Scheduler;
enum ThreadPriority : u32 {
THREADPRIO_HIGHEST = 0, ///< Highest thread priority
THREADPRIO_USERLAND_MAX = 24, ///< Highest thread priority for userland apps
THREADPRIO_DEFAULT = 44, ///< Default thread priority for userland apps
THREADPRIO_LOWEST = 63, ///< Lowest thread priority
THREADPRIO_COUNT = 64, ///< Total number of possible thread priorities.
THREADPRIO_HIGHEST = 0, ///< Highest thread priority
THREADPRIO_MAX_CORE_MIGRATION = 2, ///< Highest priority for a core migration
THREADPRIO_USERLAND_MAX = 24, ///< Highest thread priority for userland apps
THREADPRIO_DEFAULT = 44, ///< Default thread priority for userland apps
THREADPRIO_LOWEST = 63, ///< Lowest thread priority
THREADPRIO_COUNT = 64, ///< Total number of possible thread priorities.
};
enum ThreadType : u32 {
THREADTYPE_USER = 0x1,
THREADTYPE_KERNEL = 0x2,
THREADTYPE_HLE = 0x4,
THREADTYPE_IDLE = 0x8,
THREADTYPE_SUSPEND = 0x10,
};
enum ThreadProcessorId : s32 {
@ -107,26 +128,45 @@ public:
using ThreadSynchronizationObjects = std::vector<std::shared_ptr<SynchronizationObject>>;
using WakeupCallback =
std::function<bool(ThreadWakeupReason reason, std::shared_ptr<Thread> thread,
std::shared_ptr<SynchronizationObject> object, std::size_t index)>;
using HLECallback = std::function<bool(std::shared_ptr<Thread> thread)>;
/**
* Creates and returns a new thread. The new thread is immediately scheduled
* @param kernel The kernel instance this thread will be created under.
* @param system The instance of the whole system
* @param name The friendly name desired for the thread
* @param entry_point The address at which the thread should start execution
* @param priority The thread's priority
* @param arg User data to pass to the thread
* @param processor_id The ID(s) of the processors on which the thread is desired to be run
* @param stack_top The address of the thread's stack top
* @param owner_process The parent process for the thread
* @param owner_process The parent process for the thread, if null, it's a kernel thread
* @return A shared pointer to the newly created thread
*/
static ResultVal<std::shared_ptr<Thread>> Create(KernelCore& kernel, std::string name,
VAddr entry_point, u32 priority, u64 arg,
s32 processor_id, VAddr stack_top,
Process& owner_process);
static ResultVal<std::shared_ptr<Thread>> Create(Core::System& system, ThreadType type_flags,
std::string name, VAddr entry_point,
u32 priority, u64 arg, s32 processor_id,
VAddr stack_top, Process* owner_process);
/**
* Creates and returns a new thread. The new thread is immediately scheduled
* @param system The instance of the whole system
* @param name The friendly name desired for the thread
* @param entry_point The address at which the thread should start execution
* @param priority The thread's priority
* @param arg User data to pass to the thread
* @param processor_id The ID(s) of the processors on which the thread is desired to be run
* @param stack_top The address of the thread's stack top
* @param owner_process The parent process for the thread, if null, it's a kernel thread
* @param thread_start_func The function where the host context will start.
* @param thread_start_parameter The parameter which will passed to host context on init
* @return A shared pointer to the newly created thread
*/
static ResultVal<std::shared_ptr<Thread>> Create(Core::System& system, ThreadType type_flags,
std::string name, VAddr entry_point,
u32 priority, u64 arg, s32 processor_id,
VAddr stack_top, Process* owner_process,
std::function<void(void*)>&& thread_start_func,
void* thread_start_parameter);
std::string GetName() const override {
return name;
@ -181,7 +221,7 @@ public:
void UpdatePriority();
/// Changes the core that the thread is running or scheduled to run on.
void ChangeCore(u32 core, u64 mask);
ResultCode SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask);
/**
* Gets the thread's thread ID
@ -194,6 +234,10 @@ public:
/// Resumes a thread from waiting
void ResumeFromWait();
void OnWakeUp();
ResultCode Start();
/// Cancels a waiting operation that this thread may or may not be within.
///
/// When the thread is within a waiting state, this will set the thread's
@ -202,26 +246,19 @@ public:
///
void CancelWait();
/**
* Schedules an event to wake up the specified thread after the specified delay
* @param nanoseconds The time this thread will be allowed to sleep for
*/
void WakeAfterDelay(s64 nanoseconds);
void SetSynchronizationResults(SynchronizationObject* object, ResultCode result);
/// Cancel any outstanding wakeup events for this thread
void CancelWakeupTimer();
Core::ARM_Interface& ArmInterface();
/**
* Sets the result after the thread awakens (from svcWaitSynchronization)
* @param result Value to set to the returned result
*/
void SetWaitSynchronizationResult(ResultCode result);
const Core::ARM_Interface& ArmInterface() const;
/**
* Sets the output parameter value after the thread awakens (from svcWaitSynchronization)
* @param output Value to set to the output parameter
*/
void SetWaitSynchronizationOutput(s32 output);
SynchronizationObject* GetSignalingObject() const {
return signaling_object;
}
ResultCode GetSignalingResult() const {
return signaling_result;
}
/**
* Retrieves the index that this particular object occupies in the list of objects
@ -269,11 +306,6 @@ public:
*/
VAddr GetCommandBufferAddress() const;
/// Returns whether this thread is waiting on objects from a WaitSynchronization call.
bool IsSleepingOnWait() const {
return status == ThreadStatus::WaitSynch;
}
ThreadContext32& GetContext32() {
return context_32;
}
@ -290,6 +322,28 @@ public:
return context_64;
}
bool IsHLEThread() const {
return (type & THREADTYPE_HLE) != 0;
}
bool IsSuspendThread() const {
return (type & THREADTYPE_SUSPEND) != 0;
}
bool IsIdleThread() const {
return (type & THREADTYPE_IDLE) != 0;
}
bool WasRunning() const {
return was_running;
}
void SetWasRunning(bool value) {
was_running = value;
}
std::shared_ptr<Common::Fiber>& GetHostContext();
ThreadStatus GetStatus() const {
return status;
}
@ -325,18 +379,18 @@ public:
}
const ThreadSynchronizationObjects& GetSynchronizationObjects() const {
return wait_objects;
return *wait_objects;
}
void SetSynchronizationObjects(ThreadSynchronizationObjects objects) {
wait_objects = std::move(objects);
void SetSynchronizationObjects(ThreadSynchronizationObjects* objects) {
wait_objects = objects;
}
void ClearSynchronizationObjects() {
for (const auto& waiting_object : wait_objects) {
for (const auto& waiting_object : *wait_objects) {
waiting_object->RemoveWaitingThread(SharedFrom(this));
}
wait_objects.clear();
wait_objects->clear();
}
/// Determines whether all the objects this thread is waiting on are ready.
@ -386,26 +440,35 @@ public:
arb_wait_address = address;
}
bool HasWakeupCallback() const {
return wakeup_callback != nullptr;
bool HasHLECallback() const {
return hle_callback != nullptr;
}
void SetWakeupCallback(WakeupCallback callback) {
wakeup_callback = std::move(callback);
void SetHLECallback(HLECallback callback) {
hle_callback = std::move(callback);
}
void InvalidateWakeupCallback() {
SetWakeupCallback(nullptr);
void SetHLETimeEvent(Handle time_event) {
hle_time_event = time_event;
}
/**
* Invokes the thread's wakeup callback.
*
* @pre A valid wakeup callback has been set. Violating this precondition
* will cause an assertion to trigger.
*/
bool InvokeWakeupCallback(ThreadWakeupReason reason, std::shared_ptr<Thread> thread,
std::shared_ptr<SynchronizationObject> object, std::size_t index);
void SetHLESyncObject(SynchronizationObject* object) {
hle_object = object;
}
Handle GetHLETimeEvent() const {
return hle_time_event;
}
SynchronizationObject* GetHLESyncObject() const {
return hle_object;
}
void InvalidateHLECallback() {
SetHLECallback(nullptr);
}
bool InvokeHLECallback(std::shared_ptr<Thread> thread);
u32 GetIdealCore() const {
return ideal_core;
@ -415,23 +478,19 @@ public:
return affinity_mask;
}
ThreadActivity GetActivity() const {
return activity;
}
void SetActivity(ThreadActivity value);
ResultCode SetActivity(ThreadActivity value);
/// Sleeps this thread for the given amount of nanoseconds.
void Sleep(s64 nanoseconds);
ResultCode Sleep(s64 nanoseconds);
/// Yields this thread without rebalancing loads.
bool YieldSimple();
std::pair<ResultCode, bool> YieldSimple();
/// Yields this thread and does a load rebalancing.
bool YieldAndBalanceLoad();
std::pair<ResultCode, bool> YieldAndBalanceLoad();
/// Yields this thread and if the core is left idle, loads are rebalanced
bool YieldAndWaitForLoadBalancing();
std::pair<ResultCode, bool> YieldAndWaitForLoadBalancing();
void IncrementYieldCount() {
yield_count++;
@ -446,6 +505,10 @@ public:
static_cast<u32>(ThreadSchedMasks::LowMask));
}
bool IsRunnable() const {
return scheduling_state == static_cast<u32>(ThreadSchedStatus::Runnable);
}
bool IsRunning() const {
return is_running;
}
@ -466,17 +529,67 @@ public:
return global_handle;
}
private:
void SetSchedulingStatus(ThreadSchedStatus new_status);
void SetCurrentPriority(u32 new_priority);
ResultCode SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask);
bool IsWaitingForArbitration() const {
return waiting_for_arbitration;
}
void WaitForArbitration(bool set) {
waiting_for_arbitration = set;
}
bool IsWaitingSync() const {
return is_waiting_on_sync;
}
void SetWaitingSync(bool is_waiting) {
is_waiting_on_sync = is_waiting;
}
bool IsPendingTermination() const {
return will_be_terminated || GetSchedulingStatus() == ThreadSchedStatus::Exited;
}
bool IsPaused() const {
return pausing_state != 0;
}
bool IsContinuousOnSVC() const {
return is_continuous_on_svc;
}
void SetContinuousOnSVC(bool is_continuous) {
is_continuous_on_svc = is_continuous;
}
bool IsPhantomMode() const {
return is_phantom_mode;
}
void SetPhantomMode(bool phantom) {
is_phantom_mode = phantom;
}
bool HasExited() const {
return has_exited;
}
private:
friend class GlobalScheduler;
friend class Scheduler;
void SetSchedulingStatus(ThreadSchedStatus new_status);
void AddSchedulingFlag(ThreadSchedFlags flag);
void RemoveSchedulingFlag(ThreadSchedFlags flag);
void SetCurrentPriority(u32 new_priority);
void AdjustSchedulingOnStatus(u32 old_flags);
void AdjustSchedulingOnPriority(u32 old_priority);
void AdjustSchedulingOnAffinity(u64 old_affinity_mask, s32 old_core);
Common::SpinLock context_guard{};
ThreadContext32 context_32{};
ThreadContext64 context_64{};
std::unique_ptr<Core::ARM_Interface> arm_interface{};
std::shared_ptr<Common::Fiber> host_context{};
u64 thread_id = 0;
@ -485,6 +598,8 @@ private:
VAddr entry_point = 0;
VAddr stack_top = 0;
ThreadType type;
/// Nominal thread priority, as set by the emulated application.
/// The nominal priority is the thread priority without priority
/// inheritance taken into account.
@ -509,7 +624,10 @@ private:
/// Objects that the thread is waiting on, in the same order as they were
/// passed to WaitSynchronization.
ThreadSynchronizationObjects wait_objects;
ThreadSynchronizationObjects* wait_objects;
SynchronizationObject* signaling_object;
ResultCode signaling_result{RESULT_SUCCESS};
/// List of threads that are waiting for a mutex that is held by this thread.
MutexWaitingThreads wait_mutex_threads;
@ -526,30 +644,39 @@ private:
/// If waiting for an AddressArbiter, this is the address being waited on.
VAddr arb_wait_address{0};
bool waiting_for_arbitration{};
/// Handle used as userdata to reference this object when inserting into the CoreTiming queue.
Handle global_handle = 0;
/// Callback that will be invoked when the thread is resumed from a waiting state. If the thread
/// was waiting via WaitSynchronization then the object will be the last object that became
/// available. In case of a timeout, the object will be nullptr.
WakeupCallback wakeup_callback;
/// Callback for HLE Events
HLECallback hle_callback;
Handle hle_time_event;
SynchronizationObject* hle_object;
Scheduler* scheduler = nullptr;
u32 ideal_core{0xFFFFFFFF};
u64 affinity_mask{0x1};
ThreadActivity activity = ThreadActivity::Normal;
s32 ideal_core_override = -1;
u64 affinity_mask_override = 0x1;
u32 affinity_override_count = 0;
u32 scheduling_state = 0;
u32 pausing_state = 0;
bool is_running = false;
bool is_waiting_on_sync = false;
bool is_sync_cancelled = false;
bool is_continuous_on_svc = false;
bool will_be_terminated = false;
bool is_phantom_mode = false;
bool has_exited = false;
bool was_running = false;
std::string name;
};

23
src/core/hle/kernel/time_manager.cpp
View file

@ -8,30 +8,37 @@
#include "core/core_timing_util.h"
#include "core/hle/kernel/handle_table.h"
#include "core/hle/kernel/kernel.h"
#include "core/hle/kernel/scheduler.h"
#include "core/hle/kernel/thread.h"
#include "core/hle/kernel/time_manager.h"
namespace Kernel {
TimeManager::TimeManager(Core::System& system) : system{system} {
TimeManager::TimeManager(Core::System& system_) : system{system_} {
time_manager_event_type = Core::Timing::CreateEvent(
"Kernel::TimeManagerCallback", [this](u64 thread_handle, [[maybe_unused]] s64 cycles_late) {
SchedulerLock lock(system.Kernel());
Handle proper_handle = static_cast<Handle>(thread_handle);
if (cancelled_events[proper_handle]) {
return;
}
std::shared_ptr<Thread> thread =
this->system.Kernel().RetrieveThreadFromGlobalHandleTable(proper_handle);
thread->ResumeFromWait();
thread->OnWakeUp();
});
}
void TimeManager::ScheduleTimeEvent(Handle& event_handle, Thread* timetask, s64 nanoseconds) {
event_handle = timetask->GetGlobalHandle();
if (nanoseconds > 0) {
ASSERT(timetask);
event_handle = timetask->GetGlobalHandle();
const s64 cycles = Core::Timing::nsToCycles(std::chrono::nanoseconds{nanoseconds});
system.CoreTiming().ScheduleEvent(cycles, time_manager_event_type, event_handle);
ASSERT(timetask->GetStatus() != ThreadStatus::Ready);
ASSERT(timetask->GetStatus() != ThreadStatus::WaitMutex);
system.CoreTiming().ScheduleEvent(nanoseconds, time_manager_event_type, event_handle);
} else {
event_handle = InvalidHandle;
}
cancelled_events[event_handle] = false;
}
void TimeManager::UnscheduleTimeEvent(Handle event_handle) {
@ -39,6 +46,12 @@ void TimeManager::UnscheduleTimeEvent(Handle event_handle) {
return;
}
system.CoreTiming().UnscheduleEvent(time_manager_event_type, event_handle);
cancelled_events[event_handle] = true;
}
void TimeManager::CancelTimeEvent(Thread* time_task) {
Handle event_handle = time_task->GetGlobalHandle();
UnscheduleTimeEvent(event_handle);
}
} // namespace Kernel

4
src/core/hle/kernel/time_manager.h
View file

@ -5,6 +5,7 @@
#pragma once
#include <memory>
#include <unordered_map>
#include "core/hle/kernel/object.h"
@ -35,9 +36,12 @@ public:
/// Unschedule an existing time event
void UnscheduleTimeEvent(Handle event_handle);
void CancelTimeEvent(Thread* time_task);
private:
Core::System& system;
std::shared_ptr<Core::Timing::EventType> time_manager_event_type;
std::unordered_map<Handle, bool> cancelled_events;
};
} // namespace Kernel

2
src/core/hle/service/hid/controllers/debug_pad.cpp
View file

@ -23,7 +23,7 @@ void Controller_DebugPad::OnRelease() {}
void Controller_DebugPad::OnUpdate(const Core::Timing::CoreTiming& core_timing, u8* data,
std::size_t size) {
shared_memory.header.timestamp = core_timing.GetTicks();
shared_memory.header.timestamp = core_timing.GetCPUTicks();
shared_memory.header.total_entry_count = 17;
if (!IsControllerActivated()) {

2
src/core/hle/service/hid/controllers/gesture.cpp
View file

@ -19,7 +19,7 @@ void Controller_Gesture::OnRelease() {}
void Controller_Gesture::OnUpdate(const Core::Timing::CoreTiming& core_timing, u8* data,
std::size_t size) {
shared_memory.header.timestamp = core_timing.GetTicks();
shared_memory.header.timestamp = core_timing.GetCPUTicks();
shared_memory.header.total_entry_count = 17;
if (!IsControllerActivated()) {

2
src/core/hle/service/hid/controllers/keyboard.cpp
View file

@ -21,7 +21,7 @@ void Controller_Keyboard::OnRelease() {}
void Controller_Keyboard::OnUpdate(const Core::Timing::CoreTiming& core_timing, u8* data,
std::size_t size) {
shared_memory.header.timestamp = core_timing.GetTicks();
shared_memory.header.timestamp = core_timing.GetCPUTicks();
shared_memory.header.total_entry_count = 17;
if (!IsControllerActivated()) {

2
src/core/hle/service/hid/controllers/mouse.cpp
View file

@ -19,7 +19,7 @@ void Controller_Mouse::OnRelease() {}
void Controller_Mouse::OnUpdate(const Core::Timing::CoreTiming& core_timing, u8* data,
std::size_t size) {
shared_memory.header.timestamp = core_timing.GetTicks();
shared_memory.header.timestamp = core_timing.GetCPUTicks();
shared_memory.header.total_entry_count = 17;
if (!IsControllerActivated()) {

2
src/core/hle/service/hid/controllers/npad.cpp
View file

@ -328,7 +328,7 @@ void Controller_NPad::OnUpdate(const Core::Timing::CoreTiming& core_timing, u8*
const auto& last_entry =
main_controller->npad[main_controller->common.last_entry_index];
main_controller->common.timestamp = core_timing.GetTicks();
main_controller->common.timestamp = core_timing.GetCPUTicks();
main_controller->common.last_entry_index =
(main_controller->common.last_entry_index + 1) % 17;

2
src/core/hle/service/hid/controllers/stubbed.cpp
View file

@ -23,7 +23,7 @@ void Controller_Stubbed::OnUpdate(const Core::Timing::CoreTiming& core_timing, u
}
CommonHeader header{};
header.timestamp = core_timing.GetTicks();
header.timestamp = core_timing.GetCPUTicks();
header.total_entry_count = 17;
header.entry_count = 0;
header.last_entry_index = 0;

4
src/core/hle/service/hid/controllers/touchscreen.cpp
View file

@ -22,7 +22,7 @@ void Controller_Touchscreen::OnRelease() {}
void Controller_Touchscreen::OnUpdate(const Core::Timing::CoreTiming& core_timing, u8* data,
std::size_t size) {
shared_memory.header.timestamp = core_timing.GetTicks();
shared_memory.header.timestamp = core_timing.GetCPUTicks();
shared_memory.header.total_entry_count = 17;
if (!IsControllerActivated()) {
@ -49,7 +49,7 @@ void Controller_Touchscreen::OnUpdate(const Core::Timing::CoreTiming& core_timin
touch_entry.diameter_x = Settings::values.touchscreen.diameter_x;
touch_entry.diameter_y = Settings::values.touchscreen.diameter_y;
touch_entry.rotation_angle = Settings::values.touchscreen.rotation_angle;
const u64 tick = core_timing.GetTicks();
const u64 tick = core_timing.GetCPUTicks();
touch_entry.delta_time = tick - last_touch;
last_touch = tick;
touch_entry.finger = Settings::values.touchscreen.finger;

2
src/core/hle/service/hid/controllers/xpad.cpp
View file

@ -20,7 +20,7 @@ void Controller_XPad::OnRelease() {}
void Controller_XPad::OnUpdate(const Core::Timing::CoreTiming& core_timing, u8* data,
std::size_t size) {
for (auto& xpad_entry : shared_memory.shared_memory_entries) {
xpad_entry.header.timestamp = core_timing.GetTicks();
xpad_entry.header.timestamp = core_timing.GetCPUTicks();
xpad_entry.header.total_entry_count = 17;
if (!IsControllerActivated()) {

16
src/core/hle/service/hid/hid.cpp
View file

@ -39,11 +39,9 @@ namespace Service::HID {
// Updating period for each HID device.
// TODO(ogniK): Find actual polling rate of hid
constexpr s64 pad_update_ticks = static_cast<s64>(Core::Hardware::BASE_CLOCK_RATE / 66);
[[maybe_unused]] constexpr s64 accelerometer_update_ticks =
static_cast<s64>(Core::Hardware::BASE_CLOCK_RATE / 100);
[[maybe_unused]] constexpr s64 gyroscope_update_ticks =
static_cast<s64>(Core::Hardware::BASE_CLOCK_RATE / 100);
constexpr s64 pad_update_ticks = static_cast<s64>(1000000000 / 66);
[[maybe_unused]] constexpr s64 accelerometer_update_ticks = static_cast<s64>(1000000000 / 100);
[[maybe_unused]] constexpr s64 gyroscope_update_ticks = static_cast<s64>(1000000000 / 100);
constexpr std::size_t SHARED_MEMORY_SIZE = 0x40000;
IAppletResource::IAppletResource(Core::System& system)
@ -78,8 +76,8 @@ IAppletResource::IAppletResource(Core::System& system)
// Register update callbacks
pad_update_event =
Core::Timing::CreateEvent("HID::UpdatePadCallback", [this](u64 userdata, s64 cycles_late) {
UpdateControllers(userdata, cycles_late);
Core::Timing::CreateEvent("HID::UpdatePadCallback", [this](u64 userdata, s64 ns_late) {
UpdateControllers(userdata, ns_late);
});
// TODO(shinyquagsire23): Other update callbacks? (accel, gyro?)
@ -109,7 +107,7 @@ void IAppletResource::GetSharedMemoryHandle(Kernel::HLERequestContext& ctx) {
rb.PushCopyObjects(shared_mem);
}
void IAppletResource::UpdateControllers(u64 userdata, s64 cycles_late) {
void IAppletResource::UpdateControllers(u64 userdata, s64 ns_late) {
auto& core_timing = system.CoreTiming();
const bool should_reload = Settings::values.is_device_reload_pending.exchange(false);
@ -120,7 +118,7 @@ void IAppletResource::UpdateControllers(u64 userdata, s64 cycles_late) {
controller->OnUpdate(core_timing, shared_mem->GetPointer(), SHARED_MEMORY_SIZE);
}
core_timing.ScheduleEvent(pad_update_ticks - cycles_late, pad_update_event);
core_timing.ScheduleEvent(pad_update_ticks - ns_late, pad_update_event);
}
class IActiveVibrationDeviceList final : public ServiceFramework<IActiveVibrationDeviceList> {

2
src/core/hle/service/hid/irs.cpp
View file

@ -98,7 +98,7 @@ void IRS::GetImageTransferProcessorState(Kernel::HLERequestContext& ctx) {
IPC::ResponseBuilder rb{ctx, 5};
rb.Push(RESULT_SUCCESS);
rb.PushRaw<u64>(system.CoreTiming().GetTicks());
rb.PushRaw<u64>(system.CoreTiming().GetCPUTicks());
rb.PushRaw<u32>(0);
}

3
src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp
View file

@ -200,8 +200,7 @@ u32 nvhost_ctrl_gpu::GetGpuTime(const std::vector<u8>& input, std::vector<u8>& o
IoctlGetGpuTime params{};
std::memcpy(&params, input.data(), input.size());
const auto ns = Core::Timing::CyclesToNs(system.CoreTiming().GetTicks());
params.gpu_time = static_cast<u64_le>(ns.count());
params.gpu_time = static_cast<u64_le>(system.CoreTiming().GetGlobalTimeNs().count());
std::memcpy(output.data(), &params, output.size());
return 0;
}

64
src/core/hle/service/nvflinger/nvflinger.cpp
View file

@ -9,6 +9,7 @@
#include "common/logging/log.h"
#include "common/microprofile.h"
#include "common/scope_exit.h"
#include "common/thread.h"
#include "core/core.h"
#include "core/core_timing.h"
#include "core/core_timing_util.h"
@ -27,8 +28,35 @@
namespace Service::NVFlinger {
constexpr s64 frame_ticks = static_cast<s64>(Core::Hardware::BASE_CLOCK_RATE / 60);
constexpr s64 frame_ticks_30fps = static_cast<s64>(Core::Hardware::BASE_CLOCK_RATE / 30);
constexpr s64 frame_ticks = static_cast<s64>(1000000000 / 60);
constexpr s64 frame_ticks_30fps = static_cast<s64>(1000000000 / 30);
void NVFlinger::VSyncThread(NVFlinger& nv_flinger) {
nv_flinger.SplitVSync();
}
void NVFlinger::SplitVSync() {
system.RegisterHostThread();
std::string name = "yuzu:VSyncThread";
MicroProfileOnThreadCreate(name.c_str());
Common::SetCurrentThreadName(name.c_str());
Common::SetCurrentThreadPriority(Common::ThreadPriority::High);
s64 delay = 0;
while (is_running) {
guard->lock();
const s64 time_start = system.CoreTiming().GetGlobalTimeNs().count();
Compose();
const auto ticks = GetNextTicks();
const s64 time_end = system.CoreTiming().GetGlobalTimeNs().count();
const s64 time_passed = time_end - time_start;
const s64 next_time = std::max<s64>(0, ticks - time_passed - delay);
guard->unlock();
if (next_time > 0) {
wait_event->WaitFor(std::chrono::nanoseconds{next_time});
}
delay = (system.CoreTiming().GetGlobalTimeNs().count() - time_end) - next_time;
}
}
NVFlinger::NVFlinger(Core::System& system) : system(system) {
displays.emplace_back(0, "Default", system);
@ -36,22 +64,36 @@ NVFlinger::NVFlinger(Core::System& system) : system(system) {
displays.emplace_back(2, "Edid", system);
displays.emplace_back(3, "Internal", system);
displays.emplace_back(4, "Null", system);
guard = std::make_shared<std::mutex>();
// Schedule the screen composition events
composition_event =
Core::Timing::CreateEvent("ScreenComposition", [this](u64 userdata, s64 cycles_late) {
Core::Timing::CreateEvent("ScreenComposition", [this](u64 userdata, s64 ns_late) {
Lock();
Compose();
const auto ticks =
Settings::values.force_30fps_mode ? frame_ticks_30fps : GetNextTicks();
this->system.CoreTiming().ScheduleEvent(std::max<s64>(0LL, ticks - cycles_late),
const auto ticks = GetNextTicks();
this->system.CoreTiming().ScheduleEvent(std::max<s64>(0LL, ticks - ns_late),
composition_event);
});
system.CoreTiming().ScheduleEvent(frame_ticks, composition_event);
if (system.IsMulticore()) {
is_running = true;
wait_event = std::make_unique<Common::Event>();
vsync_thread = std::make_unique<std::thread>(VSyncThread, std::ref(*this));
} else {
system.CoreTiming().ScheduleEvent(frame_ticks, composition_event);
}
}
NVFlinger::~NVFlinger() {
system.CoreTiming().UnscheduleEvent(composition_event, 0);
if (system.IsMulticore()) {
is_running = false;
wait_event->Set();
vsync_thread->join();
vsync_thread.reset();
wait_event.reset();
} else {
system.CoreTiming().UnscheduleEvent(composition_event, 0);
}
}
void NVFlinger::SetNVDrvInstance(std::shared_ptr<Nvidia::Module> instance) {
@ -199,10 +241,12 @@ void NVFlinger::Compose() {
auto& gpu = system.GPU();
const auto& multi_fence = buffer->get().multi_fence;
guard->unlock();
for (u32 fence_id = 0; fence_id < multi_fence.num_fences; fence_id++) {
const auto& fence = multi_fence.fences[fence_id];
gpu.WaitFence(fence.id, fence.value);
}
guard->lock();
MicroProfileFlip();
@ -223,7 +267,7 @@ void NVFlinger::Compose() {
s64 NVFlinger::GetNextTicks() const {
constexpr s64 max_hertz = 120LL;
return (Core::Hardware::BASE_CLOCK_RATE * (1LL << swap_interval)) / max_hertz;
return (1000000000 * (1LL << swap_interval)) / max_hertz;
}
} // namespace Service::NVFlinger

21
src/core/hle/service/nvflinger/nvflinger.h
View file

@ -4,15 +4,22 @@
#pragma once
#include <atomic>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <string_view>
#include <thread>
#include <vector>
#include "common/common_types.h"
#include "core/hle/kernel/object.h"
namespace Common {
class Event;
} // namespace Common
namespace Core::Timing {
class CoreTiming;
struct EventType;
@ -79,6 +86,10 @@ public:
s64 GetNextTicks() const;
std::unique_lock<std::mutex> Lock() {
return std::unique_lock{*guard};
}
private:
/// Finds the display identified by the specified ID.
VI::Display* FindDisplay(u64 display_id);
@ -92,6 +103,10 @@ private:
/// Finds the layer identified by the specified ID in the desired display.
const VI::Layer* FindLayer(u64 display_id, u64 layer_id) const;
static void VSyncThread(NVFlinger& nv_flinger);
void SplitVSync();
std::shared_ptr<Nvidia::Module> nvdrv;
std::vector<VI::Display> displays;
@ -108,7 +123,13 @@ private:
/// Event that handles screen composition.
std::shared_ptr<Core::Timing::EventType> composition_event;
std::shared_ptr<std::mutex> guard;
Core::System& system;
std::unique_ptr<std::thread> vsync_thread;
std::unique_ptr<Common::Event> wait_event;
std::atomic<bool> is_running{};
};
} // namespace Service::NVFlinger

2
src/core/hle/service/sm/sm.cpp
View file

@ -142,7 +142,7 @@ void SM::GetService(Kernel::HLERequestContext& ctx) {
}
// Wake the threads waiting on the ServerPort
server_port->WakeupAllWaitingThreads();
server_port->Signal();
LOG_DEBUG(Service_SM, "called service={} -> session={}", name, client->GetObjectId());
IPC::ResponseBuilder rb{ctx, 2, 0, 1, IPC::ResponseBuilder::Flags::AlwaysMoveHandles};

5
src/core/hle/service/time/standard_steady_clock_core.cpp
View file

@ -11,9 +11,8 @@
namespace Service::Time::Clock {
TimeSpanType StandardSteadyClockCore::GetCurrentRawTimePoint(Core::System& system) {
const TimeSpanType ticks_time_span{TimeSpanType::FromTicks(
Core::Timing::CpuCyclesToClockCycles(system.CoreTiming().GetTicks()),
Core::Hardware::CNTFREQ)};
const TimeSpanType ticks_time_span{
TimeSpanType::FromTicks(system.CoreTiming().GetClockTicks(), Core::Hardware::CNTFREQ)};
TimeSpanType raw_time_point{setup_value.nanoseconds + ticks_time_span.nanoseconds};
if (raw_time_point.nanoseconds < cached_raw_time_point.nanoseconds) {

5
src/core/hle/service/time/tick_based_steady_clock_core.cpp
View file

@ -11,9 +11,8 @@
namespace Service::Time::Clock {
SteadyClockTimePoint TickBasedSteadyClockCore::GetTimePoint(Core::System& system) {
const TimeSpanType ticks_time_span{TimeSpanType::FromTicks(
Core::Timing::CpuCyclesToClockCycles(system.CoreTiming().GetTicks()),
Core::Hardware::CNTFREQ)};
const TimeSpanType ticks_time_span{
TimeSpanType::FromTicks(system.CoreTiming().GetClockTicks(), Core::Hardware::CNTFREQ)};
return {ticks_time_span.ToSeconds(), GetClockSourceId()};
}

5
src/core/hle/service/time/time.cpp
View file

@ -234,9 +234,8 @@ void Module::Interface::CalculateMonotonicSystemClockBaseTimePoint(Kernel::HLERe
const auto current_time_point{steady_clock_core.GetCurrentTimePoint(system)};
if (current_time_point.clock_source_id == context.steady_time_point.clock_source_id) {
const auto ticks{Clock::TimeSpanType::FromTicks(
Core::Timing::CpuCyclesToClockCycles(system.CoreTiming().GetTicks()),
Core::Hardware::CNTFREQ)};
const auto ticks{Clock::TimeSpanType::FromTicks(system.CoreTiming().GetClockTicks(),
Core::Hardware::CNTFREQ)};
const s64 base_time_point{context.offset + current_time_point.time_point -
ticks.ToSeconds()};
IPC::ResponseBuilder rb{ctx, (sizeof(s64) / 4) + 2};

3
src/core/hle/service/time/time_sharedmemory.cpp
View file

@ -30,8 +30,7 @@ void SharedMemory::SetupStandardSteadyClock(Core::System& system,
const Common::UUID& clock_source_id,
Clock::TimeSpanType current_time_point) {
const Clock::TimeSpanType ticks_time_span{Clock::TimeSpanType::FromTicks(
Core::Timing::CpuCyclesToClockCycles(system.CoreTiming().GetTicks()),
Core::Hardware::CNTFREQ)};
system.CoreTiming().GetClockTicks(), Core::Hardware::CNTFREQ)};
const Clock::SteadyClockContext context{
static_cast<u64>(current_time_point.nanoseconds - ticks_time_span.nanoseconds),
clock_source_id};

2
src/core/hle/service/vi/vi.cpp
View file

@ -511,6 +511,7 @@ private:
LOG_DEBUG(Service_VI, "called. id=0x{:08X} transaction={:X}, flags=0x{:08X}", id,
static_cast<u32>(transaction), flags);
nv_flinger->Lock();
auto& buffer_queue = nv_flinger->FindBufferQueue(id);
switch (transaction) {
@ -550,6 +551,7 @@ private:
[=](std::shared_ptr<Kernel::Thread> thread, Kernel::HLERequestContext& ctx,
Kernel::ThreadWakeupReason reason) {
// Repeat TransactParcel DequeueBuffer when a buffer is available
nv_flinger->Lock();
auto& buffer_queue = nv_flinger->FindBufferQueue(id);
auto result = buffer_queue.DequeueBuffer(width, height);
ASSERT_MSG(result != std::nullopt, "Could not dequeue buffer.");