diff options
Diffstat (limited to 'src/core/hle/kernel')
62 files changed, 6297 insertions, 3984 deletions
diff --git a/src/core/hle/kernel/address_arbiter.cpp b/src/core/hle/kernel/address_arbiter.cpp index 8475b698c..b882eaa0f 100644 --- a/src/core/hle/kernel/address_arbiter.cpp +++ b/src/core/hle/kernel/address_arbiter.cpp @@ -7,11 +7,15 @@ #include "common/assert.h" #include "common/common_types.h" +#include "core/arm/exclusive_monitor.h" #include "core/core.h" #include "core/hle/kernel/address_arbiter.h" #include "core/hle/kernel/errors.h" +#include "core/hle/kernel/handle_table.h" +#include "core/hle/kernel/kernel.h" #include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/thread.h" +#include "core/hle/kernel/time_manager.h" #include "core/hle/result.h" #include "core/memory.h" @@ -29,12 +33,10 @@ void AddressArbiter::WakeThreads(const std::vector<std::shared_ptr<Thread>>& wai // Signal the waiting threads. for (std::size_t i = 0; i < last; i++) { - ASSERT(waiting_threads[i]->GetStatus() == ThreadStatus::WaitArb); - waiting_threads[i]->SetWaitSynchronizationResult(RESULT_SUCCESS); + waiting_threads[i]->SetSynchronizationResults(nullptr, RESULT_SUCCESS); RemoveThread(waiting_threads[i]); - waiting_threads[i]->SetArbiterWaitAddress(0); + waiting_threads[i]->WaitForArbitration(false); waiting_threads[i]->ResumeFromWait(); - system.PrepareReschedule(waiting_threads[i]->GetProcessorID()); } } @@ -56,6 +58,7 @@ ResultCode AddressArbiter::SignalToAddress(VAddr address, SignalType type, s32 v } ResultCode AddressArbiter::SignalToAddressOnly(VAddr address, s32 num_to_wake) { + SchedulerLock lock(system.Kernel()); const std::vector<std::shared_ptr<Thread>> waiting_threads = GetThreadsWaitingOnAddress(address); WakeThreads(waiting_threads, num_to_wake); @@ -64,6 +67,7 @@ ResultCode AddressArbiter::SignalToAddressOnly(VAddr address, s32 num_to_wake) { ResultCode AddressArbiter::IncrementAndSignalToAddressIfEqual(VAddr address, s32 value, s32 num_to_wake) { + SchedulerLock lock(system.Kernel()); auto& memory = system.Memory(); // Ensure that we can write to the address. @@ -71,16 +75,24 @@ ResultCode AddressArbiter::IncrementAndSignalToAddressIfEqual(VAddr address, s32 return ERR_INVALID_ADDRESS_STATE; } - if (static_cast<s32>(memory.Read32(address)) != value) { - return ERR_INVALID_STATE; - } + const std::size_t current_core = system.CurrentCoreIndex(); + auto& monitor = system.Monitor(); + u32 current_value; + do { + current_value = monitor.ExclusiveRead32(current_core, address); + + if (current_value != static_cast<u32>(value)) { + return ERR_INVALID_STATE; + } + current_value++; + } while (!monitor.ExclusiveWrite32(current_core, address, current_value)); - memory.Write32(address, static_cast<u32>(value + 1)); return SignalToAddressOnly(address, num_to_wake); } ResultCode AddressArbiter::ModifyByWaitingCountAndSignalToAddressIfEqual(VAddr address, s32 value, s32 num_to_wake) { + SchedulerLock lock(system.Kernel()); auto& memory = system.Memory(); // Ensure that we can write to the address. @@ -92,29 +104,33 @@ ResultCode AddressArbiter::ModifyByWaitingCountAndSignalToAddressIfEqual(VAddr a const std::vector<std::shared_ptr<Thread>> waiting_threads = GetThreadsWaitingOnAddress(address); - // Determine the modified value depending on the waiting count. + const std::size_t current_core = system.CurrentCoreIndex(); + auto& monitor = system.Monitor(); s32 updated_value; - if (num_to_wake <= 0) { - if (waiting_threads.empty()) { - updated_value = value + 1; - } else { - updated_value = value - 1; + do { + updated_value = monitor.ExclusiveRead32(current_core, address); + + if (updated_value != value) { + return ERR_INVALID_STATE; } - } else { - if (waiting_threads.empty()) { - updated_value = value + 1; - } else if (waiting_threads.size() <= static_cast<u32>(num_to_wake)) { - updated_value = value - 1; + // Determine the modified value depending on the waiting count. + if (num_to_wake <= 0) { + if (waiting_threads.empty()) { + updated_value = value + 1; + } else { + updated_value = value - 1; + } } else { - updated_value = value; + if (waiting_threads.empty()) { + updated_value = value + 1; + } else if (waiting_threads.size() <= static_cast<u32>(num_to_wake)) { + updated_value = value - 1; + } else { + updated_value = value; + } } - } + } while (!monitor.ExclusiveWrite32(current_core, address, updated_value)); - if (static_cast<s32>(memory.Read32(address)) != value) { - return ERR_INVALID_STATE; - } - - memory.Write32(address, static_cast<u32>(updated_value)); WakeThreads(waiting_threads, num_to_wake); return RESULT_SUCCESS; } @@ -136,60 +152,127 @@ ResultCode AddressArbiter::WaitForAddress(VAddr address, ArbitrationType type, s ResultCode AddressArbiter::WaitForAddressIfLessThan(VAddr address, s32 value, s64 timeout, bool should_decrement) { auto& memory = system.Memory(); + auto& kernel = system.Kernel(); + Thread* current_thread = system.CurrentScheduler().GetCurrentThread(); - // Ensure that we can read the address. - if (!memory.IsValidVirtualAddress(address)) { - return ERR_INVALID_ADDRESS_STATE; - } + Handle event_handle = InvalidHandle; + { + SchedulerLockAndSleep lock(kernel, event_handle, current_thread, timeout); + + if (current_thread->IsPendingTermination()) { + lock.CancelSleep(); + return ERR_THREAD_TERMINATING; + } + + // Ensure that we can read the address. + if (!memory.IsValidVirtualAddress(address)) { + lock.CancelSleep(); + return ERR_INVALID_ADDRESS_STATE; + } + + s32 current_value = static_cast<s32>(memory.Read32(address)); + if (current_value >= value) { + lock.CancelSleep(); + return ERR_INVALID_STATE; + } + + current_thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT); + + s32 decrement_value; + + const std::size_t current_core = system.CurrentCoreIndex(); + auto& monitor = system.Monitor(); + do { + current_value = static_cast<s32>(monitor.ExclusiveRead32(current_core, address)); + if (should_decrement) { + decrement_value = current_value - 1; + } else { + decrement_value = current_value; + } + } while ( + !monitor.ExclusiveWrite32(current_core, address, static_cast<u32>(decrement_value))); + + // Short-circuit without rescheduling, if timeout is zero. + if (timeout == 0) { + lock.CancelSleep(); + return RESULT_TIMEOUT; + } - const s32 cur_value = static_cast<s32>(memory.Read32(address)); - if (cur_value >= value) { - return ERR_INVALID_STATE; + current_thread->SetArbiterWaitAddress(address); + InsertThread(SharedFrom(current_thread)); + current_thread->SetStatus(ThreadStatus::WaitArb); + current_thread->WaitForArbitration(true); } - if (should_decrement) { - memory.Write32(address, static_cast<u32>(cur_value - 1)); + if (event_handle != InvalidHandle) { + auto& time_manager = kernel.TimeManager(); + time_manager.UnscheduleTimeEvent(event_handle); } - // Short-circuit without rescheduling, if timeout is zero. - if (timeout == 0) { - return RESULT_TIMEOUT; + { + SchedulerLock lock(kernel); + if (current_thread->IsWaitingForArbitration()) { + RemoveThread(SharedFrom(current_thread)); + current_thread->WaitForArbitration(false); + } } - return WaitForAddressImpl(address, timeout); + return current_thread->GetSignalingResult(); } ResultCode AddressArbiter::WaitForAddressIfEqual(VAddr address, s32 value, s64 timeout) { auto& memory = system.Memory(); + auto& kernel = system.Kernel(); + Thread* current_thread = system.CurrentScheduler().GetCurrentThread(); - // Ensure that we can read the address. - if (!memory.IsValidVirtualAddress(address)) { - return ERR_INVALID_ADDRESS_STATE; - } + Handle event_handle = InvalidHandle; + { + SchedulerLockAndSleep lock(kernel, event_handle, current_thread, timeout); + + if (current_thread->IsPendingTermination()) { + lock.CancelSleep(); + return ERR_THREAD_TERMINATING; + } + + // Ensure that we can read the address. + if (!memory.IsValidVirtualAddress(address)) { + lock.CancelSleep(); + return ERR_INVALID_ADDRESS_STATE; + } - // Only wait for the address if equal. - if (static_cast<s32>(memory.Read32(address)) != value) { - return ERR_INVALID_STATE; + s32 current_value = static_cast<s32>(memory.Read32(address)); + if (current_value != value) { + lock.CancelSleep(); + return ERR_INVALID_STATE; + } + + // Short-circuit without rescheduling, if timeout is zero. + if (timeout == 0) { + lock.CancelSleep(); + return RESULT_TIMEOUT; + } + + current_thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT); + current_thread->SetArbiterWaitAddress(address); + InsertThread(SharedFrom(current_thread)); + current_thread->SetStatus(ThreadStatus::WaitArb); + current_thread->WaitForArbitration(true); } - // Short-circuit without rescheduling if timeout is zero. - if (timeout == 0) { - return RESULT_TIMEOUT; + if (event_handle != InvalidHandle) { + auto& time_manager = kernel.TimeManager(); + time_manager.UnscheduleTimeEvent(event_handle); } - return WaitForAddressImpl(address, timeout); -} + { + SchedulerLock lock(kernel); + if (current_thread->IsWaitingForArbitration()) { + RemoveThread(SharedFrom(current_thread)); + current_thread->WaitForArbitration(false); + } + } -ResultCode AddressArbiter::WaitForAddressImpl(VAddr address, s64 timeout) { - Thread* current_thread = system.CurrentScheduler().GetCurrentThread(); - current_thread->SetArbiterWaitAddress(address); - InsertThread(SharedFrom(current_thread)); - current_thread->SetStatus(ThreadStatus::WaitArb); - current_thread->InvalidateWakeupCallback(); - current_thread->WakeAfterDelay(timeout); - - system.PrepareReschedule(current_thread->GetProcessorID()); - return RESULT_TIMEOUT; + return current_thread->GetSignalingResult(); } void AddressArbiter::HandleWakeupThread(std::shared_ptr<Thread> thread) { @@ -221,9 +304,9 @@ void AddressArbiter::RemoveThread(std::shared_ptr<Thread> thread) { const auto iter = std::find_if(thread_list.cbegin(), thread_list.cend(), [&thread](const auto& entry) { return thread == entry; }); - ASSERT(iter != thread_list.cend()); - - thread_list.erase(iter); + if (iter != thread_list.cend()) { + thread_list.erase(iter); + } } std::vector<std::shared_ptr<Thread>> AddressArbiter::GetThreadsWaitingOnAddress( diff --git a/src/core/hle/kernel/address_arbiter.h b/src/core/hle/kernel/address_arbiter.h index f958eee5a..0b05d533c 100644 --- a/src/core/hle/kernel/address_arbiter.h +++ b/src/core/hle/kernel/address_arbiter.h @@ -73,9 +73,6 @@ private: /// Waits on an address if the value passed is equal to the argument value. ResultCode WaitForAddressIfEqual(VAddr address, s32 value, s64 timeout); - // Waits on the given address with a timeout in nanoseconds - ResultCode WaitForAddressImpl(VAddr address, s64 timeout); - /// Wake up num_to_wake (or all) threads in a vector. void WakeThreads(const std::vector<std::shared_ptr<Thread>>& waiting_threads, s32 num_to_wake); diff --git a/src/core/hle/kernel/client_port.cpp b/src/core/hle/kernel/client_port.cpp index 5498fd313..8aff2227a 100644 --- a/src/core/hle/kernel/client_port.cpp +++ b/src/core/hle/kernel/client_port.cpp @@ -34,7 +34,7 @@ ResultVal<std::shared_ptr<ClientSession>> ClientPort::Connect() { } // Wake the threads waiting on the ServerPort - server_port->WakeupAllWaitingThreads(); + server_port->Signal(); return MakeResult(std::move(client)); } diff --git a/src/core/hle/kernel/client_session.cpp b/src/core/hle/kernel/client_session.cpp index 6d66276bc..be9eba519 100644 --- a/src/core/hle/kernel/client_session.cpp +++ b/src/core/hle/kernel/client_session.cpp @@ -47,14 +47,16 @@ ResultVal<std::shared_ptr<ClientSession>> ClientSession::Create(KernelCore& kern return MakeResult(std::move(client_session)); } -ResultCode ClientSession::SendSyncRequest(std::shared_ptr<Thread> thread, Memory::Memory& memory) { +ResultCode ClientSession::SendSyncRequest(std::shared_ptr<Thread> thread, + Core::Memory::Memory& memory, + Core::Timing::CoreTiming& core_timing) { // Keep ServerSession alive until we're done working with it. if (!parent->Server()) { return ERR_SESSION_CLOSED_BY_REMOTE; } // Signal the server session that new data is available - return parent->Server()->HandleSyncRequest(std::move(thread), memory); + return parent->Server()->HandleSyncRequest(std::move(thread), memory, core_timing); } } // namespace Kernel diff --git a/src/core/hle/kernel/client_session.h b/src/core/hle/kernel/client_session.h index d15b09554..e5e0690c2 100644 --- a/src/core/hle/kernel/client_session.h +++ b/src/core/hle/kernel/client_session.h @@ -12,10 +12,14 @@ union ResultCode; -namespace Memory { +namespace Core::Memory { class Memory; } +namespace Core::Timing { +class CoreTiming; +} + namespace Kernel { class KernelCore; @@ -42,7 +46,8 @@ public: return HANDLE_TYPE; } - ResultCode SendSyncRequest(std::shared_ptr<Thread> thread, Memory::Memory& memory); + ResultCode SendSyncRequest(std::shared_ptr<Thread> thread, Core::Memory::Memory& memory, + Core::Timing::CoreTiming& core_timing); bool ShouldWait(const Thread* thread) const override; diff --git a/src/core/hle/kernel/errors.h b/src/core/hle/kernel/errors.h index 8097b3863..d4e5d88cf 100644 --- a/src/core/hle/kernel/errors.h +++ b/src/core/hle/kernel/errors.h @@ -12,8 +12,10 @@ namespace Kernel { constexpr ResultCode ERR_MAX_CONNECTIONS_REACHED{ErrorModule::Kernel, 7}; constexpr ResultCode ERR_INVALID_CAPABILITY_DESCRIPTOR{ErrorModule::Kernel, 14}; +constexpr ResultCode ERR_THREAD_TERMINATING{ErrorModule::Kernel, 59}; constexpr ResultCode ERR_INVALID_SIZE{ErrorModule::Kernel, 101}; constexpr ResultCode ERR_INVALID_ADDRESS{ErrorModule::Kernel, 102}; +constexpr ResultCode ERR_OUT_OF_RESOURCES{ErrorModule::Kernel, 103}; constexpr ResultCode ERR_OUT_OF_MEMORY{ErrorModule::Kernel, 104}; constexpr ResultCode ERR_HANDLE_TABLE_FULL{ErrorModule::Kernel, 105}; constexpr ResultCode ERR_INVALID_ADDRESS_STATE{ErrorModule::Kernel, 106}; diff --git a/src/core/hle/kernel/handle_table.cpp b/src/core/hle/kernel/handle_table.cpp index e441a27fc..3e745c18b 100644 --- a/src/core/hle/kernel/handle_table.cpp +++ b/src/core/hle/kernel/handle_table.cpp @@ -8,7 +8,9 @@ #include "core/core.h" #include "core/hle/kernel/errors.h" #include "core/hle/kernel/handle_table.h" +#include "core/hle/kernel/kernel.h" #include "core/hle/kernel/process.h" +#include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/thread.h" namespace Kernel { @@ -22,7 +24,7 @@ constexpr u16 GetGeneration(Handle handle) { } } // Anonymous namespace -HandleTable::HandleTable() { +HandleTable::HandleTable(KernelCore& kernel) : kernel{kernel} { Clear(); } @@ -30,6 +32,7 @@ HandleTable::~HandleTable() = default; ResultCode HandleTable::SetSize(s32 handle_table_size) { if (static_cast<u32>(handle_table_size) > MAX_COUNT) { + LOG_ERROR(Kernel, "Handle table size {} is greater than {}", handle_table_size, MAX_COUNT); return ERR_OUT_OF_MEMORY; } @@ -80,6 +83,7 @@ ResultVal<Handle> HandleTable::Duplicate(Handle handle) { ResultCode HandleTable::Close(Handle handle) { if (!IsValid(handle)) { + LOG_ERROR(Kernel, "Handle is not valid! handle={:08X}", handle); return ERR_INVALID_HANDLE; } @@ -101,9 +105,9 @@ bool HandleTable::IsValid(Handle handle) const { std::shared_ptr<Object> HandleTable::GetGeneric(Handle handle) const { if (handle == CurrentThread) { - return SharedFrom(GetCurrentThread()); + return SharedFrom(kernel.CurrentScheduler().GetCurrentThread()); } else if (handle == CurrentProcess) { - return SharedFrom(Core::System::GetInstance().CurrentProcess()); + return SharedFrom(kernel.CurrentProcess()); } if (!IsValid(handle)) { @@ -114,7 +118,7 @@ std::shared_ptr<Object> HandleTable::GetGeneric(Handle handle) const { void HandleTable::Clear() { for (u16 i = 0; i < table_size; ++i) { - generations[i] = i + 1; + generations[i] = static_cast<u16>(i + 1); objects[i] = nullptr; } next_free_slot = 0; diff --git a/src/core/hle/kernel/handle_table.h b/src/core/hle/kernel/handle_table.h index 8029660ed..c9dab8cdd 100644 --- a/src/core/hle/kernel/handle_table.h +++ b/src/core/hle/kernel/handle_table.h @@ -14,6 +14,8 @@ namespace Kernel { +class KernelCore; + enum KernelHandle : Handle { InvalidHandle = 0, CurrentThread = 0xFFFF8000, @@ -48,7 +50,7 @@ public: /// This is the maximum limit of handles allowed per process in Horizon static constexpr std::size_t MAX_COUNT = 1024; - HandleTable(); + explicit HandleTable(KernelCore& kernel); ~HandleTable(); /** @@ -134,6 +136,9 @@ private: /// Head of the free slots linked list. u16 next_free_slot = 0; + + /// Underlying kernel instance that this handle table operates under. + KernelCore& kernel; }; } // namespace Kernel diff --git a/src/core/hle/kernel/hle_ipc.cpp b/src/core/hle/kernel/hle_ipc.cpp index c558a2f33..81f85643b 100644 --- a/src/core/hle/kernel/hle_ipc.cpp +++ b/src/core/hle/kernel/hle_ipc.cpp @@ -13,16 +13,18 @@ #include "common/common_funcs.h" #include "common/common_types.h" #include "common/logging/log.h" -#include "core/core.h" #include "core/hle/ipc_helpers.h" +#include "core/hle/kernel/errors.h" #include "core/hle/kernel/handle_table.h" #include "core/hle/kernel/hle_ipc.h" #include "core/hle/kernel/kernel.h" #include "core/hle/kernel/object.h" #include "core/hle/kernel/process.h" #include "core/hle/kernel/readable_event.h" +#include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/server_session.h" #include "core/hle/kernel/thread.h" +#include "core/hle/kernel/time_manager.h" #include "core/hle/kernel/writable_event.h" #include "core/memory.h" @@ -47,31 +49,33 @@ std::shared_ptr<WritableEvent> HLERequestContext::SleepClientThread( const std::string& reason, u64 timeout, WakeupCallback&& callback, std::shared_ptr<WritableEvent> writable_event) { // Put the client thread to sleep until the wait event is signaled or the timeout expires. - thread->SetWakeupCallback( - [context = *this, callback](ThreadWakeupReason reason, std::shared_ptr<Thread> thread, - std::shared_ptr<SynchronizationObject> object, - std::size_t index) mutable -> bool { - ASSERT(thread->GetStatus() == ThreadStatus::WaitHLEEvent); - callback(thread, context, reason); - context.WriteToOutgoingCommandBuffer(*thread); - return true; - }); - - auto& kernel = Core::System::GetInstance().Kernel(); + if (!writable_event) { // Create event if not provided const auto pair = WritableEvent::CreateEventPair(kernel, "HLE Pause Event: " + reason); writable_event = pair.writable; } - const auto readable_event{writable_event->GetReadableEvent()}; - writable_event->Clear(); - thread->SetStatus(ThreadStatus::WaitHLEEvent); - thread->SetSynchronizationObjects({readable_event}); - readable_event->AddWaitingThread(thread); - - if (timeout > 0) { - thread->WakeAfterDelay(timeout); + { + Handle event_handle = InvalidHandle; + SchedulerLockAndSleep lock(kernel, event_handle, thread.get(), timeout); + thread->SetHLECallback( + [context = *this, callback](std::shared_ptr<Thread> thread) mutable -> bool { + ThreadWakeupReason reason = thread->GetSignalingResult() == RESULT_TIMEOUT + ? ThreadWakeupReason::Timeout + : ThreadWakeupReason::Signal; + callback(thread, context, reason); + context.WriteToOutgoingCommandBuffer(*thread); + return true; + }); + const auto readable_event{writable_event->GetReadableEvent()}; + writable_event->Clear(); + thread->SetHLESyncObject(readable_event.get()); + thread->SetStatus(ThreadStatus::WaitHLEEvent); + thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT); + readable_event->AddWaitingThread(thread); + lock.Release(); + thread->SetHLETimeEvent(event_handle); } is_thread_waiting = true; @@ -79,9 +83,11 @@ std::shared_ptr<WritableEvent> HLERequestContext::SleepClientThread( return writable_event; } -HLERequestContext::HLERequestContext(std::shared_ptr<Kernel::ServerSession> server_session, +HLERequestContext::HLERequestContext(KernelCore& kernel, Core::Memory::Memory& memory, + std::shared_ptr<ServerSession> server_session, std::shared_ptr<Thread> thread) - : server_session(std::move(server_session)), thread(std::move(thread)) { + : server_session(std::move(server_session)), + thread(std::move(thread)), kernel{kernel}, memory{memory} { cmd_buf[0] = 0; } @@ -216,7 +222,6 @@ ResultCode HLERequestContext::PopulateFromIncomingCommandBuffer(const HandleTabl ResultCode HLERequestContext::WriteToOutgoingCommandBuffer(Thread& thread) { auto& owner_process = *thread.GetOwnerProcess(); auto& handle_table = owner_process.GetHandleTable(); - auto& memory = Core::System::GetInstance().Memory(); std::array<u32, IPC::COMMAND_BUFFER_LENGTH> dst_cmdbuf; memory.ReadBlock(owner_process, thread.GetTLSAddress(), dst_cmdbuf.data(), @@ -282,20 +287,21 @@ ResultCode HLERequestContext::WriteToOutgoingCommandBuffer(Thread& thread) { return RESULT_SUCCESS; } -std::vector<u8> HLERequestContext::ReadBuffer(int buffer_index) const { - std::vector<u8> buffer; +std::vector<u8> HLERequestContext::ReadBuffer(std::size_t buffer_index) const { + std::vector<u8> buffer{}; const bool is_buffer_a{BufferDescriptorA().size() > buffer_index && BufferDescriptorA()[buffer_index].Size()}; - auto& memory = Core::System::GetInstance().Memory(); if (is_buffer_a) { - ASSERT_MSG(BufferDescriptorA().size() > buffer_index, - "BufferDescriptorA invalid buffer_index {}", buffer_index); + ASSERT_OR_EXECUTE_MSG( + BufferDescriptorA().size() > buffer_index, { return buffer; }, + "BufferDescriptorA invalid buffer_index {}", buffer_index); buffer.resize(BufferDescriptorA()[buffer_index].Size()); memory.ReadBlock(BufferDescriptorA()[buffer_index].Address(), buffer.data(), buffer.size()); } else { - ASSERT_MSG(BufferDescriptorX().size() > buffer_index, - "BufferDescriptorX invalid buffer_index {}", buffer_index); + ASSERT_OR_EXECUTE_MSG( + BufferDescriptorX().size() > buffer_index, { return buffer; }, + "BufferDescriptorX invalid buffer_index {}", buffer_index); buffer.resize(BufferDescriptorX()[buffer_index].Size()); memory.ReadBlock(BufferDescriptorX()[buffer_index].Address(), buffer.data(), buffer.size()); } @@ -304,7 +310,7 @@ std::vector<u8> HLERequestContext::ReadBuffer(int buffer_index) const { } std::size_t HLERequestContext::WriteBuffer(const void* buffer, std::size_t size, - int buffer_index) const { + std::size_t buffer_index) const { if (size == 0) { LOG_WARNING(Core, "skip empty buffer write"); return 0; @@ -319,54 +325,54 @@ std::size_t HLERequestContext::WriteBuffer(const void* buffer, std::size_t size, size = buffer_size; // TODO(bunnei): This needs to be HW tested } - auto& memory = Core::System::GetInstance().Memory(); if (is_buffer_b) { - ASSERT_MSG(BufferDescriptorB().size() > buffer_index, - "BufferDescriptorB invalid buffer_index {}", buffer_index); - ASSERT_MSG(BufferDescriptorB()[buffer_index].Size() >= size, - "BufferDescriptorB buffer_index {} is not large enough", buffer_index); + ASSERT_OR_EXECUTE_MSG( + BufferDescriptorB().size() > buffer_index && + BufferDescriptorB()[buffer_index].Size() >= size, + { return 0; }, "BufferDescriptorB is invalid, index={}, size={}", buffer_index, size); memory.WriteBlock(BufferDescriptorB()[buffer_index].Address(), buffer, size); } else { - ASSERT_MSG(BufferDescriptorC().size() > buffer_index, - "BufferDescriptorC invalid buffer_index {}", buffer_index); - ASSERT_MSG(BufferDescriptorC()[buffer_index].Size() >= size, - "BufferDescriptorC buffer_index {} is not large enough", buffer_index); + ASSERT_OR_EXECUTE_MSG( + BufferDescriptorC().size() > buffer_index && + BufferDescriptorC()[buffer_index].Size() >= size, + { return 0; }, "BufferDescriptorC is invalid, index={}, size={}", buffer_index, size); memory.WriteBlock(BufferDescriptorC()[buffer_index].Address(), buffer, size); } return size; } -std::size_t HLERequestContext::GetReadBufferSize(int buffer_index) const { +std::size_t HLERequestContext::GetReadBufferSize(std::size_t buffer_index) const { const bool is_buffer_a{BufferDescriptorA().size() > buffer_index && BufferDescriptorA()[buffer_index].Size()}; if (is_buffer_a) { - ASSERT_MSG(BufferDescriptorA().size() > buffer_index, - "BufferDescriptorA invalid buffer_index {}", buffer_index); - ASSERT_MSG(BufferDescriptorA()[buffer_index].Size() > 0, - "BufferDescriptorA buffer_index {} is empty", buffer_index); + ASSERT_OR_EXECUTE_MSG( + BufferDescriptorA().size() > buffer_index, { return 0; }, + "BufferDescriptorA invalid buffer_index {}", buffer_index); return BufferDescriptorA()[buffer_index].Size(); } else { - ASSERT_MSG(BufferDescriptorX().size() > buffer_index, - "BufferDescriptorX invalid buffer_index {}", buffer_index); - ASSERT_MSG(BufferDescriptorX()[buffer_index].Size() > 0, - "BufferDescriptorX buffer_index {} is empty", buffer_index); + ASSERT_OR_EXECUTE_MSG( + BufferDescriptorX().size() > buffer_index, { return 0; }, + "BufferDescriptorX invalid buffer_index {}", buffer_index); return BufferDescriptorX()[buffer_index].Size(); } } -std::size_t HLERequestContext::GetWriteBufferSize(int buffer_index) const { +std::size_t HLERequestContext::GetWriteBufferSize(std::size_t buffer_index) const { const bool is_buffer_b{BufferDescriptorB().size() > buffer_index && BufferDescriptorB()[buffer_index].Size()}; if (is_buffer_b) { - ASSERT_MSG(BufferDescriptorB().size() > buffer_index, - "BufferDescriptorB invalid buffer_index {}", buffer_index); + ASSERT_OR_EXECUTE_MSG( + BufferDescriptorB().size() > buffer_index, { return 0; }, + "BufferDescriptorB invalid buffer_index {}", buffer_index); return BufferDescriptorB()[buffer_index].Size(); } else { - ASSERT_MSG(BufferDescriptorC().size() > buffer_index, - "BufferDescriptorC invalid buffer_index {}", buffer_index); + ASSERT_OR_EXECUTE_MSG( + BufferDescriptorC().size() > buffer_index, { return 0; }, + "BufferDescriptorC invalid buffer_index {}", buffer_index); return BufferDescriptorC()[buffer_index].Size(); } + return 0; } std::string HLERequestContext::Description() const { diff --git a/src/core/hle/kernel/hle_ipc.h b/src/core/hle/kernel/hle_ipc.h index 050ad8fd7..c31a65476 100644 --- a/src/core/hle/kernel/hle_ipc.h +++ b/src/core/hle/kernel/hle_ipc.h @@ -13,12 +13,21 @@ #include <vector> #include <boost/container/small_vector.hpp> #include "common/common_types.h" +#include "common/concepts.h" #include "common/swap.h" #include "core/hle/ipc.h" #include "core/hle/kernel/object.h" union ResultCode; +namespace Core::Memory { +class Memory; +} + +namespace IPC { +class ResponseBuilder; +} + namespace Service { class ServiceFrameworkBase; } @@ -28,6 +37,7 @@ namespace Kernel { class Domain; class HandleTable; class HLERequestContext; +class KernelCore; class Process; class ServerSession; class Thread; @@ -98,7 +108,8 @@ protected: */ class HLERequestContext { public: - explicit HLERequestContext(std::shared_ptr<ServerSession> session, + explicit HLERequestContext(KernelCore& kernel, Core::Memory::Memory& memory, + std::shared_ptr<ServerSession> session, std::shared_ptr<Thread> thread); ~HLERequestContext(); @@ -179,36 +190,39 @@ public: } /// Helper function to read a buffer using the appropriate buffer descriptor - std::vector<u8> ReadBuffer(int buffer_index = 0) const; + std::vector<u8> ReadBuffer(std::size_t buffer_index = 0) const; /// Helper function to write a buffer using the appropriate buffer descriptor - std::size_t WriteBuffer(const void* buffer, std::size_t size, int buffer_index = 0) const; + std::size_t WriteBuffer(const void* buffer, std::size_t size, + std::size_t buffer_index = 0) const; /* Helper function to write a buffer using the appropriate buffer descriptor * - * @tparam ContiguousContainer an arbitrary container that satisfies the - * ContiguousContainer concept in the C++ standard library. + * @tparam T an arbitrary container that satisfies the + * ContiguousContainer concept in the C++ standard library or a trivially copyable type. * - * @param container The container to write the data of into a buffer. + * @param data The container/data to write into a buffer. * @param buffer_index The buffer in particular to write to. */ - template <typename ContiguousContainer, - typename = std::enable_if_t<!std::is_pointer_v<ContiguousContainer>>> - std::size_t WriteBuffer(const ContiguousContainer& container, int buffer_index = 0) const { - using ContiguousType = typename ContiguousContainer::value_type; - - static_assert(std::is_trivially_copyable_v<ContiguousType>, - "Container to WriteBuffer must contain trivially copyable objects"); - - return WriteBuffer(std::data(container), std::size(container) * sizeof(ContiguousType), - buffer_index); + template <typename T, typename = std::enable_if_t<!std::is_pointer_v<T>>> + std::size_t WriteBuffer(const T& data, std::size_t buffer_index = 0) const { + if constexpr (Common::IsSTLContainer<T>) { + using ContiguousType = typename T::value_type; + static_assert(std::is_trivially_copyable_v<ContiguousType>, + "Container to WriteBuffer must contain trivially copyable objects"); + return WriteBuffer(std::data(data), std::size(data) * sizeof(ContiguousType), + buffer_index); + } else { + static_assert(std::is_trivially_copyable_v<T>, "T must be trivially copyable"); + return WriteBuffer(&data, sizeof(T), buffer_index); + } } /// Helper function to get the size of the input buffer - std::size_t GetReadBufferSize(int buffer_index = 0) const; + std::size_t GetReadBufferSize(std::size_t buffer_index = 0) const; /// Helper function to get the size of the output buffer - std::size_t GetWriteBufferSize(int buffer_index = 0) const; + std::size_t GetWriteBufferSize(std::size_t buffer_index = 0) const; template <typename T> std::shared_ptr<T> GetCopyObject(std::size_t index) { @@ -277,6 +291,8 @@ public: } private: + friend class IPC::ResponseBuilder; + void ParseCommandBuffer(const HandleTable& handle_table, u32_le* src_cmdbuf, bool incoming); std::array<u32, IPC::COMMAND_BUFFER_LENGTH> cmd_buf; @@ -303,6 +319,9 @@ private: std::vector<std::shared_ptr<SessionRequestHandler>> domain_request_handlers; bool is_thread_waiting{}; + + KernelCore& kernel; + Core::Memory::Memory& memory; }; } // namespace Kernel diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index 014d647cf..bb3e312a7 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -2,31 +2,40 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include <array> #include <atomic> #include <bitset> #include <functional> #include <memory> -#include <mutex> #include <thread> #include <unordered_map> #include <utility> #include "common/assert.h" #include "common/logging/log.h" +#include "common/microprofile.h" +#include "common/thread.h" #include "core/arm/arm_interface.h" +#include "core/arm/cpu_interrupt_handler.h" #include "core/arm/exclusive_monitor.h" #include "core/core.h" #include "core/core_timing.h" #include "core/core_timing_util.h" +#include "core/cpu_manager.h" +#include "core/device_memory.h" #include "core/hardware_properties.h" #include "core/hle/kernel/client_port.h" #include "core/hle/kernel/errors.h" #include "core/hle/kernel/handle_table.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/memory/memory_layout.h" +#include "core/hle/kernel/memory/memory_manager.h" +#include "core/hle/kernel/memory/slab_heap.h" #include "core/hle/kernel/physical_core.h" #include "core/hle/kernel/process.h" #include "core/hle/kernel/resource_limit.h" #include "core/hle/kernel/scheduler.h" +#include "core/hle/kernel/shared_memory.h" #include "core/hle/kernel/synchronization.h" #include "core/hle/kernel/thread.h" #include "core/hle/kernel/time_manager.h" @@ -34,84 +43,29 @@ #include "core/hle/result.h" #include "core/memory.h" -namespace Kernel { - -/** - * Callback that will wake up the thread it was scheduled for - * @param thread_handle The handle of the thread that's been awoken - * @param cycles_late The number of CPU cycles that have passed since the desired wakeup time - */ -static void ThreadWakeupCallback(u64 thread_handle, [[maybe_unused]] s64 cycles_late) { - const auto proper_handle = static_cast<Handle>(thread_handle); - const auto& system = Core::System::GetInstance(); - - // Lock the global kernel mutex when we enter the kernel HLE. - std::lock_guard lock{HLE::g_hle_lock}; - - std::shared_ptr<Thread> thread = - system.Kernel().RetrieveThreadFromGlobalHandleTable(proper_handle); - if (thread == nullptr) { - LOG_CRITICAL(Kernel, "Callback fired for invalid thread {:08X}", proper_handle); - return; - } - - bool resume = true; - - if (thread->GetStatus() == ThreadStatus::WaitSynch || - thread->GetStatus() == ThreadStatus::WaitHLEEvent) { - // Remove the thread from each of its waiting objects' waitlists - for (const auto& object : thread->GetSynchronizationObjects()) { - object->RemoveWaitingThread(thread); - } - thread->ClearSynchronizationObjects(); - - // Invoke the wakeup callback before clearing the wait objects - if (thread->HasWakeupCallback()) { - resume = thread->InvokeWakeupCallback(ThreadWakeupReason::Timeout, thread, nullptr, 0); - } - } else if (thread->GetStatus() == ThreadStatus::WaitMutex || - thread->GetStatus() == ThreadStatus::WaitCondVar) { - thread->SetMutexWaitAddress(0); - thread->SetWaitHandle(0); - if (thread->GetStatus() == ThreadStatus::WaitCondVar) { - thread->GetOwnerProcess()->RemoveConditionVariableThread(thread); - thread->SetCondVarWaitAddress(0); - } - - auto* const lock_owner = thread->GetLockOwner(); - // Threads waking up by timeout from WaitProcessWideKey do not perform priority inheritance - // and don't have a lock owner unless SignalProcessWideKey was called first and the thread - // wasn't awakened due to the mutex already being acquired. - if (lock_owner != nullptr) { - lock_owner->RemoveMutexWaiter(thread); - } - } - - if (thread->GetStatus() == ThreadStatus::WaitArb) { - auto& address_arbiter = thread->GetOwnerProcess()->GetAddressArbiter(); - address_arbiter.HandleWakeupThread(thread); - } +MICROPROFILE_DEFINE(Kernel_SVC, "Kernel", "SVC", MP_RGB(70, 200, 70)); - if (resume) { - if (thread->GetStatus() == ThreadStatus::WaitCondVar || - thread->GetStatus() == ThreadStatus::WaitArb) { - thread->SetWaitSynchronizationResult(RESULT_TIMEOUT); - } - thread->ResumeFromWait(); - } -} +namespace Kernel { struct KernelCore::Impl { explicit Impl(Core::System& system, KernelCore& kernel) - : global_scheduler{kernel}, synchronization{system}, time_manager{system}, system{system} {} + : global_scheduler{kernel}, synchronization{system}, time_manager{system}, + global_handle_table{kernel}, system{system} {} + + void SetMulticore(bool is_multicore) { + this->is_multicore = is_multicore; + } void Initialize(KernelCore& kernel) { Shutdown(); + RegisterHostThread(); InitializePhysicalCores(); InitializeSystemResourceLimit(kernel); - InitializeThreads(); - InitializePreemption(); + InitializeMemoryLayout(); + InitializePreemption(kernel); + InitializeSchedulers(); + InitializeSuspendThreads(); } void Shutdown() { @@ -120,13 +74,24 @@ struct KernelCore::Impl { next_user_process_id = Process::ProcessIDMin; next_thread_id = 1; + for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { + if (suspend_threads[i]) { + suspend_threads[i].reset(); + } + } + + for (std::size_t i = 0; i < cores.size(); i++) { + cores[i].Shutdown(); + schedulers[i].reset(); + } + cores.clear(); + process_list.clear(); current_process = nullptr; system_resource_limit = nullptr; global_handle_table.Clear(); - thread_wakeup_event_type = nullptr; preemption_event = nullptr; global_scheduler.Shutdown(); @@ -139,13 +104,25 @@ struct KernelCore::Impl { cores.clear(); exclusive_monitor.reset(); + + num_host_threads = 0; + std::fill(register_host_thread_keys.begin(), register_host_thread_keys.end(), + std::thread::id{}); + std::fill(register_host_thread_values.begin(), register_host_thread_values.end(), 0); } void InitializePhysicalCores() { exclusive_monitor = Core::MakeExclusiveMonitor(system.Memory(), Core::Hardware::NUM_CPU_CORES); for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { - cores.emplace_back(system, i, *exclusive_monitor); + schedulers[i] = std::make_unique<Kernel::Scheduler>(system, i); + cores.emplace_back(system, i, *schedulers[i], interrupts[i]); + } + } + + void InitializeSchedulers() { + for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { + cores[i].Scheduler().Initialize(); } } @@ -154,71 +131,105 @@ struct KernelCore::Impl { system_resource_limit = ResourceLimit::Create(kernel); // If setting the default system values fails, then something seriously wrong has occurred. - ASSERT(system_resource_limit->SetLimitValue(ResourceType::PhysicalMemory, 0x200000000) + ASSERT(system_resource_limit->SetLimitValue(ResourceType::PhysicalMemory, 0x100000000) .IsSuccess()); ASSERT(system_resource_limit->SetLimitValue(ResourceType::Threads, 800).IsSuccess()); ASSERT(system_resource_limit->SetLimitValue(ResourceType::Events, 700).IsSuccess()); ASSERT(system_resource_limit->SetLimitValue(ResourceType::TransferMemory, 200).IsSuccess()); ASSERT(system_resource_limit->SetLimitValue(ResourceType::Sessions, 900).IsSuccess()); - } - void InitializeThreads() { - thread_wakeup_event_type = - Core::Timing::CreateEvent("ThreadWakeupCallback", ThreadWakeupCallback); + if (!system_resource_limit->Reserve(ResourceType::PhysicalMemory, 0) || + !system_resource_limit->Reserve(ResourceType::PhysicalMemory, 0x60000)) { + UNREACHABLE(); + } } - void InitializePreemption() { - preemption_event = - Core::Timing::CreateEvent("PreemptionCallback", [this](u64 userdata, s64 cycles_late) { - global_scheduler.PreemptThreads(); - s64 time_interval = Core::Timing::msToCycles(std::chrono::milliseconds(10)); + void InitializePreemption(KernelCore& kernel) { + preemption_event = Core::Timing::CreateEvent( + "PreemptionCallback", [this, &kernel](std::uintptr_t, std::chrono::nanoseconds) { + { + SchedulerLock lock(kernel); + global_scheduler.PreemptThreads(); + } + const auto time_interval = std::chrono::nanoseconds{ + Core::Timing::msToCycles(std::chrono::milliseconds(10))}; system.CoreTiming().ScheduleEvent(time_interval, preemption_event); }); - s64 time_interval = Core::Timing::msToCycles(std::chrono::milliseconds(10)); + const auto time_interval = + std::chrono::nanoseconds{Core::Timing::msToCycles(std::chrono::milliseconds(10))}; system.CoreTiming().ScheduleEvent(time_interval, preemption_event); } + void InitializeSuspendThreads() { + for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { + std::string name = "Suspend Thread Id:" + std::to_string(i); + std::function<void(void*)> init_func = Core::CpuManager::GetSuspendThreadStartFunc(); + void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater(); + const auto type = + static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_SUSPEND); + auto thread_res = + Thread::Create(system, type, std::move(name), 0, 0, 0, static_cast<u32>(i), 0, + nullptr, std::move(init_func), init_func_parameter); + + suspend_threads[i] = std::move(thread_res).Unwrap(); + } + } + void MakeCurrentProcess(Process* process) { current_process = process; - if (process == nullptr) { return; } - - for (auto& core : cores) { - core.SetIs64Bit(process->Is64BitProcess()); + const u32 core_id = GetCurrentHostThreadID(); + if (core_id < Core::Hardware::NUM_CPU_CORES) { + system.Memory().SetCurrentPageTable(*process, core_id); } - - system.Memory().SetCurrentPageTable(*process); } void RegisterCoreThread(std::size_t core_id) { - std::unique_lock lock{register_thread_mutex}; const std::thread::id this_id = std::this_thread::get_id(); - const auto it = host_thread_ids.find(this_id); + if (!is_multicore) { + single_core_thread_id = this_id; + } + const auto end = + register_host_thread_keys.begin() + static_cast<ptrdiff_t>(num_host_threads); + const auto it = std::find(register_host_thread_keys.begin(), end, this_id); ASSERT(core_id < Core::Hardware::NUM_CPU_CORES); - ASSERT(it == host_thread_ids.end()); - ASSERT(!registered_core_threads[core_id]); - host_thread_ids[this_id] = static_cast<u32>(core_id); - registered_core_threads.set(core_id); + ASSERT(it == end); + InsertHostThread(static_cast<u32>(core_id)); } void RegisterHostThread() { - std::unique_lock lock{register_thread_mutex}; const std::thread::id this_id = std::this_thread::get_id(); - const auto it = host_thread_ids.find(this_id); - ASSERT(it == host_thread_ids.end()); - host_thread_ids[this_id] = registered_thread_ids++; + const auto end = + register_host_thread_keys.begin() + static_cast<ptrdiff_t>(num_host_threads); + const auto it = std::find(register_host_thread_keys.begin(), end, this_id); + if (it == end) { + InsertHostThread(registered_thread_ids++); + } + } + + void InsertHostThread(u32 value) { + const size_t index = num_host_threads++; + ASSERT_MSG(index < NUM_REGISTRABLE_HOST_THREADS, "Too many host threads"); + register_host_thread_values[index] = value; + register_host_thread_keys[index] = std::this_thread::get_id(); } - u32 GetCurrentHostThreadID() const { + [[nodiscard]] u32 GetCurrentHostThreadID() const { const std::thread::id this_id = std::this_thread::get_id(); - const auto it = host_thread_ids.find(this_id); - if (it == host_thread_ids.end()) { + if (!is_multicore && single_core_thread_id == this_id) { + return static_cast<u32>(system.GetCpuManager().CurrentCore()); + } + const auto end = + register_host_thread_keys.begin() + static_cast<ptrdiff_t>(num_host_threads); + const auto it = std::find(register_host_thread_keys.begin(), end, this_id); + if (it == end) { return Core::INVALID_HOST_THREAD_ID; } - return it->second; + return register_host_thread_values[static_cast<size_t>( + std::distance(register_host_thread_keys.begin(), it))]; } Core::EmuThreadHandle GetCurrentEmuThreadID() const { @@ -229,7 +240,7 @@ struct KernelCore::Impl { } const Kernel::Scheduler& sched = cores[result.host_handle].Scheduler(); const Kernel::Thread* current = sched.GetCurrentThread(); - if (current != nullptr) { + if (current != nullptr && !current->IsPhantomMode()) { result.guest_handle = current->GetGlobalHandle(); } else { result.guest_handle = InvalidHandle; @@ -237,6 +248,57 @@ struct KernelCore::Impl { return result; } + void InitializeMemoryLayout() { + // Initialize memory layout + constexpr Memory::MemoryLayout layout{Memory::MemoryLayout::GetDefaultLayout()}; + constexpr std::size_t hid_size{0x40000}; + constexpr std::size_t font_size{0x1100000}; + constexpr std::size_t irs_size{0x8000}; + constexpr std::size_t time_size{0x1000}; + constexpr PAddr hid_addr{layout.System().StartAddress()}; + constexpr PAddr font_pa{layout.System().StartAddress() + hid_size}; + constexpr PAddr irs_addr{layout.System().StartAddress() + hid_size + font_size}; + constexpr PAddr time_addr{layout.System().StartAddress() + hid_size + font_size + irs_size}; + + // Initialize memory manager + memory_manager = std::make_unique<Memory::MemoryManager>(); + memory_manager->InitializeManager(Memory::MemoryManager::Pool::Application, + layout.Application().StartAddress(), + layout.Application().EndAddress()); + memory_manager->InitializeManager(Memory::MemoryManager::Pool::Applet, + layout.Applet().StartAddress(), + layout.Applet().EndAddress()); + memory_manager->InitializeManager(Memory::MemoryManager::Pool::System, + layout.System().StartAddress(), + layout.System().EndAddress()); + + hid_shared_mem = Kernel::SharedMemory::Create( + system.Kernel(), system.DeviceMemory(), nullptr, + {hid_addr, hid_size / Memory::PageSize}, Memory::MemoryPermission::None, + Memory::MemoryPermission::Read, hid_addr, hid_size, "HID:SharedMemory"); + font_shared_mem = Kernel::SharedMemory::Create( + system.Kernel(), system.DeviceMemory(), nullptr, + {font_pa, font_size / Memory::PageSize}, Memory::MemoryPermission::None, + Memory::MemoryPermission::Read, font_pa, font_size, "Font:SharedMemory"); + irs_shared_mem = Kernel::SharedMemory::Create( + system.Kernel(), system.DeviceMemory(), nullptr, + {irs_addr, irs_size / Memory::PageSize}, Memory::MemoryPermission::None, + Memory::MemoryPermission::Read, irs_addr, irs_size, "IRS:SharedMemory"); + time_shared_mem = Kernel::SharedMemory::Create( + system.Kernel(), system.DeviceMemory(), nullptr, + {time_addr, time_size / Memory::PageSize}, Memory::MemoryPermission::None, + Memory::MemoryPermission::Read, time_addr, time_size, "Time:SharedMemory"); + + // Allocate slab heaps + user_slab_heap_pages = std::make_unique<Memory::SlabHeap<Memory::Page>>(); + + // Initialize slab heaps + constexpr u64 user_slab_heap_size{0x3de000}; + user_slab_heap_pages->Initialize( + system.DeviceMemory().GetPointer(Core::DramMemoryMap::SlabHeapBase), + user_slab_heap_size); + } + std::atomic<u32> next_object_id{0}; std::atomic<u64> next_kernel_process_id{Process::InitialKIPIDMin}; std::atomic<u64> next_user_process_id{Process::ProcessIDMin}; @@ -251,12 +313,11 @@ struct KernelCore::Impl { std::shared_ptr<ResourceLimit> system_resource_limit; - std::shared_ptr<Core::Timing::EventType> thread_wakeup_event_type; std::shared_ptr<Core::Timing::EventType> preemption_event; // This is the kernel's handle table or supervisor handle table which // stores all the objects in place. - Kernel::HandleTable global_handle_table; + HandleTable global_handle_table; /// Map of named ports managed by the kernel, which can be retrieved using /// the ConnectToPort SVC. @@ -266,10 +327,33 @@ struct KernelCore::Impl { std::vector<Kernel::PhysicalCore> cores; // 0-3 IDs represent core threads, >3 represent others - std::unordered_map<std::thread::id, u32> host_thread_ids; - u32 registered_thread_ids{Core::Hardware::NUM_CPU_CORES}; - std::bitset<Core::Hardware::NUM_CPU_CORES> registered_core_threads; - std::mutex register_thread_mutex; + std::atomic<u32> registered_thread_ids{Core::Hardware::NUM_CPU_CORES}; + + // Number of host threads is a relatively high number to avoid overflowing + static constexpr size_t NUM_REGISTRABLE_HOST_THREADS = 64; + std::atomic<size_t> num_host_threads{0}; + std::array<std::atomic<std::thread::id>, NUM_REGISTRABLE_HOST_THREADS> + register_host_thread_keys{}; + std::array<std::atomic<u32>, NUM_REGISTRABLE_HOST_THREADS> register_host_thread_values{}; + + // Kernel memory management + std::unique_ptr<Memory::MemoryManager> memory_manager; + std::unique_ptr<Memory::SlabHeap<Memory::Page>> user_slab_heap_pages; + + // Shared memory for services + std::shared_ptr<Kernel::SharedMemory> hid_shared_mem; + std::shared_ptr<Kernel::SharedMemory> font_shared_mem; + std::shared_ptr<Kernel::SharedMemory> irs_shared_mem; + std::shared_ptr<Kernel::SharedMemory> time_shared_mem; + + std::array<std::shared_ptr<Thread>, Core::Hardware::NUM_CPU_CORES> suspend_threads{}; + std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES> interrupts{}; + std::array<std::unique_ptr<Kernel::Scheduler>, Core::Hardware::NUM_CPU_CORES> schedulers{}; + + bool is_multicore{}; + std::thread::id single_core_thread_id{}; + + std::array<u64, Core::Hardware::NUM_CPU_CORES> svc_ticks{}; // System context Core::System& system; @@ -280,6 +364,10 @@ KernelCore::~KernelCore() { Shutdown(); } +void KernelCore::SetMulticore(bool is_multicore) { + impl->SetMulticore(is_multicore); +} + void KernelCore::Initialize() { impl->Initialize(*this); } @@ -325,11 +413,11 @@ const Kernel::GlobalScheduler& KernelCore::GlobalScheduler() const { } Kernel::Scheduler& KernelCore::Scheduler(std::size_t id) { - return impl->cores[id].Scheduler(); + return *impl->schedulers[id]; } const Kernel::Scheduler& KernelCore::Scheduler(std::size_t id) const { - return impl->cores[id].Scheduler(); + return *impl->schedulers[id]; } Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) { @@ -340,6 +428,39 @@ const Kernel::PhysicalCore& KernelCore::PhysicalCore(std::size_t id) const { return impl->cores[id]; } +Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() { + u32 core_id = impl->GetCurrentHostThreadID(); + ASSERT(core_id < Core::Hardware::NUM_CPU_CORES); + return impl->cores[core_id]; +} + +const Kernel::PhysicalCore& KernelCore::CurrentPhysicalCore() const { + u32 core_id = impl->GetCurrentHostThreadID(); + ASSERT(core_id < Core::Hardware::NUM_CPU_CORES); + return impl->cores[core_id]; +} + +Kernel::Scheduler& KernelCore::CurrentScheduler() { + u32 core_id = impl->GetCurrentHostThreadID(); + ASSERT(core_id < Core::Hardware::NUM_CPU_CORES); + return *impl->schedulers[core_id]; +} + +const Kernel::Scheduler& KernelCore::CurrentScheduler() const { + u32 core_id = impl->GetCurrentHostThreadID(); + ASSERT(core_id < Core::Hardware::NUM_CPU_CORES); + return *impl->schedulers[core_id]; +} + +std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& KernelCore::Interrupts() { + return impl->interrupts; +} + +const std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& KernelCore::Interrupts() + const { + return impl->interrupts; +} + Kernel::Synchronization& KernelCore::Synchronization() { return impl->synchronization; } @@ -365,15 +486,17 @@ const Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() const { } void KernelCore::InvalidateAllInstructionCaches() { - for (std::size_t i = 0; i < impl->global_scheduler.CpuCoresCount(); i++) { - PhysicalCore(i).ArmInterface().ClearInstructionCache(); + auto& threads = GlobalScheduler().GetThreadList(); + for (auto& thread : threads) { + if (!thread->IsHLEThread()) { + auto& arm_interface = thread->ArmInterface(); + arm_interface.ClearInstructionCache(); + } } } void KernelCore::PrepareReschedule(std::size_t id) { - if (id < impl->global_scheduler.CpuCoresCount()) { - impl->cores[id].Stop(); - } + // TODO: Reimplement, this } void KernelCore::AddNamedPort(std::string name, std::shared_ptr<ClientPort> port) { @@ -409,10 +532,6 @@ u64 KernelCore::CreateNewUserProcessID() { return impl->next_user_process_id++; } -const std::shared_ptr<Core::Timing::EventType>& KernelCore::ThreadWakeupCallbackEventType() const { - return impl->thread_wakeup_event_type; -} - Kernel::HandleTable& KernelCore::GlobalHandleTable() { return impl->global_handle_table; } @@ -437,4 +556,82 @@ Core::EmuThreadHandle KernelCore::GetCurrentEmuThreadID() const { return impl->GetCurrentEmuThreadID(); } +Memory::MemoryManager& KernelCore::MemoryManager() { + return *impl->memory_manager; +} + +const Memory::MemoryManager& KernelCore::MemoryManager() const { + return *impl->memory_manager; +} + +Memory::SlabHeap<Memory::Page>& KernelCore::GetUserSlabHeapPages() { + return *impl->user_slab_heap_pages; +} + +const Memory::SlabHeap<Memory::Page>& KernelCore::GetUserSlabHeapPages() const { + return *impl->user_slab_heap_pages; +} + +Kernel::SharedMemory& KernelCore::GetHidSharedMem() { + return *impl->hid_shared_mem; +} + +const Kernel::SharedMemory& KernelCore::GetHidSharedMem() const { + return *impl->hid_shared_mem; +} + +Kernel::SharedMemory& KernelCore::GetFontSharedMem() { + return *impl->font_shared_mem; +} + +const Kernel::SharedMemory& KernelCore::GetFontSharedMem() const { + return *impl->font_shared_mem; +} + +Kernel::SharedMemory& KernelCore::GetIrsSharedMem() { + return *impl->irs_shared_mem; +} + +const Kernel::SharedMemory& KernelCore::GetIrsSharedMem() const { + return *impl->irs_shared_mem; +} + +Kernel::SharedMemory& KernelCore::GetTimeSharedMem() { + return *impl->time_shared_mem; +} + +const Kernel::SharedMemory& KernelCore::GetTimeSharedMem() const { + return *impl->time_shared_mem; +} + +void KernelCore::Suspend(bool in_suspention) { + const bool should_suspend = exception_exited || in_suspention; + { + SchedulerLock lock(*this); + ThreadStatus status = should_suspend ? ThreadStatus::Ready : ThreadStatus::WaitSleep; + for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { + impl->suspend_threads[i]->SetStatus(status); + } + } +} + +bool KernelCore::IsMulticore() const { + return impl->is_multicore; +} + +void KernelCore::ExceptionalExit() { + exception_exited = true; + Suspend(true); +} + +void KernelCore::EnterSVCProfile() { + std::size_t core = impl->GetCurrentHostThreadID(); + impl->svc_ticks[core] = MicroProfileEnter(MICROPROFILE_TOKEN(Kernel_SVC)); +} + +void KernelCore::ExitSVCProfile() { + std::size_t core = impl->GetCurrentHostThreadID(); + MicroProfileLeave(MICROPROFILE_TOKEN(Kernel_SVC), impl->svc_ticks[core]); +} + } // namespace Kernel diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h index c4f78ab71..16285c3f0 100644 --- a/src/core/hle/kernel/kernel.h +++ b/src/core/hle/kernel/kernel.h @@ -4,14 +4,18 @@ #pragma once +#include <array> #include <memory> #include <string> #include <unordered_map> #include <vector> +#include "core/arm/cpu_interrupt_handler.h" +#include "core/hardware_properties.h" +#include "core/hle/kernel/memory/memory_types.h" #include "core/hle/kernel/object.h" namespace Core { -struct EmuThreadHandle; +class CPUInterruptHandler; class ExclusiveMonitor; class System; } // namespace Core @@ -23,6 +27,12 @@ struct EventType; namespace Kernel { +namespace Memory { +class MemoryManager; +template <typename T> +class SlabHeap; +} // namespace Memory + class AddressArbiter; class ClientPort; class GlobalScheduler; @@ -31,6 +41,7 @@ class PhysicalCore; class Process; class ResourceLimit; class Scheduler; +class SharedMemory; class Synchronization; class Thread; class TimeManager; @@ -57,6 +68,9 @@ public: KernelCore(KernelCore&&) = delete; KernelCore& operator=(KernelCore&&) = delete; + /// Sets if emulation is multicore or single core, must be set before Initialize + void SetMulticore(bool is_multicore); + /// Resets the kernel to a clean slate for use. void Initialize(); @@ -102,6 +116,18 @@ public: /// Gets the an instance of the respective physical CPU core. const Kernel::PhysicalCore& PhysicalCore(std::size_t id) const; + /// Gets the sole instance of the Scheduler at the current running core. + Kernel::Scheduler& CurrentScheduler(); + + /// Gets the sole instance of the Scheduler at the current running core. + const Kernel::Scheduler& CurrentScheduler() const; + + /// Gets the an instance of the current physical CPU core. + Kernel::PhysicalCore& CurrentPhysicalCore(); + + /// Gets the an instance of the current physical CPU core. + const Kernel::PhysicalCore& CurrentPhysicalCore() const; + /// Gets the an instance of the Synchronization Interface. Kernel::Synchronization& Synchronization(); @@ -121,6 +147,10 @@ public: const Core::ExclusiveMonitor& GetExclusiveMonitor() const; + std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& Interrupts(); + + const std::array<Core::CPUInterruptHandler, Core::Hardware::NUM_CPU_CORES>& Interrupts() const; + void InvalidateAllInstructionCaches(); /// Adds a port to the named port table @@ -147,6 +177,54 @@ public: /// Register the current thread as a non CPU core thread. void RegisterHostThread(); + /// Gets the virtual memory manager for the kernel. + Memory::MemoryManager& MemoryManager(); + + /// Gets the virtual memory manager for the kernel. + const Memory::MemoryManager& MemoryManager() const; + + /// Gets the slab heap allocated for user space pages. + Memory::SlabHeap<Memory::Page>& GetUserSlabHeapPages(); + + /// Gets the slab heap allocated for user space pages. + const Memory::SlabHeap<Memory::Page>& GetUserSlabHeapPages() const; + + /// Gets the shared memory object for HID services. + Kernel::SharedMemory& GetHidSharedMem(); + + /// Gets the shared memory object for HID services. + const Kernel::SharedMemory& GetHidSharedMem() const; + + /// Gets the shared memory object for font services. + Kernel::SharedMemory& GetFontSharedMem(); + + /// Gets the shared memory object for font services. + const Kernel::SharedMemory& GetFontSharedMem() const; + + /// Gets the shared memory object for IRS services. + Kernel::SharedMemory& GetIrsSharedMem(); + + /// Gets the shared memory object for IRS services. + const Kernel::SharedMemory& GetIrsSharedMem() const; + + /// Gets the shared memory object for Time services. + Kernel::SharedMemory& GetTimeSharedMem(); + + /// Gets the shared memory object for Time services. + const Kernel::SharedMemory& GetTimeSharedMem() const; + + /// Suspend/unsuspend the OS. + void Suspend(bool in_suspention); + + /// Exceptional exit the OS. + void ExceptionalExit(); + + bool IsMulticore() const; + + void EnterSVCProfile(); + + void ExitSVCProfile(); + private: friend class Object; friend class Process; @@ -164,9 +242,6 @@ private: /// Creates a new thread ID, incrementing the internal thread ID counter. u64 CreateNewThreadID(); - /// Retrieves the event type used for thread wakeup callbacks. - const std::shared_ptr<Core::Timing::EventType>& ThreadWakeupCallbackEventType() const; - /// Provides a reference to the global handle table. Kernel::HandleTable& GlobalHandleTable(); @@ -175,6 +250,7 @@ private: struct Impl; std::unique_ptr<Impl> impl; + bool exception_exited{}; }; } // namespace Kernel diff --git a/src/core/hle/kernel/memory/address_space_info.cpp b/src/core/hle/kernel/memory/address_space_info.cpp new file mode 100644 index 000000000..e4288cab4 --- /dev/null +++ b/src/core/hle/kernel/memory/address_space_info.cpp @@ -0,0 +1,117 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +// This file references various implementation details from Atmosphere, an open-source firmware for +// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. + +#include <array> + +#include "common/assert.h" +#include "core/hle/kernel/memory/address_space_info.h" + +namespace Kernel::Memory { + +namespace { + +enum : u64 { + Size_1_MB = 0x100000, + Size_2_MB = 2 * Size_1_MB, + Size_128_MB = 128 * Size_1_MB, + Size_1_GB = 0x40000000, + Size_2_GB = 2 * Size_1_GB, + Size_4_GB = 4 * Size_1_GB, + Size_6_GB = 6 * Size_1_GB, + Size_64_GB = 64 * Size_1_GB, + Size_512_GB = 512 * Size_1_GB, + Invalid = std::numeric_limits<u64>::max(), +}; + +// clang-format off +constexpr std::array<AddressSpaceInfo, 13> AddressSpaceInfos{{ + { .bit_width = 32, .address = Size_2_MB , .size = Size_1_GB - Size_2_MB , .type = AddressSpaceInfo::Type::Is32Bit, }, + { .bit_width = 32, .address = Size_1_GB , .size = Size_4_GB - Size_1_GB , .type = AddressSpaceInfo::Type::Small64Bit, }, + { .bit_width = 32, .address = Invalid , .size = Size_1_GB , .type = AddressSpaceInfo::Type::Heap, }, + { .bit_width = 32, .address = Invalid , .size = Size_1_GB , .type = AddressSpaceInfo::Type::Alias, }, + { .bit_width = 36, .address = Size_128_MB, .size = Size_2_GB - Size_128_MB, .type = AddressSpaceInfo::Type::Is32Bit, }, + { .bit_width = 36, .address = Size_2_GB , .size = Size_64_GB - Size_2_GB , .type = AddressSpaceInfo::Type::Small64Bit, }, + { .bit_width = 36, .address = Invalid , .size = Size_6_GB , .type = AddressSpaceInfo::Type::Heap, }, + { .bit_width = 36, .address = Invalid , .size = Size_6_GB , .type = AddressSpaceInfo::Type::Alias, }, + { .bit_width = 39, .address = Size_128_MB, .size = Size_512_GB - Size_128_MB, .type = AddressSpaceInfo::Type::Large64Bit, }, + { .bit_width = 39, .address = Invalid , .size = Size_64_GB , .type = AddressSpaceInfo::Type::Is32Bit }, + { .bit_width = 39, .address = Invalid , .size = Size_6_GB , .type = AddressSpaceInfo::Type::Heap, }, + { .bit_width = 39, .address = Invalid , .size = Size_64_GB , .type = AddressSpaceInfo::Type::Alias, }, + { .bit_width = 39, .address = Invalid , .size = Size_2_GB , .type = AddressSpaceInfo::Type::Stack, }, +}}; +// clang-format on + +constexpr bool IsAllowedIndexForAddress(std::size_t index) { + return index < AddressSpaceInfos.size() && AddressSpaceInfos[index].address != Invalid; +} + +using IndexArray = std::array<std::size_t, static_cast<std::size_t>(AddressSpaceInfo::Type::Count)>; + +constexpr IndexArray AddressSpaceIndices32Bit{ + 0, 1, 0, 2, 0, 3, +}; + +constexpr IndexArray AddressSpaceIndices36Bit{ + 4, 5, 4, 6, 4, 7, +}; + +constexpr IndexArray AddressSpaceIndices39Bit{ + 9, 8, 8, 10, 12, 11, +}; + +constexpr bool IsAllowed32BitType(AddressSpaceInfo::Type type) { + return type < AddressSpaceInfo::Type::Count && type != AddressSpaceInfo::Type::Large64Bit && + type != AddressSpaceInfo::Type::Stack; +} + +constexpr bool IsAllowed36BitType(AddressSpaceInfo::Type type) { + return type < AddressSpaceInfo::Type::Count && type != AddressSpaceInfo::Type::Large64Bit && + type != AddressSpaceInfo::Type::Stack; +} + +constexpr bool IsAllowed39BitType(AddressSpaceInfo::Type type) { + return type < AddressSpaceInfo::Type::Count && type != AddressSpaceInfo::Type::Small64Bit; +} + +} // namespace + +u64 AddressSpaceInfo::GetAddressSpaceStart(std::size_t width, Type type) { + const std::size_t index{static_cast<std::size_t>(type)}; + switch (width) { + case 32: + ASSERT(IsAllowed32BitType(type)); + ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices32Bit[index])); + return AddressSpaceInfos[AddressSpaceIndices32Bit[index]].address; + case 36: + ASSERT(IsAllowed36BitType(type)); + ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices36Bit[index])); + return AddressSpaceInfos[AddressSpaceIndices36Bit[index]].address; + case 39: + ASSERT(IsAllowed39BitType(type)); + ASSERT(IsAllowedIndexForAddress(AddressSpaceIndices39Bit[index])); + return AddressSpaceInfos[AddressSpaceIndices39Bit[index]].address; + } + UNREACHABLE(); +} + +std::size_t AddressSpaceInfo::GetAddressSpaceSize(std::size_t width, Type type) { + const std::size_t index{static_cast<std::size_t>(type)}; + switch (width) { + case 32: + ASSERT(IsAllowed32BitType(type)); + return AddressSpaceInfos[AddressSpaceIndices32Bit[index]].size; + case 36: + ASSERT(IsAllowed36BitType(type)); + return AddressSpaceInfos[AddressSpaceIndices36Bit[index]].size; + case 39: + ASSERT(IsAllowed39BitType(type)); + return AddressSpaceInfos[AddressSpaceIndices39Bit[index]].size; + } + UNREACHABLE(); +} + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/address_space_info.h b/src/core/hle/kernel/memory/address_space_info.h new file mode 100644 index 000000000..a4e6e91e5 --- /dev/null +++ b/src/core/hle/kernel/memory/address_space_info.h @@ -0,0 +1,34 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +// This file references various implementation details from Atmosphere, an open-source firmware for +// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. + +#pragma once + +#include "common/common_types.h" + +namespace Kernel::Memory { + +struct AddressSpaceInfo final { + enum class Type : u32 { + Is32Bit = 0, + Small64Bit = 1, + Large64Bit = 2, + Heap = 3, + Stack = 4, + Alias = 5, + Count, + }; + + static u64 GetAddressSpaceStart(std::size_t width, Type type); + static std::size_t GetAddressSpaceSize(std::size_t width, Type type); + + const std::size_t bit_width{}; + const std::size_t address{}; + const std::size_t size{}; + const Type type{}; +}; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_block.h b/src/core/hle/kernel/memory/memory_block.h new file mode 100644 index 000000000..9d7839d08 --- /dev/null +++ b/src/core/hle/kernel/memory/memory_block.h @@ -0,0 +1,335 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +// This file references various implementation details from Atmosphere, an open-source firmware for +// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. + +#pragma once + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/common_types.h" +#include "core/hle/kernel/memory/memory_types.h" +#include "core/hle/kernel/svc_types.h" + +namespace Kernel::Memory { + +enum class MemoryState : u32 { + None = 0, + Mask = 0xFF, + All = ~None, + + FlagCanReprotect = (1 << 8), + FlagCanDebug = (1 << 9), + FlagCanUseIpc = (1 << 10), + FlagCanUseNonDeviceIpc = (1 << 11), + FlagCanUseNonSecureIpc = (1 << 12), + FlagMapped = (1 << 13), + FlagCode = (1 << 14), + FlagCanAlias = (1 << 15), + FlagCanCodeAlias = (1 << 16), + FlagCanTransfer = (1 << 17), + FlagCanQueryPhysical = (1 << 18), + FlagCanDeviceMap = (1 << 19), + FlagCanAlignedDeviceMap = (1 << 20), + FlagCanIpcUserBuffer = (1 << 21), + FlagReferenceCounted = (1 << 22), + FlagCanMapProcess = (1 << 23), + FlagCanChangeAttribute = (1 << 24), + FlagCanCodeMemory = (1 << 25), + + FlagsData = FlagCanReprotect | FlagCanUseIpc | FlagCanUseNonDeviceIpc | FlagCanUseNonSecureIpc | + FlagMapped | FlagCanAlias | FlagCanTransfer | FlagCanQueryPhysical | + FlagCanDeviceMap | FlagCanAlignedDeviceMap | FlagCanIpcUserBuffer | + FlagReferenceCounted | FlagCanChangeAttribute, + + FlagsCode = FlagCanDebug | FlagCanUseIpc | FlagCanUseNonDeviceIpc | FlagCanUseNonSecureIpc | + FlagMapped | FlagCode | FlagCanQueryPhysical | FlagCanDeviceMap | + FlagCanAlignedDeviceMap | FlagReferenceCounted, + + FlagsMisc = FlagMapped | FlagReferenceCounted | FlagCanQueryPhysical | FlagCanDeviceMap, + + Free = static_cast<u32>(Svc::MemoryState::Free), + Io = static_cast<u32>(Svc::MemoryState::Io) | FlagMapped, + Static = static_cast<u32>(Svc::MemoryState::Static) | FlagMapped | FlagCanQueryPhysical, + Code = static_cast<u32>(Svc::MemoryState::Code) | FlagsCode | FlagCanMapProcess, + CodeData = static_cast<u32>(Svc::MemoryState::CodeData) | FlagsData | FlagCanMapProcess | + FlagCanCodeMemory, + Shared = static_cast<u32>(Svc::MemoryState::Shared) | FlagMapped | FlagReferenceCounted, + Normal = static_cast<u32>(Svc::MemoryState::Normal) | FlagsData | FlagCanCodeMemory, + + AliasCode = static_cast<u32>(Svc::MemoryState::AliasCode) | FlagsCode | FlagCanMapProcess | + FlagCanCodeAlias, + AliasCodeData = static_cast<u32>(Svc::MemoryState::AliasCodeData) | FlagsData | + FlagCanMapProcess | FlagCanCodeAlias | FlagCanCodeMemory, + + Ipc = static_cast<u32>(Svc::MemoryState::Ipc) | FlagsMisc | FlagCanAlignedDeviceMap | + FlagCanUseIpc | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + Stack = static_cast<u32>(Svc::MemoryState::Stack) | FlagsMisc | FlagCanAlignedDeviceMap | + FlagCanUseIpc | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + ThreadLocal = + static_cast<u32>(Svc::MemoryState::ThreadLocal) | FlagMapped | FlagReferenceCounted, + + Transfered = static_cast<u32>(Svc::MemoryState::Transfered) | FlagsMisc | + FlagCanAlignedDeviceMap | FlagCanChangeAttribute | FlagCanUseIpc | + FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + SharedTransfered = static_cast<u32>(Svc::MemoryState::SharedTransfered) | FlagsMisc | + FlagCanAlignedDeviceMap | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + SharedCode = static_cast<u32>(Svc::MemoryState::SharedCode) | FlagMapped | + FlagReferenceCounted | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + Inaccessible = static_cast<u32>(Svc::MemoryState::Inaccessible), + + NonSecureIpc = static_cast<u32>(Svc::MemoryState::NonSecureIpc) | FlagsMisc | + FlagCanAlignedDeviceMap | FlagCanUseNonSecureIpc | FlagCanUseNonDeviceIpc, + + NonDeviceIpc = + static_cast<u32>(Svc::MemoryState::NonDeviceIpc) | FlagsMisc | FlagCanUseNonDeviceIpc, + + Kernel = static_cast<u32>(Svc::MemoryState::Kernel) | FlagMapped, + + GeneratedCode = static_cast<u32>(Svc::MemoryState::GeneratedCode) | FlagMapped | + FlagReferenceCounted | FlagCanDebug, + CodeOut = static_cast<u32>(Svc::MemoryState::CodeOut) | FlagMapped | FlagReferenceCounted, +}; +DECLARE_ENUM_FLAG_OPERATORS(MemoryState); + +static_assert(static_cast<u32>(MemoryState::Free) == 0x00000000); +static_assert(static_cast<u32>(MemoryState::Io) == 0x00002001); +static_assert(static_cast<u32>(MemoryState::Static) == 0x00042002); +static_assert(static_cast<u32>(MemoryState::Code) == 0x00DC7E03); +static_assert(static_cast<u32>(MemoryState::CodeData) == 0x03FEBD04); +static_assert(static_cast<u32>(MemoryState::Normal) == 0x037EBD05); +static_assert(static_cast<u32>(MemoryState::Shared) == 0x00402006); +static_assert(static_cast<u32>(MemoryState::AliasCode) == 0x00DD7E08); +static_assert(static_cast<u32>(MemoryState::AliasCodeData) == 0x03FFBD09); +static_assert(static_cast<u32>(MemoryState::Ipc) == 0x005C3C0A); +static_assert(static_cast<u32>(MemoryState::Stack) == 0x005C3C0B); +static_assert(static_cast<u32>(MemoryState::ThreadLocal) == 0x0040200C); +static_assert(static_cast<u32>(MemoryState::Transfered) == 0x015C3C0D); +static_assert(static_cast<u32>(MemoryState::SharedTransfered) == 0x005C380E); +static_assert(static_cast<u32>(MemoryState::SharedCode) == 0x0040380F); +static_assert(static_cast<u32>(MemoryState::Inaccessible) == 0x00000010); +static_assert(static_cast<u32>(MemoryState::NonSecureIpc) == 0x005C3811); +static_assert(static_cast<u32>(MemoryState::NonDeviceIpc) == 0x004C2812); +static_assert(static_cast<u32>(MemoryState::Kernel) == 0x00002013); +static_assert(static_cast<u32>(MemoryState::GeneratedCode) == 0x00402214); +static_assert(static_cast<u32>(MemoryState::CodeOut) == 0x00402015); + +enum class MemoryPermission : u8 { + None = 0, + Mask = static_cast<u8>(~None), + + Read = 1 << 0, + Write = 1 << 1, + Execute = 1 << 2, + + ReadAndWrite = Read | Write, + ReadAndExecute = Read | Execute, + + UserMask = static_cast<u8>(Svc::MemoryPermission::Read | Svc::MemoryPermission::Write | + Svc::MemoryPermission::Execute), +}; +DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission); + +enum class MemoryAttribute : u8 { + None = 0x00, + Mask = 0x7F, + All = Mask, + DontCareMask = 0x80, + + Locked = static_cast<u8>(Svc::MemoryAttribute::Locked), + IpcLocked = static_cast<u8>(Svc::MemoryAttribute::IpcLocked), + DeviceShared = static_cast<u8>(Svc::MemoryAttribute::DeviceShared), + Uncached = static_cast<u8>(Svc::MemoryAttribute::Uncached), + + IpcAndDeviceMapped = IpcLocked | DeviceShared, + LockedAndIpcLocked = Locked | IpcLocked, + DeviceSharedAndUncached = DeviceShared | Uncached +}; +DECLARE_ENUM_FLAG_OPERATORS(MemoryAttribute); + +static_assert((static_cast<u8>(MemoryAttribute::Mask) & + static_cast<u8>(MemoryAttribute::DontCareMask)) == 0); + +struct MemoryInfo { + VAddr addr{}; + std::size_t size{}; + MemoryState state{}; + MemoryPermission perm{}; + MemoryAttribute attribute{}; + MemoryPermission original_perm{}; + u16 ipc_lock_count{}; + u16 device_use_count{}; + + constexpr Svc::MemoryInfo GetSvcMemoryInfo() const { + return { + addr, + size, + static_cast<Svc::MemoryState>(state & MemoryState::Mask), + static_cast<Svc::MemoryAttribute>(attribute & MemoryAttribute::Mask), + static_cast<Svc::MemoryPermission>(perm & MemoryPermission::UserMask), + ipc_lock_count, + device_use_count, + }; + } + + constexpr VAddr GetAddress() const { + return addr; + } + constexpr std::size_t GetSize() const { + return size; + } + constexpr std::size_t GetNumPages() const { + return GetSize() / PageSize; + } + constexpr VAddr GetEndAddress() const { + return GetAddress() + GetSize(); + } + constexpr VAddr GetLastAddress() const { + return GetEndAddress() - 1; + } +}; + +class MemoryBlock final { + friend class MemoryBlockManager; + +private: + VAddr addr{}; + std::size_t num_pages{}; + MemoryState state{MemoryState::None}; + u16 ipc_lock_count{}; + u16 device_use_count{}; + MemoryPermission perm{MemoryPermission::None}; + MemoryPermission original_perm{MemoryPermission::None}; + MemoryAttribute attribute{MemoryAttribute::None}; + +public: + static constexpr int Compare(const MemoryBlock& lhs, const MemoryBlock& rhs) { + if (lhs.GetAddress() < rhs.GetAddress()) { + return -1; + } else if (lhs.GetAddress() <= rhs.GetLastAddress()) { + return 0; + } else { + return 1; + } + } + +public: + constexpr MemoryBlock() = default; + constexpr MemoryBlock(VAddr addr, std::size_t num_pages, MemoryState state, + MemoryPermission perm, MemoryAttribute attribute) + : addr{addr}, num_pages(num_pages), state{state}, perm{perm}, attribute{attribute} {} + + constexpr VAddr GetAddress() const { + return addr; + } + + constexpr std::size_t GetNumPages() const { + return num_pages; + } + + constexpr std::size_t GetSize() const { + return GetNumPages() * PageSize; + } + + constexpr VAddr GetEndAddress() const { + return GetAddress() + GetSize(); + } + + constexpr VAddr GetLastAddress() const { + return GetEndAddress() - 1; + } + + constexpr MemoryInfo GetMemoryInfo() const { + return { + GetAddress(), GetSize(), state, perm, + attribute, original_perm, ipc_lock_count, device_use_count, + }; + } + + void ShareToDevice(MemoryPermission /*new_perm*/) { + ASSERT((attribute & MemoryAttribute::DeviceShared) == MemoryAttribute::DeviceShared || + device_use_count == 0); + attribute |= MemoryAttribute::DeviceShared; + const u16 new_use_count{++device_use_count}; + ASSERT(new_use_count > 0); + } + + void UnshareToDevice(MemoryPermission /*new_perm*/) { + ASSERT((attribute & MemoryAttribute::DeviceShared) == MemoryAttribute::DeviceShared); + const u16 prev_use_count{device_use_count--}; + ASSERT(prev_use_count > 0); + if (prev_use_count == 1) { + attribute &= ~MemoryAttribute::DeviceShared; + } + } + +private: + constexpr bool HasProperties(MemoryState s, MemoryPermission p, MemoryAttribute a) const { + constexpr MemoryAttribute AttributeIgnoreMask{MemoryAttribute::DontCareMask | + MemoryAttribute::IpcLocked | + MemoryAttribute::DeviceShared}; + return state == s && perm == p && + (attribute | AttributeIgnoreMask) == (a | AttributeIgnoreMask); + } + + constexpr bool HasSameProperties(const MemoryBlock& rhs) const { + return state == rhs.state && perm == rhs.perm && original_perm == rhs.original_perm && + attribute == rhs.attribute && ipc_lock_count == rhs.ipc_lock_count && + device_use_count == rhs.device_use_count; + } + + constexpr bool Contains(VAddr start) const { + return GetAddress() <= start && start <= GetEndAddress(); + } + + constexpr void Add(std::size_t count) { + ASSERT(count > 0); + ASSERT(GetAddress() + count * PageSize - 1 < GetEndAddress() + count * PageSize - 1); + + num_pages += count; + } + + constexpr void Update(MemoryState new_state, MemoryPermission new_perm, + MemoryAttribute new_attribute) { + ASSERT(original_perm == MemoryPermission::None); + ASSERT((attribute & MemoryAttribute::IpcLocked) == MemoryAttribute::None); + + state = new_state; + perm = new_perm; + + attribute = static_cast<MemoryAttribute>( + new_attribute | + (attribute & (MemoryAttribute::IpcLocked | MemoryAttribute::DeviceShared))); + } + + constexpr MemoryBlock Split(VAddr split_addr) { + ASSERT(GetAddress() < split_addr); + ASSERT(Contains(split_addr)); + ASSERT(Common::IsAligned(split_addr, PageSize)); + + MemoryBlock block; + block.addr = addr; + block.num_pages = (split_addr - GetAddress()) / PageSize; + block.state = state; + block.ipc_lock_count = ipc_lock_count; + block.device_use_count = device_use_count; + block.perm = perm; + block.original_perm = original_perm; + block.attribute = attribute; + + addr = split_addr; + num_pages -= block.num_pages; + + return block; + } +}; +static_assert(std::is_trivially_destructible<MemoryBlock>::value); + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_block_manager.cpp b/src/core/hle/kernel/memory/memory_block_manager.cpp new file mode 100644 index 000000000..0732fa5a1 --- /dev/null +++ b/src/core/hle/kernel/memory/memory_block_manager.cpp @@ -0,0 +1,223 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "core/hle/kernel/memory/memory_block_manager.h" +#include "core/hle/kernel/memory/memory_types.h" + +namespace Kernel::Memory { + +MemoryBlockManager::MemoryBlockManager(VAddr start_addr, VAddr end_addr) + : start_addr{start_addr}, end_addr{end_addr} { + const u64 num_pages{(end_addr - start_addr) / PageSize}; + memory_block_tree.emplace_back(start_addr, num_pages, MemoryState::Free, MemoryPermission::None, + MemoryAttribute::None); +} + +MemoryBlockManager::iterator MemoryBlockManager::FindIterator(VAddr addr) { + auto node{memory_block_tree.begin()}; + while (node != end()) { + const VAddr end_addr{node->GetNumPages() * PageSize + node->GetAddress()}; + if (node->GetAddress() <= addr && end_addr - 1 >= addr) { + return node; + } + node = std::next(node); + } + return end(); +} + +VAddr MemoryBlockManager::FindFreeArea(VAddr region_start, std::size_t region_num_pages, + std::size_t num_pages, std::size_t align, std::size_t offset, + std::size_t guard_pages) { + if (num_pages == 0) { + return {}; + } + + const VAddr region_end{region_start + region_num_pages * PageSize}; + const VAddr region_last{region_end - 1}; + for (auto it{FindIterator(region_start)}; it != memory_block_tree.cend(); it++) { + const auto info{it->GetMemoryInfo()}; + if (region_last < info.GetAddress()) { + break; + } + + if (info.state != MemoryState::Free) { + continue; + } + + VAddr area{(info.GetAddress() <= region_start) ? region_start : info.GetAddress()}; + area += guard_pages * PageSize; + + const VAddr offset_area{Common::AlignDown(area, align) + offset}; + area = (area <= offset_area) ? offset_area : offset_area + align; + + const VAddr area_end{area + num_pages * PageSize + guard_pages * PageSize}; + const VAddr area_last{area_end - 1}; + + if (info.GetAddress() <= area && area < area_last && area_last <= region_last && + area_last <= info.GetLastAddress()) { + return area; + } + } + + return {}; +} + +void MemoryBlockManager::Update(VAddr addr, std::size_t num_pages, MemoryState prev_state, + MemoryPermission prev_perm, MemoryAttribute prev_attribute, + MemoryState state, MemoryPermission perm, + MemoryAttribute attribute) { + const VAddr end_addr{addr + num_pages * PageSize}; + iterator node{memory_block_tree.begin()}; + + prev_attribute |= MemoryAttribute::IpcAndDeviceMapped; + + while (node != memory_block_tree.end()) { + MemoryBlock* block{&(*node)}; + iterator next_node{std::next(node)}; + const VAddr cur_addr{block->GetAddress()}; + const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + + if (addr < cur_end_addr && cur_addr < end_addr) { + if (!block->HasProperties(prev_state, prev_perm, prev_attribute)) { + node = next_node; + continue; + } + + iterator new_node{node}; + if (addr > cur_addr) { + memory_block_tree.insert(node, block->Split(addr)); + } + + if (end_addr < cur_end_addr) { + new_node = memory_block_tree.insert(node, block->Split(end_addr)); + } + + new_node->Update(state, perm, attribute); + + MergeAdjacent(new_node, next_node); + } + + if (cur_end_addr - 1 >= end_addr - 1) { + break; + } + + node = next_node; + } +} + +void MemoryBlockManager::Update(VAddr addr, std::size_t num_pages, MemoryState state, + MemoryPermission perm, MemoryAttribute attribute) { + const VAddr end_addr{addr + num_pages * PageSize}; + iterator node{memory_block_tree.begin()}; + + while (node != memory_block_tree.end()) { + MemoryBlock* block{&(*node)}; + iterator next_node{std::next(node)}; + const VAddr cur_addr{block->GetAddress()}; + const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + + if (addr < cur_end_addr && cur_addr < end_addr) { + iterator new_node{node}; + + if (addr > cur_addr) { + memory_block_tree.insert(node, block->Split(addr)); + } + + if (end_addr < cur_end_addr) { + new_node = memory_block_tree.insert(node, block->Split(end_addr)); + } + + new_node->Update(state, perm, attribute); + + MergeAdjacent(new_node, next_node); + } + + if (cur_end_addr - 1 >= end_addr - 1) { + break; + } + + node = next_node; + } +} + +void MemoryBlockManager::UpdateLock(VAddr addr, std::size_t num_pages, LockFunc&& lock_func, + MemoryPermission perm) { + const VAddr end_addr{addr + num_pages * PageSize}; + iterator node{memory_block_tree.begin()}; + + while (node != memory_block_tree.end()) { + MemoryBlock* block{&(*node)}; + iterator next_node{std::next(node)}; + const VAddr cur_addr{block->GetAddress()}; + const VAddr cur_end_addr{block->GetNumPages() * PageSize + cur_addr}; + + if (addr < cur_end_addr && cur_addr < end_addr) { + iterator new_node{node}; + + if (addr > cur_addr) { + memory_block_tree.insert(node, block->Split(addr)); + } + + if (end_addr < cur_end_addr) { + new_node = memory_block_tree.insert(node, block->Split(end_addr)); + } + + lock_func(new_node, perm); + + MergeAdjacent(new_node, next_node); + } + + if (cur_end_addr - 1 >= end_addr - 1) { + break; + } + + node = next_node; + } +} + +void MemoryBlockManager::IterateForRange(VAddr start, VAddr end, IterateFunc&& func) { + const_iterator it{FindIterator(start)}; + MemoryInfo info{}; + do { + info = it->GetMemoryInfo(); + func(info); + it = std::next(it); + } while (info.addr + info.size - 1 < end - 1 && it != cend()); +} + +void MemoryBlockManager::MergeAdjacent(iterator it, iterator& next_it) { + MemoryBlock* block{&(*it)}; + + auto EraseIt = [&](const iterator it_to_erase) { + if (next_it == it_to_erase) { + next_it = std::next(next_it); + } + memory_block_tree.erase(it_to_erase); + }; + + if (it != memory_block_tree.begin()) { + MemoryBlock* prev{&(*std::prev(it))}; + + if (block->HasSameProperties(*prev)) { + const iterator prev_it{std::prev(it)}; + + prev->Add(block->GetNumPages()); + EraseIt(it); + + it = prev_it; + block = prev; + } + } + + if (it != cend()) { + const MemoryBlock* const next{&(*std::next(it))}; + + if (block->HasSameProperties(*next)) { + block->Add(next->GetNumPages()); + EraseIt(std::next(it)); + } + } +} + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_block_manager.h b/src/core/hle/kernel/memory/memory_block_manager.h new file mode 100644 index 000000000..6e1d41075 --- /dev/null +++ b/src/core/hle/kernel/memory/memory_block_manager.h @@ -0,0 +1,66 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include <functional> +#include <list> + +#include "common/common_types.h" +#include "core/hle/kernel/memory/memory_block.h" + +namespace Kernel::Memory { + +class MemoryBlockManager final { +public: + using MemoryBlockTree = std::list<MemoryBlock>; + using iterator = MemoryBlockTree::iterator; + using const_iterator = MemoryBlockTree::const_iterator; + +public: + MemoryBlockManager(VAddr start_addr, VAddr end_addr); + + iterator end() { + return memory_block_tree.end(); + } + const_iterator end() const { + return memory_block_tree.end(); + } + const_iterator cend() const { + return memory_block_tree.cend(); + } + + iterator FindIterator(VAddr addr); + + VAddr FindFreeArea(VAddr region_start, std::size_t region_num_pages, std::size_t num_pages, + std::size_t align, std::size_t offset, std::size_t guard_pages); + + void Update(VAddr addr, std::size_t num_pages, MemoryState prev_state, + MemoryPermission prev_perm, MemoryAttribute prev_attribute, MemoryState state, + MemoryPermission perm, MemoryAttribute attribute); + + void Update(VAddr addr, std::size_t num_pages, MemoryState state, + MemoryPermission perm = MemoryPermission::None, + MemoryAttribute attribute = MemoryAttribute::None); + + using LockFunc = std::function<void(iterator, MemoryPermission)>; + void UpdateLock(VAddr addr, std::size_t num_pages, LockFunc&& lock_func, MemoryPermission perm); + + using IterateFunc = std::function<void(const MemoryInfo&)>; + void IterateForRange(VAddr start, VAddr end, IterateFunc&& func); + + MemoryBlock& FindBlock(VAddr addr) { + return *FindIterator(addr); + } + +private: + void MergeAdjacent(iterator it, iterator& next_it); + + const VAddr start_addr; + const VAddr end_addr; + + MemoryBlockTree memory_block_tree; +}; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_layout.h b/src/core/hle/kernel/memory/memory_layout.h new file mode 100644 index 000000000..9b3d6267a --- /dev/null +++ b/src/core/hle/kernel/memory/memory_layout.h @@ -0,0 +1,71 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include "common/common_types.h" + +namespace Kernel::Memory { + +class MemoryRegion final { + friend class MemoryLayout; + +public: + constexpr PAddr StartAddress() const { + return start_address; + } + + constexpr PAddr EndAddress() const { + return end_address; + } + +private: + constexpr MemoryRegion() = default; + constexpr MemoryRegion(PAddr start_address, PAddr end_address) + : start_address{start_address}, end_address{end_address} {} + + const PAddr start_address{}; + const PAddr end_address{}; +}; + +class MemoryLayout final { +public: + constexpr const MemoryRegion& Application() const { + return application; + } + + constexpr const MemoryRegion& Applet() const { + return applet; + } + + constexpr const MemoryRegion& System() const { + return system; + } + + static constexpr MemoryLayout GetDefaultLayout() { + constexpr std::size_t application_size{0xcd500000}; + constexpr std::size_t applet_size{0x1fb00000}; + constexpr PAddr application_start_address{Core::DramMemoryMap::End - application_size}; + constexpr PAddr application_end_address{Core::DramMemoryMap::End}; + constexpr PAddr applet_start_address{application_start_address - applet_size}; + constexpr PAddr applet_end_address{applet_start_address + applet_size}; + constexpr PAddr system_start_address{Core::DramMemoryMap::SlabHeapEnd}; + constexpr PAddr system_end_address{applet_start_address}; + return {application_start_address, application_end_address, applet_start_address, + applet_end_address, system_start_address, system_end_address}; + } + +private: + constexpr MemoryLayout(PAddr application_start_address, std::size_t application_size, + PAddr applet_start_address, std::size_t applet_size, + PAddr system_start_address, std::size_t system_size) + : application{application_start_address, application_size}, + applet{applet_start_address, applet_size}, system{system_start_address, system_size} {} + + const MemoryRegion application; + const MemoryRegion applet; + const MemoryRegion system; +}; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_manager.cpp b/src/core/hle/kernel/memory/memory_manager.cpp new file mode 100644 index 000000000..acf13585c --- /dev/null +++ b/src/core/hle/kernel/memory/memory_manager.cpp @@ -0,0 +1,175 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include <algorithm> + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/common_types.h" +#include "common/scope_exit.h" +#include "core/hle/kernel/errors.h" +#include "core/hle/kernel/memory/memory_manager.h" +#include "core/hle/kernel/memory/page_linked_list.h" + +namespace Kernel::Memory { + +std::size_t MemoryManager::Impl::Initialize(Pool new_pool, u64 start_address, u64 end_address) { + const auto size{end_address - start_address}; + + // Calculate metadata sizes + const auto ref_count_size{(size / PageSize) * sizeof(u16)}; + const auto optimize_map_size{(Common::AlignUp((size / PageSize), 64) / 64) * sizeof(u64)}; + const auto manager_size{Common::AlignUp(optimize_map_size + ref_count_size, PageSize)}; + const auto page_heap_size{PageHeap::CalculateMetadataOverheadSize(size)}; + const auto total_metadata_size{manager_size + page_heap_size}; + ASSERT(manager_size <= total_metadata_size); + ASSERT(Common::IsAligned(total_metadata_size, PageSize)); + + // Setup region + pool = new_pool; + + // Initialize the manager's KPageHeap + heap.Initialize(start_address, size, page_heap_size); + + // Free the memory to the heap + heap.Free(start_address, size / PageSize); + + // Update the heap's used size + heap.UpdateUsedSize(); + + return total_metadata_size; +} + +void MemoryManager::InitializeManager(Pool pool, u64 start_address, u64 end_address) { + ASSERT(pool < Pool::Count); + managers[static_cast<std::size_t>(pool)].Initialize(pool, start_address, end_address); +} + +VAddr MemoryManager::AllocateContinuous(std::size_t num_pages, std::size_t align_pages, Pool pool, + Direction dir) { + // Early return if we're allocating no pages + if (num_pages == 0) { + return {}; + } + + // Lock the pool that we're allocating from + const auto pool_index{static_cast<std::size_t>(pool)}; + std::lock_guard lock{pool_locks[pool_index]}; + + // Choose a heap based on our page size request + const s32 heap_index{PageHeap::GetAlignedBlockIndex(num_pages, align_pages)}; + + // Loop, trying to iterate from each block + // TODO (bunnei): Support multiple managers + Impl& chosen_manager{managers[pool_index]}; + VAddr allocated_block{chosen_manager.AllocateBlock(heap_index)}; + + // If we failed to allocate, quit now + if (!allocated_block) { + return {}; + } + + // If we allocated more than we need, free some + const auto allocated_pages{PageHeap::GetBlockNumPages(heap_index)}; + if (allocated_pages > num_pages) { + chosen_manager.Free(allocated_block + num_pages * PageSize, allocated_pages - num_pages); + } + + return allocated_block; +} + +ResultCode MemoryManager::Allocate(PageLinkedList& page_list, std::size_t num_pages, Pool pool, + Direction dir) { + ASSERT(page_list.GetNumPages() == 0); + + // Early return if we're allocating no pages + if (num_pages == 0) { + return RESULT_SUCCESS; + } + + // Lock the pool that we're allocating from + const auto pool_index{static_cast<std::size_t>(pool)}; + std::lock_guard lock{pool_locks[pool_index]}; + + // Choose a heap based on our page size request + const s32 heap_index{PageHeap::GetBlockIndex(num_pages)}; + if (heap_index < 0) { + return ERR_OUT_OF_MEMORY; + } + + // TODO (bunnei): Support multiple managers + Impl& chosen_manager{managers[pool_index]}; + + // Ensure that we don't leave anything un-freed + auto group_guard = detail::ScopeExit([&] { + for (const auto& it : page_list.Nodes()) { + const auto min_num_pages{std::min<size_t>( + it.GetNumPages(), (chosen_manager.GetEndAddress() - it.GetAddress()) / PageSize)}; + chosen_manager.Free(it.GetAddress(), min_num_pages); + } + }); + + // Keep allocating until we've allocated all our pages + for (s32 index{heap_index}; index >= 0 && num_pages > 0; index--) { + const auto pages_per_alloc{PageHeap::GetBlockNumPages(index)}; + + while (num_pages >= pages_per_alloc) { + // Allocate a block + VAddr allocated_block{chosen_manager.AllocateBlock(index)}; + if (!allocated_block) { + break; + } + + // Safely add it to our group + { + auto block_guard = detail::ScopeExit( + [&] { chosen_manager.Free(allocated_block, pages_per_alloc); }); + + if (const ResultCode result{page_list.AddBlock(allocated_block, pages_per_alloc)}; + result.IsError()) { + return result; + } + + block_guard.Cancel(); + } + + num_pages -= pages_per_alloc; + } + } + + // Only succeed if we allocated as many pages as we wanted + if (num_pages) { + return ERR_OUT_OF_MEMORY; + } + + // We succeeded! + group_guard.Cancel(); + return RESULT_SUCCESS; +} + +ResultCode MemoryManager::Free(PageLinkedList& page_list, std::size_t num_pages, Pool pool, + Direction dir) { + // Early return if we're freeing no pages + if (!num_pages) { + return RESULT_SUCCESS; + } + + // Lock the pool that we're freeing from + const auto pool_index{static_cast<std::size_t>(pool)}; + std::lock_guard lock{pool_locks[pool_index]}; + + // TODO (bunnei): Support multiple managers + Impl& chosen_manager{managers[pool_index]}; + + // Free all of the pages + for (const auto& it : page_list.Nodes()) { + const auto min_num_pages{std::min<size_t>( + it.GetNumPages(), (chosen_manager.GetEndAddress() - it.GetAddress()) / PageSize)}; + chosen_manager.Free(it.GetAddress(), min_num_pages); + } + + return RESULT_SUCCESS; +} + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_manager.h b/src/core/hle/kernel/memory/memory_manager.h new file mode 100644 index 000000000..3cf444857 --- /dev/null +++ b/src/core/hle/kernel/memory/memory_manager.h @@ -0,0 +1,96 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include <array> +#include <mutex> + +#include "common/common_types.h" +#include "core/hle/kernel/memory/page_heap.h" +#include "core/hle/result.h" + +namespace Kernel::Memory { + +class PageLinkedList; + +class MemoryManager final : NonCopyable { +public: + enum class Pool : u32 { + Application = 0, + Applet = 1, + System = 2, + SystemNonSecure = 3, + + Count, + + Shift = 4, + Mask = (0xF << Shift), + }; + + enum class Direction : u32 { + FromFront = 0, + FromBack = 1, + + Shift = 0, + Mask = (0xF << Shift), + }; + + MemoryManager() = default; + + constexpr std::size_t GetSize(Pool pool) const { + return managers[static_cast<std::size_t>(pool)].GetSize(); + } + + void InitializeManager(Pool pool, u64 start_address, u64 end_address); + VAddr AllocateContinuous(std::size_t num_pages, std::size_t align_pages, Pool pool, + Direction dir = Direction::FromFront); + ResultCode Allocate(PageLinkedList& page_list, std::size_t num_pages, Pool pool, + Direction dir = Direction::FromFront); + ResultCode Free(PageLinkedList& page_list, std::size_t num_pages, Pool pool, + Direction dir = Direction::FromFront); + + static constexpr std::size_t MaxManagerCount = 10; + +private: + class Impl final : NonCopyable { + private: + using RefCount = u16; + + private: + PageHeap heap; + Pool pool{}; + + public: + Impl() = default; + + std::size_t Initialize(Pool new_pool, u64 start_address, u64 end_address); + + VAddr AllocateBlock(s32 index) { + return heap.AllocateBlock(index); + } + + void Free(VAddr addr, std::size_t num_pages) { + heap.Free(addr, num_pages); + } + + constexpr std::size_t GetSize() const { + return heap.GetSize(); + } + + constexpr VAddr GetAddress() const { + return heap.GetAddress(); + } + + constexpr VAddr GetEndAddress() const { + return heap.GetEndAddress(); + } + }; + +private: + std::array<std::mutex, static_cast<std::size_t>(Pool::Count)> pool_locks; + std::array<Impl, MaxManagerCount> managers; +}; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/memory_types.h b/src/core/hle/kernel/memory/memory_types.h new file mode 100644 index 000000000..a75bf77c0 --- /dev/null +++ b/src/core/hle/kernel/memory/memory_types.h @@ -0,0 +1,18 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include <array> + +#include "common/common_types.h" + +namespace Kernel::Memory { + +constexpr std::size_t PageBits{12}; +constexpr std::size_t PageSize{1 << PageBits}; + +using Page = std::array<u8, PageSize>; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_heap.cpp b/src/core/hle/kernel/memory/page_heap.cpp new file mode 100644 index 000000000..0ab1f7205 --- /dev/null +++ b/src/core/hle/kernel/memory/page_heap.cpp @@ -0,0 +1,119 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +// This file references various implementation details from Atmosphere, an open-source firmware for +// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. + +#include "core/core.h" +#include "core/hle/kernel/memory/page_heap.h" +#include "core/memory.h" + +namespace Kernel::Memory { + +void PageHeap::Initialize(VAddr address, std::size_t size, std::size_t metadata_size) { + // Check our assumptions + ASSERT(Common::IsAligned((address), PageSize)); + ASSERT(Common::IsAligned(size, PageSize)); + + // Set our members + heap_address = address; + heap_size = size; + + // Setup bitmaps + metadata.resize(metadata_size / sizeof(u64)); + u64* cur_bitmap_storage{metadata.data()}; + for (std::size_t i = 0; i < MemoryBlockPageShifts.size(); i++) { + const std::size_t cur_block_shift{MemoryBlockPageShifts[i]}; + const std::size_t next_block_shift{ + (i != MemoryBlockPageShifts.size() - 1) ? MemoryBlockPageShifts[i + 1] : 0}; + cur_bitmap_storage = blocks[i].Initialize(heap_address, heap_size, cur_block_shift, + next_block_shift, cur_bitmap_storage); + } +} + +VAddr PageHeap::AllocateBlock(s32 index) { + const std::size_t needed_size{blocks[index].GetSize()}; + + for (s32 i{index}; i < static_cast<s32>(MemoryBlockPageShifts.size()); i++) { + if (const VAddr addr{blocks[i].PopBlock()}; addr) { + if (const std::size_t allocated_size{blocks[i].GetSize()}; + allocated_size > needed_size) { + Free(addr + needed_size, (allocated_size - needed_size) / PageSize); + } + return addr; + } + } + + return 0; +} + +void PageHeap::FreeBlock(VAddr block, s32 index) { + do { + block = blocks[index++].PushBlock(block); + } while (block != 0); +} + +void PageHeap::Free(VAddr addr, std::size_t num_pages) { + // Freeing no pages is a no-op + if (num_pages == 0) { + return; + } + + // Find the largest block size that we can free, and free as many as possible + s32 big_index{static_cast<s32>(MemoryBlockPageShifts.size()) - 1}; + const VAddr start{addr}; + const VAddr end{(num_pages * PageSize) + addr}; + VAddr before_start{start}; + VAddr before_end{start}; + VAddr after_start{end}; + VAddr after_end{end}; + while (big_index >= 0) { + const std::size_t block_size{blocks[big_index].GetSize()}; + const VAddr big_start{Common::AlignUp((start), block_size)}; + const VAddr big_end{Common::AlignDown((end), block_size)}; + if (big_start < big_end) { + // Free as many big blocks as we can + for (auto block{big_start}; block < big_end; block += block_size) { + FreeBlock(block, big_index); + } + before_end = big_start; + after_start = big_end; + break; + } + big_index--; + } + ASSERT(big_index >= 0); + + // Free space before the big blocks + for (s32 i{big_index - 1}; i >= 0; i--) { + const std::size_t block_size{blocks[i].GetSize()}; + while (before_start + block_size <= before_end) { + before_end -= block_size; + FreeBlock(before_end, i); + } + } + + // Free space after the big blocks + for (s32 i{big_index - 1}; i >= 0; i--) { + const std::size_t block_size{blocks[i].GetSize()}; + while (after_start + block_size <= after_end) { + FreeBlock(after_start, i); + after_start += block_size; + } + } +} + +std::size_t PageHeap::CalculateMetadataOverheadSize(std::size_t region_size) { + std::size_t overhead_size = 0; + for (std::size_t i = 0; i < MemoryBlockPageShifts.size(); i++) { + const std::size_t cur_block_shift{MemoryBlockPageShifts[i]}; + const std::size_t next_block_shift{ + (i != MemoryBlockPageShifts.size() - 1) ? MemoryBlockPageShifts[i + 1] : 0}; + overhead_size += PageHeap::Block::CalculateMetadataOverheadSize( + region_size, cur_block_shift, next_block_shift); + } + return Common::AlignUp(overhead_size, PageSize); +} + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_heap.h b/src/core/hle/kernel/memory/page_heap.h new file mode 100644 index 000000000..22b0de860 --- /dev/null +++ b/src/core/hle/kernel/memory/page_heap.h @@ -0,0 +1,370 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +// This file references various implementation details from Atmosphere, an open-source firmware for +// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. + +#pragma once + +#include <array> +#include <vector> + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/bit_util.h" +#include "common/common_funcs.h" +#include "common/common_types.h" +#include "core/hle/kernel/memory/memory_types.h" + +namespace Kernel::Memory { + +class PageHeap final : NonCopyable { +public: + static constexpr s32 GetAlignedBlockIndex(std::size_t num_pages, std::size_t align_pages) { + const auto target_pages{std::max(num_pages, align_pages)}; + for (std::size_t i = 0; i < NumMemoryBlockPageShifts; i++) { + if (target_pages <= + (static_cast<std::size_t>(1) << MemoryBlockPageShifts[i]) / PageSize) { + return static_cast<s32>(i); + } + } + return -1; + } + + static constexpr s32 GetBlockIndex(std::size_t num_pages) { + for (s32 i{static_cast<s32>(NumMemoryBlockPageShifts) - 1}; i >= 0; i--) { + if (num_pages >= (static_cast<std::size_t>(1) << MemoryBlockPageShifts[i]) / PageSize) { + return i; + } + } + return -1; + } + + static constexpr std::size_t GetBlockSize(std::size_t index) { + return static_cast<std::size_t>(1) << MemoryBlockPageShifts[index]; + } + + static constexpr std::size_t GetBlockNumPages(std::size_t index) { + return GetBlockSize(index) / PageSize; + } + +private: + static constexpr std::size_t NumMemoryBlockPageShifts{7}; + static constexpr std::array<std::size_t, NumMemoryBlockPageShifts> MemoryBlockPageShifts{ + 0xC, 0x10, 0x15, 0x16, 0x19, 0x1D, 0x1E, + }; + + class Block final : NonCopyable { + private: + class Bitmap final : NonCopyable { + public: + static constexpr std::size_t MaxDepth{4}; + + private: + std::array<u64*, MaxDepth> bit_storages{}; + std::size_t num_bits{}; + std::size_t used_depths{}; + + public: + constexpr Bitmap() = default; + + constexpr std::size_t GetNumBits() const { + return num_bits; + } + constexpr s32 GetHighestDepthIndex() const { + return static_cast<s32>(used_depths) - 1; + } + + constexpr u64* Initialize(u64* storage, std::size_t size) { + //* Initially, everything is un-set + num_bits = 0; + + // Calculate the needed bitmap depth + used_depths = static_cast<std::size_t>(GetRequiredDepth(size)); + ASSERT(used_depths <= MaxDepth); + + // Set the bitmap pointers + for (s32 depth{GetHighestDepthIndex()}; depth >= 0; depth--) { + bit_storages[depth] = storage; + size = Common::AlignUp(size, 64) / 64; + storage += size; + } + + return storage; + } + + s64 FindFreeBlock() const { + uintptr_t offset{}; + s32 depth{}; + + do { + const u64 v{bit_storages[depth][offset]}; + if (v == 0) { + // Non-zero depth indicates that a previous level had a free block + ASSERT(depth == 0); + return -1; + } + offset = offset * 64 + Common::CountTrailingZeroes64(v); + ++depth; + } while (depth < static_cast<s32>(used_depths)); + + return static_cast<s64>(offset); + } + + constexpr void SetBit(std::size_t offset) { + SetBit(GetHighestDepthIndex(), offset); + num_bits++; + } + + constexpr void ClearBit(std::size_t offset) { + ClearBit(GetHighestDepthIndex(), offset); + num_bits--; + } + + constexpr bool ClearRange(std::size_t offset, std::size_t count) { + const s32 depth{GetHighestDepthIndex()}; + const auto bit_ind{offset / 64}; + u64* bits{bit_storages[depth]}; + if (count < 64) { + const auto shift{offset % 64}; + ASSERT(shift + count <= 64); + // Check that all the bits are set + const u64 mask{((1ULL << count) - 1) << shift}; + u64 v{bits[bit_ind]}; + if ((v & mask) != mask) { + return false; + } + + // Clear the bits + v &= ~mask; + bits[bit_ind] = v; + if (v == 0) { + ClearBit(depth - 1, bit_ind); + } + } else { + ASSERT(offset % 64 == 0); + ASSERT(count % 64 == 0); + // Check that all the bits are set + std::size_t remaining{count}; + std::size_t i = 0; + do { + if (bits[bit_ind + i++] != ~u64(0)) { + return false; + } + remaining -= 64; + } while (remaining > 0); + + // Clear the bits + remaining = count; + i = 0; + do { + bits[bit_ind + i] = 0; + ClearBit(depth - 1, bit_ind + i); + i++; + remaining -= 64; + } while (remaining > 0); + } + + num_bits -= count; + return true; + } + + private: + constexpr void SetBit(s32 depth, std::size_t offset) { + while (depth >= 0) { + const auto ind{offset / 64}; + const auto which{offset % 64}; + const u64 mask{1ULL << which}; + + u64* bit{std::addressof(bit_storages[depth][ind])}; + const u64 v{*bit}; + ASSERT((v & mask) == 0); + *bit = v | mask; + if (v) { + break; + } + offset = ind; + depth--; + } + } + + constexpr void ClearBit(s32 depth, std::size_t offset) { + while (depth >= 0) { + const auto ind{offset / 64}; + const auto which{offset % 64}; + const u64 mask{1ULL << which}; + + u64* bit{std::addressof(bit_storages[depth][ind])}; + u64 v{*bit}; + ASSERT((v & mask) != 0); + v &= ~mask; + *bit = v; + if (v) { + break; + } + offset = ind; + depth--; + } + } + + private: + static constexpr s32 GetRequiredDepth(std::size_t region_size) { + s32 depth = 0; + while (true) { + region_size /= 64; + depth++; + if (region_size == 0) { + return depth; + } + } + } + + public: + static constexpr std::size_t CalculateMetadataOverheadSize(std::size_t region_size) { + std::size_t overhead_bits = 0; + for (s32 depth{GetRequiredDepth(region_size) - 1}; depth >= 0; depth--) { + region_size = Common::AlignUp(region_size, 64) / 64; + overhead_bits += region_size; + } + return overhead_bits * sizeof(u64); + } + }; + + private: + Bitmap bitmap; + VAddr heap_address{}; + uintptr_t end_offset{}; + std::size_t block_shift{}; + std::size_t next_block_shift{}; + + public: + constexpr Block() = default; + + constexpr std::size_t GetShift() const { + return block_shift; + } + constexpr std::size_t GetNextShift() const { + return next_block_shift; + } + constexpr std::size_t GetSize() const { + return static_cast<std::size_t>(1) << GetShift(); + } + constexpr std::size_t GetNumPages() const { + return GetSize() / PageSize; + } + constexpr std::size_t GetNumFreeBlocks() const { + return bitmap.GetNumBits(); + } + constexpr std::size_t GetNumFreePages() const { + return GetNumFreeBlocks() * GetNumPages(); + } + + constexpr u64* Initialize(VAddr addr, std::size_t size, std::size_t bs, std::size_t nbs, + u64* bit_storage) { + // Set shifts + block_shift = bs; + next_block_shift = nbs; + + // Align up the address + VAddr end{addr + size}; + const auto align{(next_block_shift != 0) ? (1ULL << next_block_shift) + : (1ULL << block_shift)}; + addr = Common::AlignDown((addr), align); + end = Common::AlignUp((end), align); + + heap_address = addr; + end_offset = (end - addr) / (1ULL << block_shift); + return bitmap.Initialize(bit_storage, end_offset); + } + + constexpr VAddr PushBlock(VAddr address) { + // Set the bit for the free block + std::size_t offset{(address - heap_address) >> GetShift()}; + bitmap.SetBit(offset); + + // If we have a next shift, try to clear the blocks below and return the address + if (GetNextShift()) { + const auto diff{1ULL << (GetNextShift() - GetShift())}; + offset = Common::AlignDown(offset, diff); + if (bitmap.ClearRange(offset, diff)) { + return heap_address + (offset << GetShift()); + } + } + + // We couldn't coalesce, or we're already as big as possible + return 0; + } + + VAddr PopBlock() { + // Find a free block + const s64 soffset{bitmap.FindFreeBlock()}; + if (soffset < 0) { + return 0; + } + const auto offset{static_cast<std::size_t>(soffset)}; + + // Update our tracking and return it + bitmap.ClearBit(offset); + return heap_address + (offset << GetShift()); + } + + public: + static constexpr std::size_t CalculateMetadataOverheadSize(std::size_t region_size, + std::size_t cur_block_shift, + std::size_t next_block_shift) { + const auto cur_block_size{(1ULL << cur_block_shift)}; + const auto next_block_size{(1ULL << next_block_shift)}; + const auto align{(next_block_shift != 0) ? next_block_size : cur_block_size}; + return Bitmap::CalculateMetadataOverheadSize( + (align * 2 + Common::AlignUp(region_size, align)) / cur_block_size); + } + }; + +public: + PageHeap() = default; + + constexpr VAddr GetAddress() const { + return heap_address; + } + constexpr std::size_t GetSize() const { + return heap_size; + } + constexpr VAddr GetEndAddress() const { + return GetAddress() + GetSize(); + } + constexpr std::size_t GetPageOffset(VAddr block) const { + return (block - GetAddress()) / PageSize; + } + + void Initialize(VAddr heap_address, std::size_t heap_size, std::size_t metadata_size); + VAddr AllocateBlock(s32 index); + void Free(VAddr addr, std::size_t num_pages); + + void UpdateUsedSize() { + used_size = heap_size - (GetNumFreePages() * PageSize); + } + + static std::size_t CalculateMetadataOverheadSize(std::size_t region_size); + +private: + constexpr std::size_t GetNumFreePages() const { + std::size_t num_free{}; + + for (const auto& block : blocks) { + num_free += block.GetNumFreePages(); + } + + return num_free; + } + + void FreeBlock(VAddr block, s32 index); + + VAddr heap_address{}; + std::size_t heap_size{}; + std::size_t used_size{}; + std::array<Block, NumMemoryBlockPageShifts> blocks{}; + std::vector<u64> metadata; +}; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_linked_list.h b/src/core/hle/kernel/memory/page_linked_list.h new file mode 100644 index 000000000..45dc13eaf --- /dev/null +++ b/src/core/hle/kernel/memory/page_linked_list.h @@ -0,0 +1,92 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include <list> + +#include "common/assert.h" +#include "common/common_types.h" +#include "core/hle/kernel/memory/memory_types.h" +#include "core/hle/result.h" + +namespace Kernel::Memory { + +class PageLinkedList final { +public: + class Node final { + public: + constexpr Node(u64 addr, std::size_t num_pages) : addr{addr}, num_pages{num_pages} {} + + constexpr u64 GetAddress() const { + return addr; + } + + constexpr std::size_t GetNumPages() const { + return num_pages; + } + + private: + u64 addr{}; + std::size_t num_pages{}; + }; + +public: + PageLinkedList() = default; + PageLinkedList(u64 address, u64 num_pages) { + ASSERT(AddBlock(address, num_pages).IsSuccess()); + } + + constexpr std::list<Node>& Nodes() { + return nodes; + } + + constexpr const std::list<Node>& Nodes() const { + return nodes; + } + + std::size_t GetNumPages() const { + std::size_t num_pages = 0; + for (const Node& node : nodes) { + num_pages += node.GetNumPages(); + } + return num_pages; + } + + bool IsEqual(PageLinkedList& other) const { + auto this_node = nodes.begin(); + auto other_node = other.nodes.begin(); + while (this_node != nodes.end() && other_node != other.nodes.end()) { + if (this_node->GetAddress() != other_node->GetAddress() || + this_node->GetNumPages() != other_node->GetNumPages()) { + return false; + } + this_node = std::next(this_node); + other_node = std::next(other_node); + } + + return this_node == nodes.end() && other_node == other.nodes.end(); + } + + ResultCode AddBlock(u64 address, u64 num_pages) { + if (!num_pages) { + return RESULT_SUCCESS; + } + if (!nodes.empty()) { + const auto node = nodes.back(); + if (node.GetAddress() + node.GetNumPages() * PageSize == address) { + address = node.GetAddress(); + num_pages += node.GetNumPages(); + nodes.pop_back(); + } + } + nodes.push_back({address, num_pages}); + return RESULT_SUCCESS; + } + +private: + std::list<Node> nodes; +}; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_table.cpp b/src/core/hle/kernel/memory/page_table.cpp new file mode 100644 index 000000000..a3fadb533 --- /dev/null +++ b/src/core/hle/kernel/memory/page_table.cpp @@ -0,0 +1,1174 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/scope_exit.h" +#include "core/core.h" +#include "core/hle/kernel/errors.h" +#include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/memory/address_space_info.h" +#include "core/hle/kernel/memory/memory_block.h" +#include "core/hle/kernel/memory/memory_block_manager.h" +#include "core/hle/kernel/memory/page_linked_list.h" +#include "core/hle/kernel/memory/page_table.h" +#include "core/hle/kernel/memory/system_control.h" +#include "core/hle/kernel/process.h" +#include "core/hle/kernel/resource_limit.h" +#include "core/memory.h" + +namespace Kernel::Memory { + +namespace { + +constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType as_type) { + switch (as_type) { + case FileSys::ProgramAddressSpaceType::Is32Bit: + case FileSys::ProgramAddressSpaceType::Is32BitNoMap: + return 32; + case FileSys::ProgramAddressSpaceType::Is36Bit: + return 36; + case FileSys::ProgramAddressSpaceType::Is39Bit: + return 39; + default: + UNREACHABLE(); + return {}; + } +} + +constexpr u64 GetAddressInRange(const MemoryInfo& info, VAddr addr) { + if (info.GetAddress() < addr) { + return addr; + } + return info.GetAddress(); +} + +constexpr std::size_t GetSizeInRange(const MemoryInfo& info, VAddr start, VAddr end) { + std::size_t size{info.GetSize()}; + if (info.GetAddress() < start) { + size -= start - info.GetAddress(); + } + if (info.GetEndAddress() > end) { + size -= info.GetEndAddress() - end; + } + return size; +} + +} // namespace + +PageTable::PageTable(Core::System& system) : system{system} {} + +ResultCode PageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, + bool enable_aslr, VAddr code_addr, std::size_t code_size, + Memory::MemoryManager::Pool pool) { + + const auto GetSpaceStart = [this](AddressSpaceInfo::Type type) { + return AddressSpaceInfo::GetAddressSpaceStart(address_space_width, type); + }; + const auto GetSpaceSize = [this](AddressSpaceInfo::Type type) { + return AddressSpaceInfo::GetAddressSpaceSize(address_space_width, type); + }; + + // Set our width and heap/alias sizes + address_space_width = GetAddressSpaceWidthFromType(as_type); + const VAddr start = 0; + const VAddr end{1ULL << address_space_width}; + std::size_t alias_region_size{GetSpaceSize(AddressSpaceInfo::Type::Alias)}; + std::size_t heap_region_size{GetSpaceSize(AddressSpaceInfo::Type::Heap)}; + + ASSERT(start <= code_addr); + ASSERT(code_addr < code_addr + code_size); + ASSERT(code_addr + code_size - 1 <= end - 1); + + // Adjust heap/alias size if we don't have an alias region + if (as_type == FileSys::ProgramAddressSpaceType::Is32BitNoMap) { + heap_region_size += alias_region_size; + alias_region_size = 0; + } + + // Set code regions and determine remaining + constexpr std::size_t RegionAlignment{2 * 1024 * 1024}; + VAddr process_code_start{}; + VAddr process_code_end{}; + std::size_t stack_region_size{}; + std::size_t kernel_map_region_size{}; + + if (address_space_width == 39) { + alias_region_size = GetSpaceSize(AddressSpaceInfo::Type::Alias); + heap_region_size = GetSpaceSize(AddressSpaceInfo::Type::Heap); + stack_region_size = GetSpaceSize(AddressSpaceInfo::Type::Stack); + kernel_map_region_size = GetSpaceSize(AddressSpaceInfo::Type::Is32Bit); + code_region_start = GetSpaceStart(AddressSpaceInfo::Type::Large64Bit); + code_region_end = code_region_start + GetSpaceSize(AddressSpaceInfo::Type::Large64Bit); + alias_code_region_start = code_region_start; + alias_code_region_end = code_region_end; + process_code_start = Common::AlignDown(code_addr, RegionAlignment); + process_code_end = Common::AlignUp(code_addr + code_size, RegionAlignment); + } else { + stack_region_size = 0; + kernel_map_region_size = 0; + code_region_start = GetSpaceStart(AddressSpaceInfo::Type::Is32Bit); + code_region_end = code_region_start + GetSpaceSize(AddressSpaceInfo::Type::Is32Bit); + stack_region_start = code_region_start; + alias_code_region_start = code_region_start; + alias_code_region_end = GetSpaceStart(AddressSpaceInfo::Type::Small64Bit) + + GetSpaceSize(AddressSpaceInfo::Type::Small64Bit); + stack_region_end = code_region_end; + kernel_map_region_start = code_region_start; + kernel_map_region_end = code_region_end; + process_code_start = code_region_start; + process_code_end = code_region_end; + } + + // Set other basic fields + is_aslr_enabled = enable_aslr; + address_space_start = start; + address_space_end = end; + is_kernel = false; + + // Determine the region we can place our undetermineds in + VAddr alloc_start{}; + std::size_t alloc_size{}; + if ((process_code_start - code_region_start) >= (end - process_code_end)) { + alloc_start = code_region_start; + alloc_size = process_code_start - code_region_start; + } else { + alloc_start = process_code_end; + alloc_size = end - process_code_end; + } + const std::size_t needed_size{ + (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size)}; + if (alloc_size < needed_size) { + UNREACHABLE(); + return ERR_OUT_OF_MEMORY; + } + + const std::size_t remaining_size{alloc_size - needed_size}; + + // Determine random placements for each region + std::size_t alias_rnd{}, heap_rnd{}, stack_rnd{}, kmap_rnd{}; + if (enable_aslr) { + alias_rnd = SystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * + RegionAlignment; + heap_rnd = SystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * + RegionAlignment; + stack_rnd = SystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * + RegionAlignment; + kmap_rnd = SystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * + RegionAlignment; + } + + // Setup heap and alias regions + alias_region_start = alloc_start + alias_rnd; + alias_region_end = alias_region_start + alias_region_size; + heap_region_start = alloc_start + heap_rnd; + heap_region_end = heap_region_start + heap_region_size; + + if (alias_rnd <= heap_rnd) { + heap_region_start += alias_region_size; + heap_region_end += alias_region_size; + } else { + alias_region_start += heap_region_size; + alias_region_end += heap_region_size; + } + + // Setup stack region + if (stack_region_size) { + stack_region_start = alloc_start + stack_rnd; + stack_region_end = stack_region_start + stack_region_size; + + if (alias_rnd < stack_rnd) { + stack_region_start += alias_region_size; + stack_region_end += alias_region_size; + } else { + alias_region_start += stack_region_size; + alias_region_end += stack_region_size; + } + + if (heap_rnd < stack_rnd) { + stack_region_start += heap_region_size; + stack_region_end += heap_region_size; + } else { + heap_region_start += stack_region_size; + heap_region_end += stack_region_size; + } + } + + // Setup kernel map region + if (kernel_map_region_size) { + kernel_map_region_start = alloc_start + kmap_rnd; + kernel_map_region_end = kernel_map_region_start + kernel_map_region_size; + + if (alias_rnd < kmap_rnd) { + kernel_map_region_start += alias_region_size; + kernel_map_region_end += alias_region_size; + } else { + alias_region_start += kernel_map_region_size; + alias_region_end += kernel_map_region_size; + } + + if (heap_rnd < kmap_rnd) { + kernel_map_region_start += heap_region_size; + kernel_map_region_end += heap_region_size; + } else { + heap_region_start += kernel_map_region_size; + heap_region_end += kernel_map_region_size; + } + + if (stack_region_size) { + if (stack_rnd < kmap_rnd) { + kernel_map_region_start += stack_region_size; + kernel_map_region_end += stack_region_size; + } else { + stack_region_start += kernel_map_region_size; + stack_region_end += kernel_map_region_size; + } + } + } + + // Set heap members + current_heap_end = heap_region_start; + max_heap_size = 0; + max_physical_memory_size = 0; + + // Ensure that we regions inside our address space + auto IsInAddressSpace = [&](VAddr addr) { + return address_space_start <= addr && addr <= address_space_end; + }; + ASSERT(IsInAddressSpace(alias_region_start)); + ASSERT(IsInAddressSpace(alias_region_end)); + ASSERT(IsInAddressSpace(heap_region_start)); + ASSERT(IsInAddressSpace(heap_region_end)); + ASSERT(IsInAddressSpace(stack_region_start)); + ASSERT(IsInAddressSpace(stack_region_end)); + ASSERT(IsInAddressSpace(kernel_map_region_start)); + ASSERT(IsInAddressSpace(kernel_map_region_end)); + + // Ensure that we selected regions that don't overlap + const VAddr alias_start{alias_region_start}; + const VAddr alias_last{alias_region_end - 1}; + const VAddr heap_start{heap_region_start}; + const VAddr heap_last{heap_region_end - 1}; + const VAddr stack_start{stack_region_start}; + const VAddr stack_last{stack_region_end - 1}; + const VAddr kmap_start{kernel_map_region_start}; + const VAddr kmap_last{kernel_map_region_end - 1}; + ASSERT(alias_last < heap_start || heap_last < alias_start); + ASSERT(alias_last < stack_start || stack_last < alias_start); + ASSERT(alias_last < kmap_start || kmap_last < alias_start); + ASSERT(heap_last < stack_start || stack_last < heap_start); + ASSERT(heap_last < kmap_start || kmap_last < heap_start); + + current_heap_addr = heap_region_start; + heap_capacity = 0; + physical_memory_usage = 0; + memory_pool = pool; + + page_table_impl.Resize(address_space_width, PageBits, true); + + return InitializeMemoryLayout(start, end); +} + +ResultCode PageTable::MapProcessCode(VAddr addr, std::size_t num_pages, MemoryState state, + MemoryPermission perm) { + std::lock_guard lock{page_table_lock}; + + const u64 size{num_pages * PageSize}; + + if (!CanContain(addr, size, state)) { + return ERR_INVALID_ADDRESS_STATE; + } + + if (IsRegionMapped(addr, size)) { + return ERR_INVALID_ADDRESS_STATE; + } + + PageLinkedList page_linked_list; + CASCADE_CODE( + system.Kernel().MemoryManager().Allocate(page_linked_list, num_pages, memory_pool)); + CASCADE_CODE(Operate(addr, num_pages, page_linked_list, OperationType::MapGroup)); + + block_manager->Update(addr, num_pages, state, perm); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::MapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + const std::size_t num_pages{size / PageSize}; + + MemoryState state{}; + MemoryPermission perm{}; + CASCADE_CODE(CheckMemoryState(&state, &perm, nullptr, src_addr, size, MemoryState::All, + MemoryState::Normal, MemoryPermission::Mask, + MemoryPermission::ReadAndWrite, MemoryAttribute::Mask, + MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + + if (IsRegionMapped(dst_addr, size)) { + return ERR_INVALID_ADDRESS_STATE; + } + + PageLinkedList page_linked_list; + AddRegionToPages(src_addr, num_pages, page_linked_list); + + { + auto block_guard = detail::ScopeExit( + [&] { Operate(src_addr, num_pages, perm, OperationType::ChangePermissions); }); + + CASCADE_CODE( + Operate(src_addr, num_pages, MemoryPermission::None, OperationType::ChangePermissions)); + CASCADE_CODE(MapPages(dst_addr, page_linked_list, MemoryPermission::None)); + + block_guard.Cancel(); + } + + block_manager->Update(src_addr, num_pages, state, MemoryPermission::None, + MemoryAttribute::Locked); + block_manager->Update(dst_addr, num_pages, MemoryState::AliasCode); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::UnmapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + if (!size) { + return RESULT_SUCCESS; + } + + const std::size_t num_pages{size / PageSize}; + + CASCADE_CODE(CheckMemoryState(nullptr, nullptr, nullptr, src_addr, size, MemoryState::All, + MemoryState::Normal, MemoryPermission::None, + MemoryPermission::None, MemoryAttribute::Mask, + MemoryAttribute::Locked, MemoryAttribute::IpcAndDeviceMapped)); + + MemoryState state{}; + CASCADE_CODE(CheckMemoryState( + &state, nullptr, nullptr, dst_addr, PageSize, MemoryState::FlagCanCodeAlias, + MemoryState::FlagCanCodeAlias, MemoryPermission::None, MemoryPermission::None, + MemoryAttribute::Mask, MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + CASCADE_CODE(CheckMemoryState(dst_addr, size, MemoryState::All, state, MemoryPermission::None, + MemoryPermission::None, MemoryAttribute::Mask, + MemoryAttribute::None)); + CASCADE_CODE(Operate(dst_addr, num_pages, MemoryPermission::None, OperationType::Unmap)); + + block_manager->Update(dst_addr, num_pages, MemoryState::Free); + block_manager->Update(src_addr, num_pages, MemoryState::Normal, MemoryPermission::ReadAndWrite); + + return RESULT_SUCCESS; +} + +void PageTable::MapPhysicalMemory(PageLinkedList& page_linked_list, VAddr start, VAddr end) { + auto node{page_linked_list.Nodes().begin()}; + PAddr map_addr{node->GetAddress()}; + std::size_t src_num_pages{node->GetNumPages()}; + + block_manager->IterateForRange(start, end, [&](const MemoryInfo& info) { + if (info.state != MemoryState::Free) { + return; + } + + std::size_t dst_num_pages{GetSizeInRange(info, start, end) / PageSize}; + VAddr dst_addr{GetAddressInRange(info, start)}; + + while (dst_num_pages) { + if (!src_num_pages) { + node = std::next(node); + map_addr = node->GetAddress(); + src_num_pages = node->GetNumPages(); + } + + const std::size_t num_pages{std::min(src_num_pages, dst_num_pages)}; + Operate(dst_addr, num_pages, MemoryPermission::ReadAndWrite, OperationType::Map, + map_addr); + + dst_addr += num_pages * PageSize; + map_addr += num_pages * PageSize; + src_num_pages -= num_pages; + dst_num_pages -= num_pages; + } + }); +} + +ResultCode PageTable::MapPhysicalMemory(VAddr addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + std::size_t mapped_size{}; + const VAddr end_addr{addr + size}; + + block_manager->IterateForRange(addr, end_addr, [&](const MemoryInfo& info) { + if (info.state != MemoryState::Free) { + mapped_size += GetSizeInRange(info, addr, end_addr); + } + }); + + if (mapped_size == size) { + return RESULT_SUCCESS; + } + + auto process{system.Kernel().CurrentProcess()}; + const std::size_t remaining_size{size - mapped_size}; + const std::size_t remaining_pages{remaining_size / PageSize}; + + if (process->GetResourceLimit() && + !process->GetResourceLimit()->Reserve(ResourceType::PhysicalMemory, remaining_size)) { + return ERR_RESOURCE_LIMIT_EXCEEDED; + } + + PageLinkedList page_linked_list; + { + auto block_guard = detail::ScopeExit([&] { + system.Kernel().MemoryManager().Free(page_linked_list, remaining_pages, memory_pool); + process->GetResourceLimit()->Release(ResourceType::PhysicalMemory, remaining_size); + }); + + CASCADE_CODE(system.Kernel().MemoryManager().Allocate(page_linked_list, remaining_pages, + memory_pool)); + + block_guard.Cancel(); + } + + MapPhysicalMemory(page_linked_list, addr, end_addr); + + physical_memory_usage += remaining_size; + + const std::size_t num_pages{size / PageSize}; + block_manager->Update(addr, num_pages, MemoryState::Free, MemoryPermission::None, + MemoryAttribute::None, MemoryState::Normal, + MemoryPermission::ReadAndWrite, MemoryAttribute::None); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::UnmapPhysicalMemory(VAddr addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + const VAddr end_addr{addr + size}; + ResultCode result{RESULT_SUCCESS}; + std::size_t mapped_size{}; + + // Verify that the region can be unmapped + block_manager->IterateForRange(addr, end_addr, [&](const MemoryInfo& info) { + if (info.state == MemoryState::Normal) { + if (info.attribute != MemoryAttribute::None) { + result = ERR_INVALID_ADDRESS_STATE; + return; + } + mapped_size += GetSizeInRange(info, addr, end_addr); + } else if (info.state != MemoryState::Free) { + result = ERR_INVALID_ADDRESS_STATE; + } + }); + + if (result.IsError()) { + return result; + } + + if (!mapped_size) { + return RESULT_SUCCESS; + } + + CASCADE_CODE(UnmapMemory(addr, size)); + + auto process{system.Kernel().CurrentProcess()}; + process->GetResourceLimit()->Release(ResourceType::PhysicalMemory, mapped_size); + physical_memory_usage -= mapped_size; + + return RESULT_SUCCESS; +} + +ResultCode PageTable::UnmapMemory(VAddr addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + const VAddr end_addr{addr + size}; + ResultCode result{RESULT_SUCCESS}; + PageLinkedList page_linked_list; + + // Unmap each region within the range + block_manager->IterateForRange(addr, end_addr, [&](const MemoryInfo& info) { + if (info.state == MemoryState::Normal) { + const std::size_t block_size{GetSizeInRange(info, addr, end_addr)}; + const std::size_t block_num_pages{block_size / PageSize}; + const VAddr block_addr{GetAddressInRange(info, addr)}; + + AddRegionToPages(block_addr, block_size / PageSize, page_linked_list); + + if (result = Operate(block_addr, block_num_pages, MemoryPermission::None, + OperationType::Unmap); + result.IsError()) { + return; + } + } + }); + + if (result.IsError()) { + return result; + } + + const std::size_t num_pages{size / PageSize}; + system.Kernel().MemoryManager().Free(page_linked_list, num_pages, memory_pool); + + block_manager->Update(addr, num_pages, MemoryState::Free); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::Map(VAddr dst_addr, VAddr src_addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + MemoryState src_state{}; + CASCADE_CODE(CheckMemoryState( + &src_state, nullptr, nullptr, src_addr, size, MemoryState::FlagCanAlias, + MemoryState::FlagCanAlias, MemoryPermission::Mask, MemoryPermission::ReadAndWrite, + MemoryAttribute::Mask, MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + + if (IsRegionMapped(dst_addr, size)) { + return ERR_INVALID_ADDRESS_STATE; + } + + PageLinkedList page_linked_list; + const std::size_t num_pages{size / PageSize}; + + AddRegionToPages(src_addr, num_pages, page_linked_list); + + { + auto block_guard = detail::ScopeExit([&] { + Operate(src_addr, num_pages, MemoryPermission::ReadAndWrite, + OperationType::ChangePermissions); + }); + + CASCADE_CODE( + Operate(src_addr, num_pages, MemoryPermission::None, OperationType::ChangePermissions)); + CASCADE_CODE(MapPages(dst_addr, page_linked_list, MemoryPermission::ReadAndWrite)); + + block_guard.Cancel(); + } + + block_manager->Update(src_addr, num_pages, src_state, MemoryPermission::None, + MemoryAttribute::Locked); + block_manager->Update(dst_addr, num_pages, MemoryState::Stack, MemoryPermission::ReadAndWrite); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::Unmap(VAddr dst_addr, VAddr src_addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + MemoryState src_state{}; + CASCADE_CODE(CheckMemoryState( + &src_state, nullptr, nullptr, src_addr, size, MemoryState::FlagCanAlias, + MemoryState::FlagCanAlias, MemoryPermission::Mask, MemoryPermission::None, + MemoryAttribute::Mask, MemoryAttribute::Locked, MemoryAttribute::IpcAndDeviceMapped)); + + MemoryPermission dst_perm{}; + CASCADE_CODE(CheckMemoryState(nullptr, &dst_perm, nullptr, dst_addr, size, MemoryState::All, + MemoryState::Stack, MemoryPermission::None, + MemoryPermission::None, MemoryAttribute::Mask, + MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + + PageLinkedList src_pages; + PageLinkedList dst_pages; + const std::size_t num_pages{size / PageSize}; + + AddRegionToPages(src_addr, num_pages, src_pages); + AddRegionToPages(dst_addr, num_pages, dst_pages); + + if (!dst_pages.IsEqual(src_pages)) { + return ERR_INVALID_MEMORY_RANGE; + } + + { + auto block_guard = detail::ScopeExit([&] { MapPages(dst_addr, dst_pages, dst_perm); }); + + CASCADE_CODE(Operate(dst_addr, num_pages, MemoryPermission::None, OperationType::Unmap)); + CASCADE_CODE(Operate(src_addr, num_pages, MemoryPermission::ReadAndWrite, + OperationType::ChangePermissions)); + + block_guard.Cancel(); + } + + block_manager->Update(src_addr, num_pages, src_state, MemoryPermission::ReadAndWrite); + block_manager->Update(dst_addr, num_pages, MemoryState::Free); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::MapPages(VAddr addr, const PageLinkedList& page_linked_list, + MemoryPermission perm) { + VAddr cur_addr{addr}; + + for (const auto& node : page_linked_list.Nodes()) { + if (const auto result{ + Operate(cur_addr, node.GetNumPages(), perm, OperationType::Map, node.GetAddress())}; + result.IsError()) { + const std::size_t num_pages{(addr - cur_addr) / PageSize}; + + ASSERT( + Operate(addr, num_pages, MemoryPermission::None, OperationType::Unmap).IsSuccess()); + + return result; + } + + cur_addr += node.GetNumPages() * PageSize; + } + + return RESULT_SUCCESS; +} + +ResultCode PageTable::MapPages(VAddr addr, PageLinkedList& page_linked_list, MemoryState state, + MemoryPermission perm) { + std::lock_guard lock{page_table_lock}; + + const std::size_t num_pages{page_linked_list.GetNumPages()}; + const std::size_t size{num_pages * PageSize}; + + if (!CanContain(addr, size, state)) { + return ERR_INVALID_ADDRESS_STATE; + } + + if (IsRegionMapped(addr, num_pages * PageSize)) { + return ERR_INVALID_ADDRESS_STATE; + } + + CASCADE_CODE(MapPages(addr, page_linked_list, perm)); + + block_manager->Update(addr, num_pages, state, perm); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::SetCodeMemoryPermission(VAddr addr, std::size_t size, MemoryPermission perm) { + + std::lock_guard lock{page_table_lock}; + + MemoryState prev_state{}; + MemoryPermission prev_perm{}; + + CASCADE_CODE(CheckMemoryState( + &prev_state, &prev_perm, nullptr, addr, size, MemoryState::FlagCode, MemoryState::FlagCode, + MemoryPermission::None, MemoryPermission::None, MemoryAttribute::Mask, + MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped)); + + MemoryState state{prev_state}; + + // Ensure state is mutable if permission allows write + if ((perm & MemoryPermission::Write) != MemoryPermission::None) { + if (prev_state == MemoryState::Code) { + state = MemoryState::CodeData; + } else if (prev_state == MemoryState::AliasCode) { + state = MemoryState::AliasCodeData; + } else { + UNREACHABLE(); + } + } + + // Return early if there is nothing to change + if (state == prev_state && perm == prev_perm) { + return RESULT_SUCCESS; + } + + const std::size_t num_pages{size / PageSize}; + const OperationType operation{(perm & MemoryPermission::Execute) != MemoryPermission::None + ? OperationType::ChangePermissionsAndRefresh + : OperationType::ChangePermissions}; + + CASCADE_CODE(Operate(addr, num_pages, perm, operation)); + + block_manager->Update(addr, num_pages, state, perm); + + return RESULT_SUCCESS; +} + +MemoryInfo PageTable::QueryInfoImpl(VAddr addr) { + std::lock_guard lock{page_table_lock}; + + return block_manager->FindBlock(addr).GetMemoryInfo(); +} + +MemoryInfo PageTable::QueryInfo(VAddr addr) { + if (!Contains(addr, 1)) { + return {address_space_end, 0 - address_space_end, MemoryState::Inaccessible, + MemoryPermission::None, MemoryAttribute::None, MemoryPermission::None}; + } + + return QueryInfoImpl(addr); +} + +ResultCode PageTable::ReserveTransferMemory(VAddr addr, std::size_t size, MemoryPermission perm) { + std::lock_guard lock{page_table_lock}; + + MemoryState state{}; + MemoryAttribute attribute{}; + + CASCADE_CODE(CheckMemoryState(&state, nullptr, &attribute, addr, size, + MemoryState::FlagCanTransfer | MemoryState::FlagReferenceCounted, + MemoryState::FlagCanTransfer | MemoryState::FlagReferenceCounted, + MemoryPermission::Mask, MemoryPermission::ReadAndWrite, + MemoryAttribute::Mask, MemoryAttribute::None, + MemoryAttribute::IpcAndDeviceMapped)); + + block_manager->Update(addr, size / PageSize, state, perm, attribute | MemoryAttribute::Locked); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::ResetTransferMemory(VAddr addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + MemoryState state{}; + + CASCADE_CODE(CheckMemoryState(&state, nullptr, nullptr, addr, size, + MemoryState::FlagCanTransfer | MemoryState::FlagReferenceCounted, + MemoryState::FlagCanTransfer | MemoryState::FlagReferenceCounted, + MemoryPermission::None, MemoryPermission::None, + MemoryAttribute::Mask, MemoryAttribute::Locked, + MemoryAttribute::IpcAndDeviceMapped)); + + block_manager->Update(addr, size / PageSize, state, MemoryPermission::ReadAndWrite); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::SetMemoryAttribute(VAddr addr, std::size_t size, MemoryAttribute mask, + MemoryAttribute value) { + std::lock_guard lock{page_table_lock}; + + MemoryState state{}; + MemoryPermission perm{}; + MemoryAttribute attribute{}; + + CASCADE_CODE(CheckMemoryState(&state, &perm, &attribute, addr, size, + MemoryState::FlagCanChangeAttribute, + MemoryState::FlagCanChangeAttribute, MemoryPermission::None, + MemoryPermission::None, MemoryAttribute::LockedAndIpcLocked, + MemoryAttribute::None, MemoryAttribute::DeviceSharedAndUncached)); + + attribute = attribute & ~mask; + attribute = attribute | (mask & value); + + block_manager->Update(addr, size / PageSize, state, perm, attribute); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::SetHeapCapacity(std::size_t new_heap_capacity) { + std::lock_guard lock{page_table_lock}; + heap_capacity = new_heap_capacity; + return RESULT_SUCCESS; +} + +ResultVal<VAddr> PageTable::SetHeapSize(std::size_t size) { + + if (size > heap_region_end - heap_region_start) { + return ERR_OUT_OF_MEMORY; + } + + const u64 previous_heap_size{GetHeapSize()}; + + UNIMPLEMENTED_IF_MSG(previous_heap_size > size, "Heap shrink is unimplemented"); + + // Increase the heap size + { + std::lock_guard lock{page_table_lock}; + + const u64 delta{size - previous_heap_size}; + + auto process{system.Kernel().CurrentProcess()}; + if (process->GetResourceLimit() && delta != 0 && + !process->GetResourceLimit()->Reserve(ResourceType::PhysicalMemory, delta)) { + return ERR_RESOURCE_LIMIT_EXCEEDED; + } + + PageLinkedList page_linked_list; + const std::size_t num_pages{delta / PageSize}; + + CASCADE_CODE( + system.Kernel().MemoryManager().Allocate(page_linked_list, num_pages, memory_pool)); + + if (IsRegionMapped(current_heap_addr, delta)) { + return ERR_INVALID_ADDRESS_STATE; + } + + CASCADE_CODE( + Operate(current_heap_addr, num_pages, page_linked_list, OperationType::MapGroup)); + + block_manager->Update(current_heap_addr, num_pages, MemoryState::Normal, + MemoryPermission::ReadAndWrite); + + current_heap_addr = heap_region_start + size; + } + + return MakeResult<VAddr>(heap_region_start); +} + +ResultVal<VAddr> PageTable::AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, + bool is_map_only, VAddr region_start, + std::size_t region_num_pages, MemoryState state, + MemoryPermission perm, PAddr map_addr) { + std::lock_guard lock{page_table_lock}; + + if (!CanContain(region_start, region_num_pages * PageSize, state)) { + return ERR_INVALID_ADDRESS_STATE; + } + + if (region_num_pages <= needed_num_pages) { + return ERR_OUT_OF_MEMORY; + } + + const VAddr addr{ + AllocateVirtualMemory(region_start, region_num_pages, needed_num_pages, align)}; + if (!addr) { + return ERR_OUT_OF_MEMORY; + } + + if (is_map_only) { + CASCADE_CODE(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); + } else { + PageLinkedList page_group; + CASCADE_CODE( + system.Kernel().MemoryManager().Allocate(page_group, needed_num_pages, memory_pool)); + CASCADE_CODE(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); + } + + block_manager->Update(addr, needed_num_pages, state, perm); + + return MakeResult<VAddr>(addr); +} + +ResultCode PageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + MemoryPermission perm{}; + if (const ResultCode result{CheckMemoryState( + nullptr, &perm, nullptr, addr, size, MemoryState::FlagCanChangeAttribute, + MemoryState::FlagCanChangeAttribute, MemoryPermission::None, MemoryPermission::None, + MemoryAttribute::LockedAndIpcLocked, MemoryAttribute::None, + MemoryAttribute::DeviceSharedAndUncached)}; + result.IsError()) { + return result; + } + + block_manager->UpdateLock( + addr, size / PageSize, + [](MemoryBlockManager::iterator block, MemoryPermission perm) { + block->ShareToDevice(perm); + }, + perm); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) { + std::lock_guard lock{page_table_lock}; + + MemoryPermission perm{}; + if (const ResultCode result{CheckMemoryState( + nullptr, &perm, nullptr, addr, size, MemoryState::FlagCanChangeAttribute, + MemoryState::FlagCanChangeAttribute, MemoryPermission::None, MemoryPermission::None, + MemoryAttribute::LockedAndIpcLocked, MemoryAttribute::None, + MemoryAttribute::DeviceSharedAndUncached)}; + result.IsError()) { + return result; + } + + block_manager->UpdateLock( + addr, size / PageSize, + [](MemoryBlockManager::iterator block, MemoryPermission perm) { + block->UnshareToDevice(perm); + }, + perm); + + return RESULT_SUCCESS; +} + +ResultCode PageTable::InitializeMemoryLayout(VAddr start, VAddr end) { + block_manager = std::make_unique<MemoryBlockManager>(start, end); + + return RESULT_SUCCESS; +} + +bool PageTable::IsRegionMapped(VAddr address, u64 size) { + return CheckMemoryState(address, size, MemoryState::All, MemoryState::Free, + MemoryPermission::Mask, MemoryPermission::None, MemoryAttribute::Mask, + MemoryAttribute::None, MemoryAttribute::IpcAndDeviceMapped) + .IsError(); +} + +bool PageTable::IsRegionContiguous(VAddr addr, u64 size) const { + auto start_ptr = system.Memory().GetPointer(addr); + for (u64 offset{}; offset < size; offset += PageSize) { + if (start_ptr != system.Memory().GetPointer(addr + offset)) { + return false; + } + start_ptr += PageSize; + } + return true; +} + +void PageTable::AddRegionToPages(VAddr start, std::size_t num_pages, + PageLinkedList& page_linked_list) { + VAddr addr{start}; + while (addr < start + (num_pages * PageSize)) { + const PAddr paddr{GetPhysicalAddr(addr)}; + if (!paddr) { + UNREACHABLE(); + } + page_linked_list.AddBlock(paddr, 1); + addr += PageSize; + } +} + +VAddr PageTable::AllocateVirtualMemory(VAddr start, std::size_t region_num_pages, + u64 needed_num_pages, std::size_t align) { + if (is_aslr_enabled) { + UNIMPLEMENTED(); + } + return block_manager->FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, + IsKernel() ? 1 : 4); +} + +ResultCode PageTable::Operate(VAddr addr, std::size_t num_pages, const PageLinkedList& page_group, + OperationType operation) { + std::lock_guard lock{page_table_lock}; + + ASSERT(Common::IsAligned(addr, PageSize)); + ASSERT(num_pages > 0); + ASSERT(num_pages == page_group.GetNumPages()); + + for (const auto& node : page_group.Nodes()) { + const std::size_t size{node.GetNumPages() * PageSize}; + + switch (operation) { + case OperationType::MapGroup: + system.Memory().MapMemoryRegion(page_table_impl, addr, size, node.GetAddress()); + break; + default: + UNREACHABLE(); + } + + addr += size; + } + + return RESULT_SUCCESS; +} + +ResultCode PageTable::Operate(VAddr addr, std::size_t num_pages, MemoryPermission perm, + OperationType operation, PAddr map_addr) { + std::lock_guard lock{page_table_lock}; + + ASSERT(num_pages > 0); + ASSERT(Common::IsAligned(addr, PageSize)); + ASSERT(ContainsPages(addr, num_pages)); + + switch (operation) { + case OperationType::Unmap: + system.Memory().UnmapRegion(page_table_impl, addr, num_pages * PageSize); + break; + case OperationType::Map: { + ASSERT(map_addr); + ASSERT(Common::IsAligned(map_addr, PageSize)); + system.Memory().MapMemoryRegion(page_table_impl, addr, num_pages * PageSize, map_addr); + break; + } + case OperationType::ChangePermissions: + case OperationType::ChangePermissionsAndRefresh: + break; + default: + UNREACHABLE(); + } + return RESULT_SUCCESS; +} + +constexpr VAddr PageTable::GetRegionAddress(MemoryState state) const { + switch (state) { + case MemoryState::Free: + case MemoryState::Kernel: + return address_space_start; + case MemoryState::Normal: + return heap_region_start; + case MemoryState::Ipc: + case MemoryState::NonSecureIpc: + case MemoryState::NonDeviceIpc: + return alias_region_start; + case MemoryState::Stack: + return stack_region_start; + case MemoryState::Io: + case MemoryState::Static: + case MemoryState::ThreadLocal: + return kernel_map_region_start; + case MemoryState::Shared: + case MemoryState::AliasCode: + case MemoryState::AliasCodeData: + case MemoryState::Transfered: + case MemoryState::SharedTransfered: + case MemoryState::SharedCode: + case MemoryState::GeneratedCode: + case MemoryState::CodeOut: + return alias_code_region_start; + case MemoryState::Code: + case MemoryState::CodeData: + return code_region_start; + default: + UNREACHABLE(); + return {}; + } +} + +constexpr std::size_t PageTable::GetRegionSize(MemoryState state) const { + switch (state) { + case MemoryState::Free: + case MemoryState::Kernel: + return address_space_end - address_space_start; + case MemoryState::Normal: + return heap_region_end - heap_region_start; + case MemoryState::Ipc: + case MemoryState::NonSecureIpc: + case MemoryState::NonDeviceIpc: + return alias_region_end - alias_region_start; + case MemoryState::Stack: + return stack_region_end - stack_region_start; + case MemoryState::Io: + case MemoryState::Static: + case MemoryState::ThreadLocal: + return kernel_map_region_end - kernel_map_region_start; + case MemoryState::Shared: + case MemoryState::AliasCode: + case MemoryState::AliasCodeData: + case MemoryState::Transfered: + case MemoryState::SharedTransfered: + case MemoryState::SharedCode: + case MemoryState::GeneratedCode: + case MemoryState::CodeOut: + return alias_code_region_end - alias_code_region_start; + case MemoryState::Code: + case MemoryState::CodeData: + return code_region_end - code_region_start; + default: + UNREACHABLE(); + return {}; + } +} + +constexpr bool PageTable::CanContain(VAddr addr, std::size_t size, MemoryState state) const { + const VAddr end{addr + size}; + const VAddr last{end - 1}; + const VAddr region_start{GetRegionAddress(state)}; + const std::size_t region_size{GetRegionSize(state)}; + const bool is_in_region{region_start <= addr && addr < end && + last <= region_start + region_size - 1}; + const bool is_in_heap{!(end <= heap_region_start || heap_region_end <= addr)}; + const bool is_in_alias{!(end <= alias_region_start || alias_region_end <= addr)}; + + switch (state) { + case MemoryState::Free: + case MemoryState::Kernel: + return is_in_region; + case MemoryState::Io: + case MemoryState::Static: + case MemoryState::Code: + case MemoryState::CodeData: + case MemoryState::Shared: + case MemoryState::AliasCode: + case MemoryState::AliasCodeData: + case MemoryState::Stack: + case MemoryState::ThreadLocal: + case MemoryState::Transfered: + case MemoryState::SharedTransfered: + case MemoryState::SharedCode: + case MemoryState::GeneratedCode: + case MemoryState::CodeOut: + return is_in_region && !is_in_heap && !is_in_alias; + case MemoryState::Normal: + ASSERT(is_in_heap); + return is_in_region && !is_in_alias; + case MemoryState::Ipc: + case MemoryState::NonSecureIpc: + case MemoryState::NonDeviceIpc: + ASSERT(is_in_alias); + return is_in_region && !is_in_heap; + default: + return false; + } +} + +constexpr ResultCode PageTable::CheckMemoryState(const MemoryInfo& info, MemoryState state_mask, + MemoryState state, MemoryPermission perm_mask, + MemoryPermission perm, MemoryAttribute attr_mask, + MemoryAttribute attr) const { + // Validate the states match expectation + if ((info.state & state_mask) != state) { + return ERR_INVALID_ADDRESS_STATE; + } + if ((info.perm & perm_mask) != perm) { + return ERR_INVALID_ADDRESS_STATE; + } + if ((info.attribute & attr_mask) != attr) { + return ERR_INVALID_ADDRESS_STATE; + } + + return RESULT_SUCCESS; +} + +ResultCode PageTable::CheckMemoryState(MemoryState* out_state, MemoryPermission* out_perm, + MemoryAttribute* out_attr, VAddr addr, std::size_t size, + MemoryState state_mask, MemoryState state, + MemoryPermission perm_mask, MemoryPermission perm, + MemoryAttribute attr_mask, MemoryAttribute attr, + MemoryAttribute ignore_attr) { + std::lock_guard lock{page_table_lock}; + + // Get information about the first block + const VAddr last_addr{addr + size - 1}; + MemoryBlockManager::const_iterator it{block_manager->FindIterator(addr)}; + MemoryInfo info{it->GetMemoryInfo()}; + + // Validate all blocks in the range have correct state + const MemoryState first_state{info.state}; + const MemoryPermission first_perm{info.perm}; + const MemoryAttribute first_attr{info.attribute}; + + while (true) { + // Validate the current block + if (!(info.state == first_state)) { + return ERR_INVALID_ADDRESS_STATE; + } + if (!(info.perm == first_perm)) { + return ERR_INVALID_ADDRESS_STATE; + } + if (!((info.attribute | static_cast<MemoryAttribute>(ignore_attr)) == + (first_attr | static_cast<MemoryAttribute>(ignore_attr)))) { + return ERR_INVALID_ADDRESS_STATE; + } + + // Validate against the provided masks + CASCADE_CODE(CheckMemoryState(info, state_mask, state, perm_mask, perm, attr_mask, attr)); + + // Break once we're done + if (last_addr <= info.GetLastAddress()) { + break; + } + + // Advance our iterator + it++; + ASSERT(it != block_manager->cend()); + info = it->GetMemoryInfo(); + } + + // Write output state + if (out_state) { + *out_state = first_state; + } + if (out_perm) { + *out_perm = first_perm; + } + if (out_attr) { + *out_attr = first_attr & static_cast<MemoryAttribute>(~ignore_attr); + } + + return RESULT_SUCCESS; +} + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_table.h b/src/core/hle/kernel/memory/page_table.h new file mode 100644 index 000000000..ce0d38849 --- /dev/null +++ b/src/core/hle/kernel/memory/page_table.h @@ -0,0 +1,277 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include <memory> +#include <mutex> + +#include "common/common_types.h" +#include "common/page_table.h" +#include "core/file_sys/program_metadata.h" +#include "core/hle/kernel/memory/memory_block.h" +#include "core/hle/kernel/memory/memory_manager.h" +#include "core/hle/result.h" + +namespace Core { +class System; +} + +namespace Kernel::Memory { + +class MemoryBlockManager; + +class PageTable final : NonCopyable { +public: + explicit PageTable(Core::System& system); + + ResultCode InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, + VAddr code_addr, std::size_t code_size, + Memory::MemoryManager::Pool pool); + ResultCode MapProcessCode(VAddr addr, std::size_t pages_count, MemoryState state, + MemoryPermission perm); + ResultCode MapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); + ResultCode UnmapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); + ResultCode MapPhysicalMemory(VAddr addr, std::size_t size); + ResultCode UnmapPhysicalMemory(VAddr addr, std::size_t size); + ResultCode UnmapMemory(VAddr addr, std::size_t size); + ResultCode Map(VAddr dst_addr, VAddr src_addr, std::size_t size); + ResultCode Unmap(VAddr dst_addr, VAddr src_addr, std::size_t size); + ResultCode MapPages(VAddr addr, PageLinkedList& page_linked_list, MemoryState state, + MemoryPermission perm); + ResultCode SetCodeMemoryPermission(VAddr addr, std::size_t size, MemoryPermission perm); + MemoryInfo QueryInfo(VAddr addr); + ResultCode ReserveTransferMemory(VAddr addr, std::size_t size, MemoryPermission perm); + ResultCode ResetTransferMemory(VAddr addr, std::size_t size); + ResultCode SetMemoryAttribute(VAddr addr, std::size_t size, MemoryAttribute mask, + MemoryAttribute value); + ResultCode SetHeapCapacity(std::size_t new_heap_capacity); + ResultVal<VAddr> SetHeapSize(std::size_t size); + ResultVal<VAddr> AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, + bool is_map_only, VAddr region_start, + std::size_t region_num_pages, MemoryState state, + MemoryPermission perm, PAddr map_addr = 0); + ResultCode LockForDeviceAddressSpace(VAddr addr, std::size_t size); + ResultCode UnlockForDeviceAddressSpace(VAddr addr, std::size_t size); + + Common::PageTable& PageTableImpl() { + return page_table_impl; + } + + const Common::PageTable& PageTableImpl() const { + return page_table_impl; + } + +private: + enum class OperationType : u32 { + Map, + MapGroup, + Unmap, + ChangePermissions, + ChangePermissionsAndRefresh, + }; + + static constexpr MemoryAttribute DefaultMemoryIgnoreAttr = + MemoryAttribute::DontCareMask | MemoryAttribute::IpcLocked | MemoryAttribute::DeviceShared; + + ResultCode InitializeMemoryLayout(VAddr start, VAddr end); + ResultCode MapPages(VAddr addr, const PageLinkedList& page_linked_list, MemoryPermission perm); + void MapPhysicalMemory(PageLinkedList& page_linked_list, VAddr start, VAddr end); + bool IsRegionMapped(VAddr address, u64 size); + bool IsRegionContiguous(VAddr addr, u64 size) const; + void AddRegionToPages(VAddr start, std::size_t num_pages, PageLinkedList& page_linked_list); + MemoryInfo QueryInfoImpl(VAddr addr); + VAddr AllocateVirtualMemory(VAddr start, std::size_t region_num_pages, u64 needed_num_pages, + std::size_t align); + ResultCode Operate(VAddr addr, std::size_t num_pages, const PageLinkedList& page_group, + OperationType operation); + ResultCode Operate(VAddr addr, std::size_t num_pages, MemoryPermission perm, + OperationType operation, PAddr map_addr = 0); + constexpr VAddr GetRegionAddress(MemoryState state) const; + constexpr std::size_t GetRegionSize(MemoryState state) const; + constexpr bool CanContain(VAddr addr, std::size_t size, MemoryState state) const; + + constexpr ResultCode CheckMemoryState(const MemoryInfo& info, MemoryState state_mask, + MemoryState state, MemoryPermission perm_mask, + MemoryPermission perm, MemoryAttribute attr_mask, + MemoryAttribute attr) const; + ResultCode CheckMemoryState(MemoryState* out_state, MemoryPermission* out_perm, + MemoryAttribute* out_attr, VAddr addr, std::size_t size, + MemoryState state_mask, MemoryState state, + MemoryPermission perm_mask, MemoryPermission perm, + MemoryAttribute attr_mask, MemoryAttribute attr, + MemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr); + ResultCode CheckMemoryState(VAddr addr, std::size_t size, MemoryState state_mask, + MemoryState state, MemoryPermission perm_mask, + MemoryPermission perm, MemoryAttribute attr_mask, + MemoryAttribute attr, + MemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) { + return CheckMemoryState(nullptr, nullptr, nullptr, addr, size, state_mask, state, perm_mask, + perm, attr_mask, attr, ignore_attr); + } + + std::recursive_mutex page_table_lock; + std::unique_ptr<MemoryBlockManager> block_manager; + +public: + constexpr VAddr GetAddressSpaceStart() const { + return address_space_start; + } + constexpr VAddr GetAddressSpaceEnd() const { + return address_space_end; + } + constexpr std::size_t GetAddressSpaceSize() const { + return address_space_end - address_space_start; + } + constexpr VAddr GetHeapRegionStart() const { + return heap_region_start; + } + constexpr VAddr GetHeapRegionEnd() const { + return heap_region_end; + } + constexpr std::size_t GetHeapRegionSize() const { + return heap_region_end - heap_region_start; + } + constexpr VAddr GetAliasRegionStart() const { + return alias_region_start; + } + constexpr VAddr GetAliasRegionEnd() const { + return alias_region_end; + } + constexpr std::size_t GetAliasRegionSize() const { + return alias_region_end - alias_region_start; + } + constexpr VAddr GetStackRegionStart() const { + return stack_region_start; + } + constexpr VAddr GetStackRegionEnd() const { + return stack_region_end; + } + constexpr std::size_t GetStackRegionSize() const { + return stack_region_end - stack_region_start; + } + constexpr VAddr GetKernelMapRegionStart() const { + return kernel_map_region_start; + } + constexpr VAddr GetKernelMapRegionEnd() const { + return kernel_map_region_end; + } + constexpr VAddr GetCodeRegionStart() const { + return code_region_start; + } + constexpr VAddr GetCodeRegionEnd() const { + return code_region_end; + } + constexpr VAddr GetAliasCodeRegionStart() const { + return alias_code_region_start; + } + constexpr VAddr GetAliasCodeRegionSize() const { + return alias_code_region_end - alias_code_region_start; + } + constexpr std::size_t GetAddressSpaceWidth() const { + return address_space_width; + } + constexpr std::size_t GetHeapSize() { + return current_heap_addr - heap_region_start; + } + constexpr std::size_t GetTotalHeapSize() { + return GetHeapSize() + physical_memory_usage; + } + constexpr bool IsInsideAddressSpace(VAddr address, std::size_t size) const { + return address_space_start <= address && address + size - 1 <= address_space_end - 1; + } + constexpr bool IsOutsideAliasRegion(VAddr address, std::size_t size) const { + return alias_region_start > address || address + size - 1 > alias_region_end - 1; + } + constexpr bool IsOutsideStackRegion(VAddr address, std::size_t size) const { + return stack_region_start > address || address + size - 1 > stack_region_end - 1; + } + constexpr bool IsInvalidRegion(VAddr address, std::size_t size) const { + return address + size - 1 > GetAliasCodeRegionStart() + GetAliasCodeRegionSize() - 1; + } + constexpr bool IsInsideHeapRegion(VAddr address, std::size_t size) const { + return address + size > heap_region_start && heap_region_end > address; + } + constexpr bool IsInsideAliasRegion(VAddr address, std::size_t size) const { + return address + size > alias_region_start && alias_region_end > address; + } + constexpr bool IsOutsideASLRRegion(VAddr address, std::size_t size) const { + if (IsInvalidRegion(address, size)) { + return true; + } + if (IsInsideHeapRegion(address, size)) { + return true; + } + if (IsInsideAliasRegion(address, size)) { + return true; + } + return {}; + } + constexpr bool IsInsideASLRRegion(VAddr address, std::size_t size) const { + return !IsOutsideASLRRegion(address, size); + } + constexpr PAddr GetPhysicalAddr(VAddr addr) { + return page_table_impl.backing_addr[addr >> Memory::PageBits] + addr; + } + +private: + constexpr bool Contains(VAddr addr) const { + return address_space_start <= addr && addr <= address_space_end - 1; + } + constexpr bool Contains(VAddr addr, std::size_t size) const { + return address_space_start <= addr && addr < addr + size && + addr + size - 1 <= address_space_end - 1; + } + constexpr bool IsKernel() const { + return is_kernel; + } + constexpr bool IsAslrEnabled() const { + return is_aslr_enabled; + } + + constexpr std::size_t GetNumGuardPages() const { + return IsKernel() ? 1 : 4; + } + + constexpr bool ContainsPages(VAddr addr, std::size_t num_pages) const { + return (address_space_start <= addr) && + (num_pages <= (address_space_end - address_space_start) / PageSize) && + (addr + num_pages * PageSize - 1 <= address_space_end - 1); + } + +private: + VAddr address_space_start{}; + VAddr address_space_end{}; + VAddr heap_region_start{}; + VAddr heap_region_end{}; + VAddr current_heap_end{}; + VAddr alias_region_start{}; + VAddr alias_region_end{}; + VAddr stack_region_start{}; + VAddr stack_region_end{}; + VAddr kernel_map_region_start{}; + VAddr kernel_map_region_end{}; + VAddr code_region_start{}; + VAddr code_region_end{}; + VAddr alias_code_region_start{}; + VAddr alias_code_region_end{}; + VAddr current_heap_addr{}; + + std::size_t heap_capacity{}; + std::size_t physical_memory_usage{}; + std::size_t max_heap_size{}; + std::size_t max_physical_memory_size{}; + std::size_t address_space_width{}; + + bool is_kernel{}; + bool is_aslr_enabled{}; + + MemoryManager::Pool memory_pool{MemoryManager::Pool::Application}; + + Common::PageTable page_table_impl; + + Core::System& system; +}; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/slab_heap.h b/src/core/hle/kernel/memory/slab_heap.h new file mode 100644 index 000000000..465eaddb3 --- /dev/null +++ b/src/core/hle/kernel/memory/slab_heap.h @@ -0,0 +1,163 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +// This file references various implementation details from Atmosphere, an open-source firmware for +// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. + +#pragma once + +#include <atomic> + +#include "common/assert.h" +#include "common/common_types.h" + +namespace Kernel::Memory { + +namespace impl { + +class SlabHeapImpl final : NonCopyable { +public: + struct Node { + Node* next{}; + }; + + constexpr SlabHeapImpl() = default; + + void Initialize(std::size_t size) { + ASSERT(head == nullptr); + obj_size = size; + } + + constexpr std::size_t GetObjectSize() const { + return obj_size; + } + + Node* GetHead() const { + return head; + } + + void* Allocate() { + Node* ret = head.load(); + + do { + if (ret == nullptr) { + break; + } + } while (!head.compare_exchange_weak(ret, ret->next)); + + return ret; + } + + void Free(void* obj) { + Node* node = static_cast<Node*>(obj); + + Node* cur_head = head.load(); + do { + node->next = cur_head; + } while (!head.compare_exchange_weak(cur_head, node)); + } + +private: + std::atomic<Node*> head{}; + std::size_t obj_size{}; +}; + +} // namespace impl + +class SlabHeapBase : NonCopyable { +public: + constexpr SlabHeapBase() = default; + + constexpr bool Contains(uintptr_t addr) const { + return start <= addr && addr < end; + } + + constexpr std::size_t GetSlabHeapSize() const { + return (end - start) / GetObjectSize(); + } + + constexpr std::size_t GetObjectSize() const { + return impl.GetObjectSize(); + } + + constexpr uintptr_t GetSlabHeapAddress() const { + return start; + } + + std::size_t GetObjectIndexImpl(const void* obj) const { + return (reinterpret_cast<uintptr_t>(obj) - start) / GetObjectSize(); + } + + std::size_t GetPeakIndex() const { + return GetObjectIndexImpl(reinterpret_cast<const void*>(peak)); + } + + void* AllocateImpl() { + return impl.Allocate(); + } + + void FreeImpl(void* obj) { + // Don't allow freeing an object that wasn't allocated from this heap + ASSERT(Contains(reinterpret_cast<uintptr_t>(obj))); + impl.Free(obj); + } + + void InitializeImpl(std::size_t obj_size, void* memory, std::size_t memory_size) { + // Ensure we don't initialize a slab using null memory + ASSERT(memory != nullptr); + + // Initialize the base allocator + impl.Initialize(obj_size); + + // Set our tracking variables + const std::size_t num_obj = (memory_size / obj_size); + start = reinterpret_cast<uintptr_t>(memory); + end = start + num_obj * obj_size; + peak = start; + + // Free the objects + u8* cur = reinterpret_cast<u8*>(end); + + for (std::size_t i{}; i < num_obj; i++) { + cur -= obj_size; + impl.Free(cur); + } + } + +private: + using Impl = impl::SlabHeapImpl; + + Impl impl; + uintptr_t peak{}; + uintptr_t start{}; + uintptr_t end{}; +}; + +template <typename T> +class SlabHeap final : public SlabHeapBase { +public: + constexpr SlabHeap() : SlabHeapBase() {} + + void Initialize(void* memory, std::size_t memory_size) { + InitializeImpl(sizeof(T), memory, memory_size); + } + + T* Allocate() { + T* obj = static_cast<T*>(AllocateImpl()); + if (obj != nullptr) { + new (obj) T(); + } + return obj; + } + + void Free(T* obj) { + FreeImpl(obj); + } + + constexpr std::size_t GetObjectIndex(const T* obj) const { + return GetObjectIndexImpl(obj); + } +}; + +} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/system_control.cpp b/src/core/hle/kernel/memory/system_control.cpp new file mode 100644 index 000000000..11d204bc2 --- /dev/null +++ b/src/core/hle/kernel/memory/system_control.cpp @@ -0,0 +1,40 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include <random> + +#include "core/hle/kernel/memory/system_control.h" + +namespace Kernel::Memory::SystemControl { +namespace { +template <typename F> +u64 GenerateUniformRange(u64 min, u64 max, F f) { + // Handle the case where the difference is too large to represent. + if (max == std::numeric_limits<u64>::max() && min == std::numeric_limits<u64>::min()) { + return f(); + } + + // Iterate until we get a value in range. + const u64 range_size = ((max + 1) - min); + const u64 effective_max = (std::numeric_limits<u64>::max() / range_size) * range_size; + while (true) { + if (const u64 rnd = f(); rnd < effective_max) { + return min + (rnd % range_size); + } + } +} + +u64 GenerateRandomU64ForInit() { + static std::random_device device; + static std::mt19937 gen(device()); + static std::uniform_int_distribution<u64> distribution(1, std::numeric_limits<u64>::max()); + return distribution(gen); +} +} // Anonymous namespace + +u64 GenerateRandomRange(u64 min, u64 max) { + return GenerateUniformRange(min, max, GenerateRandomU64ForInit); +} + +} // namespace Kernel::Memory::SystemControl diff --git a/src/core/hle/kernel/memory/system_control.h b/src/core/hle/kernel/memory/system_control.h new file mode 100644 index 000000000..19cab8cbc --- /dev/null +++ b/src/core/hle/kernel/memory/system_control.h @@ -0,0 +1,13 @@ +// Copyright 2020 yuzu Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include "common/common_types.h" + +namespace Kernel::Memory::SystemControl { + +u64 GenerateRandomRange(u64 min, u64 max); + +} // namespace Kernel::Memory::SystemControl diff --git a/src/core/hle/kernel/mutex.cpp b/src/core/hle/kernel/mutex.cpp index eff4e45b0..8f6c944d1 100644 --- a/src/core/hle/kernel/mutex.cpp +++ b/src/core/hle/kernel/mutex.cpp @@ -7,6 +7,7 @@ #include <vector> #include "common/assert.h" +#include "common/logging/log.h" #include "core/core.h" #include "core/hle/kernel/errors.h" #include "core/hle/kernel/handle_table.h" @@ -33,8 +34,6 @@ static std::pair<std::shared_ptr<Thread>, u32> GetHighestPriorityMutexWaitingThr if (thread->GetMutexWaitAddress() != mutex_addr) continue; - ASSERT(thread->GetStatus() == ThreadStatus::WaitMutex); - ++num_waiters; if (highest_priority_thread == nullptr || thread->GetPriority() < highest_priority_thread->GetPriority()) { @@ -48,6 +47,7 @@ static std::pair<std::shared_ptr<Thread>, u32> GetHighestPriorityMutexWaitingThr /// Update the mutex owner field of all threads waiting on the mutex to point to the new owner. static void TransferMutexOwnership(VAddr mutex_addr, std::shared_ptr<Thread> current_thread, std::shared_ptr<Thread> new_owner) { + current_thread->RemoveMutexWaiter(new_owner); const auto threads = current_thread->GetMutexWaitingThreads(); for (const auto& thread : threads) { if (thread->GetMutexWaitAddress() != mutex_addr) @@ -67,85 +67,104 @@ ResultCode Mutex::TryAcquire(VAddr address, Handle holding_thread_handle, Handle requesting_thread_handle) { // The mutex address must be 4-byte aligned if ((address % sizeof(u32)) != 0) { + LOG_ERROR(Kernel, "Address is not 4-byte aligned! address={:016X}", address); return ERR_INVALID_ADDRESS; } - const auto& handle_table = system.Kernel().CurrentProcess()->GetHandleTable(); + auto& kernel = system.Kernel(); std::shared_ptr<Thread> current_thread = - SharedFrom(system.CurrentScheduler().GetCurrentThread()); - std::shared_ptr<Thread> holding_thread = handle_table.Get<Thread>(holding_thread_handle); - std::shared_ptr<Thread> requesting_thread = handle_table.Get<Thread>(requesting_thread_handle); + SharedFrom(kernel.CurrentScheduler().GetCurrentThread()); + { + SchedulerLock lock(kernel); + // The mutex address must be 4-byte aligned + if ((address % sizeof(u32)) != 0) { + return ERR_INVALID_ADDRESS; + } - // TODO(Subv): It is currently unknown if it is possible to lock a mutex in behalf of another - // thread. - ASSERT(requesting_thread == current_thread); + const auto& handle_table = kernel.CurrentProcess()->GetHandleTable(); + std::shared_ptr<Thread> holding_thread = handle_table.Get<Thread>(holding_thread_handle); + std::shared_ptr<Thread> requesting_thread = + handle_table.Get<Thread>(requesting_thread_handle); - const u32 addr_value = system.Memory().Read32(address); + // TODO(Subv): It is currently unknown if it is possible to lock a mutex in behalf of + // another thread. + ASSERT(requesting_thread == current_thread); - // If the mutex isn't being held, just return success. - if (addr_value != (holding_thread_handle | Mutex::MutexHasWaitersFlag)) { - return RESULT_SUCCESS; - } + current_thread->SetSynchronizationResults(nullptr, RESULT_SUCCESS); - if (holding_thread == nullptr) { - return ERR_INVALID_HANDLE; - } + const u32 addr_value = system.Memory().Read32(address); + + // If the mutex isn't being held, just return success. + if (addr_value != (holding_thread_handle | Mutex::MutexHasWaitersFlag)) { + return RESULT_SUCCESS; + } - // Wait until the mutex is released - current_thread->SetMutexWaitAddress(address); - current_thread->SetWaitHandle(requesting_thread_handle); + if (holding_thread == nullptr) { + return ERR_INVALID_HANDLE; + } - current_thread->SetStatus(ThreadStatus::WaitMutex); - current_thread->InvalidateWakeupCallback(); + // Wait until the mutex is released + current_thread->SetMutexWaitAddress(address); + current_thread->SetWaitHandle(requesting_thread_handle); - // Update the lock holder thread's priority to prevent priority inversion. - holding_thread->AddMutexWaiter(current_thread); + current_thread->SetStatus(ThreadStatus::WaitMutex); - system.PrepareReschedule(); + // Update the lock holder thread's priority to prevent priority inversion. + holding_thread->AddMutexWaiter(current_thread); + } - return RESULT_SUCCESS; + { + SchedulerLock lock(kernel); + auto* owner = current_thread->GetLockOwner(); + if (owner != nullptr) { + owner->RemoveMutexWaiter(current_thread); + } + } + return current_thread->GetSignalingResult(); } -ResultCode Mutex::Release(VAddr address) { +std::pair<ResultCode, std::shared_ptr<Thread>> Mutex::Unlock(std::shared_ptr<Thread> owner, + VAddr address) { // The mutex address must be 4-byte aligned if ((address % sizeof(u32)) != 0) { - return ERR_INVALID_ADDRESS; + LOG_ERROR(Kernel, "Address is not 4-byte aligned! address={:016X}", address); + return {ERR_INVALID_ADDRESS, nullptr}; } - std::shared_ptr<Thread> current_thread = - SharedFrom(system.CurrentScheduler().GetCurrentThread()); - auto [thread, num_waiters] = GetHighestPriorityMutexWaitingThread(current_thread, address); - - // There are no more threads waiting for the mutex, release it completely. - if (thread == nullptr) { + auto [new_owner, num_waiters] = GetHighestPriorityMutexWaitingThread(owner, address); + if (new_owner == nullptr) { system.Memory().Write32(address, 0); - return RESULT_SUCCESS; + return {RESULT_SUCCESS, nullptr}; } - // Transfer the ownership of the mutex from the previous owner to the new one. - TransferMutexOwnership(address, current_thread, thread); - - u32 mutex_value = thread->GetWaitHandle(); - + TransferMutexOwnership(address, owner, new_owner); + u32 mutex_value = new_owner->GetWaitHandle(); if (num_waiters >= 2) { // Notify the guest that there are still some threads waiting for the mutex mutex_value |= Mutex::MutexHasWaitersFlag; } + new_owner->SetSynchronizationResults(nullptr, RESULT_SUCCESS); + new_owner->SetLockOwner(nullptr); + new_owner->ResumeFromWait(); - // Grant the mutex to the next waiting thread and resume it. system.Memory().Write32(address, mutex_value); + return {RESULT_SUCCESS, new_owner}; +} - ASSERT(thread->GetStatus() == ThreadStatus::WaitMutex); - thread->ResumeFromWait(); +ResultCode Mutex::Release(VAddr address) { + auto& kernel = system.Kernel(); + SchedulerLock lock(kernel); - thread->SetLockOwner(nullptr); - thread->SetCondVarWaitAddress(0); - thread->SetMutexWaitAddress(0); - thread->SetWaitHandle(0); - thread->SetWaitSynchronizationResult(RESULT_SUCCESS); + std::shared_ptr<Thread> current_thread = + SharedFrom(kernel.CurrentScheduler().GetCurrentThread()); - system.PrepareReschedule(); + auto [result, new_owner] = Unlock(current_thread, address); - return RESULT_SUCCESS; + if (result != RESULT_SUCCESS && new_owner != nullptr) { + new_owner->SetSynchronizationResults(nullptr, result); + } + + return result; } + } // namespace Kernel diff --git a/src/core/hle/kernel/mutex.h b/src/core/hle/kernel/mutex.h index b904de2e8..3b81dc3df 100644 --- a/src/core/hle/kernel/mutex.h +++ b/src/core/hle/kernel/mutex.h @@ -28,6 +28,10 @@ public: ResultCode TryAcquire(VAddr address, Handle holding_thread_handle, Handle requesting_thread_handle); + /// Unlocks a mutex for owner at address + std::pair<ResultCode, std::shared_ptr<Thread>> Unlock(std::shared_ptr<Thread> owner, + VAddr address); + /// Releases the mutex at the specified address. ResultCode Release(VAddr address); diff --git a/src/core/hle/kernel/physical_core.cpp b/src/core/hle/kernel/physical_core.cpp index aa2787467..6e04d025f 100644 --- a/src/core/hle/kernel/physical_core.cpp +++ b/src/core/hle/kernel/physical_core.cpp @@ -2,63 +2,43 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. -#include "common/logging/log.h" -#include "core/arm/arm_interface.h" -#ifdef ARCHITECTURE_x86_64 -#include "core/arm/dynarmic/arm_dynarmic_32.h" -#include "core/arm/dynarmic/arm_dynarmic_64.h" -#endif -#include "core/arm/exclusive_monitor.h" -#include "core/arm/unicorn/arm_unicorn.h" +#include "common/spin_lock.h" +#include "core/arm/cpu_interrupt_handler.h" #include "core/core.h" #include "core/hle/kernel/physical_core.h" #include "core/hle/kernel/scheduler.h" -#include "core/hle/kernel/thread.h" namespace Kernel { -PhysicalCore::PhysicalCore(Core::System& system, std::size_t id, - Core::ExclusiveMonitor& exclusive_monitor) - : core_index{id} { -#ifdef ARCHITECTURE_x86_64 - arm_interface_32 = - std::make_unique<Core::ARM_Dynarmic_32>(system, exclusive_monitor, core_index); - arm_interface_64 = - std::make_unique<Core::ARM_Dynarmic_64>(system, exclusive_monitor, core_index); - -#else - arm_interface = std::make_shared<Core::ARM_Unicorn>(system); - LOG_WARNING(Core, "CPU JIT requested, but Dynarmic not available"); -#endif - - scheduler = std::make_unique<Kernel::Scheduler>(system, core_index); -} +PhysicalCore::PhysicalCore(Core::System& system, std::size_t id, Kernel::Scheduler& scheduler, + Core::CPUInterruptHandler& interrupt_handler) + : interrupt_handler{interrupt_handler}, + core_index{id}, scheduler{scheduler}, guard{std::make_unique<Common::SpinLock>()} {} PhysicalCore::~PhysicalCore() = default; -void PhysicalCore::Run() { - arm_interface->Run(); - arm_interface->ClearExclusiveState(); +void PhysicalCore::Idle() { + interrupt_handler.AwaitInterrupt(); } -void PhysicalCore::Step() { - arm_interface->Step(); +void PhysicalCore::Shutdown() { + scheduler.Shutdown(); } -void PhysicalCore::Stop() { - arm_interface->PrepareReschedule(); +bool PhysicalCore::IsInterrupted() const { + return interrupt_handler.IsInterrupted(); } -void PhysicalCore::Shutdown() { - scheduler->Shutdown(); +void PhysicalCore::Interrupt() { + guard->lock(); + interrupt_handler.SetInterrupt(true); + guard->unlock(); } -void PhysicalCore::SetIs64Bit(bool is_64_bit) { - if (is_64_bit) { - arm_interface = arm_interface_64.get(); - } else { - arm_interface = arm_interface_32.get(); - } +void PhysicalCore::ClearInterrupt() { + guard->lock(); + interrupt_handler.SetInterrupt(false); + guard->unlock(); } } // namespace Kernel diff --git a/src/core/hle/kernel/physical_core.h b/src/core/hle/kernel/physical_core.h index 3269166be..d7a7a951c 100644 --- a/src/core/hle/kernel/physical_core.h +++ b/src/core/hle/kernel/physical_core.h @@ -7,12 +7,17 @@ #include <cstddef> #include <memory> +namespace Common { +class SpinLock; +} + namespace Kernel { class Scheduler; } // namespace Kernel namespace Core { class ARM_Interface; +class CPUInterruptHandler; class ExclusiveMonitor; class System; } // namespace Core @@ -21,7 +26,8 @@ namespace Kernel { class PhysicalCore { public: - PhysicalCore(Core::System& system, std::size_t id, Core::ExclusiveMonitor& exclusive_monitor); + PhysicalCore(Core::System& system, std::size_t id, Kernel::Scheduler& scheduler, + Core::CPUInterruptHandler& interrupt_handler); ~PhysicalCore(); PhysicalCore(const PhysicalCore&) = delete; @@ -30,23 +36,18 @@ public: PhysicalCore(PhysicalCore&&) = default; PhysicalCore& operator=(PhysicalCore&&) = default; - /// Execute current jit state - void Run(); - /// Execute a single instruction in current jit. - void Step(); - /// Stop JIT execution/exit - void Stop(); + void Idle(); + /// Interrupt this physical core. + void Interrupt(); - // Shutdown this physical core. - void Shutdown(); + /// Clear this core's interrupt + void ClearInterrupt(); - Core::ARM_Interface& ArmInterface() { - return *arm_interface; - } + /// Check if this core is interrupted + bool IsInterrupted() const; - const Core::ARM_Interface& ArmInterface() const { - return *arm_interface; - } + // Shutdown this physical core. + void Shutdown(); bool IsMainCore() const { return core_index == 0; @@ -61,21 +62,18 @@ public: } Kernel::Scheduler& Scheduler() { - return *scheduler; + return scheduler; } const Kernel::Scheduler& Scheduler() const { - return *scheduler; + return scheduler; } - void SetIs64Bit(bool is_64_bit); - private: + Core::CPUInterruptHandler& interrupt_handler; std::size_t core_index; - std::unique_ptr<Core::ARM_Interface> arm_interface_32; - std::unique_ptr<Core::ARM_Interface> arm_interface_64; - std::unique_ptr<Kernel::Scheduler> scheduler; - Core::ARM_Interface* arm_interface{}; + Kernel::Scheduler& scheduler; + std::unique_ptr<Common::SpinLock> guard; }; } // namespace Kernel diff --git a/src/core/hle/kernel/physical_memory.h b/src/core/hle/kernel/physical_memory.h index b689e8e8b..7a0266780 100644 --- a/src/core/hle/kernel/physical_memory.h +++ b/src/core/hle/kernel/physical_memory.h @@ -4,6 +4,8 @@ #pragma once +#include <vector> + #include "common/alignment.h" namespace Kernel { diff --git a/src/core/hle/kernel/process.cpp b/src/core/hle/kernel/process.cpp index edc414d69..b17529dee 100644 --- a/src/core/hle/kernel/process.cpp +++ b/src/core/hle/kernel/process.cpp @@ -4,21 +4,26 @@ #include <algorithm> #include <bitset> +#include <ctime> #include <memory> #include <random> #include "common/alignment.h" #include "common/assert.h" #include "common/logging/log.h" #include "core/core.h" +#include "core/device_memory.h" #include "core/file_sys/program_metadata.h" #include "core/hle/kernel/code_set.h" #include "core/hle/kernel/errors.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/memory/memory_block_manager.h" +#include "core/hle/kernel/memory/page_table.h" +#include "core/hle/kernel/memory/slab_heap.h" #include "core/hle/kernel/process.h" #include "core/hle/kernel/resource_limit.h" #include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/thread.h" -#include "core/hle/kernel/vm_manager.h" +#include "core/hle/lock.h" #include "core/memory.h" #include "core/settings.h" @@ -27,26 +32,31 @@ namespace { /** * Sets up the primary application thread * + * @param system The system instance to create the main thread under. * @param owner_process The parent process for the main thread - * @param kernel The kernel instance to create the main thread under. * @param priority The priority to give the main thread */ -void SetupMainThread(Process& owner_process, KernelCore& kernel, u32 priority) { - const auto& vm_manager = owner_process.VMManager(); - const VAddr entry_point = vm_manager.GetCodeRegionBaseAddress(); - const VAddr stack_top = vm_manager.GetTLSIORegionEndAddress(); - auto thread_res = Thread::Create(kernel, "main", entry_point, priority, 0, - owner_process.GetIdealCore(), stack_top, owner_process); +void SetupMainThread(Core::System& system, Process& owner_process, u32 priority, VAddr stack_top) { + const VAddr entry_point = owner_process.PageTable().GetCodeRegionStart(); + ThreadType type = THREADTYPE_USER; + auto thread_res = Thread::Create(system, type, "main", entry_point, priority, 0, + owner_process.GetIdealCore(), stack_top, &owner_process); std::shared_ptr<Thread> thread = std::move(thread_res).Unwrap(); // Register 1 must be a handle to the main thread const Handle thread_handle = owner_process.GetHandleTable().Create(thread).Unwrap(); + thread->GetContext32().cpu_registers[0] = 0; + thread->GetContext64().cpu_registers[0] = 0; thread->GetContext32().cpu_registers[1] = thread_handle; thread->GetContext64().cpu_registers[1] = thread_handle; + auto& kernel = system.Kernel(); // Threads by default are dormant, wake up the main thread so it runs when the scheduler fires - thread->ResumeFromWait(); + { + SchedulerLock lock{kernel}; + thread->SetStatus(ThreadStatus::Ready); + } } } // Anonymous namespace @@ -57,7 +67,8 @@ void SetupMainThread(Process& owner_process, KernelCore& kernel, u32 priority) { // (whichever page happens to have an available slot). class TLSPage { public: - static constexpr std::size_t num_slot_entries = Memory::PAGE_SIZE / Memory::TLS_ENTRY_SIZE; + static constexpr std::size_t num_slot_entries = + Core::Memory::PAGE_SIZE / Core::Memory::TLS_ENTRY_SIZE; explicit TLSPage(VAddr address) : base_address{address} {} @@ -76,7 +87,7 @@ public: } is_slot_used[i] = true; - return base_address + (i * Memory::TLS_ENTRY_SIZE); + return base_address + (i * Core::Memory::TLS_ENTRY_SIZE); } return std::nullopt; @@ -86,15 +97,15 @@ public: // Ensure that all given addresses are consistent with how TLS pages // are intended to be used when releasing slots. ASSERT(IsWithinPage(address)); - ASSERT((address % Memory::TLS_ENTRY_SIZE) == 0); + ASSERT((address % Core::Memory::TLS_ENTRY_SIZE) == 0); - const std::size_t index = (address - base_address) / Memory::TLS_ENTRY_SIZE; + const std::size_t index = (address - base_address) / Core::Memory::TLS_ENTRY_SIZE; is_slot_used[index] = false; } private: bool IsWithinPage(VAddr address) const { - return base_address <= address && address < base_address + Memory::PAGE_SIZE; + return base_address <= address && address < base_address + Core::Memory::PAGE_SIZE; } VAddr base_address; @@ -106,14 +117,14 @@ std::shared_ptr<Process> Process::Create(Core::System& system, std::string name, std::shared_ptr<Process> process = std::make_shared<Process>(system); process->name = std::move(name); - process->resource_limit = kernel.GetSystemResourceLimit(); + process->resource_limit = ResourceLimit::Create(kernel); process->status = ProcessStatus::Created; process->program_id = 0; process->process_id = type == ProcessType::KernelInternal ? kernel.CreateNewKernelProcessID() : kernel.CreateNewUserProcessID(); process->capabilities.InitializeForMetadatalessProcess(); - std::mt19937 rng(Settings::values.rng_seed.value_or(0)); + std::mt19937 rng(Settings::values.rng_seed.GetValue().value_or(std::time(nullptr))); std::uniform_int_distribution<u64> distribution; std::generate(process->random_entropy.begin(), process->random_entropy.end(), [&] { return distribution(rng); }); @@ -127,7 +138,15 @@ std::shared_ptr<ResourceLimit> Process::GetResourceLimit() const { } u64 Process::GetTotalPhysicalMemoryAvailable() const { - return vm_manager.GetTotalPhysicalMemoryAvailable(); + const u64 capacity{resource_limit->GetCurrentResourceValue(ResourceType::PhysicalMemory) + + page_table->GetTotalHeapSize() + GetSystemResourceSize() + image_size + + main_thread_stack_size}; + + if (capacity < memory_usage_capacity) { + return capacity; + } + + return memory_usage_capacity; } u64 Process::GetTotalPhysicalMemoryAvailableWithoutSystemResource() const { @@ -135,8 +154,8 @@ u64 Process::GetTotalPhysicalMemoryAvailableWithoutSystemResource() const { } u64 Process::GetTotalPhysicalMemoryUsed() const { - return vm_manager.GetCurrentHeapSize() + main_thread_stack_size + code_memory_size + - GetSystemResourceUsage(); + return image_size + main_thread_stack_size + page_table->GetTotalHeapSize() + + GetSystemResourceSize(); } u64 Process::GetTotalPhysicalMemoryUsedWithoutSystemResource() const { @@ -170,7 +189,6 @@ void Process::RemoveConditionVariableThread(std::shared_ptr<Thread> thread) { } ++it; } - UNREACHABLE(); } std::vector<std::shared_ptr<Thread>> Process::GetConditionVariableThreads( @@ -195,6 +213,7 @@ void Process::UnregisterThread(const Thread* thread) { } ResultCode Process::ClearSignalState() { + SchedulerLock lock(system.Kernel()); if (status == ProcessStatus::Exited) { LOG_ERROR(Kernel, "called on a terminated process instance."); return ERR_INVALID_STATE; @@ -209,33 +228,82 @@ ResultCode Process::ClearSignalState() { return RESULT_SUCCESS; } -ResultCode Process::LoadFromMetadata(const FileSys::ProgramMetadata& metadata) { +ResultCode Process::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, + std::size_t code_size) { program_id = metadata.GetTitleID(); ideal_core = metadata.GetMainThreadCore(); is_64bit_process = metadata.Is64BitProgram(); system_resource_size = metadata.GetSystemResourceSize(); + image_size = code_size; + + // Initialize proces address space + if (const ResultCode result{ + page_table->InitializeForProcess(metadata.GetAddressSpaceType(), false, 0x8000000, + code_size, Memory::MemoryManager::Pool::Application)}; + result.IsError()) { + return result; + } - vm_manager.Reset(metadata.GetAddressSpaceType()); + // Map process code region + if (const ResultCode result{page_table->MapProcessCode( + page_table->GetCodeRegionStart(), code_size / Memory::PageSize, + Memory::MemoryState::Code, Memory::MemoryPermission::None)}; + result.IsError()) { + return result; + } + + // Initialize process capabilities + const auto& caps{metadata.GetKernelCapabilities()}; + if (const ResultCode result{ + capabilities.InitializeForUserProcess(caps.data(), caps.size(), *page_table)}; + result.IsError()) { + return result; + } - const auto& caps = metadata.GetKernelCapabilities(); - const auto capability_init_result = - capabilities.InitializeForUserProcess(caps.data(), caps.size(), vm_manager); - if (capability_init_result.IsError()) { - return capability_init_result; + // Set memory usage capacity + switch (metadata.GetAddressSpaceType()) { + case FileSys::ProgramAddressSpaceType::Is32Bit: + case FileSys::ProgramAddressSpaceType::Is36Bit: + case FileSys::ProgramAddressSpaceType::Is39Bit: + memory_usage_capacity = page_table->GetHeapRegionEnd() - page_table->GetHeapRegionStart(); + break; + + case FileSys::ProgramAddressSpaceType::Is32BitNoMap: + memory_usage_capacity = page_table->GetHeapRegionEnd() - page_table->GetHeapRegionStart() + + page_table->GetAliasRegionEnd() - page_table->GetAliasRegionStart(); + break; + + default: + UNREACHABLE(); } + // Set initial resource limits + resource_limit->SetLimitValue( + ResourceType::PhysicalMemory, + kernel.MemoryManager().GetSize(Memory::MemoryManager::Pool::Application)); + resource_limit->SetLimitValue(ResourceType::Threads, 608); + resource_limit->SetLimitValue(ResourceType::Events, 700); + resource_limit->SetLimitValue(ResourceType::TransferMemory, 128); + resource_limit->SetLimitValue(ResourceType::Sessions, 894); + ASSERT(resource_limit->Reserve(ResourceType::PhysicalMemory, code_size)); + + // Create TLS region + tls_region_address = CreateTLSRegion(); + return handle_table.SetSize(capabilities.GetHandleTableSize()); } void Process::Run(s32 main_thread_priority, u64 stack_size) { AllocateMainThreadStack(stack_size); - tls_region_address = CreateTLSRegion(); - vm_manager.LogLayout(); + const std::size_t heap_capacity{memory_usage_capacity - main_thread_stack_size - image_size}; + ASSERT(!page_table->SetHeapCapacity(heap_capacity).IsError()); ChangeStatus(ProcessStatus::Running); - SetupMainThread(*this, kernel, main_thread_priority); + SetupMainThread(system, *this, main_thread_priority, main_thread_stack_top); + resource_limit->Reserve(ResourceType::Threads, 1); + resource_limit->Reserve(ResourceType::PhysicalMemory, main_thread_stack_size); } void Process::PrepareForTermination() { @@ -279,32 +347,39 @@ static auto FindTLSPageWithAvailableSlots(std::vector<TLSPage>& tls_pages) { } VAddr Process::CreateTLSRegion() { - auto tls_page_iter = FindTLSPageWithAvailableSlots(tls_pages); + SchedulerLock lock(system.Kernel()); + if (auto tls_page_iter{FindTLSPageWithAvailableSlots(tls_pages)}; + tls_page_iter != tls_pages.cend()) { + return *tls_page_iter->ReserveSlot(); + } - if (tls_page_iter == tls_pages.cend()) { - const auto region_address = - vm_manager.FindFreeRegion(vm_manager.GetTLSIORegionBaseAddress(), - vm_manager.GetTLSIORegionEndAddress(), Memory::PAGE_SIZE); - ASSERT(region_address.Succeeded()); + Memory::Page* const tls_page_ptr{kernel.GetUserSlabHeapPages().Allocate()}; + ASSERT(tls_page_ptr); - const auto map_result = vm_manager.MapMemoryBlock( - *region_address, std::make_shared<PhysicalMemory>(Memory::PAGE_SIZE), 0, - Memory::PAGE_SIZE, MemoryState::ThreadLocal); - ASSERT(map_result.Succeeded()); + const VAddr start{page_table->GetKernelMapRegionStart()}; + const VAddr size{page_table->GetKernelMapRegionEnd() - start}; + const PAddr tls_map_addr{system.DeviceMemory().GetPhysicalAddr(tls_page_ptr)}; + const VAddr tls_page_addr{ + page_table + ->AllocateAndMapMemory(1, Memory::PageSize, true, start, size / Memory::PageSize, + Memory::MemoryState::ThreadLocal, + Memory::MemoryPermission::ReadAndWrite, tls_map_addr) + .ValueOr(0)}; - tls_pages.emplace_back(*region_address); + ASSERT(tls_page_addr); - const auto reserve_result = tls_pages.back().ReserveSlot(); - ASSERT(reserve_result.has_value()); + std::memset(tls_page_ptr, 0, Memory::PageSize); + tls_pages.emplace_back(tls_page_addr); - return *reserve_result; - } + const auto reserve_result{tls_pages.back().ReserveSlot()}; + ASSERT(reserve_result.has_value()); - return *tls_page_iter->ReserveSlot(); + return *reserve_result; } void Process::FreeTLSRegion(VAddr tls_address) { - const VAddr aligned_address = Common::AlignDown(tls_address, Memory::PAGE_SIZE); + SchedulerLock lock(system.Kernel()); + const VAddr aligned_address = Common::AlignDown(tls_address, Core::Memory::PAGE_SIZE); auto iter = std::find_if(tls_pages.begin(), tls_pages.end(), [aligned_address](const auto& page) { return page.GetBaseAddress() == aligned_address; @@ -317,29 +392,24 @@ void Process::FreeTLSRegion(VAddr tls_address) { iter->ReleaseSlot(tls_address); } -void Process::LoadModule(CodeSet module_, VAddr base_addr) { - code_memory_size += module_.memory.size(); - - const auto memory = std::make_shared<PhysicalMemory>(std::move(module_.memory)); - - const auto MapSegment = [&](const CodeSet::Segment& segment, VMAPermission permissions, - MemoryState memory_state) { - const auto vma = vm_manager - .MapMemoryBlock(segment.addr + base_addr, memory, segment.offset, - segment.size, memory_state) - .Unwrap(); - vm_manager.Reprotect(vma, permissions); +void Process::LoadModule(CodeSet code_set, VAddr base_addr) { + std::lock_guard lock{HLE::g_hle_lock}; + const auto ReprotectSegment = [&](const CodeSet::Segment& segment, + Memory::MemoryPermission permission) { + page_table->SetCodeMemoryPermission(segment.addr + base_addr, segment.size, permission); }; - // Map CodeSet segments - MapSegment(module_.CodeSegment(), VMAPermission::ReadExecute, MemoryState::Code); - MapSegment(module_.RODataSegment(), VMAPermission::Read, MemoryState::CodeData); - MapSegment(module_.DataSegment(), VMAPermission::ReadWrite, MemoryState::CodeData); + system.Memory().WriteBlock(*this, base_addr, code_set.memory.data(), code_set.memory.size()); + + ReprotectSegment(code_set.CodeSegment(), Memory::MemoryPermission::ReadAndExecute); + ReprotectSegment(code_set.RODataSegment(), Memory::MemoryPermission::Read); + ReprotectSegment(code_set.DataSegment(), Memory::MemoryPermission::ReadAndWrite); } Process::Process(Core::System& system) - : SynchronizationObject{system.Kernel()}, vm_manager{system}, - address_arbiter{system}, mutex{system}, system{system} {} + : SynchronizationObject{system.Kernel()}, page_table{std::make_unique<Memory::PageTable>( + system)}, + handle_table{system.Kernel()}, address_arbiter{system}, mutex{system}, system{system} {} Process::~Process() = default; @@ -361,16 +431,24 @@ void Process::ChangeStatus(ProcessStatus new_status) { Signal(); } -void Process::AllocateMainThreadStack(u64 stack_size) { +ResultCode Process::AllocateMainThreadStack(std::size_t stack_size) { + ASSERT(stack_size); + // The kernel always ensures that the given stack size is page aligned. - main_thread_stack_size = Common::AlignUp(stack_size, Memory::PAGE_SIZE); - - // Allocate and map the main thread stack - const VAddr mapping_address = vm_manager.GetTLSIORegionEndAddress() - main_thread_stack_size; - vm_manager - .MapMemoryBlock(mapping_address, std::make_shared<PhysicalMemory>(main_thread_stack_size), - 0, main_thread_stack_size, MemoryState::Stack) - .Unwrap(); + main_thread_stack_size = Common::AlignUp(stack_size, Memory::PageSize); + + const VAddr start{page_table->GetStackRegionStart()}; + const std::size_t size{page_table->GetStackRegionEnd() - start}; + + CASCADE_RESULT(main_thread_stack_top, + page_table->AllocateAndMapMemory( + main_thread_stack_size / Memory::PageSize, Memory::PageSize, false, start, + size / Memory::PageSize, Memory::MemoryState::Stack, + Memory::MemoryPermission::ReadAndWrite)); + + main_thread_stack_top += main_thread_stack_size; + + return RESULT_SUCCESS; } } // namespace Kernel diff --git a/src/core/hle/kernel/process.h b/src/core/hle/kernel/process.h index 4887132a7..f45cb5674 100644 --- a/src/core/hle/kernel/process.h +++ b/src/core/hle/kernel/process.h @@ -16,7 +16,6 @@ #include "core/hle/kernel/mutex.h" #include "core/hle/kernel/process_capability.h" #include "core/hle/kernel/synchronization_object.h" -#include "core/hle/kernel/vm_manager.h" #include "core/hle/result.h" namespace Core { @@ -36,6 +35,10 @@ class TLSPage; struct CodeSet; +namespace Memory { +class PageTable; +} + enum class MemoryRegion : u16 { APPLICATION = 1, SYSTEM = 2, @@ -100,14 +103,14 @@ public: return HANDLE_TYPE; } - /// Gets a reference to the process' memory manager. - Kernel::VMManager& VMManager() { - return vm_manager; + /// Gets a reference to the process' page table. + Memory::PageTable& PageTable() { + return *page_table; } - /// Gets a const reference to the process' memory manager. - const Kernel::VMManager& VMManager() const { - return vm_manager; + /// Gets const a reference to the process' page table. + const Memory::PageTable& PageTable() const { + return *page_table; } /// Gets a reference to the process' handle table. @@ -273,7 +276,7 @@ public: * @returns RESULT_SUCCESS if all relevant metadata was able to be * loaded and parsed. Otherwise, an error code is returned. */ - ResultCode LoadFromMetadata(const FileSys::ProgramMetadata& metadata); + ResultCode LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std::size_t code_size); /** * Starts the main application thread for this process. @@ -289,7 +292,7 @@ public: */ void PrepareForTermination(); - void LoadModule(CodeSet module_, VAddr base_addr); + void LoadModule(CodeSet code_set, VAddr base_addr); /////////////////////////////////////////////////////////////////////////////////////////////// // Thread-local storage management @@ -313,16 +316,10 @@ private: void ChangeStatus(ProcessStatus new_status); /// Allocates the main thread stack for the process, given the stack size in bytes. - void AllocateMainThreadStack(u64 stack_size); - - /// Memory manager for this process. - Kernel::VMManager vm_manager; + ResultCode AllocateMainThreadStack(std::size_t stack_size); - /// Size of the main thread's stack in bytes. - u64 main_thread_stack_size = 0; - - /// Size of the loaded code memory in bytes. - u64 code_memory_size = 0; + /// Memory manager for this process + std::unique_ptr<Memory::PageTable> page_table; /// Current status of the process ProcessStatus status{}; @@ -385,11 +382,23 @@ private: /// List of threads waiting for a condition variable std::unordered_map<VAddr, std::list<std::shared_ptr<Thread>>> cond_var_threads; - /// System context - Core::System& system; + /// Address of the top of the main thread's stack + VAddr main_thread_stack_top{}; + + /// Size of the main thread's stack + std::size_t main_thread_stack_size{}; + + /// Memory usage capacity for the process + std::size_t memory_usage_capacity{}; + + /// Process total image size + std::size_t image_size{}; /// Name of this process std::string name; + + /// System context + Core::System& system; }; } // namespace Kernel diff --git a/src/core/hle/kernel/process_capability.cpp b/src/core/hle/kernel/process_capability.cpp index 583e35b79..63880f13d 100644 --- a/src/core/hle/kernel/process_capability.cpp +++ b/src/core/hle/kernel/process_capability.cpp @@ -3,10 +3,11 @@ // Refer to the license.txt file included. #include "common/bit_util.h" +#include "common/logging/log.h" #include "core/hle/kernel/errors.h" #include "core/hle/kernel/handle_table.h" +#include "core/hle/kernel/memory/page_table.h" #include "core/hle/kernel/process_capability.h" -#include "core/hle/kernel/vm_manager.h" namespace Kernel { namespace { @@ -66,7 +67,7 @@ u32 GetFlagBitOffset(CapabilityType type) { ResultCode ProcessCapabilities::InitializeForKernelProcess(const u32* capabilities, std::size_t num_capabilities, - VMManager& vm_manager) { + Memory::PageTable& page_table) { Clear(); // Allow all cores and priorities. @@ -74,15 +75,15 @@ ResultCode ProcessCapabilities::InitializeForKernelProcess(const u32* capabiliti priority_mask = 0xFFFFFFFFFFFFFFFF; kernel_version = PackedKernelVersion; - return ParseCapabilities(capabilities, num_capabilities, vm_manager); + return ParseCapabilities(capabilities, num_capabilities, page_table); } ResultCode ProcessCapabilities::InitializeForUserProcess(const u32* capabilities, std::size_t num_capabilities, - VMManager& vm_manager) { + Memory::PageTable& page_table) { Clear(); - return ParseCapabilities(capabilities, num_capabilities, vm_manager); + return ParseCapabilities(capabilities, num_capabilities, page_table); } void ProcessCapabilities::InitializeForMetadatalessProcess() { @@ -105,7 +106,7 @@ void ProcessCapabilities::InitializeForMetadatalessProcess() { ResultCode ProcessCapabilities::ParseCapabilities(const u32* capabilities, std::size_t num_capabilities, - VMManager& vm_manager) { + Memory::PageTable& page_table) { u32 set_flags = 0; u32 set_svc_bits = 0; @@ -119,22 +120,30 @@ ResultCode ProcessCapabilities::ParseCapabilities(const u32* capabilities, // The MapPhysical type uses two descriptor flags for its parameters. // If there's only one, then there's a problem. if (i >= num_capabilities) { + LOG_ERROR(Kernel, "Invalid combination! i={}", i); return ERR_INVALID_COMBINATION; } const auto size_flags = capabilities[i]; if (GetCapabilityType(size_flags) != CapabilityType::MapPhysical) { + LOG_ERROR(Kernel, "Invalid capability type! size_flags={}", size_flags); return ERR_INVALID_COMBINATION; } - const auto result = HandleMapPhysicalFlags(descriptor, size_flags, vm_manager); + const auto result = HandleMapPhysicalFlags(descriptor, size_flags, page_table); if (result.IsError()) { + LOG_ERROR(Kernel, "Failed to map physical flags! descriptor={}, size_flags={}", + descriptor, size_flags); return result; } } else { const auto result = - ParseSingleFlagCapability(set_flags, set_svc_bits, descriptor, vm_manager); + ParseSingleFlagCapability(set_flags, set_svc_bits, descriptor, page_table); if (result.IsError()) { + LOG_ERROR( + Kernel, + "Failed to parse capability flag! set_flags={}, set_svc_bits={}, descriptor={}", + set_flags, set_svc_bits, descriptor); return result; } } @@ -144,7 +153,7 @@ ResultCode ProcessCapabilities::ParseCapabilities(const u32* capabilities, } ResultCode ProcessCapabilities::ParseSingleFlagCapability(u32& set_flags, u32& set_svc_bits, - u32 flag, VMManager& vm_manager) { + u32 flag, Memory::PageTable& page_table) { const auto type = GetCapabilityType(flag); if (type == CapabilityType::Unset) { @@ -162,6 +171,9 @@ ResultCode ProcessCapabilities::ParseSingleFlagCapability(u32& set_flags, u32& s const u32 flag_length = GetFlagBitOffset(type); const u32 set_flag = 1U << flag_length; if ((set_flag & set_flags & InitializeOnceMask) != 0) { + LOG_ERROR(Kernel, + "Attempted to initialize flags that may only be initialized once. set_flags={}", + set_flags); return ERR_INVALID_COMBINATION; } set_flags |= set_flag; @@ -172,7 +184,7 @@ ResultCode ProcessCapabilities::ParseSingleFlagCapability(u32& set_flags, u32& s case CapabilityType::Syscall: return HandleSyscallFlags(set_svc_bits, flag); case CapabilityType::MapIO: - return HandleMapIOFlags(flag, vm_manager); + return HandleMapIOFlags(flag, page_table); case CapabilityType::Interrupt: return HandleInterruptFlags(flag); case CapabilityType::ProgramType: @@ -187,6 +199,7 @@ ResultCode ProcessCapabilities::ParseSingleFlagCapability(u32& set_flags, u32& s break; } + LOG_ERROR(Kernel, "Invalid capability type! type={}", static_cast<u32>(type)); return ERR_INVALID_CAPABILITY_DESCRIPTOR; } @@ -208,23 +221,31 @@ void ProcessCapabilities::Clear() { ResultCode ProcessCapabilities::HandlePriorityCoreNumFlags(u32 flags) { if (priority_mask != 0 || core_mask != 0) { + LOG_ERROR(Kernel, "Core or priority mask are not zero! priority_mask={}, core_mask={}", + priority_mask, core_mask); return ERR_INVALID_CAPABILITY_DESCRIPTOR; } const u32 core_num_min = (flags >> 16) & 0xFF; const u32 core_num_max = (flags >> 24) & 0xFF; if (core_num_min > core_num_max) { + LOG_ERROR(Kernel, "Core min is greater than core max! core_num_min={}, core_num_max={}", + core_num_min, core_num_max); return ERR_INVALID_COMBINATION; } const u32 priority_min = (flags >> 10) & 0x3F; const u32 priority_max = (flags >> 4) & 0x3F; if (priority_min > priority_max) { + LOG_ERROR(Kernel, + "Priority min is greater than priority max! priority_min={}, priority_max={}", + core_num_min, priority_max); return ERR_INVALID_COMBINATION; } // The switch only has 4 usable cores. if (core_num_max >= 4) { + LOG_ERROR(Kernel, "Invalid max cores specified! core_num_max={}", core_num_max); return ERR_INVALID_PROCESSOR_ID; } @@ -259,6 +280,7 @@ ResultCode ProcessCapabilities::HandleSyscallFlags(u32& set_svc_bits, u32 flags) } if (svc_number >= svc_capabilities.size()) { + LOG_ERROR(Kernel, "Process svc capability is out of range! svc_number={}", svc_number); return ERR_OUT_OF_RANGE; } @@ -269,12 +291,12 @@ ResultCode ProcessCapabilities::HandleSyscallFlags(u32& set_svc_bits, u32 flags) } ResultCode ProcessCapabilities::HandleMapPhysicalFlags(u32 flags, u32 size_flags, - VMManager& vm_manager) { + Memory::PageTable& page_table) { // TODO(Lioncache): Implement once the memory manager can handle this. return RESULT_SUCCESS; } -ResultCode ProcessCapabilities::HandleMapIOFlags(u32 flags, VMManager& vm_manager) { +ResultCode ProcessCapabilities::HandleMapIOFlags(u32 flags, Memory::PageTable& page_table) { // TODO(Lioncache): Implement once the memory manager can handle this. return RESULT_SUCCESS; } @@ -295,6 +317,8 @@ ResultCode ProcessCapabilities::HandleInterruptFlags(u32 flags) { // emulate that, it's sufficient to mark every interrupt as defined. if (interrupt >= interrupt_capabilities.size()) { + LOG_ERROR(Kernel, "Process interrupt capability is out of range! svc_number={}", + interrupt); return ERR_OUT_OF_RANGE; } @@ -307,6 +331,7 @@ ResultCode ProcessCapabilities::HandleInterruptFlags(u32 flags) { ResultCode ProcessCapabilities::HandleProgramTypeFlags(u32 flags) { const u32 reserved = flags >> 17; if (reserved != 0) { + LOG_ERROR(Kernel, "Reserved value is non-zero! reserved={}", reserved); return ERR_RESERVED_VALUE; } @@ -324,6 +349,9 @@ ResultCode ProcessCapabilities::HandleKernelVersionFlags(u32 flags) { const u32 major_version = kernel_version >> 19; if (major_version != 0 || flags < 0x80000) { + LOG_ERROR(Kernel, + "Kernel version is non zero or flags are too small! major_version={}, flags={}", + major_version, flags); return ERR_INVALID_CAPABILITY_DESCRIPTOR; } @@ -334,6 +362,7 @@ ResultCode ProcessCapabilities::HandleKernelVersionFlags(u32 flags) { ResultCode ProcessCapabilities::HandleHandleTableFlags(u32 flags) { const u32 reserved = flags >> 26; if (reserved != 0) { + LOG_ERROR(Kernel, "Reserved value is non-zero! reserved={}", reserved); return ERR_RESERVED_VALUE; } @@ -344,6 +373,7 @@ ResultCode ProcessCapabilities::HandleHandleTableFlags(u32 flags) { ResultCode ProcessCapabilities::HandleDebugFlags(u32 flags) { const u32 reserved = flags >> 19; if (reserved != 0) { + LOG_ERROR(Kernel, "Reserved value is non-zero! reserved={}", reserved); return ERR_RESERVED_VALUE; } diff --git a/src/core/hle/kernel/process_capability.h b/src/core/hle/kernel/process_capability.h index 5cdd80747..ea9d12c16 100644 --- a/src/core/hle/kernel/process_capability.h +++ b/src/core/hle/kernel/process_capability.h @@ -12,7 +12,9 @@ union ResultCode; namespace Kernel { -class VMManager; +namespace Memory { +class PageTable; +} /// The possible types of programs that may be indicated /// by the program type capability descriptor. @@ -81,27 +83,27 @@ public: /// /// @param capabilities The capabilities to parse /// @param num_capabilities The number of capabilities to parse. - /// @param vm_manager The memory manager to use for handling any mapping-related + /// @param page_table The memory manager to use for handling any mapping-related /// operations (such as mapping IO memory, etc). /// /// @returns RESULT_SUCCESS if this capabilities instance was able to be initialized, /// otherwise, an error code upon failure. /// ResultCode InitializeForKernelProcess(const u32* capabilities, std::size_t num_capabilities, - VMManager& vm_manager); + Memory::PageTable& page_table); /// Initializes this process capabilities instance for a userland process. /// /// @param capabilities The capabilities to parse. /// @param num_capabilities The total number of capabilities to parse. - /// @param vm_manager The memory manager to use for handling any mapping-related + /// @param page_table The memory manager to use for handling any mapping-related /// operations (such as mapping IO memory, etc). /// /// @returns RESULT_SUCCESS if this capabilities instance was able to be initialized, /// otherwise, an error code upon failure. /// ResultCode InitializeForUserProcess(const u32* capabilities, std::size_t num_capabilities, - VMManager& vm_manager); + Memory::PageTable& page_table); /// Initializes this process capabilities instance for a process that does not /// have any metadata to parse. @@ -181,13 +183,13 @@ private: /// /// @param capabilities The sequence of capability descriptors to parse. /// @param num_capabilities The number of descriptors within the given sequence. - /// @param vm_manager The memory manager that will perform any memory + /// @param page_table The memory manager that will perform any memory /// mapping if necessary. /// /// @return RESULT_SUCCESS if no errors occur, otherwise an error code. /// ResultCode ParseCapabilities(const u32* capabilities, std::size_t num_capabilities, - VMManager& vm_manager); + Memory::PageTable& page_table); /// Attempts to parse a capability descriptor that is only represented by a /// single flag set. @@ -196,13 +198,13 @@ private: /// flags being initialized more than once when they shouldn't be. /// @param set_svc_bits Running set of bits representing the allowed supervisor calls mask. /// @param flag The flag to attempt to parse. - /// @param vm_manager The memory manager that will perform any memory + /// @param page_table The memory manager that will perform any memory /// mapping if necessary. /// /// @return RESULT_SUCCESS if no errors occurred, otherwise an error code. /// ResultCode ParseSingleFlagCapability(u32& set_flags, u32& set_svc_bits, u32 flag, - VMManager& vm_manager); + Memory::PageTable& page_table); /// Clears the internal state of this process capability instance. Necessary, /// to have a sane starting point due to us allowing running executables without @@ -226,10 +228,10 @@ private: ResultCode HandleSyscallFlags(u32& set_svc_bits, u32 flags); /// Handles flags related to mapping physical memory pages. - ResultCode HandleMapPhysicalFlags(u32 flags, u32 size_flags, VMManager& vm_manager); + ResultCode HandleMapPhysicalFlags(u32 flags, u32 size_flags, Memory::PageTable& page_table); /// Handles flags related to mapping IO pages. - ResultCode HandleMapIOFlags(u32 flags, VMManager& vm_manager); + ResultCode HandleMapIOFlags(u32 flags, Memory::PageTable& page_table); /// Handles flags related to the interrupt capability flags. ResultCode HandleInterruptFlags(u32 flags); diff --git a/src/core/hle/kernel/readable_event.cpp b/src/core/hle/kernel/readable_event.cpp index 9d3d3a81b..6e286419e 100644 --- a/src/core/hle/kernel/readable_event.cpp +++ b/src/core/hle/kernel/readable_event.cpp @@ -4,9 +4,12 @@ #include <algorithm> #include "common/assert.h" +#include "common/logging/log.h" #include "core/hle/kernel/errors.h" +#include "core/hle/kernel/kernel.h" #include "core/hle/kernel/object.h" #include "core/hle/kernel/readable_event.h" +#include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/thread.h" namespace Kernel { @@ -23,10 +26,12 @@ void ReadableEvent::Acquire(Thread* thread) { } void ReadableEvent::Signal() { - if (!is_signaled) { - is_signaled = true; - SynchronizationObject::Signal(); - }; + if (is_signaled) { + return; + } + + is_signaled = true; + SynchronizationObject::Signal(); } void ReadableEvent::Clear() { @@ -34,7 +39,10 @@ void ReadableEvent::Clear() { } ResultCode ReadableEvent::Reset() { + SchedulerLock lock(kernel); if (!is_signaled) { + LOG_TRACE(Kernel, "Handle is not signaled! object_id={}, object_type={}, object_name={}", + GetObjectId(), GetTypeName(), GetName()); return ERR_INVALID_STATE; } diff --git a/src/core/hle/kernel/resource_limit.cpp b/src/core/hle/kernel/resource_limit.cpp index b53423462..212e442f4 100644 --- a/src/core/hle/kernel/resource_limit.cpp +++ b/src/core/hle/kernel/resource_limit.cpp @@ -16,26 +16,58 @@ constexpr std::size_t ResourceTypeToIndex(ResourceType type) { ResourceLimit::ResourceLimit(KernelCore& kernel) : Object{kernel} {} ResourceLimit::~ResourceLimit() = default; +bool ResourceLimit::Reserve(ResourceType resource, s64 amount) { + return Reserve(resource, amount, 10000000000); +} + +bool ResourceLimit::Reserve(ResourceType resource, s64 amount, u64 timeout) { + const std::size_t index{ResourceTypeToIndex(resource)}; + + s64 new_value = current[index] + amount; + if (new_value > limit[index] && available[index] + amount <= limit[index]) { + // TODO(bunnei): This is wrong for multicore, we should wait the calling thread for timeout + new_value = current[index] + amount; + } + + if (new_value <= limit[index]) { + current[index] = new_value; + return true; + } + return false; +} + +void ResourceLimit::Release(ResourceType resource, u64 amount) { + Release(resource, amount, amount); +} + +void ResourceLimit::Release(ResourceType resource, u64 used_amount, u64 available_amount) { + const std::size_t index{ResourceTypeToIndex(resource)}; + + current[index] -= used_amount; + available[index] -= available_amount; +} + std::shared_ptr<ResourceLimit> ResourceLimit::Create(KernelCore& kernel) { return std::make_shared<ResourceLimit>(kernel); } s64 ResourceLimit::GetCurrentResourceValue(ResourceType resource) const { - return values.at(ResourceTypeToIndex(resource)); + return limit.at(ResourceTypeToIndex(resource)) - current.at(ResourceTypeToIndex(resource)); } s64 ResourceLimit::GetMaxResourceValue(ResourceType resource) const { - return limits.at(ResourceTypeToIndex(resource)); + return limit.at(ResourceTypeToIndex(resource)); } ResultCode ResourceLimit::SetLimitValue(ResourceType resource, s64 value) { - const auto index = ResourceTypeToIndex(resource); - - if (value < values[index]) { + const std::size_t index{ResourceTypeToIndex(resource)}; + if (current[index] <= value) { + limit[index] = value; + return RESULT_SUCCESS; + } else { + LOG_ERROR(Kernel, "Limit value is too large! resource={}, value={}, index={}", + static_cast<u32>(resource), value, index); return ERR_INVALID_STATE; } - - values[index] = value; - return RESULT_SUCCESS; } } // namespace Kernel diff --git a/src/core/hle/kernel/resource_limit.h b/src/core/hle/kernel/resource_limit.h index 53b89e621..936cc4d0f 100644 --- a/src/core/hle/kernel/resource_limit.h +++ b/src/core/hle/kernel/resource_limit.h @@ -51,6 +51,11 @@ public: return HANDLE_TYPE; } + bool Reserve(ResourceType resource, s64 amount); + bool Reserve(ResourceType resource, s64 amount, u64 timeout); + void Release(ResourceType resource, u64 amount); + void Release(ResourceType resource, u64 used_amount, u64 available_amount); + /** * Gets the current value for the specified resource. * @param resource Requested resource type @@ -91,10 +96,9 @@ private: using ResourceArray = std::array<s64, static_cast<std::size_t>(ResourceType::ResourceTypeCount)>; - /// Maximum values a resource type may reach. - ResourceArray limits{}; - /// Current resource limit values. - ResourceArray values{}; + ResourceArray limit{}; + ResourceArray current{}; + ResourceArray available{}; }; } // namespace Kernel diff --git a/src/core/hle/kernel/scheduler.cpp b/src/core/hle/kernel/scheduler.cpp index 1140c72a3..6b7db5372 100644 --- a/src/core/hle/kernel/scheduler.cpp +++ b/src/core/hle/kernel/scheduler.cpp @@ -6,16 +6,21 @@ // licensed under GPLv2 or later under exception provided by the author. #include <algorithm> +#include <mutex> #include <set> #include <unordered_set> #include <utility> #include "common/assert.h" +#include "common/bit_util.h" +#include "common/fiber.h" #include "common/logging/log.h" #include "core/arm/arm_interface.h" #include "core/core.h" #include "core/core_timing.h" +#include "core/cpu_manager.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/physical_core.h" #include "core/hle/kernel/process.h" #include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/time_manager.h" @@ -27,103 +32,149 @@ GlobalScheduler::GlobalScheduler(KernelCore& kernel) : kernel{kernel} {} GlobalScheduler::~GlobalScheduler() = default; void GlobalScheduler::AddThread(std::shared_ptr<Thread> thread) { + std::scoped_lock lock{global_list_guard}; thread_list.push_back(std::move(thread)); } void GlobalScheduler::RemoveThread(std::shared_ptr<Thread> thread) { + std::scoped_lock lock{global_list_guard}; thread_list.erase(std::remove(thread_list.begin(), thread_list.end(), thread), thread_list.end()); } -void GlobalScheduler::UnloadThread(std::size_t core) { - Scheduler& sched = kernel.Scheduler(core); - sched.UnloadThread(); -} - -void GlobalScheduler::SelectThread(std::size_t core) { +u32 GlobalScheduler::SelectThreads() { + ASSERT(is_locked); const auto update_thread = [](Thread* thread, Scheduler& sched) { - if (thread != sched.selected_thread.get()) { + std::scoped_lock lock{sched.guard}; + if (thread != sched.selected_thread_set.get()) { if (thread == nullptr) { ++sched.idle_selection_count; } - sched.selected_thread = SharedFrom(thread); + sched.selected_thread_set = SharedFrom(thread); } - sched.is_context_switch_pending = sched.selected_thread != sched.current_thread; + const bool reschedule_pending = + sched.is_context_switch_pending || (sched.selected_thread_set != sched.current_thread); + sched.is_context_switch_pending = reschedule_pending; std::atomic_thread_fence(std::memory_order_seq_cst); + return reschedule_pending; }; - Scheduler& sched = kernel.Scheduler(core); - Thread* current_thread = nullptr; - // Step 1: Get top thread in schedule queue. - current_thread = scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front(); - if (current_thread) { - update_thread(current_thread, sched); - return; + if (!is_reselection_pending.load()) { + return 0; } - // Step 2: Try selecting a suggested thread. - Thread* winner = nullptr; - std::set<s32> sug_cores; - for (auto thread : suggested_queue[core]) { - s32 this_core = thread->GetProcessorID(); - Thread* thread_on_core = nullptr; - if (this_core >= 0) { - thread_on_core = scheduled_queue[this_core].front(); - } - if (this_core < 0 || thread != thread_on_core) { - winner = thread; - break; + std::array<Thread*, Core::Hardware::NUM_CPU_CORES> top_threads{}; + + u32 idle_cores{}; + + // Step 1: Get top thread in schedule queue. + for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { + Thread* top_thread = + scheduled_queue[core].empty() ? nullptr : scheduled_queue[core].front(); + if (top_thread != nullptr) { + // TODO(Blinkhawk): Implement Thread Pinning + } else { + idle_cores |= (1U << core); } - sug_cores.insert(this_core); + top_threads[core] = top_thread; } - // if we got a suggested thread, select it, else do a second pass. - if (winner && winner->GetPriority() > 2) { - if (winner->IsRunning()) { - UnloadThread(static_cast<u32>(winner->GetProcessorID())); + + while (idle_cores != 0) { + u32 core_id = Common::CountTrailingZeroes32(idle_cores); + + if (!suggested_queue[core_id].empty()) { + std::array<s32, Core::Hardware::NUM_CPU_CORES> migration_candidates{}; + std::size_t num_candidates = 0; + auto iter = suggested_queue[core_id].begin(); + Thread* suggested = nullptr; + // Step 2: Try selecting a suggested thread. + while (iter != suggested_queue[core_id].end()) { + suggested = *iter; + iter++; + s32 suggested_core_id = suggested->GetProcessorID(); + Thread* top_thread = + suggested_core_id >= 0 ? top_threads[suggested_core_id] : nullptr; + if (top_thread != suggested) { + if (top_thread != nullptr && + top_thread->GetPriority() < THREADPRIO_MAX_CORE_MIGRATION) { + suggested = nullptr; + break; + // There's a too high thread to do core migration, cancel + } + TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id), suggested); + break; + } + suggested = nullptr; + migration_candidates[num_candidates++] = suggested_core_id; + } + // Step 3: Select a suggested thread from another core + if (suggested == nullptr) { + for (std::size_t i = 0; i < num_candidates; i++) { + s32 candidate_core = migration_candidates[i]; + suggested = top_threads[candidate_core]; + auto it = scheduled_queue[candidate_core].begin(); + it++; + Thread* next = it != scheduled_queue[candidate_core].end() ? *it : nullptr; + if (next != nullptr) { + TransferToCore(suggested->GetPriority(), static_cast<s32>(core_id), + suggested); + top_threads[candidate_core] = next; + break; + } else { + suggested = nullptr; + } + } + } + top_threads[core_id] = suggested; } - TransferToCore(winner->GetPriority(), static_cast<s32>(core), winner); - update_thread(winner, sched); - return; + + idle_cores &= ~(1U << core_id); } - // Step 3: Select a suggested thread from another core - for (auto& src_core : sug_cores) { - auto it = scheduled_queue[src_core].begin(); - it++; - if (it != scheduled_queue[src_core].end()) { - Thread* thread_on_core = scheduled_queue[src_core].front(); - Thread* to_change = *it; - if (thread_on_core->IsRunning() || to_change->IsRunning()) { - UnloadThread(static_cast<u32>(src_core)); - } - TransferToCore(thread_on_core->GetPriority(), static_cast<s32>(core), thread_on_core); - current_thread = thread_on_core; - break; + u32 cores_needing_context_switch{}; + for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { + Scheduler& sched = kernel.Scheduler(core); + ASSERT(top_threads[core] == nullptr || + static_cast<u32>(top_threads[core]->GetProcessorID()) == core); + if (update_thread(top_threads[core], sched)) { + cores_needing_context_switch |= (1U << core); } } - update_thread(current_thread, sched); + return cores_needing_context_switch; } bool GlobalScheduler::YieldThread(Thread* yielding_thread) { + ASSERT(is_locked); // Note: caller should use critical section, etc. + if (!yielding_thread->IsRunnable()) { + // Normally this case shouldn't happen except for SetThreadActivity. + is_reselection_pending.store(true, std::memory_order_release); + return false; + } const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID()); const u32 priority = yielding_thread->GetPriority(); // Yield the thread - const Thread* const winner = scheduled_queue[core_id].front(priority); - ASSERT_MSG(yielding_thread == winner, "Thread yielding without being in front"); - scheduled_queue[core_id].yield(priority); + Reschedule(priority, core_id, yielding_thread); + const Thread* const winner = scheduled_queue[core_id].front(); + if (kernel.GetCurrentHostThreadID() != core_id) { + is_reselection_pending.store(true, std::memory_order_release); + } return AskForReselectionOrMarkRedundant(yielding_thread, winner); } bool GlobalScheduler::YieldThreadAndBalanceLoad(Thread* yielding_thread) { + ASSERT(is_locked); // Note: caller should check if !thread.IsSchedulerOperationRedundant and use critical section, // etc. + if (!yielding_thread->IsRunnable()) { + // Normally this case shouldn't happen except for SetThreadActivity. + is_reselection_pending.store(true, std::memory_order_release); + return false; + } const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID()); const u32 priority = yielding_thread->GetPriority(); // Yield the thread - ASSERT_MSG(yielding_thread == scheduled_queue[core_id].front(priority), - "Thread yielding without being in front"); - scheduled_queue[core_id].yield(priority); + Reschedule(priority, core_id, yielding_thread); std::array<Thread*, Core::Hardware::NUM_CPU_CORES> current_threads; for (std::size_t i = 0; i < current_threads.size(); i++) { @@ -153,21 +204,28 @@ bool GlobalScheduler::YieldThreadAndBalanceLoad(Thread* yielding_thread) { if (winner != nullptr) { if (winner != yielding_thread) { - if (winner->IsRunning()) { - UnloadThread(static_cast<u32>(winner->GetProcessorID())); - } TransferToCore(winner->GetPriority(), s32(core_id), winner); } } else { winner = next_thread; } + if (kernel.GetCurrentHostThreadID() != core_id) { + is_reselection_pending.store(true, std::memory_order_release); + } + return AskForReselectionOrMarkRedundant(yielding_thread, winner); } bool GlobalScheduler::YieldThreadAndWaitForLoadBalancing(Thread* yielding_thread) { + ASSERT(is_locked); // Note: caller should check if !thread.IsSchedulerOperationRedundant and use critical section, // etc. + if (!yielding_thread->IsRunnable()) { + // Normally this case shouldn't happen except for SetThreadActivity. + is_reselection_pending.store(true, std::memory_order_release); + return false; + } Thread* winner = nullptr; const u32 core_id = static_cast<u32>(yielding_thread->GetProcessorID()); @@ -195,25 +253,31 @@ bool GlobalScheduler::YieldThreadAndWaitForLoadBalancing(Thread* yielding_thread } if (winner != nullptr) { if (winner != yielding_thread) { - if (winner->IsRunning()) { - UnloadThread(static_cast<u32>(winner->GetProcessorID())); - } TransferToCore(winner->GetPriority(), static_cast<s32>(core_id), winner); } } else { winner = yielding_thread; } + } else { + winner = scheduled_queue[core_id].front(); + } + + if (kernel.GetCurrentHostThreadID() != core_id) { + is_reselection_pending.store(true, std::memory_order_release); } return AskForReselectionOrMarkRedundant(yielding_thread, winner); } void GlobalScheduler::PreemptThreads() { + ASSERT(is_locked); for (std::size_t core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) { const u32 priority = preemption_priorities[core_id]; if (scheduled_queue[core_id].size(priority) > 0) { - scheduled_queue[core_id].front(priority)->IncrementYieldCount(); + if (scheduled_queue[core_id].size(priority) > 1) { + scheduled_queue[core_id].front(priority)->IncrementYieldCount(); + } scheduled_queue[core_id].yield(priority); if (scheduled_queue[core_id].size(priority) > 1) { scheduled_queue[core_id].front(priority)->IncrementYieldCount(); @@ -247,9 +311,6 @@ void GlobalScheduler::PreemptThreads() { } if (winner != nullptr) { - if (winner->IsRunning()) { - UnloadThread(static_cast<u32>(winner->GetProcessorID())); - } TransferToCore(winner->GetPriority(), s32(core_id), winner); current_thread = winner->GetPriority() <= current_thread->GetPriority() ? winner : current_thread; @@ -280,9 +341,6 @@ void GlobalScheduler::PreemptThreads() { } if (winner != nullptr) { - if (winner->IsRunning()) { - UnloadThread(static_cast<u32>(winner->GetProcessorID())); - } TransferToCore(winner->GetPriority(), s32(core_id), winner); current_thread = winner; } @@ -292,34 +350,65 @@ void GlobalScheduler::PreemptThreads() { } } +void GlobalScheduler::EnableInterruptAndSchedule(u32 cores_pending_reschedule, + Core::EmuThreadHandle global_thread) { + u32 current_core = global_thread.host_handle; + bool must_context_switch = global_thread.guest_handle != InvalidHandle && + (current_core < Core::Hardware::NUM_CPU_CORES); + while (cores_pending_reschedule != 0) { + u32 core = Common::CountTrailingZeroes32(cores_pending_reschedule); + ASSERT(core < Core::Hardware::NUM_CPU_CORES); + if (!must_context_switch || core != current_core) { + auto& phys_core = kernel.PhysicalCore(core); + phys_core.Interrupt(); + } else { + must_context_switch = true; + } + cores_pending_reschedule &= ~(1U << core); + } + if (must_context_switch) { + auto& core_scheduler = kernel.CurrentScheduler(); + kernel.ExitSVCProfile(); + core_scheduler.TryDoContextSwitch(); + kernel.EnterSVCProfile(); + } +} + void GlobalScheduler::Suggest(u32 priority, std::size_t core, Thread* thread) { + ASSERT(is_locked); suggested_queue[core].add(thread, priority); } void GlobalScheduler::Unsuggest(u32 priority, std::size_t core, Thread* thread) { + ASSERT(is_locked); suggested_queue[core].remove(thread, priority); } void GlobalScheduler::Schedule(u32 priority, std::size_t core, Thread* thread) { + ASSERT(is_locked); ASSERT_MSG(thread->GetProcessorID() == s32(core), "Thread must be assigned to this core."); scheduled_queue[core].add(thread, priority); } void GlobalScheduler::SchedulePrepend(u32 priority, std::size_t core, Thread* thread) { + ASSERT(is_locked); ASSERT_MSG(thread->GetProcessorID() == s32(core), "Thread must be assigned to this core."); scheduled_queue[core].add(thread, priority, false); } void GlobalScheduler::Reschedule(u32 priority, std::size_t core, Thread* thread) { + ASSERT(is_locked); scheduled_queue[core].remove(thread, priority); scheduled_queue[core].add(thread, priority); } void GlobalScheduler::Unschedule(u32 priority, std::size_t core, Thread* thread) { + ASSERT(is_locked); scheduled_queue[core].remove(thread, priority); } void GlobalScheduler::TransferToCore(u32 priority, s32 destination_core, Thread* thread) { + ASSERT(is_locked); const bool schedulable = thread->GetPriority() < THREADPRIO_COUNT; const s32 source_core = thread->GetProcessorID(); if (source_core == destination_core || !schedulable) { @@ -349,6 +438,108 @@ bool GlobalScheduler::AskForReselectionOrMarkRedundant(Thread* current_thread, } } +void GlobalScheduler::AdjustSchedulingOnStatus(Thread* thread, u32 old_flags) { + if (old_flags == thread->scheduling_state) { + return; + } + ASSERT(is_locked); + + if (old_flags == static_cast<u32>(ThreadSchedStatus::Runnable)) { + // In this case the thread was running, now it's pausing/exitting + if (thread->processor_id >= 0) { + Unschedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread); + } + + for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { + if (core != static_cast<u32>(thread->processor_id) && + ((thread->affinity_mask >> core) & 1) != 0) { + Unsuggest(thread->current_priority, core, thread); + } + } + } else if (thread->scheduling_state == static_cast<u32>(ThreadSchedStatus::Runnable)) { + // The thread is now set to running from being stopped + if (thread->processor_id >= 0) { + Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread); + } + + for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { + if (core != static_cast<u32>(thread->processor_id) && + ((thread->affinity_mask >> core) & 1) != 0) { + Suggest(thread->current_priority, core, thread); + } + } + } + + SetReselectionPending(); +} + +void GlobalScheduler::AdjustSchedulingOnPriority(Thread* thread, u32 old_priority) { + if (thread->scheduling_state != static_cast<u32>(ThreadSchedStatus::Runnable)) { + return; + } + ASSERT(is_locked); + if (thread->processor_id >= 0) { + Unschedule(old_priority, static_cast<u32>(thread->processor_id), thread); + } + + for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { + if (core != static_cast<u32>(thread->processor_id) && + ((thread->affinity_mask >> core) & 1) != 0) { + Unsuggest(old_priority, core, thread); + } + } + + if (thread->processor_id >= 0) { + if (thread == kernel.CurrentScheduler().GetCurrentThread()) { + SchedulePrepend(thread->current_priority, static_cast<u32>(thread->processor_id), + thread); + } else { + Schedule(thread->current_priority, static_cast<u32>(thread->processor_id), thread); + } + } + + for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { + if (core != static_cast<u32>(thread->processor_id) && + ((thread->affinity_mask >> core) & 1) != 0) { + Suggest(thread->current_priority, core, thread); + } + } + thread->IncrementYieldCount(); + SetReselectionPending(); +} + +void GlobalScheduler::AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask, + s32 old_core) { + if (thread->scheduling_state != static_cast<u32>(ThreadSchedStatus::Runnable) || + thread->current_priority >= THREADPRIO_COUNT) { + return; + } + ASSERT(is_locked); + + for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { + if (((old_affinity_mask >> core) & 1) != 0) { + if (core == static_cast<u32>(old_core)) { + Unschedule(thread->current_priority, core, thread); + } else { + Unsuggest(thread->current_priority, core, thread); + } + } + } + + for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { + if (((thread->affinity_mask >> core) & 1) != 0) { + if (core == static_cast<u32>(thread->processor_id)) { + Schedule(thread->current_priority, core, thread); + } else { + Suggest(thread->current_priority, core, thread); + } + } + } + + thread->IncrementYieldCount(); + SetReselectionPending(); +} + void GlobalScheduler::Shutdown() { for (std::size_t core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { scheduled_queue[core].clear(); @@ -359,10 +550,12 @@ void GlobalScheduler::Shutdown() { void GlobalScheduler::Lock() { Core::EmuThreadHandle current_thread = kernel.GetCurrentEmuThreadID(); + ASSERT(!current_thread.IsInvalid()); if (current_thread == current_owner) { ++scope_lock; } else { inner_lock.lock(); + is_locked = true; current_owner = current_thread; ASSERT(current_owner != Core::EmuThreadHandle::InvalidHandle()); scope_lock = 1; @@ -374,17 +567,18 @@ void GlobalScheduler::Unlock() { ASSERT(scope_lock > 0); return; } - for (std::size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { - SelectThread(i); - } + u32 cores_pending_reschedule = SelectThreads(); + Core::EmuThreadHandle leaving_thread = current_owner; current_owner = Core::EmuThreadHandle::InvalidHandle(); scope_lock = 1; + is_locked = false; inner_lock.unlock(); - // TODO(Blinkhawk): Setup the interrupts and change context on current core. + EnableInterruptAndSchedule(cores_pending_reschedule, leaving_thread); } -Scheduler::Scheduler(Core::System& system, std::size_t core_id) - : system{system}, core_id{core_id} {} +Scheduler::Scheduler(Core::System& system, std::size_t core_id) : system(system), core_id(core_id) { + switch_fiber = std::make_shared<Common::Fiber>(std::function<void(void*)>(OnSwitch), this); +} Scheduler::~Scheduler() = default; @@ -393,56 +587,122 @@ bool Scheduler::HaveReadyThreads() const { } Thread* Scheduler::GetCurrentThread() const { - return current_thread.get(); + if (current_thread) { + return current_thread.get(); + } + return idle_thread.get(); } Thread* Scheduler::GetSelectedThread() const { return selected_thread.get(); } -void Scheduler::SelectThreads() { - system.GlobalScheduler().SelectThread(core_id); -} - u64 Scheduler::GetLastContextSwitchTicks() const { return last_context_switch_time; } void Scheduler::TryDoContextSwitch() { + auto& phys_core = system.Kernel().CurrentPhysicalCore(); + if (phys_core.IsInterrupted()) { + phys_core.ClearInterrupt(); + } + guard.lock(); if (is_context_switch_pending) { SwitchContext(); + } else { + guard.unlock(); } } -void Scheduler::UnloadThread() { - Thread* const previous_thread = GetCurrentThread(); - Process* const previous_process = system.Kernel().CurrentProcess(); +void Scheduler::OnThreadStart() { + SwitchContextStep2(); +} - UpdateLastContextSwitchTime(previous_thread, previous_process); +void Scheduler::Unload() { + Thread* thread = current_thread.get(); + if (thread) { + thread->SetContinuousOnSVC(false); + thread->last_running_ticks = system.CoreTiming().GetCPUTicks(); + thread->SetIsRunning(false); + if (!thread->IsHLEThread() && !thread->HasExited()) { + Core::ARM_Interface& cpu_core = thread->ArmInterface(); + cpu_core.SaveContext(thread->GetContext32()); + cpu_core.SaveContext(thread->GetContext64()); + // Save the TPIDR_EL0 system register in case it was modified. + thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0()); + cpu_core.ClearExclusiveState(); + } + thread->context_guard.unlock(); + } +} - // Save context for previous thread - if (previous_thread) { - system.ArmInterface(core_id).SaveContext(previous_thread->GetContext32()); - system.ArmInterface(core_id).SaveContext(previous_thread->GetContext64()); - // Save the TPIDR_EL0 system register in case it was modified. - previous_thread->SetTPIDR_EL0(system.ArmInterface(core_id).GetTPIDR_EL0()); +void Scheduler::Reload() { + Thread* thread = current_thread.get(); + if (thread) { + ASSERT_MSG(thread->GetSchedulingStatus() == ThreadSchedStatus::Runnable, + "Thread must be runnable."); + + // Cancel any outstanding wakeup events for this thread + thread->SetIsRunning(true); + thread->SetWasRunning(false); + thread->last_running_ticks = system.CoreTiming().GetCPUTicks(); - if (previous_thread->GetStatus() == ThreadStatus::Running) { - // This is only the case when a reschedule is triggered without the current thread - // yielding execution (i.e. an event triggered, system core time-sliced, etc) - previous_thread->SetStatus(ThreadStatus::Ready); + auto* const thread_owner_process = thread->GetOwnerProcess(); + if (thread_owner_process != nullptr) { + system.Kernel().MakeCurrentProcess(thread_owner_process); + } + if (!thread->IsHLEThread()) { + Core::ARM_Interface& cpu_core = thread->ArmInterface(); + cpu_core.LoadContext(thread->GetContext32()); + cpu_core.LoadContext(thread->GetContext64()); + cpu_core.SetTlsAddress(thread->GetTLSAddress()); + cpu_core.SetTPIDR_EL0(thread->GetTPIDR_EL0()); + cpu_core.ChangeProcessorID(this->core_id); + cpu_core.ClearExclusiveState(); } - previous_thread->SetIsRunning(false); } - current_thread = nullptr; +} + +void Scheduler::SwitchContextStep2() { + // Load context of new thread + if (selected_thread) { + ASSERT_MSG(selected_thread->GetSchedulingStatus() == ThreadSchedStatus::Runnable, + "Thread must be runnable."); + + // Cancel any outstanding wakeup events for this thread + selected_thread->SetIsRunning(true); + selected_thread->last_running_ticks = system.CoreTiming().GetCPUTicks(); + selected_thread->SetWasRunning(false); + + auto* const thread_owner_process = current_thread->GetOwnerProcess(); + if (thread_owner_process != nullptr) { + system.Kernel().MakeCurrentProcess(thread_owner_process); + } + if (!selected_thread->IsHLEThread()) { + Core::ARM_Interface& cpu_core = selected_thread->ArmInterface(); + cpu_core.LoadContext(selected_thread->GetContext32()); + cpu_core.LoadContext(selected_thread->GetContext64()); + cpu_core.SetTlsAddress(selected_thread->GetTLSAddress()); + cpu_core.SetTPIDR_EL0(selected_thread->GetTPIDR_EL0()); + cpu_core.ChangeProcessorID(this->core_id); + cpu_core.ClearExclusiveState(); + } + } + + TryDoContextSwitch(); } void Scheduler::SwitchContext() { - Thread* const previous_thread = GetCurrentThread(); - Thread* const new_thread = GetSelectedThread(); + current_thread_prev = current_thread; + selected_thread = selected_thread_set; + Thread* previous_thread = current_thread_prev.get(); + Thread* new_thread = selected_thread.get(); + current_thread = selected_thread; is_context_switch_pending = false; + if (new_thread == previous_thread) { + guard.unlock(); return; } @@ -452,51 +712,80 @@ void Scheduler::SwitchContext() { // Save context for previous thread if (previous_thread) { - system.ArmInterface(core_id).SaveContext(previous_thread->GetContext32()); - system.ArmInterface(core_id).SaveContext(previous_thread->GetContext64()); - // Save the TPIDR_EL0 system register in case it was modified. - previous_thread->SetTPIDR_EL0(system.ArmInterface(core_id).GetTPIDR_EL0()); - - if (previous_thread->GetStatus() == ThreadStatus::Running) { - // This is only the case when a reschedule is triggered without the current thread - // yielding execution (i.e. an event triggered, system core time-sliced, etc) - previous_thread->SetStatus(ThreadStatus::Ready); + if (new_thread != nullptr && new_thread->IsSuspendThread()) { + previous_thread->SetWasRunning(true); } + previous_thread->SetContinuousOnSVC(false); + previous_thread->last_running_ticks = system.CoreTiming().GetCPUTicks(); previous_thread->SetIsRunning(false); + if (!previous_thread->IsHLEThread() && !previous_thread->HasExited()) { + Core::ARM_Interface& cpu_core = previous_thread->ArmInterface(); + cpu_core.SaveContext(previous_thread->GetContext32()); + cpu_core.SaveContext(previous_thread->GetContext64()); + // Save the TPIDR_EL0 system register in case it was modified. + previous_thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0()); + cpu_core.ClearExclusiveState(); + } + previous_thread->context_guard.unlock(); } - // Load context of new thread - if (new_thread) { - ASSERT_MSG(new_thread->GetProcessorID() == s32(this->core_id), - "Thread must be assigned to this core."); - ASSERT_MSG(new_thread->GetStatus() == ThreadStatus::Ready, - "Thread must be ready to become running."); + std::shared_ptr<Common::Fiber>* old_context; + if (previous_thread != nullptr) { + old_context = &previous_thread->GetHostContext(); + } else { + old_context = &idle_thread->GetHostContext(); + } + guard.unlock(); - // Cancel any outstanding wakeup events for this thread - new_thread->CancelWakeupTimer(); - current_thread = SharedFrom(new_thread); - new_thread->SetStatus(ThreadStatus::Running); - new_thread->SetIsRunning(true); + Common::Fiber::YieldTo(*old_context, switch_fiber); + /// When a thread wakes up, the scheduler may have changed to other in another core. + auto& next_scheduler = system.Kernel().CurrentScheduler(); + next_scheduler.SwitchContextStep2(); +} - auto* const thread_owner_process = current_thread->GetOwnerProcess(); - if (previous_process != thread_owner_process) { - system.Kernel().MakeCurrentProcess(thread_owner_process); - } +void Scheduler::OnSwitch(void* this_scheduler) { + Scheduler* sched = static_cast<Scheduler*>(this_scheduler); + sched->SwitchToCurrent(); +} - system.ArmInterface(core_id).LoadContext(new_thread->GetContext32()); - system.ArmInterface(core_id).LoadContext(new_thread->GetContext64()); - system.ArmInterface(core_id).SetTlsAddress(new_thread->GetTLSAddress()); - system.ArmInterface(core_id).SetTPIDR_EL0(new_thread->GetTPIDR_EL0()); - } else { - current_thread = nullptr; - // Note: We do not reset the current process and current page table when idling because - // technically we haven't changed processes, our threads are just paused. +void Scheduler::SwitchToCurrent() { + while (true) { + { + std::scoped_lock lock{guard}; + selected_thread = selected_thread_set; + current_thread = selected_thread; + is_context_switch_pending = false; + } + const auto is_switch_pending = [this] { + std::scoped_lock lock{guard}; + return is_context_switch_pending; + }; + do { + if (current_thread != nullptr && !current_thread->IsHLEThread()) { + current_thread->context_guard.lock(); + if (!current_thread->IsRunnable()) { + current_thread->context_guard.unlock(); + break; + } + if (static_cast<u32>(current_thread->GetProcessorID()) != core_id) { + current_thread->context_guard.unlock(); + break; + } + } + std::shared_ptr<Common::Fiber>* next_context; + if (current_thread != nullptr) { + next_context = ¤t_thread->GetHostContext(); + } else { + next_context = &idle_thread->GetHostContext(); + } + Common::Fiber::YieldTo(switch_fiber, *next_context); + } while (!is_switch_pending()); } } void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) { const u64 prev_switch_ticks = last_context_switch_time; - const u64 most_recent_switch_ticks = system.CoreTiming().GetTicks(); + const u64 most_recent_switch_ticks = system.CoreTiming().GetCPUTicks(); const u64 update_ticks = most_recent_switch_ticks - prev_switch_ticks; if (thread != nullptr) { @@ -510,6 +799,16 @@ void Scheduler::UpdateLastContextSwitchTime(Thread* thread, Process* process) { last_context_switch_time = most_recent_switch_ticks; } +void Scheduler::Initialize() { + std::string name = "Idle Thread Id:" + std::to_string(core_id); + std::function<void(void*)> init_func = Core::CpuManager::GetIdleThreadStartFunc(); + void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater(); + ThreadType type = static_cast<ThreadType>(THREADTYPE_KERNEL | THREADTYPE_HLE | THREADTYPE_IDLE); + auto thread_res = Thread::Create(system, type, name, 0, 64, 0, static_cast<u32>(core_id), 0, + nullptr, std::move(init_func), init_func_parameter); + idle_thread = std::move(thread_res).Unwrap(); +} + void Scheduler::Shutdown() { current_thread = nullptr; selected_thread = nullptr; @@ -538,4 +837,13 @@ SchedulerLockAndSleep::~SchedulerLockAndSleep() { time_manager.ScheduleTimeEvent(event_handle, time_task, nanoseconds); } +void SchedulerLockAndSleep::Release() { + if (sleep_cancelled) { + return; + } + auto& time_manager = kernel.TimeManager(); + time_manager.ScheduleTimeEvent(event_handle, time_task, nanoseconds); + sleep_cancelled = true; +} + } // namespace Kernel diff --git a/src/core/hle/kernel/scheduler.h b/src/core/hle/kernel/scheduler.h index 07df33f9c..b6f04dcea 100644 --- a/src/core/hle/kernel/scheduler.h +++ b/src/core/hle/kernel/scheduler.h @@ -11,9 +11,14 @@ #include "common/common_types.h" #include "common/multi_level_queue.h" +#include "common/spin_lock.h" #include "core/hardware_properties.h" #include "core/hle/kernel/thread.h" +namespace Common { +class Fiber; +} + namespace Core { class ARM_Interface; class System; @@ -41,41 +46,17 @@ public: return thread_list; } - /** - * Add a thread to the suggested queue of a cpu core. Suggested threads may be - * picked if no thread is scheduled to run on the core. - */ - void Suggest(u32 priority, std::size_t core, Thread* thread); - - /** - * Remove a thread to the suggested queue of a cpu core. Suggested threads may be - * picked if no thread is scheduled to run on the core. - */ - void Unsuggest(u32 priority, std::size_t core, Thread* thread); - - /** - * Add a thread to the scheduling queue of a cpu core. The thread is added at the - * back the queue in its priority level. - */ - void Schedule(u32 priority, std::size_t core, Thread* thread); - - /** - * Add a thread to the scheduling queue of a cpu core. The thread is added at the - * front the queue in its priority level. - */ - void SchedulePrepend(u32 priority, std::size_t core, Thread* thread); + /// Notify the scheduler a thread's status has changed. + void AdjustSchedulingOnStatus(Thread* thread, u32 old_flags); - /// Reschedule an already scheduled thread based on a new priority - void Reschedule(u32 priority, std::size_t core, Thread* thread); - - /// Unschedules a thread. - void Unschedule(u32 priority, std::size_t core, Thread* thread); + /// Notify the scheduler a thread's priority has changed. + void AdjustSchedulingOnPriority(Thread* thread, u32 old_priority); - /// Selects a core and forces it to unload its current thread's context - void UnloadThread(std::size_t core); + /// Notify the scheduler a thread's core and/or affinity mask has changed. + void AdjustSchedulingOnAffinity(Thread* thread, u64 old_affinity_mask, s32 old_core); /** - * Takes care of selecting the new scheduled thread in three steps: + * Takes care of selecting the new scheduled threads in three steps: * * 1. First a thread is selected from the top of the priority queue. If no thread * is obtained then we move to step two, else we are done. @@ -85,8 +66,10 @@ public: * * 3. Third is no suggested thread is found, we do a second pass and pick a running * thread in another core and swap it with its current thread. + * + * returns the cores needing scheduling. */ - void SelectThread(std::size_t core); + u32 SelectThreads(); bool HaveReadyThreads(std::size_t core_id) const { return !scheduled_queue[core_id].empty(); @@ -149,6 +132,40 @@ private: /// Unlocks the scheduler, reselects threads, interrupts cores for rescheduling /// and reschedules current core if needed. void Unlock(); + + void EnableInterruptAndSchedule(u32 cores_pending_reschedule, + Core::EmuThreadHandle global_thread); + + /** + * Add a thread to the suggested queue of a cpu core. Suggested threads may be + * picked if no thread is scheduled to run on the core. + */ + void Suggest(u32 priority, std::size_t core, Thread* thread); + + /** + * Remove a thread to the suggested queue of a cpu core. Suggested threads may be + * picked if no thread is scheduled to run on the core. + */ + void Unsuggest(u32 priority, std::size_t core, Thread* thread); + + /** + * Add a thread to the scheduling queue of a cpu core. The thread is added at the + * back the queue in its priority level. + */ + void Schedule(u32 priority, std::size_t core, Thread* thread); + + /** + * Add a thread to the scheduling queue of a cpu core. The thread is added at the + * front the queue in its priority level. + */ + void SchedulePrepend(u32 priority, std::size_t core, Thread* thread); + + /// Reschedule an already scheduled thread based on a new priority + void Reschedule(u32 priority, std::size_t core, Thread* thread); + + /// Unschedules a thread. + void Unschedule(u32 priority, std::size_t core, Thread* thread); + /** * Transfers a thread into an specific core. If the destination_core is -1 * it will be unscheduled from its source code and added into its suggested @@ -170,10 +187,13 @@ private: std::array<u32, Core::Hardware::NUM_CPU_CORES> preemption_priorities = {59, 59, 59, 62}; /// Scheduler lock mechanisms. - std::mutex inner_lock{}; // TODO(Blinkhawk): Replace for a SpinLock + bool is_locked{}; + std::mutex inner_lock; std::atomic<s64> scope_lock{}; Core::EmuThreadHandle current_owner{Core::EmuThreadHandle::InvalidHandle()}; + Common::SpinLock global_list_guard{}; + /// Lists all thread ids that aren't deleted/etc. std::vector<std::shared_ptr<Thread>> thread_list; KernelCore& kernel; @@ -190,11 +210,11 @@ public: /// Reschedules to the next available thread (call after current thread is suspended) void TryDoContextSwitch(); - /// Unloads currently running thread - void UnloadThread(); - - /// Select the threads in top of the scheduling multilist. - void SelectThreads(); + /// The next two are for SingleCore Only. + /// Unload current thread before preempting core. + void Unload(); + /// Reload current thread after core preemption. + void Reload(); /// Gets the current running thread Thread* GetCurrentThread() const; @@ -209,15 +229,30 @@ public: return is_context_switch_pending; } + void Initialize(); + /// Shutdowns the scheduler. void Shutdown(); + void OnThreadStart(); + + std::shared_ptr<Common::Fiber>& ControlContext() { + return switch_fiber; + } + + const std::shared_ptr<Common::Fiber>& ControlContext() const { + return switch_fiber; + } + private: friend class GlobalScheduler; /// Switches the CPU's active thread context to that of the specified thread void SwitchContext(); + /// When a thread wakes up, it must run this through it's new scheduler + void SwitchContextStep2(); + /** * Called on every context switch to update the internal timestamp * This also updates the running time ticks for the given thread and @@ -231,20 +266,30 @@ private: */ void UpdateLastContextSwitchTime(Thread* thread, Process* process); + static void OnSwitch(void* this_scheduler); + void SwitchToCurrent(); + std::shared_ptr<Thread> current_thread = nullptr; std::shared_ptr<Thread> selected_thread = nullptr; + std::shared_ptr<Thread> current_thread_prev = nullptr; + std::shared_ptr<Thread> selected_thread_set = nullptr; + std::shared_ptr<Thread> idle_thread = nullptr; + + std::shared_ptr<Common::Fiber> switch_fiber = nullptr; Core::System& system; u64 last_context_switch_time = 0; u64 idle_selection_count = 0; const std::size_t core_id; + Common::SpinLock guard{}; + bool is_context_switch_pending = false; }; class SchedulerLock { public: - explicit SchedulerLock(KernelCore& kernel); + [[nodiscard]] explicit SchedulerLock(KernelCore& kernel); ~SchedulerLock(); protected: @@ -261,6 +306,8 @@ public: sleep_cancelled = true; } + void Release(); + private: Handle& event_handle; Thread* time_task; diff --git a/src/core/hle/kernel/server_session.cpp b/src/core/hle/kernel/server_session.cpp index 4604e35c5..8c19f2534 100644 --- a/src/core/hle/kernel/server_session.cpp +++ b/src/core/hle/kernel/server_session.cpp @@ -8,7 +8,6 @@ #include "common/assert.h" #include "common/common_types.h" #include "common/logging/log.h" -#include "core/core.h" #include "core/core_timing.h" #include "core/hle/ipc_helpers.h" #include "core/hle/kernel/client_port.h" @@ -17,6 +16,7 @@ #include "core/hle/kernel/hle_ipc.h" #include "core/hle/kernel/kernel.h" #include "core/hle/kernel/process.h" +#include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/server_session.h" #include "core/hle/kernel/session.h" #include "core/hle/kernel/thread.h" @@ -32,8 +32,10 @@ ResultVal<std::shared_ptr<ServerSession>> ServerSession::Create(KernelCore& kern std::string name) { std::shared_ptr<ServerSession> session{std::make_shared<ServerSession>(kernel)}; - session->request_event = Core::Timing::CreateEvent( - name, [session](u64 userdata, s64 cycles_late) { session->CompleteSyncRequest(); }); + session->request_event = + Core::Timing::CreateEvent(name, [session](std::uintptr_t, std::chrono::nanoseconds) { + session->CompleteSyncRequest(); + }); session->name = std::move(name); session->parent = std::move(parent); @@ -134,10 +136,11 @@ ResultCode ServerSession::HandleDomainSyncRequest(Kernel::HLERequestContext& con return RESULT_SUCCESS; } -ResultCode ServerSession::QueueSyncRequest(std::shared_ptr<Thread> thread, Memory::Memory& memory) { +ResultCode ServerSession::QueueSyncRequest(std::shared_ptr<Thread> thread, + Core::Memory::Memory& memory) { u32* cmd_buf{reinterpret_cast<u32*>(memory.GetPointer(thread->GetTLSAddress()))}; - std::shared_ptr<Kernel::HLERequestContext> context{ - std::make_shared<Kernel::HLERequestContext>(SharedFrom(this), std::move(thread))}; + auto context = + std::make_shared<HLERequestContext>(kernel, memory, SharedFrom(this), std::move(thread)); context->PopulateFromIncomingCommandBuffer(kernel.CurrentProcess()->GetHandleTable(), cmd_buf); request_queue.Push(std::move(context)); @@ -167,9 +170,12 @@ ResultCode ServerSession::CompleteSyncRequest() { } // Some service requests require the thread to block - if (!context.IsThreadWaiting()) { - context.GetThread().ResumeFromWait(); - context.GetThread().SetWaitSynchronizationResult(result); + { + SchedulerLock lock(kernel); + if (!context.IsThreadWaiting()) { + context.GetThread().ResumeFromWait(); + context.GetThread().SetSynchronizationResults(nullptr, result); + } } request_queue.Pop(); @@ -178,9 +184,12 @@ ResultCode ServerSession::CompleteSyncRequest() { } ResultCode ServerSession::HandleSyncRequest(std::shared_ptr<Thread> thread, - Memory::Memory& memory) { - Core::System::GetInstance().CoreTiming().ScheduleEvent(20000, request_event, {}); - return QueueSyncRequest(std::move(thread), memory); + Core::Memory::Memory& memory, + Core::Timing::CoreTiming& core_timing) { + const ResultCode result = QueueSyncRequest(std::move(thread), memory); + const auto delay = std::chrono::nanoseconds{kernel.IsMulticore() ? 0 : 20000}; + core_timing.ScheduleEvent(delay, request_event, {}); + return result; } } // namespace Kernel diff --git a/src/core/hle/kernel/server_session.h b/src/core/hle/kernel/server_session.h index 77e4f6721..d23e9ec68 100644 --- a/src/core/hle/kernel/server_session.h +++ b/src/core/hle/kernel/server_session.h @@ -13,13 +13,14 @@ #include "core/hle/kernel/synchronization_object.h" #include "core/hle/result.h" -namespace Memory { +namespace Core::Memory { class Memory; } namespace Core::Timing { +class CoreTiming; struct EventType; -} +} // namespace Core::Timing namespace Kernel { @@ -87,12 +88,14 @@ public: /** * Handle a sync request from the emulated application. * - * @param thread Thread that initiated the request. - * @param memory Memory context to handle the sync request under. + * @param thread Thread that initiated the request. + * @param memory Memory context to handle the sync request under. + * @param core_timing Core timing context to schedule the request event under. * * @returns ResultCode from the operation. */ - ResultCode HandleSyncRequest(std::shared_ptr<Thread> thread, Memory::Memory& memory); + ResultCode HandleSyncRequest(std::shared_ptr<Thread> thread, Core::Memory::Memory& memory, + Core::Timing::CoreTiming& core_timing); bool ShouldWait(const Thread* thread) const override; @@ -126,7 +129,7 @@ public: private: /// Queues a sync request from the emulated application. - ResultCode QueueSyncRequest(std::shared_ptr<Thread> thread, Memory::Memory& memory); + ResultCode QueueSyncRequest(std::shared_ptr<Thread> thread, Core::Memory::Memory& memory); /// Completes a sync request from the emulated application. ResultCode CompleteSyncRequest(); diff --git a/src/core/hle/kernel/shared_memory.cpp b/src/core/hle/kernel/shared_memory.cpp index afb2e3fc2..0cd467110 100644 --- a/src/core/hle/kernel/shared_memory.cpp +++ b/src/core/hle/kernel/shared_memory.cpp @@ -2,149 +2,56 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. -#include <utility> - #include "common/assert.h" -#include "common/logging/log.h" -#include "core/hle/kernel/errors.h" +#include "core/core.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/memory/page_table.h" #include "core/hle/kernel/shared_memory.h" namespace Kernel { -SharedMemory::SharedMemory(KernelCore& kernel) : Object{kernel} {} -SharedMemory::~SharedMemory() = default; - -std::shared_ptr<SharedMemory> SharedMemory::Create(KernelCore& kernel, Process* owner_process, - u64 size, MemoryPermission permissions, - MemoryPermission other_permissions, - VAddr address, MemoryRegion region, - std::string name) { - std::shared_ptr<SharedMemory> shared_memory = std::make_shared<SharedMemory>(kernel); - - shared_memory->owner_process = owner_process; - shared_memory->name = std::move(name); - shared_memory->size = size; - shared_memory->permissions = permissions; - shared_memory->other_permissions = other_permissions; - - if (address == 0) { - shared_memory->backing_block = std::make_shared<Kernel::PhysicalMemory>(size); - shared_memory->backing_block_offset = 0; - - // Refresh the address mappings for the current process. - if (kernel.CurrentProcess() != nullptr) { - kernel.CurrentProcess()->VMManager().RefreshMemoryBlockMappings( - shared_memory->backing_block.get()); - } - } else { - const auto& vm_manager = shared_memory->owner_process->VMManager(); +SharedMemory::SharedMemory(KernelCore& kernel, Core::DeviceMemory& device_memory) + : Object{kernel}, device_memory{device_memory} {} - // The memory is already available and mapped in the owner process. - const auto vma = vm_manager.FindVMA(address); - ASSERT_MSG(vm_manager.IsValidHandle(vma), "Invalid memory address"); - ASSERT_MSG(vma->second.backing_block, "Backing block doesn't exist for address"); - - // The returned VMA might be a bigger one encompassing the desired address. - const auto vma_offset = address - vma->first; - ASSERT_MSG(vma_offset + size <= vma->second.size, - "Shared memory exceeds bounds of mapped block"); - - shared_memory->backing_block = vma->second.backing_block; - shared_memory->backing_block_offset = vma->second.offset + vma_offset; - } - - shared_memory->base_address = address; +SharedMemory::~SharedMemory() = default; - return shared_memory; -} +std::shared_ptr<SharedMemory> SharedMemory::Create( + KernelCore& kernel, Core::DeviceMemory& device_memory, Process* owner_process, + Memory::PageLinkedList&& page_list, Memory::MemoryPermission owner_permission, + Memory::MemoryPermission user_permission, PAddr physical_address, std::size_t size, + std::string name) { -std::shared_ptr<SharedMemory> SharedMemory::CreateForApplet( - KernelCore& kernel, std::shared_ptr<Kernel::PhysicalMemory> heap_block, std::size_t offset, - u64 size, MemoryPermission permissions, MemoryPermission other_permissions, std::string name) { - std::shared_ptr<SharedMemory> shared_memory = std::make_shared<SharedMemory>(kernel); + std::shared_ptr<SharedMemory> shared_memory{ + std::make_shared<SharedMemory>(kernel, device_memory)}; - shared_memory->owner_process = nullptr; - shared_memory->name = std::move(name); + shared_memory->owner_process = owner_process; + shared_memory->page_list = std::move(page_list); + shared_memory->owner_permission = owner_permission; + shared_memory->user_permission = user_permission; + shared_memory->physical_address = physical_address; shared_memory->size = size; - shared_memory->permissions = permissions; - shared_memory->other_permissions = other_permissions; - shared_memory->backing_block = std::move(heap_block); - shared_memory->backing_block_offset = offset; - shared_memory->base_address = - kernel.CurrentProcess()->VMManager().GetHeapRegionBaseAddress() + offset; + shared_memory->name = name; return shared_memory; } -ResultCode SharedMemory::Map(Process& target_process, VAddr address, MemoryPermission permissions, - MemoryPermission other_permissions) { - const MemoryPermission own_other_permissions = - &target_process == owner_process ? this->permissions : this->other_permissions; - - // Automatically allocated memory blocks can only be mapped with other_permissions = DontCare - if (base_address == 0 && other_permissions != MemoryPermission::DontCare) { - return ERR_INVALID_MEMORY_PERMISSIONS; - } - - // Error out if the requested permissions don't match what the creator process allows. - if (static_cast<u32>(permissions) & ~static_cast<u32>(own_other_permissions)) { - LOG_ERROR(Kernel, "cannot map id={}, address=0x{:X} name={}, permissions don't match", - GetObjectId(), address, name); - return ERR_INVALID_MEMORY_PERMISSIONS; - } +ResultCode SharedMemory::Map(Process& target_process, VAddr address, std::size_t size, + Memory::MemoryPermission permissions) { + const u64 page_count{(size + Memory::PageSize - 1) / Memory::PageSize}; - // Error out if the provided permissions are not compatible with what the creator process needs. - if (other_permissions != MemoryPermission::DontCare && - static_cast<u32>(this->permissions) & ~static_cast<u32>(other_permissions)) { - LOG_ERROR(Kernel, "cannot map id={}, address=0x{:X} name={}, permissions don't match", - GetObjectId(), address, name); - return ERR_INVALID_MEMORY_PERMISSIONS; + if (page_list.GetNumPages() != page_count) { + UNIMPLEMENTED_MSG("Page count does not match"); } - VAddr target_address = address; + const Memory::MemoryPermission expected = + &target_process == owner_process ? owner_permission : user_permission; - // Map the memory block into the target process - auto result = target_process.VMManager().MapMemoryBlock( - target_address, backing_block, backing_block_offset, size, MemoryState::Shared); - if (result.Failed()) { - LOG_ERROR( - Kernel, - "cannot map id={}, target_address=0x{:X} name={}, error mapping to virtual memory", - GetObjectId(), target_address, name); - return result.Code(); + if (permissions != expected) { + UNIMPLEMENTED_MSG("Permission does not match"); } - return target_process.VMManager().ReprotectRange(target_address, size, - ConvertPermissions(permissions)); -} - -ResultCode SharedMemory::Unmap(Process& target_process, VAddr address, u64 unmap_size) { - if (unmap_size != size) { - LOG_ERROR(Kernel, - "Invalid size passed to Unmap. Size must be equal to the size of the " - "memory managed. Shared memory size=0x{:016X}, Unmap size=0x{:016X}", - size, unmap_size); - return ERR_INVALID_SIZE; - } - - // TODO(Subv): Verify what happens if the application tries to unmap an address that is not - // mapped to a SharedMemory. - return target_process.VMManager().UnmapRange(address, size); -} - -VMAPermission SharedMemory::ConvertPermissions(MemoryPermission permission) { - u32 masked_permissions = - static_cast<u32>(permission) & static_cast<u32>(MemoryPermission::ReadWriteExecute); - return static_cast<VMAPermission>(masked_permissions); -} - -u8* SharedMemory::GetPointer(std::size_t offset) { - return backing_block->data() + backing_block_offset + offset; -} - -const u8* SharedMemory::GetPointer(std::size_t offset) const { - return backing_block->data() + backing_block_offset + offset; + return target_process.PageTable().MapPages(address, page_list, Memory::MemoryState::Shared, + permissions); } } // namespace Kernel diff --git a/src/core/hle/kernel/shared_memory.h b/src/core/hle/kernel/shared_memory.h index 014951d82..0ef87235c 100644 --- a/src/core/hle/kernel/shared_memory.h +++ b/src/core/hle/kernel/shared_memory.h @@ -8,8 +8,10 @@ #include <string> #include "common/common_types.h" +#include "core/device_memory.h" +#include "core/hle/kernel/memory/memory_block.h" +#include "core/hle/kernel/memory/page_linked_list.h" #include "core/hle/kernel/object.h" -#include "core/hle/kernel/physical_memory.h" #include "core/hle/kernel/process.h" #include "core/hle/result.h" @@ -17,63 +19,21 @@ namespace Kernel { class KernelCore; -/// Permissions for mapped shared memory blocks -enum class MemoryPermission : u32 { - None = 0, - Read = (1u << 0), - Write = (1u << 1), - ReadWrite = (Read | Write), - Execute = (1u << 2), - ReadExecute = (Read | Execute), - WriteExecute = (Write | Execute), - ReadWriteExecute = (Read | Write | Execute), - DontCare = (1u << 28) -}; - class SharedMemory final : public Object { public: - explicit SharedMemory(KernelCore& kernel); + explicit SharedMemory(KernelCore& kernel, Core::DeviceMemory& device_memory); ~SharedMemory() override; - /** - * Creates a shared memory object. - * @param kernel The kernel instance to create a shared memory instance under. - * @param owner_process Process that created this shared memory object. - * @param size Size of the memory block. Must be page-aligned. - * @param permissions Permission restrictions applied to the process which created the block. - * @param other_permissions Permission restrictions applied to other processes mapping the - * block. - * @param address The address from which to map the Shared Memory. - * @param region If the address is 0, the shared memory will be allocated in this region of the - * linear heap. - * @param name Optional object name, used for debugging purposes. - */ - static std::shared_ptr<SharedMemory> Create(KernelCore& kernel, Process* owner_process, - u64 size, MemoryPermission permissions, - MemoryPermission other_permissions, - VAddr address = 0, - MemoryRegion region = MemoryRegion::BASE, - std::string name = "Unknown"); - - /** - * Creates a shared memory object from a block of memory managed by an HLE applet. - * @param kernel The kernel instance to create a shared memory instance under. - * @param heap_block Heap block of the HLE applet. - * @param offset The offset into the heap block that the SharedMemory will map. - * @param size Size of the memory block. Must be page-aligned. - * @param permissions Permission restrictions applied to the process which created the block. - * @param other_permissions Permission restrictions applied to other processes mapping the - * block. - * @param name Optional object name, used for debugging purposes. - */ - static std::shared_ptr<SharedMemory> CreateForApplet( - KernelCore& kernel, std::shared_ptr<Kernel::PhysicalMemory> heap_block, std::size_t offset, - u64 size, MemoryPermission permissions, MemoryPermission other_permissions, - std::string name = "Unknown Applet"); + static std::shared_ptr<SharedMemory> Create( + KernelCore& kernel, Core::DeviceMemory& device_memory, Process* owner_process, + Memory::PageLinkedList&& page_list, Memory::MemoryPermission owner_permission, + Memory::MemoryPermission user_permission, PAddr physical_address, std::size_t size, + std::string name); std::string GetTypeName() const override { return "SharedMemory"; } + std::string GetName() const override { return name; } @@ -83,71 +43,42 @@ public: return HANDLE_TYPE; } - /// Gets the size of the underlying memory block in bytes. - u64 GetSize() const { - return size; - } - - /** - * Converts the specified MemoryPermission into the equivalent VMAPermission. - * @param permission The MemoryPermission to convert. - */ - static VMAPermission ConvertPermissions(MemoryPermission permission); - /** * Maps a shared memory block to an address in the target process' address space - * @param target_process Process on which to map the memory block. + * @param target_process Process on which to map the memory block * @param address Address in system memory to map shared memory block to + * @param size Size of the shared memory block to map * @param permissions Memory block map permissions (specified by SVC field) - * @param other_permissions Memory block map other permissions (specified by SVC field) - */ - ResultCode Map(Process& target_process, VAddr address, MemoryPermission permissions, - MemoryPermission other_permissions); - - /** - * Unmaps a shared memory block from the specified address in system memory - * - * @param target_process Process from which to unmap the memory block. - * @param address Address in system memory where the shared memory block is mapped. - * @param unmap_size The amount of bytes to unmap from this shared memory instance. - * - * @return Result code of the unmap operation - * - * @pre The given size to unmap must be the same size as the amount of memory managed by - * the SharedMemory instance itself, otherwise ERR_INVALID_SIZE will be returned. */ - ResultCode Unmap(Process& target_process, VAddr address, u64 unmap_size); + ResultCode Map(Process& target_process, VAddr address, std::size_t size, + Memory::MemoryPermission permissions); /** * Gets a pointer to the shared memory block * @param offset Offset from the start of the shared memory block to get pointer * @return A pointer to the shared memory block from the specified offset */ - u8* GetPointer(std::size_t offset = 0); + u8* GetPointer(std::size_t offset = 0) { + return device_memory.GetPointer(physical_address + offset); + } /** - * Gets a constant pointer to the shared memory block + * Gets a pointer to the shared memory block * @param offset Offset from the start of the shared memory block to get pointer - * @return A constant pointer to the shared memory block from the specified offset + * @return A pointer to the shared memory block from the specified offset */ - const u8* GetPointer(std::size_t offset = 0) const; + const u8* GetPointer(std::size_t offset = 0) const { + return device_memory.GetPointer(physical_address + offset); + } private: - /// Backing memory for this shared memory block. - std::shared_ptr<PhysicalMemory> backing_block; - /// Offset into the backing block for this shared memory. - std::size_t backing_block_offset = 0; - /// Size of the memory block. Page-aligned. - u64 size = 0; - /// Permission restrictions applied to the process which created the block. - MemoryPermission permissions{}; - /// Permission restrictions applied to other processes mapping the block. - MemoryPermission other_permissions{}; - /// Process that created this shared memory block. - Process* owner_process; - /// Address of shared memory block in the owner process if specified. - VAddr base_address = 0; - /// Name of shared memory object. + Core::DeviceMemory& device_memory; + Process* owner_process{}; + Memory::PageLinkedList page_list; + Memory::MemoryPermission owner_permission{}; + Memory::MemoryPermission user_permission{}; + PAddr physical_address{}; + std::size_t size{}; std::string name; }; diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp index 4ffc113c2..e3b770d66 100644 --- a/src/core/hle/kernel/svc.cpp +++ b/src/core/hle/kernel/svc.cpp @@ -10,30 +10,36 @@ #include "common/alignment.h" #include "common/assert.h" +#include "common/fiber.h" #include "common/logging/log.h" #include "common/microprofile.h" #include "common/string_util.h" #include "core/arm/exclusive_monitor.h" #include "core/core.h" -#include "core/core_manager.h" #include "core/core_timing.h" #include "core/core_timing_util.h" +#include "core/cpu_manager.h" #include "core/hle/kernel/address_arbiter.h" #include "core/hle/kernel/client_port.h" #include "core/hle/kernel/client_session.h" #include "core/hle/kernel/errors.h" #include "core/hle/kernel/handle_table.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/memory/memory_block.h" +#include "core/hle/kernel/memory/page_table.h" #include "core/hle/kernel/mutex.h" +#include "core/hle/kernel/physical_core.h" #include "core/hle/kernel/process.h" #include "core/hle/kernel/readable_event.h" #include "core/hle/kernel/resource_limit.h" #include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/shared_memory.h" #include "core/hle/kernel/svc.h" +#include "core/hle/kernel/svc_types.h" #include "core/hle/kernel/svc_wrap.h" #include "core/hle/kernel/synchronization.h" #include "core/hle/kernel/thread.h" +#include "core/hle/kernel/time_manager.h" #include "core/hle/kernel/transfer_memory.h" #include "core/hle/kernel/writable_event.h" #include "core/hle/lock.h" @@ -42,7 +48,7 @@ #include "core/memory.h" #include "core/reporter.h" -namespace Kernel { +namespace Kernel::Svc { namespace { // Checks if address + size is greater than the given address @@ -52,14 +58,11 @@ constexpr bool IsValidAddressRange(VAddr address, u64 size) { return address + size > address; } -// 8 GiB -constexpr u64 MAIN_MEMORY_SIZE = 0x200000000; - // Helper function that performs the common sanity checks for svcMapMemory // and svcUnmapMemory. This is doable, as both functions perform their sanitizing // in the same order. -ResultCode MapUnmapMemorySanityChecks(const VMManager& vm_manager, VAddr dst_addr, VAddr src_addr, - u64 size) { +ResultCode MapUnmapMemorySanityChecks(const Memory::PageTable& manager, VAddr dst_addr, + VAddr src_addr, u64 size) { if (!Common::Is4KBAligned(dst_addr)) { LOG_ERROR(Kernel_SVC, "Destination address is not aligned to 4KB, 0x{:016X}", dst_addr); return ERR_INVALID_ADDRESS; @@ -93,36 +96,33 @@ ResultCode MapUnmapMemorySanityChecks(const VMManager& vm_manager, VAddr dst_add return ERR_INVALID_ADDRESS_STATE; } - if (!vm_manager.IsWithinAddressSpace(src_addr, size)) { + if (!manager.IsInsideAddressSpace(src_addr, size)) { LOG_ERROR(Kernel_SVC, "Source is not within the address space, addr=0x{:016X}, size=0x{:016X}", src_addr, size); return ERR_INVALID_ADDRESS_STATE; } - if (!vm_manager.IsWithinStackRegion(dst_addr, size)) { + if (manager.IsOutsideStackRegion(dst_addr, size)) { LOG_ERROR(Kernel_SVC, "Destination is not within the stack region, addr=0x{:016X}, size=0x{:016X}", dst_addr, size); return ERR_INVALID_MEMORY_RANGE; } - const VAddr dst_end_address = dst_addr + size; - if (dst_end_address > vm_manager.GetHeapRegionBaseAddress() && - vm_manager.GetHeapRegionEndAddress() > dst_addr) { + if (manager.IsInsideHeapRegion(dst_addr, size)) { LOG_ERROR(Kernel_SVC, "Destination does not fit within the heap region, addr=0x{:016X}, " - "size=0x{:016X}, end_addr=0x{:016X}", - dst_addr, size, dst_end_address); + "size=0x{:016X}", + dst_addr, size); return ERR_INVALID_MEMORY_RANGE; } - if (dst_end_address > vm_manager.GetMapRegionBaseAddress() && - vm_manager.GetMapRegionEndAddress() > dst_addr) { + if (manager.IsInsideAliasRegion(dst_addr, size)) { LOG_ERROR(Kernel_SVC, "Destination does not fit within the map region, addr=0x{:016X}, " - "size=0x{:016X}, end_addr=0x{:016X}", - dst_addr, size, dst_end_address); + "size=0x{:016X}", + dst_addr, size); return ERR_INVALID_MEMORY_RANGE; } @@ -136,6 +136,7 @@ enum class ResourceLimitValueType { ResultVal<s64> RetrieveResourceLimitValue(Core::System& system, Handle resource_limit, u32 resource_type, ResourceLimitValueType value_type) { + std::lock_guard lock{HLE::g_hle_lock}; const auto type = static_cast<ResourceType>(resource_type); if (!IsValidResourceType(type)) { LOG_ERROR(Kernel_SVC, "Invalid resource limit type: '{}'", resource_type); @@ -163,6 +164,7 @@ ResultVal<s64> RetrieveResourceLimitValue(Core::System& system, Handle resource_ /// Set the process heap to a given Size. It can both extend and shrink the heap. static ResultCode SetHeapSize(Core::System& system, VAddr* heap_addr, u64 heap_size) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_TRACE(Kernel_SVC, "called, heap_size=0x{:X}", heap_size); // Size must be a multiple of 0x200000 (2MB) and be equal to or less than 8GB. @@ -177,13 +179,10 @@ static ResultCode SetHeapSize(Core::System& system, VAddr* heap_addr, u64 heap_s return ERR_INVALID_SIZE; } - auto& vm_manager = system.Kernel().CurrentProcess()->VMManager(); - const auto alloc_result = vm_manager.SetHeapSize(heap_size); - if (alloc_result.Failed()) { - return alloc_result.Code(); - } + auto& page_table{system.Kernel().CurrentProcess()->PageTable()}; + + CASCADE_RESULT(*heap_addr, page_table.SetHeapSize(heap_size)); - *heap_addr = *alloc_result; return RESULT_SUCCESS; } @@ -194,65 +193,9 @@ static ResultCode SetHeapSize32(Core::System& system, u32* heap_addr, u32 heap_s return result; } -static ResultCode SetMemoryPermission(Core::System& system, VAddr addr, u64 size, u32 prot) { - LOG_TRACE(Kernel_SVC, "called, addr=0x{:X}, size=0x{:X}, prot=0x{:X}", addr, size, prot); - - if (!Common::Is4KBAligned(addr)) { - LOG_ERROR(Kernel_SVC, "Address is not aligned to 4KB, addr=0x{:016X}", addr); - return ERR_INVALID_ADDRESS; - } - - if (size == 0) { - LOG_ERROR(Kernel_SVC, "Size is 0"); - return ERR_INVALID_SIZE; - } - - if (!Common::Is4KBAligned(size)) { - LOG_ERROR(Kernel_SVC, "Size is not aligned to 4KB, size=0x{:016X}", size); - return ERR_INVALID_SIZE; - } - - if (!IsValidAddressRange(addr, size)) { - LOG_ERROR(Kernel_SVC, "Region is not a valid address range, addr=0x{:016X}, size=0x{:016X}", - addr, size); - return ERR_INVALID_ADDRESS_STATE; - } - - const auto permission = static_cast<MemoryPermission>(prot); - if (permission != MemoryPermission::None && permission != MemoryPermission::Read && - permission != MemoryPermission::ReadWrite) { - LOG_ERROR(Kernel_SVC, "Invalid memory permission specified, Got memory permission=0x{:08X}", - static_cast<u32>(permission)); - return ERR_INVALID_MEMORY_PERMISSIONS; - } - - auto* const current_process = system.Kernel().CurrentProcess(); - auto& vm_manager = current_process->VMManager(); - - if (!vm_manager.IsWithinAddressSpace(addr, size)) { - LOG_ERROR(Kernel_SVC, - "Source is not within the address space, addr=0x{:016X}, size=0x{:016X}", addr, - size); - return ERR_INVALID_ADDRESS_STATE; - } - - const VMManager::VMAHandle iter = vm_manager.FindVMA(addr); - if (!vm_manager.IsValidHandle(iter)) { - LOG_ERROR(Kernel_SVC, "Unable to find VMA for address=0x{:016X}", addr); - return ERR_INVALID_ADDRESS_STATE; - } - - LOG_WARNING(Kernel_SVC, "Uniformity check on protected memory is not implemented."); - // TODO: Performs a uniformity check to make sure only protected memory is changed (it doesn't - // make sense to allow changing permissions on kernel memory itself, etc). - - const auto converted_permissions = SharedMemory::ConvertPermissions(permission); - - return vm_manager.ReprotectRange(addr, size, converted_permissions); -} - static ResultCode SetMemoryAttribute(Core::System& system, VAddr address, u64 size, u32 mask, u32 attribute) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_DEBUG(Kernel_SVC, "called, address=0x{:016X}, size=0x{:X}, mask=0x{:08X}, attribute=0x{:08X}", address, size, mask, attribute); @@ -274,72 +217,73 @@ static ResultCode SetMemoryAttribute(Core::System& system, VAddr address, u64 si return ERR_INVALID_ADDRESS_STATE; } - const auto mem_attribute = static_cast<MemoryAttribute>(attribute); - const auto mem_mask = static_cast<MemoryAttribute>(mask); - const auto attribute_with_mask = mem_attribute | mem_mask; - - if (attribute_with_mask != mem_mask) { + const auto attributes{static_cast<Memory::MemoryAttribute>(mask | attribute)}; + if (attributes != static_cast<Memory::MemoryAttribute>(mask) || + (attributes | Memory::MemoryAttribute::Uncached) != Memory::MemoryAttribute::Uncached) { LOG_ERROR(Kernel_SVC, "Memory attribute doesn't match the given mask (Attribute: 0x{:X}, Mask: {:X}", attribute, mask); return ERR_INVALID_COMBINATION; } - if ((attribute_with_mask | MemoryAttribute::Uncached) != MemoryAttribute::Uncached) { - LOG_ERROR(Kernel_SVC, "Specified attribute isn't equal to MemoryAttributeUncached (8)."); - return ERR_INVALID_COMBINATION; - } + auto& page_table{system.Kernel().CurrentProcess()->PageTable()}; - auto& vm_manager = system.Kernel().CurrentProcess()->VMManager(); - if (!vm_manager.IsWithinAddressSpace(address, size)) { - LOG_ERROR(Kernel_SVC, - "Given address (0x{:016X}) is outside the bounds of the address space.", address); - return ERR_INVALID_ADDRESS_STATE; - } + return page_table.SetMemoryAttribute(address, size, static_cast<Memory::MemoryAttribute>(mask), + static_cast<Memory::MemoryAttribute>(attribute)); +} - return vm_manager.SetMemoryAttribute(address, size, mem_mask, mem_attribute); +static ResultCode SetMemoryAttribute32(Core::System& system, u32 address, u32 size, u32 mask, + u32 attribute) { + return SetMemoryAttribute(system, static_cast<VAddr>(address), static_cast<std::size_t>(size), + mask, attribute); } /// Maps a memory range into a different range. static ResultCode MapMemory(Core::System& system, VAddr dst_addr, VAddr src_addr, u64 size) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_TRACE(Kernel_SVC, "called, dst_addr=0x{:X}, src_addr=0x{:X}, size=0x{:X}", dst_addr, src_addr, size); - auto& vm_manager = system.Kernel().CurrentProcess()->VMManager(); - const auto result = MapUnmapMemorySanityChecks(vm_manager, dst_addr, src_addr, size); + auto& page_table{system.Kernel().CurrentProcess()->PageTable()}; - if (result.IsError()) { + if (const ResultCode result{MapUnmapMemorySanityChecks(page_table, dst_addr, src_addr, size)}; + result.IsError()) { return result; } - return vm_manager.MirrorMemory(dst_addr, src_addr, size, MemoryState::Stack); + return page_table.Map(dst_addr, src_addr, size); +} + +static ResultCode MapMemory32(Core::System& system, u32 dst_addr, u32 src_addr, u32 size) { + return MapMemory(system, static_cast<VAddr>(dst_addr), static_cast<VAddr>(src_addr), + static_cast<std::size_t>(size)); } /// Unmaps a region that was previously mapped with svcMapMemory static ResultCode UnmapMemory(Core::System& system, VAddr dst_addr, VAddr src_addr, u64 size) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_TRACE(Kernel_SVC, "called, dst_addr=0x{:X}, src_addr=0x{:X}, size=0x{:X}", dst_addr, src_addr, size); - auto& vm_manager = system.Kernel().CurrentProcess()->VMManager(); - const auto result = MapUnmapMemorySanityChecks(vm_manager, dst_addr, src_addr, size); + auto& page_table{system.Kernel().CurrentProcess()->PageTable()}; - if (result.IsError()) { + if (const ResultCode result{MapUnmapMemorySanityChecks(page_table, dst_addr, src_addr, size)}; + result.IsError()) { return result; } - const auto unmap_res = vm_manager.UnmapRange(dst_addr, size); - - // Reprotect the source mapping on success - if (unmap_res.IsSuccess()) { - ASSERT(vm_manager.ReprotectRange(src_addr, size, VMAPermission::ReadWrite).IsSuccess()); - } + return page_table.Unmap(dst_addr, src_addr, size); +} - return unmap_res; +static ResultCode UnmapMemory32(Core::System& system, u32 dst_addr, u32 src_addr, u32 size) { + return UnmapMemory(system, static_cast<VAddr>(dst_addr), static_cast<VAddr>(src_addr), + static_cast<std::size_t>(size)); } /// Connect to an OS service given the port name, returns the handle to the port to out static ResultCode ConnectToNamedPort(Core::System& system, Handle* out_handle, VAddr port_name_address) { + std::lock_guard lock{HLE::g_hle_lock}; auto& memory = system.Memory(); if (!memory.IsValidVirtualAddress(port_name_address)) { @@ -367,6 +311,8 @@ static ResultCode ConnectToNamedPort(Core::System& system, Handle* out_handle, return ERR_NOT_FOUND; } + ASSERT(kernel.CurrentProcess()->GetResourceLimit()->Reserve(ResourceType::Sessions, 1)); + auto client_port = it->second; std::shared_ptr<ClientSession> client_session; @@ -396,11 +342,30 @@ static ResultCode SendSyncRequest(Core::System& system, Handle handle) { LOG_TRACE(Kernel_SVC, "called handle=0x{:08X}({})", handle, session->GetName()); auto thread = system.CurrentScheduler().GetCurrentThread(); - thread->InvalidateWakeupCallback(); - thread->SetStatus(ThreadStatus::WaitIPC); - system.PrepareReschedule(thread->GetProcessorID()); + { + SchedulerLock lock(system.Kernel()); + thread->InvalidateHLECallback(); + thread->SetStatus(ThreadStatus::WaitIPC); + session->SendSyncRequest(SharedFrom(thread), system.Memory(), system.CoreTiming()); + } + + if (thread->HasHLECallback()) { + Handle event_handle = thread->GetHLETimeEvent(); + if (event_handle != InvalidHandle) { + auto& time_manager = system.Kernel().TimeManager(); + time_manager.UnscheduleTimeEvent(event_handle); + } + + { + SchedulerLock lock(system.Kernel()); + auto* sync_object = thread->GetHLESyncObject(); + sync_object->RemoveWaitingThread(SharedFrom(thread)); + } + + thread->InvokeHLECallback(SharedFrom(thread)); + } - return session->SendSyncRequest(SharedFrom(thread), system.Memory()); + return thread->GetSignalingResult(); } static ResultCode SendSyncRequest32(Core::System& system, Handle handle) { @@ -462,6 +427,15 @@ static ResultCode GetProcessId(Core::System& system, u64* process_id, Handle han return ERR_INVALID_HANDLE; } +static ResultCode GetProcessId32(Core::System& system, u32* process_id_low, u32* process_id_high, + Handle handle) { + u64 process_id{}; + const auto result = GetProcessId(system, &process_id, handle); + *process_id_low = static_cast<u32>(process_id); + *process_id_high = static_cast<u32>(process_id >> 32); + return result; +} + /// Wait for the given handles to synchronize, timeout after the specified nanoseconds static ResultCode WaitSynchronization(Core::System& system, Handle* index, VAddr handles_address, u64 handle_count, s64 nano_seconds) { @@ -484,9 +458,7 @@ static ResultCode WaitSynchronization(Core::System& system, Handle* index, VAddr return ERR_OUT_OF_RANGE; } - auto* const thread = system.CurrentScheduler().GetCurrentThread(); auto& kernel = system.Kernel(); - using ObjectPtr = Thread::ThreadSynchronizationObjects::value_type; Thread::ThreadSynchronizationObjects objects(handle_count); const auto& handle_table = kernel.CurrentProcess()->GetHandleTable(); @@ -526,10 +498,13 @@ static ResultCode CancelSynchronization(Core::System& system, Handle thread_hand } thread->CancelWait(); - system.PrepareReschedule(thread->GetProcessorID()); return RESULT_SUCCESS; } +static ResultCode CancelSynchronization32(Core::System& system, Handle thread_handle) { + return CancelSynchronization(system, thread_handle); +} + /// Attempts to locks a mutex, creating it if it does not already exist static ResultCode ArbitrateLock(Core::System& system, Handle holding_thread_handle, VAddr mutex_addr, Handle requesting_thread_handle) { @@ -538,7 +513,7 @@ static ResultCode ArbitrateLock(Core::System& system, Handle holding_thread_hand "requesting_current_thread_handle=0x{:08X}", holding_thread_handle, mutex_addr, requesting_thread_handle); - if (Memory::IsKernelVirtualAddress(mutex_addr)) { + if (Core::Memory::IsKernelVirtualAddress(mutex_addr)) { LOG_ERROR(Kernel_SVC, "Mutex Address is a kernel virtual address, mutex_addr={:016X}", mutex_addr); return ERR_INVALID_ADDRESS_STATE; @@ -554,11 +529,17 @@ static ResultCode ArbitrateLock(Core::System& system, Handle holding_thread_hand requesting_thread_handle); } +static ResultCode ArbitrateLock32(Core::System& system, Handle holding_thread_handle, + u32 mutex_addr, Handle requesting_thread_handle) { + return ArbitrateLock(system, holding_thread_handle, static_cast<VAddr>(mutex_addr), + requesting_thread_handle); +} + /// Unlock a mutex static ResultCode ArbitrateUnlock(Core::System& system, VAddr mutex_addr) { LOG_TRACE(Kernel_SVC, "called mutex_addr=0x{:X}", mutex_addr); - if (Memory::IsKernelVirtualAddress(mutex_addr)) { + if (Core::Memory::IsKernelVirtualAddress(mutex_addr)) { LOG_ERROR(Kernel_SVC, "Mutex Address is a kernel virtual address, mutex_addr={:016X}", mutex_addr); return ERR_INVALID_ADDRESS_STATE; @@ -573,6 +554,10 @@ static ResultCode ArbitrateUnlock(Core::System& system, VAddr mutex_addr) { return current_process->GetMutex().Release(mutex_addr); } +static ResultCode ArbitrateUnlock32(Core::System& system, u32 mutex_addr) { + return ArbitrateUnlock(system, static_cast<VAddr>(mutex_addr)); +} + enum class BreakType : u32 { Panic = 0, AssertionFailed = 1, @@ -673,6 +658,7 @@ static void Break(Core::System& system, u32 reason, u64 info1, u64 info2) { info2, has_dumped_buffer ? std::make_optional(debug_buffer) : std::nullopt); if (!break_reason.signal_debugger) { + SchedulerLock lock(system.Kernel()); LOG_CRITICAL( Debug_Emulated, "Emulated program broke execution! reason=0x{:016X}, info1=0x{:016X}, info2=0x{:016X}", @@ -683,18 +669,19 @@ static void Break(Core::System& system, u32 reason, u64 info1, u64 info2) { auto* const current_thread = system.CurrentScheduler().GetCurrentThread(); const auto thread_processor_id = current_thread->GetProcessorID(); system.ArmInterface(static_cast<std::size_t>(thread_processor_id)).LogBacktrace(); - ASSERT(false); - - system.Kernel().CurrentProcess()->PrepareForTermination(); // Kill the current thread + system.Kernel().ExceptionalExit(); current_thread->Stop(); - system.PrepareReschedule(); } } +static void Break32(Core::System& system, u32 reason, u32 info1, u32 info2) { + Break(system, reason, static_cast<u64>(info1), static_cast<u64>(info2)); +} + /// Used to output a message on a debug hardware unit - does nothing on a retail unit -static void OutputDebugString([[maybe_unused]] Core::System& system, VAddr address, u64 len) { +static void OutputDebugString(Core::System& system, VAddr address, u64 len) { if (len == 0) { return; } @@ -707,6 +694,7 @@ static void OutputDebugString([[maybe_unused]] Core::System& system, VAddr addre /// Gets system/memory information for the current process static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 handle, u64 info_sub_id) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_TRACE(Kernel_SVC, "called info_id=0x{:X}, info_sub_id=0x{:X}, handle=0x{:08X}", info_id, info_sub_id, handle); @@ -765,6 +753,8 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha case GetInfoType::TotalPhysicalMemoryAvailableWithoutSystemResource: case GetInfoType::TotalPhysicalMemoryUsedWithoutSystemResource: { if (info_sub_id != 0) { + LOG_ERROR(Kernel_SVC, "Info sub id is non zero! info_id={}, info_sub_id={}", info_id, + info_sub_id); return ERR_INVALID_ENUM_VALUE; } @@ -772,6 +762,8 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha system.Kernel().CurrentProcess()->GetHandleTable(); const auto process = current_process_handle_table.Get<Process>(static_cast<Handle>(handle)); if (!process) { + LOG_ERROR(Kernel_SVC, "Process is not valid! info_id={}, info_sub_id={}, handle={:08X}", + info_id, info_sub_id, handle); return ERR_INVALID_HANDLE; } @@ -785,35 +777,35 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha return RESULT_SUCCESS; case GetInfoType::MapRegionBaseAddr: - *result = process->VMManager().GetMapRegionBaseAddress(); + *result = process->PageTable().GetAliasRegionStart(); return RESULT_SUCCESS; case GetInfoType::MapRegionSize: - *result = process->VMManager().GetMapRegionSize(); + *result = process->PageTable().GetAliasRegionSize(); return RESULT_SUCCESS; case GetInfoType::HeapRegionBaseAddr: - *result = process->VMManager().GetHeapRegionBaseAddress(); + *result = process->PageTable().GetHeapRegionStart(); return RESULT_SUCCESS; case GetInfoType::HeapRegionSize: - *result = process->VMManager().GetHeapRegionSize(); + *result = process->PageTable().GetHeapRegionSize(); return RESULT_SUCCESS; case GetInfoType::ASLRRegionBaseAddr: - *result = process->VMManager().GetASLRRegionBaseAddress(); + *result = process->PageTable().GetAliasCodeRegionStart(); return RESULT_SUCCESS; case GetInfoType::ASLRRegionSize: - *result = process->VMManager().GetASLRRegionSize(); + *result = process->PageTable().GetAliasCodeRegionSize(); return RESULT_SUCCESS; case GetInfoType::StackRegionBaseAddr: - *result = process->VMManager().GetStackRegionBaseAddress(); + *result = process->PageTable().GetStackRegionStart(); return RESULT_SUCCESS; case GetInfoType::StackRegionSize: - *result = process->VMManager().GetStackRegionSize(); + *result = process->PageTable().GetStackRegionSize(); return RESULT_SUCCESS; case GetInfoType::TotalPhysicalMemoryAvailable: @@ -853,7 +845,7 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha break; } - LOG_WARNING(Kernel_SVC, "(STUBBED) Unimplemented svcGetInfo id=0x{:016X}", info_id); + LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id=0x{:016X}", info_id); return ERR_INVALID_ENUM_VALUE; } @@ -863,10 +855,13 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha case GetInfoType::RegisterResourceLimit: { if (handle != 0) { + LOG_ERROR(Kernel, "Handle is non zero! handle={:08X}", handle); return ERR_INVALID_HANDLE; } if (info_sub_id != 0) { + LOG_ERROR(Kernel, "Info sub id is non zero! info_id={}, info_sub_id={}", info_id, + info_sub_id); return ERR_INVALID_COMBINATION; } @@ -936,9 +931,9 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha if (same_thread && info_sub_id == 0xFFFFFFFFFFFFFFFF) { const u64 thread_ticks = current_thread->GetTotalCPUTimeTicks(); - out_ticks = thread_ticks + (core_timing.GetTicks() - prev_ctx_ticks); + out_ticks = thread_ticks + (core_timing.GetCPUTicks() - prev_ctx_ticks); } else if (same_thread && info_sub_id == system.CurrentCoreIndex()) { - out_ticks = core_timing.GetTicks() - prev_ctx_ticks; + out_ticks = core_timing.GetCPUTicks() - prev_ctx_ticks; } *result = out_ticks; @@ -946,7 +941,7 @@ static ResultCode GetInfo(Core::System& system, u64* result, u64 info_id, u64 ha } default: - LOG_WARNING(Kernel_SVC, "(STUBBED) Unimplemented svcGetInfo id=0x{:016X}", info_id); + LOG_ERROR(Kernel_SVC, "Unimplemented svcGetInfo id=0x{:016X}", info_id); return ERR_INVALID_ENUM_VALUE; } } @@ -965,6 +960,7 @@ static ResultCode GetInfo32(Core::System& system, u32* result_low, u32* result_h /// Maps memory at a desired address static ResultCode MapPhysicalMemory(Core::System& system, VAddr addr, u64 size) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_DEBUG(Kernel_SVC, "called, addr=0x{:016X}, size=0x{:X}", addr, size); if (!Common::Is4KBAligned(addr)) { @@ -987,24 +983,38 @@ static ResultCode MapPhysicalMemory(Core::System& system, VAddr addr, u64 size) return ERR_INVALID_MEMORY_RANGE; } - Process* const current_process = system.Kernel().CurrentProcess(); - auto& vm_manager = current_process->VMManager(); + Process* const current_process{system.Kernel().CurrentProcess()}; + auto& page_table{current_process->PageTable()}; if (current_process->GetSystemResourceSize() == 0) { LOG_ERROR(Kernel_SVC, "System Resource Size is zero"); return ERR_INVALID_STATE; } - if (!vm_manager.IsWithinMapRegion(addr, size)) { - LOG_ERROR(Kernel_SVC, "Range not within map region"); + if (!page_table.IsInsideAddressSpace(addr, size)) { + LOG_ERROR(Kernel_SVC, + "Address is not within the address space, addr=0x{:016X}, size=0x{:016X}", addr, + size); + return ERR_INVALID_MEMORY_RANGE; + } + + if (page_table.IsOutsideAliasRegion(addr, size)) { + LOG_ERROR(Kernel_SVC, + "Address is not within the alias region, addr=0x{:016X}, size=0x{:016X}", addr, + size); return ERR_INVALID_MEMORY_RANGE; } - return vm_manager.MapPhysicalMemory(addr, size); + return page_table.MapPhysicalMemory(addr, size); +} + +static ResultCode MapPhysicalMemory32(Core::System& system, u32 addr, u32 size) { + return MapPhysicalMemory(system, static_cast<VAddr>(addr), static_cast<std::size_t>(size)); } /// Unmaps memory previously mapped via MapPhysicalMemory static ResultCode UnmapPhysicalMemory(Core::System& system, VAddr addr, u64 size) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_DEBUG(Kernel_SVC, "called, addr=0x{:016X}, size=0x{:X}", addr, size); if (!Common::Is4KBAligned(addr)) { @@ -1027,20 +1037,33 @@ static ResultCode UnmapPhysicalMemory(Core::System& system, VAddr addr, u64 size return ERR_INVALID_MEMORY_RANGE; } - Process* const current_process = system.Kernel().CurrentProcess(); - auto& vm_manager = current_process->VMManager(); + Process* const current_process{system.Kernel().CurrentProcess()}; + auto& page_table{current_process->PageTable()}; if (current_process->GetSystemResourceSize() == 0) { LOG_ERROR(Kernel_SVC, "System Resource Size is zero"); return ERR_INVALID_STATE; } - if (!vm_manager.IsWithinMapRegion(addr, size)) { - LOG_ERROR(Kernel_SVC, "Range not within map region"); + if (!page_table.IsInsideAddressSpace(addr, size)) { + LOG_ERROR(Kernel_SVC, + "Address is not within the address space, addr=0x{:016X}, size=0x{:016X}", addr, + size); + return ERR_INVALID_MEMORY_RANGE; + } + + if (page_table.IsOutsideAliasRegion(addr, size)) { + LOG_ERROR(Kernel_SVC, + "Address is not within the alias region, addr=0x{:016X}, size=0x{:016X}", addr, + size); return ERR_INVALID_MEMORY_RANGE; } - return vm_manager.UnmapPhysicalMemory(addr, size); + return page_table.UnmapPhysicalMemory(addr, size); +} + +static ResultCode UnmapPhysicalMemory32(Core::System& system, u32 addr, u32 size) { + return UnmapPhysicalMemory(system, static_cast<VAddr>(addr), static_cast<std::size_t>(size)); } /// Sets the thread activity @@ -1072,10 +1095,11 @@ static ResultCode SetThreadActivity(Core::System& system, Handle handle, u32 act return ERR_BUSY; } - thread->SetActivity(static_cast<ThreadActivity>(activity)); + return thread->SetActivity(static_cast<ThreadActivity>(activity)); +} - system.PrepareReschedule(thread->GetProcessorID()); - return RESULT_SUCCESS; +static ResultCode SetThreadActivity32(Core::System& system, Handle handle, u32 activity) { + return SetThreadActivity(system, handle, activity); } /// Gets the thread context @@ -1119,6 +1143,10 @@ static ResultCode GetThreadContext(Core::System& system, VAddr thread_context, H return RESULT_SUCCESS; } +static ResultCode GetThreadContext32(Core::System& system, u32 thread_context, Handle handle) { + return GetThreadContext(system, static_cast<VAddr>(thread_context), handle); +} + /// Gets the priority for the specified thread static ResultCode GetThreadPriority(Core::System& system, u32* priority, Handle handle) { LOG_TRACE(Kernel_SVC, "called"); @@ -1126,6 +1154,7 @@ static ResultCode GetThreadPriority(Core::System& system, u32* priority, Handle const auto& handle_table = system.Kernel().CurrentProcess()->GetHandleTable(); const std::shared_ptr<Thread> thread = handle_table.Get<Thread>(handle); if (!thread) { + *priority = 0; LOG_ERROR(Kernel_SVC, "Thread handle does not exist, handle=0x{:08X}", handle); return ERR_INVALID_HANDLE; } @@ -1160,18 +1189,26 @@ static ResultCode SetThreadPriority(Core::System& system, Handle handle, u32 pri thread->SetPriority(priority); - system.PrepareReschedule(thread->GetProcessorID()); return RESULT_SUCCESS; } +static ResultCode SetThreadPriority32(Core::System& system, Handle handle, u32 priority) { + return SetThreadPriority(system, handle, priority); +} + /// Get which CPU core is executing the current thread static u32 GetCurrentProcessorNumber(Core::System& system) { LOG_TRACE(Kernel_SVC, "called"); - return system.CurrentScheduler().GetCurrentThread()->GetProcessorID(); + return static_cast<u32>(system.CurrentPhysicalCore().CoreIndex()); +} + +static u32 GetCurrentProcessorNumber32(Core::System& system) { + return GetCurrentProcessorNumber(system); } static ResultCode MapSharedMemory(Core::System& system, Handle shared_memory_handle, VAddr addr, u64 size, u32 permissions) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_TRACE(Kernel_SVC, "called, shared_memory_handle=0x{:X}, addr=0x{:X}, size=0x{:X}, permissions=0x{:08X}", shared_memory_handle, addr, size, permissions); @@ -1197,79 +1234,61 @@ static ResultCode MapSharedMemory(Core::System& system, Handle shared_memory_han return ERR_INVALID_ADDRESS_STATE; } - const auto permissions_type = static_cast<MemoryPermission>(permissions); - if (permissions_type != MemoryPermission::Read && - permissions_type != MemoryPermission::ReadWrite) { + const auto permission_type = static_cast<Memory::MemoryPermission>(permissions); + if ((permission_type | Memory::MemoryPermission::Write) != + Memory::MemoryPermission::ReadAndWrite) { LOG_ERROR(Kernel_SVC, "Expected Read or ReadWrite permission but got permissions=0x{:08X}", permissions); return ERR_INVALID_MEMORY_PERMISSIONS; } - auto* const current_process = system.Kernel().CurrentProcess(); - auto shared_memory = current_process->GetHandleTable().Get<SharedMemory>(shared_memory_handle); - if (!shared_memory) { - LOG_ERROR(Kernel_SVC, "Shared memory does not exist, shared_memory_handle=0x{:08X}", - shared_memory_handle); - return ERR_INVALID_HANDLE; - } + auto* const current_process{system.Kernel().CurrentProcess()}; + auto& page_table{current_process->PageTable()}; - const auto& vm_manager = current_process->VMManager(); - if (!vm_manager.IsWithinASLRRegion(addr, size)) { - LOG_ERROR(Kernel_SVC, "Region is not within the ASLR region. addr=0x{:016X}, size={:016X}", + if (page_table.IsInvalidRegion(addr, size)) { + LOG_ERROR(Kernel_SVC, + "Addr does not fit within the valid region, addr=0x{:016X}, " + "size=0x{:016X}", addr, size); return ERR_INVALID_MEMORY_RANGE; } - return shared_memory->Map(*current_process, addr, permissions_type, MemoryPermission::DontCare); -} - -static ResultCode UnmapSharedMemory(Core::System& system, Handle shared_memory_handle, VAddr addr, - u64 size) { - LOG_WARNING(Kernel_SVC, "called, shared_memory_handle=0x{:08X}, addr=0x{:X}, size=0x{:X}", - shared_memory_handle, addr, size); - - if (!Common::Is4KBAligned(addr)) { - LOG_ERROR(Kernel_SVC, "Address is not aligned to 4KB, addr=0x{:016X}", addr); - return ERR_INVALID_ADDRESS; - } - - if (size == 0) { - LOG_ERROR(Kernel_SVC, "Size is 0"); - return ERR_INVALID_SIZE; - } - - if (!Common::Is4KBAligned(size)) { - LOG_ERROR(Kernel_SVC, "Size is not aligned to 4KB, size=0x{:016X}", size); - return ERR_INVALID_SIZE; + if (page_table.IsInsideHeapRegion(addr, size)) { + LOG_ERROR(Kernel_SVC, + "Addr does not fit within the heap region, addr=0x{:016X}, " + "size=0x{:016X}", + addr, size); + return ERR_INVALID_MEMORY_RANGE; } - if (!IsValidAddressRange(addr, size)) { - LOG_ERROR(Kernel_SVC, "Region is not a valid address range, addr=0x{:016X}, size=0x{:016X}", + if (page_table.IsInsideAliasRegion(addr, size)) { + LOG_ERROR(Kernel_SVC, + "Address does not fit within the map region, addr=0x{:016X}, " + "size=0x{:016X}", addr, size); - return ERR_INVALID_ADDRESS_STATE; + return ERR_INVALID_MEMORY_RANGE; } - auto* const current_process = system.Kernel().CurrentProcess(); - auto shared_memory = current_process->GetHandleTable().Get<SharedMemory>(shared_memory_handle); + auto shared_memory{current_process->GetHandleTable().Get<SharedMemory>(shared_memory_handle)}; if (!shared_memory) { LOG_ERROR(Kernel_SVC, "Shared memory does not exist, shared_memory_handle=0x{:08X}", shared_memory_handle); return ERR_INVALID_HANDLE; } - const auto& vm_manager = current_process->VMManager(); - if (!vm_manager.IsWithinASLRRegion(addr, size)) { - LOG_ERROR(Kernel_SVC, "Region is not within the ASLR region. addr=0x{:016X}, size={:016X}", - addr, size); - return ERR_INVALID_MEMORY_RANGE; - } + return shared_memory->Map(*current_process, addr, size, permission_type); +} - return shared_memory->Unmap(*current_process, addr, size); +static ResultCode MapSharedMemory32(Core::System& system, Handle shared_memory_handle, u32 addr, + u32 size, u32 permissions) { + return MapSharedMemory(system, shared_memory_handle, static_cast<VAddr>(addr), + static_cast<std::size_t>(size), permissions); } static ResultCode QueryProcessMemory(Core::System& system, VAddr memory_info_address, VAddr page_info_address, Handle process_handle, VAddr address) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_TRACE(Kernel_SVC, "called process=0x{:08X} address={:X}", process_handle, address); const auto& handle_table = system.Kernel().CurrentProcess()->GetHandleTable(); std::shared_ptr<Process> process = handle_table.Get<Process>(process_handle); @@ -1279,18 +1298,17 @@ static ResultCode QueryProcessMemory(Core::System& system, VAddr memory_info_add return ERR_INVALID_HANDLE; } - auto& memory = system.Memory(); - const auto& vm_manager = process->VMManager(); - const MemoryInfo memory_info = vm_manager.QueryMemory(address); - - memory.Write64(memory_info_address, memory_info.base_address); - memory.Write64(memory_info_address + 8, memory_info.size); - memory.Write32(memory_info_address + 16, memory_info.state); - memory.Write32(memory_info_address + 20, memory_info.attributes); - memory.Write32(memory_info_address + 24, memory_info.permission); - memory.Write32(memory_info_address + 32, memory_info.ipc_ref_count); - memory.Write32(memory_info_address + 28, memory_info.device_ref_count); - memory.Write32(memory_info_address + 36, 0); + auto& memory{system.Memory()}; + const auto memory_info{process->PageTable().QueryInfo(address).GetSvcMemoryInfo()}; + + memory.Write64(memory_info_address + 0x00, memory_info.addr); + memory.Write64(memory_info_address + 0x08, memory_info.size); + memory.Write32(memory_info_address + 0x10, static_cast<u32>(memory_info.state) & 0xff); + memory.Write32(memory_info_address + 0x14, static_cast<u32>(memory_info.attr)); + memory.Write32(memory_info_address + 0x18, static_cast<u32>(memory_info.perm)); + memory.Write32(memory_info_address + 0x1c, memory_info.ipc_refcount); + memory.Write32(memory_info_address + 0x20, memory_info.device_refcount); + memory.Write32(memory_info_address + 0x24, 0); // Page info appears to be currently unused by the kernel and is always set to zero. memory.Write32(page_info_address, 0); @@ -1362,8 +1380,8 @@ static ResultCode MapProcessCodeMemory(Core::System& system, Handle process_hand return ERR_INVALID_HANDLE; } - auto& vm_manager = process->VMManager(); - if (!vm_manager.IsWithinAddressSpace(src_address, size)) { + auto& page_table = process->PageTable(); + if (!page_table.IsInsideAddressSpace(src_address, size)) { LOG_ERROR(Kernel_SVC, "Source address range is not within the address space (src_address=0x{:016X}, " "size=0x{:016X}).", @@ -1371,7 +1389,7 @@ static ResultCode MapProcessCodeMemory(Core::System& system, Handle process_hand return ERR_INVALID_ADDRESS_STATE; } - if (!vm_manager.IsWithinASLRRegion(dst_address, size)) { + if (!page_table.IsInsideASLRRegion(dst_address, size)) { LOG_ERROR(Kernel_SVC, "Destination address range is not within the ASLR region (dst_address=0x{:016X}, " "size=0x{:016X}).", @@ -1379,7 +1397,7 @@ static ResultCode MapProcessCodeMemory(Core::System& system, Handle process_hand return ERR_INVALID_MEMORY_RANGE; } - return vm_manager.MapCodeMemory(dst_address, src_address, size); + return page_table.MapProcessCodeMemory(dst_address, src_address, size); } static ResultCode UnmapProcessCodeMemory(Core::System& system, Handle process_handle, @@ -1430,8 +1448,8 @@ static ResultCode UnmapProcessCodeMemory(Core::System& system, Handle process_ha return ERR_INVALID_HANDLE; } - auto& vm_manager = process->VMManager(); - if (!vm_manager.IsWithinAddressSpace(src_address, size)) { + auto& page_table = process->PageTable(); + if (!page_table.IsInsideAddressSpace(src_address, size)) { LOG_ERROR(Kernel_SVC, "Source address range is not within the address space (src_address=0x{:016X}, " "size=0x{:016X}).", @@ -1439,7 +1457,7 @@ static ResultCode UnmapProcessCodeMemory(Core::System& system, Handle process_ha return ERR_INVALID_ADDRESS_STATE; } - if (!vm_manager.IsWithinASLRRegion(dst_address, size)) { + if (!page_table.IsInsideASLRRegion(dst_address, size)) { LOG_ERROR(Kernel_SVC, "Destination address range is not within the ASLR region (dst_address=0x{:016X}, " "size=0x{:016X}).", @@ -1447,12 +1465,13 @@ static ResultCode UnmapProcessCodeMemory(Core::System& system, Handle process_ha return ERR_INVALID_MEMORY_RANGE; } - return vm_manager.UnmapCodeMemory(dst_address, src_address, size); + return page_table.UnmapProcessCodeMemory(dst_address, src_address, size); } /// Exits the current process static void ExitProcess(Core::System& system) { auto* current_process = system.Kernel().CurrentProcess(); + UNIMPLEMENTED(); LOG_INFO(Kernel_SVC, "Process {} exiting", current_process->GetProcessID()); ASSERT_MSG(current_process->GetStatus() == ProcessStatus::Running, @@ -1462,8 +1481,10 @@ static void ExitProcess(Core::System& system) { // Kill the current thread system.CurrentScheduler().GetCurrentThread()->Stop(); +} - system.PrepareReschedule(); +static void ExitProcess32(Core::System& system) { + ExitProcess(system); } /// Creates a new thread @@ -1506,9 +1527,13 @@ static ResultCode CreateThread(Core::System& system, Handle* out_handle, VAddr e } auto& kernel = system.Kernel(); + + ASSERT(kernel.CurrentProcess()->GetResourceLimit()->Reserve(ResourceType::Threads, 1)); + + ThreadType type = THREADTYPE_USER; CASCADE_RESULT(std::shared_ptr<Thread> thread, - Thread::Create(kernel, "", entry_point, priority, arg, processor_id, stack_top, - *current_process)); + Thread::Create(system, type, "", entry_point, priority, arg, processor_id, + stack_top, current_process)); const auto new_thread_handle = current_process->GetHandleTable().Create(thread); if (new_thread_handle.Failed()) { @@ -1522,11 +1547,15 @@ static ResultCode CreateThread(Core::System& system, Handle* out_handle, VAddr e thread->SetName( fmt::format("thread[entry_point={:X}, handle={:X}]", entry_point, *new_thread_handle)); - system.PrepareReschedule(thread->GetProcessorID()); - return RESULT_SUCCESS; } +static ResultCode CreateThread32(Core::System& system, Handle* out_handle, u32 priority, + u32 entry_point, u32 arg, u32 stack_top, s32 processor_id) { + return CreateThread(system, out_handle, static_cast<VAddr>(entry_point), static_cast<u64>(arg), + static_cast<VAddr>(stack_top), priority, processor_id); +} + /// Starts the thread for the provided handle static ResultCode StartThread(Core::System& system, Handle thread_handle) { LOG_DEBUG(Kernel_SVC, "called thread=0x{:08X}", thread_handle); @@ -1541,13 +1570,11 @@ static ResultCode StartThread(Core::System& system, Handle thread_handle) { ASSERT(thread->GetStatus() == ThreadStatus::Dormant); - thread->ResumeFromWait(); - - if (thread->GetStatus() == ThreadStatus::Ready) { - system.PrepareReschedule(thread->GetProcessorID()); - } + return thread->Start(); +} - return RESULT_SUCCESS; +static ResultCode StartThread32(Core::System& system, Handle thread_handle) { + return StartThread(system, thread_handle); } /// Called when a thread exits @@ -1555,9 +1582,12 @@ static void ExitThread(Core::System& system) { LOG_DEBUG(Kernel_SVC, "called, pc=0x{:08X}", system.CurrentArmInterface().GetPC()); auto* const current_thread = system.CurrentScheduler().GetCurrentThread(); - current_thread->Stop(); system.GlobalScheduler().RemoveThread(SharedFrom(current_thread)); - system.PrepareReschedule(); + current_thread->Stop(); +} + +static void ExitThread32(Core::System& system) { + ExitThread(system); } /// Sleep the current thread @@ -1576,15 +1606,21 @@ static void SleepThread(Core::System& system, s64 nanoseconds) { if (nanoseconds <= 0) { switch (static_cast<SleepType>(nanoseconds)) { - case SleepType::YieldWithoutLoadBalancing: - is_redundant = current_thread->YieldSimple(); + case SleepType::YieldWithoutLoadBalancing: { + auto pair = current_thread->YieldSimple(); + is_redundant = pair.second; break; - case SleepType::YieldWithLoadBalancing: - is_redundant = current_thread->YieldAndBalanceLoad(); + } + case SleepType::YieldWithLoadBalancing: { + auto pair = current_thread->YieldAndBalanceLoad(); + is_redundant = pair.second; break; - case SleepType::YieldAndWaitForLoadBalancing: - is_redundant = current_thread->YieldAndWaitForLoadBalancing(); + } + case SleepType::YieldAndWaitForLoadBalancing: { + auto pair = current_thread->YieldAndWaitForLoadBalancing(); + is_redundant = pair.second; break; + } default: UNREACHABLE_MSG("Unimplemented sleep yield type '{:016X}'!", nanoseconds); } @@ -1592,13 +1628,18 @@ static void SleepThread(Core::System& system, s64 nanoseconds) { current_thread->Sleep(nanoseconds); } - if (is_redundant) { - // If it's redundant, the core is pretty much idle. Some games keep idling - // a core while it's doing nothing, we advance timing to avoid costly continuous - // calls. - system.CoreTiming().AddTicks(2000); + if (is_redundant && !system.Kernel().IsMulticore()) { + system.Kernel().ExitSVCProfile(); + system.CoreTiming().AddTicks(1000U); + system.GetCpuManager().PreemptSingleCore(); + system.Kernel().EnterSVCProfile(); } - system.PrepareReschedule(current_thread->GetProcessorID()); +} + +static void SleepThread32(Core::System& system, u32 nanoseconds_low, u32 nanoseconds_high) { + const s64 nanoseconds = static_cast<s64>(static_cast<u64>(nanoseconds_low) | + (static_cast<u64>(nanoseconds_high) << 32)); + SleepThread(system, nanoseconds); } /// Wait process wide key atomic @@ -1610,7 +1651,7 @@ static ResultCode WaitProcessWideKeyAtomic(Core::System& system, VAddr mutex_add "called mutex_addr={:X}, condition_variable_addr={:X}, thread_handle=0x{:08X}, timeout={}", mutex_addr, condition_variable_addr, thread_handle, nano_seconds); - if (Memory::IsKernelVirtualAddress(mutex_addr)) { + if (Core::Memory::IsKernelVirtualAddress(mutex_addr)) { LOG_ERROR( Kernel_SVC, "Given mutex address must not be within the kernel address space. address=0x{:016X}", @@ -1625,31 +1666,69 @@ static ResultCode WaitProcessWideKeyAtomic(Core::System& system, VAddr mutex_add } ASSERT(condition_variable_addr == Common::AlignDown(condition_variable_addr, 4)); - + auto& kernel = system.Kernel(); + Handle event_handle; + Thread* current_thread = system.CurrentScheduler().GetCurrentThread(); auto* const current_process = system.Kernel().CurrentProcess(); - const auto& handle_table = current_process->GetHandleTable(); - std::shared_ptr<Thread> thread = handle_table.Get<Thread>(thread_handle); - ASSERT(thread); + { + SchedulerLockAndSleep lock(kernel, event_handle, current_thread, nano_seconds); + const auto& handle_table = current_process->GetHandleTable(); + std::shared_ptr<Thread> thread = handle_table.Get<Thread>(thread_handle); + ASSERT(thread); + + current_thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT); + + if (thread->IsPendingTermination()) { + lock.CancelSleep(); + return ERR_THREAD_TERMINATING; + } + + const auto release_result = current_process->GetMutex().Release(mutex_addr); + if (release_result.IsError()) { + lock.CancelSleep(); + return release_result; + } - const auto release_result = current_process->GetMutex().Release(mutex_addr); - if (release_result.IsError()) { - return release_result; + if (nano_seconds == 0) { + lock.CancelSleep(); + return RESULT_TIMEOUT; + } + + current_thread->SetCondVarWaitAddress(condition_variable_addr); + current_thread->SetMutexWaitAddress(mutex_addr); + current_thread->SetWaitHandle(thread_handle); + current_thread->SetStatus(ThreadStatus::WaitCondVar); + current_process->InsertConditionVariableThread(SharedFrom(current_thread)); } - Thread* current_thread = system.CurrentScheduler().GetCurrentThread(); - current_thread->SetCondVarWaitAddress(condition_variable_addr); - current_thread->SetMutexWaitAddress(mutex_addr); - current_thread->SetWaitHandle(thread_handle); - current_thread->SetStatus(ThreadStatus::WaitCondVar); - current_thread->InvalidateWakeupCallback(); - current_process->InsertConditionVariableThread(SharedFrom(current_thread)); + if (event_handle != InvalidHandle) { + auto& time_manager = kernel.TimeManager(); + time_manager.UnscheduleTimeEvent(event_handle); + } + + { + SchedulerLock lock(kernel); - current_thread->WakeAfterDelay(nano_seconds); + auto* owner = current_thread->GetLockOwner(); + if (owner != nullptr) { + owner->RemoveMutexWaiter(SharedFrom(current_thread)); + } + current_process->RemoveConditionVariableThread(SharedFrom(current_thread)); + } // Note: Deliberately don't attempt to inherit the lock owner's priority. - system.PrepareReschedule(current_thread->GetProcessorID()); - return RESULT_SUCCESS; + return current_thread->GetSignalingResult(); +} + +static ResultCode WaitProcessWideKeyAtomic32(Core::System& system, u32 mutex_addr, + u32 condition_variable_addr, Handle thread_handle, + u32 nanoseconds_low, u32 nanoseconds_high) { + const s64 nanoseconds = + static_cast<s64>(nanoseconds_low | (static_cast<u64>(nanoseconds_high) << 32)); + return WaitProcessWideKeyAtomic(system, static_cast<VAddr>(mutex_addr), + static_cast<VAddr>(condition_variable_addr), thread_handle, + nanoseconds); } /// Signal process wide key @@ -1660,16 +1739,18 @@ static void SignalProcessWideKey(Core::System& system, VAddr condition_variable_ ASSERT(condition_variable_addr == Common::AlignDown(condition_variable_addr, 4)); // Retrieve a list of all threads that are waiting for this condition variable. - auto* const current_process = system.Kernel().CurrentProcess(); + auto& kernel = system.Kernel(); + SchedulerLock lock(kernel); + auto* const current_process = kernel.CurrentProcess(); std::vector<std::shared_ptr<Thread>> waiting_threads = current_process->GetConditionVariableThreads(condition_variable_addr); // Only process up to 'target' threads, unless 'target' is less equal 0, in which case process // them all. std::size_t last = waiting_threads.size(); - if (target > 0) + if (target > 0) { last = std::min(waiting_threads.size(), static_cast<std::size_t>(target)); - + } for (std::size_t index = 0; index < last; ++index) { auto& thread = waiting_threads[index]; @@ -1677,21 +1758,17 @@ static void SignalProcessWideKey(Core::System& system, VAddr condition_variable_ // liberate Cond Var Thread. current_process->RemoveConditionVariableThread(thread); - thread->SetCondVarWaitAddress(0); const std::size_t current_core = system.CurrentCoreIndex(); auto& monitor = system.Monitor(); - auto& memory = system.Memory(); // Atomically read the value of the mutex. u32 mutex_val = 0; u32 update_val = 0; const VAddr mutex_address = thread->GetMutexWaitAddress(); do { - monitor.SetExclusive(current_core, mutex_address); - // If the mutex is not yet acquired, acquire it. - mutex_val = memory.Read32(mutex_address); + mutex_val = monitor.ExclusiveRead32(current_core, mutex_address); if (mutex_val != 0) { update_val = mutex_val | Mutex::MutexHasWaitersFlag; @@ -1699,33 +1776,28 @@ static void SignalProcessWideKey(Core::System& system, VAddr condition_variable_ update_val = thread->GetWaitHandle(); } } while (!monitor.ExclusiveWrite32(current_core, mutex_address, update_val)); + monitor.ClearExclusive(); if (mutex_val == 0) { // We were able to acquire the mutex, resume this thread. - ASSERT(thread->GetStatus() == ThreadStatus::WaitCondVar); - thread->ResumeFromWait(); - auto* const lock_owner = thread->GetLockOwner(); if (lock_owner != nullptr) { lock_owner->RemoveMutexWaiter(thread); } thread->SetLockOwner(nullptr); - thread->SetMutexWaitAddress(0); - thread->SetWaitHandle(0); - thread->SetWaitSynchronizationResult(RESULT_SUCCESS); - system.PrepareReschedule(thread->GetProcessorID()); + thread->SetSynchronizationResults(nullptr, RESULT_SUCCESS); + thread->ResumeFromWait(); } else { // The mutex is already owned by some other thread, make this thread wait on it. const Handle owner_handle = static_cast<Handle>(mutex_val & Mutex::MutexOwnerMask); const auto& handle_table = system.Kernel().CurrentProcess()->GetHandleTable(); auto owner = handle_table.Get<Thread>(owner_handle); ASSERT(owner); - ASSERT(thread->GetStatus() == ThreadStatus::WaitCondVar); - thread->InvalidateWakeupCallback(); - thread->SetStatus(ThreadStatus::WaitMutex); + if (thread->GetStatus() == ThreadStatus::WaitCondVar) { + thread->SetStatus(ThreadStatus::WaitMutex); + } owner->AddMutexWaiter(thread); - system.PrepareReschedule(thread->GetProcessorID()); } } } @@ -1741,7 +1813,7 @@ static ResultCode WaitForAddress(Core::System& system, VAddr address, u32 type, type, value, timeout); // If the passed address is a kernel virtual address, return invalid memory state. - if (Memory::IsKernelVirtualAddress(address)) { + if (Core::Memory::IsKernelVirtualAddress(address)) { LOG_ERROR(Kernel_SVC, "Address is a kernel virtual address, address={:016X}", address); return ERR_INVALID_ADDRESS_STATE; } @@ -1756,12 +1828,15 @@ static ResultCode WaitForAddress(Core::System& system, VAddr address, u32 type, auto& address_arbiter = system.Kernel().CurrentProcess()->GetAddressArbiter(); const ResultCode result = address_arbiter.WaitForAddress(address, arbitration_type, value, timeout); - if (result == RESULT_SUCCESS) { - system.PrepareReschedule(); - } return result; } +static ResultCode WaitForAddress32(Core::System& system, u32 address, u32 type, s32 value, + u32 timeout_low, u32 timeout_high) { + s64 timeout = static_cast<s64>(timeout_low | (static_cast<u64>(timeout_high) << 32)); + return WaitForAddress(system, static_cast<VAddr>(address), type, value, timeout); +} + // Signals to an address (via Address Arbiter) static ResultCode SignalToAddress(Core::System& system, VAddr address, u32 type, s32 value, s32 num_to_wake) { @@ -1769,7 +1844,7 @@ static ResultCode SignalToAddress(Core::System& system, VAddr address, u32 type, address, type, value, num_to_wake); // If the passed address is a kernel virtual address, return invalid memory state. - if (Memory::IsKernelVirtualAddress(address)) { + if (Core::Memory::IsKernelVirtualAddress(address)) { LOG_ERROR(Kernel_SVC, "Address is a kernel virtual address, address={:016X}", address); return ERR_INVALID_ADDRESS_STATE; } @@ -1785,6 +1860,11 @@ static ResultCode SignalToAddress(Core::System& system, VAddr address, u32 type, return address_arbiter.SignalToAddress(address, signal_type, value, num_to_wake); } +static ResultCode SignalToAddress32(Core::System& system, u32 address, u32 type, s32 value, + s32 num_to_wake) { + return SignalToAddress(system, static_cast<VAddr>(address), type, value, num_to_wake); +} + static void KernelDebug([[maybe_unused]] Core::System& system, [[maybe_unused]] u32 kernel_debug_type, [[maybe_unused]] u64 param1, [[maybe_unused]] u64 param2, [[maybe_unused]] u64 param3) { @@ -1803,14 +1883,21 @@ static u64 GetSystemTick(Core::System& system) { auto& core_timing = system.CoreTiming(); // Returns the value of cntpct_el0 (https://switchbrew.org/wiki/SVC#svcGetSystemTick) - const u64 result{Core::Timing::CpuCyclesToClockCycles(system.CoreTiming().GetTicks())}; + const u64 result{system.CoreTiming().GetClockTicks()}; - // Advance time to defeat dumb games that busy-wait for the frame to end. - core_timing.AddTicks(400); + if (!system.Kernel().IsMulticore()) { + core_timing.AddTicks(400U); + } return result; } +static void GetSystemTick32(Core::System& system, u32* time_low, u32* time_high) { + u64 time = GetSystemTick(system); + *time_low = static_cast<u32>(time); + *time_high = static_cast<u32>(time >> 32); +} + /// Close a handle static ResultCode CloseHandle(Core::System& system, Handle handle) { LOG_TRACE(Kernel_SVC, "Closing handle 0x{:08X}", handle); @@ -1843,9 +1930,14 @@ static ResultCode ResetSignal(Core::System& system, Handle handle) { return ERR_INVALID_HANDLE; } +static ResultCode ResetSignal32(Core::System& system, Handle handle) { + return ResetSignal(system, handle); +} + /// Creates a TransferMemory object static ResultCode CreateTransferMemory(Core::System& system, Handle* handle, VAddr addr, u64 size, u32 permissions) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_DEBUG(Kernel_SVC, "called addr=0x{:X}, size=0x{:X}, perms=0x{:08X}", addr, size, permissions); @@ -1865,9 +1957,9 @@ static ResultCode CreateTransferMemory(Core::System& system, Handle* handle, VAd return ERR_INVALID_ADDRESS_STATE; } - const auto perms = static_cast<MemoryPermission>(permissions); - if (perms != MemoryPermission::None && perms != MemoryPermission::Read && - perms != MemoryPermission::ReadWrite) { + const auto perms{static_cast<Memory::MemoryPermission>(permissions)}; + if (perms > Memory::MemoryPermission::ReadAndWrite || + perms == Memory::MemoryPermission::Write) { LOG_ERROR(Kernel_SVC, "Invalid memory permissions for transfer memory! (perms={:08X})", permissions); return ERR_INVALID_MEMORY_PERMISSIONS; @@ -1890,109 +1982,10 @@ static ResultCode CreateTransferMemory(Core::System& system, Handle* handle, VAd return RESULT_SUCCESS; } -static ResultCode MapTransferMemory(Core::System& system, Handle handle, VAddr address, u64 size, - u32 permission_raw) { - LOG_DEBUG(Kernel_SVC, - "called. handle=0x{:08X}, address=0x{:016X}, size=0x{:016X}, permissions=0x{:08X}", - handle, address, size, permission_raw); - - if (!Common::Is4KBAligned(address)) { - LOG_ERROR(Kernel_SVC, "Transfer memory addresses must be 4KB aligned (size=0x{:016X}).", - address); - return ERR_INVALID_ADDRESS; - } - - if (size == 0 || !Common::Is4KBAligned(size)) { - LOG_ERROR(Kernel_SVC, - "Transfer memory sizes must be 4KB aligned and not be zero (size=0x{:016X}).", - size); - return ERR_INVALID_SIZE; - } - - if (!IsValidAddressRange(address, size)) { - LOG_ERROR(Kernel_SVC, - "Given address and size overflows the 64-bit range (address=0x{:016X}, " - "size=0x{:016X}).", - address, size); - return ERR_INVALID_ADDRESS_STATE; - } - - const auto permissions = static_cast<MemoryPermission>(permission_raw); - if (permissions != MemoryPermission::None && permissions != MemoryPermission::Read && - permissions != MemoryPermission::ReadWrite) { - LOG_ERROR(Kernel_SVC, "Invalid transfer memory permissions given (permissions=0x{:08X}).", - permission_raw); - return ERR_INVALID_STATE; - } - - const auto& kernel = system.Kernel(); - const auto* const current_process = kernel.CurrentProcess(); - const auto& handle_table = current_process->GetHandleTable(); - - auto transfer_memory = handle_table.Get<TransferMemory>(handle); - if (!transfer_memory) { - LOG_ERROR(Kernel_SVC, "Nonexistent transfer memory handle given (handle=0x{:08X}).", - handle); - return ERR_INVALID_HANDLE; - } - - if (!current_process->VMManager().IsWithinASLRRegion(address, size)) { - LOG_ERROR(Kernel_SVC, - "Given address and size don't fully fit within the ASLR region " - "(address=0x{:016X}, size=0x{:016X}).", - address, size); - return ERR_INVALID_MEMORY_RANGE; - } - - return transfer_memory->MapMemory(address, size, permissions); -} - -static ResultCode UnmapTransferMemory(Core::System& system, Handle handle, VAddr address, - u64 size) { - LOG_DEBUG(Kernel_SVC, "called. handle=0x{:08X}, address=0x{:016X}, size=0x{:016X}", handle, - address, size); - - if (!Common::Is4KBAligned(address)) { - LOG_ERROR(Kernel_SVC, "Transfer memory addresses must be 4KB aligned (size=0x{:016X}).", - address); - return ERR_INVALID_ADDRESS; - } - - if (size == 0 || !Common::Is4KBAligned(size)) { - LOG_ERROR(Kernel_SVC, - "Transfer memory sizes must be 4KB aligned and not be zero (size=0x{:016X}).", - size); - return ERR_INVALID_SIZE; - } - - if (!IsValidAddressRange(address, size)) { - LOG_ERROR(Kernel_SVC, - "Given address and size overflows the 64-bit range (address=0x{:016X}, " - "size=0x{:016X}).", - address, size); - return ERR_INVALID_ADDRESS_STATE; - } - - const auto& kernel = system.Kernel(); - const auto* const current_process = kernel.CurrentProcess(); - const auto& handle_table = current_process->GetHandleTable(); - - auto transfer_memory = handle_table.Get<TransferMemory>(handle); - if (!transfer_memory) { - LOG_ERROR(Kernel_SVC, "Nonexistent transfer memory handle given (handle=0x{:08X}).", - handle); - return ERR_INVALID_HANDLE; - } - - if (!current_process->VMManager().IsWithinASLRRegion(address, size)) { - LOG_ERROR(Kernel_SVC, - "Given address and size don't fully fit within the ASLR region " - "(address=0x{:016X}, size=0x{:016X}).", - address, size); - return ERR_INVALID_MEMORY_RANGE; - } - - return transfer_memory->UnmapMemory(address, size); +static ResultCode CreateTransferMemory32(Core::System& system, Handle* handle, u32 addr, u32 size, + u32 permissions) { + return CreateTransferMemory(system, handle, static_cast<VAddr>(addr), + static_cast<std::size_t>(size), permissions); } static ResultCode GetThreadCoreMask(Core::System& system, Handle thread_handle, u32* core, @@ -2004,6 +1997,8 @@ static ResultCode GetThreadCoreMask(Core::System& system, Handle thread_handle, if (!thread) { LOG_ERROR(Kernel_SVC, "Thread handle does not exist, thread_handle=0x{:08X}", thread_handle); + *core = 0; + *mask = 0; return ERR_INVALID_HANDLE; } @@ -2013,6 +2008,15 @@ static ResultCode GetThreadCoreMask(Core::System& system, Handle thread_handle, return RESULT_SUCCESS; } +static ResultCode GetThreadCoreMask32(Core::System& system, Handle thread_handle, u32* core, + u32* mask_low, u32* mask_high) { + u64 mask{}; + const auto result = GetThreadCoreMask(system, thread_handle, core, &mask); + *mask_high = static_cast<u32>(mask >> 32); + *mask_low = static_cast<u32>(mask); + return result; +} + static ResultCode SetThreadCoreMask(Core::System& system, Handle thread_handle, u32 core, u64 affinity_mask) { LOG_DEBUG(Kernel_SVC, "called, handle=0x{:08X}, core=0x{:X}, affinity_mask=0x{:016X}", @@ -2044,7 +2048,7 @@ static ResultCode SetThreadCoreMask(Core::System& system, Handle thread_handle, return ERR_INVALID_COMBINATION; } - if (core < Core::NUM_CPU_CORES) { + if (core < Core::Hardware::NUM_CPU_CORES) { if ((affinity_mask & (1ULL << core)) == 0) { LOG_ERROR(Kernel_SVC, "Core is not enabled for the current mask, core={}, mask={:016X}", core, @@ -2066,57 +2070,14 @@ static ResultCode SetThreadCoreMask(Core::System& system, Handle thread_handle, return ERR_INVALID_HANDLE; } - system.PrepareReschedule(thread->GetProcessorID()); - thread->ChangeCore(core, affinity_mask); - system.PrepareReschedule(thread->GetProcessorID()); - - return RESULT_SUCCESS; + return thread->SetCoreAndAffinityMask(core, affinity_mask); } -static ResultCode CreateSharedMemory(Core::System& system, Handle* handle, u64 size, - u32 local_permissions, u32 remote_permissions) { - LOG_TRACE(Kernel_SVC, "called, size=0x{:X}, localPerms=0x{:08X}, remotePerms=0x{:08X}", size, - local_permissions, remote_permissions); - if (size == 0) { - LOG_ERROR(Kernel_SVC, "Size is 0"); - return ERR_INVALID_SIZE; - } - if (!Common::Is4KBAligned(size)) { - LOG_ERROR(Kernel_SVC, "Size is not aligned to 4KB, 0x{:016X}", size); - return ERR_INVALID_SIZE; - } - - if (size >= MAIN_MEMORY_SIZE) { - LOG_ERROR(Kernel_SVC, "Size is not less than 8GB, 0x{:016X}", size); - return ERR_INVALID_SIZE; - } - - const auto local_perms = static_cast<MemoryPermission>(local_permissions); - if (local_perms != MemoryPermission::Read && local_perms != MemoryPermission::ReadWrite) { - LOG_ERROR(Kernel_SVC, - "Invalid local memory permissions, expected Read or ReadWrite but got " - "local_permissions={}", - static_cast<u32>(local_permissions)); - return ERR_INVALID_MEMORY_PERMISSIONS; - } - - const auto remote_perms = static_cast<MemoryPermission>(remote_permissions); - if (remote_perms != MemoryPermission::Read && remote_perms != MemoryPermission::ReadWrite && - remote_perms != MemoryPermission::DontCare) { - LOG_ERROR(Kernel_SVC, - "Invalid remote memory permissions, expected Read, ReadWrite or DontCare but got " - "remote_permissions={}", - static_cast<u32>(remote_permissions)); - return ERR_INVALID_MEMORY_PERMISSIONS; - } - - auto& kernel = system.Kernel(); - auto process = kernel.CurrentProcess(); - auto& handle_table = process->GetHandleTable(); - auto shared_mem_handle = SharedMemory::Create(kernel, process, size, local_perms, remote_perms); - - CASCADE_RESULT(*handle, handle_table.Create(shared_mem_handle)); - return RESULT_SUCCESS; +static ResultCode SetThreadCoreMask32(Core::System& system, Handle thread_handle, u32 core, + u32 affinity_mask_low, u32 affinity_mask_high) { + const u64 affinity_mask = + static_cast<u64>(affinity_mask_low) | (static_cast<u64>(affinity_mask_high) << 32); + return SetThreadCoreMask(system, thread_handle, core, affinity_mask); } static ResultCode CreateEvent(Core::System& system, Handle* write_handle, Handle* read_handle) { @@ -2147,6 +2108,10 @@ static ResultCode CreateEvent(Core::System& system, Handle* write_handle, Handle return RESULT_SUCCESS; } +static ResultCode CreateEvent32(Core::System& system, Handle* write_handle, Handle* read_handle) { + return CreateEvent(system, write_handle, read_handle); +} + static ResultCode ClearEvent(Core::System& system, Handle handle) { LOG_TRACE(Kernel_SVC, "called, event=0x{:08X}", handle); @@ -2168,6 +2133,10 @@ static ResultCode ClearEvent(Core::System& system, Handle handle) { return ERR_INVALID_HANDLE; } +static ResultCode ClearEvent32(Core::System& system, Handle handle) { + return ClearEvent(system, handle); +} + static ResultCode SignalEvent(Core::System& system, Handle handle) { LOG_DEBUG(Kernel_SVC, "called. Handle=0x{:08X}", handle); @@ -2180,10 +2149,13 @@ static ResultCode SignalEvent(Core::System& system, Handle handle) { } writable_event->Signal(); - system.PrepareReschedule(); return RESULT_SUCCESS; } +static ResultCode SignalEvent32(Core::System& system, Handle handle) { + return SignalEvent(system, handle); +} + static ResultCode GetProcessInfo(Core::System& system, u64* out, Handle process_handle, u32 type) { LOG_DEBUG(Kernel_SVC, "called, handle=0x{:08X}, type=0x{:X}", process_handle, type); @@ -2211,6 +2183,7 @@ static ResultCode GetProcessInfo(Core::System& system, u64* out, Handle process_ } static ResultCode CreateResourceLimit(Core::System& system, Handle* out_handle) { + std::lock_guard lock{HLE::g_hle_lock}; LOG_DEBUG(Kernel_SVC, "called"); auto& kernel = system.Kernel(); @@ -2305,11 +2278,10 @@ static ResultCode GetProcessList(Core::System& system, u32* out_num_processes, } const auto& kernel = system.Kernel(); - const auto& vm_manager = kernel.CurrentProcess()->VMManager(); const auto total_copy_size = out_process_ids_size * sizeof(u64); - if (out_process_ids_size > 0 && - !vm_manager.IsWithinAddressSpace(out_process_ids, total_copy_size)) { + if (out_process_ids_size > 0 && !kernel.CurrentProcess()->PageTable().IsInsideAddressSpace( + out_process_ids, total_copy_size)) { LOG_ERROR(Kernel_SVC, "Address range outside address space. begin=0x{:016X}, end=0x{:016X}", out_process_ids, out_process_ids + total_copy_size); return ERR_INVALID_ADDRESS_STATE; @@ -2345,11 +2317,10 @@ static ResultCode GetThreadList(Core::System& system, u32* out_num_threads, VAdd } const auto* const current_process = system.Kernel().CurrentProcess(); - const auto& vm_manager = current_process->VMManager(); const auto total_copy_size = out_thread_ids_size * sizeof(u64); if (out_thread_ids_size > 0 && - !vm_manager.IsWithinAddressSpace(out_thread_ids, total_copy_size)) { + !current_process->PageTable().IsInsideAddressSpace(out_thread_ids, total_copy_size)) { LOG_ERROR(Kernel_SVC, "Address range outside address space. begin=0x{:016X}, end=0x{:016X}", out_thread_ids, out_thread_ids + total_copy_size); return ERR_INVALID_ADDRESS_STATE; @@ -2370,6 +2341,15 @@ static ResultCode GetThreadList(Core::System& system, u32* out_num_threads, VAdd return RESULT_SUCCESS; } +static ResultCode FlushProcessDataCache32(Core::System& system, Handle handle, u32 address, + u32 size) { + // Note(Blinkhawk): For emulation purposes of the data cache this is mostly a nope + // as all emulation is done in the same cache level in host architecture, thus data cache + // does not need flushing. + LOG_DEBUG(Kernel_SVC, "called"); + return RESULT_SUCCESS; +} + namespace { struct FunctionDef { using Func = void(Core::System&); @@ -2384,57 +2364,57 @@ static const FunctionDef SVC_Table_32[] = { {0x00, nullptr, "Unknown"}, {0x01, SvcWrap32<SetHeapSize32>, "SetHeapSize32"}, {0x02, nullptr, "Unknown"}, - {0x03, nullptr, "SetMemoryAttribute32"}, - {0x04, nullptr, "MapMemory32"}, - {0x05, nullptr, "UnmapMemory32"}, + {0x03, SvcWrap32<SetMemoryAttribute32>, "SetMemoryAttribute32"}, + {0x04, SvcWrap32<MapMemory32>, "MapMemory32"}, + {0x05, SvcWrap32<UnmapMemory32>, "UnmapMemory32"}, {0x06, SvcWrap32<QueryMemory32>, "QueryMemory32"}, - {0x07, nullptr, "ExitProcess32"}, - {0x08, nullptr, "CreateThread32"}, - {0x09, nullptr, "StartThread32"}, - {0x0a, nullptr, "ExitThread32"}, - {0x0b, nullptr, "SleepThread32"}, + {0x07, SvcWrap32<ExitProcess32>, "ExitProcess32"}, + {0x08, SvcWrap32<CreateThread32>, "CreateThread32"}, + {0x09, SvcWrap32<StartThread32>, "StartThread32"}, + {0x0a, SvcWrap32<ExitThread32>, "ExitThread32"}, + {0x0b, SvcWrap32<SleepThread32>, "SleepThread32"}, {0x0c, SvcWrap32<GetThreadPriority32>, "GetThreadPriority32"}, - {0x0d, nullptr, "SetThreadPriority32"}, - {0x0e, nullptr, "GetThreadCoreMask32"}, - {0x0f, nullptr, "SetThreadCoreMask32"}, - {0x10, nullptr, "GetCurrentProcessorNumber32"}, - {0x11, nullptr, "SignalEvent32"}, - {0x12, nullptr, "ClearEvent32"}, - {0x13, nullptr, "MapSharedMemory32"}, + {0x0d, SvcWrap32<SetThreadPriority32>, "SetThreadPriority32"}, + {0x0e, SvcWrap32<GetThreadCoreMask32>, "GetThreadCoreMask32"}, + {0x0f, SvcWrap32<SetThreadCoreMask32>, "SetThreadCoreMask32"}, + {0x10, SvcWrap32<GetCurrentProcessorNumber32>, "GetCurrentProcessorNumber32"}, + {0x11, SvcWrap32<SignalEvent32>, "SignalEvent32"}, + {0x12, SvcWrap32<ClearEvent32>, "ClearEvent32"}, + {0x13, SvcWrap32<MapSharedMemory32>, "MapSharedMemory32"}, {0x14, nullptr, "UnmapSharedMemory32"}, - {0x15, nullptr, "CreateTransferMemory32"}, + {0x15, SvcWrap32<CreateTransferMemory32>, "CreateTransferMemory32"}, {0x16, SvcWrap32<CloseHandle32>, "CloseHandle32"}, - {0x17, nullptr, "ResetSignal32"}, + {0x17, SvcWrap32<ResetSignal32>, "ResetSignal32"}, {0x18, SvcWrap32<WaitSynchronization32>, "WaitSynchronization32"}, - {0x19, nullptr, "CancelSynchronization32"}, - {0x1a, nullptr, "ArbitrateLock32"}, - {0x1b, nullptr, "ArbitrateUnlock32"}, - {0x1c, nullptr, "WaitProcessWideKeyAtomic32"}, + {0x19, SvcWrap32<CancelSynchronization32>, "CancelSynchronization32"}, + {0x1a, SvcWrap32<ArbitrateLock32>, "ArbitrateLock32"}, + {0x1b, SvcWrap32<ArbitrateUnlock32>, "ArbitrateUnlock32"}, + {0x1c, SvcWrap32<WaitProcessWideKeyAtomic32>, "WaitProcessWideKeyAtomic32"}, {0x1d, SvcWrap32<SignalProcessWideKey32>, "SignalProcessWideKey32"}, - {0x1e, nullptr, "GetSystemTick32"}, + {0x1e, SvcWrap32<GetSystemTick32>, "GetSystemTick32"}, {0x1f, SvcWrap32<ConnectToNamedPort32>, "ConnectToNamedPort32"}, {0x20, nullptr, "Unknown"}, {0x21, SvcWrap32<SendSyncRequest32>, "SendSyncRequest32"}, {0x22, nullptr, "SendSyncRequestWithUserBuffer32"}, {0x23, nullptr, "Unknown"}, - {0x24, nullptr, "GetProcessId32"}, + {0x24, SvcWrap32<GetProcessId32>, "GetProcessId32"}, {0x25, SvcWrap32<GetThreadId32>, "GetThreadId32"}, - {0x26, nullptr, "Break32"}, + {0x26, SvcWrap32<Break32>, "Break32"}, {0x27, nullptr, "OutputDebugString32"}, {0x28, nullptr, "Unknown"}, {0x29, SvcWrap32<GetInfo32>, "GetInfo32"}, {0x2a, nullptr, "Unknown"}, {0x2b, nullptr, "Unknown"}, - {0x2c, nullptr, "MapPhysicalMemory32"}, - {0x2d, nullptr, "UnmapPhysicalMemory32"}, + {0x2c, SvcWrap32<MapPhysicalMemory32>, "MapPhysicalMemory32"}, + {0x2d, SvcWrap32<UnmapPhysicalMemory32>, "UnmapPhysicalMemory32"}, {0x2e, nullptr, "Unknown"}, {0x2f, nullptr, "Unknown"}, {0x30, nullptr, "Unknown"}, {0x31, nullptr, "Unknown"}, - {0x32, nullptr, "SetThreadActivity32"}, - {0x33, nullptr, "GetThreadContext32"}, - {0x34, nullptr, "WaitForAddress32"}, - {0x35, nullptr, "SignalToAddress32"}, + {0x32, SvcWrap32<SetThreadActivity32>, "SetThreadActivity32"}, + {0x33, SvcWrap32<GetThreadContext32>, "GetThreadContext32"}, + {0x34, SvcWrap32<WaitForAddress32>, "WaitForAddress32"}, + {0x35, SvcWrap32<SignalToAddress32>, "SignalToAddress32"}, {0x36, nullptr, "Unknown"}, {0x37, nullptr, "Unknown"}, {0x38, nullptr, "Unknown"}, @@ -2450,7 +2430,7 @@ static const FunctionDef SVC_Table_32[] = { {0x42, nullptr, "Unknown"}, {0x43, nullptr, "ReplyAndReceive32"}, {0x44, nullptr, "Unknown"}, - {0x45, nullptr, "CreateEvent32"}, + {0x45, SvcWrap32<CreateEvent32>, "CreateEvent32"}, {0x46, nullptr, "Unknown"}, {0x47, nullptr, "Unknown"}, {0x48, nullptr, "Unknown"}, @@ -2476,7 +2456,7 @@ static const FunctionDef SVC_Table_32[] = { {0x5c, nullptr, "Unknown"}, {0x5d, nullptr, "Unknown"}, {0x5e, nullptr, "Unknown"}, - {0x5F, nullptr, "FlushProcessDataCache32"}, + {0x5F, SvcWrap32<FlushProcessDataCache32>, "FlushProcessDataCache32"}, {0x60, nullptr, "Unknown"}, {0x61, nullptr, "Unknown"}, {0x62, nullptr, "Unknown"}, @@ -2510,7 +2490,7 @@ static const FunctionDef SVC_Table_32[] = { static const FunctionDef SVC_Table_64[] = { {0x00, nullptr, "Unknown"}, {0x01, SvcWrap64<SetHeapSize>, "SetHeapSize"}, - {0x02, SvcWrap64<SetMemoryPermission>, "SetMemoryPermission"}, + {0x02, nullptr, "SetMemoryPermission"}, {0x03, SvcWrap64<SetMemoryAttribute>, "SetMemoryAttribute"}, {0x04, SvcWrap64<MapMemory>, "MapMemory"}, {0x05, SvcWrap64<UnmapMemory>, "UnmapMemory"}, @@ -2528,7 +2508,7 @@ static const FunctionDef SVC_Table_64[] = { {0x11, SvcWrap64<SignalEvent>, "SignalEvent"}, {0x12, SvcWrap64<ClearEvent>, "ClearEvent"}, {0x13, SvcWrap64<MapSharedMemory>, "MapSharedMemory"}, - {0x14, SvcWrap64<UnmapSharedMemory>, "UnmapSharedMemory"}, + {0x14, nullptr, "UnmapSharedMemory"}, {0x15, SvcWrap64<CreateTransferMemory>, "CreateTransferMemory"}, {0x16, SvcWrap64<CloseHandle>, "CloseHandle"}, {0x17, SvcWrap64<ResetSignal>, "ResetSignal"}, @@ -2588,9 +2568,9 @@ static const FunctionDef SVC_Table_64[] = { {0x4D, nullptr, "SleepSystem"}, {0x4E, nullptr, "ReadWriteRegister"}, {0x4F, nullptr, "SetProcessActivity"}, - {0x50, SvcWrap64<CreateSharedMemory>, "CreateSharedMemory"}, - {0x51, SvcWrap64<MapTransferMemory>, "MapTransferMemory"}, - {0x52, SvcWrap64<UnmapTransferMemory>, "UnmapTransferMemory"}, + {0x50, nullptr, "CreateSharedMemory"}, + {0x51, nullptr, "MapTransferMemory"}, + {0x52, nullptr, "UnmapTransferMemory"}, {0x53, nullptr, "CreateInterruptEvent"}, {0x54, nullptr, "QueryPhysicalAddress"}, {0x55, nullptr, "QueryIoMapping"}, @@ -2654,13 +2634,10 @@ static const FunctionDef* GetSVCInfo64(u32 func_num) { return &SVC_Table_64[func_num]; } -MICROPROFILE_DEFINE(Kernel_SVC, "Kernel", "SVC", MP_RGB(70, 200, 70)); - -void CallSVC(Core::System& system, u32 immediate) { - MICROPROFILE_SCOPE(Kernel_SVC); - - // Lock the global kernel mutex when we enter the kernel HLE. - std::lock_guard lock{HLE::g_hle_lock}; +void Call(Core::System& system, u32 immediate) { + system.ExitDynarmicProfile(); + auto& kernel = system.Kernel(); + kernel.EnterSVCProfile(); const FunctionDef* info = system.CurrentProcess()->Is64BitProcess() ? GetSVCInfo64(immediate) : GetSVCInfo32(immediate); @@ -2673,6 +2650,9 @@ void CallSVC(Core::System& system, u32 immediate) { } else { LOG_CRITICAL(Kernel_SVC, "Unknown SVC function 0x{:X}", immediate); } + + kernel.ExitSVCProfile(); + system.EnterDynarmicProfile(); } -} // namespace Kernel +} // namespace Kernel::Svc diff --git a/src/core/hle/kernel/svc.h b/src/core/hle/kernel/svc.h index c5539ac1c..46e64277e 100644 --- a/src/core/hle/kernel/svc.h +++ b/src/core/hle/kernel/svc.h @@ -10,8 +10,8 @@ namespace Core { class System; } -namespace Kernel { +namespace Kernel::Svc { -void CallSVC(Core::System& system, u32 immediate); +void Call(Core::System& system, u32 immediate); -} // namespace Kernel +} // namespace Kernel::Svc diff --git a/src/core/hle/kernel/svc_types.h b/src/core/hle/kernel/svc_types.h new file mode 100644 index 000000000..986724beb --- /dev/null +++ b/src/core/hle/kernel/svc_types.h @@ -0,0 +1,68 @@ +// Copyright 2020 yuzu emulator team +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include "common/common_funcs.h" +#include "common/common_types.h" + +namespace Kernel::Svc { + +enum class MemoryState : u32 { + Free = 0x00, + Io = 0x01, + Static = 0x02, + Code = 0x03, + CodeData = 0x04, + Normal = 0x05, + Shared = 0x06, + Alias = 0x07, + AliasCode = 0x08, + AliasCodeData = 0x09, + Ipc = 0x0A, + Stack = 0x0B, + ThreadLocal = 0x0C, + Transfered = 0x0D, + SharedTransfered = 0x0E, + SharedCode = 0x0F, + Inaccessible = 0x10, + NonSecureIpc = 0x11, + NonDeviceIpc = 0x12, + Kernel = 0x13, + GeneratedCode = 0x14, + CodeOut = 0x15, +}; +DECLARE_ENUM_FLAG_OPERATORS(MemoryState); + +enum class MemoryAttribute : u32 { + Locked = (1 << 0), + IpcLocked = (1 << 1), + DeviceShared = (1 << 2), + Uncached = (1 << 3), +}; +DECLARE_ENUM_FLAG_OPERATORS(MemoryAttribute); + +enum class MemoryPermission : u32 { + None = (0 << 0), + Read = (1 << 0), + Write = (1 << 1), + Execute = (1 << 2), + ReadWrite = Read | Write, + ReadExecute = Read | Execute, + DontCare = (1 << 28), +}; +DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission); + +struct MemoryInfo { + u64 addr{}; + u64 size{}; + MemoryState state{}; + MemoryAttribute attr{}; + MemoryPermission perm{}; + u32 ipc_refcount{}; + u32 device_refcount{}; + u32 padding{}; +}; + +} // namespace Kernel::Svc diff --git a/src/core/hle/kernel/svc_wrap.h b/src/core/hle/kernel/svc_wrap.h index 7d735e3fa..0b6dd9df0 100644 --- a/src/core/hle/kernel/svc_wrap.h +++ b/src/core/hle/kernel/svc_wrap.h @@ -350,13 +350,50 @@ void SvcWrap64(Core::System& system) { func(system, static_cast<u32>(Param(system, 0)), Param(system, 1), Param(system, 2)); } -// Used by QueryMemory32 +// Used by QueryMemory32, ArbitrateLock32 template <ResultCode func(Core::System&, u32, u32, u32)> void SvcWrap32(Core::System& system) { FuncReturn32(system, func(system, Param32(system, 0), Param32(system, 1), Param32(system, 2)).raw); } +// Used by Break32 +template <void func(Core::System&, u32, u32, u32)> +void SvcWrap32(Core::System& system) { + func(system, Param32(system, 0), Param32(system, 1), Param32(system, 2)); +} + +// Used by ExitProcess32, ExitThread32 +template <void func(Core::System&)> +void SvcWrap32(Core::System& system) { + func(system); +} + +// Used by GetCurrentProcessorNumber32 +template <u32 func(Core::System&)> +void SvcWrap32(Core::System& system) { + FuncReturn32(system, func(system)); +} + +// Used by SleepThread32 +template <void func(Core::System&, u32, u32)> +void SvcWrap32(Core::System& system) { + func(system, Param32(system, 0), Param32(system, 1)); +} + +// Used by CreateThread32 +template <ResultCode func(Core::System&, Handle*, u32, u32, u32, u32, s32)> +void SvcWrap32(Core::System& system) { + Handle param_1 = 0; + + const u32 retval = func(system, ¶m_1, Param32(system, 0), Param32(system, 1), + Param32(system, 2), Param32(system, 3), Param32(system, 4)) + .raw; + + system.CurrentArmInterface().SetReg(1, param_1); + FuncReturn(system, retval); +} + // Used by GetInfo32 template <ResultCode func(Core::System&, u32*, u32*, u32, u32, u32, u32)> void SvcWrap32(Core::System& system) { @@ -393,18 +430,114 @@ void SvcWrap32(Core::System& system) { FuncReturn(system, retval); } +// Used by GetSystemTick32 +template <void func(Core::System&, u32*, u32*)> +void SvcWrap32(Core::System& system) { + u32 param_1 = 0; + u32 param_2 = 0; + + func(system, ¶m_1, ¶m_2); + system.CurrentArmInterface().SetReg(0, param_1); + system.CurrentArmInterface().SetReg(1, param_2); +} + +// Used by CreateEvent32 +template <ResultCode func(Core::System&, Handle*, Handle*)> +void SvcWrap32(Core::System& system) { + Handle param_1 = 0; + Handle param_2 = 0; + + const u32 retval = func(system, ¶m_1, ¶m_2).raw; + system.CurrentArmInterface().SetReg(1, param_1); + system.CurrentArmInterface().SetReg(2, param_2); + FuncReturn(system, retval); +} + +// Used by GetThreadId32 +template <ResultCode func(Core::System&, Handle, u32*, u32*, u32*)> +void SvcWrap32(Core::System& system) { + u32 param_1 = 0; + u32 param_2 = 0; + u32 param_3 = 0; + + const u32 retval = func(system, Param32(system, 2), ¶m_1, ¶m_2, ¶m_3).raw; + system.CurrentArmInterface().SetReg(1, param_1); + system.CurrentArmInterface().SetReg(2, param_2); + system.CurrentArmInterface().SetReg(3, param_3); + FuncReturn(system, retval); +} + // Used by SignalProcessWideKey32 template <void func(Core::System&, u32, s32)> void SvcWrap32(Core::System& system) { func(system, static_cast<u32>(Param(system, 0)), static_cast<s32>(Param(system, 1))); } -// Used by SendSyncRequest32 +// Used by SetThreadPriority32 +template <ResultCode func(Core::System&, Handle, u32)> +void SvcWrap32(Core::System& system) { + const u32 retval = + func(system, static_cast<Handle>(Param(system, 0)), static_cast<u32>(Param(system, 1))).raw; + FuncReturn(system, retval); +} + +// Used by SetThreadCoreMask32 +template <ResultCode func(Core::System&, Handle, u32, u32, u32)> +void SvcWrap32(Core::System& system) { + const u32 retval = + func(system, static_cast<Handle>(Param(system, 0)), static_cast<u32>(Param(system, 1)), + static_cast<u32>(Param(system, 2)), static_cast<u32>(Param(system, 3))) + .raw; + FuncReturn(system, retval); +} + +// Used by WaitProcessWideKeyAtomic32 +template <ResultCode func(Core::System&, u32, u32, Handle, u32, u32)> +void SvcWrap32(Core::System& system) { + const u32 retval = + func(system, static_cast<u32>(Param(system, 0)), static_cast<u32>(Param(system, 1)), + static_cast<Handle>(Param(system, 2)), static_cast<u32>(Param(system, 3)), + static_cast<u32>(Param(system, 4))) + .raw; + FuncReturn(system, retval); +} + +// Used by WaitForAddress32 +template <ResultCode func(Core::System&, u32, u32, s32, u32, u32)> +void SvcWrap32(Core::System& system) { + const u32 retval = func(system, static_cast<u32>(Param(system, 0)), + static_cast<u32>(Param(system, 1)), static_cast<s32>(Param(system, 2)), + static_cast<u32>(Param(system, 3)), static_cast<u32>(Param(system, 4))) + .raw; + FuncReturn(system, retval); +} + +// Used by SignalToAddress32 +template <ResultCode func(Core::System&, u32, u32, s32, s32)> +void SvcWrap32(Core::System& system) { + const u32 retval = + func(system, static_cast<u32>(Param(system, 0)), static_cast<u32>(Param(system, 1)), + static_cast<s32>(Param(system, 2)), static_cast<s32>(Param(system, 3))) + .raw; + FuncReturn(system, retval); +} + +// Used by SendSyncRequest32, ArbitrateUnlock32 template <ResultCode func(Core::System&, u32)> void SvcWrap32(Core::System& system) { FuncReturn(system, func(system, static_cast<u32>(Param(system, 0))).raw); } +// Used by CreateTransferMemory32 +template <ResultCode func(Core::System&, Handle*, u32, u32, u32)> +void SvcWrap32(Core::System& system) { + Handle handle = 0; + const u32 retval = + func(system, &handle, Param32(system, 1), Param32(system, 2), Param32(system, 3)).raw; + system.CurrentArmInterface().SetReg(1, handle); + FuncReturn(system, retval); +} + // Used by WaitSynchronization32 template <ResultCode func(Core::System&, u32, u32, s32, u32, Handle*)> void SvcWrap32(Core::System& system) { diff --git a/src/core/hle/kernel/synchronization.cpp b/src/core/hle/kernel/synchronization.cpp index dc37fad1a..8b875d853 100644 --- a/src/core/hle/kernel/synchronization.cpp +++ b/src/core/hle/kernel/synchronization.cpp @@ -10,78 +10,106 @@ #include "core/hle/kernel/synchronization.h" #include "core/hle/kernel/synchronization_object.h" #include "core/hle/kernel/thread.h" +#include "core/hle/kernel/time_manager.h" namespace Kernel { -/// Default thread wakeup callback for WaitSynchronization -static bool DefaultThreadWakeupCallback(ThreadWakeupReason reason, std::shared_ptr<Thread> thread, - std::shared_ptr<SynchronizationObject> object, - std::size_t index) { - ASSERT(thread->GetStatus() == ThreadStatus::WaitSynch); - - if (reason == ThreadWakeupReason::Timeout) { - thread->SetWaitSynchronizationResult(RESULT_TIMEOUT); - return true; - } - - ASSERT(reason == ThreadWakeupReason::Signal); - thread->SetWaitSynchronizationResult(RESULT_SUCCESS); - thread->SetWaitSynchronizationOutput(static_cast<u32>(index)); - return true; -} - Synchronization::Synchronization(Core::System& system) : system{system} {} void Synchronization::SignalObject(SynchronizationObject& obj) const { + auto& kernel = system.Kernel(); + SchedulerLock lock(kernel); if (obj.IsSignaled()) { - obj.WakeupAllWaitingThreads(); + for (auto thread : obj.GetWaitingThreads()) { + if (thread->GetSchedulingStatus() == ThreadSchedStatus::Paused) { + if (thread->GetStatus() != ThreadStatus::WaitHLEEvent) { + ASSERT(thread->GetStatus() == ThreadStatus::WaitSynch); + ASSERT(thread->IsWaitingSync()); + } + thread->SetSynchronizationResults(&obj, RESULT_SUCCESS); + thread->ResumeFromWait(); + } + } + obj.ClearWaitingThreads(); } } std::pair<ResultCode, Handle> Synchronization::WaitFor( std::vector<std::shared_ptr<SynchronizationObject>>& sync_objects, s64 nano_seconds) { + auto& kernel = system.Kernel(); auto* const thread = system.CurrentScheduler().GetCurrentThread(); - // Find the first object that is acquirable in the provided list of objects - const auto itr = std::find_if(sync_objects.begin(), sync_objects.end(), - [thread](const std::shared_ptr<SynchronizationObject>& object) { - return object->IsSignaled(); - }); - - if (itr != sync_objects.end()) { - // We found a ready object, acquire it and set the result value - SynchronizationObject* object = itr->get(); - object->Acquire(thread); - const u32 index = static_cast<s32>(std::distance(sync_objects.begin(), itr)); - return {RESULT_SUCCESS, index}; + Handle event_handle = InvalidHandle; + { + SchedulerLockAndSleep lock(kernel, event_handle, thread, nano_seconds); + const auto itr = + std::find_if(sync_objects.begin(), sync_objects.end(), + [thread](const std::shared_ptr<SynchronizationObject>& object) { + return object->IsSignaled(); + }); + + if (itr != sync_objects.end()) { + // We found a ready object, acquire it and set the result value + SynchronizationObject* object = itr->get(); + object->Acquire(thread); + const u32 index = static_cast<s32>(std::distance(sync_objects.begin(), itr)); + lock.CancelSleep(); + return {RESULT_SUCCESS, index}; + } + + if (nano_seconds == 0) { + lock.CancelSleep(); + return {RESULT_TIMEOUT, InvalidHandle}; + } + + if (thread->IsPendingTermination()) { + lock.CancelSleep(); + return {ERR_THREAD_TERMINATING, InvalidHandle}; + } + + if (thread->IsSyncCancelled()) { + thread->SetSyncCancelled(false); + lock.CancelSleep(); + return {ERR_SYNCHRONIZATION_CANCELED, InvalidHandle}; + } + + for (auto& object : sync_objects) { + object->AddWaitingThread(SharedFrom(thread)); + } + + thread->SetSynchronizationObjects(&sync_objects); + thread->SetSynchronizationResults(nullptr, RESULT_TIMEOUT); + thread->SetStatus(ThreadStatus::WaitSynch); + thread->SetWaitingSync(true); } + thread->SetWaitingSync(false); - // No objects were ready to be acquired, prepare to suspend the thread. - - // If a timeout value of 0 was provided, just return the Timeout error code instead of - // suspending the thread. - if (nano_seconds == 0) { - return {RESULT_TIMEOUT, InvalidHandle}; + if (event_handle != InvalidHandle) { + auto& time_manager = kernel.TimeManager(); + time_manager.UnscheduleTimeEvent(event_handle); } - if (thread->IsSyncCancelled()) { - thread->SetSyncCancelled(false); - return {ERR_SYNCHRONIZATION_CANCELED, InvalidHandle}; + { + SchedulerLock lock(kernel); + ResultCode signaling_result = thread->GetSignalingResult(); + SynchronizationObject* signaling_object = thread->GetSignalingObject(); + thread->SetSynchronizationObjects(nullptr); + auto shared_thread = SharedFrom(thread); + for (auto& obj : sync_objects) { + obj->RemoveWaitingThread(shared_thread); + } + if (signaling_object != nullptr) { + const auto itr = std::find_if( + sync_objects.begin(), sync_objects.end(), + [signaling_object](const std::shared_ptr<SynchronizationObject>& object) { + return object.get() == signaling_object; + }); + ASSERT(itr != sync_objects.end()); + signaling_object->Acquire(thread); + const u32 index = static_cast<s32>(std::distance(sync_objects.begin(), itr)); + return {signaling_result, index}; + } + return {signaling_result, -1}; } - - for (auto& object : sync_objects) { - object->AddWaitingThread(SharedFrom(thread)); - } - - thread->SetSynchronizationObjects(std::move(sync_objects)); - thread->SetStatus(ThreadStatus::WaitSynch); - - // Create an event to wake the thread up after the specified nanosecond delay has passed - thread->WakeAfterDelay(nano_seconds); - thread->SetWakeupCallback(DefaultThreadWakeupCallback); - - system.PrepareReschedule(thread->GetProcessorID()); - - return {RESULT_TIMEOUT, InvalidHandle}; } } // namespace Kernel diff --git a/src/core/hle/kernel/synchronization_object.cpp b/src/core/hle/kernel/synchronization_object.cpp index 43f3eef18..ba4d39157 100644 --- a/src/core/hle/kernel/synchronization_object.cpp +++ b/src/core/hle/kernel/synchronization_object.cpp @@ -38,68 +38,8 @@ void SynchronizationObject::RemoveWaitingThread(std::shared_ptr<Thread> thread) waiting_threads.erase(itr); } -std::shared_ptr<Thread> SynchronizationObject::GetHighestPriorityReadyThread() const { - Thread* candidate = nullptr; - u32 candidate_priority = THREADPRIO_LOWEST + 1; - - for (const auto& thread : waiting_threads) { - const ThreadStatus thread_status = thread->GetStatus(); - - // The list of waiting threads must not contain threads that are not waiting to be awakened. - ASSERT_MSG(thread_status == ThreadStatus::WaitSynch || - thread_status == ThreadStatus::WaitHLEEvent, - "Inconsistent thread statuses in waiting_threads"); - - if (thread->GetPriority() >= candidate_priority) - continue; - - if (ShouldWait(thread.get())) - continue; - - candidate = thread.get(); - candidate_priority = thread->GetPriority(); - } - - return SharedFrom(candidate); -} - -void SynchronizationObject::WakeupWaitingThread(std::shared_ptr<Thread> thread) { - ASSERT(!ShouldWait(thread.get())); - - if (!thread) { - return; - } - - if (thread->IsSleepingOnWait()) { - for (const auto& object : thread->GetSynchronizationObjects()) { - ASSERT(!object->ShouldWait(thread.get())); - object->Acquire(thread.get()); - } - } else { - Acquire(thread.get()); - } - - const std::size_t index = thread->GetSynchronizationObjectIndex(SharedFrom(this)); - - thread->ClearSynchronizationObjects(); - - thread->CancelWakeupTimer(); - - bool resume = true; - if (thread->HasWakeupCallback()) { - resume = thread->InvokeWakeupCallback(ThreadWakeupReason::Signal, thread, SharedFrom(this), - index); - } - if (resume) { - thread->ResumeFromWait(); - kernel.PrepareReschedule(thread->GetProcessorID()); - } -} - -void SynchronizationObject::WakeupAllWaitingThreads() { - while (auto thread = GetHighestPriorityReadyThread()) { - WakeupWaitingThread(thread); - } +void SynchronizationObject::ClearWaitingThreads() { + waiting_threads.clear(); } const std::vector<std::shared_ptr<Thread>>& SynchronizationObject::GetWaitingThreads() const { diff --git a/src/core/hle/kernel/synchronization_object.h b/src/core/hle/kernel/synchronization_object.h index 741c31faf..f89b24204 100644 --- a/src/core/hle/kernel/synchronization_object.h +++ b/src/core/hle/kernel/synchronization_object.h @@ -12,6 +12,7 @@ namespace Kernel { class KernelCore; +class Synchronization; class Thread; /// Class that represents a Kernel object that a thread can be waiting on @@ -49,24 +50,11 @@ public: */ void RemoveWaitingThread(std::shared_ptr<Thread> thread); - /** - * Wake up all threads waiting on this object that can be awoken, in priority order, - * and set the synchronization result and output of the thread. - */ - void WakeupAllWaitingThreads(); - - /** - * Wakes up a single thread waiting on this object. - * @param thread Thread that is waiting on this object to wakeup. - */ - void WakeupWaitingThread(std::shared_ptr<Thread> thread); - - /// Obtains the highest priority thread that is ready to run from this object's waiting list. - std::shared_ptr<Thread> GetHighestPriorityReadyThread() const; - /// Get a const reference to the waiting threads list for debug use const std::vector<std::shared_ptr<Thread>>& GetWaitingThreads() const; + void ClearWaitingThreads(); + protected: bool is_signaled{}; // Tells if this sync object is signalled; diff --git a/src/core/hle/kernel/thread.cpp b/src/core/hle/kernel/thread.cpp index 83e956036..da0cb26b6 100644 --- a/src/core/hle/kernel/thread.cpp +++ b/src/core/hle/kernel/thread.cpp @@ -9,12 +9,12 @@ #include "common/assert.h" #include "common/common_types.h" +#include "common/fiber.h" #include "common/logging/log.h" #include "common/thread_queue_list.h" #include "core/arm/arm_interface.h" #include "core/core.h" -#include "core/core_timing.h" -#include "core/core_timing_util.h" +#include "core/cpu_manager.h" #include "core/hardware_properties.h" #include "core/hle/kernel/errors.h" #include "core/hle/kernel/handle_table.h" @@ -23,9 +23,15 @@ #include "core/hle/kernel/process.h" #include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/thread.h" +#include "core/hle/kernel/time_manager.h" #include "core/hle/result.h" #include "core/memory.h" +#ifdef ARCHITECTURE_x86_64 +#include "core/arm/dynarmic/arm_dynarmic_32.h" +#include "core/arm/dynarmic/arm_dynarmic_64.h" +#endif + namespace Kernel { bool Thread::ShouldWait(const Thread* thread) const { @@ -44,47 +50,28 @@ Thread::Thread(KernelCore& kernel) : SynchronizationObject{kernel} {} Thread::~Thread() = default; void Thread::Stop() { - // Cancel any outstanding wakeup events for this thread - Core::System::GetInstance().CoreTiming().UnscheduleEvent(kernel.ThreadWakeupCallbackEventType(), - global_handle); - kernel.GlobalHandleTable().Close(global_handle); - global_handle = 0; - SetStatus(ThreadStatus::Dead); - Signal(); - - // Clean up any dangling references in objects that this thread was waiting for - for (auto& wait_object : wait_objects) { - wait_object->RemoveWaitingThread(SharedFrom(this)); - } - wait_objects.clear(); - - owner_process->UnregisterThread(this); - - // Mark the TLS slot in the thread's page as free. - owner_process->FreeTLSRegion(tls_address); -} + { + SchedulerLock lock(kernel); + SetStatus(ThreadStatus::Dead); + Signal(); + kernel.GlobalHandleTable().Close(global_handle); -void Thread::WakeAfterDelay(s64 nanoseconds) { - // Don't schedule a wakeup if the thread wants to wait forever - if (nanoseconds == -1) - return; - - // This function might be called from any thread so we have to be cautious and use the - // thread-safe version of ScheduleEvent. - const s64 cycles = Core::Timing::nsToCycles(std::chrono::nanoseconds{nanoseconds}); - Core::System::GetInstance().CoreTiming().ScheduleEvent( - cycles, kernel.ThreadWakeupCallbackEventType(), global_handle); -} + if (owner_process) { + owner_process->UnregisterThread(this); -void Thread::CancelWakeupTimer() { - Core::System::GetInstance().CoreTiming().UnscheduleEvent(kernel.ThreadWakeupCallbackEventType(), - global_handle); + // Mark the TLS slot in the thread's page as free. + owner_process->FreeTLSRegion(tls_address); + } + arm_interface.reset(); + has_exited = true; + } + global_handle = 0; } void Thread::ResumeFromWait() { - ASSERT_MSG(wait_objects.empty(), "Thread is waking up while waiting for objects"); - + SchedulerLock lock(kernel); switch (status) { + case ThreadStatus::Paused: case ThreadStatus::WaitSynch: case ThreadStatus::WaitHLEEvent: case ThreadStatus::WaitSleep: @@ -92,12 +79,13 @@ void Thread::ResumeFromWait() { case ThreadStatus::WaitMutex: case ThreadStatus::WaitCondVar: case ThreadStatus::WaitArb: + case ThreadStatus::Dormant: break; case ThreadStatus::Ready: // The thread's wakeup callback must have already been cleared when the thread was first // awoken. - ASSERT(wakeup_callback == nullptr); + ASSERT(hle_callback == nullptr); // If the thread is waiting on multiple wait objects, it might be awoken more than once // before actually resuming. We can ignore subsequent wakeups if the thread status has // already been set to ThreadStatus::Ready. @@ -113,24 +101,31 @@ void Thread::ResumeFromWait() { return; } - wakeup_callback = nullptr; + SetStatus(ThreadStatus::Ready); +} - if (activity == ThreadActivity::Paused) { - SetStatus(ThreadStatus::Paused); - return; - } +void Thread::OnWakeUp() { + SchedulerLock lock(kernel); + + SetStatus(ThreadStatus::Ready); +} +ResultCode Thread::Start() { + SchedulerLock lock(kernel); SetStatus(ThreadStatus::Ready); + return RESULT_SUCCESS; } void Thread::CancelWait() { - if (GetSchedulingStatus() != ThreadSchedStatus::Paused) { + SchedulerLock lock(kernel); + if (GetSchedulingStatus() != ThreadSchedStatus::Paused || !is_waiting_on_sync) { is_sync_cancelled = true; return; } + // TODO(Blinkhawk): Implement cancel of server session is_sync_cancelled = false; - SetWaitSynchronizationResult(ERR_SYNCHRONIZATION_CANCELED); - ResumeFromWait(); + SetSynchronizationResults(nullptr, ERR_SYNCHRONIZATION_CANCELED); + SetStatus(ThreadStatus::Ready); } static void ResetThreadContext32(Core::ARM_Interface::ThreadContext32& context, u32 stack_top, @@ -148,16 +143,32 @@ static void ResetThreadContext64(Core::ARM_Interface::ThreadContext64& context, context.pc = entry_point; context.sp = stack_top; // TODO(merry): Perform a hardware test to determine the below value. - // AHP = 0, DN = 1, FTZ = 1, RMode = Round towards zero - context.fpcr = 0x03C00000; + context.fpcr = 0; +} + +std::shared_ptr<Common::Fiber>& Thread::GetHostContext() { + return host_context; +} + +ResultVal<std::shared_ptr<Thread>> Thread::Create(Core::System& system, ThreadType type_flags, + std::string name, VAddr entry_point, u32 priority, + u64 arg, s32 processor_id, VAddr stack_top, + Process* owner_process) { + std::function<void(void*)> init_func = Core::CpuManager::GetGuestThreadStartFunc(); + void* init_func_parameter = system.GetCpuManager().GetStartFuncParamater(); + return Create(system, type_flags, name, entry_point, priority, arg, processor_id, stack_top, + owner_process, std::move(init_func), init_func_parameter); } -ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::string name, - VAddr entry_point, u32 priority, u64 arg, - s32 processor_id, VAddr stack_top, - Process& owner_process) { +ResultVal<std::shared_ptr<Thread>> Thread::Create(Core::System& system, ThreadType type_flags, + std::string name, VAddr entry_point, u32 priority, + u64 arg, s32 processor_id, VAddr stack_top, + Process* owner_process, + std::function<void(void*)>&& thread_start_func, + void* thread_start_parameter) { + auto& kernel = system.Kernel(); // Check if priority is in ranged. Lowest priority -> highest priority id. - if (priority > THREADPRIO_LOWEST) { + if (priority > THREADPRIO_LOWEST && ((type_flags & THREADTYPE_IDLE) == 0)) { LOG_ERROR(Kernel_SVC, "Invalid thread priority: {}", priority); return ERR_INVALID_THREAD_PRIORITY; } @@ -167,11 +178,12 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin return ERR_INVALID_PROCESSOR_ID; } - auto& system = Core::System::GetInstance(); - if (!system.Memory().IsValidVirtualAddress(owner_process, entry_point)) { - LOG_ERROR(Kernel_SVC, "(name={}): invalid entry {:016X}", name, entry_point); - // TODO (bunnei): Find the correct error code to use here - return RESULT_UNKNOWN; + if (owner_process) { + if (!system.Memory().IsValidVirtualAddress(*owner_process, entry_point)) { + LOG_ERROR(Kernel_SVC, "(name={}): invalid entry {:016X}", name, entry_point); + // TODO (bunnei): Find the correct error code to use here + return RESULT_UNKNOWN; + } } std::shared_ptr<Thread> thread = std::make_shared<Thread>(kernel); @@ -182,51 +194,72 @@ ResultVal<std::shared_ptr<Thread>> Thread::Create(KernelCore& kernel, std::strin thread->stack_top = stack_top; thread->tpidr_el0 = 0; thread->nominal_priority = thread->current_priority = priority; - thread->last_running_ticks = system.CoreTiming().GetTicks(); + thread->last_running_ticks = 0; thread->processor_id = processor_id; thread->ideal_core = processor_id; thread->affinity_mask = 1ULL << processor_id; - thread->wait_objects.clear(); + thread->wait_objects = nullptr; thread->mutex_wait_address = 0; thread->condvar_wait_address = 0; thread->wait_handle = 0; thread->name = std::move(name); thread->global_handle = kernel.GlobalHandleTable().Create(thread).Unwrap(); - thread->owner_process = &owner_process; - auto& scheduler = kernel.GlobalScheduler(); - scheduler.AddThread(thread); - thread->tls_address = thread->owner_process->CreateTLSRegion(); + thread->owner_process = owner_process; + thread->type = type_flags; + if ((type_flags & THREADTYPE_IDLE) == 0) { + auto& scheduler = kernel.GlobalScheduler(); + scheduler.AddThread(thread); + } + if (owner_process) { + thread->tls_address = thread->owner_process->CreateTLSRegion(); + thread->owner_process->RegisterThread(thread.get()); + } else { + thread->tls_address = 0; + } - thread->owner_process->RegisterThread(thread.get()); + thread->arm_interface.reset(); + if ((type_flags & THREADTYPE_HLE) == 0) { +#ifdef ARCHITECTURE_x86_64 + if (owner_process && !owner_process->Is64BitProcess()) { + thread->arm_interface = std::make_unique<Core::ARM_Dynarmic_32>( + system, kernel.Interrupts(), kernel.IsMulticore(), kernel.GetExclusiveMonitor(), + processor_id); + } else { + thread->arm_interface = std::make_unique<Core::ARM_Dynarmic_64>( + system, kernel.Interrupts(), kernel.IsMulticore(), kernel.GetExclusiveMonitor(), + processor_id); + } +#else +#error Platform not supported yet. +#endif - ResetThreadContext32(thread->context_32, static_cast<u32>(stack_top), - static_cast<u32>(entry_point), static_cast<u32>(arg)); - ResetThreadContext64(thread->context_64, stack_top, entry_point, arg); + ResetThreadContext32(thread->context_32, static_cast<u32>(stack_top), + static_cast<u32>(entry_point), static_cast<u32>(arg)); + ResetThreadContext64(thread->context_64, stack_top, entry_point, arg); + } + thread->host_context = + std::make_shared<Common::Fiber>(std::move(thread_start_func), thread_start_parameter); return MakeResult<std::shared_ptr<Thread>>(std::move(thread)); } void Thread::SetPriority(u32 priority) { + SchedulerLock lock(kernel); ASSERT_MSG(priority <= THREADPRIO_LOWEST && priority >= THREADPRIO_HIGHEST, "Invalid priority value."); nominal_priority = priority; UpdatePriority(); } -void Thread::SetWaitSynchronizationResult(ResultCode result) { - context_32.cpu_registers[0] = result.raw; - context_64.cpu_registers[0] = result.raw; -} - -void Thread::SetWaitSynchronizationOutput(s32 output) { - context_32.cpu_registers[1] = output; - context_64.cpu_registers[1] = output; +void Thread::SetSynchronizationResults(SynchronizationObject* object, ResultCode result) { + signaling_object = object; + signaling_result = result; } s32 Thread::GetSynchronizationObjectIndex(std::shared_ptr<SynchronizationObject> object) const { - ASSERT_MSG(!wait_objects.empty(), "Thread is not waiting for anything"); - const auto match = std::find(wait_objects.rbegin(), wait_objects.rend(), object); - return static_cast<s32>(std::distance(match, wait_objects.rend()) - 1); + ASSERT_MSG(!wait_objects->empty(), "Thread is not waiting for anything"); + const auto match = std::find(wait_objects->rbegin(), wait_objects->rend(), object); + return static_cast<s32>(std::distance(match, wait_objects->rend()) - 1); } VAddr Thread::GetCommandBufferAddress() const { @@ -235,6 +268,14 @@ VAddr Thread::GetCommandBufferAddress() const { return GetTLSAddress() + command_header_offset; } +Core::ARM_Interface& Thread::ArmInterface() { + return *arm_interface; +} + +const Core::ARM_Interface& Thread::ArmInterface() const { + return *arm_interface; +} + void Thread::SetStatus(ThreadStatus new_status) { if (new_status == status) { return; @@ -256,10 +297,6 @@ void Thread::SetStatus(ThreadStatus new_status) { break; } - if (status == ThreadStatus::Running) { - last_running_ticks = Core::System::GetInstance().CoreTiming().GetTicks(); - } - status = new_status; } @@ -340,75 +377,116 @@ void Thread::UpdatePriority() { lock_owner->UpdatePriority(); } -void Thread::ChangeCore(u32 core, u64 mask) { - SetCoreAndAffinityMask(core, mask); -} - bool Thread::AllSynchronizationObjectsReady() const { - return std::none_of(wait_objects.begin(), wait_objects.end(), + return std::none_of(wait_objects->begin(), wait_objects->end(), [this](const std::shared_ptr<SynchronizationObject>& object) { return object->ShouldWait(this); }); } -bool Thread::InvokeWakeupCallback(ThreadWakeupReason reason, std::shared_ptr<Thread> thread, - std::shared_ptr<SynchronizationObject> object, - std::size_t index) { - ASSERT(wakeup_callback); - return wakeup_callback(reason, std::move(thread), std::move(object), index); +bool Thread::InvokeHLECallback(std::shared_ptr<Thread> thread) { + ASSERT(hle_callback); + return hle_callback(std::move(thread)); } -void Thread::SetActivity(ThreadActivity value) { - activity = value; +ResultCode Thread::SetActivity(ThreadActivity value) { + SchedulerLock lock(kernel); + + auto sched_status = GetSchedulingStatus(); + + if (sched_status != ThreadSchedStatus::Runnable && sched_status != ThreadSchedStatus::Paused) { + return ERR_INVALID_STATE; + } + + if (IsPendingTermination()) { + return RESULT_SUCCESS; + } if (value == ThreadActivity::Paused) { - // Set status if not waiting - if (status == ThreadStatus::Ready || status == ThreadStatus::Running) { - SetStatus(ThreadStatus::Paused); - kernel.PrepareReschedule(processor_id); + if ((pausing_state & static_cast<u32>(ThreadSchedFlags::ThreadPauseFlag)) != 0) { + return ERR_INVALID_STATE; + } + AddSchedulingFlag(ThreadSchedFlags::ThreadPauseFlag); + } else { + if ((pausing_state & static_cast<u32>(ThreadSchedFlags::ThreadPauseFlag)) == 0) { + return ERR_INVALID_STATE; } - } else if (status == ThreadStatus::Paused) { - // Ready to reschedule - ResumeFromWait(); + RemoveSchedulingFlag(ThreadSchedFlags::ThreadPauseFlag); } + return RESULT_SUCCESS; } -void Thread::Sleep(s64 nanoseconds) { - // Sleep current thread and check for next thread to schedule - SetStatus(ThreadStatus::WaitSleep); +ResultCode Thread::Sleep(s64 nanoseconds) { + Handle event_handle{}; + { + SchedulerLockAndSleep lock(kernel, event_handle, this, nanoseconds); + SetStatus(ThreadStatus::WaitSleep); + } - // Create an event to wake the thread up after the specified nanosecond delay has passed - WakeAfterDelay(nanoseconds); + if (event_handle != InvalidHandle) { + auto& time_manager = kernel.TimeManager(); + time_manager.UnscheduleTimeEvent(event_handle); + } + return RESULT_SUCCESS; } -bool Thread::YieldSimple() { - auto& scheduler = kernel.GlobalScheduler(); - return scheduler.YieldThread(this); +std::pair<ResultCode, bool> Thread::YieldSimple() { + bool is_redundant = false; + { + SchedulerLock lock(kernel); + is_redundant = kernel.GlobalScheduler().YieldThread(this); + } + return {RESULT_SUCCESS, is_redundant}; +} + +std::pair<ResultCode, bool> Thread::YieldAndBalanceLoad() { + bool is_redundant = false; + { + SchedulerLock lock(kernel); + is_redundant = kernel.GlobalScheduler().YieldThreadAndBalanceLoad(this); + } + return {RESULT_SUCCESS, is_redundant}; +} + +std::pair<ResultCode, bool> Thread::YieldAndWaitForLoadBalancing() { + bool is_redundant = false; + { + SchedulerLock lock(kernel); + is_redundant = kernel.GlobalScheduler().YieldThreadAndWaitForLoadBalancing(this); + } + return {RESULT_SUCCESS, is_redundant}; } -bool Thread::YieldAndBalanceLoad() { - auto& scheduler = kernel.GlobalScheduler(); - return scheduler.YieldThreadAndBalanceLoad(this); +void Thread::AddSchedulingFlag(ThreadSchedFlags flag) { + const u32 old_state = scheduling_state; + pausing_state |= static_cast<u32>(flag); + const u32 base_scheduling = static_cast<u32>(GetSchedulingStatus()); + scheduling_state = base_scheduling | pausing_state; + kernel.GlobalScheduler().AdjustSchedulingOnStatus(this, old_state); } -bool Thread::YieldAndWaitForLoadBalancing() { - auto& scheduler = kernel.GlobalScheduler(); - return scheduler.YieldThreadAndWaitForLoadBalancing(this); +void Thread::RemoveSchedulingFlag(ThreadSchedFlags flag) { + const u32 old_state = scheduling_state; + pausing_state &= ~static_cast<u32>(flag); + const u32 base_scheduling = static_cast<u32>(GetSchedulingStatus()); + scheduling_state = base_scheduling | pausing_state; + kernel.GlobalScheduler().AdjustSchedulingOnStatus(this, old_state); } void Thread::SetSchedulingStatus(ThreadSchedStatus new_status) { - const u32 old_flags = scheduling_state; + const u32 old_state = scheduling_state; scheduling_state = (scheduling_state & static_cast<u32>(ThreadSchedMasks::HighMask)) | static_cast<u32>(new_status); - AdjustSchedulingOnStatus(old_flags); + kernel.GlobalScheduler().AdjustSchedulingOnStatus(this, old_state); } void Thread::SetCurrentPriority(u32 new_priority) { const u32 old_priority = std::exchange(current_priority, new_priority); - AdjustSchedulingOnPriority(old_priority); + kernel.GlobalScheduler().AdjustSchedulingOnPriority(this, old_priority); } ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) { + SchedulerLock lock(kernel); const auto HighestSetCore = [](u64 mask, u32 max_cores) { for (s32 core = static_cast<s32>(max_cores - 1); core >= 0; core--) { if (((mask >> core) & 1) != 0) { @@ -422,6 +500,8 @@ ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) { if (new_core == THREADPROCESSORID_DONT_UPDATE) { new_core = use_override ? ideal_core_override : ideal_core; if ((new_affinity_mask & (1ULL << new_core)) == 0) { + LOG_ERROR(Kernel, "New affinity mask is incorrect! new_core={}, new_affinity_mask={}", + new_core, new_affinity_mask); return ERR_INVALID_COMBINATION; } } @@ -440,118 +520,10 @@ ResultCode Thread::SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask) { processor_id = ideal_core; } } - AdjustSchedulingOnAffinity(old_affinity_mask, old_core); + kernel.GlobalScheduler().AdjustSchedulingOnAffinity(this, old_affinity_mask, old_core); } } return RESULT_SUCCESS; } -void Thread::AdjustSchedulingOnStatus(u32 old_flags) { - if (old_flags == scheduling_state) { - return; - } - - auto& scheduler = kernel.GlobalScheduler(); - if (static_cast<ThreadSchedStatus>(old_flags & static_cast<u32>(ThreadSchedMasks::LowMask)) == - ThreadSchedStatus::Runnable) { - // In this case the thread was running, now it's pausing/exitting - if (processor_id >= 0) { - scheduler.Unschedule(current_priority, static_cast<u32>(processor_id), this); - } - - for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { - if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) { - scheduler.Unsuggest(current_priority, core, this); - } - } - } else if (GetSchedulingStatus() == ThreadSchedStatus::Runnable) { - // The thread is now set to running from being stopped - if (processor_id >= 0) { - scheduler.Schedule(current_priority, static_cast<u32>(processor_id), this); - } - - for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { - if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) { - scheduler.Suggest(current_priority, core, this); - } - } - } - - scheduler.SetReselectionPending(); -} - -void Thread::AdjustSchedulingOnPriority(u32 old_priority) { - if (GetSchedulingStatus() != ThreadSchedStatus::Runnable) { - return; - } - auto& scheduler = kernel.GlobalScheduler(); - if (processor_id >= 0) { - scheduler.Unschedule(old_priority, static_cast<u32>(processor_id), this); - } - - for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { - if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) { - scheduler.Unsuggest(old_priority, core, this); - } - } - - // Add thread to the new priority queues. - Thread* current_thread = GetCurrentThread(); - - if (processor_id >= 0) { - if (current_thread == this) { - scheduler.SchedulePrepend(current_priority, static_cast<u32>(processor_id), this); - } else { - scheduler.Schedule(current_priority, static_cast<u32>(processor_id), this); - } - } - - for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { - if (core != static_cast<u32>(processor_id) && ((affinity_mask >> core) & 1) != 0) { - scheduler.Suggest(current_priority, core, this); - } - } - - scheduler.SetReselectionPending(); -} - -void Thread::AdjustSchedulingOnAffinity(u64 old_affinity_mask, s32 old_core) { - auto& scheduler = kernel.GlobalScheduler(); - if (GetSchedulingStatus() != ThreadSchedStatus::Runnable || - current_priority >= THREADPRIO_COUNT) { - return; - } - - for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { - if (((old_affinity_mask >> core) & 1) != 0) { - if (core == static_cast<u32>(old_core)) { - scheduler.Unschedule(current_priority, core, this); - } else { - scheduler.Unsuggest(current_priority, core, this); - } - } - } - - for (u32 core = 0; core < Core::Hardware::NUM_CPU_CORES; core++) { - if (((affinity_mask >> core) & 1) != 0) { - if (core == static_cast<u32>(processor_id)) { - scheduler.Schedule(current_priority, core, this); - } else { - scheduler.Suggest(current_priority, core, this); - } - } - } - - scheduler.SetReselectionPending(); -} - -//////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * Gets the current thread - */ -Thread* GetCurrentThread() { - return Core::System::GetInstance().CurrentScheduler().GetCurrentThread(); -} - } // namespace Kernel diff --git a/src/core/hle/kernel/thread.h b/src/core/hle/kernel/thread.h index 23fdef8a4..8daf79fac 100644 --- a/src/core/hle/kernel/thread.h +++ b/src/core/hle/kernel/thread.h @@ -6,26 +6,47 @@ #include <functional> #include <string> +#include <utility> #include <vector> #include "common/common_types.h" +#include "common/spin_lock.h" #include "core/arm/arm_interface.h" #include "core/hle/kernel/object.h" #include "core/hle/kernel/synchronization_object.h" #include "core/hle/result.h" +namespace Common { +class Fiber; +} + +namespace Core { +class ARM_Interface; +class System; +} // namespace Core + namespace Kernel { +class GlobalScheduler; class KernelCore; class Process; class Scheduler; enum ThreadPriority : u32 { - THREADPRIO_HIGHEST = 0, ///< Highest thread priority - THREADPRIO_USERLAND_MAX = 24, ///< Highest thread priority for userland apps - THREADPRIO_DEFAULT = 44, ///< Default thread priority for userland apps - THREADPRIO_LOWEST = 63, ///< Lowest thread priority - THREADPRIO_COUNT = 64, ///< Total number of possible thread priorities. + THREADPRIO_HIGHEST = 0, ///< Highest thread priority + THREADPRIO_MAX_CORE_MIGRATION = 2, ///< Highest priority for a core migration + THREADPRIO_USERLAND_MAX = 24, ///< Highest thread priority for userland apps + THREADPRIO_DEFAULT = 44, ///< Default thread priority for userland apps + THREADPRIO_LOWEST = 63, ///< Lowest thread priority + THREADPRIO_COUNT = 64, ///< Total number of possible thread priorities. +}; + +enum ThreadType : u32 { + THREADTYPE_USER = 0x1, + THREADTYPE_KERNEL = 0x2, + THREADTYPE_HLE = 0x4, + THREADTYPE_IDLE = 0x8, + THREADTYPE_SUSPEND = 0x10, }; enum ThreadProcessorId : s32 { @@ -107,26 +128,45 @@ public: using ThreadSynchronizationObjects = std::vector<std::shared_ptr<SynchronizationObject>>; - using WakeupCallback = - std::function<bool(ThreadWakeupReason reason, std::shared_ptr<Thread> thread, - std::shared_ptr<SynchronizationObject> object, std::size_t index)>; + using HLECallback = std::function<bool(std::shared_ptr<Thread> thread)>; + + /** + * Creates and returns a new thread. The new thread is immediately scheduled + * @param system The instance of the whole system + * @param name The friendly name desired for the thread + * @param entry_point The address at which the thread should start execution + * @param priority The thread's priority + * @param arg User data to pass to the thread + * @param processor_id The ID(s) of the processors on which the thread is desired to be run + * @param stack_top The address of the thread's stack top + * @param owner_process The parent process for the thread, if null, it's a kernel thread + * @return A shared pointer to the newly created thread + */ + static ResultVal<std::shared_ptr<Thread>> Create(Core::System& system, ThreadType type_flags, + std::string name, VAddr entry_point, + u32 priority, u64 arg, s32 processor_id, + VAddr stack_top, Process* owner_process); /** * Creates and returns a new thread. The new thread is immediately scheduled - * @param kernel The kernel instance this thread will be created under. + * @param system The instance of the whole system * @param name The friendly name desired for the thread * @param entry_point The address at which the thread should start execution * @param priority The thread's priority * @param arg User data to pass to the thread * @param processor_id The ID(s) of the processors on which the thread is desired to be run * @param stack_top The address of the thread's stack top - * @param owner_process The parent process for the thread + * @param owner_process The parent process for the thread, if null, it's a kernel thread + * @param thread_start_func The function where the host context will start. + * @param thread_start_parameter The parameter which will passed to host context on init * @return A shared pointer to the newly created thread */ - static ResultVal<std::shared_ptr<Thread>> Create(KernelCore& kernel, std::string name, - VAddr entry_point, u32 priority, u64 arg, - s32 processor_id, VAddr stack_top, - Process& owner_process); + static ResultVal<std::shared_ptr<Thread>> Create(Core::System& system, ThreadType type_flags, + std::string name, VAddr entry_point, + u32 priority, u64 arg, s32 processor_id, + VAddr stack_top, Process* owner_process, + std::function<void(void*)>&& thread_start_func, + void* thread_start_parameter); std::string GetName() const override { return name; @@ -181,7 +221,7 @@ public: void UpdatePriority(); /// Changes the core that the thread is running or scheduled to run on. - void ChangeCore(u32 core, u64 mask); + ResultCode SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask); /** * Gets the thread's thread ID @@ -194,6 +234,10 @@ public: /// Resumes a thread from waiting void ResumeFromWait(); + void OnWakeUp(); + + ResultCode Start(); + /// Cancels a waiting operation that this thread may or may not be within. /// /// When the thread is within a waiting state, this will set the thread's @@ -202,26 +246,19 @@ public: /// void CancelWait(); - /** - * Schedules an event to wake up the specified thread after the specified delay - * @param nanoseconds The time this thread will be allowed to sleep for - */ - void WakeAfterDelay(s64 nanoseconds); + void SetSynchronizationResults(SynchronizationObject* object, ResultCode result); - /// Cancel any outstanding wakeup events for this thread - void CancelWakeupTimer(); + Core::ARM_Interface& ArmInterface(); - /** - * Sets the result after the thread awakens (from svcWaitSynchronization) - * @param result Value to set to the returned result - */ - void SetWaitSynchronizationResult(ResultCode result); + const Core::ARM_Interface& ArmInterface() const; - /** - * Sets the output parameter value after the thread awakens (from svcWaitSynchronization) - * @param output Value to set to the output parameter - */ - void SetWaitSynchronizationOutput(s32 output); + SynchronizationObject* GetSignalingObject() const { + return signaling_object; + } + + ResultCode GetSignalingResult() const { + return signaling_result; + } /** * Retrieves the index that this particular object occupies in the list of objects @@ -269,11 +306,6 @@ public: */ VAddr GetCommandBufferAddress() const; - /// Returns whether this thread is waiting on objects from a WaitSynchronization call. - bool IsSleepingOnWait() const { - return status == ThreadStatus::WaitSynch; - } - ThreadContext32& GetContext32() { return context_32; } @@ -290,6 +322,28 @@ public: return context_64; } + bool IsHLEThread() const { + return (type & THREADTYPE_HLE) != 0; + } + + bool IsSuspendThread() const { + return (type & THREADTYPE_SUSPEND) != 0; + } + + bool IsIdleThread() const { + return (type & THREADTYPE_IDLE) != 0; + } + + bool WasRunning() const { + return was_running; + } + + void SetWasRunning(bool value) { + was_running = value; + } + + std::shared_ptr<Common::Fiber>& GetHostContext(); + ThreadStatus GetStatus() const { return status; } @@ -325,18 +379,18 @@ public: } const ThreadSynchronizationObjects& GetSynchronizationObjects() const { - return wait_objects; + return *wait_objects; } - void SetSynchronizationObjects(ThreadSynchronizationObjects objects) { - wait_objects = std::move(objects); + void SetSynchronizationObjects(ThreadSynchronizationObjects* objects) { + wait_objects = objects; } void ClearSynchronizationObjects() { - for (const auto& waiting_object : wait_objects) { + for (const auto& waiting_object : *wait_objects) { waiting_object->RemoveWaitingThread(SharedFrom(this)); } - wait_objects.clear(); + wait_objects->clear(); } /// Determines whether all the objects this thread is waiting on are ready. @@ -386,26 +440,35 @@ public: arb_wait_address = address; } - bool HasWakeupCallback() const { - return wakeup_callback != nullptr; + bool HasHLECallback() const { + return hle_callback != nullptr; } - void SetWakeupCallback(WakeupCallback callback) { - wakeup_callback = std::move(callback); + void SetHLECallback(HLECallback callback) { + hle_callback = std::move(callback); } - void InvalidateWakeupCallback() { - SetWakeupCallback(nullptr); + void SetHLETimeEvent(Handle time_event) { + hle_time_event = time_event; } - /** - * Invokes the thread's wakeup callback. - * - * @pre A valid wakeup callback has been set. Violating this precondition - * will cause an assertion to trigger. - */ - bool InvokeWakeupCallback(ThreadWakeupReason reason, std::shared_ptr<Thread> thread, - std::shared_ptr<SynchronizationObject> object, std::size_t index); + void SetHLESyncObject(SynchronizationObject* object) { + hle_object = object; + } + + Handle GetHLETimeEvent() const { + return hle_time_event; + } + + SynchronizationObject* GetHLESyncObject() const { + return hle_object; + } + + void InvalidateHLECallback() { + SetHLECallback(nullptr); + } + + bool InvokeHLECallback(std::shared_ptr<Thread> thread); u32 GetIdealCore() const { return ideal_core; @@ -415,23 +478,19 @@ public: return affinity_mask; } - ThreadActivity GetActivity() const { - return activity; - } - - void SetActivity(ThreadActivity value); + ResultCode SetActivity(ThreadActivity value); /// Sleeps this thread for the given amount of nanoseconds. - void Sleep(s64 nanoseconds); + ResultCode Sleep(s64 nanoseconds); /// Yields this thread without rebalancing loads. - bool YieldSimple(); + std::pair<ResultCode, bool> YieldSimple(); /// Yields this thread and does a load rebalancing. - bool YieldAndBalanceLoad(); + std::pair<ResultCode, bool> YieldAndBalanceLoad(); /// Yields this thread and if the core is left idle, loads are rebalanced - bool YieldAndWaitForLoadBalancing(); + std::pair<ResultCode, bool> YieldAndWaitForLoadBalancing(); void IncrementYieldCount() { yield_count++; @@ -446,6 +505,10 @@ public: static_cast<u32>(ThreadSchedMasks::LowMask)); } + bool IsRunnable() const { + return scheduling_state == static_cast<u32>(ThreadSchedStatus::Runnable); + } + bool IsRunning() const { return is_running; } @@ -466,17 +529,65 @@ public: return global_handle; } + bool IsWaitingForArbitration() const { + return waiting_for_arbitration; + } + + void WaitForArbitration(bool set) { + waiting_for_arbitration = set; + } + + bool IsWaitingSync() const { + return is_waiting_on_sync; + } + + void SetWaitingSync(bool is_waiting) { + is_waiting_on_sync = is_waiting; + } + + bool IsPendingTermination() const { + return will_be_terminated || GetSchedulingStatus() == ThreadSchedStatus::Exited; + } + + bool IsPaused() const { + return pausing_state != 0; + } + + bool IsContinuousOnSVC() const { + return is_continuous_on_svc; + } + + void SetContinuousOnSVC(bool is_continuous) { + is_continuous_on_svc = is_continuous; + } + + bool IsPhantomMode() const { + return is_phantom_mode; + } + + void SetPhantomMode(bool phantom) { + is_phantom_mode = phantom; + } + + bool HasExited() const { + return has_exited; + } + private: + friend class GlobalScheduler; + friend class Scheduler; + void SetSchedulingStatus(ThreadSchedStatus new_status); - void SetCurrentPriority(u32 new_priority); - ResultCode SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask); + void AddSchedulingFlag(ThreadSchedFlags flag); + void RemoveSchedulingFlag(ThreadSchedFlags flag); - void AdjustSchedulingOnStatus(u32 old_flags); - void AdjustSchedulingOnPriority(u32 old_priority); - void AdjustSchedulingOnAffinity(u64 old_affinity_mask, s32 old_core); + void SetCurrentPriority(u32 new_priority); + Common::SpinLock context_guard{}; ThreadContext32 context_32{}; ThreadContext64 context_64{}; + std::unique_ptr<Core::ARM_Interface> arm_interface{}; + std::shared_ptr<Common::Fiber> host_context{}; u64 thread_id = 0; @@ -485,6 +596,8 @@ private: VAddr entry_point = 0; VAddr stack_top = 0; + ThreadType type; + /// Nominal thread priority, as set by the emulated application. /// The nominal priority is the thread priority without priority /// inheritance taken into account. @@ -509,7 +622,10 @@ private: /// Objects that the thread is waiting on, in the same order as they were /// passed to WaitSynchronization. - ThreadSynchronizationObjects wait_objects; + ThreadSynchronizationObjects* wait_objects; + + SynchronizationObject* signaling_object; + ResultCode signaling_result{RESULT_SUCCESS}; /// List of threads that are waiting for a mutex that is held by this thread. MutexWaitingThreads wait_mutex_threads; @@ -526,36 +642,40 @@ private: /// If waiting for an AddressArbiter, this is the address being waited on. VAddr arb_wait_address{0}; + bool waiting_for_arbitration{}; /// Handle used as userdata to reference this object when inserting into the CoreTiming queue. Handle global_handle = 0; - /// Callback that will be invoked when the thread is resumed from a waiting state. If the thread - /// was waiting via WaitSynchronization then the object will be the last object that became - /// available. In case of a timeout, the object will be nullptr. - WakeupCallback wakeup_callback; + /// Callback for HLE Events + HLECallback hle_callback; + Handle hle_time_event; + SynchronizationObject* hle_object; Scheduler* scheduler = nullptr; u32 ideal_core{0xFFFFFFFF}; u64 affinity_mask{0x1}; - ThreadActivity activity = ThreadActivity::Normal; - s32 ideal_core_override = -1; u64 affinity_mask_override = 0x1; u32 affinity_override_count = 0; u32 scheduling_state = 0; + u32 pausing_state = 0; bool is_running = false; + bool is_waiting_on_sync = false; bool is_sync_cancelled = false; + bool is_continuous_on_svc = false; + + bool will_be_terminated = false; + bool is_phantom_mode = false; + bool has_exited = false; + + bool was_running = false; + std::string name; }; -/** - * Gets the current thread - */ -Thread* GetCurrentThread(); - } // namespace Kernel diff --git a/src/core/hle/kernel/time_manager.cpp b/src/core/hle/kernel/time_manager.cpp index 21b290468..95f2446c9 100644 --- a/src/core/hle/kernel/time_manager.cpp +++ b/src/core/hle/kernel/time_manager.cpp @@ -8,30 +8,38 @@ #include "core/core_timing_util.h" #include "core/hle/kernel/handle_table.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/scheduler.h" #include "core/hle/kernel/thread.h" #include "core/hle/kernel/time_manager.h" namespace Kernel { -TimeManager::TimeManager(Core::System& system) : system{system} { +TimeManager::TimeManager(Core::System& system_) : system{system_} { time_manager_event_type = Core::Timing::CreateEvent( - "Kernel::TimeManagerCallback", [this](u64 thread_handle, [[maybe_unused]] s64 cycles_late) { - Handle proper_handle = static_cast<Handle>(thread_handle); - std::shared_ptr<Thread> thread = - this->system.Kernel().RetrieveThreadFromGlobalHandleTable(proper_handle); - thread->ResumeFromWait(); + "Kernel::TimeManagerCallback", + [this](std::uintptr_t thread_handle, std::chrono::nanoseconds) { + const SchedulerLock lock(system.Kernel()); + const auto proper_handle = static_cast<Handle>(thread_handle); + if (cancelled_events[proper_handle]) { + return; + } + auto thread = this->system.Kernel().RetrieveThreadFromGlobalHandleTable(proper_handle); + thread->OnWakeUp(); }); } void TimeManager::ScheduleTimeEvent(Handle& event_handle, Thread* timetask, s64 nanoseconds) { + event_handle = timetask->GetGlobalHandle(); if (nanoseconds > 0) { ASSERT(timetask); - event_handle = timetask->GetGlobalHandle(); - const s64 cycles = Core::Timing::nsToCycles(std::chrono::nanoseconds{nanoseconds}); - system.CoreTiming().ScheduleEvent(cycles, time_manager_event_type, event_handle); + ASSERT(timetask->GetStatus() != ThreadStatus::Ready); + ASSERT(timetask->GetStatus() != ThreadStatus::WaitMutex); + system.CoreTiming().ScheduleEvent(std::chrono::nanoseconds{nanoseconds}, + time_manager_event_type, event_handle); } else { event_handle = InvalidHandle; } + cancelled_events[event_handle] = false; } void TimeManager::UnscheduleTimeEvent(Handle event_handle) { @@ -39,6 +47,12 @@ void TimeManager::UnscheduleTimeEvent(Handle event_handle) { return; } system.CoreTiming().UnscheduleEvent(time_manager_event_type, event_handle); + cancelled_events[event_handle] = true; +} + +void TimeManager::CancelTimeEvent(Thread* time_task) { + Handle event_handle = time_task->GetGlobalHandle(); + UnscheduleTimeEvent(event_handle); } } // namespace Kernel diff --git a/src/core/hle/kernel/time_manager.h b/src/core/hle/kernel/time_manager.h index eaec486d1..307a18765 100644 --- a/src/core/hle/kernel/time_manager.h +++ b/src/core/hle/kernel/time_manager.h @@ -5,6 +5,7 @@ #pragma once #include <memory> +#include <unordered_map> #include "core/hle/kernel/object.h" @@ -35,9 +36,12 @@ public: /// Unschedule an existing time event void UnscheduleTimeEvent(Handle event_handle); + void CancelTimeEvent(Thread* time_task); + private: Core::System& system; std::shared_ptr<Core::Timing::EventType> time_manager_event_type; + std::unordered_map<Handle, bool> cancelled_events; }; } // namespace Kernel diff --git a/src/core/hle/kernel/transfer_memory.cpp b/src/core/hle/kernel/transfer_memory.cpp index f2d3f8b49..765f408c3 100644 --- a/src/core/hle/kernel/transfer_memory.cpp +++ b/src/core/hle/kernel/transfer_memory.cpp @@ -2,17 +2,16 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. -#include "core/hle/kernel/errors.h" #include "core/hle/kernel/kernel.h" +#include "core/hle/kernel/memory/page_table.h" #include "core/hle/kernel/process.h" -#include "core/hle/kernel/shared_memory.h" #include "core/hle/kernel/transfer_memory.h" #include "core/hle/result.h" #include "core/memory.h" namespace Kernel { -TransferMemory::TransferMemory(KernelCore& kernel, Memory::Memory& memory) +TransferMemory::TransferMemory(KernelCore& kernel, Core::Memory::Memory& memory) : Object{kernel}, memory{memory} {} TransferMemory::~TransferMemory() { @@ -20,14 +19,15 @@ TransferMemory::~TransferMemory() { Reset(); } -std::shared_ptr<TransferMemory> TransferMemory::Create(KernelCore& kernel, Memory::Memory& memory, - VAddr base_address, u64 size, - MemoryPermission permissions) { +std::shared_ptr<TransferMemory> TransferMemory::Create(KernelCore& kernel, + Core::Memory::Memory& memory, + VAddr base_address, std::size_t size, + Memory::MemoryPermission permissions) { std::shared_ptr<TransferMemory> transfer_memory{ std::make_shared<TransferMemory>(kernel, memory)}; transfer_memory->base_address = base_address; - transfer_memory->memory_size = size; + transfer_memory->size = size; transfer_memory->owner_permissions = permissions; transfer_memory->owner_process = kernel.CurrentProcess(); @@ -38,98 +38,12 @@ const u8* TransferMemory::GetPointer() const { return memory.GetPointer(base_address); } -u64 TransferMemory::GetSize() const { - return memory_size; -} - -ResultCode TransferMemory::MapMemory(VAddr address, u64 size, MemoryPermission permissions) { - if (memory_size != size) { - return ERR_INVALID_SIZE; - } - - if (owner_permissions != permissions) { - return ERR_INVALID_STATE; - } - - if (is_mapped) { - return ERR_INVALID_STATE; - } - - backing_block = std::make_shared<PhysicalMemory>(size); - - const auto map_state = owner_permissions == MemoryPermission::None - ? MemoryState::TransferMemoryIsolated - : MemoryState::TransferMemory; - auto& vm_manager = owner_process->VMManager(); - const auto map_result = vm_manager.MapMemoryBlock(address, backing_block, 0, size, map_state); - if (map_result.Failed()) { - return map_result.Code(); - } - - is_mapped = true; - return RESULT_SUCCESS; -} - ResultCode TransferMemory::Reserve() { - auto& vm_manager{owner_process->VMManager()}; - const auto check_range_result{vm_manager.CheckRangeState( - base_address, memory_size, MemoryState::FlagTransfer | MemoryState::FlagMemoryPoolAllocated, - MemoryState::FlagTransfer | MemoryState::FlagMemoryPoolAllocated, VMAPermission::All, - VMAPermission::ReadWrite, MemoryAttribute::Mask, MemoryAttribute::None, - MemoryAttribute::IpcAndDeviceMapped)}; - - if (check_range_result.Failed()) { - return check_range_result.Code(); - } - - auto [state_, permissions_, attribute] = *check_range_result; - - if (const auto result{vm_manager.ReprotectRange( - base_address, memory_size, SharedMemory::ConvertPermissions(owner_permissions))}; - result.IsError()) { - return result; - } - - return vm_manager.SetMemoryAttribute(base_address, memory_size, MemoryAttribute::Mask, - attribute | MemoryAttribute::Locked); + return owner_process->PageTable().ReserveTransferMemory(base_address, size, owner_permissions); } ResultCode TransferMemory::Reset() { - auto& vm_manager{owner_process->VMManager()}; - if (const auto result{vm_manager.CheckRangeState( - base_address, memory_size, - MemoryState::FlagTransfer | MemoryState::FlagMemoryPoolAllocated, - MemoryState::FlagTransfer | MemoryState::FlagMemoryPoolAllocated, VMAPermission::None, - VMAPermission::None, MemoryAttribute::Mask, MemoryAttribute::Locked, - MemoryAttribute::IpcAndDeviceMapped)}; - result.Failed()) { - return result.Code(); - } - - if (const auto result{ - vm_manager.ReprotectRange(base_address, memory_size, VMAPermission::ReadWrite)}; - result.IsError()) { - return result; - } - - return vm_manager.SetMemoryAttribute(base_address, memory_size, MemoryAttribute::Mask, - MemoryAttribute::None); -} - -ResultCode TransferMemory::UnmapMemory(VAddr address, u64 size) { - if (memory_size != size) { - return ERR_INVALID_SIZE; - } - - auto& vm_manager = owner_process->VMManager(); - const auto result = vm_manager.UnmapRange(address, size); - - if (result.IsError()) { - return result; - } - - is_mapped = false; - return RESULT_SUCCESS; + return owner_process->PageTable().ResetTransferMemory(base_address, size); } } // namespace Kernel diff --git a/src/core/hle/kernel/transfer_memory.h b/src/core/hle/kernel/transfer_memory.h index 6e388536a..05e9f7464 100644 --- a/src/core/hle/kernel/transfer_memory.h +++ b/src/core/hle/kernel/transfer_memory.h @@ -6,12 +6,13 @@ #include <memory> +#include "core/hle/kernel/memory/memory_block.h" #include "core/hle/kernel/object.h" #include "core/hle/kernel/physical_memory.h" union ResultCode; -namespace Memory { +namespace Core::Memory { class Memory; } @@ -20,8 +21,6 @@ namespace Kernel { class KernelCore; class Process; -enum class MemoryPermission : u32; - /// Defines the interface for transfer memory objects. /// /// Transfer memory is typically used for the purpose of @@ -30,14 +29,14 @@ enum class MemoryPermission : u32; /// class TransferMemory final : public Object { public: - explicit TransferMemory(KernelCore& kernel, Memory::Memory& memory); + explicit TransferMemory(KernelCore& kernel, Core::Memory::Memory& memory); ~TransferMemory() override; static constexpr HandleType HANDLE_TYPE = HandleType::TransferMemory; - static std::shared_ptr<TransferMemory> Create(KernelCore& kernel, Memory::Memory& memory, - VAddr base_address, u64 size, - MemoryPermission permissions); + static std::shared_ptr<TransferMemory> Create(KernelCore& kernel, Core::Memory::Memory& memory, + VAddr base_address, std::size_t size, + Memory::MemoryPermission permissions); TransferMemory(const TransferMemory&) = delete; TransferMemory& operator=(const TransferMemory&) = delete; @@ -61,29 +60,9 @@ public: const u8* GetPointer() const; /// Gets the size of the memory backing this instance in bytes. - u64 GetSize() const; - - /// Attempts to map transfer memory with the given range and memory permissions. - /// - /// @param address The base address to being mapping memory at. - /// @param size The size of the memory to map, in bytes. - /// @param permissions The memory permissions to check against when mapping memory. - /// - /// @pre The given address, size, and memory permissions must all match - /// the same values that were given when creating the transfer memory - /// instance. - /// - ResultCode MapMemory(VAddr address, u64 size, MemoryPermission permissions); - - /// Unmaps the transfer memory with the given range - /// - /// @param address The base address to begin unmapping memory at. - /// @param size The size of the memory to unmap, in bytes. - /// - /// @pre The given address and size must be the same as the ones used - /// to create the transfer memory instance. - /// - ResultCode UnmapMemory(VAddr address, u64 size); + constexpr std::size_t GetSize() const { + return size; + } /// Reserves the region to be used for the transfer memory, called after the transfer memory is /// created. @@ -94,25 +73,19 @@ public: ResultCode Reset(); private: - /// Memory block backing this instance. - std::shared_ptr<PhysicalMemory> backing_block; - /// The base address for the memory managed by this instance. - VAddr base_address = 0; + VAddr base_address{}; /// Size of the memory, in bytes, that this instance manages. - u64 memory_size = 0; + std::size_t size{}; /// The memory permissions that are applied to this instance. - MemoryPermission owner_permissions{}; + Memory::MemoryPermission owner_permissions{}; /// The process that this transfer memory instance was created under. - Process* owner_process = nullptr; - - /// Whether or not this transfer memory instance has mapped memory. - bool is_mapped = false; + Process* owner_process{}; - Memory::Memory& memory; + Core::Memory::Memory& memory; }; } // namespace Kernel diff --git a/src/core/hle/kernel/vm_manager.cpp b/src/core/hle/kernel/vm_manager.cpp deleted file mode 100644 index 024c22901..000000000 --- a/src/core/hle/kernel/vm_manager.cpp +++ /dev/null @@ -1,1175 +0,0 @@ -// Copyright 2015 Citra Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -#include <algorithm> -#include <cstring> -#include <iterator> -#include <utility> -#include "common/alignment.h" -#include "common/assert.h" -#include "common/logging/log.h" -#include "common/memory_hook.h" -#include "core/core.h" -#include "core/file_sys/program_metadata.h" -#include "core/hle/kernel/errors.h" -#include "core/hle/kernel/process.h" -#include "core/hle/kernel/resource_limit.h" -#include "core/hle/kernel/vm_manager.h" -#include "core/memory.h" - -namespace Kernel { -namespace { -const char* GetMemoryStateName(MemoryState state) { - static constexpr const char* names[] = { - "Unmapped", "Io", - "Normal", "Code", - "CodeData", "Heap", - "Shared", "Unknown1", - "ModuleCode", "ModuleCodeData", - "IpcBuffer0", "Stack", - "ThreadLocal", "TransferMemoryIsolated", - "TransferMemory", "ProcessMemory", - "Inaccessible", "IpcBuffer1", - "IpcBuffer3", "KernelStack", - }; - - return names[ToSvcMemoryState(state)]; -} - -// Checks if a given address range lies within a larger address range. -constexpr bool IsInsideAddressRange(VAddr address, u64 size, VAddr address_range_begin, - VAddr address_range_end) { - const VAddr end_address = address + size - 1; - return address_range_begin <= address && end_address <= address_range_end - 1; -} -} // Anonymous namespace - -bool VirtualMemoryArea::CanBeMergedWith(const VirtualMemoryArea& next) const { - ASSERT(base + size == next.base); - if (permissions != next.permissions || state != next.state || attribute != next.attribute || - type != next.type) { - return false; - } - if ((attribute & MemoryAttribute::DeviceMapped) == MemoryAttribute::DeviceMapped) { - // TODO: Can device mapped memory be merged sanely? - // Not merging it may cause inaccuracies versus hardware when memory layout is queried. - return false; - } - if (type == VMAType::AllocatedMemoryBlock) { - return true; - } - if (type == VMAType::BackingMemory && backing_memory + size != next.backing_memory) { - return false; - } - if (type == VMAType::MMIO && paddr + size != next.paddr) { - return false; - } - return true; -} - -VMManager::VMManager(Core::System& system) : system{system} { - // Default to assuming a 39-bit address space. This way we have a sane - // starting point with executables that don't provide metadata. - Reset(FileSys::ProgramAddressSpaceType::Is39Bit); -} - -VMManager::~VMManager() = default; - -void VMManager::Reset(FileSys::ProgramAddressSpaceType type) { - Clear(); - - InitializeMemoryRegionRanges(type); - - page_table.Resize(address_space_width); - - // Initialize the map with a single free region covering the entire managed space. - VirtualMemoryArea initial_vma; - initial_vma.size = address_space_end; - vma_map.emplace(initial_vma.base, initial_vma); - - UpdatePageTableForVMA(initial_vma); -} - -VMManager::VMAHandle VMManager::FindVMA(VAddr target) const { - if (target >= address_space_end) { - return vma_map.end(); - } else { - return std::prev(vma_map.upper_bound(target)); - } -} - -bool VMManager::IsValidHandle(VMAHandle handle) const { - return handle != vma_map.cend(); -} - -ResultVal<VMManager::VMAHandle> VMManager::MapMemoryBlock(VAddr target, - std::shared_ptr<PhysicalMemory> block, - std::size_t offset, u64 size, - MemoryState state, VMAPermission perm) { - ASSERT(block != nullptr); - ASSERT(offset + size <= block->size()); - - // This is the appropriately sized VMA that will turn into our allocation. - CASCADE_RESULT(VMAIter vma_handle, CarveVMA(target, size)); - VirtualMemoryArea& final_vma = vma_handle->second; - ASSERT(final_vma.size == size); - - final_vma.type = VMAType::AllocatedMemoryBlock; - final_vma.permissions = perm; - final_vma.state = state; - final_vma.backing_block = std::move(block); - final_vma.offset = offset; - UpdatePageTableForVMA(final_vma); - - return MakeResult<VMAHandle>(MergeAdjacent(vma_handle)); -} - -ResultVal<VMManager::VMAHandle> VMManager::MapBackingMemory(VAddr target, u8* memory, u64 size, - MemoryState state) { - ASSERT(memory != nullptr); - - // This is the appropriately sized VMA that will turn into our allocation. - CASCADE_RESULT(VMAIter vma_handle, CarveVMA(target, size)); - VirtualMemoryArea& final_vma = vma_handle->second; - ASSERT(final_vma.size == size); - - final_vma.type = VMAType::BackingMemory; - final_vma.permissions = VMAPermission::ReadWrite; - final_vma.state = state; - final_vma.backing_memory = memory; - UpdatePageTableForVMA(final_vma); - - return MakeResult<VMAHandle>(MergeAdjacent(vma_handle)); -} - -ResultVal<VAddr> VMManager::FindFreeRegion(u64 size) const { - return FindFreeRegion(GetASLRRegionBaseAddress(), GetASLRRegionEndAddress(), size); -} - -ResultVal<VAddr> VMManager::FindFreeRegion(VAddr begin, VAddr end, u64 size) const { - ASSERT(begin < end); - ASSERT(size <= end - begin); - - const VMAHandle vma_handle = - std::find_if(vma_map.begin(), vma_map.end(), [begin, end, size](const auto& vma) { - if (vma.second.type != VMAType::Free) { - return false; - } - const VAddr vma_base = vma.second.base; - const VAddr vma_end = vma_base + vma.second.size; - const VAddr assumed_base = (begin < vma_base) ? vma_base : begin; - const VAddr used_range = assumed_base + size; - - return vma_base <= assumed_base && assumed_base < used_range && used_range < end && - used_range <= vma_end; - }); - - if (vma_handle == vma_map.cend()) { - // TODO(Subv): Find the correct error code here. - return RESULT_UNKNOWN; - } - - const VAddr target = std::max(begin, vma_handle->second.base); - return MakeResult<VAddr>(target); -} - -ResultVal<VMManager::VMAHandle> VMManager::MapMMIO(VAddr target, PAddr paddr, u64 size, - MemoryState state, - Common::MemoryHookPointer mmio_handler) { - // This is the appropriately sized VMA that will turn into our allocation. - CASCADE_RESULT(VMAIter vma_handle, CarveVMA(target, size)); - VirtualMemoryArea& final_vma = vma_handle->second; - ASSERT(final_vma.size == size); - - final_vma.type = VMAType::MMIO; - final_vma.permissions = VMAPermission::ReadWrite; - final_vma.state = state; - final_vma.paddr = paddr; - final_vma.mmio_handler = std::move(mmio_handler); - UpdatePageTableForVMA(final_vma); - - return MakeResult<VMAHandle>(MergeAdjacent(vma_handle)); -} - -VMManager::VMAIter VMManager::Unmap(VMAIter vma_handle) { - VirtualMemoryArea& vma = vma_handle->second; - vma.type = VMAType::Free; - vma.permissions = VMAPermission::None; - vma.state = MemoryState::Unmapped; - vma.attribute = MemoryAttribute::None; - - vma.backing_block = nullptr; - vma.offset = 0; - vma.backing_memory = nullptr; - vma.paddr = 0; - - UpdatePageTableForVMA(vma); - - return MergeAdjacent(vma_handle); -} - -ResultCode VMManager::UnmapRange(VAddr target, u64 size) { - CASCADE_RESULT(VMAIter vma, CarveVMARange(target, size)); - const VAddr target_end = target + size; - - const VMAIter end = vma_map.end(); - // The comparison against the end of the range must be done using addresses since VMAs can be - // merged during this process, causing invalidation of the iterators. - while (vma != end && vma->second.base < target_end) { - vma = std::next(Unmap(vma)); - } - - ASSERT(FindVMA(target)->second.size >= size); - - return RESULT_SUCCESS; -} - -VMManager::VMAHandle VMManager::Reprotect(VMAHandle vma_handle, VMAPermission new_perms) { - VMAIter iter = StripIterConstness(vma_handle); - - VirtualMemoryArea& vma = iter->second; - vma.permissions = new_perms; - UpdatePageTableForVMA(vma); - - return MergeAdjacent(iter); -} - -ResultCode VMManager::ReprotectRange(VAddr target, u64 size, VMAPermission new_perms) { - CASCADE_RESULT(VMAIter vma, CarveVMARange(target, size)); - const VAddr target_end = target + size; - - const VMAIter end = vma_map.end(); - // The comparison against the end of the range must be done using addresses since VMAs can be - // merged during this process, causing invalidation of the iterators. - while (vma != end && vma->second.base < target_end) { - vma = std::next(StripIterConstness(Reprotect(vma, new_perms))); - } - - return RESULT_SUCCESS; -} - -ResultVal<VAddr> VMManager::SetHeapSize(u64 size) { - if (size > GetHeapRegionSize()) { - return ERR_OUT_OF_MEMORY; - } - - // No need to do any additional work if the heap is already the given size. - if (size == GetCurrentHeapSize()) { - return MakeResult(heap_region_base); - } - - if (heap_memory == nullptr) { - // Initialize heap - heap_memory = std::make_shared<PhysicalMemory>(size); - heap_end = heap_region_base + size; - } else { - UnmapRange(heap_region_base, GetCurrentHeapSize()); - } - - // If necessary, expand backing vector to cover new heap extents in - // the case of allocating. Otherwise, shrink the backing memory, - // if a smaller heap has been requested. - heap_memory->resize(size); - heap_memory->shrink_to_fit(); - RefreshMemoryBlockMappings(heap_memory.get()); - - heap_end = heap_region_base + size; - ASSERT(GetCurrentHeapSize() == heap_memory->size()); - - const auto mapping_result = - MapMemoryBlock(heap_region_base, heap_memory, 0, size, MemoryState::Heap); - if (mapping_result.Failed()) { - return mapping_result.Code(); - } - - return MakeResult<VAddr>(heap_region_base); -} - -ResultCode VMManager::MapPhysicalMemory(VAddr target, u64 size) { - // Check how much memory we've already mapped. - const auto mapped_size_result = SizeOfAllocatedVMAsInRange(target, size); - if (mapped_size_result.Failed()) { - return mapped_size_result.Code(); - } - - // If we've already mapped the desired amount, return early. - const std::size_t mapped_size = *mapped_size_result; - if (mapped_size == size) { - return RESULT_SUCCESS; - } - - // Check that we can map the memory we want. - const auto res_limit = system.CurrentProcess()->GetResourceLimit(); - const u64 physmem_remaining = res_limit->GetMaxResourceValue(ResourceType::PhysicalMemory) - - res_limit->GetCurrentResourceValue(ResourceType::PhysicalMemory); - if (physmem_remaining < (size - mapped_size)) { - return ERR_RESOURCE_LIMIT_EXCEEDED; - } - - // Keep track of the memory regions we unmap. - std::vector<std::pair<u64, u64>> mapped_regions; - ResultCode result = RESULT_SUCCESS; - - // Iterate, trying to map memory. - { - const auto end_addr = target + size; - const auto last_addr = end_addr - 1; - VAddr cur_addr = target; - - auto iter = FindVMA(target); - ASSERT(iter != vma_map.end()); - - while (true) { - const auto& vma = iter->second; - const auto vma_start = vma.base; - const auto vma_end = vma_start + vma.size; - const auto vma_last = vma_end - 1; - - // Map the memory block - const auto map_size = std::min(end_addr - cur_addr, vma_end - cur_addr); - if (vma.state == MemoryState::Unmapped) { - const auto map_res = - MapMemoryBlock(cur_addr, std::make_shared<PhysicalMemory>(map_size), 0, - map_size, MemoryState::Heap, VMAPermission::ReadWrite); - result = map_res.Code(); - if (result.IsError()) { - break; - } - - mapped_regions.emplace_back(cur_addr, map_size); - } - - // Break once we hit the end of the range. - if (last_addr <= vma_last) { - break; - } - - // Advance to the next block. - cur_addr = vma_end; - iter = FindVMA(cur_addr); - ASSERT(iter != vma_map.end()); - } - } - - // If we failed, unmap memory. - if (result.IsError()) { - for (const auto [unmap_address, unmap_size] : mapped_regions) { - ASSERT_MSG(UnmapRange(unmap_address, unmap_size).IsSuccess(), - "Failed to unmap memory range."); - } - - return result; - } - - // Update amount of mapped physical memory. - physical_memory_mapped += size - mapped_size; - - return RESULT_SUCCESS; -} - -ResultCode VMManager::UnmapPhysicalMemory(VAddr target, u64 size) { - // Check how much memory is currently mapped. - const auto mapped_size_result = SizeOfUnmappablePhysicalMemoryInRange(target, size); - if (mapped_size_result.Failed()) { - return mapped_size_result.Code(); - } - - // If we've already unmapped all the memory, return early. - const std::size_t mapped_size = *mapped_size_result; - if (mapped_size == 0) { - return RESULT_SUCCESS; - } - - // Keep track of the memory regions we unmap. - std::vector<std::pair<u64, u64>> unmapped_regions; - ResultCode result = RESULT_SUCCESS; - - // Try to unmap regions. - { - const auto end_addr = target + size; - const auto last_addr = end_addr - 1; - VAddr cur_addr = target; - - auto iter = FindVMA(target); - ASSERT(iter != vma_map.end()); - - while (true) { - const auto& vma = iter->second; - const auto vma_start = vma.base; - const auto vma_end = vma_start + vma.size; - const auto vma_last = vma_end - 1; - - // Unmap the memory block - const auto unmap_size = std::min(end_addr - cur_addr, vma_end - cur_addr); - if (vma.state == MemoryState::Heap) { - result = UnmapRange(cur_addr, unmap_size); - if (result.IsError()) { - break; - } - - unmapped_regions.emplace_back(cur_addr, unmap_size); - } - - // Break once we hit the end of the range. - if (last_addr <= vma_last) { - break; - } - - // Advance to the next block. - cur_addr = vma_end; - iter = FindVMA(cur_addr); - ASSERT(iter != vma_map.end()); - } - } - - // If we failed, re-map regions. - // TODO: Preserve memory contents? - if (result.IsError()) { - for (const auto [map_address, map_size] : unmapped_regions) { - const auto remap_res = - MapMemoryBlock(map_address, std::make_shared<PhysicalMemory>(map_size), 0, map_size, - MemoryState::Heap, VMAPermission::None); - ASSERT_MSG(remap_res.Succeeded(), "Failed to remap a memory block."); - } - - return result; - } - - // Update mapped amount - physical_memory_mapped -= mapped_size; - - return RESULT_SUCCESS; -} - -ResultCode VMManager::MapCodeMemory(VAddr dst_address, VAddr src_address, u64 size) { - constexpr auto ignore_attribute = MemoryAttribute::LockedForIPC | MemoryAttribute::DeviceMapped; - const auto src_check_result = CheckRangeState( - src_address, size, MemoryState::All, MemoryState::Heap, VMAPermission::All, - VMAPermission::ReadWrite, MemoryAttribute::Mask, MemoryAttribute::None, ignore_attribute); - - if (src_check_result.Failed()) { - return src_check_result.Code(); - } - - const auto mirror_result = - MirrorMemory(dst_address, src_address, size, MemoryState::ModuleCode); - if (mirror_result.IsError()) { - return mirror_result; - } - - // Ensure we lock the source memory region. - const auto src_vma_result = CarveVMARange(src_address, size); - if (src_vma_result.Failed()) { - return src_vma_result.Code(); - } - auto src_vma_iter = *src_vma_result; - src_vma_iter->second.attribute = MemoryAttribute::Locked; - Reprotect(src_vma_iter, VMAPermission::Read); - - // The destination memory region is fine as is, however we need to make it read-only. - return ReprotectRange(dst_address, size, VMAPermission::Read); -} - -ResultCode VMManager::UnmapCodeMemory(VAddr dst_address, VAddr src_address, u64 size) { - constexpr auto ignore_attribute = MemoryAttribute::LockedForIPC | MemoryAttribute::DeviceMapped; - const auto src_check_result = CheckRangeState( - src_address, size, MemoryState::All, MemoryState::Heap, VMAPermission::None, - VMAPermission::None, MemoryAttribute::Mask, MemoryAttribute::Locked, ignore_attribute); - - if (src_check_result.Failed()) { - return src_check_result.Code(); - } - - // Yes, the kernel only checks the first page of the region. - const auto dst_check_result = - CheckRangeState(dst_address, Memory::PAGE_SIZE, MemoryState::FlagModule, - MemoryState::FlagModule, VMAPermission::None, VMAPermission::None, - MemoryAttribute::Mask, MemoryAttribute::None, ignore_attribute); - - if (dst_check_result.Failed()) { - return dst_check_result.Code(); - } - - const auto dst_memory_state = std::get<MemoryState>(*dst_check_result); - const auto dst_contiguous_check_result = CheckRangeState( - dst_address, size, MemoryState::All, dst_memory_state, VMAPermission::None, - VMAPermission::None, MemoryAttribute::Mask, MemoryAttribute::None, ignore_attribute); - - if (dst_contiguous_check_result.Failed()) { - return dst_contiguous_check_result.Code(); - } - - const auto unmap_result = UnmapRange(dst_address, size); - if (unmap_result.IsError()) { - return unmap_result; - } - - // With the mirrored portion unmapped, restore the original region's traits. - const auto src_vma_result = CarveVMARange(src_address, size); - if (src_vma_result.Failed()) { - return src_vma_result.Code(); - } - auto src_vma_iter = *src_vma_result; - src_vma_iter->second.state = MemoryState::Heap; - src_vma_iter->second.attribute = MemoryAttribute::None; - Reprotect(src_vma_iter, VMAPermission::ReadWrite); - - if (dst_memory_state == MemoryState::ModuleCode) { - system.InvalidateCpuInstructionCaches(); - } - - return unmap_result; -} - -MemoryInfo VMManager::QueryMemory(VAddr address) const { - const auto vma = FindVMA(address); - MemoryInfo memory_info{}; - - if (IsValidHandle(vma)) { - memory_info.base_address = vma->second.base; - memory_info.attributes = ToSvcMemoryAttribute(vma->second.attribute); - memory_info.permission = static_cast<u32>(vma->second.permissions); - memory_info.size = vma->second.size; - memory_info.state = ToSvcMemoryState(vma->second.state); - } else { - memory_info.base_address = address_space_end; - memory_info.permission = static_cast<u32>(VMAPermission::None); - memory_info.size = 0 - address_space_end; - memory_info.state = static_cast<u32>(MemoryState::Inaccessible); - } - - return memory_info; -} - -ResultCode VMManager::SetMemoryAttribute(VAddr address, u64 size, MemoryAttribute mask, - MemoryAttribute attribute) { - constexpr auto ignore_mask = - MemoryAttribute::Uncached | MemoryAttribute::DeviceMapped | MemoryAttribute::Locked; - constexpr auto attribute_mask = ~ignore_mask; - - const auto result = CheckRangeState( - address, size, MemoryState::FlagUncached, MemoryState::FlagUncached, VMAPermission::None, - VMAPermission::None, attribute_mask, MemoryAttribute::None, ignore_mask); - - if (result.Failed()) { - return result.Code(); - } - - const auto [prev_state, prev_permissions, prev_attributes] = *result; - const auto new_attribute = (prev_attributes & ~mask) | (mask & attribute); - - const auto carve_result = CarveVMARange(address, size); - if (carve_result.Failed()) { - return carve_result.Code(); - } - - auto vma_iter = *carve_result; - vma_iter->second.attribute = new_attribute; - - MergeAdjacent(vma_iter); - return RESULT_SUCCESS; -} - -ResultCode VMManager::MirrorMemory(VAddr dst_addr, VAddr src_addr, u64 size, MemoryState state) { - const auto vma = FindVMA(src_addr); - - ASSERT_MSG(vma != vma_map.end(), "Invalid memory address"); - ASSERT_MSG(vma->second.backing_block, "Backing block doesn't exist for address"); - - // The returned VMA might be a bigger one encompassing the desired address. - const auto vma_offset = src_addr - vma->first; - ASSERT_MSG(vma_offset + size <= vma->second.size, - "Shared memory exceeds bounds of mapped block"); - - const std::shared_ptr<PhysicalMemory>& backing_block = vma->second.backing_block; - const std::size_t backing_block_offset = vma->second.offset + vma_offset; - - CASCADE_RESULT(auto new_vma, - MapMemoryBlock(dst_addr, backing_block, backing_block_offset, size, state)); - // Protect mirror with permissions from old region - Reprotect(new_vma, vma->second.permissions); - // Remove permissions from old region - ReprotectRange(src_addr, size, VMAPermission::None); - - return RESULT_SUCCESS; -} - -void VMManager::RefreshMemoryBlockMappings(const PhysicalMemory* block) { - // If this ever proves to have a noticeable performance impact, allow users of the function to - // specify a specific range of addresses to limit the scan to. - for (const auto& p : vma_map) { - const VirtualMemoryArea& vma = p.second; - if (block == vma.backing_block.get()) { - UpdatePageTableForVMA(vma); - } - } -} - -void VMManager::LogLayout() const { - for (const auto& p : vma_map) { - const VirtualMemoryArea& vma = p.second; - LOG_DEBUG(Kernel, "{:016X} - {:016X} size: {:016X} {}{}{} {}", vma.base, - vma.base + vma.size, vma.size, - (u8)vma.permissions & (u8)VMAPermission::Read ? 'R' : '-', - (u8)vma.permissions & (u8)VMAPermission::Write ? 'W' : '-', - (u8)vma.permissions & (u8)VMAPermission::Execute ? 'X' : '-', - GetMemoryStateName(vma.state)); - } -} - -VMManager::VMAIter VMManager::StripIterConstness(const VMAHandle& iter) { - // This uses a neat C++ trick to convert a const_iterator to a regular iterator, given - // non-const access to its container. - return vma_map.erase(iter, iter); // Erases an empty range of elements -} - -ResultVal<VMManager::VMAIter> VMManager::CarveVMA(VAddr base, u64 size) { - ASSERT_MSG((size & Memory::PAGE_MASK) == 0, "non-page aligned size: 0x{:016X}", size); - ASSERT_MSG((base & Memory::PAGE_MASK) == 0, "non-page aligned base: 0x{:016X}", base); - - VMAIter vma_handle = StripIterConstness(FindVMA(base)); - if (vma_handle == vma_map.end()) { - // Target address is outside the range managed by the kernel - return ERR_INVALID_ADDRESS; - } - - const VirtualMemoryArea& vma = vma_handle->second; - if (vma.type != VMAType::Free) { - // Region is already allocated - return ERR_INVALID_ADDRESS_STATE; - } - - const VAddr start_in_vma = base - vma.base; - const VAddr end_in_vma = start_in_vma + size; - - if (end_in_vma > vma.size) { - // Requested allocation doesn't fit inside VMA - return ERR_INVALID_ADDRESS_STATE; - } - - if (end_in_vma != vma.size) { - // Split VMA at the end of the allocated region - SplitVMA(vma_handle, end_in_vma); - } - if (start_in_vma != 0) { - // Split VMA at the start of the allocated region - vma_handle = SplitVMA(vma_handle, start_in_vma); - } - - return MakeResult<VMAIter>(vma_handle); -} - -ResultVal<VMManager::VMAIter> VMManager::CarveVMARange(VAddr target, u64 size) { - ASSERT_MSG((size & Memory::PAGE_MASK) == 0, "non-page aligned size: 0x{:016X}", size); - ASSERT_MSG((target & Memory::PAGE_MASK) == 0, "non-page aligned base: 0x{:016X}", target); - - const VAddr target_end = target + size; - ASSERT(target_end >= target); - ASSERT(target_end <= address_space_end); - ASSERT(size > 0); - - VMAIter begin_vma = StripIterConstness(FindVMA(target)); - const VMAIter i_end = vma_map.lower_bound(target_end); - if (std::any_of(begin_vma, i_end, - [](const auto& entry) { return entry.second.type == VMAType::Free; })) { - return ERR_INVALID_ADDRESS_STATE; - } - - if (target != begin_vma->second.base) { - begin_vma = SplitVMA(begin_vma, target - begin_vma->second.base); - } - - VMAIter end_vma = StripIterConstness(FindVMA(target_end)); - if (end_vma != vma_map.end() && target_end != end_vma->second.base) { - end_vma = SplitVMA(end_vma, target_end - end_vma->second.base); - } - - return MakeResult<VMAIter>(begin_vma); -} - -VMManager::VMAIter VMManager::SplitVMA(VMAIter vma_handle, u64 offset_in_vma) { - VirtualMemoryArea& old_vma = vma_handle->second; - VirtualMemoryArea new_vma = old_vma; // Make a copy of the VMA - - // For now, don't allow no-op VMA splits (trying to split at a boundary) because it's probably - // a bug. This restriction might be removed later. - ASSERT(offset_in_vma < old_vma.size); - ASSERT(offset_in_vma > 0); - - old_vma.size = offset_in_vma; - new_vma.base += offset_in_vma; - new_vma.size -= offset_in_vma; - - switch (new_vma.type) { - case VMAType::Free: - break; - case VMAType::AllocatedMemoryBlock: - new_vma.offset += offset_in_vma; - break; - case VMAType::BackingMemory: - new_vma.backing_memory += offset_in_vma; - break; - case VMAType::MMIO: - new_vma.paddr += offset_in_vma; - break; - } - - ASSERT(old_vma.CanBeMergedWith(new_vma)); - - return vma_map.emplace_hint(std::next(vma_handle), new_vma.base, new_vma); -} - -VMManager::VMAIter VMManager::MergeAdjacent(VMAIter iter) { - const VMAIter next_vma = std::next(iter); - if (next_vma != vma_map.end() && iter->second.CanBeMergedWith(next_vma->second)) { - MergeAdjacentVMA(iter->second, next_vma->second); - vma_map.erase(next_vma); - } - - if (iter != vma_map.begin()) { - VMAIter prev_vma = std::prev(iter); - if (prev_vma->second.CanBeMergedWith(iter->second)) { - MergeAdjacentVMA(prev_vma->second, iter->second); - vma_map.erase(iter); - iter = prev_vma; - } - } - - return iter; -} - -void VMManager::MergeAdjacentVMA(VirtualMemoryArea& left, const VirtualMemoryArea& right) { - ASSERT(left.CanBeMergedWith(right)); - - // Always merge allocated memory blocks, even when they don't share the same backing block. - if (left.type == VMAType::AllocatedMemoryBlock && - (left.backing_block != right.backing_block || left.offset + left.size != right.offset)) { - - // Check if we can save work. - if (left.offset == 0 && left.size == left.backing_block->size()) { - // Fast case: left is an entire backing block. - left.backing_block->resize(left.size + right.size); - std::memcpy(left.backing_block->data() + left.size, - right.backing_block->data() + right.offset, right.size); - } else { - // Slow case: make a new memory block for left and right. - auto new_memory = std::make_shared<PhysicalMemory>(); - new_memory->resize(left.size + right.size); - std::memcpy(new_memory->data(), left.backing_block->data() + left.offset, left.size); - std::memcpy(new_memory->data() + left.size, right.backing_block->data() + right.offset, - right.size); - - left.backing_block = std::move(new_memory); - left.offset = 0; - } - - // Page table update is needed, because backing memory changed. - left.size += right.size; - UpdatePageTableForVMA(left); - } else { - // Just update the size. - left.size += right.size; - } -} - -void VMManager::UpdatePageTableForVMA(const VirtualMemoryArea& vma) { - auto& memory = system.Memory(); - - switch (vma.type) { - case VMAType::Free: - memory.UnmapRegion(page_table, vma.base, vma.size); - break; - case VMAType::AllocatedMemoryBlock: - memory.MapMemoryRegion(page_table, vma.base, vma.size, *vma.backing_block, vma.offset); - break; - case VMAType::BackingMemory: - memory.MapMemoryRegion(page_table, vma.base, vma.size, vma.backing_memory); - break; - case VMAType::MMIO: - memory.MapIoRegion(page_table, vma.base, vma.size, vma.mmio_handler); - break; - } -} - -void VMManager::InitializeMemoryRegionRanges(FileSys::ProgramAddressSpaceType type) { - u64 map_region_size = 0; - u64 heap_region_size = 0; - u64 stack_region_size = 0; - u64 tls_io_region_size = 0; - - u64 stack_and_tls_io_end = 0; - - switch (type) { - case FileSys::ProgramAddressSpaceType::Is32Bit: - case FileSys::ProgramAddressSpaceType::Is32BitNoMap: - address_space_width = 32; - code_region_base = 0x200000; - code_region_end = code_region_base + 0x3FE00000; - aslr_region_base = 0x200000; - aslr_region_end = aslr_region_base + 0xFFE00000; - if (type == FileSys::ProgramAddressSpaceType::Is32Bit) { - map_region_size = 0x40000000; - heap_region_size = 0x40000000; - } else { - map_region_size = 0; - heap_region_size = 0x80000000; - } - stack_and_tls_io_end = 0x40000000; - break; - case FileSys::ProgramAddressSpaceType::Is36Bit: - address_space_width = 36; - code_region_base = 0x8000000; - code_region_end = code_region_base + 0x78000000; - aslr_region_base = 0x8000000; - aslr_region_end = aslr_region_base + 0xFF8000000; - map_region_size = 0x180000000; - heap_region_size = 0x180000000; - stack_and_tls_io_end = 0x80000000; - break; - case FileSys::ProgramAddressSpaceType::Is39Bit: - address_space_width = 39; - code_region_base = 0x8000000; - code_region_end = code_region_base + 0x80000000; - aslr_region_base = 0x8000000; - aslr_region_end = aslr_region_base + 0x7FF8000000; - map_region_size = 0x1000000000; - heap_region_size = 0x180000000; - stack_region_size = 0x80000000; - tls_io_region_size = 0x1000000000; - break; - default: - UNREACHABLE_MSG("Invalid address space type specified: {}", static_cast<u32>(type)); - return; - } - - const u64 stack_and_tls_io_begin = aslr_region_base; - - address_space_base = 0; - address_space_end = 1ULL << address_space_width; - - map_region_base = code_region_end; - map_region_end = map_region_base + map_region_size; - - heap_region_base = map_region_end; - heap_region_end = heap_region_base + heap_region_size; - heap_end = heap_region_base; - - stack_region_base = heap_region_end; - stack_region_end = stack_region_base + stack_region_size; - - tls_io_region_base = stack_region_end; - tls_io_region_end = tls_io_region_base + tls_io_region_size; - - if (stack_region_size == 0) { - stack_region_base = stack_and_tls_io_begin; - stack_region_end = stack_and_tls_io_end; - } - - if (tls_io_region_size == 0) { - tls_io_region_base = stack_and_tls_io_begin; - tls_io_region_end = stack_and_tls_io_end; - } -} - -void VMManager::Clear() { - ClearVMAMap(); - ClearPageTable(); -} - -void VMManager::ClearVMAMap() { - vma_map.clear(); -} - -void VMManager::ClearPageTable() { - std::fill(page_table.pointers.begin(), page_table.pointers.end(), nullptr); - page_table.special_regions.clear(); - std::fill(page_table.attributes.begin(), page_table.attributes.end(), - Common::PageType::Unmapped); -} - -VMManager::CheckResults VMManager::CheckRangeState(VAddr address, u64 size, MemoryState state_mask, - MemoryState state, VMAPermission permission_mask, - VMAPermission permissions, - MemoryAttribute attribute_mask, - MemoryAttribute attribute, - MemoryAttribute ignore_mask) const { - auto iter = FindVMA(address); - - // If we don't have a valid VMA handle at this point, then it means this is - // being called with an address outside of the address space, which is definitely - // indicative of a bug, as this function only operates on mapped memory regions. - DEBUG_ASSERT(IsValidHandle(iter)); - - const VAddr end_address = address + size - 1; - const MemoryAttribute initial_attributes = iter->second.attribute; - const VMAPermission initial_permissions = iter->second.permissions; - const MemoryState initial_state = iter->second.state; - - while (true) { - // The iterator should be valid throughout the traversal. Hitting the end of - // the mapped VMA regions is unquestionably indicative of a bug. - DEBUG_ASSERT(IsValidHandle(iter)); - - const auto& vma = iter->second; - - if (vma.state != initial_state) { - return ERR_INVALID_ADDRESS_STATE; - } - - if ((vma.state & state_mask) != state) { - return ERR_INVALID_ADDRESS_STATE; - } - - if (vma.permissions != initial_permissions) { - return ERR_INVALID_ADDRESS_STATE; - } - - if ((vma.permissions & permission_mask) != permissions) { - return ERR_INVALID_ADDRESS_STATE; - } - - if ((vma.attribute | ignore_mask) != (initial_attributes | ignore_mask)) { - return ERR_INVALID_ADDRESS_STATE; - } - - if ((vma.attribute & attribute_mask) != attribute) { - return ERR_INVALID_ADDRESS_STATE; - } - - if (end_address <= vma.EndAddress()) { - break; - } - - ++iter; - } - - return MakeResult( - std::make_tuple(initial_state, initial_permissions, initial_attributes & ~ignore_mask)); -} - -ResultVal<std::size_t> VMManager::SizeOfAllocatedVMAsInRange(VAddr address, - std::size_t size) const { - const VAddr end_addr = address + size; - const VAddr last_addr = end_addr - 1; - std::size_t mapped_size = 0; - - VAddr cur_addr = address; - auto iter = FindVMA(cur_addr); - ASSERT(iter != vma_map.end()); - - while (true) { - const auto& vma = iter->second; - const VAddr vma_start = vma.base; - const VAddr vma_end = vma_start + vma.size; - const VAddr vma_last = vma_end - 1; - - // Add size if relevant. - if (vma.state != MemoryState::Unmapped) { - mapped_size += std::min(end_addr - cur_addr, vma_end - cur_addr); - } - - // Break once we hit the end of the range. - if (last_addr <= vma_last) { - break; - } - - // Advance to the next block. - cur_addr = vma_end; - iter = std::next(iter); - ASSERT(iter != vma_map.end()); - } - - return MakeResult(mapped_size); -} - -ResultVal<std::size_t> VMManager::SizeOfUnmappablePhysicalMemoryInRange(VAddr address, - std::size_t size) const { - const VAddr end_addr = address + size; - const VAddr last_addr = end_addr - 1; - std::size_t mapped_size = 0; - - VAddr cur_addr = address; - auto iter = FindVMA(cur_addr); - ASSERT(iter != vma_map.end()); - - while (true) { - const auto& vma = iter->second; - const auto vma_start = vma.base; - const auto vma_end = vma_start + vma.size; - const auto vma_last = vma_end - 1; - const auto state = vma.state; - const auto attr = vma.attribute; - - // Memory within region must be free or mapped heap. - if (!((state == MemoryState::Heap && attr == MemoryAttribute::None) || - (state == MemoryState::Unmapped))) { - return ERR_INVALID_ADDRESS_STATE; - } - - // Add size if relevant. - if (state != MemoryState::Unmapped) { - mapped_size += std::min(end_addr - cur_addr, vma_end - cur_addr); - } - - // Break once we hit the end of the range. - if (last_addr <= vma_last) { - break; - } - - // Advance to the next block. - cur_addr = vma_end; - iter = std::next(iter); - ASSERT(iter != vma_map.end()); - } - - return MakeResult(mapped_size); -} - -u64 VMManager::GetTotalPhysicalMemoryAvailable() const { - LOG_WARNING(Kernel, "(STUBBED) called"); - return 0xF8000000; -} - -VAddr VMManager::GetAddressSpaceBaseAddress() const { - return address_space_base; -} - -VAddr VMManager::GetAddressSpaceEndAddress() const { - return address_space_end; -} - -u64 VMManager::GetAddressSpaceSize() const { - return address_space_end - address_space_base; -} - -u64 VMManager::GetAddressSpaceWidth() const { - return address_space_width; -} - -bool VMManager::IsWithinAddressSpace(VAddr address, u64 size) const { - return IsInsideAddressRange(address, size, GetAddressSpaceBaseAddress(), - GetAddressSpaceEndAddress()); -} - -VAddr VMManager::GetASLRRegionBaseAddress() const { - return aslr_region_base; -} - -VAddr VMManager::GetASLRRegionEndAddress() const { - return aslr_region_end; -} - -u64 VMManager::GetASLRRegionSize() const { - return aslr_region_end - aslr_region_base; -} - -bool VMManager::IsWithinASLRRegion(VAddr begin, u64 size) const { - const VAddr range_end = begin + size; - const VAddr aslr_start = GetASLRRegionBaseAddress(); - const VAddr aslr_end = GetASLRRegionEndAddress(); - - if (aslr_start > begin || begin > range_end || range_end - 1 > aslr_end - 1) { - return false; - } - - if (range_end > heap_region_base && heap_region_end > begin) { - return false; - } - - if (range_end > map_region_base && map_region_end > begin) { - return false; - } - - return true; -} - -VAddr VMManager::GetCodeRegionBaseAddress() const { - return code_region_base; -} - -VAddr VMManager::GetCodeRegionEndAddress() const { - return code_region_end; -} - -u64 VMManager::GetCodeRegionSize() const { - return code_region_end - code_region_base; -} - -bool VMManager::IsWithinCodeRegion(VAddr address, u64 size) const { - return IsInsideAddressRange(address, size, GetCodeRegionBaseAddress(), - GetCodeRegionEndAddress()); -} - -VAddr VMManager::GetHeapRegionBaseAddress() const { - return heap_region_base; -} - -VAddr VMManager::GetHeapRegionEndAddress() const { - return heap_region_end; -} - -u64 VMManager::GetHeapRegionSize() const { - return heap_region_end - heap_region_base; -} - -u64 VMManager::GetCurrentHeapSize() const { - return heap_end - heap_region_base; -} - -bool VMManager::IsWithinHeapRegion(VAddr address, u64 size) const { - return IsInsideAddressRange(address, size, GetHeapRegionBaseAddress(), - GetHeapRegionEndAddress()); -} - -VAddr VMManager::GetMapRegionBaseAddress() const { - return map_region_base; -} - -VAddr VMManager::GetMapRegionEndAddress() const { - return map_region_end; -} - -u64 VMManager::GetMapRegionSize() const { - return map_region_end - map_region_base; -} - -bool VMManager::IsWithinMapRegion(VAddr address, u64 size) const { - return IsInsideAddressRange(address, size, GetMapRegionBaseAddress(), GetMapRegionEndAddress()); -} - -VAddr VMManager::GetStackRegionBaseAddress() const { - return stack_region_base; -} - -VAddr VMManager::GetStackRegionEndAddress() const { - return stack_region_end; -} - -u64 VMManager::GetStackRegionSize() const { - return stack_region_end - stack_region_base; -} - -bool VMManager::IsWithinStackRegion(VAddr address, u64 size) const { - return IsInsideAddressRange(address, size, GetStackRegionBaseAddress(), - GetStackRegionEndAddress()); -} - -VAddr VMManager::GetTLSIORegionBaseAddress() const { - return tls_io_region_base; -} - -VAddr VMManager::GetTLSIORegionEndAddress() const { - return tls_io_region_end; -} - -u64 VMManager::GetTLSIORegionSize() const { - return tls_io_region_end - tls_io_region_base; -} - -bool VMManager::IsWithinTLSIORegion(VAddr address, u64 size) const { - return IsInsideAddressRange(address, size, GetTLSIORegionBaseAddress(), - GetTLSIORegionEndAddress()); -} - -} // namespace Kernel diff --git a/src/core/hle/kernel/vm_manager.h b/src/core/hle/kernel/vm_manager.h deleted file mode 100644 index 90b4b006a..000000000 --- a/src/core/hle/kernel/vm_manager.h +++ /dev/null @@ -1,796 +0,0 @@ -// Copyright 2015 Citra Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -#pragma once - -#include <map> -#include <memory> -#include <tuple> -#include <vector> -#include "common/common_types.h" -#include "common/memory_hook.h" -#include "common/page_table.h" -#include "core/hle/kernel/physical_memory.h" -#include "core/hle/result.h" -#include "core/memory.h" - -namespace Core { -class System; -} - -namespace FileSys { -enum class ProgramAddressSpaceType : u8; -} - -namespace Kernel { - -enum class VMAType : u8 { - /// VMA represents an unmapped region of the address space. - Free, - /// VMA is backed by a ref-counted allocate memory block. - AllocatedMemoryBlock, - /// VMA is backed by a raw, unmanaged pointer. - BackingMemory, - /// VMA is mapped to MMIO registers at a fixed PAddr. - MMIO, - // TODO(yuriks): Implement MemoryAlias to support MAP/UNMAP -}; - -/// Permissions for mapped memory blocks -enum class VMAPermission : u8 { - None = 0, - Read = 1, - Write = 2, - Execute = 4, - - ReadWrite = Read | Write, - ReadExecute = Read | Execute, - WriteExecute = Write | Execute, - ReadWriteExecute = Read | Write | Execute, - - // Used as a wildcard when checking permissions across memory ranges - All = 0xFF, -}; - -constexpr VMAPermission operator|(VMAPermission lhs, VMAPermission rhs) { - return static_cast<VMAPermission>(u32(lhs) | u32(rhs)); -} - -constexpr VMAPermission operator&(VMAPermission lhs, VMAPermission rhs) { - return static_cast<VMAPermission>(u32(lhs) & u32(rhs)); -} - -constexpr VMAPermission operator^(VMAPermission lhs, VMAPermission rhs) { - return static_cast<VMAPermission>(u32(lhs) ^ u32(rhs)); -} - -constexpr VMAPermission operator~(VMAPermission permission) { - return static_cast<VMAPermission>(~u32(permission)); -} - -constexpr VMAPermission& operator|=(VMAPermission& lhs, VMAPermission rhs) { - lhs = lhs | rhs; - return lhs; -} - -constexpr VMAPermission& operator&=(VMAPermission& lhs, VMAPermission rhs) { - lhs = lhs & rhs; - return lhs; -} - -constexpr VMAPermission& operator^=(VMAPermission& lhs, VMAPermission rhs) { - lhs = lhs ^ rhs; - return lhs; -} - -/// Attribute flags that can be applied to a VMA -enum class MemoryAttribute : u32 { - Mask = 0xFF, - - /// No particular qualities - None = 0, - /// Memory locked/borrowed for use. e.g. This would be used by transfer memory. - Locked = 1, - /// Memory locked for use by IPC-related internals. - LockedForIPC = 2, - /// Mapped as part of the device address space. - DeviceMapped = 4, - /// Uncached memory - Uncached = 8, - - IpcAndDeviceMapped = LockedForIPC | DeviceMapped, -}; - -constexpr MemoryAttribute operator|(MemoryAttribute lhs, MemoryAttribute rhs) { - return static_cast<MemoryAttribute>(u32(lhs) | u32(rhs)); -} - -constexpr MemoryAttribute operator&(MemoryAttribute lhs, MemoryAttribute rhs) { - return static_cast<MemoryAttribute>(u32(lhs) & u32(rhs)); -} - -constexpr MemoryAttribute operator^(MemoryAttribute lhs, MemoryAttribute rhs) { - return static_cast<MemoryAttribute>(u32(lhs) ^ u32(rhs)); -} - -constexpr MemoryAttribute operator~(MemoryAttribute attribute) { - return static_cast<MemoryAttribute>(~u32(attribute)); -} - -constexpr MemoryAttribute& operator|=(MemoryAttribute& lhs, MemoryAttribute rhs) { - lhs = lhs | rhs; - return lhs; -} - -constexpr MemoryAttribute& operator&=(MemoryAttribute& lhs, MemoryAttribute rhs) { - lhs = lhs & rhs; - return lhs; -} - -constexpr MemoryAttribute& operator^=(MemoryAttribute& lhs, MemoryAttribute rhs) { - lhs = lhs ^ rhs; - return lhs; -} - -constexpr u32 ToSvcMemoryAttribute(MemoryAttribute attribute) { - return static_cast<u32>(attribute & MemoryAttribute::Mask); -} - -// clang-format off -/// Represents memory states and any relevant flags, as used by the kernel. -/// svcQueryMemory interprets these by masking away all but the first eight -/// bits when storing memory state into a MemoryInfo instance. -enum class MemoryState : u32 { - Mask = 0xFF, - FlagProtect = 1U << 8, - FlagDebug = 1U << 9, - FlagIPC0 = 1U << 10, - FlagIPC3 = 1U << 11, - FlagIPC1 = 1U << 12, - FlagMapped = 1U << 13, - FlagCode = 1U << 14, - FlagAlias = 1U << 15, - FlagModule = 1U << 16, - FlagTransfer = 1U << 17, - FlagQueryPhysicalAddressAllowed = 1U << 18, - FlagSharedDevice = 1U << 19, - FlagSharedDeviceAligned = 1U << 20, - FlagIPCBuffer = 1U << 21, - FlagMemoryPoolAllocated = 1U << 22, - FlagMapProcess = 1U << 23, - FlagUncached = 1U << 24, - FlagCodeMemory = 1U << 25, - - // Wildcard used in range checking to indicate all states. - All = 0xFFFFFFFF, - - // Convenience flag sets to reduce repetition - IPCFlags = FlagIPC0 | FlagIPC3 | FlagIPC1, - - CodeFlags = FlagDebug | IPCFlags | FlagMapped | FlagCode | FlagQueryPhysicalAddressAllowed | - FlagSharedDevice | FlagSharedDeviceAligned | FlagMemoryPoolAllocated, - - DataFlags = FlagProtect | IPCFlags | FlagMapped | FlagAlias | FlagTransfer | - FlagQueryPhysicalAddressAllowed | FlagSharedDevice | FlagSharedDeviceAligned | - FlagMemoryPoolAllocated | FlagIPCBuffer | FlagUncached, - - Unmapped = 0x00, - Io = 0x01 | FlagMapped, - Normal = 0x02 | FlagMapped | FlagQueryPhysicalAddressAllowed, - Code = 0x03 | CodeFlags | FlagMapProcess, - CodeData = 0x04 | DataFlags | FlagMapProcess | FlagCodeMemory, - Heap = 0x05 | DataFlags | FlagCodeMemory, - Shared = 0x06 | FlagMapped | FlagMemoryPoolAllocated, - ModuleCode = 0x08 | CodeFlags | FlagModule | FlagMapProcess, - ModuleCodeData = 0x09 | DataFlags | FlagModule | FlagMapProcess | FlagCodeMemory, - - IpcBuffer0 = 0x0A | FlagMapped | FlagQueryPhysicalAddressAllowed | FlagMemoryPoolAllocated | - IPCFlags | FlagSharedDevice | FlagSharedDeviceAligned, - - Stack = 0x0B | FlagMapped | IPCFlags | FlagQueryPhysicalAddressAllowed | - FlagSharedDevice | FlagSharedDeviceAligned | FlagMemoryPoolAllocated, - - ThreadLocal = 0x0C | FlagMapped | FlagMemoryPoolAllocated, - - TransferMemoryIsolated = 0x0D | IPCFlags | FlagMapped | FlagQueryPhysicalAddressAllowed | - FlagSharedDevice | FlagSharedDeviceAligned | FlagMemoryPoolAllocated | - FlagUncached, - - TransferMemory = 0x0E | FlagIPC3 | FlagIPC1 | FlagMapped | FlagQueryPhysicalAddressAllowed | - FlagSharedDevice | FlagSharedDeviceAligned | FlagMemoryPoolAllocated, - - ProcessMemory = 0x0F | FlagIPC3 | FlagIPC1 | FlagMapped | FlagMemoryPoolAllocated, - - // Used to signify an inaccessible or invalid memory region with memory queries - Inaccessible = 0x10, - - IpcBuffer1 = 0x11 | FlagIPC3 | FlagIPC1 | FlagMapped | FlagQueryPhysicalAddressAllowed | - FlagSharedDevice | FlagSharedDeviceAligned | FlagMemoryPoolAllocated, - - IpcBuffer3 = 0x12 | FlagIPC3 | FlagMapped | FlagQueryPhysicalAddressAllowed | - FlagSharedDeviceAligned | FlagMemoryPoolAllocated, - - KernelStack = 0x13 | FlagMapped, -}; -// clang-format on - -constexpr MemoryState operator|(MemoryState lhs, MemoryState rhs) { - return static_cast<MemoryState>(u32(lhs) | u32(rhs)); -} - -constexpr MemoryState operator&(MemoryState lhs, MemoryState rhs) { - return static_cast<MemoryState>(u32(lhs) & u32(rhs)); -} - -constexpr MemoryState operator^(MemoryState lhs, MemoryState rhs) { - return static_cast<MemoryState>(u32(lhs) ^ u32(rhs)); -} - -constexpr MemoryState operator~(MemoryState lhs) { - return static_cast<MemoryState>(~u32(lhs)); -} - -constexpr MemoryState& operator|=(MemoryState& lhs, MemoryState rhs) { - lhs = lhs | rhs; - return lhs; -} - -constexpr MemoryState& operator&=(MemoryState& lhs, MemoryState rhs) { - lhs = lhs & rhs; - return lhs; -} - -constexpr MemoryState& operator^=(MemoryState& lhs, MemoryState rhs) { - lhs = lhs ^ rhs; - return lhs; -} - -constexpr u32 ToSvcMemoryState(MemoryState state) { - return static_cast<u32>(state & MemoryState::Mask); -} - -struct MemoryInfo { - u64 base_address; - u64 size; - u32 state; - u32 attributes; - u32 permission; - u32 ipc_ref_count; - u32 device_ref_count; -}; -static_assert(sizeof(MemoryInfo) == 0x28, "MemoryInfo has incorrect size."); - -struct PageInfo { - u32 flags; -}; - -/** - * Represents a VMA in an address space. A VMA is a contiguous region of virtual addressing space - * with homogeneous attributes across its extents. In this particular implementation each VMA is - * also backed by a single host memory allocation. - */ -struct VirtualMemoryArea { - /// Gets the starting (base) address of this VMA. - VAddr StartAddress() const { - return base; - } - - /// Gets the ending address of this VMA. - VAddr EndAddress() const { - return base + size - 1; - } - - /// Virtual base address of the region. - VAddr base = 0; - /// Size of the region. - u64 size = 0; - - VMAType type = VMAType::Free; - VMAPermission permissions = VMAPermission::None; - MemoryState state = MemoryState::Unmapped; - MemoryAttribute attribute = MemoryAttribute::None; - - // Settings for type = AllocatedMemoryBlock - /// Memory block backing this VMA. - std::shared_ptr<PhysicalMemory> backing_block = nullptr; - /// Offset into the backing_memory the mapping starts from. - std::size_t offset = 0; - - // Settings for type = BackingMemory - /// Pointer backing this VMA. It will not be destroyed or freed when the VMA is removed. - u8* backing_memory = nullptr; - - // Settings for type = MMIO - /// Physical address of the register area this VMA maps to. - PAddr paddr = 0; - Common::MemoryHookPointer mmio_handler = nullptr; - - /// Tests if this area can be merged to the right with `next`. - bool CanBeMergedWith(const VirtualMemoryArea& next) const; -}; - -/** - * Manages a process' virtual addressing space. This class maintains a list of allocated and free - * regions in the address space, along with their attributes, and allows kernel clients to - * manipulate it, adjusting the page table to match. - * - * This is similar in idea and purpose to the VM manager present in operating system kernels, with - * the main difference being that it doesn't have to support swapping or memory mapping of files. - * The implementation is also simplified by not having to allocate page frames. See these articles - * about the Linux kernel for an explantion of the concept and implementation: - * - http://duartes.org/gustavo/blog/post/how-the-kernel-manages-your-memory/ - * - http://duartes.org/gustavo/blog/post/page-cache-the-affair-between-memory-and-files/ - */ -class VMManager final { - using VMAMap = std::map<VAddr, VirtualMemoryArea>; - -public: - using VMAHandle = VMAMap::const_iterator; - - explicit VMManager(Core::System& system); - ~VMManager(); - - /// Clears the address space map, re-initializing with a single free area. - void Reset(FileSys::ProgramAddressSpaceType type); - - /// Finds the VMA in which the given address is included in, or `vma_map.end()`. - VMAHandle FindVMA(VAddr target) const; - - /// Indicates whether or not the given handle is within the VMA map. - bool IsValidHandle(VMAHandle handle) const; - - // TODO(yuriks): Should these functions actually return the handle? - - /** - * Maps part of a ref-counted block of memory at a given address. - * - * @param target The guest address to start the mapping at. - * @param block The block to be mapped. - * @param offset Offset into `block` to map from. - * @param size Size of the mapping. - * @param state MemoryState tag to attach to the VMA. - */ - ResultVal<VMAHandle> MapMemoryBlock(VAddr target, std::shared_ptr<PhysicalMemory> block, - std::size_t offset, u64 size, MemoryState state, - VMAPermission perm = VMAPermission::ReadWrite); - - /** - * Maps an unmanaged host memory pointer at a given address. - * - * @param target The guest address to start the mapping at. - * @param memory The memory to be mapped. - * @param size Size of the mapping. - * @param state MemoryState tag to attach to the VMA. - */ - ResultVal<VMAHandle> MapBackingMemory(VAddr target, u8* memory, u64 size, MemoryState state); - - /** - * Finds the first free memory region of the given size within - * the user-addressable ASLR memory region. - * - * @param size The size of the desired region in bytes. - * - * @returns If successful, the base address of the free region with - * the given size. - */ - ResultVal<VAddr> FindFreeRegion(u64 size) const; - - /** - * Finds the first free address range that can hold a region of the desired size - * - * @param begin The starting address of the range. - * This is treated as an inclusive beginning address. - * - * @param end The ending address of the range. - * This is treated as an exclusive ending address. - * - * @param size The size of the free region to attempt to locate, - * in bytes. - * - * @returns If successful, the base address of the free region with - * the given size. - * - * @returns If unsuccessful, a result containing an error code. - * - * @pre The starting address must be less than the ending address. - * @pre The size must not exceed the address range itself. - */ - ResultVal<VAddr> FindFreeRegion(VAddr begin, VAddr end, u64 size) const; - - /** - * Maps a memory-mapped IO region at a given address. - * - * @param target The guest address to start the mapping at. - * @param paddr The physical address where the registers are present. - * @param size Size of the mapping. - * @param state MemoryState tag to attach to the VMA. - * @param mmio_handler The handler that will implement read and write for this MMIO region. - */ - ResultVal<VMAHandle> MapMMIO(VAddr target, PAddr paddr, u64 size, MemoryState state, - Common::MemoryHookPointer mmio_handler); - - /// Unmaps a range of addresses, splitting VMAs as necessary. - ResultCode UnmapRange(VAddr target, u64 size); - - /// Changes the permissions of the given VMA. - VMAHandle Reprotect(VMAHandle vma, VMAPermission new_perms); - - /// Changes the permissions of a range of addresses, splitting VMAs as necessary. - ResultCode ReprotectRange(VAddr target, u64 size, VMAPermission new_perms); - - ResultCode MirrorMemory(VAddr dst_addr, VAddr src_addr, u64 size, MemoryState state); - - /// Attempts to allocate a heap with the given size. - /// - /// @param size The size of the heap to allocate in bytes. - /// - /// @note If a heap is currently allocated, and this is called - /// with a size that is equal to the size of the current heap, - /// then this function will do nothing and return the current - /// heap's starting address, as there's no need to perform - /// any additional heap allocation work. - /// - /// @note If a heap is currently allocated, and this is called - /// with a size less than the current heap's size, then - /// this function will attempt to shrink the heap. - /// - /// @note If a heap is currently allocated, and this is called - /// with a size larger than the current heap's size, then - /// this function will attempt to extend the size of the heap. - /// - /// @returns A result indicating either success or failure. - /// <p> - /// If successful, this function will return a result - /// containing the starting address to the allocated heap. - /// <p> - /// If unsuccessful, this function will return a result - /// containing an error code. - /// - /// @pre The given size must lie within the allowable heap - /// memory region managed by this VMManager instance. - /// Failure to abide by this will result in ERR_OUT_OF_MEMORY - /// being returned as the result. - /// - ResultVal<VAddr> SetHeapSize(u64 size); - - /// Maps memory at a given address. - /// - /// @param target The virtual address to map memory at. - /// @param size The amount of memory to map. - /// - /// @note The destination address must lie within the Map region. - /// - /// @note This function requires that SystemResourceSize be non-zero, - /// however, this is just because if it were not then the - /// resulting page tables could be exploited on hardware by - /// a malicious program. SystemResource usage does not need - /// to be explicitly checked or updated here. - ResultCode MapPhysicalMemory(VAddr target, u64 size); - - /// Unmaps memory at a given address. - /// - /// @param target The virtual address to unmap memory at. - /// @param size The amount of memory to unmap. - /// - /// @note The destination address must lie within the Map region. - /// - /// @note This function requires that SystemResourceSize be non-zero, - /// however, this is just because if it were not then the - /// resulting page tables could be exploited on hardware by - /// a malicious program. SystemResource usage does not need - /// to be explicitly checked or updated here. - ResultCode UnmapPhysicalMemory(VAddr target, u64 size); - - /// Maps a region of memory as code memory. - /// - /// @param dst_address The base address of the region to create the aliasing memory region. - /// @param src_address The base address of the region to be aliased. - /// @param size The total amount of memory to map in bytes. - /// - /// @pre Both memory regions lie within the actual addressable address space. - /// - /// @post After this function finishes execution, assuming success, then the address range - /// [dst_address, dst_address+size) will alias the memory region, - /// [src_address, src_address+size). - /// <p> - /// What this also entails is as follows: - /// 1. The aliased region gains the Locked memory attribute. - /// 2. The aliased region becomes read-only. - /// 3. The aliasing region becomes read-only. - /// 4. The aliasing region is created with a memory state of MemoryState::CodeModule. - /// - ResultCode MapCodeMemory(VAddr dst_address, VAddr src_address, u64 size); - - /// Unmaps a region of memory designated as code module memory. - /// - /// @param dst_address The base address of the memory region aliasing the source memory region. - /// @param src_address The base address of the memory region being aliased. - /// @param size The size of the memory region to unmap in bytes. - /// - /// @pre Both memory ranges lie within the actual addressable address space. - /// - /// @pre The memory region being unmapped has been previously been mapped - /// by a call to MapCodeMemory. - /// - /// @post After execution of the function, if successful. the aliasing memory region - /// will be unmapped and the aliased region will have various traits about it - /// restored to what they were prior to the original mapping call preceding - /// this function call. - /// <p> - /// What this also entails is as follows: - /// 1. The state of the memory region will now indicate a general heap region. - /// 2. All memory attributes for the memory region are cleared. - /// 3. Memory permissions for the region are restored to user read/write. - /// - ResultCode UnmapCodeMemory(VAddr dst_address, VAddr src_address, u64 size); - - /// Queries the memory manager for information about the given address. - /// - /// @param address The address to query the memory manager about for information. - /// - /// @return A MemoryInfo instance containing information about the given address. - /// - MemoryInfo QueryMemory(VAddr address) const; - - /// Sets an attribute across the given address range. - /// - /// @param address The starting address - /// @param size The size of the range to set the attribute on. - /// @param mask The attribute mask - /// @param attribute The attribute to set across the given address range - /// - /// @returns RESULT_SUCCESS if successful - /// @returns ERR_INVALID_ADDRESS_STATE if the attribute could not be set. - /// - ResultCode SetMemoryAttribute(VAddr address, u64 size, MemoryAttribute mask, - MemoryAttribute attribute); - - /** - * Scans all VMAs and updates the page table range of any that use the given vector as backing - * memory. This should be called after any operation that causes reallocation of the vector. - */ - void RefreshMemoryBlockMappings(const PhysicalMemory* block); - - /// Dumps the address space layout to the log, for debugging - void LogLayout() const; - - /// Gets the total memory usage, used by svcGetInfo - u64 GetTotalPhysicalMemoryAvailable() const; - - /// Gets the address space base address - VAddr GetAddressSpaceBaseAddress() const; - - /// Gets the address space end address - VAddr GetAddressSpaceEndAddress() const; - - /// Gets the total address space address size in bytes - u64 GetAddressSpaceSize() const; - - /// Gets the address space width in bits. - u64 GetAddressSpaceWidth() const; - - /// Determines whether or not the given address range lies within the address space. - bool IsWithinAddressSpace(VAddr address, u64 size) const; - - /// Gets the base address of the ASLR region. - VAddr GetASLRRegionBaseAddress() const; - - /// Gets the end address of the ASLR region. - VAddr GetASLRRegionEndAddress() const; - - /// Gets the size of the ASLR region - u64 GetASLRRegionSize() const; - - /// Determines whether or not the specified address range is within the ASLR region. - bool IsWithinASLRRegion(VAddr address, u64 size) const; - - /// Gets the base address of the code region. - VAddr GetCodeRegionBaseAddress() const; - - /// Gets the end address of the code region. - VAddr GetCodeRegionEndAddress() const; - - /// Gets the total size of the code region in bytes. - u64 GetCodeRegionSize() const; - - /// Determines whether or not the specified range is within the code region. - bool IsWithinCodeRegion(VAddr address, u64 size) const; - - /// Gets the base address of the heap region. - VAddr GetHeapRegionBaseAddress() const; - - /// Gets the end address of the heap region; - VAddr GetHeapRegionEndAddress() const; - - /// Gets the total size of the heap region in bytes. - u64 GetHeapRegionSize() const; - - /// Gets the total size of the current heap in bytes. - /// - /// @note This is the current allocated heap size, not the size - /// of the region it's allowed to exist within. - /// - u64 GetCurrentHeapSize() const; - - /// Determines whether or not the specified range is within the heap region. - bool IsWithinHeapRegion(VAddr address, u64 size) const; - - /// Gets the base address of the map region. - VAddr GetMapRegionBaseAddress() const; - - /// Gets the end address of the map region. - VAddr GetMapRegionEndAddress() const; - - /// Gets the total size of the map region in bytes. - u64 GetMapRegionSize() const; - - /// Determines whether or not the specified range is within the map region. - bool IsWithinMapRegion(VAddr address, u64 size) const; - - /// Gets the base address of the stack region. - VAddr GetStackRegionBaseAddress() const; - - /// Gets the end address of the stack region. - VAddr GetStackRegionEndAddress() const; - - /// Gets the total size of the stack region in bytes. - u64 GetStackRegionSize() const; - - /// Determines whether or not the given address range is within the stack region - bool IsWithinStackRegion(VAddr address, u64 size) const; - - /// Gets the base address of the TLS IO region. - VAddr GetTLSIORegionBaseAddress() const; - - /// Gets the end address of the TLS IO region. - VAddr GetTLSIORegionEndAddress() const; - - /// Gets the total size of the TLS IO region in bytes. - u64 GetTLSIORegionSize() const; - - /// Determines if the given address range is within the TLS IO region. - bool IsWithinTLSIORegion(VAddr address, u64 size) const; - - /// Each VMManager has its own page table, which is set as the main one when the owning process - /// is scheduled. - Common::PageTable page_table{Memory::PAGE_BITS}; - - using CheckResults = ResultVal<std::tuple<MemoryState, VMAPermission, MemoryAttribute>>; - - /// Checks if an address range adheres to the specified states provided. - /// - /// @param address The starting address of the address range. - /// @param size The size of the address range. - /// @param state_mask The memory state mask. - /// @param state The state to compare the individual VMA states against, - /// which is done in the form of: (vma.state & state_mask) != state. - /// @param permission_mask The memory permissions mask. - /// @param permissions The permission to compare the individual VMA permissions against, - /// which is done in the form of: - /// (vma.permission & permission_mask) != permission. - /// @param attribute_mask The memory attribute mask. - /// @param attribute The memory attributes to compare the individual VMA attributes - /// against, which is done in the form of: - /// (vma.attributes & attribute_mask) != attribute. - /// @param ignore_mask The memory attributes to ignore during the check. - /// - /// @returns If successful, returns a tuple containing the memory attributes - /// (with ignored bits specified by ignore_mask unset), memory permissions, and - /// memory state across the memory range. - /// @returns If not successful, returns ERR_INVALID_ADDRESS_STATE. - /// - CheckResults CheckRangeState(VAddr address, u64 size, MemoryState state_mask, MemoryState state, - VMAPermission permission_mask, VMAPermission permissions, - MemoryAttribute attribute_mask, MemoryAttribute attribute, - MemoryAttribute ignore_mask) const; - -private: - using VMAIter = VMAMap::iterator; - - /// Converts a VMAHandle to a mutable VMAIter. - VMAIter StripIterConstness(const VMAHandle& iter); - - /// Unmaps the given VMA. - VMAIter Unmap(VMAIter vma); - - /** - * Carves a VMA of a specific size at the specified address by splitting Free VMAs while doing - * the appropriate error checking. - */ - ResultVal<VMAIter> CarveVMA(VAddr base, u64 size); - - /** - * Splits the edges of the given range of non-Free VMAs so that there is a VMA split at each - * end of the range. - */ - ResultVal<VMAIter> CarveVMARange(VAddr base, u64 size); - - /** - * Splits a VMA in two, at the specified offset. - * @returns the right side of the split, with the original iterator becoming the left side. - */ - VMAIter SplitVMA(VMAIter vma, u64 offset_in_vma); - - /** - * Checks for and merges the specified VMA with adjacent ones if possible. - * @returns the merged VMA or the original if no merging was possible. - */ - VMAIter MergeAdjacent(VMAIter vma); - - /** - * Merges two adjacent VMAs. - */ - void MergeAdjacentVMA(VirtualMemoryArea& left, const VirtualMemoryArea& right); - - /// Updates the pages corresponding to this VMA so they match the VMA's attributes. - void UpdatePageTableForVMA(const VirtualMemoryArea& vma); - - /// Initializes memory region ranges to adhere to a given address space type. - void InitializeMemoryRegionRanges(FileSys::ProgramAddressSpaceType type); - - /// Clears the underlying map and page table. - void Clear(); - - /// Clears out the VMA map, unmapping any previously mapped ranges. - void ClearVMAMap(); - - /// Clears out the page table - void ClearPageTable(); - - /// Gets the amount of memory currently mapped (state != Unmapped) in a range. - ResultVal<std::size_t> SizeOfAllocatedVMAsInRange(VAddr address, std::size_t size) const; - - /// Gets the amount of memory unmappable by UnmapPhysicalMemory in a range. - ResultVal<std::size_t> SizeOfUnmappablePhysicalMemoryInRange(VAddr address, - std::size_t size) const; - - /** - * A map covering the entirety of the managed address space, keyed by the `base` field of each - * VMA. It must always be modified by splitting or merging VMAs, so that the invariant - * `elem.base + elem.size == next.base` is preserved, and mergeable regions must always be - * merged when possible so that no two similar and adjacent regions exist that have not been - * merged. - */ - VMAMap vma_map; - - u32 address_space_width = 0; - VAddr address_space_base = 0; - VAddr address_space_end = 0; - - VAddr aslr_region_base = 0; - VAddr aslr_region_end = 0; - - VAddr code_region_base = 0; - VAddr code_region_end = 0; - - VAddr heap_region_base = 0; - VAddr heap_region_end = 0; - - VAddr map_region_base = 0; - VAddr map_region_end = 0; - - VAddr stack_region_base = 0; - VAddr stack_region_end = 0; - - VAddr tls_io_region_base = 0; - VAddr tls_io_region_end = 0; - - // Memory used to back the allocations in the regular heap. A single vector is used to cover - // the entire virtual address space extents that bound the allocations, including any holes. - // This makes deallocation and reallocation of holes fast and keeps process memory contiguous - // in the emulator address space, allowing Memory::GetPointer to be reasonably safe. - std::shared_ptr<PhysicalMemory> heap_memory; - - // The end of the currently allocated heap. This is not an inclusive - // end of the range. This is essentially 'base_address + current_size'. - VAddr heap_end = 0; - - // The current amount of memory mapped via MapPhysicalMemory. - // This is used here (and in Nintendo's kernel) only for debugging, and does not impact - // any behavior. - u64 physical_memory_mapped = 0; - - Core::System& system; -}; -} // namespace Kernel |