1 files changed, 274 insertions, 223 deletions

diff --git a/src/core/hle/kernel/thread.h b/src/core/hle/kernel/thread.h
index 11ef29888..6b66c9a0e 100644
--- a/src/core/hle/kernel/thread.h
+++ b/src/core/hle/kernel/thread.h
@@ -6,16 +6,21 @@
 
 #include <array>
 #include <functional>
+#include <span>
 #include <string>
 #include <utility>
 #include <vector>
 
+#include <boost/intrusive/list.hpp>
+
 #include "common/common_types.h"
+#include "common/intrusive_red_black_tree.h"
 #include "common/spin_lock.h"
 #include "core/arm/arm_interface.h"
 #include "core/hle/kernel/k_affinity_mask.h"
+#include "core/hle/kernel/k_synchronization_object.h"
 #include "core/hle/kernel/object.h"
-#include "core/hle/kernel/synchronization_object.h"
+#include "core/hle/kernel/svc_common.h"
 #include "core/hle/result.h"
 
 namespace Common {
@@ -73,19 +78,24 @@ enum ThreadProcessorId : s32 {
                                      (1 << THREADPROCESSORID_2) | (1 << THREADPROCESSORID_3)
 };
 
-enum class ThreadStatus {
-    Ready,        ///< Ready to run
-    Paused,       ///< Paused by SetThreadActivity or debug
-    WaitHLEEvent, ///< Waiting for hle event to finish
-    WaitSleep,    ///< Waiting due to a SleepThread SVC
-    WaitIPC,      ///< Waiting for the reply from an IPC request
-    WaitSynch,    ///< Waiting due to WaitSynchronization
-    WaitMutex,    ///< Waiting due to an ArbitrateLock svc
-    WaitCondVar,  ///< Waiting due to an WaitProcessWideKey svc
-    WaitArb,      ///< Waiting due to a SignalToAddress/WaitForAddress svc
-    Dormant,      ///< Created but not yet made ready
-    Dead          ///< Run to completion, or forcefully terminated
+enum class ThreadState : u16 {
+    Initialized = 0,
+    Waiting = 1,
+    Runnable = 2,
+    Terminated = 3,
+
+    SuspendShift = 4,
+    Mask = (1 << SuspendShift) - 1,
+
+    ProcessSuspended = (1 << (0 + SuspendShift)),
+    ThreadSuspended = (1 << (1 + SuspendShift)),
+    DebugSuspended = (1 << (2 + SuspendShift)),
+    BacktraceSuspended = (1 << (3 + SuspendShift)),
+    InitSuspended = (1 << (4 + SuspendShift)),
+
+    SuspendFlagMask = ((1 << 5) - 1) << SuspendShift,
 };
+DECLARE_ENUM_FLAG_OPERATORS(ThreadState);
 
 enum class ThreadWakeupReason {
     Signal, // The thread was woken up by WakeupAllWaitingThreads due to an object signal.
@@ -97,13 +107,6 @@ enum class ThreadActivity : u32 {
     Paused = 1,
 };
 
-enum class ThreadSchedStatus : u32 {
-    None = 0,
-    Paused = 1,
-    Runnable = 2,
-    Exited = 3,
-};
-
 enum class ThreadSchedFlags : u32 {
     ProcessPauseFlag = 1 << 4,
     ThreadPauseFlag = 1 << 5,
@@ -111,13 +114,20 @@ enum class ThreadSchedFlags : u32 {
     KernelInitPauseFlag = 1 << 8,
 };
 
-enum class ThreadSchedMasks : u32 {
-    LowMask = 0x000f,
-    HighMask = 0xfff0,
-    ForcePauseMask = 0x0070,
+enum class ThreadWaitReasonForDebugging : u32 {
+    None,            ///< Thread is not waiting
+    Sleep,           ///< Thread is waiting due to a SleepThread SVC
+    IPC,             ///< Thread is waiting for the reply from an IPC request
+    Synchronization, ///< Thread is waiting due to a WaitSynchronization SVC
+    ConditionVar,    ///< Thread is waiting due to a WaitProcessWideKey SVC
+    Arbitration,     ///< Thread is waiting due to a SignalToAddress/WaitForAddress SVC
+    Suspended,       ///< Thread is waiting due to process suspension
 };
 
-class Thread final : public SynchronizationObject {
+class Thread final : public KSynchronizationObject, public boost::intrusive::list_base_hook<> {
+    friend class KScheduler;
+    friend class Process;
+
 public:
     explicit Thread(KernelCore& kernel);
     ~Thread() override;
@@ -127,10 +137,6 @@ public:
     using ThreadContext32 = Core::ARM_Interface::ThreadContext32;
     using ThreadContext64 = Core::ARM_Interface::ThreadContext64;
 
-    using ThreadSynchronizationObjects = std::vector<std::shared_ptr<SynchronizationObject>>;
-
-    using HLECallback = std::function<bool(std::shared_ptr<Thread> thread)>;
-
     /**
      * Creates and returns a new thread. The new thread is immediately scheduled
      * @param system The instance of the whole system
@@ -186,59 +192,54 @@ public:
         return HANDLE_TYPE;
     }
 
-    bool ShouldWait(const Thread* thread) const override;
-    void Acquire(Thread* thread) override;
-    bool IsSignaled() const override;
-
     /**
      * Gets the thread's current priority
      * @return The current thread's priority
      */
-    u32 GetPriority() const {
+    [[nodiscard]] s32 GetPriority() const {
         return current_priority;
     }
 
     /**
+     * Sets the thread's current priority.
+     * @param priority The new priority.
+     */
+    void SetPriority(s32 priority) {
+        current_priority = priority;
+    }
+
+    /**
      * Gets the thread's nominal priority.
      * @return The current thread's nominal priority.
      */
-    u32 GetNominalPriority() const {
-        return nominal_priority;
+    [[nodiscard]] s32 GetBasePriority() const {
+        return base_priority;
     }
 
     /**
-     * Sets the thread's current priority
-     * @param priority The new priority
+     * Sets the thread's nominal priority.
+     * @param priority The new priority.
      */
-    void SetPriority(u32 priority);
-
-    /// Adds a thread to the list of threads that are waiting for a lock held by this thread.
-    void AddMutexWaiter(std::shared_ptr<Thread> thread);
-
-    /// Removes a thread from the list of threads that are waiting for a lock held by this thread.
-    void RemoveMutexWaiter(std::shared_ptr<Thread> thread);
-
-    /// Recalculates the current priority taking into account priority inheritance.
-    void UpdatePriority();
+    void SetBasePriority(u32 priority);
 
     /// Changes the core that the thread is running or scheduled to run on.
-    ResultCode SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask);
+    [[nodiscard]] ResultCode SetCoreAndAffinityMask(s32 new_core, u64 new_affinity_mask);
 
     /**
      * Gets the thread's thread ID
      * @return The thread's ID
      */
-    u64 GetThreadID() const {
+    [[nodiscard]] u64 GetThreadID() const {
         return thread_id;
     }
 
     /// Resumes a thread from waiting
-    void ResumeFromWait();
-
-    void OnWakeUp();
+    void Wakeup();
 
     ResultCode Start();
 
+    virtual bool IsSignaled() const override;
+
     /// Cancels a waiting operation that this thread may or may not be within.
     ///
     /// When the thread is within a waiting state, this will set the thread's
@@ -247,29 +248,20 @@ public:
     ///
     void CancelWait();
 
-    void SetSynchronizationResults(SynchronizationObject* object, ResultCode result);
+    void SetSynchronizationResults(KSynchronizationObject* object, ResultCode result);
 
-    SynchronizationObject* GetSignalingObject() const {
-        return signaling_object;
+    void SetSyncedObject(KSynchronizationObject* object, ResultCode result) {
+        SetSynchronizationResults(object, result);
     }
 
-    ResultCode GetSignalingResult() const {
+    ResultCode GetWaitResult(KSynchronizationObject** out) const {
+        *out = signaling_object;
         return signaling_result;
     }
 
-    /**
-     * Retrieves the index that this particular object occupies in the list of objects
-     * that the thread passed to WaitSynchronization, starting the search from the last element.
-     *
-     * It is used to set the output index of WaitSynchronization when the thread is awakened.
-     *
-     * When a thread wakes up due to an object signal, the kernel will use the index of the last
-     * matching object in the wait objects list in case of having multiple instances of the same
-     * object in the list.
-     *
-     * @param object Object to query the index of.
-     */
-    s32 GetSynchronizationObjectIndex(std::shared_ptr<SynchronizationObject> object) const;
+    ResultCode GetSignalingResult() const {
+        return signaling_result;
+    }
 
     /**
      * Stops a thread, invalidating it from further use
@@ -341,18 +333,22 @@ public:
 
     std::shared_ptr<Common::Fiber>& GetHostContext();
 
-    ThreadStatus GetStatus() const {
-        return status;
+    ThreadState GetState() const {
+        return thread_state & ThreadState::Mask;
+    }
+
+    ThreadState GetRawState() const {
+        return thread_state;
     }
 
-    void SetStatus(ThreadStatus new_status);
+    void SetState(ThreadState state);
 
     s64 GetLastScheduledTick() const {
-        return this->last_scheduled_tick;
+        return last_scheduled_tick;
     }
 
     void SetLastScheduledTick(s64 tick) {
-        this->last_scheduled_tick = tick;
+        last_scheduled_tick = tick;
     }
 
     u64 GetTotalCPUTimeTicks() const {
@@ -387,98 +383,18 @@ public:
         return owner_process;
     }
 
-    const ThreadSynchronizationObjects& GetSynchronizationObjects() const {
-        return *wait_objects;
-    }
-
-    void SetSynchronizationObjects(ThreadSynchronizationObjects* objects) {
-        wait_objects = objects;
-    }
-
-    void ClearSynchronizationObjects() {
-        for (const auto& waiting_object : *wait_objects) {
-            waiting_object->RemoveWaitingThread(SharedFrom(this));
-        }
-        wait_objects->clear();
-    }
-
-    /// Determines whether all the objects this thread is waiting on are ready.
-    bool AllSynchronizationObjectsReady() const;
-
     const MutexWaitingThreads& GetMutexWaitingThreads() const {
         return wait_mutex_threads;
     }
 
     Thread* GetLockOwner() const {
-        return lock_owner.get();
-    }
-
-    void SetLockOwner(std::shared_ptr<Thread> owner) {
-        lock_owner = std::move(owner);
-    }
-
-    VAddr GetCondVarWaitAddress() const {
-        return condvar_wait_address;
-    }
-
-    void SetCondVarWaitAddress(VAddr address) {
-        condvar_wait_address = address;
-    }
-
-    VAddr GetMutexWaitAddress() const {
-        return mutex_wait_address;
-    }
-
-    void SetMutexWaitAddress(VAddr address) {
-        mutex_wait_address = address;
-    }
-
-    Handle GetWaitHandle() const {
-        return wait_handle;
-    }
-
-    void SetWaitHandle(Handle handle) {
-        wait_handle = handle;
-    }
-
-    VAddr GetArbiterWaitAddress() const {
-        return arb_wait_address;
-    }
-
-    void SetArbiterWaitAddress(VAddr address) {
-        arb_wait_address = address;
-    }
-
-    bool HasHLECallback() const {
-        return hle_callback != nullptr;
-    }
-
-    void SetHLECallback(HLECallback callback) {
-        hle_callback = std::move(callback);
-    }
-
-    void SetHLETimeEvent(Handle time_event) {
-        hle_time_event = time_event;
-    }
-
-    void SetHLESyncObject(SynchronizationObject* object) {
-        hle_object = object;
-    }
-
-    Handle GetHLETimeEvent() const {
-        return hle_time_event;
-    }
-
-    SynchronizationObject* GetHLESyncObject() const {
-        return hle_object;
+        return lock_owner;
     }
 
-    void InvalidateHLECallback() {
-        SetHLECallback(nullptr);
+    void SetLockOwner(Thread* owner) {
+        lock_owner = owner;
     }
 
-    bool InvokeHLECallback(std::shared_ptr<Thread> thread);
-
     u32 GetIdealCore() const {
         return ideal_core;
     }
@@ -493,20 +409,11 @@ public:
     ResultCode Sleep(s64 nanoseconds);
 
     s64 GetYieldScheduleCount() const {
-        return this->schedule_count;
+        return schedule_count;
     }
 
     void SetYieldScheduleCount(s64 count) {
-        this->schedule_count = count;
-    }
-
-    ThreadSchedStatus GetSchedulingStatus() const {
-        return static_cast<ThreadSchedStatus>(scheduling_state &
-                                              static_cast<u32>(ThreadSchedMasks::LowMask));
-    }
-
-    bool IsRunnable() const {
-        return scheduling_state == static_cast<u32>(ThreadSchedStatus::Runnable);
+        schedule_count = count;
     }
 
     bool IsRunning() const {
@@ -517,36 +424,32 @@ public:
         is_running = value;
     }
 
-    bool IsSyncCancelled() const {
+    bool IsWaitCancelled() const {
         return is_sync_cancelled;
     }
 
-    void SetSyncCancelled(bool value) {
-        is_sync_cancelled = value;
+    void ClearWaitCancelled() {
+        is_sync_cancelled = false;
     }
 
     Handle GetGlobalHandle() const {
         return global_handle;
     }
 
-    bool IsWaitingForArbitration() const {
-        return waiting_for_arbitration;
+    bool IsCancellable() const {
+        return is_cancellable;
     }
 
-    void WaitForArbitration(bool set) {
-        waiting_for_arbitration = set;
+    void SetCancellable() {
+        is_cancellable = true;
     }
 
-    bool IsWaitingSync() const {
-        return is_waiting_on_sync;
+    void ClearCancellable() {
+        is_cancellable = false;
     }
 
-    void SetWaitingSync(bool is_waiting) {
-        is_waiting_on_sync = is_waiting;
-    }
-
-    bool IsPendingTermination() const {
-        return will_be_terminated || GetSchedulingStatus() == ThreadSchedStatus::Exited;
+    bool IsTerminationRequested() const {
+        return will_be_terminated || GetRawState() == ThreadState::Terminated;
     }
 
     bool IsPaused() const {
@@ -578,21 +481,21 @@ public:
         constexpr QueueEntry() = default;
 
         constexpr void Initialize() {
-            this->prev = nullptr;
-            this->next = nullptr;
+            prev = nullptr;
+            next = nullptr;
         }
 
         constexpr Thread* GetPrev() const {
-            return this->prev;
+            return prev;
         }
         constexpr Thread* GetNext() const {
-            return this->next;
+            return next;
         }
         constexpr void SetPrev(Thread* thread) {
-            this->prev = thread;
+            prev = thread;
         }
         constexpr void SetNext(Thread* thread) {
-            this->next = thread;
+            next = thread;
         }
 
     private:
@@ -601,11 +504,11 @@ public:
     };
 
     QueueEntry& GetPriorityQueueEntry(s32 core) {
-        return this->per_core_priority_queue_entry[core];
+        return per_core_priority_queue_entry[core];
     }
 
     const QueueEntry& GetPriorityQueueEntry(s32 core) const {
-        return this->per_core_priority_queue_entry[core];
+        return per_core_priority_queue_entry[core];
     }
 
     s32 GetDisableDispatchCount() const {
@@ -622,24 +525,170 @@ public:
         disable_count--;
     }
 
+    void SetWaitReasonForDebugging(ThreadWaitReasonForDebugging reason) {
+        wait_reason_for_debugging = reason;
+    }
+
+    [[nodiscard]] ThreadWaitReasonForDebugging GetWaitReasonForDebugging() const {
+        return wait_reason_for_debugging;
+    }
+
+    void SetWaitObjectsForDebugging(const std::span<KSynchronizationObject*>& objects) {
+        wait_objects_for_debugging.clear();
+        wait_objects_for_debugging.reserve(objects.size());
+        for (const auto& object : objects) {
+            wait_objects_for_debugging.emplace_back(object);
+        }
+    }
+
+    [[nodiscard]] const std::vector<KSynchronizationObject*>& GetWaitObjectsForDebugging() const {
+        return wait_objects_for_debugging;
+    }
+
+    void SetMutexWaitAddressForDebugging(VAddr address) {
+        mutex_wait_address_for_debugging = address;
+    }
+
+    [[nodiscard]] VAddr GetMutexWaitAddressForDebugging() const {
+        return mutex_wait_address_for_debugging;
+    }
+
+    void AddWaiter(Thread* thread);
+
+    void RemoveWaiter(Thread* thread);
+
+    [[nodiscard]] Thread* RemoveWaiterByKey(s32* out_num_waiters, VAddr key);
+
+    [[nodiscard]] VAddr GetAddressKey() const {
+        return address_key;
+    }
+
+    [[nodiscard]] u32 GetAddressKeyValue() const {
+        return address_key_value;
+    }
+
+    void SetAddressKey(VAddr key) {
+        address_key = key;
+    }
+
+    void SetAddressKey(VAddr key, u32 val) {
+        address_key = key;
+        address_key_value = val;
+    }
+
 private:
-    friend class GlobalSchedulerContext;
-    friend class KScheduler;
-    friend class Process;
+    static constexpr size_t PriorityInheritanceCountMax = 10;
+    union SyncObjectBuffer {
+        std::array<KSynchronizationObject*, Svc::ArgumentHandleCountMax> sync_objects{};
+        std::array<Handle,
+                   Svc::ArgumentHandleCountMax*(sizeof(KSynchronizationObject*) / sizeof(Handle))>
+            handles;
+        constexpr SyncObjectBuffer() {}
+    };
+    static_assert(sizeof(SyncObjectBuffer::sync_objects) == sizeof(SyncObjectBuffer::handles));
+
+    struct ConditionVariableComparator {
+        struct LightCompareType {
+            u64 cv_key{};
+            s32 priority{};
+
+            [[nodiscard]] constexpr u64 GetConditionVariableKey() const {
+                return cv_key;
+            }
+
+            [[nodiscard]] constexpr s32 GetPriority() const {
+                return priority;
+            }
+        };
+
+        template <typename T>
+        requires(
+            std::same_as<T, Thread> ||
+            std::same_as<T, LightCompareType>) static constexpr int Compare(const T& lhs,
+                                                                            const Thread& rhs) {
+            const uintptr_t l_key = lhs.GetConditionVariableKey();
+            const uintptr_t r_key = rhs.GetConditionVariableKey();
+
+            if (l_key < r_key) {
+                // Sort first by key
+                return -1;
+            } else if (l_key == r_key && lhs.GetPriority() < rhs.GetPriority()) {
+                // And then by priority.
+                return -1;
+            } else {
+                return 1;
+            }
+        }
+    };
+
+    Common::IntrusiveRedBlackTreeNode condvar_arbiter_tree_node{};
+
+    using ConditionVariableThreadTreeTraits =
+        Common::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&Thread::condvar_arbiter_tree_node>;
+    using ConditionVariableThreadTree =
+        ConditionVariableThreadTreeTraits::TreeType<ConditionVariableComparator>;
+
+public:
+    using ConditionVariableThreadTreeType = ConditionVariableThreadTree;
+
+    [[nodiscard]] uintptr_t GetConditionVariableKey() const {
+        return condvar_key;
+    }
+
+    [[nodiscard]] uintptr_t GetAddressArbiterKey() const {
+        return condvar_key;
+    }
 
-    void SetSchedulingStatus(ThreadSchedStatus new_status);
+    void SetConditionVariable(ConditionVariableThreadTree* tree, VAddr address, uintptr_t cv_key,
+                              u32 value) {
+        condvar_tree = tree;
+        condvar_key = cv_key;
+        address_key = address;
+        address_key_value = value;
+    }
+
+    void ClearConditionVariable() {
+        condvar_tree = nullptr;
+    }
+
+    [[nodiscard]] bool IsWaitingForConditionVariable() const {
+        return condvar_tree != nullptr;
+    }
+
+    void SetAddressArbiter(ConditionVariableThreadTree* tree, uintptr_t address) {
+        condvar_tree = tree;
+        condvar_key = address;
+    }
+
+    void ClearAddressArbiter() {
+        condvar_tree = nullptr;
+    }
+
+    [[nodiscard]] bool IsWaitingForAddressArbiter() const {
+        return condvar_tree != nullptr;
+    }
+
+    [[nodiscard]] ConditionVariableThreadTree* GetConditionVariableTree() const {
+        return condvar_tree;
+    }
+
+    [[nodiscard]] bool HasWaiters() const {
+        return !waiter_list.empty();
+    }
+
+private:
     void AddSchedulingFlag(ThreadSchedFlags flag);
     void RemoveSchedulingFlag(ThreadSchedFlags flag);
-
-    void SetCurrentPriority(u32 new_priority);
+    void AddWaiterImpl(Thread* thread);
+    void RemoveWaiterImpl(Thread* thread);
+    static void RestorePriority(KernelCore& kernel, Thread* thread);
 
     Common::SpinLock context_guard{};
     ThreadContext32 context_32{};
     ThreadContext64 context_64{};
     std::shared_ptr<Common::Fiber> host_context{};
 
-    ThreadStatus status = ThreadStatus::Dormant;
-    u32 scheduling_state = 0;
+    ThreadState thread_state = ThreadState::Initialized;
 
     u64 thread_id = 0;
 
@@ -652,11 +701,11 @@ private:
     /// Nominal thread priority, as set by the emulated application.
     /// The nominal priority is the thread priority without priority
     /// inheritance taken into account.
-    u32 nominal_priority = 0;
+    s32 base_priority{};
 
     /// Current thread priority. This may change over the course of the
     /// thread's lifetime in order to facilitate priority inheritance.
-    u32 current_priority = 0;
+    s32 current_priority{};
 
     u64 total_cpu_time_ticks = 0; ///< Total CPU running ticks.
     s64 schedule_count{};
@@ -671,37 +720,27 @@ private:
     Process* owner_process;
 
     /// Objects that the thread is waiting on, in the same order as they were
-    /// passed to WaitSynchronization.
-    ThreadSynchronizationObjects* wait_objects;
+    /// passed to WaitSynchronization. This is used for debugging only.
+    std::vector<KSynchronizationObject*> wait_objects_for_debugging;
 
-    SynchronizationObject* signaling_object;
+    /// The current mutex wait address. This is used for debugging only.
+    VAddr mutex_wait_address_for_debugging{};
+
+    /// The reason the thread is waiting. This is used for debugging only.
+    ThreadWaitReasonForDebugging wait_reason_for_debugging{};
+
+    KSynchronizationObject* signaling_object;
     ResultCode signaling_result{RESULT_SUCCESS};
 
     /// List of threads that are waiting for a mutex that is held by this thread.
     MutexWaitingThreads wait_mutex_threads;
 
     /// Thread that owns the lock that this thread is waiting for.
-    std::shared_ptr<Thread> lock_owner;
-
-    /// If waiting on a ConditionVariable, this is the ConditionVariable address
-    VAddr condvar_wait_address = 0;
-    /// If waiting on a Mutex, this is the mutex address
-    VAddr mutex_wait_address = 0;
-    /// The handle used to wait for the mutex.
-    Handle wait_handle = 0;
-
-    /// If waiting for an AddressArbiter, this is the address being waited on.
-    VAddr arb_wait_address{0};
-    bool waiting_for_arbitration{};
+    Thread* lock_owner{};
 
     /// Handle used as userdata to reference this object when inserting into the CoreTiming queue.
     Handle global_handle = 0;
 
-    /// Callback for HLE Events
-    HLECallback hle_callback;
-    Handle hle_time_event;
-    SynchronizationObject* hle_object;
-
     KScheduler* scheduler = nullptr;
 
     std::array<QueueEntry, Core::Hardware::NUM_CPU_CORES> per_core_priority_queue_entry{};
@@ -714,7 +753,7 @@ private:
 
     u32 pausing_state = 0;
     bool is_running = false;
-    bool is_waiting_on_sync = false;
+    bool is_cancellable = false;
     bool is_sync_cancelled = false;
 
     bool is_continuous_on_svc = false;
@@ -725,6 +764,18 @@ private:
 
     bool was_running = false;
 
+    bool signaled{};
+
+    ConditionVariableThreadTree* condvar_tree{};
+    uintptr_t condvar_key{};
+    VAddr address_key{};
+    u32 address_key_value{};
+    s32 num_kernel_waiters{};
+
+    using WaiterList = boost::intrusive::list<Thread>;
+    WaiterList waiter_list{};
+    WaiterList pinned_waiter_list{};
+
     std::string name;
 };