diff options
| author | bunnei <bunneidev@gmail.com> | 2021-02-12 17:58:31 -0800 |
|---|---|---|
| committer | bunnei <bunneidev@gmail.com> | 2021-02-18 16:16:25 -0800 |
| commit | 93e20867b0ab2e737e231a9b5bb29d40947fb311 (patch) | |
| tree | 84df7c7b56e9c78ce92809328f260edd68ccd490 /src/core/hle/kernel/memory | |
| parent | b1e27890e8728c19ceef404aa6352cd6f2913ded (diff) | |
hle: kernel: Migrate PageHeap/PageTable to KPageHeap/KPageTable.
Diffstat (limited to 'src/core/hle/kernel/memory')
| -rw-r--r-- | src/core/hle/kernel/memory/page_heap.cpp | 119 | ||||
| -rw-r--r-- | src/core/hle/kernel/memory/page_heap.h | 196 | ||||
| -rw-r--r-- | src/core/hle/kernel/memory/page_table.cpp | 1189 | ||||
| -rw-r--r-- | src/core/hle/kernel/memory/page_table.h | 281 |
4 files changed, 0 insertions, 1785 deletions
diff --git a/src/core/hle/kernel/memory/page_heap.cpp b/src/core/hle/kernel/memory/page_heap.cpp deleted file mode 100644 index 8fb53a0e8..000000000 --- a/src/core/hle/kernel/memory/page_heap.cpp +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -// This file references various implementation details from Atmosphere, an open-source firmware for -// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. - -#include "core/core.h" -#include "core/hle/kernel/memory/page_heap.h" -#include "core/memory.h" - -namespace Kernel::Memory { - -void PageHeap::Initialize(VAddr address, std::size_t size, std::size_t metadata_size) { - // Check our assumptions - ASSERT(Common::IsAligned((address), PageSize)); - ASSERT(Common::IsAligned(size, PageSize)); - - // Set our members - heap_address = address; - heap_size = size; - - // Setup bitmaps - metadata.resize(metadata_size / sizeof(u64)); - u64* cur_bitmap_storage{metadata.data()}; - for (std::size_t i = 0; i < MemoryBlockPageShifts.size(); i++) { - const std::size_t cur_block_shift{MemoryBlockPageShifts[i]}; - const std::size_t next_block_shift{ - (i != MemoryBlockPageShifts.size() - 1) ? MemoryBlockPageShifts[i + 1] : 0}; - cur_bitmap_storage = blocks[i].Initialize(heap_address, heap_size, cur_block_shift, - next_block_shift, cur_bitmap_storage); - } -} - -VAddr PageHeap::AllocateBlock(s32 index, bool random) { - const std::size_t needed_size{blocks[index].GetSize()}; - - for (s32 i{index}; i < static_cast<s32>(MemoryBlockPageShifts.size()); i++) { - if (const VAddr addr{blocks[i].PopBlock(random)}; addr) { - if (const std::size_t allocated_size{blocks[i].GetSize()}; - allocated_size > needed_size) { - Free(addr + needed_size, (allocated_size - needed_size) / PageSize); - } - return addr; - } - } - - return 0; -} - -void PageHeap::FreeBlock(VAddr block, s32 index) { - do { - block = blocks[index++].PushBlock(block); - } while (block != 0); -} - -void PageHeap::Free(VAddr addr, std::size_t num_pages) { - // Freeing no pages is a no-op - if (num_pages == 0) { - return; - } - - // Find the largest block size that we can free, and free as many as possible - s32 big_index{static_cast<s32>(MemoryBlockPageShifts.size()) - 1}; - const VAddr start{addr}; - const VAddr end{(num_pages * PageSize) + addr}; - VAddr before_start{start}; - VAddr before_end{start}; - VAddr after_start{end}; - VAddr after_end{end}; - while (big_index >= 0) { - const std::size_t block_size{blocks[big_index].GetSize()}; - const VAddr big_start{Common::AlignUp((start), block_size)}; - const VAddr big_end{Common::AlignDown((end), block_size)}; - if (big_start < big_end) { - // Free as many big blocks as we can - for (auto block{big_start}; block < big_end; block += block_size) { - FreeBlock(block, big_index); - } - before_end = big_start; - after_start = big_end; - break; - } - big_index--; - } - ASSERT(big_index >= 0); - - // Free space before the big blocks - for (s32 i{big_index - 1}; i >= 0; i--) { - const std::size_t block_size{blocks[i].GetSize()}; - while (before_start + block_size <= before_end) { - before_end -= block_size; - FreeBlock(before_end, i); - } - } - - // Free space after the big blocks - for (s32 i{big_index - 1}; i >= 0; i--) { - const std::size_t block_size{blocks[i].GetSize()}; - while (after_start + block_size <= after_end) { - FreeBlock(after_start, i); - after_start += block_size; - } - } -} - -std::size_t PageHeap::CalculateManagementOverheadSize(std::size_t region_size) { - std::size_t overhead_size = 0; - for (std::size_t i = 0; i < MemoryBlockPageShifts.size(); i++) { - const std::size_t cur_block_shift{MemoryBlockPageShifts[i]}; - const std::size_t next_block_shift{ - (i != MemoryBlockPageShifts.size() - 1) ? MemoryBlockPageShifts[i + 1] : 0}; - overhead_size += PageHeap::Block::CalculateManagementOverheadSize( - region_size, cur_block_shift, next_block_shift); - } - return Common::AlignUp(overhead_size, PageSize); -} - -} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_heap.h b/src/core/hle/kernel/memory/page_heap.h deleted file mode 100644 index e21d60a54..000000000 --- a/src/core/hle/kernel/memory/page_heap.h +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -// This file references various implementation details from Atmosphere, an open-source firmware for -// the Nintendo Switch. Copyright 2018-2020 Atmosphere-NX. - -#pragma once - -#include <array> -#include <bit> -#include <vector> - -#include "common/alignment.h" -#include "common/assert.h" -#include "common/common_funcs.h" -#include "common/common_types.h" -#include "core/hle/kernel/k_page_bitmap.h" -#include "core/hle/kernel/memory_types.h" - -namespace Kernel::Memory { - -class PageHeap final : NonCopyable { -public: - static constexpr s32 GetAlignedBlockIndex(std::size_t num_pages, std::size_t align_pages) { - const auto target_pages{std::max(num_pages, align_pages)}; - for (std::size_t i = 0; i < NumMemoryBlockPageShifts; i++) { - if (target_pages <= - (static_cast<std::size_t>(1) << MemoryBlockPageShifts[i]) / PageSize) { - return static_cast<s32>(i); - } - } - return -1; - } - - static constexpr s32 GetBlockIndex(std::size_t num_pages) { - for (s32 i{static_cast<s32>(NumMemoryBlockPageShifts) - 1}; i >= 0; i--) { - if (num_pages >= (static_cast<std::size_t>(1) << MemoryBlockPageShifts[i]) / PageSize) { - return i; - } - } - return -1; - } - - static constexpr std::size_t GetBlockSize(std::size_t index) { - return static_cast<std::size_t>(1) << MemoryBlockPageShifts[index]; - } - - static constexpr std::size_t GetBlockNumPages(std::size_t index) { - return GetBlockSize(index) / PageSize; - } - -private: - static constexpr std::size_t NumMemoryBlockPageShifts{7}; - static constexpr std::array<std::size_t, NumMemoryBlockPageShifts> MemoryBlockPageShifts{ - 0xC, 0x10, 0x15, 0x16, 0x19, 0x1D, 0x1E, - }; - - class Block final : NonCopyable { - private: - KPageBitmap bitmap; - VAddr heap_address{}; - uintptr_t end_offset{}; - std::size_t block_shift{}; - std::size_t next_block_shift{}; - - public: - Block() = default; - - constexpr std::size_t GetShift() const { - return block_shift; - } - constexpr std::size_t GetNextShift() const { - return next_block_shift; - } - constexpr std::size_t GetSize() const { - return static_cast<std::size_t>(1) << GetShift(); - } - constexpr std::size_t GetNumPages() const { - return GetSize() / PageSize; - } - constexpr std::size_t GetNumFreeBlocks() const { - return bitmap.GetNumBits(); - } - constexpr std::size_t GetNumFreePages() const { - return GetNumFreeBlocks() * GetNumPages(); - } - - u64* Initialize(VAddr addr, std::size_t size, std::size_t bs, std::size_t nbs, - u64* bit_storage) { - // Set shifts - block_shift = bs; - next_block_shift = nbs; - - // Align up the address - VAddr end{addr + size}; - const auto align{(next_block_shift != 0) ? (1ULL << next_block_shift) - : (1ULL << block_shift)}; - addr = Common::AlignDown((addr), align); - end = Common::AlignUp((end), align); - - heap_address = addr; - end_offset = (end - addr) / (1ULL << block_shift); - return bitmap.Initialize(bit_storage, end_offset); - } - - VAddr PushBlock(VAddr address) { - // Set the bit for the free block - std::size_t offset{(address - heap_address) >> GetShift()}; - bitmap.SetBit(offset); - - // If we have a next shift, try to clear the blocks below and return the address - if (GetNextShift()) { - const auto diff{1ULL << (GetNextShift() - GetShift())}; - offset = Common::AlignDown(offset, diff); - if (bitmap.ClearRange(offset, diff)) { - return heap_address + (offset << GetShift()); - } - } - - // We couldn't coalesce, or we're already as big as possible - return 0; - } - - VAddr PopBlock(bool random) { - // Find a free block - const s64 soffset{bitmap.FindFreeBlock(random)}; - if (soffset < 0) { - return 0; - } - const auto offset{static_cast<std::size_t>(soffset)}; - - // Update our tracking and return it - bitmap.ClearBit(offset); - return heap_address + (offset << GetShift()); - } - - public: - static constexpr std::size_t CalculateManagementOverheadSize(std::size_t region_size, - std::size_t cur_block_shift, - std::size_t next_block_shift) { - const auto cur_block_size{(1ULL << cur_block_shift)}; - const auto next_block_size{(1ULL << next_block_shift)}; - const auto align{(next_block_shift != 0) ? next_block_size : cur_block_size}; - return KPageBitmap::CalculateManagementOverheadSize( - (align * 2 + Common::AlignUp(region_size, align)) / cur_block_size); - } - }; - -public: - PageHeap() = default; - - constexpr VAddr GetAddress() const { - return heap_address; - } - constexpr std::size_t GetSize() const { - return heap_size; - } - constexpr VAddr GetEndAddress() const { - return GetAddress() + GetSize(); - } - constexpr std::size_t GetPageOffset(VAddr block) const { - return (block - GetAddress()) / PageSize; - } - - void Initialize(VAddr heap_address, std::size_t heap_size, std::size_t metadata_size); - VAddr AllocateBlock(s32 index, bool random); - void Free(VAddr addr, std::size_t num_pages); - - void UpdateUsedSize() { - used_size = heap_size - (GetNumFreePages() * PageSize); - } - - static std::size_t CalculateManagementOverheadSize(std::size_t region_size); - -private: - constexpr std::size_t GetNumFreePages() const { - std::size_t num_free{}; - - for (const auto& block : blocks) { - num_free += block.GetNumFreePages(); - } - - return num_free; - } - - void FreeBlock(VAddr block, s32 index); - - VAddr heap_address{}; - std::size_t heap_size{}; - std::size_t used_size{}; - std::array<Block, NumMemoryBlockPageShifts> blocks{}; - std::vector<u64> metadata; -}; - -} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_table.cpp b/src/core/hle/kernel/memory/page_table.cpp deleted file mode 100644 index ef9d97413..000000000 --- a/src/core/hle/kernel/memory/page_table.cpp +++ /dev/null @@ -1,1189 +0,0 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -#include "common/alignment.h" -#include "common/assert.h" -#include "common/scope_exit.h" -#include "core/core.h" -#include "core/hle/kernel/k_address_space_info.h" -#include "core/hle/kernel/k_memory_block.h" -#include "core/hle/kernel/k_memory_block_manager.h" -#include "core/hle/kernel/k_page_linked_list.h" -#include "core/hle/kernel/k_resource_limit.h" -#include "core/hle/kernel/k_scoped_resource_reservation.h" -#include "core/hle/kernel/k_system_control.h" -#include "core/hle/kernel/kernel.h" -#include "core/hle/kernel/memory/page_table.h" -#include "core/hle/kernel/process.h" -#include "core/hle/kernel/svc_results.h" -#include "core/memory.h" - -namespace Kernel::Memory { - -namespace { - -constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType as_type) { - switch (as_type) { - case FileSys::ProgramAddressSpaceType::Is32Bit: - case FileSys::ProgramAddressSpaceType::Is32BitNoMap: - return 32; - case FileSys::ProgramAddressSpaceType::Is36Bit: - return 36; - case FileSys::ProgramAddressSpaceType::Is39Bit: - return 39; - default: - UNREACHABLE(); - return {}; - } -} - -constexpr u64 GetAddressInRange(const KMemoryInfo& info, VAddr addr) { - if (info.GetAddress() < addr) { - return addr; - } - return info.GetAddress(); -} - -constexpr std::size_t GetSizeInRange(const KMemoryInfo& info, VAddr start, VAddr end) { - std::size_t size{info.GetSize()}; - if (info.GetAddress() < start) { - size -= start - info.GetAddress(); - } - if (info.GetEndAddress() > end) { - size -= info.GetEndAddress() - end; - } - return size; -} - -} // namespace - -PageTable::PageTable(Core::System& system) : system{system} {} - -ResultCode PageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, - bool enable_aslr, VAddr code_addr, std::size_t code_size, - KMemoryManager::Pool pool) { - - const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) { - return KAddressSpaceInfo::GetAddressSpaceStart(address_space_width, type); - }; - const auto GetSpaceSize = [this](KAddressSpaceInfo::Type type) { - return KAddressSpaceInfo::GetAddressSpaceSize(address_space_width, type); - }; - - // Set our width and heap/alias sizes - address_space_width = GetAddressSpaceWidthFromType(as_type); - const VAddr start = 0; - const VAddr end{1ULL << address_space_width}; - std::size_t alias_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Alias)}; - std::size_t heap_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Heap)}; - - ASSERT(start <= code_addr); - ASSERT(code_addr < code_addr + code_size); - ASSERT(code_addr + code_size - 1 <= end - 1); - - // Adjust heap/alias size if we don't have an alias region - if (as_type == FileSys::ProgramAddressSpaceType::Is32BitNoMap) { - heap_region_size += alias_region_size; - alias_region_size = 0; - } - - // Set code regions and determine remaining - constexpr std::size_t RegionAlignment{2 * 1024 * 1024}; - VAddr process_code_start{}; - VAddr process_code_end{}; - std::size_t stack_region_size{}; - std::size_t kernel_map_region_size{}; - - if (address_space_width == 39) { - alias_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Alias); - heap_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Heap); - stack_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Stack); - kernel_map_region_size = GetSpaceSize(KAddressSpaceInfo::Type::MapSmall); - code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::Map39Bit); - code_region_end = code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::Map39Bit); - alias_code_region_start = code_region_start; - alias_code_region_end = code_region_end; - process_code_start = Common::AlignDown(code_addr, RegionAlignment); - process_code_end = Common::AlignUp(code_addr + code_size, RegionAlignment); - } else { - stack_region_size = 0; - kernel_map_region_size = 0; - code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::MapSmall); - code_region_end = code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::MapSmall); - stack_region_start = code_region_start; - alias_code_region_start = code_region_start; - alias_code_region_end = GetSpaceStart(KAddressSpaceInfo::Type::MapLarge) + - GetSpaceSize(KAddressSpaceInfo::Type::MapLarge); - stack_region_end = code_region_end; - kernel_map_region_start = code_region_start; - kernel_map_region_end = code_region_end; - process_code_start = code_region_start; - process_code_end = code_region_end; - } - - // Set other basic fields - is_aslr_enabled = enable_aslr; - address_space_start = start; - address_space_end = end; - is_kernel = false; - - // Determine the region we can place our undetermineds in - VAddr alloc_start{}; - std::size_t alloc_size{}; - if ((process_code_start - code_region_start) >= (end - process_code_end)) { - alloc_start = code_region_start; - alloc_size = process_code_start - code_region_start; - } else { - alloc_start = process_code_end; - alloc_size = end - process_code_end; - } - const std::size_t needed_size{ - (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size)}; - if (alloc_size < needed_size) { - UNREACHABLE(); - return ResultOutOfMemory; - } - - const std::size_t remaining_size{alloc_size - needed_size}; - - // Determine random placements for each region - std::size_t alias_rnd{}, heap_rnd{}, stack_rnd{}, kmap_rnd{}; - if (enable_aslr) { - alias_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * - RegionAlignment; - heap_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * - RegionAlignment; - stack_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * - RegionAlignment; - kmap_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * - RegionAlignment; - } - - // Setup heap and alias regions - alias_region_start = alloc_start + alias_rnd; - alias_region_end = alias_region_start + alias_region_size; - heap_region_start = alloc_start + heap_rnd; - heap_region_end = heap_region_start + heap_region_size; - - if (alias_rnd <= heap_rnd) { - heap_region_start += alias_region_size; - heap_region_end += alias_region_size; - } else { - alias_region_start += heap_region_size; - alias_region_end += heap_region_size; - } - - // Setup stack region - if (stack_region_size) { - stack_region_start = alloc_start + stack_rnd; - stack_region_end = stack_region_start + stack_region_size; - - if (alias_rnd < stack_rnd) { - stack_region_start += alias_region_size; - stack_region_end += alias_region_size; - } else { - alias_region_start += stack_region_size; - alias_region_end += stack_region_size; - } - - if (heap_rnd < stack_rnd) { - stack_region_start += heap_region_size; - stack_region_end += heap_region_size; - } else { - heap_region_start += stack_region_size; - heap_region_end += stack_region_size; - } - } - - // Setup kernel map region - if (kernel_map_region_size) { - kernel_map_region_start = alloc_start + kmap_rnd; - kernel_map_region_end = kernel_map_region_start + kernel_map_region_size; - - if (alias_rnd < kmap_rnd) { - kernel_map_region_start += alias_region_size; - kernel_map_region_end += alias_region_size; - } else { - alias_region_start += kernel_map_region_size; - alias_region_end += kernel_map_region_size; - } - - if (heap_rnd < kmap_rnd) { - kernel_map_region_start += heap_region_size; - kernel_map_region_end += heap_region_size; - } else { - heap_region_start += kernel_map_region_size; - heap_region_end += kernel_map_region_size; - } - - if (stack_region_size) { - if (stack_rnd < kmap_rnd) { - kernel_map_region_start += stack_region_size; - kernel_map_region_end += stack_region_size; - } else { - stack_region_start += kernel_map_region_size; - stack_region_end += kernel_map_region_size; - } - } - } - - // Set heap members - current_heap_end = heap_region_start; - max_heap_size = 0; - max_physical_memory_size = 0; - - // Ensure that we regions inside our address space - auto IsInAddressSpace = [&](VAddr addr) { - return address_space_start <= addr && addr <= address_space_end; - }; - ASSERT(IsInAddressSpace(alias_region_start)); - ASSERT(IsInAddressSpace(alias_region_end)); - ASSERT(IsInAddressSpace(heap_region_start)); - ASSERT(IsInAddressSpace(heap_region_end)); - ASSERT(IsInAddressSpace(stack_region_start)); - ASSERT(IsInAddressSpace(stack_region_end)); - ASSERT(IsInAddressSpace(kernel_map_region_start)); - ASSERT(IsInAddressSpace(kernel_map_region_end)); - - // Ensure that we selected regions that don't overlap - const VAddr alias_start{alias_region_start}; - const VAddr alias_last{alias_region_end - 1}; - const VAddr heap_start{heap_region_start}; - const VAddr heap_last{heap_region_end - 1}; - const VAddr stack_start{stack_region_start}; - const VAddr stack_last{stack_region_end - 1}; - const VAddr kmap_start{kernel_map_region_start}; - const VAddr kmap_last{kernel_map_region_end - 1}; - ASSERT(alias_last < heap_start || heap_last < alias_start); - ASSERT(alias_last < stack_start || stack_last < alias_start); - ASSERT(alias_last < kmap_start || kmap_last < alias_start); - ASSERT(heap_last < stack_start || stack_last < heap_start); - ASSERT(heap_last < kmap_start || kmap_last < heap_start); - - current_heap_addr = heap_region_start; - heap_capacity = 0; - physical_memory_usage = 0; - memory_pool = pool; - - page_table_impl.Resize(address_space_width, PageBits); - - return InitializeMemoryLayout(start, end); -} - -ResultCode PageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, - KMemoryPermission perm) { - std::lock_guard lock{page_table_lock}; - - const u64 size{num_pages * PageSize}; - - if (!CanContain(addr, size, state)) { - return ResultInvalidCurrentMemory; - } - - if (IsRegionMapped(addr, size)) { - return ResultInvalidCurrentMemory; - } - - KPageLinkedList page_linked_list; - CASCADE_CODE( - system.Kernel().MemoryManager().Allocate(page_linked_list, num_pages, memory_pool)); - CASCADE_CODE(Operate(addr, num_pages, page_linked_list, OperationType::MapGroup)); - - block_manager->Update(addr, num_pages, state, perm); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::MapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - const std::size_t num_pages{size / PageSize}; - - KMemoryState state{}; - KMemoryPermission perm{}; - CASCADE_CODE(CheckMemoryState(&state, &perm, nullptr, src_addr, size, KMemoryState::All, - KMemoryState::Normal, KMemoryPermission::Mask, - KMemoryPermission::ReadAndWrite, KMemoryAttribute::Mask, - KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); - - if (IsRegionMapped(dst_addr, size)) { - return ResultInvalidCurrentMemory; - } - - KPageLinkedList page_linked_list; - AddRegionToPages(src_addr, num_pages, page_linked_list); - - { - auto block_guard = detail::ScopeExit( - [&] { Operate(src_addr, num_pages, perm, OperationType::ChangePermissions); }); - - CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None, - OperationType::ChangePermissions)); - CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::None)); - - block_guard.Cancel(); - } - - block_manager->Update(src_addr, num_pages, state, KMemoryPermission::None, - KMemoryAttribute::Locked); - block_manager->Update(dst_addr, num_pages, KMemoryState::AliasCode); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::UnmapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - if (!size) { - return RESULT_SUCCESS; - } - - const std::size_t num_pages{size / PageSize}; - - CASCADE_CODE(CheckMemoryState(nullptr, nullptr, nullptr, src_addr, size, KMemoryState::All, - KMemoryState::Normal, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); - - KMemoryState state{}; - CASCADE_CODE(CheckMemoryState( - &state, nullptr, nullptr, dst_addr, PageSize, KMemoryState::FlagCanCodeAlias, - KMemoryState::FlagCanCodeAlias, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); - CASCADE_CODE(CheckMemoryState(dst_addr, size, KMemoryState::All, state, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::None)); - CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); - - block_manager->Update(dst_addr, num_pages, KMemoryState::Free); - block_manager->Update(src_addr, num_pages, KMemoryState::Normal, - KMemoryPermission::ReadAndWrite); - - return RESULT_SUCCESS; -} - -void PageTable::MapPhysicalMemory(KPageLinkedList& page_linked_list, VAddr start, VAddr end) { - auto node{page_linked_list.Nodes().begin()}; - PAddr map_addr{node->GetAddress()}; - std::size_t src_num_pages{node->GetNumPages()}; - - block_manager->IterateForRange(start, end, [&](const KMemoryInfo& info) { - if (info.state != KMemoryState::Free) { - return; - } - - std::size_t dst_num_pages{GetSizeInRange(info, start, end) / PageSize}; - VAddr dst_addr{GetAddressInRange(info, start)}; - - while (dst_num_pages) { - if (!src_num_pages) { - node = std::next(node); - map_addr = node->GetAddress(); - src_num_pages = node->GetNumPages(); - } - - const std::size_t num_pages{std::min(src_num_pages, dst_num_pages)}; - Operate(dst_addr, num_pages, KMemoryPermission::ReadAndWrite, OperationType::Map, - map_addr); - - dst_addr += num_pages * PageSize; - map_addr += num_pages * PageSize; - src_num_pages -= num_pages; - dst_num_pages -= num_pages; - } - }); -} - -ResultCode PageTable::MapPhysicalMemory(VAddr addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - std::size_t mapped_size{}; - const VAddr end_addr{addr + size}; - - block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { - if (info.state != KMemoryState::Free) { - mapped_size += GetSizeInRange(info, addr, end_addr); - } - }); - - if (mapped_size == size) { - return RESULT_SUCCESS; - } - - const std::size_t remaining_size{size - mapped_size}; - const std::size_t remaining_pages{remaining_size / PageSize}; - - // Reserve the memory from the process resource limit. - KScopedResourceReservation memory_reservation( - system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, - remaining_size); - if (!memory_reservation.Succeeded()) { - LOG_ERROR(Kernel, "Could not reserve remaining {:X} bytes", remaining_size); - return ResultResourceLimitedExceeded; - } - - KPageLinkedList page_linked_list; - - CASCADE_CODE( - system.Kernel().MemoryManager().Allocate(page_linked_list, remaining_pages, memory_pool)); - - // We succeeded, so commit the memory reservation. - memory_reservation.Commit(); - - MapPhysicalMemory(page_linked_list, addr, end_addr); - - physical_memory_usage += remaining_size; - - const std::size_t num_pages{size / PageSize}; - block_manager->Update(addr, num_pages, KMemoryState::Free, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryState::Normal, - KMemoryPermission::ReadAndWrite, KMemoryAttribute::None); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::UnmapPhysicalMemory(VAddr addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - const VAddr end_addr{addr + size}; - ResultCode result{RESULT_SUCCESS}; - std::size_t mapped_size{}; - - // Verify that the region can be unmapped - block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { - if (info.state == KMemoryState::Normal) { - if (info.attribute != KMemoryAttribute::None) { - result = ResultInvalidCurrentMemory; - return; - } - mapped_size += GetSizeInRange(info, addr, end_addr); - } else if (info.state != KMemoryState::Free) { - result = ResultInvalidCurrentMemory; - } - }); - - if (result.IsError()) { - return result; - } - - if (!mapped_size) { - return RESULT_SUCCESS; - } - - CASCADE_CODE(UnmapMemory(addr, size)); - - auto process{system.Kernel().CurrentProcess()}; - process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size); - physical_memory_usage -= mapped_size; - - return RESULT_SUCCESS; -} - -ResultCode PageTable::UnmapMemory(VAddr addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - const VAddr end_addr{addr + size}; - ResultCode result{RESULT_SUCCESS}; - KPageLinkedList page_linked_list; - - // Unmap each region within the range - block_manager->IterateForRange(addr, end_addr, [&](const KMemoryInfo& info) { - if (info.state == KMemoryState::Normal) { - const std::size_t block_size{GetSizeInRange(info, addr, end_addr)}; - const std::size_t block_num_pages{block_size / PageSize}; - const VAddr block_addr{GetAddressInRange(info, addr)}; - - AddRegionToPages(block_addr, block_size / PageSize, page_linked_list); - - if (result = Operate(block_addr, block_num_pages, KMemoryPermission::None, - OperationType::Unmap); - result.IsError()) { - return; - } - } - }); - - if (result.IsError()) { - return result; - } - - const std::size_t num_pages{size / PageSize}; - system.Kernel().MemoryManager().Free(page_linked_list, num_pages, memory_pool); - - block_manager->Update(addr, num_pages, KMemoryState::Free); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::Map(VAddr dst_addr, VAddr src_addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - KMemoryState src_state{}; - CASCADE_CODE(CheckMemoryState( - &src_state, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, - KMemoryState::FlagCanAlias, KMemoryPermission::Mask, KMemoryPermission::ReadAndWrite, - KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); - - if (IsRegionMapped(dst_addr, size)) { - return ResultInvalidCurrentMemory; - } - - KPageLinkedList page_linked_list; - const std::size_t num_pages{size / PageSize}; - - AddRegionToPages(src_addr, num_pages, page_linked_list); - - { - auto block_guard = detail::ScopeExit([&] { - Operate(src_addr, num_pages, KMemoryPermission::ReadAndWrite, - OperationType::ChangePermissions); - }); - - CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None, - OperationType::ChangePermissions)); - CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::ReadAndWrite)); - - block_guard.Cancel(); - } - - block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::None, - KMemoryAttribute::Locked); - block_manager->Update(dst_addr, num_pages, KMemoryState::Stack, - KMemoryPermission::ReadAndWrite); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::Unmap(VAddr dst_addr, VAddr src_addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - KMemoryState src_state{}; - CASCADE_CODE(CheckMemoryState( - &src_state, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, - KMemoryState::FlagCanAlias, KMemoryPermission::Mask, KMemoryPermission::None, - KMemoryAttribute::Mask, KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); - - KMemoryPermission dst_perm{}; - CASCADE_CODE(CheckMemoryState(nullptr, &dst_perm, nullptr, dst_addr, size, KMemoryState::All, - KMemoryState::Stack, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); - - KPageLinkedList src_pages; - KPageLinkedList dst_pages; - const std::size_t num_pages{size / PageSize}; - - AddRegionToPages(src_addr, num_pages, src_pages); - AddRegionToPages(dst_addr, num_pages, dst_pages); - - if (!dst_pages.IsEqual(src_pages)) { - return ResultInvalidMemoryRange; - } - - { - auto block_guard = detail::ScopeExit([&] { MapPages(dst_addr, dst_pages, dst_perm); }); - - CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); - CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::ReadAndWrite, - OperationType::ChangePermissions)); - - block_guard.Cancel(); - } - - block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::ReadAndWrite); - block_manager->Update(dst_addr, num_pages, KMemoryState::Free); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::MapPages(VAddr addr, const KPageLinkedList& page_linked_list, - KMemoryPermission perm) { - VAddr cur_addr{addr}; - - for (const auto& node : page_linked_list.Nodes()) { - if (const auto result{ - Operate(cur_addr, node.GetNumPages(), perm, OperationType::Map, node.GetAddress())}; - result.IsError()) { - const std::size_t num_pages{(addr - cur_addr) / PageSize}; - - ASSERT(Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap) - .IsSuccess()); - - return result; - } - - cur_addr += node.GetNumPages() * PageSize; - } - - return RESULT_SUCCESS; -} - -ResultCode PageTable::MapPages(VAddr addr, KPageLinkedList& page_linked_list, KMemoryState state, - KMemoryPermission perm) { - std::lock_guard lock{page_table_lock}; - - const std::size_t num_pages{page_linked_list.GetNumPages()}; - const std::size_t size{num_pages * PageSize}; - - if (!CanContain(addr, size, state)) { - return ResultInvalidCurrentMemory; - } - - if (IsRegionMapped(addr, num_pages * PageSize)) { - return ResultInvalidCurrentMemory; - } - - CASCADE_CODE(MapPages(addr, page_linked_list, perm)); - - block_manager->Update(addr, num_pages, state, perm); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::SetCodeMemoryPermission(VAddr addr, std::size_t size, - KMemoryPermission perm) { - - std::lock_guard lock{page_table_lock}; - - KMemoryState prev_state{}; - KMemoryPermission prev_perm{}; - - CASCADE_CODE(CheckMemoryState( - &prev_state, &prev_perm, nullptr, addr, size, KMemoryState::FlagCode, - KMemoryState::FlagCode, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); - - KMemoryState state{prev_state}; - - // Ensure state is mutable if permission allows write - if ((perm & KMemoryPermission::Write) != KMemoryPermission::None) { - if (prev_state == KMemoryState::Code) { - state = KMemoryState::CodeData; - } else if (prev_state == KMemoryState::AliasCode) { - state = KMemoryState::AliasCodeData; - } else { - UNREACHABLE(); - } - } - - // Return early if there is nothing to change - if (state == prev_state && perm == prev_perm) { - return RESULT_SUCCESS; - } - - if ((prev_perm & KMemoryPermission::Execute) != (perm & KMemoryPermission::Execute)) { - // Memory execution state is changing, invalidate CPU cache range - system.InvalidateCpuInstructionCacheRange(addr, size); - } - - const std::size_t num_pages{size / PageSize}; - const OperationType operation{(perm & KMemoryPermission::Execute) != KMemoryPermission::None - ? OperationType::ChangePermissionsAndRefresh - : OperationType::ChangePermissions}; - - CASCADE_CODE(Operate(addr, num_pages, perm, operation)); - - block_manager->Update(addr, num_pages, state, perm); - - return RESULT_SUCCESS; -} - -KMemoryInfo PageTable::QueryInfoImpl(VAddr addr) { - std::lock_guard lock{page_table_lock}; - - return block_manager->FindBlock(addr).GetMemoryInfo(); -} - -KMemoryInfo PageTable::QueryInfo(VAddr addr) { - if (!Contains(addr, 1)) { - return {address_space_end, 0 - address_space_end, KMemoryState::Inaccessible, - KMemoryPermission::None, KMemoryAttribute::None, KMemoryPermission::None}; - } - - return QueryInfoImpl(addr); -} - -ResultCode PageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm) { - std::lock_guard lock{page_table_lock}; - - KMemoryState state{}; - KMemoryAttribute attribute{}; - - CASCADE_CODE(CheckMemoryState( - &state, nullptr, &attribute, addr, size, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, KMemoryPermission::Mask, - KMemoryPermission::ReadAndWrite, KMemoryAttribute::Mask, KMemoryAttribute::None, - KMemoryAttribute::IpcAndDeviceMapped)); - - block_manager->Update(addr, size / PageSize, state, perm, attribute | KMemoryAttribute::Locked); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::ResetTransferMemory(VAddr addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - KMemoryState state{}; - - CASCADE_CODE( - CheckMemoryState(&state, nullptr, nullptr, addr, size, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); - - block_manager->Update(addr, size / PageSize, state, KMemoryPermission::ReadAndWrite); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::SetMemoryAttribute(VAddr addr, std::size_t size, KMemoryAttribute mask, - KMemoryAttribute value) { - std::lock_guard lock{page_table_lock}; - - KMemoryState state{}; - KMemoryPermission perm{}; - KMemoryAttribute attribute{}; - - CASCADE_CODE(CheckMemoryState( - &state, &perm, &attribute, addr, size, KMemoryState::FlagCanChangeAttribute, - KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, - KMemoryAttribute::DeviceSharedAndUncached)); - - attribute = attribute & ~mask; - attribute = attribute | (mask & value); - - block_manager->Update(addr, size / PageSize, state, perm, attribute); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::SetHeapCapacity(std::size_t new_heap_capacity) { - std::lock_guard lock{page_table_lock}; - heap_capacity = new_heap_capacity; - return RESULT_SUCCESS; -} - -ResultVal<VAddr> PageTable::SetHeapSize(std::size_t size) { - - if (size > heap_region_end - heap_region_start) { - return ResultOutOfMemory; - } - - const u64 previous_heap_size{GetHeapSize()}; - - UNIMPLEMENTED_IF_MSG(previous_heap_size > size, "Heap shrink is unimplemented"); - - // Increase the heap size - { - std::lock_guard lock{page_table_lock}; - - const u64 delta{size - previous_heap_size}; - - // Reserve memory for the heap extension. - KScopedResourceReservation memory_reservation( - system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, - delta); - - if (!memory_reservation.Succeeded()) { - LOG_ERROR(Kernel, "Could not reserve heap extension of size {:X} bytes", delta); - return ResultResourceLimitedExceeded; - } - - KPageLinkedList page_linked_list; - const std::size_t num_pages{delta / PageSize}; - - CASCADE_CODE( - system.Kernel().MemoryManager().Allocate(page_linked_list, num_pages, memory_pool)); - - if (IsRegionMapped(current_heap_addr, delta)) { - return ResultInvalidCurrentMemory; - } - - CASCADE_CODE( - Operate(current_heap_addr, num_pages, page_linked_list, OperationType::MapGroup)); - - // Succeeded in allocation, commit the resource reservation - memory_reservation.Commit(); - - block_manager->Update(current_heap_addr, num_pages, KMemoryState::Normal, - KMemoryPermission::ReadAndWrite); - - current_heap_addr = heap_region_start + size; - } - - return MakeResult<VAddr>(heap_region_start); -} - -ResultVal<VAddr> PageTable::AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, - bool is_map_only, VAddr region_start, - std::size_t region_num_pages, KMemoryState state, - KMemoryPermission perm, PAddr map_addr) { - std::lock_guard lock{page_table_lock}; - - if (!CanContain(region_start, region_num_pages * PageSize, state)) { - return ResultInvalidCurrentMemory; - } - - if (region_num_pages <= needed_num_pages) { - return ResultOutOfMemory; - } - - const VAddr addr{ - AllocateVirtualMemory(region_start, region_num_pages, needed_num_pages, align)}; - if (!addr) { - return ResultOutOfMemory; - } - - if (is_map_only) { - CASCADE_CODE(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); - } else { - KPageLinkedList page_group; - CASCADE_CODE( - system.Kernel().MemoryManager().Allocate(page_group, needed_num_pages, memory_pool)); - CASCADE_CODE(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); - } - - block_manager->Update(addr, needed_num_pages, state, perm); - - return MakeResult<VAddr>(addr); -} - -ResultCode PageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - KMemoryPermission perm{}; - if (const ResultCode result{CheckMemoryState( - nullptr, &perm, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, - KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, - KMemoryAttribute::DeviceSharedAndUncached)}; - result.IsError()) { - return result; - } - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission perm) { - block->ShareToDevice(perm); - }, - perm); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) { - std::lock_guard lock{page_table_lock}; - - KMemoryPermission perm{}; - if (const ResultCode result{CheckMemoryState( - nullptr, &perm, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, - KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, - KMemoryAttribute::DeviceSharedAndUncached)}; - result.IsError()) { - return result; - } - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission perm) { - block->UnshareToDevice(perm); - }, - perm); - - return RESULT_SUCCESS; -} - -ResultCode PageTable::InitializeMemoryLayout(VAddr start, VAddr end) { - block_manager = std::make_unique<KMemoryBlockManager>(start, end); - - return RESULT_SUCCESS; -} - -bool PageTable::IsRegionMapped(VAddr address, u64 size) { - return CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, - KMemoryPermission::Mask, KMemoryPermission::None, - KMemoryAttribute::Mask, KMemoryAttribute::None, - KMemoryAttribute::IpcAndDeviceMapped) - .IsError(); -} - -bool PageTable::IsRegionContiguous(VAddr addr, u64 size) const { - auto start_ptr = system.Memory().GetPointer(addr); - for (u64 offset{}; offset < size; offset += PageSize) { - if (start_ptr != system.Memory().GetPointer(addr + offset)) { - return false; - } - start_ptr += PageSize; - } - return true; -} - -void PageTable::AddRegionToPages(VAddr start, std::size_t num_pages, - KPageLinkedList& page_linked_list) { - VAddr addr{start}; - while (addr < start + (num_pages * PageSize)) { - const PAddr paddr{GetPhysicalAddr(addr)}; - if (!paddr) { - UNREACHABLE(); - } - page_linked_list.AddBlock(paddr, 1); - addr += PageSize; - } -} - -VAddr PageTable::AllocateVirtualMemory(VAddr start, std::size_t region_num_pages, - u64 needed_num_pages, std::size_t align) { - if (is_aslr_enabled) { - UNIMPLEMENTED(); - } - return block_manager->FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, - IsKernel() ? 1 : 4); -} - -ResultCode PageTable::Operate(VAddr addr, std::size_t num_pages, const KPageLinkedList& page_group, - OperationType operation) { - std::lock_guard lock{page_table_lock}; - - ASSERT(Common::IsAligned(addr, PageSize)); - ASSERT(num_pages > 0); - ASSERT(num_pages == page_group.GetNumPages()); - - for (const auto& node : page_group.Nodes()) { - const std::size_t size{node.GetNumPages() * PageSize}; - - switch (operation) { - case OperationType::MapGroup: - system.Memory().MapMemoryRegion(page_table_impl, addr, size, node.GetAddress()); - break; - default: - UNREACHABLE(); - } - - addr += size; - } - - return RESULT_SUCCESS; -} - -ResultCode PageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, - OperationType operation, PAddr map_addr) { - std::lock_guard lock{page_table_lock}; - - ASSERT(num_pages > 0); - ASSERT(Common::IsAligned(addr, PageSize)); - ASSERT(ContainsPages(addr, num_pages)); - - switch (operation) { - case OperationType::Unmap: - system.Memory().UnmapRegion(page_table_impl, addr, num_pages * PageSize); - break; - case OperationType::Map: { - ASSERT(map_addr); - ASSERT(Common::IsAligned(map_addr, PageSize)); - system.Memory().MapMemoryRegion(page_table_impl, addr, num_pages * PageSize, map_addr); - break; - } - case OperationType::ChangePermissions: - case OperationType::ChangePermissionsAndRefresh: - break; - default: - UNREACHABLE(); - } - return RESULT_SUCCESS; -} - -constexpr VAddr PageTable::GetRegionAddress(KMemoryState state) const { - switch (state) { - case KMemoryState::Free: - case KMemoryState::Kernel: - return address_space_start; - case KMemoryState::Normal: - return heap_region_start; - case KMemoryState::Ipc: - case KMemoryState::NonSecureIpc: - case KMemoryState::NonDeviceIpc: - return alias_region_start; - case KMemoryState::Stack: - return stack_region_start; - case KMemoryState::Io: - case KMemoryState::Static: - case KMemoryState::ThreadLocal: - return kernel_map_region_start; - case KMemoryState::Shared: - case KMemoryState::AliasCode: - case KMemoryState::AliasCodeData: - case KMemoryState::Transferred: - case KMemoryState::SharedTransferred: - case KMemoryState::SharedCode: - case KMemoryState::GeneratedCode: - case KMemoryState::CodeOut: - return alias_code_region_start; - case KMemoryState::Code: - case KMemoryState::CodeData: - return code_region_start; - default: - UNREACHABLE(); - return {}; - } -} - -constexpr std::size_t PageTable::GetRegionSize(KMemoryState state) const { - switch (state) { - case KMemoryState::Free: - case KMemoryState::Kernel: - return address_space_end - address_space_start; - case KMemoryState::Normal: - return heap_region_end - heap_region_start; - case KMemoryState::Ipc: - case KMemoryState::NonSecureIpc: - case KMemoryState::NonDeviceIpc: - return alias_region_end - alias_region_start; - case KMemoryState::Stack: - return stack_region_end - stack_region_start; - case KMemoryState::Io: - case KMemoryState::Static: - case KMemoryState::ThreadLocal: - return kernel_map_region_end - kernel_map_region_start; - case KMemoryState::Shared: - case KMemoryState::AliasCode: - case KMemoryState::AliasCodeData: - case KMemoryState::Transferred: - case KMemoryState::SharedTransferred: - case KMemoryState::SharedCode: - case KMemoryState::GeneratedCode: - case KMemoryState::CodeOut: - return alias_code_region_end - alias_code_region_start; - case KMemoryState::Code: - case KMemoryState::CodeData: - return code_region_end - code_region_start; - default: - UNREACHABLE(); - return {}; - } -} - -constexpr bool PageTable::CanContain(VAddr addr, std::size_t size, KMemoryState state) const { - const VAddr end{addr + size}; - const VAddr last{end - 1}; - const VAddr region_start{GetRegionAddress(state)}; - const std::size_t region_size{GetRegionSize(state)}; - const bool is_in_region{region_start <= addr && addr < end && - last <= region_start + region_size - 1}; - const bool is_in_heap{!(end <= heap_region_start || heap_region_end <= addr)}; - const bool is_in_alias{!(end <= alias_region_start || alias_region_end <= addr)}; - - switch (state) { - case KMemoryState::Free: - case KMemoryState::Kernel: - return is_in_region; - case KMemoryState::Io: - case KMemoryState::Static: - case KMemoryState::Code: - case KMemoryState::CodeData: - case KMemoryState::Shared: - case KMemoryState::AliasCode: - case KMemoryState::AliasCodeData: - case KMemoryState::Stack: - case KMemoryState::ThreadLocal: - case KMemoryState::Transferred: - case KMemoryState::SharedTransferred: - case KMemoryState::SharedCode: - case KMemoryState::GeneratedCode: - case KMemoryState::CodeOut: - return is_in_region && !is_in_heap && !is_in_alias; - case KMemoryState::Normal: - ASSERT(is_in_heap); - return is_in_region && !is_in_alias; - case KMemoryState::Ipc: - case KMemoryState::NonSecureIpc: - case KMemoryState::NonDeviceIpc: - ASSERT(is_in_alias); - return is_in_region && !is_in_heap; - default: - return false; - } -} - -constexpr ResultCode PageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, - KMemoryPermission perm, KMemoryAttribute attr_mask, - KMemoryAttribute attr) const { - // Validate the states match expectation - if ((info.state & state_mask) != state) { - return ResultInvalidCurrentMemory; - } - if ((info.perm & perm_mask) != perm) { - return ResultInvalidCurrentMemory; - } - if ((info.attribute & attr_mask) != attr) { - return ResultInvalidCurrentMemory; - } - - return RESULT_SUCCESS; -} - -ResultCode PageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, - KMemoryAttribute* out_attr, VAddr addr, std::size_t size, - KMemoryState state_mask, KMemoryState state, - KMemoryPermission perm_mask, KMemoryPermission perm, - KMemoryAttribute attr_mask, KMemoryAttribute attr, - KMemoryAttribute ignore_attr) { - std::lock_guard lock{page_table_lock}; - - // Get information about the first block - const VAddr last_addr{addr + size - 1}; - KMemoryBlockManager::const_iterator it{block_manager->FindIterator(addr)}; - KMemoryInfo info{it->GetMemoryInfo()}; - - // Validate all blocks in the range have correct state - const KMemoryState first_state{info.state}; - const KMemoryPermission first_perm{info.perm}; - const KMemoryAttribute first_attr{info.attribute}; - - while (true) { - // Validate the current block - if (!(info.state == first_state)) { - return ResultInvalidCurrentMemory; - } - if (!(info.perm == first_perm)) { - return ResultInvalidCurrentMemory; - } - if (!((info.attribute | static_cast<KMemoryAttribute>(ignore_attr)) == - (first_attr | static_cast<KMemoryAttribute>(ignore_attr)))) { - return ResultInvalidCurrentMemory; - } - - // Validate against the provided masks - CASCADE_CODE(CheckMemoryState(info, state_mask, state, perm_mask, perm, attr_mask, attr)); - - // Break once we're done - if (last_addr <= info.GetLastAddress()) { - break; - } - - // Advance our iterator - it++; - ASSERT(it != block_manager->cend()); - info = it->GetMemoryInfo(); - } - - // Write output state - if (out_state) { - *out_state = first_state; - } - if (out_perm) { - *out_perm = first_perm; - } - if (out_attr) { - *out_attr = first_attr & static_cast<KMemoryAttribute>(~ignore_attr); - } - - return RESULT_SUCCESS; -} - -} // namespace Kernel::Memory diff --git a/src/core/hle/kernel/memory/page_table.h b/src/core/hle/kernel/memory/page_table.h deleted file mode 100644 index a9e850e01..000000000 --- a/src/core/hle/kernel/memory/page_table.h +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2020 yuzu Emulator Project -// Licensed under GPLv2 or any later version -// Refer to the license.txt file included. - -#pragma once - -#include <memory> -#include <mutex> - -#include "common/common_types.h" -#include "common/page_table.h" -#include "core/file_sys/program_metadata.h" -#include "core/hle/kernel/k_memory_block.h" -#include "core/hle/kernel/k_memory_manager.h" -#include "core/hle/result.h" - -namespace Core { -class System; -} - -namespace Kernel { -class KMemoryBlockManager; -} - -namespace Kernel::Memory { - -class PageTable final : NonCopyable { -public: - explicit PageTable(Core::System& system); - - ResultCode InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, - VAddr code_addr, std::size_t code_size, - KMemoryManager::Pool pool); - ResultCode MapProcessCode(VAddr addr, std::size_t pages_count, KMemoryState state, - KMemoryPermission perm); - ResultCode MapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); - ResultCode UnmapProcessCodeMemory(VAddr dst_addr, VAddr src_addr, std::size_t size); - ResultCode MapPhysicalMemory(VAddr addr, std::size_t size); - ResultCode UnmapPhysicalMemory(VAddr addr, std::size_t size); - ResultCode UnmapMemory(VAddr addr, std::size_t size); - ResultCode Map(VAddr dst_addr, VAddr src_addr, std::size_t size); - ResultCode Unmap(VAddr dst_addr, VAddr src_addr, std::size_t size); - ResultCode MapPages(VAddr addr, KPageLinkedList& page_linked_list, KMemoryState state, - KMemoryPermission perm); - ResultCode SetCodeMemoryPermission(VAddr addr, std::size_t size, KMemoryPermission perm); - KMemoryInfo QueryInfo(VAddr addr); - ResultCode ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm); - ResultCode ResetTransferMemory(VAddr addr, std::size_t size); - ResultCode SetMemoryAttribute(VAddr addr, std::size_t size, KMemoryAttribute mask, - KMemoryAttribute value); - ResultCode SetHeapCapacity(std::size_t new_heap_capacity); - ResultVal<VAddr> SetHeapSize(std::size_t size); - ResultVal<VAddr> AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, - bool is_map_only, VAddr region_start, - std::size_t region_num_pages, KMemoryState state, - KMemoryPermission perm, PAddr map_addr = 0); - ResultCode LockForDeviceAddressSpace(VAddr addr, std::size_t size); - ResultCode UnlockForDeviceAddressSpace(VAddr addr, std::size_t size); - - Common::PageTable& PageTableImpl() { - return page_table_impl; - } - - const Common::PageTable& PageTableImpl() const { - return page_table_impl; - } - -private: - enum class OperationType : u32 { - Map, - MapGroup, - Unmap, - ChangePermissions, - ChangePermissionsAndRefresh, - }; - - static constexpr KMemoryAttribute DefaultMemoryIgnoreAttr = KMemoryAttribute::DontCareMask | - KMemoryAttribute::IpcLocked | - KMemoryAttribute::DeviceShared; - - ResultCode InitializeMemoryLayout(VAddr start, VAddr end); - ResultCode MapPages(VAddr addr, const KPageLinkedList& page_linked_list, - KMemoryPermission perm); - void MapPhysicalMemory(KPageLinkedList& page_linked_list, VAddr start, VAddr end); - bool IsRegionMapped(VAddr address, u64 size); - bool IsRegionContiguous(VAddr addr, u64 size) const; - void AddRegionToPages(VAddr start, std::size_t num_pages, KPageLinkedList& page_linked_list); - KMemoryInfo QueryInfoImpl(VAddr addr); - VAddr AllocateVirtualMemory(VAddr start, std::size_t region_num_pages, u64 needed_num_pages, - std::size_t align); - ResultCode Operate(VAddr addr, std::size_t num_pages, const KPageLinkedList& page_group, - OperationType operation); - ResultCode Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, - OperationType operation, PAddr map_addr = 0); - constexpr VAddr GetRegionAddress(KMemoryState state) const; - constexpr std::size_t GetRegionSize(KMemoryState state) const; - constexpr bool CanContain(VAddr addr, std::size_t size, KMemoryState state) const; - - constexpr ResultCode CheckMemoryState(const KMemoryInfo& info, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, - KMemoryPermission perm, KMemoryAttribute attr_mask, - KMemoryAttribute attr) const; - ResultCode CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, - KMemoryAttribute* out_attr, VAddr addr, std::size_t size, - KMemoryState state_mask, KMemoryState state, - KMemoryPermission perm_mask, KMemoryPermission perm, - KMemoryAttribute attr_mask, KMemoryAttribute attr, - KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr); - ResultCode CheckMemoryState(VAddr addr, std::size_t size, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, - KMemoryPermission perm, KMemoryAttribute attr_mask, - KMemoryAttribute attr, - KMemoryAttribute ignore_attr = DefaultMemoryIgnoreAttr) { - return CheckMemoryState(nullptr, nullptr, nullptr, addr, size, state_mask, state, perm_mask, - perm, attr_mask, attr, ignore_attr); - } - - std::recursive_mutex page_table_lock; - std::unique_ptr<KMemoryBlockManager> block_manager; - -public: - constexpr VAddr GetAddressSpaceStart() const { - return address_space_start; - } - constexpr VAddr GetAddressSpaceEnd() const { - return address_space_end; - } - constexpr std::size_t GetAddressSpaceSize() const { - return address_space_end - address_space_start; - } - constexpr VAddr GetHeapRegionStart() const { - return heap_region_start; - } - constexpr VAddr GetHeapRegionEnd() const { - return heap_region_end; - } - constexpr std::size_t GetHeapRegionSize() const { - return heap_region_end - heap_region_start; - } - constexpr VAddr GetAliasRegionStart() const { - return alias_region_start; - } - constexpr VAddr GetAliasRegionEnd() const { - return alias_region_end; - } - constexpr std::size_t GetAliasRegionSize() const { - return alias_region_end - alias_region_start; - } - constexpr VAddr GetStackRegionStart() const { - return stack_region_start; - } - constexpr VAddr GetStackRegionEnd() const { - return stack_region_end; - } - constexpr std::size_t GetStackRegionSize() const { - return stack_region_end - stack_region_start; - } - constexpr VAddr GetKernelMapRegionStart() const { - return kernel_map_region_start; - } - constexpr VAddr GetKernelMapRegionEnd() const { - return kernel_map_region_end; - } - constexpr VAddr GetCodeRegionStart() const { - return code_region_start; - } - constexpr VAddr GetCodeRegionEnd() const { - return code_region_end; - } - constexpr VAddr GetAliasCodeRegionStart() const { - return alias_code_region_start; - } - constexpr VAddr GetAliasCodeRegionSize() const { - return alias_code_region_end - alias_code_region_start; - } - constexpr std::size_t GetAddressSpaceWidth() const { - return address_space_width; - } - constexpr std::size_t GetHeapSize() { - return current_heap_addr - heap_region_start; - } - constexpr std::size_t GetTotalHeapSize() { - return GetHeapSize() + physical_memory_usage; - } - constexpr bool IsInsideAddressSpace(VAddr address, std::size_t size) const { - return address_space_start <= address && address + size - 1 <= address_space_end - 1; - } - constexpr bool IsOutsideAliasRegion(VAddr address, std::size_t size) const { - return alias_region_start > address || address + size - 1 > alias_region_end - 1; - } - constexpr bool IsOutsideStackRegion(VAddr address, std::size_t size) const { - return stack_region_start > address || address + size - 1 > stack_region_end - 1; - } - constexpr bool IsInvalidRegion(VAddr address, std::size_t size) const { - return address + size - 1 > GetAliasCodeRegionStart() + GetAliasCodeRegionSize() - 1; - } - constexpr bool IsInsideHeapRegion(VAddr address, std::size_t size) const { - return address + size > heap_region_start && heap_region_end > address; - } - constexpr bool IsInsideAliasRegion(VAddr address, std::size_t size) const { - return address + size > alias_region_start && alias_region_end > address; - } - constexpr bool IsOutsideASLRRegion(VAddr address, std::size_t size) const { - if (IsInvalidRegion(address, size)) { - return true; - } - if (IsInsideHeapRegion(address, size)) { - return true; - } - if (IsInsideAliasRegion(address, size)) { - return true; - } - return {}; - } - constexpr bool IsInsideASLRRegion(VAddr address, std::size_t size) const { - return !IsOutsideASLRRegion(address, size); - } - constexpr PAddr GetPhysicalAddr(VAddr addr) { - return page_table_impl.backing_addr[addr >> PageBits] + addr; - } - -private: - constexpr bool Contains(VAddr addr) const { - return address_space_start <= addr && addr <= address_space_end - 1; - } - constexpr bool Contains(VAddr addr, std::size_t size) const { - return address_space_start <= addr && addr < addr + size && - addr + size - 1 <= address_space_end - 1; - } - constexpr bool IsKernel() const { - return is_kernel; - } - constexpr bool IsAslrEnabled() const { - return is_aslr_enabled; - } - - constexpr std::size_t GetNumGuardPages() const { - return IsKernel() ? 1 : 4; - } - - constexpr bool ContainsPages(VAddr addr, std::size_t num_pages) const { - return (address_space_start <= addr) && - (num_pages <= (address_space_end - address_space_start) / PageSize) && - (addr + num_pages * PageSize - 1 <= address_space_end - 1); - } - -private: - VAddr address_space_start{}; - VAddr address_space_end{}; - VAddr heap_region_start{}; - VAddr heap_region_end{}; - VAddr current_heap_end{}; - VAddr alias_region_start{}; - VAddr alias_region_end{}; - VAddr stack_region_start{}; - VAddr stack_region_end{}; - VAddr kernel_map_region_start{}; - VAddr kernel_map_region_end{}; - VAddr code_region_start{}; - VAddr code_region_end{}; - VAddr alias_code_region_start{}; - VAddr alias_code_region_end{}; - VAddr current_heap_addr{}; - - std::size_t heap_capacity{}; - std::size_t physical_memory_usage{}; - std::size_t max_heap_size{}; - std::size_t max_physical_memory_size{}; - std::size_t address_space_width{}; - - bool is_kernel{}; - bool is_aslr_enabled{}; - - KMemoryManager::Pool memory_pool{KMemoryManager::Pool::Application}; - - Common::PageTable page_table_impl; - - Core::System& system; -}; - -} // namespace Kernel::Memory |
