From 47b8160666da8dcb679bb7cabe35a615a1786155 Mon Sep 17 00:00:00 2001 From: bunnei Date: Mon, 5 Sep 2022 17:42:24 -0700 Subject: core: device_memory: Templatize GetPointer(..). --- src/core/hle/kernel/k_page_table.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/core/hle/kernel/k_page_table.cpp') diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index d975de844..8ebb75338 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -1648,7 +1648,7 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { // Clear all the newly allocated pages. for (const auto& it : pg.Nodes()) { - std::memset(system.DeviceMemory().GetPointer(it.GetAddress()), heap_fill_value, + std::memset(system.DeviceMemory().GetPointer(it.GetAddress()), heap_fill_value, it.GetSize()); } @@ -1805,9 +1805,9 @@ bool KPageTable::IsRegionMapped(VAddr address, u64 size) { } bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { - auto start_ptr = system.Memory().GetPointer(addr); + auto start_ptr = system.DeviceMemory().GetPointer(addr); for (u64 offset{}; offset < size; offset += PageSize) { - if (start_ptr != system.Memory().GetPointer(addr + offset)) { + if (start_ptr != system.DeviceMemory().GetPointer(addr + offset)) { return false; } start_ptr += PageSize; -- cgit v1.2.3 From ed591934fbffa32af0151302fd07e9fce776eb17 Mon Sep 17 00:00:00 2001 From: bunnei Date: Fri, 9 Sep 2022 21:38:28 -0700 Subject: core: hle: kernel: k_page_table: Update, and integrate with new KMemoryBlockManager/SlabManager. --- src/core/hle/kernel/k_page_table.cpp | 619 +++++++++++++++++++++-------------- 1 file changed, 379 insertions(+), 240 deletions(-) (limited to 'src/core/hle/kernel/k_page_table.cpp') diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 8ebb75338..2cf46af0a 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -49,6 +49,7 @@ KPageTable::~KPageTable() = default; Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, VAddr code_addr, std::size_t code_size, + KMemoryBlockSlabManager* mem_block_slab_manager, KMemoryManager::Pool pool) { const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) { @@ -113,6 +114,7 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type address_space_start = start; address_space_end = end; is_kernel = false; + memory_block_slab_manager = mem_block_slab_manager; // Determine the region we can place our undetermineds in VAddr alloc_start{}; @@ -254,7 +256,14 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type page_table_impl.Resize(address_space_width, PageBits); - return InitializeMemoryLayout(start, end); + return memory_block_manager.Initialize(address_space_start, address_space_end, + memory_block_slab_manager); +} + +void KPageTable::Finalize() { + memory_block_manager.Finalize(memory_block_slab_manager, [&](VAddr addr, u64 size) { + system.Memory().UnmapRegion(page_table_impl, addr, size); + }); } Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, @@ -271,6 +280,13 @@ Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryStat R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager); + + // Allocate and open. KPageGroup pg; R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( &pg, num_pages, @@ -278,7 +294,10 @@ Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryStat R_TRY(Operate(addr, num_pages, pg, OperationType::MapGroup)); - block_manager->Update(addr, num_pages, state, perm); + // Update the blocks. + memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } @@ -307,6 +326,18 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + // Create an update allocator for the source. + Result src_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator src_allocator( + std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); + R_TRY(src_allocator_result); + + // Create an update allocator for the destination. + Result dst_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator dst_allocator( + std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); + R_TRY(dst_allocator_result); + // Map the code memory. { // Determine the number of pages being operated on. @@ -335,10 +366,14 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size unprot_guard.Cancel(); // Apply the memory block updates. - block_manager->Update(src_address, num_pages, src_state, new_perm, - KMemoryAttribute::Locked); - block_manager->Update(dst_address, num_pages, KMemoryState::AliasCode, new_perm, - KMemoryAttribute::None); + memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, + src_state, new_perm, KMemoryAttribute::Locked, + KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); + memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::AliasCode, new_perm, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); } return ResultSuccess; @@ -370,7 +405,7 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si // Determine whether any pages being unmapped are code. bool any_code_pages = false; { - KMemoryBlockManager::const_iterator it = block_manager->FindIterator(dst_address); + KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(dst_address); while (true) { // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -408,6 +443,20 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si // Determine the number of pages being operated on. const std::size_t num_pages = size / PageSize; + // Create an update allocator for the source. + Result src_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), + memory_block_slab_manager, + num_src_allocator_blocks); + R_TRY(src_allocator_result); + + // Create an update allocator for the destination. + Result dst_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), + memory_block_slab_manager, + num_dst_allocator_blocks); + R_TRY(dst_allocator_result); + // Unmap the aliased copy of the pages. R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); @@ -416,9 +465,14 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si OperationType::ChangePermissions)); // Apply the memory block updates. - block_manager->Update(dst_address, num_pages, KMemoryState::None); - block_manager->Update(src_address, num_pages, KMemoryState::Normal, - KMemoryPermission::UserReadWrite); + memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::None, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); + memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, + KMemoryState::Normal, KMemoryPermission::UserReadWrite, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Locked); // Note that we reprotected pages. reprotected_pages = true; @@ -434,55 +488,12 @@ VAddr KPageTable::FindFreeArea(VAddr region_start, std::size_t region_num_pages, if (num_pages <= region_num_pages) { if (this->IsAslrEnabled()) { - // Try to directly find a free area up to 8 times. - for (std::size_t i = 0; i < 8; i++) { - const std::size_t random_offset = - KSystemControl::GenerateRandomRange( - 0, (region_num_pages - num_pages - guard_pages) * PageSize / alignment) * - alignment; - const VAddr candidate = - Common::AlignDown((region_start + random_offset), alignment) + offset; - - KMemoryInfo info = this->QueryInfoImpl(candidate); - - if (info.state != KMemoryState::Free) { - continue; - } - if (region_start > candidate) { - continue; - } - if (info.GetAddress() + guard_pages * PageSize > candidate) { - continue; - } - - const VAddr candidate_end = candidate + (num_pages + guard_pages) * PageSize - 1; - if (candidate_end > info.GetLastAddress()) { - continue; - } - if (candidate_end > region_start + region_num_pages * PageSize - 1) { - continue; - } - - address = candidate; - break; - } - // Fall back to finding the first free area with a random offset. - if (address == 0) { - // NOTE: Nintendo does not account for guard pages here. - // This may theoretically cause an offset to be chosen that cannot be mapped. We - // will account for guard pages. - const std::size_t offset_pages = KSystemControl::GenerateRandomRange( - 0, region_num_pages - num_pages - guard_pages); - address = block_manager->FindFreeArea(region_start + offset_pages * PageSize, - region_num_pages - offset_pages, num_pages, - alignment, offset, guard_pages); - } + UNIMPLEMENTED(); } - // Find the first free area. if (address == 0) { - address = block_manager->FindFreeArea(region_start, region_num_pages, num_pages, - alignment, offset, guard_pages); + address = memory_block_manager.FindFreeArea(region_start, region_num_pages, num_pages, + alignment, offset, guard_pages); } } @@ -649,11 +660,19 @@ Result KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTab KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); // Apply the memory block update. - block_manager->Update(dst_addr, num_pages, KMemoryState::Free, KMemoryPermission::None, - KMemoryAttribute::None); + memory_block_manager.Update(std::addressof(allocator), dst_addr, num_pages, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); system.InvalidateCpuInstructionCaches(); @@ -682,10 +701,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { cur_address = address; mapped_size = 0; - auto it = block_manager->FindIterator(cur_address); + auto it = memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -739,10 +758,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { size_t checked_mapped_size = 0; cur_address = address; - auto it = block_manager->FindIterator(cur_address); + auto it = memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -782,6 +801,14 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { } } + // Create an update allocator. + ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, + num_allocator_blocks); + R_TRY(allocator_result); + // Reset the current tracking address, and make sure we clean up on failure. cur_address = address; auto unmap_guard = detail::ScopeExit([&] { @@ -791,10 +818,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { // Iterate, unmapping the pages. cur_address = address; - auto it = block_manager->FindIterator(cur_address); + auto it = memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -830,10 +857,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { PAddr pg_phys_addr = pg_it->GetAddress(); size_t pg_pages = pg_it->GetNumPages(); - auto it = block_manager->FindIterator(cur_address); + auto it = memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -889,10 +916,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { mapped_physical_memory_size += (size - mapped_size); // Update the relevant memory blocks. - block_manager->Update(address, size / PageSize, KMemoryState::Free, - KMemoryPermission::None, KMemoryAttribute::None, - KMemoryState::Normal, KMemoryPermission::UserReadWrite, - KMemoryAttribute::None); + memory_block_manager.UpdateIfMatch( + std::addressof(allocator), address, size / PageSize, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, KMemoryState::Normal, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None); // Cancel our guard. unmap_guard.Cancel(); @@ -924,10 +951,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { cur_address = address; mapped_size = 0; - auto it = block_manager->FindIterator(cur_address); + auto it = memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -1022,6 +1049,13 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { } ASSERT(pg.GetNumPages() == mapped_size / PageSize); + // Create an update allocator. + ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Reset the current tracking address, and make sure we clean up on failure. cur_address = address; auto remap_guard = detail::ScopeExit([&] { @@ -1030,7 +1064,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { cur_address = address; // Iterate over the memory we unmapped. - auto it = block_manager->FindIterator(cur_address); + auto it = memory_block_manager.FindIterator(cur_address); auto pg_it = pg.Nodes().begin(); PAddr pg_phys_addr = pg_it->GetAddress(); size_t pg_pages = pg_it->GetNumPages(); @@ -1085,10 +1119,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { }); // Iterate over the memory, unmapping as we go. - auto it = block_manager->FindIterator(cur_address); + auto it = memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != block_manager->end()); + ASSERT(it != memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -1120,8 +1154,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size); // Update memory blocks. - block_manager->Update(address, size / PageSize, KMemoryState::Free, KMemoryPermission::None, - KMemoryAttribute::None); + memory_block_manager.Update(std::addressof(allocator), address, size / PageSize, + KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); // TODO(bunnei): This is a workaround until the next set of changes, where we add reference // counting for mapped pages. Until then, we must manually close the reference to the page @@ -1134,83 +1170,134 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { return ResultSuccess; } -Result KPageTable::MapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { +Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, std::size_t size) { + // Lock the table. KScopedLightLock lk(general_lock); - KMemoryState src_state{}; - CASCADE_CODE(CheckMemoryState( - &src_state, nullptr, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, - KMemoryState::FlagCanAlias, KMemoryPermission::All, KMemoryPermission::UserReadWrite, - KMemoryAttribute::Mask, KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); + // Validate that the source address's state is valid. + KMemoryState src_state; + size_t num_src_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(src_state), nullptr, nullptr, + std::addressof(num_src_allocator_blocks), src_address, size, + KMemoryState::FlagCanAlias, KMemoryState::FlagCanAlias, + KMemoryPermission::All, KMemoryPermission::UserReadWrite, + KMemoryAttribute::All, KMemoryAttribute::None)); - if (IsRegionMapped(dst_addr, size)) { - return ResultInvalidCurrentMemory; - } + // Validate that the dst address's state is valid. + size_t num_dst_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_dst_allocator_blocks), dst_address, size, + KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryAttribute::None)); + // Create an update allocator for the source. + Result src_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator src_allocator( + std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); + R_TRY(src_allocator_result); + + // Create an update allocator for the destination. + Result dst_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator dst_allocator( + std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); + R_TRY(dst_allocator_result); + + // Map the memory. KPageGroup page_linked_list; const std::size_t num_pages{size / PageSize}; + const KMemoryPermission new_src_perm = static_cast( + KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); + const KMemoryAttribute new_src_attr = KMemoryAttribute::Locked; - AddRegionToPages(src_addr, num_pages, page_linked_list); - + AddRegionToPages(src_address, num_pages, page_linked_list); { + // Reprotect the source as kernel-read/not mapped. auto block_guard = detail::ScopeExit([&] { - Operate(src_addr, num_pages, KMemoryPermission::UserReadWrite, + Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, OperationType::ChangePermissions); }); - - CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::None, - OperationType::ChangePermissions)); - CASCADE_CODE(MapPages(dst_addr, page_linked_list, KMemoryPermission::UserReadWrite)); + R_TRY(Operate(src_address, num_pages, new_src_perm, OperationType::ChangePermissions)); + R_TRY(MapPages(dst_address, page_linked_list, KMemoryPermission::UserReadWrite)); block_guard.Cancel(); } - block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::None, - KMemoryAttribute::Locked); - block_manager->Update(dst_addr, num_pages, KMemoryState::Stack, - KMemoryPermission::UserReadWrite); + // Apply the memory block updates. + memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, + new_src_perm, new_src_attr, + KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); + memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::Stack, KMemoryPermission::UserReadWrite, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } -Result KPageTable::UnmapMemory(VAddr dst_addr, VAddr src_addr, std::size_t size) { +Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, std::size_t size) { + // Lock the table. KScopedLightLock lk(general_lock); - KMemoryState src_state{}; - CASCADE_CODE(CheckMemoryState( - &src_state, nullptr, nullptr, nullptr, src_addr, size, KMemoryState::FlagCanAlias, - KMemoryState::FlagCanAlias, KMemoryPermission::All, KMemoryPermission::None, - KMemoryAttribute::Mask, KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); + // Validate that the source address's state is valid. + KMemoryState src_state; + size_t num_src_allocator_blocks; + R_TRY(this->CheckMemoryState( + std::addressof(src_state), nullptr, nullptr, std::addressof(num_src_allocator_blocks), + src_address, size, KMemoryState::FlagCanAlias, KMemoryState::FlagCanAlias, + KMemoryPermission::All, KMemoryPermission::NotMapped | KMemoryPermission::KernelRead, + KMemoryAttribute::All, KMemoryAttribute::Locked)); + + // Validate that the dst address's state is valid. + KMemoryPermission dst_perm; + size_t num_dst_allocator_blocks; + R_TRY(this->CheckMemoryState( + nullptr, std::addressof(dst_perm), nullptr, std::addressof(num_dst_allocator_blocks), + dst_address, size, KMemoryState::All, KMemoryState::Stack, KMemoryPermission::None, + KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + + // Create an update allocator for the source. + Result src_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator src_allocator( + std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); + R_TRY(src_allocator_result); - KMemoryPermission dst_perm{}; - CASCADE_CODE(CheckMemoryState(nullptr, &dst_perm, nullptr, nullptr, dst_addr, size, - KMemoryState::All, KMemoryState::Stack, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped)); + // Create an update allocator for the destination. + Result dst_allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator dst_allocator( + std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); + R_TRY(dst_allocator_result); KPageGroup src_pages; KPageGroup dst_pages; const std::size_t num_pages{size / PageSize}; - AddRegionToPages(src_addr, num_pages, src_pages); - AddRegionToPages(dst_addr, num_pages, dst_pages); + AddRegionToPages(src_address, num_pages, src_pages); + AddRegionToPages(dst_address, num_pages, dst_pages); if (!dst_pages.IsEqual(src_pages)) { return ResultInvalidMemoryRegion; } { - auto block_guard = detail::ScopeExit([&] { MapPages(dst_addr, dst_pages, dst_perm); }); + auto block_guard = detail::ScopeExit([&] { MapPages(dst_address, dst_pages, dst_perm); }); - CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); - CASCADE_CODE(Operate(src_addr, num_pages, KMemoryPermission::UserReadWrite, - OperationType::ChangePermissions)); + R_TRY(Operate(dst_address, num_pages, KMemoryPermission::None, OperationType::Unmap)); + R_TRY(Operate(src_address, num_pages, KMemoryPermission::UserReadWrite, + OperationType::ChangePermissions)); block_guard.Cancel(); } - block_manager->Update(src_addr, num_pages, src_state, KMemoryPermission::UserReadWrite); - block_manager->Update(dst_addr, num_pages, KMemoryState::Free); + // Apply the memory block updates. + memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Locked); + memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::None, KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); return ResultSuccess; } @@ -1254,11 +1341,18 @@ Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemory KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager); + // Map the pages. R_TRY(MapPages(address, page_linked_list, perm)); // Update the blocks. - block_manager->Update(address, num_pages, state, perm); + memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } @@ -1288,6 +1382,11 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t KMemoryAttribute::None, KMemoryAttribute::None) .IsSuccess()); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager); + // Perform mapping operation. if (is_pa_valid) { R_TRY(this->Operate(addr, num_pages, perm, OperationType::Map, phys_addr)); @@ -1296,7 +1395,9 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t } // Update the blocks. - block_manager->Update(addr, num_pages, state, perm); + memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); // We successfully mapped the pages. *out_addr = addr; @@ -1321,25 +1422,36 @@ Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { return ResultSuccess; } -Result KPageTable::UnmapPages(VAddr addr, KPageGroup& page_linked_list, KMemoryState state) { +Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state) { // Check that the unmap is in range. const std::size_t num_pages{page_linked_list.GetNumPages()}; const std::size_t size{num_pages * PageSize}; - R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. KScopedLightLock lk(general_lock); // Check the memory state. - R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, state, KMemoryPermission::None, + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, + KMemoryState::All, state, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Perform the unmap. - R_TRY(UnmapPages(addr, page_linked_list)); + R_TRY(UnmapPages(address, page_linked_list)); // Update the blocks. - block_manager->Update(addr, num_pages, state, KMemoryPermission::None); + memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); return ResultSuccess; } @@ -1359,11 +1471,20 @@ Result KPageTable::UnmapPages(VAddr address, std::size_t num_pages, KMemoryState KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Perform the unmap. R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); // Update the blocks. - block_manager->Update(address, num_pages, KMemoryState::Free, KMemoryPermission::None); + memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); return ResultSuccess; } @@ -1435,13 +1556,21 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, // Succeed if there's nothing to do. R_SUCCEED_IF(old_perm == new_perm && old_state == new_state); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Perform mapping operation. const auto operation = was_x ? OperationType::ChangePermissionsAndRefresh : OperationType::ChangePermissions; R_TRY(Operate(addr, num_pages, new_perm, operation)); // Update the blocks. - block_manager->Update(addr, num_pages, new_state, new_perm, KMemoryAttribute::None); + memory_block_manager.Update(std::addressof(allocator), addr, num_pages, new_state, new_perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); // Ensure cache coherency, if we're setting pages as executable. if (is_x) { @@ -1454,51 +1583,30 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, KMemoryInfo KPageTable::QueryInfoImpl(VAddr addr) { KScopedLightLock lk(general_lock); - return block_manager->FindBlock(addr).GetMemoryInfo(); + return memory_block_manager.FindBlock(addr)->GetMemoryInfo(); } KMemoryInfo KPageTable::QueryInfo(VAddr addr) { if (!Contains(addr, 1)) { - return {address_space_end, 0 - address_space_end, KMemoryState::Inaccessible, - KMemoryPermission::None, KMemoryAttribute::None, KMemoryPermission::None}; + return { + .m_address = address_space_end, + .m_size = 0 - address_space_end, + .m_state = static_cast(Svc::MemoryState::Inaccessible), + .m_device_disable_merge_left_count = 0, + .m_device_disable_merge_right_count = 0, + .m_ipc_lock_count = 0, + .m_device_use_count = 0, + .m_ipc_disable_merge_count = 0, + .m_permission = KMemoryPermission::None, + .m_attribute = KMemoryAttribute::None, + .m_original_permission = KMemoryPermission::None, + .m_disable_merge_attribute = KMemoryBlockDisableMergeAttribute::None, + }; } return QueryInfoImpl(addr); } -Result KPageTable::ReserveTransferMemory(VAddr addr, std::size_t size, KMemoryPermission perm) { - KScopedLightLock lk(general_lock); - - KMemoryState state{}; - KMemoryAttribute attribute{}; - - R_TRY(CheckMemoryState(&state, nullptr, &attribute, nullptr, addr, size, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryPermission::All, KMemoryPermission::UserReadWrite, - KMemoryAttribute::Mask, KMemoryAttribute::None, - KMemoryAttribute::IpcAndDeviceMapped)); - - block_manager->Update(addr, size / PageSize, state, perm, attribute | KMemoryAttribute::Locked); - - return ResultSuccess; -} - -Result KPageTable::ResetTransferMemory(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryState state{}; - - R_TRY(CheckMemoryState(&state, nullptr, nullptr, nullptr, addr, size, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryState::FlagCanTransfer | KMemoryState::FlagReferenceCounted, - KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::Locked, KMemoryAttribute::IpcAndDeviceMapped)); - - block_manager->Update(addr, size / PageSize, state, KMemoryPermission::UserReadWrite); - return ResultSuccess; -} - Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; @@ -1509,20 +1617,30 @@ Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, // Verify we can change the memory permission. KMemoryState old_state; KMemoryPermission old_perm; - R_TRY(this->CheckMemoryState( - std::addressof(old_state), std::addressof(old_perm), nullptr, nullptr, addr, size, - KMemoryState::FlagCanReprotect, KMemoryState::FlagCanReprotect, KMemoryPermission::None, - KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::None)); + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(old_state), std::addressof(old_perm), nullptr, + std::addressof(num_allocator_blocks), addr, size, + KMemoryState::FlagCanReprotect, KMemoryState::FlagCanReprotect, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::All, KMemoryAttribute::None)); // Determine new perm. const KMemoryPermission new_perm = ConvertToKMemoryPermission(svc_perm); R_SUCCEED_IF(old_perm == new_perm); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Perform mapping operation. R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); // Update the blocks. - block_manager->Update(addr, num_pages, old_state, new_perm, KMemoryAttribute::None); + memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } @@ -1548,6 +1666,12 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u3 KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, AttributeTestMask, KMemoryAttribute::None, ~AttributeTestMask)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Determine the new attribute. const KMemoryAttribute new_attr = static_cast(((old_attr & static_cast(~mask)) | @@ -1557,7 +1681,9 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u3 this->Operate(addr, num_pages, old_perm, OperationType::ChangePermissionsAndRefresh); // Update the blocks. - block_manager->Update(addr, num_pages, old_state, old_perm, new_attr); + memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, old_perm, + new_attr, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } @@ -1603,6 +1729,12 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator( + std::addressof(allocator_result), memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Unmap the end of the heap. const auto num_pages = (GetHeapSize() - size) / PageSize; R_TRY(Operate(heap_region_start + size, num_pages, KMemoryPermission::None, @@ -1613,8 +1745,12 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { LimitableResource::PhysicalMemory, num_pages * PageSize); // Apply the memory block update. - block_manager->Update(heap_region_start + size, num_pages, KMemoryState::Free, - KMemoryPermission::None, KMemoryAttribute::None); + memory_block_manager.Update(std::addressof(allocator), heap_region_start + size, + num_pages, KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + size == 0 ? KMemoryBlockDisableMergeAttribute::Normal + : KMemoryBlockDisableMergeAttribute::None); // Update the current heap end. current_heap_end = heap_region_start + size; @@ -1667,6 +1803,12 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator( + std::addressof(allocator_result), memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Map the pages. const auto num_pages = allocation_size / PageSize; R_TRY(Operate(current_heap_end, num_pages, pg, OperationType::MapGroup)); @@ -1681,8 +1823,12 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { memory_reservation.Commit(); // Apply the memory block update. - block_manager->Update(current_heap_end, num_pages, KMemoryState::Normal, - KMemoryPermission::UserReadWrite, KMemoryAttribute::None); + memory_block_manager.Update( + std::addressof(allocator), current_heap_end, num_pages, KMemoryState::Normal, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + heap_region_start == current_heap_end ? KMemoryBlockDisableMergeAttribute::Normal + : KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); // Update the current heap end. current_heap_end = heap_region_start + size; @@ -1713,6 +1859,11 @@ ResultVal KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, return ResultOutOfMemory; } + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager); + if (is_map_only) { R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); } else { @@ -1723,53 +1874,38 @@ ResultVal KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); } - block_manager->Update(addr, needed_num_pages, state, perm); + // Update the blocks. + memory_block_manager.Update(std::addressof(allocator), addr, needed_num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return addr; } -Result KPageTable::LockForDeviceAddressSpace(VAddr addr, std::size_t size) { - KScopedLightLock lk(general_lock); - - KMemoryPermission perm{}; - if (const Result result{CheckMemoryState( - nullptr, &perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, - KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, - KMemoryAttribute::DeviceSharedAndUncached)}; - result.IsError()) { - return result; - } - - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->ShareToDevice(permission); - }, - perm); - - return ResultSuccess; -} +Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, std::size_t size) { + // Lightly validate the range before doing anything else. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); -Result KPageTable::UnlockForDeviceAddressSpace(VAddr addr, std::size_t size) { + // Lock the table. KScopedLightLock lk(general_lock); - KMemoryPermission perm{}; - if (const Result result{CheckMemoryState( - nullptr, &perm, nullptr, nullptr, addr, size, KMemoryState::FlagCanChangeAttribute, - KMemoryState::FlagCanChangeAttribute, KMemoryPermission::None, KMemoryPermission::None, - KMemoryAttribute::LockedAndIpcLocked, KMemoryAttribute::None, - KMemoryAttribute::DeviceSharedAndUncached)}; - result.IsError()) { - return result; - } + // Check the memory state. + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryStateContiguous( + std::addressof(num_allocator_blocks), address, size, KMemoryState::FlagCanDeviceMap, + KMemoryState::FlagCanDeviceMap, KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked, KMemoryAttribute::DeviceShared)); - block_manager->UpdateLock( - addr, size / PageSize, - [](KMemoryBlockManager::iterator block, KMemoryPermission permission) { - block->UnshareToDevice(permission); - }, - perm); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // Update the memory blocks. + memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, + &KMemoryBlock::UnshareToDevice, KMemoryPermission::None); return ResultSuccess; } @@ -1791,19 +1927,6 @@ Result KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size, const KPage KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg); } -Result KPageTable::InitializeMemoryLayout(VAddr start, VAddr end) { - block_manager = std::make_unique(start, end); - - return ResultSuccess; -} - -bool KPageTable::IsRegionMapped(VAddr address, u64 size) { - return CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, - KMemoryPermission::All, KMemoryPermission::None, KMemoryAttribute::Mask, - KMemoryAttribute::None, KMemoryAttribute::IpcAndDeviceMapped) - .IsError(); -} - bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { auto start_ptr = system.DeviceMemory().GetPointer(addr); for (u64 offset{}; offset < size; offset += PageSize) { @@ -1831,8 +1954,8 @@ VAddr KPageTable::AllocateVirtualMemory(VAddr start, std::size_t region_num_page if (is_aslr_enabled) { UNIMPLEMENTED(); } - return block_manager->FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, - IsKernel() ? 1 : 4); + return memory_block_manager.FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, + IsKernel() ? 1 : 4); } Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& page_group, @@ -2008,9 +2131,9 @@ Result KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_ KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr) const { // Validate the states match expectation. - R_UNLESS((info.state & state_mask) == state, ResultInvalidCurrentMemory); - R_UNLESS((info.perm & perm_mask) == perm, ResultInvalidCurrentMemory); - R_UNLESS((info.attribute & attr_mask) == attr, ResultInvalidCurrentMemory); + R_UNLESS((info.m_state & state_mask) == state, ResultInvalidCurrentMemory); + R_UNLESS((info.m_permission & perm_mask) == perm, ResultInvalidCurrentMemory); + R_UNLESS((info.m_attribute & attr_mask) == attr, ResultInvalidCurrentMemory); return ResultSuccess; } @@ -2024,7 +2147,7 @@ Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VA // Get information about the first block. const VAddr last_addr = addr + size - 1; - KMemoryBlockManager::const_iterator it = block_manager->FindIterator(addr); + KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(addr); KMemoryInfo info = it->GetMemoryInfo(); // If the start address isn't aligned, we need a block. @@ -2042,7 +2165,7 @@ Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VA // Advance our iterator. it++; - ASSERT(it != block_manager->cend()); + ASSERT(it != memory_block_manager.cend()); info = it->GetMemoryInfo(); } @@ -2067,7 +2190,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* // Get information about the first block. const VAddr last_addr = addr + size - 1; - KMemoryBlockManager::const_iterator it = block_manager->FindIterator(addr); + KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(addr); KMemoryInfo info = it->GetMemoryInfo(); // If the start address isn't aligned, we need a block. @@ -2075,14 +2198,14 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* (Common::AlignDown(addr, PageSize) != info.GetAddress()) ? 1 : 0; // Validate all blocks in the range have correct state. - const KMemoryState first_state = info.state; - const KMemoryPermission first_perm = info.perm; - const KMemoryAttribute first_attr = info.attribute; + const KMemoryState first_state = info.m_state; + const KMemoryPermission first_perm = info.m_permission; + const KMemoryAttribute first_attr = info.m_attribute; while (true) { // Validate the current block. - R_UNLESS(info.state == first_state, ResultInvalidCurrentMemory); - R_UNLESS(info.perm == first_perm, ResultInvalidCurrentMemory); - R_UNLESS((info.attribute | ignore_attr) == (first_attr | ignore_attr), + R_UNLESS(info.m_state == first_state, ResultInvalidCurrentMemory); + R_UNLESS(info.m_permission == first_perm, ResultInvalidCurrentMemory); + R_UNLESS((info.m_attribute | ignore_attr) == (first_attr | ignore_attr), ResultInvalidCurrentMemory); // Validate against the provided masks. @@ -2095,7 +2218,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* // Advance our iterator. it++; - ASSERT(it != block_manager->cend()); + ASSERT(it != memory_block_manager.cend()); info = it->GetMemoryInfo(); } @@ -2162,6 +2285,12 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr R_TRY(this->MakePageGroup(*out_pg, addr, num_pages)); } + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Decide on new perm and attr. new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; KMemoryAttribute new_attr = static_cast(old_attr | lock_attr); @@ -2172,7 +2301,9 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr } // Apply the memory block updates. - block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + new_attr, KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } @@ -2213,13 +2344,21 @@ Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask new_perm = (new_perm != KMemoryPermission::None) ? new_perm : old_perm; KMemoryAttribute new_attr = static_cast(old_attr & ~lock_attr); + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + // Update permission, if we need to. if (new_perm != old_perm) { R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); } // Apply the memory block updates. - block_manager->Update(addr, num_pages, old_state, new_perm, new_attr); + memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + new_attr, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Locked); return ResultSuccess; } -- cgit v1.2.3 From ff26190d422599ded0166cba686e7456c59163a5 Mon Sep 17 00:00:00 2001 From: bunnei Date: Sat, 1 Oct 2022 14:08:47 -0700 Subject: core: hle: kernel: k_page_table: Impl. LockForUn/MapDeviceAddressSpace, cleanup. --- src/core/hle/kernel/k_page_table.cpp | 881 +++++++++++++++++++---------------- 1 file changed, 476 insertions(+), 405 deletions(-) (limited to 'src/core/hle/kernel/k_page_table.cpp') diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 2cf46af0a..fcffc0b88 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -25,7 +25,7 @@ namespace { using namespace Common::Literals; -constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType as_type) { +constexpr size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType as_type) { switch (as_type) { case FileSys::ProgramAddressSpaceType::Is32Bit: case FileSys::ProgramAddressSpaceType::Is32BitNoMap: @@ -43,28 +43,29 @@ constexpr std::size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceT } // namespace KPageTable::KPageTable(Core::System& system_) - : general_lock{system_.Kernel()}, map_physical_memory_lock{system_.Kernel()}, system{system_} {} + : m_general_lock{system_.Kernel()}, + m_map_physical_memory_lock{system_.Kernel()}, m_system{system_} {} KPageTable::~KPageTable() = default; Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type, bool enable_aslr, - VAddr code_addr, std::size_t code_size, + VAddr code_addr, size_t code_size, KMemoryBlockSlabManager* mem_block_slab_manager, KMemoryManager::Pool pool) { const auto GetSpaceStart = [this](KAddressSpaceInfo::Type type) { - return KAddressSpaceInfo::GetAddressSpaceStart(address_space_width, type); + return KAddressSpaceInfo::GetAddressSpaceStart(m_address_space_width, type); }; const auto GetSpaceSize = [this](KAddressSpaceInfo::Type type) { - return KAddressSpaceInfo::GetAddressSpaceSize(address_space_width, type); + return KAddressSpaceInfo::GetAddressSpaceSize(m_address_space_width, type); }; // Set our width and heap/alias sizes - address_space_width = GetAddressSpaceWidthFromType(as_type); + m_address_space_width = GetAddressSpaceWidthFromType(as_type); const VAddr start = 0; - const VAddr end{1ULL << address_space_width}; - std::size_t alias_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Alias)}; - std::size_t heap_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Heap)}; + const VAddr end{1ULL << m_address_space_width}; + size_t alias_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Alias)}; + size_t heap_region_size{GetSpaceSize(KAddressSpaceInfo::Type::Heap)}; ASSERT(code_addr < code_addr + code_size); ASSERT(code_addr + code_size - 1 <= end - 1); @@ -76,67 +77,68 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type } // Set code regions and determine remaining - constexpr std::size_t RegionAlignment{2_MiB}; + constexpr size_t RegionAlignment{2_MiB}; VAddr process_code_start{}; VAddr process_code_end{}; - std::size_t stack_region_size{}; - std::size_t kernel_map_region_size{}; + size_t stack_region_size{}; + size_t kernel_map_region_size{}; - if (address_space_width == 39) { + if (m_address_space_width == 39) { alias_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Alias); heap_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Heap); stack_region_size = GetSpaceSize(KAddressSpaceInfo::Type::Stack); kernel_map_region_size = GetSpaceSize(KAddressSpaceInfo::Type::MapSmall); - code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::Map39Bit); - code_region_end = code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::Map39Bit); - alias_code_region_start = code_region_start; - alias_code_region_end = code_region_end; + m_code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::Map39Bit); + m_code_region_end = m_code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::Map39Bit); + m_alias_code_region_start = m_code_region_start; + m_alias_code_region_end = m_code_region_end; process_code_start = Common::AlignDown(code_addr, RegionAlignment); process_code_end = Common::AlignUp(code_addr + code_size, RegionAlignment); } else { stack_region_size = 0; kernel_map_region_size = 0; - code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::MapSmall); - code_region_end = code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::MapSmall); - stack_region_start = code_region_start; - alias_code_region_start = code_region_start; - alias_code_region_end = GetSpaceStart(KAddressSpaceInfo::Type::MapLarge) + - GetSpaceSize(KAddressSpaceInfo::Type::MapLarge); - stack_region_end = code_region_end; - kernel_map_region_start = code_region_start; - kernel_map_region_end = code_region_end; - process_code_start = code_region_start; - process_code_end = code_region_end; + m_code_region_start = GetSpaceStart(KAddressSpaceInfo::Type::MapSmall); + m_code_region_end = m_code_region_start + GetSpaceSize(KAddressSpaceInfo::Type::MapSmall); + m_stack_region_start = m_code_region_start; + m_alias_code_region_start = m_code_region_start; + m_alias_code_region_end = GetSpaceStart(KAddressSpaceInfo::Type::MapLarge) + + GetSpaceSize(KAddressSpaceInfo::Type::MapLarge); + m_stack_region_end = m_code_region_end; + m_kernel_map_region_start = m_code_region_start; + m_kernel_map_region_end = m_code_region_end; + process_code_start = m_code_region_start; + process_code_end = m_code_region_end; } // Set other basic fields - is_aslr_enabled = enable_aslr; - address_space_start = start; - address_space_end = end; - is_kernel = false; - memory_block_slab_manager = mem_block_slab_manager; + m_enable_aslr = enable_aslr; + m_enable_device_address_space_merge = false; + m_address_space_start = start; + m_address_space_end = end; + m_is_kernel = false; + m_memory_block_slab_manager = mem_block_slab_manager; // Determine the region we can place our undetermineds in VAddr alloc_start{}; - std::size_t alloc_size{}; - if ((process_code_start - code_region_start) >= (end - process_code_end)) { - alloc_start = code_region_start; - alloc_size = process_code_start - code_region_start; + size_t alloc_size{}; + if ((process_code_start - m_code_region_start) >= (end - process_code_end)) { + alloc_start = m_code_region_start; + alloc_size = process_code_start - m_code_region_start; } else { alloc_start = process_code_end; alloc_size = end - process_code_end; } - const std::size_t needed_size{ + const size_t needed_size{ (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size)}; if (alloc_size < needed_size) { ASSERT(false); return ResultOutOfMemory; } - const std::size_t remaining_size{alloc_size - needed_size}; + const size_t remaining_size{alloc_size - needed_size}; // Determine random placements for each region - std::size_t alias_rnd{}, heap_rnd{}, stack_rnd{}, kmap_rnd{}; + size_t alias_rnd{}, heap_rnd{}, stack_rnd{}, kmap_rnd{}; if (enable_aslr) { alias_rnd = KSystemControl::GenerateRandomRange(0, remaining_size / RegionAlignment) * RegionAlignment; @@ -149,124 +151,124 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type } // Setup heap and alias regions - alias_region_start = alloc_start + alias_rnd; - alias_region_end = alias_region_start + alias_region_size; - heap_region_start = alloc_start + heap_rnd; - heap_region_end = heap_region_start + heap_region_size; + m_alias_region_start = alloc_start + alias_rnd; + m_alias_region_end = m_alias_region_start + alias_region_size; + m_heap_region_start = alloc_start + heap_rnd; + m_heap_region_end = m_heap_region_start + heap_region_size; if (alias_rnd <= heap_rnd) { - heap_region_start += alias_region_size; - heap_region_end += alias_region_size; + m_heap_region_start += alias_region_size; + m_heap_region_end += alias_region_size; } else { - alias_region_start += heap_region_size; - alias_region_end += heap_region_size; + m_alias_region_start += heap_region_size; + m_alias_region_end += heap_region_size; } // Setup stack region if (stack_region_size) { - stack_region_start = alloc_start + stack_rnd; - stack_region_end = stack_region_start + stack_region_size; + m_stack_region_start = alloc_start + stack_rnd; + m_stack_region_end = m_stack_region_start + stack_region_size; if (alias_rnd < stack_rnd) { - stack_region_start += alias_region_size; - stack_region_end += alias_region_size; + m_stack_region_start += alias_region_size; + m_stack_region_end += alias_region_size; } else { - alias_region_start += stack_region_size; - alias_region_end += stack_region_size; + m_alias_region_start += stack_region_size; + m_alias_region_end += stack_region_size; } if (heap_rnd < stack_rnd) { - stack_region_start += heap_region_size; - stack_region_end += heap_region_size; + m_stack_region_start += heap_region_size; + m_stack_region_end += heap_region_size; } else { - heap_region_start += stack_region_size; - heap_region_end += stack_region_size; + m_heap_region_start += stack_region_size; + m_heap_region_end += stack_region_size; } } // Setup kernel map region if (kernel_map_region_size) { - kernel_map_region_start = alloc_start + kmap_rnd; - kernel_map_region_end = kernel_map_region_start + kernel_map_region_size; + m_kernel_map_region_start = alloc_start + kmap_rnd; + m_kernel_map_region_end = m_kernel_map_region_start + kernel_map_region_size; if (alias_rnd < kmap_rnd) { - kernel_map_region_start += alias_region_size; - kernel_map_region_end += alias_region_size; + m_kernel_map_region_start += alias_region_size; + m_kernel_map_region_end += alias_region_size; } else { - alias_region_start += kernel_map_region_size; - alias_region_end += kernel_map_region_size; + m_alias_region_start += kernel_map_region_size; + m_alias_region_end += kernel_map_region_size; } if (heap_rnd < kmap_rnd) { - kernel_map_region_start += heap_region_size; - kernel_map_region_end += heap_region_size; + m_kernel_map_region_start += heap_region_size; + m_kernel_map_region_end += heap_region_size; } else { - heap_region_start += kernel_map_region_size; - heap_region_end += kernel_map_region_size; + m_heap_region_start += kernel_map_region_size; + m_heap_region_end += kernel_map_region_size; } if (stack_region_size) { if (stack_rnd < kmap_rnd) { - kernel_map_region_start += stack_region_size; - kernel_map_region_end += stack_region_size; + m_kernel_map_region_start += stack_region_size; + m_kernel_map_region_end += stack_region_size; } else { - stack_region_start += kernel_map_region_size; - stack_region_end += kernel_map_region_size; + m_stack_region_start += kernel_map_region_size; + m_stack_region_end += kernel_map_region_size; } } } // Set heap members - current_heap_end = heap_region_start; - max_heap_size = 0; - max_physical_memory_size = 0; + m_current_heap_end = m_heap_region_start; + m_max_heap_size = 0; + m_max_physical_memory_size = 0; // Ensure that we regions inside our address space auto IsInAddressSpace = [&](VAddr addr) { - return address_space_start <= addr && addr <= address_space_end; + return m_address_space_start <= addr && addr <= m_address_space_end; }; - ASSERT(IsInAddressSpace(alias_region_start)); - ASSERT(IsInAddressSpace(alias_region_end)); - ASSERT(IsInAddressSpace(heap_region_start)); - ASSERT(IsInAddressSpace(heap_region_end)); - ASSERT(IsInAddressSpace(stack_region_start)); - ASSERT(IsInAddressSpace(stack_region_end)); - ASSERT(IsInAddressSpace(kernel_map_region_start)); - ASSERT(IsInAddressSpace(kernel_map_region_end)); + ASSERT(IsInAddressSpace(m_alias_region_start)); + ASSERT(IsInAddressSpace(m_alias_region_end)); + ASSERT(IsInAddressSpace(m_heap_region_start)); + ASSERT(IsInAddressSpace(m_heap_region_end)); + ASSERT(IsInAddressSpace(m_stack_region_start)); + ASSERT(IsInAddressSpace(m_stack_region_end)); + ASSERT(IsInAddressSpace(m_kernel_map_region_start)); + ASSERT(IsInAddressSpace(m_kernel_map_region_end)); // Ensure that we selected regions that don't overlap - const VAddr alias_start{alias_region_start}; - const VAddr alias_last{alias_region_end - 1}; - const VAddr heap_start{heap_region_start}; - const VAddr heap_last{heap_region_end - 1}; - const VAddr stack_start{stack_region_start}; - const VAddr stack_last{stack_region_end - 1}; - const VAddr kmap_start{kernel_map_region_start}; - const VAddr kmap_last{kernel_map_region_end - 1}; + const VAddr alias_start{m_alias_region_start}; + const VAddr alias_last{m_alias_region_end - 1}; + const VAddr heap_start{m_heap_region_start}; + const VAddr heap_last{m_heap_region_end - 1}; + const VAddr stack_start{m_stack_region_start}; + const VAddr stack_last{m_stack_region_end - 1}; + const VAddr kmap_start{m_kernel_map_region_start}; + const VAddr kmap_last{m_kernel_map_region_end - 1}; ASSERT(alias_last < heap_start || heap_last < alias_start); ASSERT(alias_last < stack_start || stack_last < alias_start); ASSERT(alias_last < kmap_start || kmap_last < alias_start); ASSERT(heap_last < stack_start || stack_last < heap_start); ASSERT(heap_last < kmap_start || kmap_last < heap_start); - current_heap_end = heap_region_start; - max_heap_size = 0; - mapped_physical_memory_size = 0; - memory_pool = pool; + m_current_heap_end = m_heap_region_start; + m_max_heap_size = 0; + m_mapped_physical_memory_size = 0; + m_memory_pool = pool; - page_table_impl.Resize(address_space_width, PageBits); + m_page_table_impl.Resize(m_address_space_width, PageBits); - return memory_block_manager.Initialize(address_space_start, address_space_end, - memory_block_slab_manager); + return m_memory_block_manager.Initialize(m_address_space_start, m_address_space_end, + m_memory_block_slab_manager); } void KPageTable::Finalize() { - memory_block_manager.Finalize(memory_block_slab_manager, [&](VAddr addr, u64 size) { - system.Memory().UnmapRegion(page_table_impl, addr, size); + m_memory_block_manager.Finalize(m_memory_block_slab_manager, [&](VAddr addr, u64 size) { + m_system.Memory().UnmapRegion(m_page_table_impl, addr, size); }); } -Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryState state, +Result KPageTable::MapProcessCode(VAddr addr, size_t num_pages, KMemoryState state, KMemoryPermission perm) { const u64 size{num_pages * PageSize}; @@ -274,7 +276,7 @@ Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryStat R_UNLESS(this->CanContain(addr, size, state), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify that the destination memory is unmapped. R_TRY(this->CheckMemoryState(addr, size, KMemoryState::All, KMemoryState::Free, @@ -284,43 +286,43 @@ Result KPageTable::MapProcessCode(VAddr addr, std::size_t num_pages, KMemoryStat // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager); + m_memory_block_slab_manager); // Allocate and open. KPageGroup pg; - R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( + R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpen( &pg, num_pages, - KMemoryManager::EncodeOption(KMemoryManager::Pool::Application, allocation_option))); + KMemoryManager::EncodeOption(KMemoryManager::Pool::Application, m_allocation_option))); R_TRY(Operate(addr, num_pages, pg, OperationType::MapGroup)); // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } -Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size) { +Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t size) { // Validate the mapping request. R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), ResultInvalidMemoryRegion); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify that the source memory is normal heap. KMemoryState src_state{}; KMemoryPermission src_perm{}; - std::size_t num_src_allocator_blocks{}; + size_t num_src_allocator_blocks{}; R_TRY(this->CheckMemoryState(&src_state, &src_perm, nullptr, &num_src_allocator_blocks, src_address, size, KMemoryState::All, KMemoryState::Normal, KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)); // Verify that the destination memory is unmapped. - std::size_t num_dst_allocator_blocks{}; + size_t num_dst_allocator_blocks{}; R_TRY(this->CheckMemoryState(&num_dst_allocator_blocks, dst_address, size, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, @@ -328,20 +330,22 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size // Create an update allocator for the source. Result src_allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator src_allocator( - std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); + KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), + m_memory_block_slab_manager, + num_src_allocator_blocks); R_TRY(src_allocator_result); // Create an update allocator for the destination. Result dst_allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator dst_allocator( - std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); + KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), + m_memory_block_slab_manager, + num_dst_allocator_blocks); R_TRY(dst_allocator_result); // Map the code memory. { // Determine the number of pages being operated on. - const std::size_t num_pages = size / PageSize; + const size_t num_pages = size / PageSize; // Create page groups for the memory being mapped. KPageGroup pg; @@ -366,37 +370,37 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, std::size unprot_guard.Cancel(); // Apply the memory block updates. - memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, - src_state, new_perm, KMemoryAttribute::Locked, - KMemoryBlockDisableMergeAttribute::Locked, - KMemoryBlockDisableMergeAttribute::None); - memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, - KMemoryState::AliasCode, new_perm, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, + src_state, new_perm, KMemoryAttribute::Locked, + KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::AliasCode, new_perm, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); } return ResultSuccess; } -Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::size_t size, +Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, size_t size, ICacheInvalidationStrategy icache_invalidation_strategy) { // Validate the mapping request. R_UNLESS(this->CanContain(dst_address, size, KMemoryState::AliasCode), ResultInvalidMemoryRegion); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify that the source memory is locked normal heap. - std::size_t num_src_allocator_blocks{}; + size_t num_src_allocator_blocks{}; R_TRY(this->CheckMemoryState(std::addressof(num_src_allocator_blocks), src_address, size, KMemoryState::All, KMemoryState::Normal, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, KMemoryAttribute::Locked)); // Verify that the destination memory is aliasable code. - std::size_t num_dst_allocator_blocks{}; + size_t num_dst_allocator_blocks{}; R_TRY(this->CheckMemoryStateContiguous( std::addressof(num_dst_allocator_blocks), dst_address, size, KMemoryState::FlagCanCodeAlias, KMemoryState::FlagCanCodeAlias, KMemoryPermission::None, KMemoryPermission::None, @@ -405,7 +409,7 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si // Determine whether any pages being unmapped are code. bool any_code_pages = false; { - KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(dst_address); + KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(dst_address); while (true) { // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -431,9 +435,9 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si SCOPE_EXIT({ if (reprotected_pages && any_code_pages) { if (icache_invalidation_strategy == ICacheInvalidationStrategy::InvalidateRange) { - system.InvalidateCpuInstructionCacheRange(dst_address, size); + m_system.InvalidateCpuInstructionCacheRange(dst_address, size); } else { - system.InvalidateCpuInstructionCaches(); + m_system.InvalidateCpuInstructionCaches(); } } }); @@ -441,19 +445,19 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si // Unmap. { // Determine the number of pages being operated on. - const std::size_t num_pages = size / PageSize; + const size_t num_pages = size / PageSize; // Create an update allocator for the source. Result src_allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), - memory_block_slab_manager, + m_memory_block_slab_manager, num_src_allocator_blocks); R_TRY(src_allocator_result); // Create an update allocator for the destination. Result dst_allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), - memory_block_slab_manager, + m_memory_block_slab_manager, num_dst_allocator_blocks); R_TRY(dst_allocator_result); @@ -465,14 +469,14 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si OperationType::ChangePermissions)); // Apply the memory block updates. - memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, - KMemoryState::None, KMemoryPermission::None, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Normal); - memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, - KMemoryState::Normal, KMemoryPermission::UserReadWrite, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Locked); + m_memory_block_manager.Update( + std::addressof(dst_allocator), dst_address, num_pages, KMemoryState::None, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Normal); + m_memory_block_manager.Update( + std::addressof(src_allocator), src_address, num_pages, KMemoryState::Normal, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Locked); // Note that we reprotected pages. reprotected_pages = true; @@ -481,9 +485,8 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, std::si return ResultSuccess; } -VAddr KPageTable::FindFreeArea(VAddr region_start, std::size_t region_num_pages, - std::size_t num_pages, std::size_t alignment, std::size_t offset, - std::size_t guard_pages) { +VAddr KPageTable::FindFreeArea(VAddr region_start, size_t region_num_pages, size_t num_pages, + size_t alignment, size_t offset, size_t guard_pages) { VAddr address = 0; if (num_pages <= region_num_pages) { @@ -492,8 +495,8 @@ VAddr KPageTable::FindFreeArea(VAddr region_start, std::size_t region_num_pages, } // Find the first free area. if (address == 0) { - address = memory_block_manager.FindFreeArea(region_start, region_num_pages, num_pages, - alignment, offset, guard_pages); + address = m_memory_block_manager.FindFreeArea(region_start, region_num_pages, num_pages, + alignment, offset, guard_pages); } } @@ -511,7 +514,8 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - R_UNLESS(page_table_impl.BeginTraversal(next_entry, context, addr), ResultInvalidCurrentMemory); + R_UNLESS(m_page_table_impl.BeginTraversal(next_entry, context, addr), + ResultInvalidCurrentMemory); // Prepare tracking variables. PAddr cur_addr = next_entry.phys_addr; @@ -519,9 +523,9 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { size_t tot_size = cur_size; // Iterate, adding to group as we go. - const auto& memory_layout = system.Kernel().MemoryLayout(); + const auto& memory_layout = m_system.Kernel().MemoryLayout(); while (tot_size < size) { - R_UNLESS(page_table_impl.ContinueTraversal(next_entry, context), + R_UNLESS(m_page_table_impl.ContinueTraversal(next_entry, context), ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -557,7 +561,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu const size_t size = num_pages * PageSize; const auto& pg = pg_ll.Nodes(); - const auto& memory_layout = system.Kernel().MemoryLayout(); + const auto& memory_layout = m_system.Kernel().MemoryLayout(); // Empty groups are necessarily invalid. if (pg.empty()) { @@ -584,7 +588,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - if (!page_table_impl.BeginTraversal(next_entry, context, addr)) { + if (!m_page_table_impl.BeginTraversal(next_entry, context, addr)) { return false; } @@ -595,7 +599,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu // Iterate, comparing expected to actual. while (tot_size < size) { - if (!page_table_impl.ContinueTraversal(next_entry, context)) { + if (!m_page_table_impl.ContinueTraversal(next_entry, context)) { return false; } @@ -641,11 +645,11 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu return cur_block_address == cur_addr && cur_block_pages == (cur_size / PageSize); } -Result KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTable& src_page_table, +Result KPageTable::UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& src_page_table, VAddr src_addr) { - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); - const std::size_t num_pages{size / PageSize}; + const size_t num_pages{size / PageSize}; // Check that the memory is mapped in the destination process. size_t num_allocator_blocks; @@ -663,48 +667,48 @@ Result KPageTable::UnmapProcessMemory(VAddr dst_addr, std::size_t size, KPageTab // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); CASCADE_CODE(Operate(dst_addr, num_pages, KMemoryPermission::None, OperationType::Unmap)); // Apply the memory block update. - memory_block_manager.Update(std::addressof(allocator), dst_addr, num_pages, KMemoryState::Free, - KMemoryPermission::None, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Normal); + m_memory_block_manager.Update(std::addressof(allocator), dst_addr, num_pages, + KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); - system.InvalidateCpuInstructionCaches(); + m_system.InvalidateCpuInstructionCaches(); return ResultSuccess; } -Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { +Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Lock the physical memory lock. - KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); + KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); // Calculate the last address for convenience. const VAddr last_address = address + size - 1; // Define iteration variables. VAddr cur_address; - std::size_t mapped_size; + size_t mapped_size; // The entire mapping process can be retried. while (true) { // Check if the memory is already mapped. { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Iterate over the memory. cur_address = address; mapped_size = 0; - auto it = memory_block_manager.FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != memory_block_manager.end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -735,20 +739,20 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { { // Reserve the memory from the process resource limit. KScopedResourceReservation memory_reservation( - system.Kernel().CurrentProcess()->GetResourceLimit(), + m_system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, size - mapped_size); R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); // Allocate pages for the new memory. KPageGroup pg; - R_TRY(system.Kernel().MemoryManager().AllocateAndOpenForProcess( + R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpenForProcess( &pg, (size - mapped_size) / PageSize, - KMemoryManager::EncodeOption(memory_pool, allocation_option), 0, 0)); + KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option), 0, 0)); // Map the memory. { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); size_t num_allocator_blocks = 0; @@ -758,10 +762,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { size_t checked_mapped_size = 0; cur_address = address; - auto it = memory_block_manager.FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != memory_block_manager.end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -805,7 +809,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); @@ -818,10 +822,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { // Iterate, unmapping the pages. cur_address = address; - auto it = memory_block_manager.FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != memory_block_manager.end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -857,10 +861,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { PAddr pg_phys_addr = pg_it->GetAddress(); size_t pg_pages = pg_it->GetNumPages(); - auto it = memory_block_manager.FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != memory_block_manager.end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -913,10 +917,10 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { memory_reservation.Commit(); // Increase our tracked mapped size. - mapped_physical_memory_size += (size - mapped_size); + m_mapped_physical_memory_size += (size - mapped_size); // Update the relevant memory blocks. - memory_block_manager.UpdateIfMatch( + m_memory_block_manager.UpdateIfMatch( std::addressof(allocator), address, size / PageSize, KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None, KMemoryState::Normal, KMemoryPermission::UserReadWrite, KMemoryAttribute::None); @@ -930,20 +934,20 @@ Result KPageTable::MapPhysicalMemory(VAddr address, std::size_t size) { } } -Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { +Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { // Lock the physical memory lock. - KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); + KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Calculate the last address for convenience. const VAddr last_address = address + size - 1; // Define iteration variables. VAddr cur_address = 0; - std::size_t mapped_size = 0; - std::size_t num_allocator_blocks = 0; + size_t mapped_size = 0; + size_t num_allocator_blocks = 0; // Check if the memory is mapped. { @@ -951,10 +955,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { cur_address = address; mapped_size = 0; - auto it = memory_block_manager.FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != memory_block_manager.end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -1053,7 +1057,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Reset the current tracking address, and make sure we clean up on failure. @@ -1064,7 +1068,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { cur_address = address; // Iterate over the memory we unmapped. - auto it = memory_block_manager.FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); auto pg_it = pg.Nodes().begin(); PAddr pg_phys_addr = pg_it->GetAddress(); size_t pg_pages = pg_it->GetNumPages(); @@ -1119,10 +1123,10 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { }); // Iterate over the memory, unmapping as we go. - auto it = memory_block_manager.FindIterator(cur_address); + auto it = m_memory_block_manager.FindIterator(cur_address); while (true) { // Check that the iterator is valid. - ASSERT(it != memory_block_manager.end()); + ASSERT(it != m_memory_block_manager.end()); // Get the memory info. const KMemoryInfo info = it->GetMemoryInfo(); @@ -1149,20 +1153,20 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { } // Release the memory resource. - mapped_physical_memory_size -= mapped_size; - auto process{system.Kernel().CurrentProcess()}; + m_mapped_physical_memory_size -= mapped_size; + auto process{m_system.Kernel().CurrentProcess()}; process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size); // Update memory blocks. - memory_block_manager.Update(std::addressof(allocator), address, size / PageSize, - KMemoryState::Free, KMemoryPermission::None, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), address, size / PageSize, + KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); // TODO(bunnei): This is a workaround until the next set of changes, where we add reference // counting for mapped pages. Until then, we must manually close the reference to the page // group. - system.Kernel().MemoryManager().Close(pg); + m_system.Kernel().MemoryManager().Close(pg); // We succeeded. remap_guard.Cancel(); @@ -1170,9 +1174,9 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, std::size_t size) { return ResultSuccess; } -Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, std::size_t size) { +Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, size_t size) { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Validate that the source address's state is valid. KMemoryState src_state; @@ -1192,19 +1196,21 @@ Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, std::size_t s // Create an update allocator for the source. Result src_allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator src_allocator( - std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); + KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), + m_memory_block_slab_manager, + num_src_allocator_blocks); R_TRY(src_allocator_result); // Create an update allocator for the destination. Result dst_allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator dst_allocator( - std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); + KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), + m_memory_block_slab_manager, + num_dst_allocator_blocks); R_TRY(dst_allocator_result); // Map the memory. KPageGroup page_linked_list; - const std::size_t num_pages{size / PageSize}; + const size_t num_pages{size / PageSize}; const KMemoryPermission new_src_perm = static_cast( KMemoryPermission::KernelRead | KMemoryPermission::NotMapped); const KMemoryAttribute new_src_attr = KMemoryAttribute::Locked; @@ -1223,21 +1229,21 @@ Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, std::size_t s } // Apply the memory block updates. - memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, - new_src_perm, new_src_attr, - KMemoryBlockDisableMergeAttribute::Locked, - KMemoryBlockDisableMergeAttribute::None); - memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, - KMemoryState::Stack, KMemoryPermission::UserReadWrite, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, + new_src_perm, new_src_attr, + KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::Stack, KMemoryPermission::UserReadWrite, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } -Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, std::size_t size) { +Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size) { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Validate that the source address's state is valid. KMemoryState src_state; @@ -1258,19 +1264,21 @@ Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, std::size_t // Create an update allocator for the source. Result src_allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator src_allocator( - std::addressof(src_allocator_result), memory_block_slab_manager, num_src_allocator_blocks); + KMemoryBlockManagerUpdateAllocator src_allocator(std::addressof(src_allocator_result), + m_memory_block_slab_manager, + num_src_allocator_blocks); R_TRY(src_allocator_result); // Create an update allocator for the destination. Result dst_allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator dst_allocator( - std::addressof(dst_allocator_result), memory_block_slab_manager, num_dst_allocator_blocks); + KMemoryBlockManagerUpdateAllocator dst_allocator(std::addressof(dst_allocator_result), + m_memory_block_slab_manager, + num_dst_allocator_blocks); R_TRY(dst_allocator_result); KPageGroup src_pages; KPageGroup dst_pages; - const std::size_t num_pages{size / PageSize}; + const size_t num_pages{size / PageSize}; AddRegionToPages(src_address, num_pages, src_pages); AddRegionToPages(dst_address, num_pages, dst_pages); @@ -1290,14 +1298,14 @@ Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, std::size_t } // Apply the memory block updates. - memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, - KMemoryPermission::UserReadWrite, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Locked); - memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, - KMemoryState::None, KMemoryPermission::None, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Normal); + m_memory_block_manager.Update(std::addressof(src_allocator), src_address, num_pages, src_state, + KMemoryPermission::UserReadWrite, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Locked); + m_memory_block_manager.Update(std::addressof(dst_allocator), dst_address, num_pages, + KMemoryState::None, KMemoryPermission::None, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); return ResultSuccess; } @@ -1312,7 +1320,7 @@ Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, if (const auto result{ Operate(cur_addr, node.GetNumPages(), perm, OperationType::Map, node.GetAddress())}; result.IsError()) { - const std::size_t num_pages{(addr - cur_addr) / PageSize}; + const size_t num_pages{(addr - cur_addr) / PageSize}; ASSERT(Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap) .IsSuccess()); @@ -1329,12 +1337,12 @@ Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state, KMemoryPermission perm) { // Check that the map is in range. - const std::size_t num_pages{page_linked_list.GetNumPages()}; - const std::size_t size{num_pages * PageSize}; + const size_t num_pages{page_linked_list.GetNumPages()}; + const size_t size{num_pages * PageSize}; R_UNLESS(this->CanContain(address, size, state), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the memory state. R_TRY(this->CheckMemoryState(address, size, KMemoryState::All, KMemoryState::Free, @@ -1344,23 +1352,22 @@ Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemory // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager); + m_memory_block_slab_manager); // Map the pages. R_TRY(MapPages(address, page_linked_list, perm)); // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } -Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t alignment, - PAddr phys_addr, bool is_pa_valid, VAddr region_start, - std::size_t region_num_pages, KMemoryState state, - KMemoryPermission perm) { +Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, + bool is_pa_valid, VAddr region_start, size_t region_num_pages, + KMemoryState state, KMemoryPermission perm) { ASSERT(Common::IsAligned(alignment, PageSize) && alignment >= PageSize); // Ensure this is a valid map request. @@ -1369,7 +1376,7 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t R_UNLESS(num_pages < region_num_pages, ResultOutOfMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Find a random address to map at. VAddr addr = this->FindFreeArea(region_start, region_num_pages, num_pages, alignment, 0, @@ -1385,7 +1392,7 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager); + m_memory_block_slab_manager); // Perform mapping operation. if (is_pa_valid) { @@ -1395,9 +1402,9 @@ Result KPageTable::MapPages(VAddr* out_addr, std::size_t num_pages, std::size_t } // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); // We successfully mapped the pages. *out_addr = addr; @@ -1424,12 +1431,12 @@ Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state) { // Check that the unmap is in range. - const std::size_t num_pages{page_linked_list.GetNumPages()}; - const std::size_t size{num_pages * PageSize}; + const size_t num_pages{page_linked_list.GetNumPages()}; + const size_t size{num_pages * PageSize}; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the memory state. size_t num_allocator_blocks; @@ -1441,31 +1448,31 @@ Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemo // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Perform the unmap. R_TRY(UnmapPages(address, page_linked_list)); // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, - KMemoryPermission::None, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Normal); + m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); return ResultSuccess; } -Result KPageTable::UnmapPages(VAddr address, std::size_t num_pages, KMemoryState state) { +Result KPageTable::UnmapPages(VAddr address, size_t num_pages, KMemoryState state) { // Check that the unmap is in range. - const std::size_t size = num_pages * PageSize; + const size_t size = num_pages * PageSize; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the memory state. - std::size_t num_allocator_blocks{}; + size_t num_allocator_blocks{}; R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, KMemoryState::All, state, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, @@ -1474,17 +1481,17 @@ Result KPageTable::UnmapPages(VAddr address, std::size_t num_pages, KMemoryState // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Perform the unmap. R_TRY(Operate(address, num_pages, KMemoryPermission::None, OperationType::Unmap)); // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, - KMemoryPermission::None, KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Normal); + m_memory_block_manager.Update(std::addressof(allocator), address, num_pages, KMemoryState::Free, + KMemoryPermission::None, KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Normal); return ResultSuccess; } @@ -1501,7 +1508,7 @@ Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t n R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check if state allows us to create the group. R_TRY(this->CheckMemoryState(address, size, state_mask | KMemoryState::FlagReferenceCounted, @@ -1514,12 +1521,12 @@ Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t n return ResultSuccess; } -Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, +Result KPageTable::SetProcessMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify we can change the memory permission. KMemoryState old_state; @@ -1559,7 +1566,7 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Perform mapping operation. @@ -1568,29 +1575,29 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, std::size_t size, R_TRY(Operate(addr, num_pages, new_perm, operation)); // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), addr, num_pages, new_state, new_perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, new_state, new_perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); // Ensure cache coherency, if we're setting pages as executable. if (is_x) { - system.InvalidateCpuInstructionCacheRange(addr, size); + m_system.InvalidateCpuInstructionCacheRange(addr, size); } return ResultSuccess; } KMemoryInfo KPageTable::QueryInfoImpl(VAddr addr) { - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); - return memory_block_manager.FindBlock(addr)->GetMemoryInfo(); + return m_memory_block_manager.FindBlock(addr)->GetMemoryInfo(); } KMemoryInfo KPageTable::QueryInfo(VAddr addr) { if (!Contains(addr, 1)) { return { - .m_address = address_space_end, - .m_size = 0 - address_space_end, + .m_address = m_address_space_end, + .m_size = 0 - m_address_space_end, .m_state = static_cast(Svc::MemoryState::Inaccessible), .m_device_disable_merge_left_count = 0, .m_device_disable_merge_right_count = 0, @@ -1607,12 +1614,11 @@ KMemoryInfo KPageTable::QueryInfo(VAddr addr) { return QueryInfoImpl(addr); } -Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, - Svc::MemoryPermission svc_perm) { +Result KPageTable::SetMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermission svc_perm) { const size_t num_pages = size / PageSize; // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify we can change the memory permission. KMemoryState old_state; @@ -1631,27 +1637,27 @@ Result KPageTable::SetMemoryPermission(VAddr addr, std::size_t size, // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Perform mapping operation. R_TRY(Operate(addr, num_pages, new_perm, OperationType::ChangePermissions)); // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } -Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u32 attr) { +Result KPageTable::SetMemoryAttribute(VAddr addr, size_t size, u32 mask, u32 attr) { const size_t num_pages = size / PageSize; ASSERT((static_cast(mask) | KMemoryAttribute::SetMask) == KMemoryAttribute::SetMask); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Verify we can change the memory attribute. KMemoryState old_state; @@ -1669,7 +1675,7 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u3 // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Determine the new attribute. @@ -1681,124 +1687,125 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, std::size_t size, u32 mask, u3 this->Operate(addr, num_pages, old_perm, OperationType::ChangePermissionsAndRefresh); // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, old_perm, - new_attr, KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, old_perm, + new_attr, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } -Result KPageTable::SetMaxHeapSize(std::size_t size) { +Result KPageTable::SetMaxHeapSize(size_t size) { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Only process page tables are allowed to set heap size. ASSERT(!this->IsKernel()); - max_heap_size = size; + m_max_heap_size = size; return ResultSuccess; } -Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { +Result KPageTable::SetHeapSize(VAddr* out, size_t size) { // Lock the physical memory mutex. - KScopedLightLock map_phys_mem_lk(map_physical_memory_lock); + KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); // Try to perform a reduction in heap, instead of an extension. VAddr cur_address{}; - std::size_t allocation_size{}; + size_t allocation_size{}; { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Validate that setting heap size is possible at all. - R_UNLESS(!is_kernel, ResultOutOfMemory); - R_UNLESS(size <= static_cast(heap_region_end - heap_region_start), + R_UNLESS(!m_is_kernel, ResultOutOfMemory); + R_UNLESS(size <= static_cast(m_heap_region_end - m_heap_region_start), ResultOutOfMemory); - R_UNLESS(size <= max_heap_size, ResultOutOfMemory); + R_UNLESS(size <= m_max_heap_size, ResultOutOfMemory); if (size < GetHeapSize()) { // The size being requested is less than the current size, so we need to free the end of // the heap. // Validate memory state. - std::size_t num_allocator_blocks; + size_t num_allocator_blocks; R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), - heap_region_start + size, GetHeapSize() - size, + m_heap_region_start + size, GetHeapSize() - size, KMemoryState::All, KMemoryState::Normal, KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None)); // Create an update allocator. Result allocator_result{ResultSuccess}; - KMemoryBlockManagerUpdateAllocator allocator( - std::addressof(allocator_result), memory_block_slab_manager, num_allocator_blocks); + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, + num_allocator_blocks); R_TRY(allocator_result); // Unmap the end of the heap. const auto num_pages = (GetHeapSize() - size) / PageSize; - R_TRY(Operate(heap_region_start + size, num_pages, KMemoryPermission::None, + R_TRY(Operate(m_heap_region_start + size, num_pages, KMemoryPermission::None, OperationType::Unmap)); // Release the memory from the resource limit. - system.Kernel().CurrentProcess()->GetResourceLimit()->Release( + m_system.Kernel().CurrentProcess()->GetResourceLimit()->Release( LimitableResource::PhysicalMemory, num_pages * PageSize); // Apply the memory block update. - memory_block_manager.Update(std::addressof(allocator), heap_region_start + size, - num_pages, KMemoryState::Free, KMemoryPermission::None, - KMemoryAttribute::None, - KMemoryBlockDisableMergeAttribute::None, - size == 0 ? KMemoryBlockDisableMergeAttribute::Normal - : KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), m_heap_region_start + size, + num_pages, KMemoryState::Free, KMemoryPermission::None, + KMemoryAttribute::None, + KMemoryBlockDisableMergeAttribute::None, + size == 0 ? KMemoryBlockDisableMergeAttribute::Normal + : KMemoryBlockDisableMergeAttribute::None); // Update the current heap end. - current_heap_end = heap_region_start + size; + m_current_heap_end = m_heap_region_start + size; // Set the output. - *out = heap_region_start; + *out = m_heap_region_start; return ResultSuccess; } else if (size == GetHeapSize()) { // The size requested is exactly the current size. - *out = heap_region_start; + *out = m_heap_region_start; return ResultSuccess; } else { // We have to allocate memory. Determine how much to allocate and where while the table // is locked. - cur_address = current_heap_end; + cur_address = m_current_heap_end; allocation_size = size - GetHeapSize(); } } // Reserve memory for the heap extension. KScopedResourceReservation memory_reservation( - system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, + m_system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, allocation_size); R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); // Allocate pages for the heap extension. KPageGroup pg; - R_TRY(system.Kernel().MemoryManager().AllocateAndOpen( + R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpen( &pg, allocation_size / PageSize, - KMemoryManager::EncodeOption(memory_pool, allocation_option))); + KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option))); // Clear all the newly allocated pages. for (const auto& it : pg.Nodes()) { - std::memset(system.DeviceMemory().GetPointer(it.GetAddress()), heap_fill_value, + std::memset(m_system.DeviceMemory().GetPointer(it.GetAddress()), m_heap_fill_value, it.GetSize()); } // Map the pages. { // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Ensure that the heap hasn't changed since we began executing. - ASSERT(cur_address == current_heap_end); + ASSERT(cur_address == m_current_heap_end); // Check the memory state. - std::size_t num_allocator_blocks{}; - R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), current_heap_end, + size_t num_allocator_blocks{}; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), m_current_heap_end, allocation_size, KMemoryState::All, KMemoryState::Free, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::None, KMemoryAttribute::None)); @@ -1806,16 +1813,16 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator( - std::addressof(allocator_result), memory_block_slab_manager, num_allocator_blocks); + std::addressof(allocator_result), m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Map the pages. const auto num_pages = allocation_size / PageSize; - R_TRY(Operate(current_heap_end, num_pages, pg, OperationType::MapGroup)); + R_TRY(Operate(m_current_heap_end, num_pages, pg, OperationType::MapGroup)); // Clear all the newly allocated pages. - for (std::size_t cur_page = 0; cur_page < num_pages; ++cur_page) { - std::memset(system.Memory().GetPointer(current_heap_end + (cur_page * PageSize)), 0, + for (size_t cur_page = 0; cur_page < num_pages; ++cur_page) { + std::memset(m_system.Memory().GetPointer(m_current_heap_end + (cur_page * PageSize)), 0, PageSize); } @@ -1823,27 +1830,27 @@ Result KPageTable::SetHeapSize(VAddr* out, std::size_t size) { memory_reservation.Commit(); // Apply the memory block update. - memory_block_manager.Update( - std::addressof(allocator), current_heap_end, num_pages, KMemoryState::Normal, + m_memory_block_manager.Update( + std::addressof(allocator), m_current_heap_end, num_pages, KMemoryState::Normal, KMemoryPermission::UserReadWrite, KMemoryAttribute::None, - heap_region_start == current_heap_end ? KMemoryBlockDisableMergeAttribute::Normal - : KMemoryBlockDisableMergeAttribute::None, + m_heap_region_start == m_current_heap_end ? KMemoryBlockDisableMergeAttribute::Normal + : KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::None); // Update the current heap end. - current_heap_end = heap_region_start + size; + m_current_heap_end = m_heap_region_start + size; // Set the output. - *out = heap_region_start; + *out = m_heap_region_start; return ResultSuccess; } } -ResultVal KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, std::size_t align, +ResultVal KPageTable::AllocateAndMapMemory(size_t needed_num_pages, size_t align, bool is_map_only, VAddr region_start, - std::size_t region_num_pages, KMemoryState state, + size_t region_num_pages, KMemoryState state, KMemoryPermission perm, PAddr map_addr) { - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); if (!CanContain(region_start, region_num_pages * PageSize, state)) { return ResultInvalidCurrentMemory; @@ -1862,33 +1869,98 @@ ResultVal KPageTable::AllocateAndMapMemory(std::size_t needed_num_pages, // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager); + m_memory_block_slab_manager); if (is_map_only) { R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); } else { KPageGroup page_group; - R_TRY(system.Kernel().MemoryManager().AllocateAndOpenForProcess( + R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpenForProcess( &page_group, needed_num_pages, - KMemoryManager::EncodeOption(memory_pool, allocation_option), 0, 0)); + KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option), 0, 0)); R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); } // Update the blocks. - memory_block_manager.Update(std::addressof(allocator), addr, needed_num_pages, state, perm, - KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, needed_num_pages, state, perm, + KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, + KMemoryBlockDisableMergeAttribute::None); return addr; } -Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, std::size_t size) { +Result KPageTable::LockForMapDeviceAddressSpace(VAddr address, size_t size, KMemoryPermission perm, + bool is_aligned) { + // Lightly validate the range before doing anything else. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Check the memory state. + const auto test_state = + (is_aligned ? KMemoryState::FlagCanAlignedDeviceMap : KMemoryState::FlagCanDeviceMap); + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, test_state, + test_state, perm, perm, + KMemoryAttribute::IpcLocked | KMemoryAttribute::Locked, + KMemoryAttribute::None, KMemoryAttribute::DeviceShared)); + + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // Update the memory blocks. + m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, + &KMemoryBlock::ShareToDevice, KMemoryPermission::None); + + return ResultSuccess; +} + +Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size) { + // Lightly validate the range before doing anything else. + const size_t num_pages = size / PageSize; + R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); + + // Lock the table. + KScopedLightLock lk(m_general_lock); + + // Check the memory state. + size_t num_allocator_blocks; + R_TRY(this->CheckMemoryStateContiguous( + std::addressof(num_allocator_blocks), address, size, + KMemoryState::FlagReferenceCounted | KMemoryState::FlagCanDeviceMap, + KMemoryState::FlagReferenceCounted | KMemoryState::FlagCanDeviceMap, + KMemoryPermission::None, KMemoryPermission::None, + KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked, KMemoryAttribute::DeviceShared)); + + // Create an update allocator. + Result allocator_result{ResultSuccess}; + KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), + m_memory_block_slab_manager, num_allocator_blocks); + R_TRY(allocator_result); + + // Update the memory blocks. + const KMemoryBlockManager::MemoryBlockLockFunction lock_func = + m_enable_device_address_space_merge + ? &KMemoryBlock::UpdateDeviceDisableMergeStateForShare + : &KMemoryBlock::UpdateDeviceDisableMergeStateForShareRight; + m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, lock_func, + KMemoryPermission::None); + + return ResultSuccess; +} + +Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, size_t size) { // Lightly validate the range before doing anything else. const size_t num_pages = size / PageSize; R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the memory state. size_t num_allocator_blocks; @@ -1900,17 +1972,17 @@ Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, std::size_t size) // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Update the memory blocks. - memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, - &KMemoryBlock::UnshareToDevice, KMemoryPermission::None); + m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, + &KMemoryBlock::UnshareToDevice, KMemoryPermission::None); return ResultSuccess; } -Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, std::size_t size) { +Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, size_t size) { return this->LockMemoryAndOpen( out, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, @@ -1920,7 +1992,7 @@ Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, std::size_t si KMemoryAttribute::Locked); } -Result KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size, const KPageGroup& pg) { +Result KPageTable::UnlockForCodeMemory(VAddr addr, size_t size, const KPageGroup& pg) { return this->UnlockMemory( addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, @@ -1928,9 +2000,9 @@ Result KPageTable::UnlockForCodeMemory(VAddr addr, std::size_t size, const KPage } bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { - auto start_ptr = system.DeviceMemory().GetPointer(addr); + auto start_ptr = m_system.DeviceMemory().GetPointer(addr); for (u64 offset{}; offset < size; offset += PageSize) { - if (start_ptr != system.DeviceMemory().GetPointer(addr + offset)) { + if (start_ptr != m_system.DeviceMemory().GetPointer(addr + offset)) { return false; } start_ptr += PageSize; @@ -1938,8 +2010,7 @@ bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { return true; } -void KPageTable::AddRegionToPages(VAddr start, std::size_t num_pages, - KPageGroup& page_linked_list) { +void KPageTable::AddRegionToPages(VAddr start, size_t num_pages, KPageGroup& page_linked_list) { VAddr addr{start}; while (addr < start + (num_pages * PageSize)) { const PAddr paddr{GetPhysicalAddr(addr)}; @@ -1949,16 +2020,16 @@ void KPageTable::AddRegionToPages(VAddr start, std::size_t num_pages, } } -VAddr KPageTable::AllocateVirtualMemory(VAddr start, std::size_t region_num_pages, - u64 needed_num_pages, std::size_t align) { - if (is_aslr_enabled) { +VAddr KPageTable::AllocateVirtualMemory(VAddr start, size_t region_num_pages, u64 needed_num_pages, + size_t align) { + if (m_enable_aslr) { UNIMPLEMENTED(); } - return memory_block_manager.FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, - IsKernel() ? 1 : 4); + return m_memory_block_manager.FindFreeArea(start, region_num_pages, needed_num_pages, align, 0, + IsKernel() ? 1 : 4); } -Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& page_group, +Result KPageTable::Operate(VAddr addr, size_t num_pages, const KPageGroup& page_group, OperationType operation) { ASSERT(this->IsLockedByCurrentThread()); @@ -1967,11 +2038,11 @@ Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& ASSERT(num_pages == page_group.GetNumPages()); for (const auto& node : page_group.Nodes()) { - const std::size_t size{node.GetNumPages() * PageSize}; + const size_t size{node.GetNumPages() * PageSize}; switch (operation) { case OperationType::MapGroup: - system.Memory().MapMemoryRegion(page_table_impl, addr, size, node.GetAddress()); + m_system.Memory().MapMemoryRegion(m_page_table_impl, addr, size, node.GetAddress()); break; default: ASSERT(false); @@ -1983,7 +2054,7 @@ Result KPageTable::Operate(VAddr addr, std::size_t num_pages, const KPageGroup& return ResultSuccess; } -Result KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission perm, +Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, OperationType operation, PAddr map_addr) { ASSERT(this->IsLockedByCurrentThread()); @@ -1993,12 +2064,12 @@ Result KPageTable::Operate(VAddr addr, std::size_t num_pages, KMemoryPermission switch (operation) { case OperationType::Unmap: - system.Memory().UnmapRegion(page_table_impl, addr, num_pages * PageSize); + m_system.Memory().UnmapRegion(m_page_table_impl, addr, num_pages * PageSize); break; case OperationType::Map: { ASSERT(map_addr); ASSERT(Common::IsAligned(map_addr, PageSize)); - system.Memory().MapMemoryRegion(page_table_impl, addr, num_pages * PageSize, map_addr); + m_system.Memory().MapMemoryRegion(m_page_table_impl, addr, num_pages * PageSize, map_addr); break; } case OperationType::ChangePermissions: @@ -2014,18 +2085,18 @@ VAddr KPageTable::GetRegionAddress(KMemoryState state) const { switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: - return address_space_start; + return m_address_space_start; case KMemoryState::Normal: - return heap_region_start; + return m_heap_region_start; case KMemoryState::Ipc: case KMemoryState::NonSecureIpc: case KMemoryState::NonDeviceIpc: - return alias_region_start; + return m_alias_region_start; case KMemoryState::Stack: - return stack_region_start; + return m_stack_region_start; case KMemoryState::Static: case KMemoryState::ThreadLocal: - return kernel_map_region_start; + return m_kernel_map_region_start; case KMemoryState::Io: case KMemoryState::Shared: case KMemoryState::AliasCode: @@ -2036,31 +2107,31 @@ VAddr KPageTable::GetRegionAddress(KMemoryState state) const { case KMemoryState::GeneratedCode: case KMemoryState::CodeOut: case KMemoryState::Coverage: - return alias_code_region_start; + return m_alias_code_region_start; case KMemoryState::Code: case KMemoryState::CodeData: - return code_region_start; + return m_code_region_start; default: UNREACHABLE(); } } -std::size_t KPageTable::GetRegionSize(KMemoryState state) const { +size_t KPageTable::GetRegionSize(KMemoryState state) const { switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: - return address_space_end - address_space_start; + return m_address_space_end - m_address_space_start; case KMemoryState::Normal: - return heap_region_end - heap_region_start; + return m_heap_region_end - m_heap_region_start; case KMemoryState::Ipc: case KMemoryState::NonSecureIpc: case KMemoryState::NonDeviceIpc: - return alias_region_end - alias_region_start; + return m_alias_region_end - m_alias_region_start; case KMemoryState::Stack: - return stack_region_end - stack_region_start; + return m_stack_region_end - m_stack_region_start; case KMemoryState::Static: case KMemoryState::ThreadLocal: - return kernel_map_region_end - kernel_map_region_start; + return m_kernel_map_region_end - m_kernel_map_region_start; case KMemoryState::Io: case KMemoryState::Shared: case KMemoryState::AliasCode: @@ -2071,16 +2142,16 @@ std::size_t KPageTable::GetRegionSize(KMemoryState state) const { case KMemoryState::GeneratedCode: case KMemoryState::CodeOut: case KMemoryState::Coverage: - return alias_code_region_end - alias_code_region_start; + return m_alias_code_region_end - m_alias_code_region_start; case KMemoryState::Code: case KMemoryState::CodeData: - return code_region_end - code_region_start; + return m_code_region_end - m_code_region_start; default: UNREACHABLE(); } } -bool KPageTable::CanContain(VAddr addr, std::size_t size, KMemoryState state) const { +bool KPageTable::CanContain(VAddr addr, size_t size, KMemoryState state) const { const VAddr end = addr + size; const VAddr last = end - 1; @@ -2089,10 +2160,10 @@ bool KPageTable::CanContain(VAddr addr, std::size_t size, KMemoryState state) co const bool is_in_region = region_start <= addr && addr < end && last <= region_start + region_size - 1; - const bool is_in_heap = !(end <= heap_region_start || heap_region_end <= addr || - heap_region_start == heap_region_end); - const bool is_in_alias = !(end <= alias_region_start || alias_region_end <= addr || - alias_region_start == alias_region_end); + const bool is_in_heap = !(end <= m_heap_region_start || m_heap_region_end <= addr || + m_heap_region_start == m_heap_region_end); + const bool is_in_alias = !(end <= m_alias_region_start || m_alias_region_end <= addr || + m_alias_region_start == m_alias_region_end); switch (state) { case KMemoryState::Free: case KMemoryState::Kernel: @@ -2138,16 +2209,16 @@ Result KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_ return ResultSuccess; } -Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VAddr addr, - std::size_t size, KMemoryState state_mask, - KMemoryState state, KMemoryPermission perm_mask, - KMemoryPermission perm, KMemoryAttribute attr_mask, +Result KPageTable::CheckMemoryStateContiguous(size_t* out_blocks_needed, VAddr addr, size_t size, + KMemoryState state_mask, KMemoryState state, + KMemoryPermission perm_mask, KMemoryPermission perm, + KMemoryAttribute attr_mask, KMemoryAttribute attr) const { ASSERT(this->IsLockedByCurrentThread()); // Get information about the first block. const VAddr last_addr = addr + size - 1; - KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(addr); + KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(addr); KMemoryInfo info = it->GetMemoryInfo(); // If the start address isn't aligned, we need a block. @@ -2165,7 +2236,7 @@ Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VA // Advance our iterator. it++; - ASSERT(it != memory_block_manager.cend()); + ASSERT(it != m_memory_block_manager.cend()); info = it->GetMemoryInfo(); } @@ -2181,8 +2252,8 @@ Result KPageTable::CheckMemoryStateContiguous(std::size_t* out_blocks_needed, VA } Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, - KMemoryAttribute* out_attr, std::size_t* out_blocks_needed, - VAddr addr, std::size_t size, KMemoryState state_mask, + KMemoryAttribute* out_attr, size_t* out_blocks_needed, + VAddr addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, KMemoryAttribute attr_mask, KMemoryAttribute attr, KMemoryAttribute ignore_attr) const { @@ -2190,7 +2261,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* // Get information about the first block. const VAddr last_addr = addr + size - 1; - KMemoryBlockManager::const_iterator it = memory_block_manager.FindIterator(addr); + KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(addr); KMemoryInfo info = it->GetMemoryInfo(); // If the start address isn't aligned, we need a block. @@ -2218,7 +2289,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* // Advance our iterator. it++; - ASSERT(it != memory_block_manager.cend()); + ASSERT(it != m_memory_block_manager.cend()); info = it->GetMemoryInfo(); } @@ -2257,7 +2328,7 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check that the output page group is empty, if it exists. if (out_pg) { @@ -2288,7 +2359,7 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Decide on new perm and attr. @@ -2301,9 +2372,9 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr } // Apply the memory block updates. - memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, - new_attr, KMemoryBlockDisableMergeAttribute::Locked, - KMemoryBlockDisableMergeAttribute::None); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + new_attr, KMemoryBlockDisableMergeAttribute::Locked, + KMemoryBlockDisableMergeAttribute::None); return ResultSuccess; } @@ -2322,7 +2393,7 @@ Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask R_UNLESS(this->Contains(addr, size), ResultInvalidCurrentMemory); // Lock the table. - KScopedLightLock lk(general_lock); + KScopedLightLock lk(m_general_lock); // Check the state. KMemoryState old_state{}; @@ -2347,7 +2418,7 @@ Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask // Create an update allocator. Result allocator_result{ResultSuccess}; KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), - memory_block_slab_manager, num_allocator_blocks); + m_memory_block_slab_manager, num_allocator_blocks); R_TRY(allocator_result); // Update permission, if we need to. @@ -2356,9 +2427,9 @@ Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask } // Apply the memory block updates. - memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, - new_attr, KMemoryBlockDisableMergeAttribute::None, - KMemoryBlockDisableMergeAttribute::Locked); + m_memory_block_manager.Update(std::addressof(allocator), addr, num_pages, old_state, new_perm, + new_attr, KMemoryBlockDisableMergeAttribute::None, + KMemoryBlockDisableMergeAttribute::Locked); return ResultSuccess; } -- cgit v1.2.3 From abcc009dff5d98b5a04229f3a82baab23d568244 Mon Sep 17 00:00:00 2001 From: bunnei Date: Sun, 2 Oct 2022 14:26:30 -0700 Subject: core: hle: kernel: k_process: Improve management of page table & cleanup. --- src/core/hle/kernel/k_page_table.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'src/core/hle/kernel/k_page_table.cpp') diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index fcffc0b88..22098c056 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -256,16 +256,21 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type m_mapped_physical_memory_size = 0; m_memory_pool = pool; - m_page_table_impl.Resize(m_address_space_width, PageBits); + m_page_table_impl = std::make_unique(); + m_page_table_impl->Resize(m_address_space_width, PageBits); return m_memory_block_manager.Initialize(m_address_space_start, m_address_space_end, m_memory_block_slab_manager); } void KPageTable::Finalize() { + // Finalize memory blocks. m_memory_block_manager.Finalize(m_memory_block_slab_manager, [&](VAddr addr, u64 size) { - m_system.Memory().UnmapRegion(m_page_table_impl, addr, size); + m_system.Memory().UnmapRegion(*m_page_table_impl, addr, size); }); + + // Close the backing page table, as the destructor is not called for guest objects. + m_page_table_impl.reset(); } Result KPageTable::MapProcessCode(VAddr addr, size_t num_pages, KMemoryState state, @@ -514,7 +519,7 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - R_UNLESS(m_page_table_impl.BeginTraversal(next_entry, context, addr), + R_UNLESS(m_page_table_impl->BeginTraversal(next_entry, context, addr), ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -525,7 +530,7 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { // Iterate, adding to group as we go. const auto& memory_layout = m_system.Kernel().MemoryLayout(); while (tot_size < size) { - R_UNLESS(m_page_table_impl.ContinueTraversal(next_entry, context), + R_UNLESS(m_page_table_impl->ContinueTraversal(next_entry, context), ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -588,7 +593,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu // Begin traversal. Common::PageTable::TraversalContext context; Common::PageTable::TraversalEntry next_entry; - if (!m_page_table_impl.BeginTraversal(next_entry, context, addr)) { + if (!m_page_table_impl->BeginTraversal(next_entry, context, addr)) { return false; } @@ -599,7 +604,7 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu // Iterate, comparing expected to actual. while (tot_size < size) { - if (!m_page_table_impl.ContinueTraversal(next_entry, context)) { + if (!m_page_table_impl->ContinueTraversal(next_entry, context)) { return false; } @@ -2042,7 +2047,7 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, const KPageGroup& page_ switch (operation) { case OperationType::MapGroup: - m_system.Memory().MapMemoryRegion(m_page_table_impl, addr, size, node.GetAddress()); + m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, size, node.GetAddress()); break; default: ASSERT(false); @@ -2064,12 +2069,12 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, switch (operation) { case OperationType::Unmap: - m_system.Memory().UnmapRegion(m_page_table_impl, addr, num_pages * PageSize); + m_system.Memory().UnmapRegion(*m_page_table_impl, addr, num_pages * PageSize); break; case OperationType::Map: { ASSERT(map_addr); ASSERT(Common::IsAligned(map_addr, PageSize)); - m_system.Memory().MapMemoryRegion(m_page_table_impl, addr, num_pages * PageSize, map_addr); + m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, num_pages * PageSize, map_addr); break; } case OperationType::ChangePermissions: -- cgit v1.2.3 From 829e82e264504696ce1d0ae9421a53d16bf104ea Mon Sep 17 00:00:00 2001 From: bunnei Date: Fri, 14 Oct 2022 22:55:51 -0700 Subject: core: hle: kernel: Use result macros for new/changed code. --- src/core/hle/kernel/k_page_table.cpp | 111 ++++++++++++++++------------------- 1 file changed, 50 insertions(+), 61 deletions(-) (limited to 'src/core/hle/kernel/k_page_table.cpp') diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp index 22098c056..307e491cb 100644 --- a/src/core/hle/kernel/k_page_table.cpp +++ b/src/core/hle/kernel/k_page_table.cpp @@ -128,12 +128,9 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type alloc_start = process_code_end; alloc_size = end - process_code_end; } - const size_t needed_size{ - (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size)}; - if (alloc_size < needed_size) { - ASSERT(false); - return ResultOutOfMemory; - } + const size_t needed_size = + (alias_region_size + heap_region_size + stack_region_size + kernel_map_region_size); + R_UNLESS(alloc_size >= needed_size, ResultOutOfMemory); const size_t remaining_size{alloc_size - needed_size}; @@ -259,8 +256,9 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type m_page_table_impl = std::make_unique(); m_page_table_impl->Resize(m_address_space_width, PageBits); - return m_memory_block_manager.Initialize(m_address_space_start, m_address_space_end, - m_memory_block_slab_manager); + // Initialize our memory block manager. + R_RETURN(m_memory_block_manager.Initialize(m_address_space_start, m_address_space_end, + m_memory_block_slab_manager)); } void KPageTable::Finalize() { @@ -306,7 +304,7 @@ Result KPageTable::MapProcessCode(VAddr addr, size_t num_pages, KMemoryState sta KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t size) { @@ -385,7 +383,7 @@ Result KPageTable::MapCodeMemory(VAddr dst_address, VAddr src_address, size_t si KMemoryBlockDisableMergeAttribute::None); } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, size_t size, @@ -487,7 +485,7 @@ Result KPageTable::UnmapCodeMemory(VAddr dst_address, VAddr src_address, size_t reprotected_pages = true; } - return ResultSuccess; + R_SUCCEED(); } VAddr KPageTable::FindFreeArea(VAddr region_start, size_t region_num_pages, size_t num_pages, @@ -558,7 +556,7 @@ Result KPageTable::MakePageGroup(KPageGroup& pg, VAddr addr, size_t num_pages) { R_UNLESS(IsHeapPhysicalAddress(memory_layout, cur_addr), ResultInvalidCurrentMemory); R_TRY(pg.AddBlock(cur_addr, cur_pages)); - return ResultSuccess; + R_SUCCEED(); } bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t num_pages) { @@ -685,7 +683,7 @@ Result KPageTable::UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& s m_system.InvalidateCpuInstructionCaches(); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { @@ -933,7 +931,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { // Cancel our guard. unmap_guard.Cancel(); - return ResultSuccess; + R_SUCCEED(); } } } @@ -1176,7 +1174,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { // We succeeded. remap_guard.Cancel(); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, size_t size) { @@ -1243,7 +1241,7 @@ Result KPageTable::MapMemory(VAddr dst_address, VAddr src_address, size_t size) KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size) { @@ -1288,9 +1286,7 @@ Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size AddRegionToPages(src_address, num_pages, src_pages); AddRegionToPages(dst_address, num_pages, dst_pages); - if (!dst_pages.IsEqual(src_pages)) { - return ResultInvalidMemoryRegion; - } + R_UNLESS(dst_pages.IsEqual(src_pages), ResultInvalidMemoryRegion); { auto block_guard = detail::ScopeExit([&] { MapPages(dst_address, dst_pages, dst_perm); }); @@ -1312,7 +1308,7 @@ Result KPageTable::UnmapMemory(VAddr dst_address, VAddr src_address, size_t size KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Normal); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, @@ -1330,13 +1326,13 @@ Result KPageTable::MapPages(VAddr addr, const KPageGroup& page_linked_list, ASSERT(Operate(addr, num_pages, KMemoryPermission::None, OperationType::Unmap) .IsSuccess()); - return result; + R_RETURN(result); } cur_addr += node.GetNumPages() * PageSize; } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state, @@ -1367,7 +1363,7 @@ Result KPageTable::MapPages(VAddr address, KPageGroup& page_linked_list, KMemory KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::Normal, KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, PAddr phys_addr, @@ -1413,7 +1409,7 @@ Result KPageTable::MapPages(VAddr* out_addr, size_t num_pages, size_t alignment, // We successfully mapped the pages. *out_addr = addr; - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { @@ -1425,13 +1421,13 @@ Result KPageTable::UnmapPages(VAddr addr, const KPageGroup& page_linked_list) { if (const auto result{Operate(cur_addr, node.GetNumPages(), KMemoryPermission::None, OperationType::Unmap)}; result.IsError()) { - return result; + R_RETURN(result); } cur_addr += node.GetNumPages() * PageSize; } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemoryState state) { @@ -1465,7 +1461,7 @@ Result KPageTable::UnmapPages(VAddr address, KPageGroup& page_linked_list, KMemo KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Normal); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnmapPages(VAddr address, size_t num_pages, KMemoryState state) { @@ -1498,7 +1494,7 @@ Result KPageTable::UnmapPages(VAddr address, size_t num_pages, KMemoryState stat KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Normal); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t num_pages, @@ -1523,7 +1519,7 @@ Result KPageTable::MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t n // Create a new page group for the region. R_TRY(this->MakePageGroup(*out, address, num_pages)); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::SetProcessMemoryPermission(VAddr addr, size_t size, @@ -1589,7 +1585,7 @@ Result KPageTable::SetProcessMemoryPermission(VAddr addr, size_t size, m_system.InvalidateCpuInstructionCacheRange(addr, size); } - return ResultSuccess; + R_SUCCEED(); } KMemoryInfo KPageTable::QueryInfoImpl(VAddr addr) { @@ -1653,7 +1649,7 @@ Result KPageTable::SetMemoryPermission(VAddr addr, size_t size, Svc::MemoryPermi KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::SetMemoryAttribute(VAddr addr, size_t size, u32 mask, u32 attr) { @@ -1696,7 +1692,7 @@ Result KPageTable::SetMemoryAttribute(VAddr addr, size_t size, u32 mask, u32 att new_attr, KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::SetMaxHeapSize(size_t size) { @@ -1708,7 +1704,7 @@ Result KPageTable::SetMaxHeapSize(size_t size) { m_max_heap_size = size; - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::SetHeapSize(VAddr* out, size_t size) { @@ -1769,11 +1765,11 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) { // Set the output. *out = m_heap_region_start; - return ResultSuccess; + R_SUCCEED(); } else if (size == GetHeapSize()) { // The size requested is exactly the current size. *out = m_heap_region_start; - return ResultSuccess; + R_SUCCEED(); } else { // We have to allocate memory. Determine how much to allocate and where while the table // is locked. @@ -1847,7 +1843,7 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) { // Set the output. *out = m_heap_region_start; - return ResultSuccess; + R_SUCCEED(); } } @@ -1857,19 +1853,12 @@ ResultVal KPageTable::AllocateAndMapMemory(size_t needed_num_pages, size_ KMemoryPermission perm, PAddr map_addr) { KScopedLightLock lk(m_general_lock); - if (!CanContain(region_start, region_num_pages * PageSize, state)) { - return ResultInvalidCurrentMemory; - } - - if (region_num_pages <= needed_num_pages) { - return ResultOutOfMemory; - } - + R_UNLESS(CanContain(region_start, region_num_pages * PageSize, state), + ResultInvalidCurrentMemory); + R_UNLESS(region_num_pages > needed_num_pages, ResultOutOfMemory); const VAddr addr{ AllocateVirtualMemory(region_start, region_num_pages, needed_num_pages, align)}; - if (!addr) { - return ResultOutOfMemory; - } + R_UNLESS(addr, ResultOutOfMemory); // Create an update allocator. Result allocator_result{ResultSuccess}; @@ -1922,7 +1911,7 @@ Result KPageTable::LockForMapDeviceAddressSpace(VAddr address, size_t size, KMem m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, &KMemoryBlock::ShareToDevice, KMemoryPermission::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size) { @@ -1956,7 +1945,7 @@ Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size) { m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, lock_func, KMemoryPermission::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, size_t size) { @@ -1984,24 +1973,24 @@ Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, size_t size) { m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, &KMemoryBlock::UnshareToDevice, KMemoryPermission::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, size_t size) { - return this->LockMemoryAndOpen( + R_RETURN(this->LockMemoryAndOpen( out, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, KMemoryPermission::All, KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None, static_cast(KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite), - KMemoryAttribute::Locked); + KMemoryAttribute::Locked)); } Result KPageTable::UnlockForCodeMemory(VAddr addr, size_t size, const KPageGroup& pg) { - return this->UnlockMemory( + R_RETURN(this->UnlockMemory( addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory, KMemoryPermission::None, KMemoryPermission::None, KMemoryAttribute::All, - KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg); + KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite, KMemoryAttribute::Locked, &pg)); } bool KPageTable::IsRegionContiguous(VAddr addr, u64 size) const { @@ -2056,7 +2045,7 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, const KPageGroup& page_ addr += size; } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, @@ -2083,7 +2072,7 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, default: ASSERT(false); } - return ResultSuccess; + R_SUCCEED(); } VAddr KPageTable::GetRegionAddress(KMemoryState state) const { @@ -2211,7 +2200,7 @@ Result KPageTable::CheckMemoryState(const KMemoryInfo& info, KMemoryState state_ R_UNLESS((info.m_permission & perm_mask) == perm, ResultInvalidCurrentMemory); R_UNLESS((info.m_attribute & attr_mask) == attr, ResultInvalidCurrentMemory); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::CheckMemoryStateContiguous(size_t* out_blocks_needed, VAddr addr, size_t size, @@ -2253,7 +2242,7 @@ Result KPageTable::CheckMemoryStateContiguous(size_t* out_blocks_needed, VAddr a *out_blocks_needed = blocks_for_start_align + blocks_for_end_align; } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* out_perm, @@ -2315,7 +2304,7 @@ Result KPageTable::CheckMemoryState(KMemoryState* out_state, KMemoryPermission* if (out_blocks_needed != nullptr) { *out_blocks_needed = blocks_for_start_align + blocks_for_end_align; } - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr addr, size_t size, @@ -2381,7 +2370,7 @@ Result KPageTable::LockMemoryAndOpen(KPageGroup* out_pg, PAddr* out_paddr, VAddr new_attr, KMemoryBlockDisableMergeAttribute::Locked, KMemoryBlockDisableMergeAttribute::None); - return ResultSuccess; + R_SUCCEED(); } Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask, @@ -2436,7 +2425,7 @@ Result KPageTable::UnlockMemory(VAddr addr, size_t size, KMemoryState state_mask new_attr, KMemoryBlockDisableMergeAttribute::None, KMemoryBlockDisableMergeAttribute::Locked); - return ResultSuccess; + R_SUCCEED(); } } // namespace Kernel -- cgit v1.2.3