From 306739c2c479b646270f7cd8000bb11483613d50 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Sun, 30 Sep 2018 18:13:22 -0400 Subject: ips_layer: Add IPSwitchCompiler to process IPSwitch format --- src/core/file_sys/ips_layer.cpp | 142 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) (limited to 'src/core/file_sys/ips_layer.cpp') diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index df933ee36..6bce138a8 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -2,7 +2,9 @@ // Licensed under GPLv2 or any later version // Refer to the license.txt file included. +#include #include "common/assert.h" +#include "common/hex_util.h" #include "common/swap.h" #include "core/file_sys/ips_layer.h" #include "core/file_sys/vfs_vector.h" @@ -85,4 +87,144 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { return std::make_shared(in_data, in->GetName(), in->GetContainingDirectory()); } +IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) + : valid(false), patch_text(std::move(patch_text_)), nso_build_id{}, is_little_endian(false), + offset_shift(0), print_values(false) { + Parse(); +} + +std::array IPSwitchCompiler::GetBuildID() const { + return nso_build_id; +} + +bool IPSwitchCompiler::IsValid() const { + return valid; +} + +static bool StartsWith(const std::string& base, const std::string& check) { + return base.size() >= check.size() && base.substr(0, check.size()) == check; +} + +void IPSwitchCompiler::Parse() { + const auto bytes = patch_text->ReadAllBytes(); + std::stringstream s; + s.write(reinterpret_cast(bytes.data()), bytes.size()); + + std::vector lines; + std::string line; + while (std::getline(s, line)) { + // Remove a trailing \r + if (!line.empty() && line[line.size() - 1] == '\r') + line = line.substr(0, line.size() - 1); + lines.push_back(line); + } + + for (std::size_t i = 0; i < lines.size(); ++i) { + auto line = lines[i]; + if (StartsWith(line, "@stop")) { + // Force stop + break; + } else if (StartsWith(line, "@nsobid-")) { + // NSO Build ID Specifier + auto raw_build_id = line.substr(8); + if (raw_build_id.size() != 0x40) + raw_build_id.resize(0x40, '0'); + nso_build_id = Common::HexStringToArray<0x20>(raw_build_id); + } else if (StartsWith(line, "@flag offset_shift ")) { + // Offset Shift Flag + offset_shift = std::stoull(line.substr(19), nullptr, 0); + } else if (StartsWith(line, "#")) { + // Mandatory Comment + LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}", + patch_text->GetName(), line.substr(1)); + } else if (StartsWith(line, "@little-endian")) { + // Set values to read as little endian + is_little_endian = true; + } else if (StartsWith(line, "@big-endian")) { + // Set values to read as big endian + is_little_endian = false; + } else if (StartsWith(line, "@flag print_values")) { + // Force printing of applied values + print_values = true; + } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) { + // Start of patch + const auto enabled = StartsWith(line, "@enabled"); + if (i == 0) + return; + const auto name = lines[i - 1].substr(3); + LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})", + patch_text->GetName(), name, line.substr(1)); + + IPSwitchPatch patch{name, enabled, {}}; + + // Read rest of patch + while (true) { + if (i + 1 >= lines.size()) + break; + line = lines[++i]; + + // 11 - 8 hex digit offset + space + minimum two digit overwrite val + if (line.length() < 11) + break; + auto offset = std::stoul(line.substr(0, 8), nullptr, 16); + offset += offset_shift; + + std::vector replace; + // 9 - first char of replacement val + if (line[9] == '\"') { + // string replacement + const auto end_index = line.find_last_of('\"'); + if (end_index == std::string::npos || end_index < 10) + return; + const auto value = line.substr(10, end_index - 10); + replace.reserve(value.size()); + std::copy(value.begin(), value.end(), std::back_inserter(replace)); + } else { + // hex replacement + const auto value = line.substr(9); + replace.reserve(value.size() / 2); + replace = Common::HexStringToVector(value, is_little_endian); + } + + if (print_values) { + LOG_INFO(Loader, + "[IPSwitchCompiler ('{}')] - Patching value at offset 0x{:08X} " + "with byte string '{}'", + patch_text->GetName(), offset, Common::HexVectorToString(replace)); + } + + patch.records.emplace(offset, replace); + } + + patches.push_back(std::move(patch)); + } + } + + valid = true; +} + +VirtualFile IPSwitchCompiler::Apply(const VirtualFile& in) const { + if (in == nullptr || !valid) + return nullptr; + + auto in_data = in->ReadAllBytes(); + + for (const auto& patch : patches) { + if (!patch.enabled) + continue; + + for (const auto& record : patch.records) { + if (record.first >= in_data.size()) + continue; + auto replace_size = record.second.size(); + if (record.first + replace_size > in_data.size()) + replace_size = in_data.size() - record.first; + for (std::size_t i = 0; i < replace_size; ++i) + in_data[i + record.first] = record.second[i]; + } + } + + return std::make_shared(in_data, in->GetName(), in->GetContainingDirectory()); +} + } // namespace FileSys -- cgit v1.2.3 From 9669cdb710e3242b7c0b705ea613587d36f79e00 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Mon, 1 Oct 2018 08:31:34 -0400 Subject: ips_layer: Add support for escape sequences and midline comments More accurately follows IPSwitch specification. --- src/core/file_sys/ips_layer.cpp | 45 ++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) (limited to 'src/core/file_sys/ips_layer.cpp') diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index 6bce138a8..6c5535f83 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -17,6 +17,11 @@ enum class IPSFileType { Error, }; +const std::map ESCAPE_CHARACTER_MAP{ + {"\\a", "\a"}, {"\\b", "\b"}, {"\\f", "\f"}, {"\\n", "\n"}, {"\\r", "\r"}, {"\\t", "\t"}, + {"\\v", "\v"}, {"\\\\", "\\"}, {"\\\'", "\'"}, {"\\\"", "\""}, {"\\\?", "\?"}, +}; + static IPSFileType IdentifyMagic(const std::vector& magic) { if (magic.size() != 5) return IPSFileType::Error; @@ -89,7 +94,7 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : valid(false), patch_text(std::move(patch_text_)), nso_build_id{}, is_little_endian(false), - offset_shift(0), print_values(false) { + offset_shift(0), print_values(false), last_comment("") { Parse(); } @@ -105,6 +110,18 @@ static bool StartsWith(const std::string& base, const std::string& check) { return base.size() >= check.size() && base.substr(0, check.size()) == check; } +static std::string EscapeStringSequences(std::string in) { + for (const auto& seq : ESCAPE_CHARACTER_MAP) { + for (auto index = in.find(seq.first); index != std::string::npos; + index = in.find(seq.first, index)) { + in.replace(index, std::strlen(seq.first), seq.second); + index += std::strlen(seq.second); + } + } + + return in; +} + void IPSwitchCompiler::Parse() { const auto bytes = patch_text->ReadAllBytes(); std::stringstream s; @@ -121,6 +138,13 @@ void IPSwitchCompiler::Parse() { for (std::size_t i = 0; i < lines.size(); ++i) { auto line = lines[i]; + + // Remove midline comments + if (!StartsWith(line, "//") && line.find("//") != std::string::npos) { + last_comment = line.substr(line.find("//") + 2); + line = line.substr(0, line.find("//")); + } + if (StartsWith(line, "@stop")) { // Force stop break; @@ -132,11 +156,18 @@ void IPSwitchCompiler::Parse() { nso_build_id = Common::HexStringToArray<0x20>(raw_build_id); } else if (StartsWith(line, "@flag offset_shift ")) { // Offset Shift Flag - offset_shift = std::stoull(line.substr(19), nullptr, 0); + offset_shift = std::stoll(line.substr(19), nullptr, 0); } else if (StartsWith(line, "#")) { // Mandatory Comment LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}", patch_text->GetName(), line.substr(1)); + } else if (StartsWith(line, "//")) { + // Normal Comment + last_comment = line.substr(2); + if (last_comment.find_first_not_of(' ') == std::string::npos) + continue; + if (last_comment.find_first_not_of(' ') != 0) + last_comment = last_comment.substr(last_comment.find_first_not_of(' ')); } else if (StartsWith(line, "@little-endian")) { // Set values to read as little endian is_little_endian = true; @@ -151,11 +182,10 @@ void IPSwitchCompiler::Parse() { const auto enabled = StartsWith(line, "@enabled"); if (i == 0) return; - const auto name = lines[i - 1].substr(3); LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Parsing patch '{}' ({})", - patch_text->GetName(), name, line.substr(1)); + patch_text->GetName(), last_comment, line.substr(1)); - IPSwitchPatch patch{name, enabled, {}}; + IPSwitchPatch patch{last_comment, enabled, {}}; // Read rest of patch while (true) { @@ -173,10 +203,11 @@ void IPSwitchCompiler::Parse() { // 9 - first char of replacement val if (line[9] == '\"') { // string replacement - const auto end_index = line.find_last_of('\"'); + const auto end_index = line.find('\"', 10); if (end_index == std::string::npos || end_index < 10) return; - const auto value = line.substr(10, end_index - 10); + auto value = line.substr(10, end_index - 10); + value = EscapeStringSequences(value); replace.reserve(value.size()); std::copy(value.begin(), value.end(), std::back_inserter(replace)); } else { -- cgit v1.2.3 From 70bd2bb1d3aec23fc052c9612f9e530c1a2de72d Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Mon, 1 Oct 2018 17:17:08 -0400 Subject: ips_layer: Deduplicate resource usage --- src/core/file_sys/ips_layer.cpp | 48 +++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 23 deletions(-) (limited to 'src/core/file_sys/ips_layer.cpp') diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index 6c5535f83..abe3dcf62 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -17,9 +17,10 @@ enum class IPSFileType { Error, }; -const std::map ESCAPE_CHARACTER_MAP{ - {"\\a", "\a"}, {"\\b", "\b"}, {"\\f", "\f"}, {"\\n", "\n"}, {"\\r", "\r"}, {"\\t", "\t"}, - {"\\v", "\v"}, {"\\\\", "\\"}, {"\\\'", "\'"}, {"\\\"", "\""}, {"\\\?", "\?"}, +constexpr std::array, 11> ESCAPE_CHARACTER_MAP{ + std::pair{"\\a", "\a"}, {"\\b", "\b"}, {"\\f", "\f"}, {"\\n", "\n"}, + {"\\r", "\r"}, {"\\t", "\t"}, {"\\v", "\v"}, {"\\\\", "\\"}, + {"\\\'", "\'"}, {"\\\"", "\""}, {"\\\?", "\?"}, }; static IPSFileType IdentifyMagic(const std::vector& magic) { @@ -92,12 +93,12 @@ VirtualFile PatchIPS(const VirtualFile& in, const VirtualFile& ips) { return std::make_shared(in_data, in->GetName(), in->GetContainingDirectory()); } -IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) - : valid(false), patch_text(std::move(patch_text_)), nso_build_id{}, is_little_endian(false), - offset_shift(0), print_values(false), last_comment("") { +IPSwitchCompiler::IPSwitchCompiler(VirtualFile patch_text_) : patch_text(std::move(patch_text_)) { Parse(); } +IPSwitchCompiler::~IPSwitchCompiler() = default; + std::array IPSwitchCompiler::GetBuildID() const { return nso_build_id; } @@ -106,7 +107,7 @@ bool IPSwitchCompiler::IsValid() const { return valid; } -static bool StartsWith(const std::string& base, const std::string& check) { +static bool StartsWith(std::string_view base, std::string_view check) { return base.size() >= check.size() && base.substr(0, check.size()) == check; } @@ -128,21 +129,22 @@ void IPSwitchCompiler::Parse() { s.write(reinterpret_cast(bytes.data()), bytes.size()); std::vector lines; - std::string line; - while (std::getline(s, line)) { + std::string stream_line; + while (std::getline(s, stream_line)) { // Remove a trailing \r - if (!line.empty() && line[line.size() - 1] == '\r') - line = line.substr(0, line.size() - 1); - lines.push_back(line); + if (!stream_line.empty() && stream_line.back() == '\r') + stream_line.pop_back(); + lines.push_back(std::move(stream_line)); } for (std::size_t i = 0; i < lines.size(); ++i) { auto line = lines[i]; // Remove midline comments - if (!StartsWith(line, "//") && line.find("//") != std::string::npos) { - last_comment = line.substr(line.find("//") + 2); - line = line.substr(0, line.find("//")); + const auto comment_index = line.find("//"); + if (!StartsWith(line, "//") && comment_index != std::string::npos) { + last_comment = line.substr(comment_index + 2); + line = line.substr(0, comment_index); } if (StartsWith(line, "@stop")) { @@ -191,28 +193,28 @@ void IPSwitchCompiler::Parse() { while (true) { if (i + 1 >= lines.size()) break; - line = lines[++i]; + const auto patch_line = lines[++i]; // 11 - 8 hex digit offset + space + minimum two digit overwrite val - if (line.length() < 11) + if (patch_line.length() < 11) break; - auto offset = std::stoul(line.substr(0, 8), nullptr, 16); + auto offset = std::stoul(patch_line.substr(0, 8), nullptr, 16); offset += offset_shift; std::vector replace; // 9 - first char of replacement val - if (line[9] == '\"') { + if (patch_line[9] == '\"') { // string replacement - const auto end_index = line.find('\"', 10); + const auto end_index = patch_line.find('\"', 10); if (end_index == std::string::npos || end_index < 10) return; - auto value = line.substr(10, end_index - 10); + auto value = patch_line.substr(10, end_index - 10); value = EscapeStringSequences(value); replace.reserve(value.size()); std::copy(value.begin(), value.end(), std::back_inserter(replace)); } else { // hex replacement - const auto value = line.substr(9); + const auto value = patch_line.substr(9); replace.reserve(value.size() / 2); replace = Common::HexStringToVector(value, is_little_endian); } @@ -224,7 +226,7 @@ void IPSwitchCompiler::Parse() { patch_text->GetName(), offset, Common::HexVectorToString(replace)); } - patch.records.emplace(offset, replace); + patch.records.emplace(offset, std::move(replace)); } patches.push_back(std::move(patch)); -- cgit v1.2.3 From 110d5784702282c594fe57f84f8d6bda21a82d50 Mon Sep 17 00:00:00 2001 From: Zach Hilman Date: Thu, 4 Oct 2018 12:23:18 -0400 Subject: ips_layer: Fix inaccuracies with comments and flags Specifically bugs/crashes that arise when putting them in positions that are legal but not typical, such as midline, between patch data, or between patch records. --- src/core/file_sys/ips_layer.cpp | 64 +++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 15 deletions(-) (limited to 'src/core/file_sys/ips_layer.cpp') diff --git a/src/core/file_sys/ips_layer.cpp b/src/core/file_sys/ips_layer.cpp index abe3dcf62..0cadbc375 100644 --- a/src/core/file_sys/ips_layer.cpp +++ b/src/core/file_sys/ips_layer.cpp @@ -123,6 +123,22 @@ static std::string EscapeStringSequences(std::string in) { return in; } +void IPSwitchCompiler::ParseFlag(const std::string& line) { + if (StartsWith(line, "@flag offset_shift ")) { + // Offset Shift Flag + offset_shift = std::stoll(line.substr(19), nullptr, 0); + } else if (StartsWith(line, "@little-endian")) { + // Set values to read as little endian + is_little_endian = true; + } else if (StartsWith(line, "@big-endian")) { + // Set values to read as big endian + is_little_endian = false; + } else if (StartsWith(line, "@flag print_values")) { + // Force printing of applied values + print_values = true; + } +} + void IPSwitchCompiler::Parse() { const auto bytes = patch_text->ReadAllBytes(); std::stringstream s; @@ -141,7 +157,17 @@ void IPSwitchCompiler::Parse() { auto line = lines[i]; // Remove midline comments - const auto comment_index = line.find("//"); + std::size_t comment_index = std::string::npos; + bool within_string = false; + for (std::size_t k = 0; k < line.size(); ++k) { + if (line[k] == '\"' && (k > 0 && line[k - 1] != '\\')) { + within_string = !within_string; + } else if (line[k] == '\\' && (k < line.size() - 1 && line[k + 1] == '\\')) { + comment_index = k; + break; + } + } + if (!StartsWith(line, "//") && comment_index != std::string::npos) { last_comment = line.substr(comment_index + 2); line = line.substr(0, comment_index); @@ -156,9 +182,6 @@ void IPSwitchCompiler::Parse() { if (raw_build_id.size() != 0x40) raw_build_id.resize(0x40, '0'); nso_build_id = Common::HexStringToArray<0x20>(raw_build_id); - } else if (StartsWith(line, "@flag offset_shift ")) { - // Offset Shift Flag - offset_shift = std::stoll(line.substr(19), nullptr, 0); } else if (StartsWith(line, "#")) { // Mandatory Comment LOG_INFO(Loader, "[IPSwitchCompiler ('{}')] Forced output comment: {}", @@ -170,15 +193,6 @@ void IPSwitchCompiler::Parse() { continue; if (last_comment.find_first_not_of(' ') != 0) last_comment = last_comment.substr(last_comment.find_first_not_of(' ')); - } else if (StartsWith(line, "@little-endian")) { - // Set values to read as little endian - is_little_endian = true; - } else if (StartsWith(line, "@big-endian")) { - // Set values to read as big endian - is_little_endian = false; - } else if (StartsWith(line, "@flag print_values")) { - // Force printing of applied values - print_values = true; } else if (StartsWith(line, "@enabled") || StartsWith(line, "@disabled")) { // Start of patch const auto enabled = StartsWith(line, "@enabled"); @@ -195,6 +209,18 @@ void IPSwitchCompiler::Parse() { break; const auto patch_line = lines[++i]; + // Start of new patch + if (StartsWith(patch_line, "@enabled") || StartsWith(patch_line, "@disabled")) { + --i; + break; + } + + // Check for a flag + if (StartsWith(patch_line, "@")) { + ParseFlag(patch_line); + continue; + } + // 11 - 8 hex digit offset + space + minimum two digit overwrite val if (patch_line.length() < 11) break; @@ -205,9 +231,15 @@ void IPSwitchCompiler::Parse() { // 9 - first char of replacement val if (patch_line[9] == '\"') { // string replacement - const auto end_index = patch_line.find('\"', 10); + auto end_index = patch_line.find('\"', 10); if (end_index == std::string::npos || end_index < 10) return; + while (patch_line[end_index - 1] == '\\') { + end_index = patch_line.find('\"', end_index + 1); + if (end_index == std::string::npos || end_index < 10) + return; + } + auto value = patch_line.substr(10, end_index - 10); value = EscapeStringSequences(value); replace.reserve(value.size()); @@ -226,10 +258,12 @@ void IPSwitchCompiler::Parse() { patch_text->GetName(), offset, Common::HexVectorToString(replace)); } - patch.records.emplace(offset, std::move(replace)); + patch.records.insert_or_assign(offset, std::move(replace)); } patches.push_back(std::move(patch)); + } else if (StartsWith(line, "@")) { + ParseFlag(line); } } -- cgit v1.2.3