From 2b0d2e0a4f16e34c5e2180dd6d35b0c01e58aca6 Mon Sep 17 00:00:00 2001 From: Pavel Shevaev Date: Fri, 25 Apr 2025 15:16:50 +0300 Subject: [PATCH] Adding basic support for fmt3 patching; Update result is now empty() if no files affected, added or removed --- config.inc.php | 6 +- pack.inc.php | 167 ++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 140 insertions(+), 33 deletions(-) diff --git a/config.inc.php b/config.inc.php index c70272b..be667a4 100644 --- a/config.inc.php +++ b/config.inc.php @@ -125,7 +125,9 @@ class ConfigUpdateResult function isEmpty() : bool { - return count($this->affected_files) == 0; + return count($this->affected_files) == 0 && + count($this->added_files) == 0 && + count($this->removed_files) == 0; } function isPatchPossible() : bool @@ -221,7 +223,7 @@ class ConfigManager $affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files); $affected_conf_files = $this->filterConfigFiles($affected_all_files); - config_log("Affected configs: {$affected_conf_files->count()}/{$req->files->count()}"); + config_log("Affected configs: {$affected_conf_files->count()} (request: {$req->files->count()})"); $update_result = new ConfigUpdateResult(); $update_result->request = $req; diff --git a/pack.inc.php b/pack.inc.php index 0344a25..cbee470 100644 --- a/pack.inc.php +++ b/pack.inc.php @@ -102,6 +102,16 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st $params->version, ); } + else if($params->binary_format == 3) + { + $patched_data = _config_patch_bundle_fmt3( + $packed_data, + $params->cache_entries, + $params->use_lz4, + $params->use_config_refs, + $params->version, + ); + } else throw new Exception("Unknown binary format: {$params->binary_format}"); @@ -236,7 +246,7 @@ function _config_pack_bundle_fmt2( return $packed_data; } -//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed. +//NOTE: Much like fmt1, but configs entries are grouped into sizeable chunks, with each chunk lz4-compressed. // This reduces overall bundle size when there are many small configs entries. function _config_pack_bundle_fmt3( array $cache_entries, @@ -351,12 +361,12 @@ function _config_patch_bundle_fmt2( { list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs); - $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); + $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); - if($header_found) + if($headers_found) { - $header_idx = key($header_found); - $header_entry = current($header_found); + $header_idx = key($headers_found); + $header_entry = current($headers_found); $current_offset = $header_entry[4]; $current_size = $header_entry[5]; @@ -414,6 +424,95 @@ function _config_patch_bundle_fmt2( return $patched_data; } +function _config_patch_bundle_fmt3( + string $packed_data, + array $patch_entries, + bool $use_lz4, + bool $use_config_refs, + int $version, +) : string +{ + if(!$use_lz4) + throw new Exception("Config bundle FMT3 is only available with LZ4 enabled"); + + list($header, $entries, $chunks_offset, $max_chunk_size) = + _config_unpack_bundle_fmt3(packed_data: $packed_data, unpack_entries: false); + + $payloads_bundle = substr($packed_data, $chunks_offset); + + foreach($patch_entries as $idx => $patch_entry) + { + list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs); + + $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); + + if($headers_found) + { + $header_idx = key($headers_found); + $header_entry = current($headers_found); + + $chunk_offset = $header_entry[4]; + $payloads_offset = $header_entry[5]; //within chunk + $payloads_size = $header_entry[6]; + + $chunk_entries = array_filter($header, fn($item) => $item[4] == $chunk_offset); + + //chunk contains only one patch entry + $patched_chunk = lz4_compress($patch_payload, 9); + if(strlen($patched_chunk) > $max_chunk_size) + $max_chunk_size = strlen($patched_chunk); + + //TODO: make a better generic algorithm for patching the whole chunk + //special case for single entry + if(count($chunk_entries) == 1 && $payloads_offset == 0) + { + $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1]; + + //we can just replace chunk in place if the size of a patched chunk is less or equal + if(strlen($patched_chunk) <= $lz4_chunk_size) + { + $payloads_bundle = substr_replace($payloads_bundle, $patched_chunk, $chunk_offset, strlen($patched_chunk)); + $header_entry[6] = strlen($patch_payload); + } + //just append to the end + else + { + $header_entry[4] = strlen($payloads_bundle); //chunk offset + $header_entry[5] = 0; + $header_entry[6] = strlen($patch_payload); + + $payloads_bundle .= $patched_chunk; + } + } + //just append to the end + else + { + $header_entry[4] = strlen($payloads_bundle); //chunk offset + $header_entry[5] = 0; + $header_entry[6] = strlen($patch_payload); + + $payloads_bundle .= $patched_chunk; + } + + $header[$header_idx] = $header_entry; + } + else + throw new Exception("Patched entry {$patch_entry->id} not found in config bundle"); + } + + $header_msgpack = config_msgpack_pack($header); + + $packed_data = + pack("C", 3) . + pack("V", $version) . + pack("V", strlen($header_msgpack)) . + pack("V", $max_chunk_size) . + $header_msgpack . + $payloads_bundle; + + return $packed_data; +} + //format: [[class_id, [data]], ...[class_id, [data]]] function config_unpack_bundle(string $packed_data) : array { @@ -431,7 +530,8 @@ function config_unpack_bundle(string $packed_data) : array } else if($info['format'] === 3) { - return _config_unpack_bundle_fmt3($packed_data); + list($_, $entries) = _config_unpack_bundle_fmt3($packed_data); + return $entries; } else throw new Exception("Unknown format: {$info['format']}"); @@ -498,12 +598,10 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries = return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle); } -function _config_unpack_bundle_fmt3(string $packed_data): array +function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries = true): array { if(ord($packed_data[0]) !== 3) - { throw new Exception("Invalid config bundle format"); - } $offset = 1; $version = unpack("V", substr($packed_data, $offset, 4))[1]; @@ -515,6 +613,7 @@ function _config_unpack_bundle_fmt3(string $packed_data): array $header_msgpack = substr($packed_data, $offset, $header_len); $offset += $header_len; + $chunks_offset = $offset; $header = config_msgpack_unpack($header_msgpack); @@ -523,39 +622,45 @@ function _config_unpack_bundle_fmt3(string $packed_data): array $chunk_buffer = ''; $chunk_id = -1; - foreach ($header as $entry_data) + foreach($header as $entry_data) { list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data; - if($entry_chunk_offset !== $chunk_id) + $unpacked_payload = null; + + if($unpack_entries) { - if($chunk_offset !== -1) + if($entry_chunk_offset !== $chunk_id) { - $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1]; - $offset+=4; - $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size); - $chunk_buffer = lz4_uncompress($lz4_chunk_data); - $offset += $lz4_chunk_size; - $chunk_offset = $offset; - } - else - { - $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1]; - $chunk_offset += 4; - $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size); - $chunk_buffer = lz4_uncompress($lz4_chunk_data); - $chunk_offset += $lz4_chunk_size; + if($chunk_offset !== -1) + { + $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1]; + $offset+=4; + $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size); + $chunk_buffer = lz4_uncompress($lz4_chunk_data); + $offset += $lz4_chunk_size; + $chunk_offset = $offset; + } + else + { + $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1]; + $chunk_offset += 4; + $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size); + $chunk_buffer = lz4_uncompress($lz4_chunk_data); + $chunk_offset += $lz4_chunk_size; + } + + $chunk_id = $entry_chunk_offset; } - $chunk_id = $entry_chunk_offset; + $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size); + $unpacked_payload = _config_unpack_payload($format, $payload); } - $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size); - - $cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); + $cache_entries[$id] = array($class_id, $unpacked_payload); } - return $cache_entries; + return array($header, $cache_entries, $chunks_offset, $max_chunk_size); } //format: [format_id, payload_data]