bit
/
taskman_config
25
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Adding basic support for fmt3 patching; Update result is now empty() if no files affected, added or removed
Publish PHP Package / docker (push) Successful in 12s Details

This commit is contained in:
Pavel Shevaev 2025-04-25 15:16:50 +03:00
parent 8c0604b9fa
commit 2b0d2e0a4f
2 changed files with 140 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config.inc.php
View File

@ -125,7 +125,9 @@ class ConfigUpdateResult
function isEmpty() : bool
{
return count($this->affected_files) == 0;
return count($this->affected_files) == 0 &&
count($this->added_files) == 0 &&
count($this->removed_files) == 0;
}
function isPatchPossible() : bool
@ -221,7 +223,7 @@ class ConfigManager
$affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
$affected_conf_files = $this->filterConfigFiles($affected_all_files);
config_log("Affected configs: {$affected_conf_files->count()}/{$req->files->count()}");
config_log("Affected configs: {$affected_conf_files->count()} (request: {$req->files->count()})");
$update_result = new ConfigUpdateResult();
$update_result->request = $req;

167
pack.inc.php
View File

@ -102,6 +102,16 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
$params->version,
);
}
else if($params->binary_format == 3)
{
$patched_data = _config_patch_bundle_fmt3(
$packed_data,
$params->cache_entries,
$params->use_lz4,
$params->use_config_refs,
$params->version,
);
}
else
throw new Exception("Unknown binary format: {$params->binary_format}");
@ -236,7 +246,7 @@ function _config_pack_bundle_fmt2(
return $packed_data;
}
//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
//NOTE: Much like fmt1, but configs entries are grouped into sizeable chunks, with each chunk lz4-compressed.
// This reduces overall bundle size when there are many small configs entries.
function _config_pack_bundle_fmt3(
array $cache_entries,
@ -351,12 +361,12 @@ function _config_patch_bundle_fmt2(
{
list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
$header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
$headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
if($header_found)
if($headers_found)
{
$header_idx = key($header_found);
$header_entry = current($header_found);
$header_idx = key($headers_found);
$header_entry = current($headers_found);
$current_offset = $header_entry[4];
$current_size = $header_entry[5];
@ -414,6 +424,95 @@ function _config_patch_bundle_fmt2(
return $patched_data;
}
function _config_patch_bundle_fmt3(
string $packed_data,
array $patch_entries,
bool $use_lz4,
bool $use_config_refs,
int $version,
) : string
{
if(!$use_lz4)
throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
list($header, $entries, $chunks_offset, $max_chunk_size) =
_config_unpack_bundle_fmt3(packed_data: $packed_data, unpack_entries: false);
$payloads_bundle = substr($packed_data, $chunks_offset);
foreach($patch_entries as $idx => $patch_entry)
{
list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
$headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
if($headers_found)
{
$header_idx = key($headers_found);
$header_entry = current($headers_found);
$chunk_offset = $header_entry[4];
$payloads_offset = $header_entry[5]; //within chunk
$payloads_size = $header_entry[6];
$chunk_entries = array_filter($header, fn($item) => $item[4] == $chunk_offset);
//chunk contains only one patch entry
$patched_chunk = lz4_compress($patch_payload, 9);
if(strlen($patched_chunk) > $max_chunk_size)
$max_chunk_size = strlen($patched_chunk);
//TODO: make a better generic algorithm for patching the whole chunk
//special case for single entry
if(count($chunk_entries) == 1 && $payloads_offset == 0)
{
$lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
//we can just replace chunk in place if the size of a patched chunk is less or equal
if(strlen($patched_chunk) <= $lz4_chunk_size)
{
$payloads_bundle = substr_replace($payloads_bundle, $patched_chunk, $chunk_offset, strlen($patched_chunk));
$header_entry[6] = strlen($patch_payload);
}
//just append to the end
else
{
$header_entry[4] = strlen($payloads_bundle); //chunk offset
$header_entry[5] = 0;
$header_entry[6] = strlen($patch_payload);
$payloads_bundle .= $patched_chunk;
}
}
//just append to the end
else
{
$header_entry[4] = strlen($payloads_bundle); //chunk offset
$header_entry[5] = 0;
$header_entry[6] = strlen($patch_payload);
$payloads_bundle .= $patched_chunk;
}
$header[$header_idx] = $header_entry;
}
else
throw new Exception("Patched entry {$patch_entry->id} not found in config bundle");
}
$header_msgpack = config_msgpack_pack($header);
$packed_data =
pack("C", 3) .
pack("V", $version) .
pack("V", strlen($header_msgpack)) .
pack("V", $max_chunk_size) .
$header_msgpack .
$payloads_bundle;
return $packed_data;
}
//format: [[class_id, [data]], ...[class_id, [data]]]
function config_unpack_bundle(string $packed_data) : array
{
@ -431,7 +530,8 @@ function config_unpack_bundle(string $packed_data) : array
}
else if($info['format'] === 3)
{
return _config_unpack_bundle_fmt3($packed_data);
list($_, $entries) = _config_unpack_bundle_fmt3($packed_data);
return $entries;
}
else
throw new Exception("Unknown format: {$info['format']}");
@ -498,12 +598,10 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries =
return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
}
function _config_unpack_bundle_fmt3(string $packed_data): array
function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries = true): array
{
if(ord($packed_data[0]) !== 3)
{
throw new Exception("Invalid config bundle format");
}
$offset = 1;
$version = unpack("V", substr($packed_data, $offset, 4))[1];
@ -515,6 +613,7 @@ function _config_unpack_bundle_fmt3(string $packed_data): array
$header_msgpack = substr($packed_data, $offset, $header_len);
$offset += $header_len;
$chunks_offset = $offset;
$header = config_msgpack_unpack($header_msgpack);
@ -523,39 +622,45 @@ function _config_unpack_bundle_fmt3(string $packed_data): array
$chunk_buffer = '';
$chunk_id = -1;
foreach ($header as $entry_data)
foreach($header as $entry_data)
{
list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
if($entry_chunk_offset !== $chunk_id)
$unpacked_payload = null;
if($unpack_entries)
{
if($chunk_offset !== -1)
if($entry_chunk_offset !== $chunk_id)
{
$lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
$offset+=4;
$lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
$offset += $lz4_chunk_size;
$chunk_offset = $offset;
}
else
{
$lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
$chunk_offset += 4;
$lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
$chunk_offset += $lz4_chunk_size;
if($chunk_offset !== -1)
{
$lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
$offset+=4;
$lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
$offset += $lz4_chunk_size;
$chunk_offset = $offset;
}
else
{
$lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
$chunk_offset += 4;
$lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
$chunk_offset += $lz4_chunk_size;
}
$chunk_id = $entry_chunk_offset;
}
$chunk_id = $entry_chunk_offset;
$payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
$unpacked_payload = _config_unpack_payload($format, $payload);
}
$payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
$cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
$cache_entries[$id] = array($class_id, $unpacked_payload);
}
return $cache_entries;
return array($header, $cache_entries, $chunks_offset, $max_chunk_size);
}
//format: [format_id, payload_data]