bit
/
taskman_config
25
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Support for configs bundle patching WIP; Migrating to iterable from array where it makes sense; Adding config_hash_changed(..)
Publish PHP Package / docker (push) Successful in 6s Details

This commit is contained in:
Pavel Shevaev 2025-03-06 13:23:43 +03:00
parent 77a63241ff
commit fd49e48a57
2 changed files with 261 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

181
pack.inc.php
View File

@ -12,8 +12,14 @@ class ConfigPackParams
public ?int $version = null;
public bool $debug = false;
function __construct(array $cache_entries, int $version, bool $use_lz4 = false,
bool $use_config_refs = false, int $binary_format = 1, bool $debug = false)
function __construct(
array $cache_entries,
int $version,
bool $use_lz4 = false,
bool $use_config_refs = false,
int $binary_format = 1,
bool $debug = false
)
{
$this->cache_entries = $cache_entries;
$this->use_lz4 = $use_lz4;
@ -45,7 +51,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
$params->cache_entries,
$params->use_lz4,
$params->use_config_refs,
$params->version
$params->version,
);
}
else
@ -61,6 +67,35 @@ function config_pack_bundle(ConfigPackParams $params) : string
return $packed_data;
}
function config_patch_bundle(ConfigPackParams $params, string $packed_data) : string
{
$t = microtime(true);
$patched_data = null;
if($params->binary_format == 2)
{
$patched_data = _config_patch_bundle_fmt2(
$packed_data,
$params->cache_entries,
$params->use_lz4,
$params->use_config_refs,
$params->version,
);
}
else
throw new Exception("Unknown binary format: {$params->binary_format}");
if($params->debug)
config_log("Patched entries: " . sizeof($params->cache_entries) . ", total: " .
kb($patched_data) . ", format: {$params->binary_format}, lz4: " .
var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) .
", CRC: " . crc32($patched_data) .
", " . round(microtime(true) - $t,2) . " sec.");
return $patched_data;
}
//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build)
function _config_pack_bundle_fmt1(
array $cache_entries,
@ -118,12 +153,14 @@ function _config_pack_bundle_fmt1(
return $packed_data;
}
//NOTE: strids are stored as lookup strings
//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
// (each path item separated by '/' is stored as an array item)
function _config_pack_bundle_fmt2(
array $cache_entries,
bool $use_lz4,
bool $use_config_refs,
int $version) : string
int $version,
) : string
{
$MAP = array();
$STRIDMAP = array();
@ -139,20 +176,8 @@ function _config_pack_bundle_fmt2(
$payloads[] = array($payloads_offset, $payload, $format, $payload_size);
$payloads_offset += $payload_size;
$strids_indices = array();
$strid_parts = explode('/', ltrim($entry->strid, '@'));
foreach($strid_parts as $strid_part)
{
if(!isset($STRIDMAP[$strid_part]))
{
$strid_index = count($STRIDLIST);
$STRIDLIST[] = $strid_part;
$STRIDMAP[$strid_part] = $strid_index;
$strids_indices[] = $strid_index;
}
else
$strids_indices[] = $STRIDMAP[$strid_part];
}
$strids_indices = _config_encode_strid_as_indices($entry->strid, $STRIDMAP, $STRIDLIST);
$strids[] = $strids_indices;
}
@ -192,6 +217,105 @@ function _config_pack_bundle_fmt2(
return $packed_data;
}
function _config_encode_strid_as_indices(string $strid, array &$STRIDMAP, array &$STRIDLIST) : array
{
$strids_indices = array();
$strid_parts = explode('/', ltrim($strid, '@'));
foreach($strid_parts as $strid_part)
{
if(!isset($STRIDMAP[$strid_part]))
{
$strid_index = count($STRIDLIST);
$STRIDLIST[] = $strid_part;
$STRIDMAP[$strid_part] = $strid_index;
$strids_indices[] = $strid_index;
}
else
$strids_indices[] = $STRIDMAP[$strid_part];
}
return $strids_indices;
}
function _config_patch_bundle_fmt2(
string $packed_data,
array $patch_entries,
bool $use_lz4,
bool $use_config_refs,
int $version,
) : string
{
list($strids, $header, $_, $payloads_bundle) =
_config_unpack_bundle_fmt2(packed_data: $packed_data, unpack_entries: false);
$stridmap = array_flip($strids);
foreach($patch_entries as $idx => $patch_entry)
{
list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
$header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
if($header_found)
{
$header_idx = key($header_found);
$header_entry = current($header_found);
$current_offset = $header_entry[4];
$current_size = $header_entry[5];
if($current_size >= strlen($patch_payload))
{
//let's do the inline patching
$payloads_bundle = substr_replace($payloads_bundle, $patch_payload, $current_offset, strlen($patch_payload));
$header_entry[0] = $patch_format;
$header_entry[5] = strlen($patch_payload);
$header[$header_idx] = $header_entry;
}
else
{
//let's add it to the end
$header_entry[0] = $patch_format;
$header_entry[4] = strlen($payloads_bundle);
$header_entry[5] = strlen($patch_payload);
$header[$header_idx] = $header_entry;
$payloads_bundle .= $patch_payload;
}
}
else
{
//let's add new entry it to the end
$strid_indices = _config_encode_strid_as_indices($patch_entry->strid, $stridmap, $strids);
$header_entry = array(
$patch_format,
$patch_entry->id,
$strid_indices,
$patch_entry->class_id,
strlen($payloads_bundle),
strlen($patch_payload),
);
$header[] = $header_entry;
$payloads_bundle .= $patch_payload;
}
}
$strids_msgpack = config_msgpack_pack($strids);
$header_msgpack = config_msgpack_pack($header);
$patched_data =
pack("C", 2) .
pack("V", $version) .
pack("V", strlen($strids_msgpack)) .
pack("V", strlen($header_msgpack)) .
$strids_msgpack .
$header_msgpack .
$payloads_bundle;
return $patched_data;
}
//format: [[class_id, [data]], ...[class_id, [data]]]
function config_unpack_bundle(string $packed_data) : array
{
@ -204,7 +328,8 @@ function config_unpack_bundle(string $packed_data) : array
}
else if($info['format'] === 2)
{
return _config_unpack_bundle_fmt2($packed_data);
list($_, $_, $entries) = _config_unpack_bundle_fmt2($packed_data);
return $entries;
}
else
throw new Exception("Unknown format: {$info['format']}");
@ -237,7 +362,7 @@ function _config_unpack_bundle_fmt1(string $packed_data) : array
return $entries;
}
function _config_unpack_bundle_fmt2(string $packed_data) : array
function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries = true) : array
{
$packed_info = substr($packed_data, 0, 1+4+4+4);
@ -255,16 +380,20 @@ function _config_unpack_bundle_fmt2(string $packed_data) : array
$payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
$entries = array();
foreach($header as $item)
if($unpack_entries)
{
list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
foreach($header as $item)
{
list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
$payload = substr($payloads_bundle, $offset, $size);
$payload = substr($payloads_bundle, $offset, $size);
$entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
$entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
}
}
return $entries;
return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
}
//format: [format_id, payload_data]

108
scan.inc.php
View File

@ -2,11 +2,24 @@
namespace taskman;
use Exception;
class ConfigScanResult
class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator
{
/*var array<string, string[]>*/
public array $base_dir2files = array();
private $iter_pos = 0;
function __construct(array $base_dir2files = array())
{
foreach($base_dir2files as $dir => $files)
$this->base_dir2files[$dir] = $files;
}
function clear()
{
$this->base_dir2files = array();
}
function isEmpty() : bool
{
return empty($this->base_dir2files);
@ -69,10 +82,94 @@ class ConfigScanResult
return $all_files;
}
//ArrayAccess interface
function offsetExists(mixed $offset) : bool
{
if(!is_int($offset))
throw new Exception("Invalid offset");
return $this->count() > $offset;
}
function offsetGet(mixed $offset) : mixed
{
if(!is_int($offset))
throw new Exception("Invalid offset");
foreach($this->base_dir2files as $base_dir => $files)
{
$n = count($files);
if($offset - $n < 0)
return $files[$offset];
$offset -= $n;
}
return null;
}
function offsetSet(mixed $offset, mixed $value) : void
{
if(!is_int($offset))
throw new Exception("Invalid offset");
foreach($this->base_dir2files as $base_dir => &$files)
{
$n = count($files);
if($offset - $n < 0)
{
$files[$offset] = $value;
return;
}
$offset -= $n;
}
}
function offsetUnset(mixed $offset) : void
{
if(!is_int($offset))
throw new Exception("Invalid offset");
foreach($this->base_dir2files as $base_dir => $files)
{
$n = count($files);
if($offset - $n < 0)
{
unset($files[$offset]);
return;
}
$offset -= $n;
}
}
//Iterator interface
function rewind() : void
{
$this->iter_pos = 0;
}
function current() : mixed
{
return $this->offsetGet($this->iter_pos);
}
function key() : mixed
{
return $this->iter_pos;
}
function next() : void
{
++$this->iter_pos;
}
function valid() : bool
{
return $this->offsetExists($this->iter_pos);
}
}
function config_scan_files(
array $base_dirs,
iterable $base_dirs,
string $ext_filter = '.conf.js',
bool $verbose = false
) : ConfigScanResult
@ -92,3 +189,10 @@ function config_scan_files(
return $result;
}
function config_hash_changed(ConfigGlobals $globals, iterable $all_files)
{
$all_crc_file = $globals->build_dir . "/configs.crc";
return names_hash_changed($all_crc_file, $all_files);
}