diff --git a/pack.inc.php b/pack.inc.php index f93cc3b..9c6d394 100644 --- a/pack.inc.php +++ b/pack.inc.php @@ -12,8 +12,14 @@ class ConfigPackParams public ?int $version = null; public bool $debug = false; - function __construct(array $cache_entries, int $version, bool $use_lz4 = false, - bool $use_config_refs = false, int $binary_format = 1, bool $debug = false) + function __construct( + array $cache_entries, + int $version, + bool $use_lz4 = false, + bool $use_config_refs = false, + int $binary_format = 1, + bool $debug = false + ) { $this->cache_entries = $cache_entries; $this->use_lz4 = $use_lz4; @@ -45,7 +51,7 @@ function config_pack_bundle(ConfigPackParams $params) : string $params->cache_entries, $params->use_lz4, $params->use_config_refs, - $params->version + $params->version, ); } else @@ -61,6 +67,35 @@ function config_pack_bundle(ConfigPackParams $params) : string return $packed_data; } +function config_patch_bundle(ConfigPackParams $params, string $packed_data) : string +{ + $t = microtime(true); + + $patched_data = null; + + if($params->binary_format == 2) + { + $patched_data = _config_patch_bundle_fmt2( + $packed_data, + $params->cache_entries, + $params->use_lz4, + $params->use_config_refs, + $params->version, + ); + } + else + throw new Exception("Unknown binary format: {$params->binary_format}"); + + if($params->debug) + config_log("Patched entries: " . sizeof($params->cache_entries) . ", total: " . + kb($patched_data) . ", format: {$params->binary_format}, lz4: " . + var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . + ", CRC: " . crc32($patched_data) . + ", " . round(microtime(true) - $t,2) . " sec."); + + return $patched_data; +} + //NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) function _config_pack_bundle_fmt1( array $cache_entries, @@ -118,12 +153,14 @@ function _config_pack_bundle_fmt1( return $packed_data; } -//NOTE: strids are stored as lookup strings +//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices +// (each path item separated by '/' is stored as an array item) function _config_pack_bundle_fmt2( array $cache_entries, bool $use_lz4, bool $use_config_refs, - int $version) : string + int $version, +) : string { $MAP = array(); $STRIDMAP = array(); @@ -139,20 +176,8 @@ function _config_pack_bundle_fmt2( $payloads[] = array($payloads_offset, $payload, $format, $payload_size); $payloads_offset += $payload_size; - $strids_indices = array(); - $strid_parts = explode('/', ltrim($entry->strid, '@')); - foreach($strid_parts as $strid_part) - { - if(!isset($STRIDMAP[$strid_part])) - { - $strid_index = count($STRIDLIST); - $STRIDLIST[] = $strid_part; - $STRIDMAP[$strid_part] = $strid_index; - $strids_indices[] = $strid_index; - } - else - $strids_indices[] = $STRIDMAP[$strid_part]; - } + $strids_indices = _config_encode_strid_as_indices($entry->strid, $STRIDMAP, $STRIDLIST); + $strids[] = $strids_indices; } @@ -192,6 +217,105 @@ function _config_pack_bundle_fmt2( return $packed_data; } +function _config_encode_strid_as_indices(string $strid, array &$STRIDMAP, array &$STRIDLIST) : array +{ + $strids_indices = array(); + $strid_parts = explode('/', ltrim($strid, '@')); + foreach($strid_parts as $strid_part) + { + if(!isset($STRIDMAP[$strid_part])) + { + $strid_index = count($STRIDLIST); + $STRIDLIST[] = $strid_part; + $STRIDMAP[$strid_part] = $strid_index; + $strids_indices[] = $strid_index; + } + else + $strids_indices[] = $STRIDMAP[$strid_part]; + } + return $strids_indices; +} + +function _config_patch_bundle_fmt2( + string $packed_data, + array $patch_entries, + bool $use_lz4, + bool $use_config_refs, + int $version, +) : string +{ + list($strids, $header, $_, $payloads_bundle) = + _config_unpack_bundle_fmt2(packed_data: $packed_data, unpack_entries: false); + + $stridmap = array_flip($strids); + + foreach($patch_entries as $idx => $patch_entry) + { + list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs); + + $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); + + if($header_found) + { + $header_idx = key($header_found); + $header_entry = current($header_found); + + $current_offset = $header_entry[4]; + $current_size = $header_entry[5]; + if($current_size >= strlen($patch_payload)) + { + //let's do the inline patching + $payloads_bundle = substr_replace($payloads_bundle, $patch_payload, $current_offset, strlen($patch_payload)); + + $header_entry[0] = $patch_format; + $header_entry[5] = strlen($patch_payload); + $header[$header_idx] = $header_entry; + } + else + { + //let's add it to the end + $header_entry[0] = $patch_format; + $header_entry[4] = strlen($payloads_bundle); + $header_entry[5] = strlen($patch_payload); + $header[$header_idx] = $header_entry; + + $payloads_bundle .= $patch_payload; + } + } + else + { + //let's add new entry it to the end + $strid_indices = _config_encode_strid_as_indices($patch_entry->strid, $stridmap, $strids); + + $header_entry = array( + $patch_format, + $patch_entry->id, + $strid_indices, + $patch_entry->class_id, + strlen($payloads_bundle), + strlen($patch_payload), + ); + + $header[] = $header_entry; + $payloads_bundle .= $patch_payload; + } + } + + $strids_msgpack = config_msgpack_pack($strids); + $header_msgpack = config_msgpack_pack($header); + + $patched_data = + pack("C", 2) . + pack("V", $version) . + pack("V", strlen($strids_msgpack)) . + pack("V", strlen($header_msgpack)) . + $strids_msgpack . + $header_msgpack . + $payloads_bundle; + + return $patched_data; +} + //format: [[class_id, [data]], ...[class_id, [data]]] function config_unpack_bundle(string $packed_data) : array { @@ -204,7 +328,8 @@ function config_unpack_bundle(string $packed_data) : array } else if($info['format'] === 2) { - return _config_unpack_bundle_fmt2($packed_data); + list($_, $_, $entries) = _config_unpack_bundle_fmt2($packed_data); + return $entries; } else throw new Exception("Unknown format: {$info['format']}"); @@ -237,7 +362,7 @@ function _config_unpack_bundle_fmt1(string $packed_data) : array return $entries; } -function _config_unpack_bundle_fmt2(string $packed_data) : array +function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries = true) : array { $packed_info = substr($packed_data, 0, 1+4+4+4); @@ -255,16 +380,20 @@ function _config_unpack_bundle_fmt2(string $packed_data) : array $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']); $entries = array(); - foreach($header as $item) + + if($unpack_entries) { - list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; + foreach($header as $item) + { + list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; - $payload = substr($payloads_bundle, $offset, $size); + $payload = substr($payloads_bundle, $offset, $size); - $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); + $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); + } } - return $entries; + return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle); } //format: [format_id, payload_data] diff --git a/scan.inc.php b/scan.inc.php index 97d2820..8e9d853 100644 --- a/scan.inc.php +++ b/scan.inc.php @@ -2,11 +2,24 @@ namespace taskman; use Exception; -class ConfigScanResult +class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator { /*var array*/ public array $base_dir2files = array(); + private $iter_pos = 0; + + function __construct(array $base_dir2files = array()) + { + foreach($base_dir2files as $dir => $files) + $this->base_dir2files[$dir] = $files; + } + + function clear() + { + $this->base_dir2files = array(); + } + function isEmpty() : bool { return empty($this->base_dir2files); @@ -69,10 +82,94 @@ class ConfigScanResult return $all_files; } + + //ArrayAccess interface + function offsetExists(mixed $offset) : bool + { + if(!is_int($offset)) + throw new Exception("Invalid offset"); + + return $this->count() > $offset; + } + + function offsetGet(mixed $offset) : mixed + { + if(!is_int($offset)) + throw new Exception("Invalid offset"); + + foreach($this->base_dir2files as $base_dir => $files) + { + $n = count($files); + if($offset - $n < 0) + return $files[$offset]; + $offset -= $n; + } + return null; + } + + function offsetSet(mixed $offset, mixed $value) : void + { + if(!is_int($offset)) + throw new Exception("Invalid offset"); + + foreach($this->base_dir2files as $base_dir => &$files) + { + $n = count($files); + if($offset - $n < 0) + { + $files[$offset] = $value; + return; + } + $offset -= $n; + } + } + + function offsetUnset(mixed $offset) : void + { + if(!is_int($offset)) + throw new Exception("Invalid offset"); + + foreach($this->base_dir2files as $base_dir => $files) + { + $n = count($files); + if($offset - $n < 0) + { + unset($files[$offset]); + return; + } + $offset -= $n; + } + } + + //Iterator interface + function rewind() : void + { + $this->iter_pos = 0; + } + + function current() : mixed + { + return $this->offsetGet($this->iter_pos); + } + + function key() : mixed + { + return $this->iter_pos; + } + + function next() : void + { + ++$this->iter_pos; + } + + function valid() : bool + { + return $this->offsetExists($this->iter_pos); + } } function config_scan_files( - array $base_dirs, + iterable $base_dirs, string $ext_filter = '.conf.js', bool $verbose = false ) : ConfigScanResult @@ -92,3 +189,10 @@ function config_scan_files( return $result; } + +function config_hash_changed(ConfigGlobals $globals, iterable $all_files) +{ + $all_crc_file = $globals->build_dir . "/configs.crc"; + return names_hash_changed($all_crc_file, $all_files); +} +