bit
/
taskman_config
25
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: bit:master
Branches Tags
bit:master
bit:6.x
bit:v6.1.0
bit:v6.0.0rc11
bit:v6.0.0rc10
bit:v6.0.0rc9
bit:v6.0.0rc8
bit:v6.0.0rc7
bit:v6.0.0rc6
bit:v6.0.0rc5
bit:v6.0.0rc4
bit:v6.0.0rc3
bit:v6.0.0rc2
bit:v6.0.0rc1
bit:v5.10.0
bit:v5.9.0
bit:v5.8.0
bit:v5.7.0
bit:v5.6.2
bit:v5.6.1
bit:v5.6.0
bit:v5.5.0
bit:v5.4.0
bit:v5.3.1
bit:v5.3.0
bit:v5.2.0
bit:v5.1.0
bit:v5.0.2
bit:v5.0.1
bit:v5.0.0
bit:v4.3.0
bit:v4.2.0
bit:v4.1.0
bit:v4.0.6
bit:v4.0.5
bit:v4.0.4
bit:v4.0.3
bit:v4.0.2
bit:v4.0.1
bit:v4.0.0
bit:v3.6.1
bit:v3.6.0
bit:v3.5.4
bit:v3.5.3
bit:v3.5.2
bit:v3.5.1
bit:v3.5.0
bit:v3.4.0
bit:v3.3.0
bit:v3.2.0
bit:v3.1.0
bit:v3.0.0
bit:v2.0.0
bit:v1.0.10
bit:v1.0.9
bit:v1.0.8
bit:v1.0.7
bit:v1.0.6
bit:v1.0.5
bit:v1.0.4
bit:v1.0.3
bit:v1.0.2
bit:v1.0.1
bit:v1.0
..
compare: bit:v6.0.0rc9
Branches Tags
bit:master
bit:6.x
bit:v6.1.0
bit:v6.0.0rc11
bit:v6.0.0rc10
bit:v6.0.0rc9
bit:v6.0.0rc8
bit:v6.0.0rc7
bit:v6.0.0rc6
bit:v6.0.0rc5
bit:v6.0.0rc4
bit:v6.0.0rc3
bit:v6.0.0rc2
bit:v6.0.0rc1
bit:v5.10.0
bit:v5.9.0
bit:v5.8.0
bit:v5.7.0
bit:v5.6.2
bit:v5.6.1
bit:v5.6.0
bit:v5.5.0
bit:v5.4.0
bit:v5.3.1
bit:v5.3.0
bit:v5.2.0
bit:v5.1.0
bit:v5.0.2
bit:v5.0.1
bit:v5.0.0
bit:v4.3.0
bit:v4.2.0
bit:v4.1.0
bit:v4.0.6
bit:v4.0.5
bit:v4.0.4
bit:v4.0.3
bit:v4.0.2
bit:v4.0.1
bit:v4.0.0
bit:v3.6.1
bit:v3.6.0
bit:v3.5.4
bit:v3.5.3
bit:v3.5.2
bit:v3.5.1
bit:v3.5.0
bit:v3.4.0
bit:v3.3.0
bit:v3.2.0
bit:v3.1.0
bit:v3.0.0
bit:v2.0.0
bit:v1.0.10
bit:v1.0.9
bit:v1.0.8
bit:v1.0.7
bit:v1.0.6
bit:v1.0.5
bit:v1.0.4
bit:v1.0.3
bit:v1.0.2
bit:v1.0.1
bit:v1.0

No commits in common. "master" and "v6.0.0rc9" have entirely different histories.
master ... v6.0.0rc9

4 changed files with 81 additions and 211 deletions
Show Stats Expand all files Collapse all files

20
cache.inc.php
View File

@ -325,24 +325,14 @@ class ConfigCacheFileMap
unset($this->file2deps[$file]);
}
function updateDepsForEntry(array $base_dirs, ConfigCacheEntry $entry)
function updateDepsForEntry(ConfigCacheEntry $entry)
{
//1. add includes
$all_refs = $entry->includes;
//2. add refs
foreach($entry->refs as $ref => $_)
foreach($entry->includes as $include)
{
$file = config_real_path($base_dirs, substr($ref, 1) . '.conf.js');
$all_refs[] = $file;
}
foreach($all_refs as $ref_file)
{
if(isset($this->file2deps[$ref_file]))
$this->file2deps[$ref_file][$entry->file] = true;
if(isset($this->file2deps[$include]))
$this->file2deps[$include][$entry->file] = true;
else
$this->file2deps[$ref_file] = [$entry->file => true];
$this->file2deps[$include] = [$entry->file => true];
}
}

98
config.inc.php
View File

@ -18,7 +18,7 @@ class ConfigGlobals
public readonly string $base_class;
public readonly string $build_dir;
//NOTE: it's a string since is serialized
public readonly ?string $files_filter;
public readonly ?string $files_overrider;
function __construct(
array $base_dirs,
@ -26,7 +26,7 @@ class ConfigGlobals
?string $worker_init_fn = null,
int $workers_num = 1,
string $base_class = '\ConfBase',
?string $files_filter = null
?string $files_overrider = null
)
{
$this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
@ -34,7 +34,7 @@ class ConfigGlobals
$this->worker_init_fn = $worker_init_fn;
$this->workers_num = $workers_num;
$this->base_class = $base_class;
$this->files_filter = $files_filter;
$this->files_overrider = $files_overrider;
}
function initWorker(bool $is_master_proc)
@ -123,13 +123,6 @@ class ConfigUpdateResult
public int $corruptions = 0;
public int $fast_jsons = 0;
function isEmpty() : bool
{
return count($this->affected_files) == 0 &&
count($this->added_files) == 0 &&
count($this->removed_files) == 0;
}
function isPatchPossible() : bool
{
return ($this->request->mode == ConfigUpdateMode::Selected ||
@ -185,8 +178,7 @@ class ConfigManager
function fetchAll(bool $force = false) : \Generator
{
$result = $this->updateAll($force);
$filtered = $this->filterConfigFiles($result->request->files);
return $this->iterateCache($filtered);
return $this->iterateCache($result->request->files);
}
function fetchByPath(string $path) : ConfigCacheEntry
@ -204,8 +196,11 @@ class ConfigManager
{
foreach($files as $file)
{
$ce = $this->cache->getOrLoadByPath($file);
yield $ce;
if(config_is_file($file))
{
$ce = $this->cache->getOrLoadByPath($file);
yield $ce;
}
}
}
@ -220,20 +215,22 @@ class ConfigManager
$removed_files = [];
$this->_checkFileMap($req, $added_files, $removed_files);
$affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
$affected_conf_files = $this->filterConfigFiles($affected_all_files);
$affected_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
config_log("Affected configs: {$affected_conf_files->count()} (requested: {$req->files->count()})");
//NOTE: at this point taking into account only config files
$affected_files->filter(fn($file) => config_is_file($file));
config_log("Affected files: {$affected_files->count()}");
$update_result = new ConfigUpdateResult();
$update_result->request = $req;
$update_result->affected_files = $affected_conf_files;
$update_result->affected_files = $affected_files;
$update_result->added_files = $added_files;
$update_result->removed_files = $removed_files;
$update_params = new ConfigCacheUpdateParams(
globals: $this->globals,
affected_files: $affected_conf_files,
affected_files: $affected_files,
verbose: $req->verbose
);
self::_updateCache($update_result, $update_params);
@ -241,11 +238,11 @@ class ConfigManager
//let's clear internal cache once the update procedure is done
$this->cache->clear();
$this->_updateFileMap($req, $affected_conf_files, $added_files, $removed_files);
$this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
if($req->return_affected)
{
foreach($affected_conf_files as $file)
foreach($affected_files as $file)
{
$entry = $this->cache->getOrLoadByPath($file);
$update_result->affected_entries[] = $entry;
@ -257,19 +254,6 @@ class ConfigManager
return $update_result;
}
function filterConfigFiles(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles
{
if($files instanceof \taskman\artefact\TaskmanDirFiles)
$files = ConfigDirFiles::makeFromArtefactFiles($files);
$filtered = $this->globals->files_filter != null ?
call_user_func($this->globals->files_filter, $files) :
new ConfigDirFiles($files->getMap());
$filtered->filter(fn($file) => config_is_file($file));
return $filtered;
}
function updateAll(bool $force = false) : ConfigUpdateResult
{
$files = $this->scanFiles(extension: '.js');
@ -341,19 +325,19 @@ class ConfigManager
}
}
private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_conf_files, array $added_files, array $removed_files)
private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
{
$fs_cache_map = $this->getFileMap();
//TODO: traverse all affected config files and update file map
foreach($affected_conf_files as $file)
//TODO: traverse all affected files and update file map
foreach($affected_files as $file)
{
$cache_entry = $this->cache->getOrLoadByPath($file);
$fs_cache_map->updateDepsForEntry($this->globals->base_dirs, $cache_entry);
$fs_cache_map->updateDepsForEntry($cache_entry);
}
if($req->mode == ConfigUpdateMode::Force ||
$affected_conf_files->count() > 0 ||
$affected_files->count() > 0 ||
$added_files ||
$removed_files)
{
@ -370,7 +354,7 @@ class ConfigManager
if($req->mode === ConfigUpdateMode::Force)
{
$affected_files = new ConfigDirFiles($req->files->getMap());
$affected_files = $req->files;
}
else if($req->mode === ConfigUpdateMode::DetectChanged)
{
@ -396,9 +380,12 @@ class ConfigManager
//if there were removed files we need to rebuild affected files
foreach($removed_files as $file)
{
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
foreach($affected_by_file as $dep)
$affected_files->addFile($dep, unique: true);
if(!config_is_file($file))
{
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
foreach($affected_by_file as $dep)
$affected_files->addFile($dep, unique: true);
}
}
}
else if($req->mode === ConfigUpdateMode::Selected)
@ -408,9 +395,13 @@ class ConfigManager
foreach($req->files as $file)
{
$affected_files->addFile($file, unique: true);
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
foreach($affected_by_file as $dep)
$affected_files->addFile($dep, unique: true);
if(!config_is_file($file))
{
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
foreach($affected_by_file as $dep)
$affected_files->addFile($dep, unique: true);
}
}
}
@ -424,7 +415,10 @@ class ConfigManager
function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
{
return config_scan_files($this->globals->base_dirs, $extension, $verbose);
$res = config_scan_files($this->globals->base_dirs, $extension, $verbose);
if($this->globals->files_overrider != null)
$res = call_user_func($this->globals->files_overrider, $res);
return $res;
}
private function _getMapPath() : string
@ -436,11 +430,8 @@ class ConfigManager
{
if(!is_file($this->_getMapPath()))
return null;
$t = microtime(true);
$data = ensure_read($this->_getMapPath());
$map = ConfigCacheFileMap::unserialize($data);
config_log("Loaded file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
return $map;
config_log("Loading file map");
return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
}
private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
@ -453,10 +444,9 @@ class ConfigManager
private function _saveFileMap()
{
$t = microtime(true);
$data = ConfigCacheFileMap::serialize($this->getFileMap());
config_log("Saving file map: " . kb($data));
ensure_write($this->_getMapPath(), $data);
config_log("Saved file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
}
}

169
pack.inc.php
View File

@ -102,16 +102,6 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
$params->version,
);
}
else if($params->binary_format == 3)
{
$patched_data = _config_patch_bundle_fmt3(
$packed_data,
$params->cache_entries,
$params->use_lz4,
$params->use_config_refs,
$params->version,
);
}
else
throw new Exception("Unknown binary format: {$params->binary_format}");
@ -246,7 +236,7 @@ function _config_pack_bundle_fmt2(
return $packed_data;
}
//NOTE: Much like fmt1, but configs entries are grouped into sizeable chunks, with each chunk lz4-compressed.
//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
// This reduces overall bundle size when there are many small configs entries.
function _config_pack_bundle_fmt3(
array $cache_entries,
@ -361,12 +351,12 @@ function _config_patch_bundle_fmt2(
{
list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
$headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
$header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
if($headers_found)
if($header_found)
{
$header_idx = key($headers_found);
$header_entry = current($headers_found);
$header_idx = key($header_found);
$header_entry = current($header_found);
$current_offset = $header_entry[4];
$current_size = $header_entry[5];
@ -424,97 +414,6 @@ function _config_patch_bundle_fmt2(
return $patched_data;
}
function _config_patch_bundle_fmt3(
string $packed_data,
array $patch_entries,
bool $use_lz4,
bool $use_config_refs,
int $version,
) : string
{
if(!$use_lz4)
throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
list($header, $entries, $chunks_offset, $max_chunk_size) =
_config_unpack_bundle_fmt3(packed_data: $packed_data, unpack_entries: false);
$payloads_bundle = substr($packed_data, $chunks_offset);
foreach($patch_entries as $idx => $patch_entry)
{
list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
$headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
if($headers_found)
{
$header_idx = key($headers_found);
$header_entry = current($headers_found);
$chunk_offset = $header_entry[4];
$payloads_offset = $header_entry[5]; //within chunk
$payloads_size = $header_entry[6];
$chunk_entries = array_filter($header, fn($item) => $item[4] == $chunk_offset);
//chunk contains only one patch entry
$patch_payload_lz4 = lz4_compress($patch_payload, 9);
$patched_chunk = pack("V", strlen($patch_payload_lz4));
$patched_chunk .= $patch_payload_lz4;
if(strlen($patch_payload) > $max_chunk_size)
$max_chunk_size = strlen($patch_payload);
//TODO: make a better generic algorithm for patching the whole chunk
//special case for single entry
if(count($chunk_entries) == 1 && $payloads_offset == 0)
{
$lz4_chunk_size = unpack("V", substr($payloads_bundle, $chunk_offset, 4))[1];
//we can just replace chunk in place if the size of a patched chunk is less or equal
if(strlen($patch_payload_lz4) <= $lz4_chunk_size)
{
$payloads_bundle = substr_replace($payloads_bundle, $patched_chunk, $chunk_offset, strlen($patched_chunk));
$header_entry[6] = strlen($patch_payload);
}
//just append to the end
else
{
$header_entry[4] = strlen($payloads_bundle); //chunk offset
$header_entry[5] = 0;
$header_entry[6] = strlen($patch_payload);
$payloads_bundle .= $patched_chunk;
}
}
//just append to the end
else
{
$header_entry[4] = strlen($payloads_bundle); //chunk offset
$header_entry[5] = 0;
$header_entry[6] = strlen($patch_payload);
$payloads_bundle .= $patched_chunk;
}
$header[$header_idx] = $header_entry;
}
else
throw new Exception("Patched entry {$patch_entry->id} not found in config bundle");
}
$header_msgpack = config_msgpack_pack($header);
$packed_data =
pack("C", 3) .
pack("V", $version) .
pack("V", strlen($header_msgpack)) .
pack("V", $max_chunk_size) .
$header_msgpack .
$payloads_bundle;
return $packed_data;
}
//format: [[class_id, [data]], ...[class_id, [data]]]
function config_unpack_bundle(string $packed_data) : array
{
@ -532,8 +431,7 @@ function config_unpack_bundle(string $packed_data) : array
}
else if($info['format'] === 3)
{
list($_, $entries) = _config_unpack_bundle_fmt3($packed_data);
return $entries;
return _config_unpack_bundle_fmt3($packed_data);
}
else
throw new Exception("Unknown format: {$info['format']}");
@ -600,10 +498,12 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries =
return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
}
function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries = true): array
function _config_unpack_bundle_fmt3(string $packed_data): array
{
if(ord($packed_data[0]) !== 3)
{
throw new Exception("Invalid config bundle format");
}
$offset = 1;
$version = unpack("V", substr($packed_data, $offset, 4))[1];
@ -615,7 +515,6 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
$header_msgpack = substr($packed_data, $offset, $header_len);
$offset += $header_len;
$chunks_offset = $offset;
$header = config_msgpack_unpack($header_msgpack);
@ -624,45 +523,39 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
$chunk_buffer = '';
$chunk_id = -1;
foreach($header as $entry_data)
foreach ($header as $entry_data)
{
list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
$unpacked_payload = null;
if($unpack_entries)
if($entry_chunk_offset !== $chunk_id)
{
if($entry_chunk_offset !== $chunk_id)
if($chunk_offset !== -1)
{
if($chunk_offset !== -1)
{
$lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
$offset+=4;
$lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
$offset += $lz4_chunk_size;
$chunk_offset = $offset;
}
else
{
$lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
$chunk_offset += 4;
$lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
$chunk_offset += $lz4_chunk_size;
}
$chunk_id = $entry_chunk_offset;
$lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
$offset+=4;
$lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
$offset += $lz4_chunk_size;
$chunk_offset = $offset;
}
else
{
$lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
$chunk_offset += 4;
$lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
$chunk_offset += $lz4_chunk_size;
}
$payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
$unpacked_payload = _config_unpack_payload($format, $payload);
$chunk_id = $entry_chunk_offset;
}
$cache_entries[$id] = array($class_id, $unpacked_payload);
$payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
$cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
}
return array($header, $cache_entries, $chunks_offset, $max_chunk_size);
return $cache_entries;
}
//format: [format_id, payload_data]

5
util.inc.php
View File

@ -28,11 +28,9 @@ function config_real_path(array $base_dirs, string $rel_path, bool $strict = tru
{
foreach($base_dirs as $dir)
if(is_file($dir . '/' . $rel_path))
return normalize_path($dir . '/' . $rel_path);
return $dir . '/' . $rel_path;
if($strict)
throw new Exception("No file for relative path '$rel_path'");
return null;
}
@ -106,7 +104,6 @@ function config_walk_fields(object $proto, callable $callback)
}
}
//TODO: doesn't really belong here
function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
{
if(!isset($map[$file]))