Cleaning API

.gitea/workflows/build_composer.yaml

@@ -0,0 +1,28 @@
+name: Publish PHP Package
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Get tag name
+      run: echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+    - name: zip and send
+      run: |
+        ls -la
+        apt-get update -y
+        apt-get install -y zip
+        cd ../
+        zip -r ${{ gitea.event.repository.name }}.zip ${{ gitea.event.repository.name }} -x '*.git*'
+        curl -v \
+        --user composer-pbl:${{ secrets.COMPOSER_PSWD }} \
+        --upload-file ${{ gitea.event.repository.name }}.zip \
+        https://git.bit5.ru/api/packages/bit/composer?version=${{ env.TAG }}  

.gitignore

@@ -0,0 +1 @@
+tags

CHANGELOG.md

@@ -0,0 +1,13 @@
+## v5.10.0
+- Added fmt3 bundle format - basically, fmt1 with chunk-based lz4 compression
+
+## v4.0.2
+- Improving exception message
+
+## v4.0.1
+- Typesafety fix
+
+## v4.0.0
+- Adding initial binary format 2 support: no more lookups by CRC28 strids
+- Removed misc obsolete stuff
+- Adding PHP type hints

cache.inc.php

@@ -0,0 +1,571 @@
+<?php
+namespace taskman;
+use Exception;
+
+class ConfigCache
+{
+  private ConfigGlobals $globals;
+
+  /** @var array<string, ConfigCacheEntry> */
+  private array $by_path = [];
+  /** @var array<int, ConfigCacheEntry> */
+  private array $by_id = [];
+  /** @var array<string, ConfigCacheEntry> */
+  private array $by_alias = [];
+
+  function __construct(ConfigGlobals $globals)
+  {
+    $this->globals = $globals;
+  }
+
+  function clear()
+  {
+    $this->by_path = [];
+    $this->by_id = [];
+    $this->by_alias = [];
+  }
+
+  function tryGetOrLoadByPath(string $path) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($path, $this->by_path))
+    {
+      $cache_file = config_get_cache_path($this->globals, $path);
+      if(!is_file($cache_file))
+        return null;
+      $ce = ConfigCacheEntry::unserialize(ensure_read($cache_file));
+      $this->by_path[$path] = $ce;
+      return $ce;
+    }
+
+    return $this->by_path[$path];
+  }
+
+  function getOrLoadByPath(string $path) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadByPath($path);
+    if($ce === null)
+      throw new Exception("Failed to find entry by path '$path'");
+    return $ce;
+  }
+
+  function tryGetOrLoadById(int $id) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($id, $this->by_id))
+    {
+      $cache_id_file = config_get_cache_id_path($this->globals, $id);
+      if(!is_file($cache_id_file))
+        return null;
+      $file = ensure_read($cache_id_file);
+      $ce = $this->tryGetOrLoadByPath($file);
+      $this->by_id[$id] = $ce;
+      return $ce;
+    }
+
+    return $this->by_id[$id];
+  }
+
+  function getOrLoadById(int $id) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadById($id);
+    if($ce === null)
+      throw new Exception("Failed to find entry by id '$id'");
+    return $ce;
+  }
+
+  function tryGetOrLoadByStrid(string $strid) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($strid, $this->by_alias))
+    {
+      $cache_strid_file = config_get_cache_strid_path($this->globals, $strid);
+      if(!is_file($cache_strid_file))
+        return null;
+      $file = ensure_read($cache_strid_file);
+      $ce = $this->tryGetOrLoadByPath($file);
+      $this->by_alias[$strid] = $ce;
+      return $ce;
+    }
+
+    return $this->by_alias[$strid];
+  }
+
+  function getOrLoadByStrid(string $strid) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadByStrid($strid);
+    if($ce === null)
+      throw new Exception("Failed to find entry by strid '$strid'");
+    return $ce;
+  }
+}
+
+/**
+ * @property object $config
+ * @property string $payload
+ */
+class ConfigCacheEntry
+{
+  const FMT_BINARY     = 0;
+  const FMT_LZ4        = 1;
+  const FMT_FILE_REF   = 2;
+
+  public string $class;
+  public int $class_id;
+  public int $id;
+  public string $strid;
+  public string $cache_file;
+  //NOTE: actual payload is stored in a separate file for faster incremental retrievals
+  public string $payload_file;
+  public string $file;
+  public array $includes = array();
+  public array $refs = array();
+  public ?ConfigCacheEntryExtras $extras;
+
+  public $_config;
+  public $_payload;
+
+  static function serialize(ConfigCacheEntry $ce) : string
+  {
+    $d = $ce->export();
+    return serialize($d);
+  }
+
+  static function unserialize(string $str) : ?ConfigCacheEntry
+  {
+    $d = @unserialize($str);
+    if(!is_array($d))
+      return null;
+    $ce = new ConfigCacheEntry();
+    $ce->import($d);
+    return $ce;
+  }
+
+  function __construct()
+  { 
+    $this->extras = ConfigCacheEntryExtras::create();
+  }
+
+  function __get(string $name)
+  {
+    if($name === "config")
+    {
+      if($this->_config === null)
+      {
+        $klass = $this->class;
+        $data = config_msgpack_unpack($this->payload);
+        $this->_config = new $klass();
+        $this->_config->import($data);
+      }
+      return $this->_config;
+    }
+    else if($name === "payload")
+    {
+      if($this->_payload !== null)
+        return $this->_payload;
+      //if payload not set directly not storing it
+      return ensure_read($this->payload_file);
+    }
+    else
+      throw new Exception("No such property '$name'");
+  }
+
+  function __set(string $name, $v)
+  {
+    if($name === "config")
+      $this->_config = $v;
+    else if($name === "payload")
+      $this->_payload = $v;
+    else
+      throw new Exception("No such property '$name'");
+  }
+
+  function export() : array
+  {
+    $d = array();
+    $d[] = $this->class;
+    $d[] = $this->class_id;
+    $d[] = $this->id;
+    $d[] = $this->strid;
+    $d[] = $this->cache_file;
+    $d[] = $this->payload_file;
+    $d[] = $this->file;
+    $d[] = $this->includes;
+    $d[] = $this->refs;
+    $d[] = $this->extras->export();
+    return $d;
+  }
+
+  function import(array $d)
+  {
+    $extras = array();
+
+    list(
+      $this->class,
+      $this->class_id,
+      $this->id,
+      $this->strid,
+      $this->cache_file,
+      $this->payload_file,
+      $this->file,
+      $this->includes,
+      $this->refs,
+      $extras,
+    ) = $d;
+
+    $this->extras->import($extras);
+  }
+}
+
+class ConfigCacheEntryExtras extends \stdClass
+{
+  private static string $klass = ConfigCacheEntryExtras::class;
+
+  static function init(string $project_specific_klass)
+  {
+    self::$klass = $project_specific_klass;
+  }
+
+  static function create() : object
+  {
+    return new self::$klass();
+  }
+
+  function export() : array
+  {
+    $as_array = get_object_vars($this);
+    return $as_array;
+  }
+
+  function import(array $as_array)
+  {
+    foreach($as_array as $field_name => $field_value)
+      $this->$field_name = $field_value;
+  }
+}
+
+class ConfigCacheFileMap
+{
+  private array $file2deps = array();
+
+  static function serialize(ConfigCacheFileMap $map) : string
+  {
+    $d = $map->export();
+    return serialize($d);
+  }
+
+  static function unserialize(string $str) : ?ConfigCacheFileMap
+  {
+    $d = @unserialize($str);
+    if(!is_array($d))
+      return null;
+    $map = new ConfigCacheFileMap();
+    $map->import($d);
+    return $map;
+  }
+
+  function count() : int
+  {
+    return count($this->file2deps);
+  }
+
+  function exists(string $file) : bool
+  {
+    return isset($this->file2deps[$file]);
+  }
+
+  function getAffectedFiles(string $file) : array
+  {
+    if(!isset($this->file2deps[$file]))
+      return [];
+    return array_keys($this->file2deps[$file]);
+  }
+
+  function differs(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles|array $files) : bool
+  {
+    if(!is_array($files))
+      $files = $files->getAllFiles();
+
+    $this_files = array_keys($this->file2deps);
+
+    return count($files) !== count($this_files) || 
+      count(array_intersect($files, $this_files)) != count($this_files);
+  }
+
+  function compare(array $files) : array
+  {
+    $this_files = array_keys($this->file2deps);
+
+    $added = array_diff($files, $this_files);
+    $removed = array_diff($this_files, $files);
+
+    return [$added, $removed];
+  }
+
+  function init(array $files)
+  {
+    $this->file2deps = [];
+    foreach($files as $file)
+      $this->file2deps[$file] = [];
+  }
+
+  function update(array $added, array $removed)
+  {
+    foreach($added as $file)
+    {
+      //NOTE: we preserve existing files
+      if(!isset($this->file2deps[$file]))
+        $this->file2deps[$file] = [];
+    }
+    foreach($removed as $file)
+      unset($this->file2deps[$file]);
+  }
+
+  function updateDepsForEntry(ConfigCacheEntry $entry)
+  {
+    foreach($entry->includes as $include)
+    {
+      if(isset($this->file2deps[$include]))
+        $this->file2deps[$include][$entry->file] = true;
+      else
+        $this->file2deps[$include] = [$entry->file => true];
+    }
+  }
+
+  function export() : array
+  {
+    $d = array();
+    $d[] = $this->file2deps;
+    return $d;
+  }
+
+  function import(array $d)
+  {
+    list($this->file2deps) = $d;
+  }
+}
+
+class ConfigCacheUpdateParams
+{
+  const MAX_DEFAULT_WORKERS = 5;
+  const FILES_THRESHOLD_ONE_WORKER = 100;
+
+  public ConfigGlobals $globals;
+  public ConfigDirFiles $affected_files;
+  public bool $verbose = false;
+  public bool $check_junk = true;
+  public int $max_errors_num = 15;
+  public bool $return_affected_entries = false;
+
+  function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, bool $verbose = false)
+  {
+    $this->globals = $globals;
+    $this->affected_files = $affected_files;
+    $this->verbose = $verbose;
+  }
+
+  function calcMaxWorkers() : int
+  {
+    if($this->affected_files->count() < self::FILES_THRESHOLD_ONE_WORKER)
+      return 1;
+    return $this->globals->max_workers ?? self::MAX_DEFAULT_WORKERS;
+  }
+
+  function splitFilesByChunks(int $max_workers, bool $sort = true) : array
+  {
+    $flat = $this->affected_files->getFlatArray();
+    if(!$flat)
+      return array();
+    if($sort)
+      usort($flat, fn($a, $b) => $a[1] <=> $b[1]);
+
+    $chunk_size = (int)ceil(count($flat)/$max_workers);
+    return array_chunk($flat, $chunk_size);
+  }
+
+  //returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ]
+  function splitJobs(bool $sort = true) : array
+  {
+    $max_workers = $this->calcMaxWorkers();
+    $jobs = array();
+    $chunks = $this->splitFilesByChunks($max_workers, $sort); 
+    foreach($chunks as $idx => $chunk)
+      $jobs[] = array($idx, microtime(true), $chunk);
+    return $jobs;
+  }
+}
+
+function _config_cache_update(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
+{
+  $jobs = $params->splitJobs(sort: true);
+
+  $serial = sizeof($jobs) == 1;
+
+  $results_by_job = _config_update_worker_run_procs($params, $jobs, $serial);
+
+  if(!$serial)
+  {
+    //NOTE: in case result unserialize error try serial processing
+    if(array_search(false, $results_by_job, true/*strict*/) !== false)
+    {
+      if($params->verbose)
+        config_log("Corrupted result, trying serial processing...");
+      $results_by_job = _config_update_worker_run_procs(params: $params, jobs: $jobs, serial: true);
+    }
+  }
+
+  _config_merge_update_results($result, $results_by_job);
+
+  if($params->verbose)
+    config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
+}
+
+function _config_merge_update_results(ConfigUpdateResult $result, array $results_by_job)
+{
+  $total_fast_jsons = 0;
+  
+  foreach($results_by_job as $results)
+  {
+    foreach($results as $item) 
+    {
+      $base_dir = $item['base_dir'];
+      $file = $item['file'];
+      $cache_file = $item['cache_file'];
+      $parser_type = $item['parser_type'];
+      $error = $item['error'];
+
+      if($error !== null)
+      {
+        $result->errors[$file] = $error;
+        continue;
+      }
+
+      if($parser_type === 1)
+        ++$total_fast_jsons;
+    }
+  }
+
+  $result->fast_jsons = $total_fast_jsons;
+}
+
+function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file, 
+  ?int &$parser_type = null) : ConfigCacheEntry
+{
+  list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
+  if(!$conf_id)
+    throw new Exception("Bad conf id: {$conf_id}");
+
+  //TODO: this is a bit ugly but kinda works for now
+  $GLOBALS['CONFIG_CURRENT_FILE'] = $file; 
+  $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id; 
+  $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
+
+  $pres = config_parse($params->globals->base_dirs, $file);
+  if($pres->error !== 0)
+    throw new Exception("Parse error({$pres->error}):\n" . $pres->error_descr);
+
+  $parser_type = $pres->parser_type;
+  $parsed_arr = $pres->parsed_arr;
+  //TODO: these hardcoded keys below probably shouldn't be part of data?
+  $parsed_arr['id'] = $conf_id;
+  $parsed_arr['strid'] = $conf_strid;
+  list($class, $class_id, $normalized_data) 
+    = config_apply_class_normalization($parsed_arr, $params->check_junk);
+
+  $cache_payload_file = config_get_cache_payload_path($params->globals, $file);
+
+  $cache_file = config_get_cache_path($params->globals, $file);
+
+  $cache_entry = new ConfigCacheEntry();
+  $cache_entry->id = $conf_id;
+  $cache_entry->strid = $conf_strid;
+  $cache_entry->cache_file = $cache_file;
+  $cache_entry->class = $class;
+  $cache_entry->class_id = $class_id;
+  $cache_entry->payload_file = $cache_payload_file;
+  //TODO: pass flag if file is actually normalized?
+  $cache_entry->file = normalize_path($file);
+  $cache_entry->includes = config_get_module_includes(cm: $pres->jsm_module);
+  //TODO: do we really need these refs?
+  $cache_entry->refs = config_content_match_refs($pres->normalized_jzon);
+  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
+
+  ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
+  ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
+  ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
+  ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
+
+  return $cache_entry;
+}
+
+function _config_update_worker_run_procs(ConfigCacheUpdateParams $params, array $jobs, bool $serial) : array
+{
+  if($serial)
+  {
+    $results_by_job = array();
+    foreach($jobs as $job)
+      $results_by_job[] = _config_update_worker_func($params, $job, $serial);
+    return $results_by_job;
+  }
+  else
+  {
+    //initializing worker for master process anyway
+    $params->globals->initWorker(true);
+
+    $workers_args = array();
+    foreach($jobs as $job)
+      $workers_args[] = array($params, $job);
+
+    return run_background_gamectl_workers('config_update_worker', $workers_args);
+  }
+}
+
+function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job, bool $is_master_proc = false) : array
+{
+  $start_time = microtime(true);
+
+  $params->globals->initWorker($is_master_proc);
+
+  list($idx, $start_time, $chunk) = $job;
+  if($params->verbose)
+    config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
+
+  $fast_parser_num = 0;
+  $results = array();
+  foreach($chunk as $file_idx => $chunk_data)
+  {
+    list($base_dir, $file) = $chunk_data;
+
+    try
+    {
+      if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
+        config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
+
+      $cache_file = config_get_cache_path($params->globals, $file);
+
+      $parser_type = null;
+      $entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
+
+      $results[] = [
+        'base_dir' => $base_dir, 
+        'file' => $file, 
+        'cache_file' => $entry->cache_file, 
+        'parser_type' => $parser_type, 
+        'error' => null
+      ];
+    }
+    catch(\Throwable $e)
+    {
+      $results[] = [
+        'base_dir' => $base_dir, 
+        'file' => $file, 
+        'cache_file' => null,
+        'parser_type' => null,
+        'error' => $e->getMessage()
+      ];
+    }
+  }
+
+  if($params->verbose)
+    config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
+
+  return $results;
+}
+

composer.json

@@ -6,6 +6,6 @@
         "php": ">=7.4"
     },
     "autoload": {
-        "classmap": ["config.inc.php", "msgpack/msgpack_custom.inc.php"]
+        "classmap": ["msgpack/msgpack_custom.inc.php"]
     }
 }

msgpack.inc.php

@@ -0,0 +1,36 @@
+<?php
+namespace taskman;
+use Exception;
+
+if(function_exists('msgpack_pack'))
+{
+  function config_msgpack_pack(array $data) : string
+  {
+    $prev = ini_set('msgpack.use_str8_serialization', '0');
+    $res =  msgpack_pack($data);
+    if($prev != '0')
+      ini_set('msgpack.use_str8_serialization', $prev);
+    return $res;
+  }
+
+  function config_msgpack_unpack(string $data) : array
+  {
+    return msgpack_unpack($data);
+  }
+}
+else
+{
+  function config_msgpack_pack(array $data) : string
+  {
+    include_once(__DIR__ . '/msgpack/msgpack_custom.inc.php');
+
+    $packer = new \MessagePackCustom(); 
+    return $packer->pack($data);
+  }
+
+  function config_msgpack_unpack(string $data) : array
+  {
+    return \MessagePack\MessagePack::unpack($data);
+  }
+}
+

pack.inc.php

@@ -0,0 +1,636 @@
+<?php
+namespace taskman;
+use Exception;
+
+class ConfigPackParams
+{
+  /*var ConfigCacheEntry[]*/
+  public iterable $cache_entries;
+  public bool $use_lz4 = false;
+  public bool $use_config_refs = false;
+  public int $binary_format = 1; //1,2 supported
+  public ?int $version = null;
+  public bool $debug = false;
+
+  public const EXTRA_FMT3_CHUNK_SIZE = "EXTRA_FMT3_CHUNK_SIZE";
+  public const EXTRA_FMT3_COMPRESSION_LEVEL = "EXTRA_FMT3_COMPRESSION_LEVEL";
+  public array $extras = array();
+
+  function __construct(
+    iterable $cache_entries, 
+    int $version, 
+    bool $use_lz4 = false, 
+    bool $use_config_refs = false, 
+    int $binary_format = 1, 
+    bool $debug = false,
+    array $extras = array()
+  )
+  {
+    $this->cache_entries = $cache_entries;
+    $this->use_lz4 = $use_lz4;
+    $this->use_config_refs = $use_config_refs;
+    $this->binary_format = $binary_format;
+    $this->version = $version;
+    $this->debug = $debug;
+    $this->extras = $extras;
+  }
+}
+
+function config_pack_bundle(ConfigPackParams $params) : string
+{
+  $t = microtime(true);
+
+  $packed_data = null;
+
+  if($params->binary_format == 1)
+  {
+    $packed_data = _config_pack_bundle_fmt1(
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version
+    );
+  }
+  else if($params->binary_format == 2)
+  {
+    $packed_data = _config_pack_bundle_fmt2(
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version,
+    );
+  }
+  else if($params->binary_format == 3)
+  {
+    $packed_data = _config_pack_bundle_fmt3(
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version,
+      $params->extras[ConfigPackParams::EXTRA_FMT3_CHUNK_SIZE],
+      $params->extras[ConfigPackParams::EXTRA_FMT3_COMPRESSION_LEVEL],
+    );
+  }
+  else
+    throw new Exception("Unknown binary format: {$params->binary_format}");
+
+  if($params->debug)
+    config_log("Packed entries: " . sizeof($params->cache_entries) . ", total: " . 
+      kb($packed_data) . ", format: {$params->binary_format}, lz4: " . 
+      var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
+      ", CRC: " . crc32($packed_data) . 
+      ", " . round(microtime(true) - $t,2) . " sec.");
+
+  return $packed_data;
+}
+
+function config_patch_bundle(ConfigPackParams $params, string $packed_data) : string
+{
+  $t = microtime(true);
+
+  $patched_data = null;
+
+  if($params->binary_format == 2)
+  {
+    $patched_data = _config_patch_bundle_fmt2(
+      $packed_data,
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version,
+    );
+  }
+  else
+    throw new Exception("Unknown binary format: {$params->binary_format}");
+
+  if($params->debug)
+    config_log("Patched entries: " . sizeof($params->cache_entries) . ", total: " . 
+      kb($patched_data) . ", format: {$params->binary_format}, lz4: " . 
+      var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
+      ", CRC: " . crc32($patched_data) . 
+      ", " . round(microtime(true) - $t,2) . " sec.");
+
+  return $patched_data;
+}
+
+//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
+function _config_pack_bundle_fmt1(
+  iterable $cache_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version) : string
+{
+  $MAP = array();
+  $STRIDMAP = array();
+
+  $payloads = array();
+  $payloads_offset = 0;
+  foreach($cache_entries as $entry)
+  {
+    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
+    $payload_size = strlen($payload);
+    $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
+    $payloads_offset += $payload_size;
+  }
+
+  $header = array();
+  foreach($cache_entries as $idx => $entry)
+  { 
+    if(isset($MAP[$entry->id]))
+      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
+    $MAP[$entry->id] = $entry->strid; 
+
+    $strid_crc = crc32($entry->strid); 
+    if(isset($STRIDMAP[$strid_crc]))
+      throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
+    $STRIDMAP[$strid_crc] = $entry->strid; 
+
+    $header[] = array(
+      $payloads[$idx][2],
+      $entry->id, 
+      crc32($entry->strid), 
+      $entry->class_id, 
+      $payloads[$idx][0],
+      $payloads[$idx][3]
+    );
+  }
+
+  $header_msgpack = config_msgpack_pack($header);
+  $payloads_bundle = '';
+  foreach($payloads as $item)
+    $payloads_bundle .= $item[1];
+
+  $packed_data =  
+         pack("C", 1) . 
+         pack("V", $version) . 
+         pack("V", strlen($header_msgpack)) .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $packed_data;
+}
+
+//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
+//      (each path item separated by '/' is stored as an array item)
+function _config_pack_bundle_fmt2(
+  iterable $cache_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version,
+) : string
+{
+  $MAP = array();
+  $STRIDMAP = array();
+  $STRIDLIST = array();
+
+  $payloads = array();
+  $strids = array();
+  $payloads_offset = 0;
+  foreach($cache_entries as $entry)
+  {
+    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
+    $payload_size = strlen($payload);
+    $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
+    $payloads_offset += $payload_size;
+
+    $strids_indices = _config_encode_strid_as_indices($entry->strid, $STRIDMAP, $STRIDLIST);
+
+    $strids[] = $strids_indices;
+  }
+
+  $header = array();
+  foreach($cache_entries as $idx => $entry)
+  { 
+    if(isset($MAP[$entry->id]))
+      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
+    $MAP[$entry->id] = $entry->strid; 
+
+    $header[] = array(
+      $payloads[$idx][2], //format
+      $entry->id, 
+      $strids[$idx], //strid as a lookup indices 
+      $entry->class_id, 
+      $payloads[$idx][0], //offset
+      $payloads[$idx][3]  //size
+    );
+  }
+
+  $strids_msgpack = config_msgpack_pack($STRIDLIST);
+
+  $header_msgpack = config_msgpack_pack($header);
+  $payloads_bundle = '';
+  foreach($payloads as $item)
+    $payloads_bundle .= $item[1];
+
+  $packed_data =  
+         pack("C", 2) . 
+         pack("V", $version) . 
+         pack("V", strlen($strids_msgpack)) .
+         pack("V", strlen($header_msgpack)) .
+         $strids_msgpack .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $packed_data;
+}
+
+//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
+//      This reduces overall bundle size when there are many small configs entries.
+function _config_pack_bundle_fmt3(
+  array $cache_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version,
+  int $chunk_size,
+  int $compression_level) : string
+{
+  if(!$use_lz4)
+    throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
+
+  if($compression_level < 0 || $compression_level > 12)
+    throw new Exception("LZ4 compression level must be in range [0, 12]");
+
+  $MAP = array();
+  $STRIDMAP = array();
+
+  $header = array();
+  $payloads_offset = 0;
+  $max_chunk_size = 0;
+  $chunk_offset = 0;
+  $payloads_bundle = '';
+  $payloads_buffer = '';
+  $count_entries = count($cache_entries);
+  foreach($cache_entries as $idx => $entry)
+  {
+    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
+    $payload_size = strlen($payload);
+    $payloads_buffer .= $payload;
+
+    if(isset($MAP[$entry->id]))
+      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
+    $MAP[$entry->id] = $entry->strid; 
+
+    $strid_crc = crc32($entry->strid); 
+    if(isset($STRIDMAP[$strid_crc]))
+      throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
+    $STRIDMAP[$strid_crc] = $entry->strid; 
+
+    $header[] = array(
+      $format,
+      $entry->id, 
+      crc32($entry->strid), 
+      $entry->class_id, 
+      $chunk_offset,
+      $payloads_offset,
+      $payload_size
+    );
+
+    $payloads_offset += $payload_size;
+    
+    if($payloads_offset >= $chunk_size || $idx == ($count_entries - 1))  
+    {
+      if($payloads_offset > $max_chunk_size)
+        $max_chunk_size = $payloads_offset;
+      $payloads_offset = 0;
+      $lz4_data = lz4_compress($payloads_buffer, $compression_level); 
+      $payloads_bundle .= pack("V", strlen($lz4_data));
+      $payloads_bundle .= $lz4_data;
+      $chunk_offset = strlen($payloads_bundle);
+      $payloads_buffer = '';
+    }
+  }
+
+  $header_msgpack = config_msgpack_pack($header);
+
+  $packed_data =  
+         pack("C", 3) . 
+         pack("V", $version) . 
+         pack("V", strlen($header_msgpack)) .
+         pack("V", $max_chunk_size) .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $packed_data;
+}
+
+function _config_encode_strid_as_indices(string $strid, array &$STRIDMAP, array &$STRIDLIST) : array
+{
+  $strids_indices = array();
+  $strid_parts = explode('/', ltrim($strid, '@'));
+  foreach($strid_parts as $strid_part)
+  {
+    if(!isset($STRIDMAP[$strid_part]))
+    {
+      $strid_index = count($STRIDLIST);
+      $STRIDLIST[] = $strid_part; 
+      $STRIDMAP[$strid_part] = $strid_index;
+      $strids_indices[] = $strid_index;
+    }
+    else
+      $strids_indices[] = $STRIDMAP[$strid_part];
+  }
+  return $strids_indices;
+}
+
+function _config_patch_bundle_fmt2(
+  string $packed_data,
+  array $patch_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version,
+) : string
+{
+  list($strids, $header, $_, $payloads_bundle) = 
+    _config_unpack_bundle_fmt2(packed_data: $packed_data, unpack_entries: false);
+
+  $stridmap = array_flip($strids);
+
+  foreach($patch_entries as $idx => $patch_entry)
+  {
+    list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
+
+    $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+
+    if($header_found)
+    {
+      $header_idx = key($header_found);
+      $header_entry = current($header_found);
+
+      $current_offset = $header_entry[4];
+      $current_size = $header_entry[5];
+      if($current_size >= strlen($patch_payload))
+      {                                         
+        //let's do the inline patching
+        $payloads_bundle = substr_replace($payloads_bundle, $patch_payload, $current_offset, strlen($patch_payload));
+
+        $header_entry[0] = $patch_format;
+        $header_entry[5] = strlen($patch_payload);
+        $header[$header_idx] = $header_entry; 
+      }
+      else
+      {
+        //let's add it to the end
+        $header_entry[0] = $patch_format;
+        $header_entry[4] = strlen($payloads_bundle);
+        $header_entry[5] = strlen($patch_payload);
+        $header[$header_idx] = $header_entry; 
+
+        $payloads_bundle .= $patch_payload;
+      }
+    }
+    else
+    {
+      //let's add new entry it to the end
+      $strid_indices = _config_encode_strid_as_indices($patch_entry->strid, $stridmap, $strids);
+
+      $header_entry = array(
+        $patch_format,
+        $patch_entry->id,
+        $strid_indices,
+        $patch_entry->class_id,
+        strlen($payloads_bundle),
+        strlen($patch_payload),
+      );
+
+      $header[] = $header_entry;
+      $payloads_bundle .= $patch_payload;
+    }
+  }
+
+  $strids_msgpack = config_msgpack_pack($strids);
+  $header_msgpack = config_msgpack_pack($header);
+
+  $patched_data =  
+         pack("C", 2) . 
+         pack("V", $version) . 
+         pack("V", strlen($strids_msgpack)) .
+         pack("V", strlen($header_msgpack)) .
+         $strids_msgpack .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $patched_data;
+}
+
+//format: [[class_id, [data]], ...[class_id, [data]]]
+function config_unpack_bundle(string $packed_data) : array
+{
+  $packed_info = substr($packed_data, 0, 1);
+  $info = unpack('Cformat', $packed_info);
+
+  if($info['format'] === 1)
+  {
+    return _config_unpack_bundle_fmt1($packed_data);
+  }
+  else if($info['format'] === 2)
+  {
+    list($_, $_, $entries) = _config_unpack_bundle_fmt2($packed_data);
+    return $entries;
+  }
+  else if($info['format'] === 3)
+  {
+    return _config_unpack_bundle_fmt3($packed_data);
+  }
+  else
+    throw new Exception("Unknown format: {$info['format']}");
+}
+
+function _config_unpack_bundle_fmt1(string $packed_data) : array
+{
+  $packed_info = substr($packed_data, 0, 1+4+4);
+
+  $info = unpack('Cformat/Vversion/Vheader_len', $packed_info);
+
+  if($info['format'] !== 1)
+    throw new Exception("Unknown format: {$info['format']}");
+
+  $header_msgpack = substr($packed_data, 1+4+4, $info['header_len']); 
+  $header = config_msgpack_unpack($header_msgpack);
+
+  $payloads_bundle = substr($packed_data, 1+4+4+$info['header_len']);
+  
+  $entries = array();
+  foreach($header as $item)
+  {
+    list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
+
+    $payload = substr($payloads_bundle, $offset, $size);
+
+    $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+  }
+
+  return $entries;
+}
+
+function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries = true) : array
+{
+  $packed_info = substr($packed_data, 0, 1+4+4+4);
+
+  $info = unpack('Cformat/Vversion/Vstrids_len/Vheader_len', $packed_info);
+
+  if($info['format'] !== 2)
+    throw new Exception("Unknown format: {$info['format']}");
+
+  $strids_msgpack = substr($packed_data, 1+4+4+4, $info['strids_len']); 
+  $strids = config_msgpack_unpack($strids_msgpack);
+
+  $header_msgpack = substr($packed_data, 1+4+4+4+$info['strids_len'], $info['header_len']); 
+  $header = config_msgpack_unpack($header_msgpack);
+
+  $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
+  
+  $entries = array();
+
+  if($unpack_entries)
+  {
+    foreach($header as $item)
+    {
+      list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
+
+      $payload = substr($payloads_bundle, $offset, $size);
+
+      $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+    }
+  }
+
+  return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
+}
+
+function _config_unpack_bundle_fmt3(string $packed_data): array
+{
+  if(ord($packed_data[0]) !== 3)
+  {
+    throw new Exception("Invalid config bundle format");
+  }
+
+  $offset = 1;
+  $version = unpack("V", substr($packed_data, $offset, 4))[1];
+  $offset += 4;
+  $header_len = unpack("V", substr($packed_data, $offset, 4))[1];
+  $offset += 4;
+  $max_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
+  $offset += 4;
+
+  $header_msgpack = substr($packed_data, $offset, $header_len);
+  $offset += $header_len;
+
+  $header = config_msgpack_unpack($header_msgpack);
+
+  $cache_entries = [];
+  $chunk_offset = 0;
+  $chunk_buffer = '';
+  $chunk_id = -1;
+
+  foreach ($header as $entry_data)
+  {
+    list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
+
+    if($entry_chunk_offset !== $chunk_id)
+    {
+      if($chunk_offset !== -1)
+      {
+        $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
+        $offset+=4;
+        $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
+        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+        $offset += $lz4_chunk_size;
+        $chunk_offset = $offset;
+      }
+      else
+      {
+        $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
+        $chunk_offset += 4;
+        $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
+        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+        $chunk_offset += $lz4_chunk_size;
+      }
+
+      $chunk_id = $entry_chunk_offset;
+    }
+
+    $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
+
+    $cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+  }
+
+  return $cache_entries;
+}
+
+//format: [format_id, payload_data]
+function _config_get_payload(ConfigCacheEntry $ce, bool $use_lz4, bool $use_config_refs) : array
+{
+  $format = ConfigCacheEntry::FMT_BINARY;
+  $payload = null;
+
+  if($use_config_refs && $ce->payload_file)
+  {
+    $format = ConfigCacheEntry::FMT_FILE_REF;
+    $payload = $ce->payload_file;
+  }
+  else
+  {
+    $payload = $ce->payload;
+    if($use_lz4 && strlen($payload) > 512)
+    {
+      $format = ConfigCacheEntry::FMT_LZ4;
+      $payload = lz4_compress($payload, 9);
+    }
+  }
+
+  return array($format, $payload);
+}
+
+function _config_unpack_payload(int $format, string $payload) : array
+{
+  $msg_packed = null;
+  if($format === ConfigCacheEntry::FMT_LZ4)
+    $msg_packed = lz4_uncompress($payload);
+  else if($format === ConfigCacheEntry::FMT_BINARY)
+    $msg_packed = $payload;
+  else if($format === ConfigCacheEntry::FMT_FILE_REF)
+    $msg_packed = ensure_read($payload);
+  else
+    throw new Exception("Bad format: $format");
+  return config_msgpack_unpack($msg_packed);
+}
+
+function config_pack_and_write_bundle(
+  /*var ConfBase[]*/
+  array $configs, 
+  string $file_path, 
+  int $version,
+  bool $use_lz4 = true, 
+  int $binary_format = 1, 
+  bool $debug = false
+)
+{
+  $cache_entries = array();
+  foreach($configs as $conf)
+  {
+    $payload = config_msgpack_pack($conf->export());
+
+    //creating fake cache entries
+    $entry = new ConfigCacheEntry();
+    $entry->id = $conf->id;
+    $entry->strid = $conf->strid;
+    $entry->class_id = $conf->getClassId();
+    $entry->class = get_class($conf);
+    $entry->payload = $payload;
+    $entry->config = $conf;
+
+    $cache_entries[] = $entry;
+  }
+
+  $packed_data = config_pack_bundle(
+    new ConfigPackParams(
+      cache_entries: $cache_entries, 
+      use_lz4: $use_lz4, 
+      binary_format: $binary_format, 
+      version: $version,
+      debug: $debug
+    )
+  );
+
+  ensure_write($file_path, $packed_data);
+}
+

parse.inc.php

@@ -0,0 +1,175 @@
+<?php
+namespace taskman;
+use Exception;
+
+class ConfigParseResult
+{
+  public int $error = 0;
+  public string $error_descr;
+  public string $normalized_jzon = '';
+  public array $parsed_arr;
+  public int $parser_type;
+  public \JSM_Module $jsm_module;
+}
+
+function config_parse(array $base_dirs, string $file) : ConfigParseResult
+{
+  $res = new ConfigParseResult();
+
+  $normalized_jzon = '';
+  try
+  {
+    $jsm = new \JSM($base_dirs, $file);
+    list($normalized_jzon, $jsm_module) = $jsm->process();
+    $res->normalized_jzon = $normalized_jzon;
+    $res->jsm_module = $jsm_module;
+  }
+  catch(Exception $e)
+  {
+    $res->error = 1;
+    $res->error_descr = $e->getMessage() . "\n" . $e->getTraceAsString(); 
+    return $res;
+  }
+
+  $decode_res = config_check_and_decode_jzon($normalized_jzon);
+  
+  if($decode_res[0] === 0)
+  {
+    $res->parsed_arr = $decode_res[2];
+    $res->parser_type = $decode_res[3];
+  }
+  else
+  {
+    $res->error = $decode_res[0];
+    $res->error_descr = $decode_res[1];
+  }
+
+  return $res;
+}
+
+function config_check_and_decode_jzon(string $json) : array
+{
+  try
+  {
+    $parser = 0;
+    $arr = \jzon_parse($json, $parser);
+    return array(0, "", $arr, $parser);
+  }
+  catch(Exception $e)
+  {
+    return array(1, $e->getMessage(), array(), 0);
+  }
+}
+
+function config_get_header(string $file, ?int &$conf_id, ?string &$strid, bool $use_cache = true) : bool
+{
+  static $cache = array();
+  if($use_cache && isset($cache[$file]))
+  {
+    list($conf_id, $strid) = $cache[$file];
+    return true;
+  }
+
+  $h = fopen($file, "r");
+  $line = fgets($h, 256);
+  fclose($h);
+
+  //TODO: rewrite it without regex usage
+  if(preg_match('~\{\s*/\*\s*proto_id\s*=\s*(\d+)\s*;\s*alias\s*=\s*([^\s]+)~', $line, $matches))
+  {
+    $conf_id = (int)$matches[1];
+    $strid = $matches[2];
+    if($use_cache)
+      $cache[$file] = array($conf_id, $strid);
+    return true;
+  }
+  else
+    return false;
+}
+
+function config_set_header(string $contents, int $conf_id, string $strid, bool &$is_success) : string
+{
+  $contents = preg_replace('~\s*\{~', "{  /* proto_id = {$conf_id} ; alias = {$strid} */", $contents, 1/*limit*/, $count);
+  $is_success = $count == 1;
+  return $contents;
+}
+
+function config_replace_header(string $contents) : string
+{
+  return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents);
+}
+
+//returns [proto_id, alias]
+function config_ensure_header(string $conf_dir, string $file, bool $force = false) : array
+{
+  if(config_get_header($file, $curr_proto_id, $curr_alias, !$force))
+  {
+    //TODO: add 'normalized' argument when we are sure both path arguments are normalized
+    $alias = config_file2strid($conf_dir, $file);
+    if($force)
+      $curr_proto_id = config_file2id($conf_dir, $file);
+
+    //NOTE: keeping current proto id intact if not forced
+    if($force || $curr_alias !== $alias)
+    {
+      $lines = file($file);
+      //TODO: why not using config_set_header(..) ?
+      $lines[0] = preg_replace(
+        '~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{  /* proto_id = {$curr_proto_id} ; alias = {$alias} */", 
+        $lines[0], 
+        1/*limit*/, 
+        $count);
+      if($count != 1)
+        throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
+      ensure_write_if_differs($file, join("", $lines));
+    }
+
+    return array($curr_proto_id, $alias);
+  }
+  else
+  {
+    $conf_id = config_file2id($conf_dir, $file);
+    $alias = config_file2strid($conf_dir, $file);
+
+    $lines = file($file);
+
+    $is_success = false;
+    $lines[0] = config_set_header($lines[0], $conf_id, $alias, $is_success);
+    if(!$is_success)
+      throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
+    ensure_write($file, join("", $lines));
+
+    return array($conf_id, $alias);
+  }
+}
+
+function config_apply_class_normalization(array $kv_data, bool $check_junk = false) : array
+{
+  $class = $kv_data['class'];
+  $norm_data = array();
+  unset($kv_data['class']);
+  call_user_func_array([$class, 'normalize'], 
+    array(&$kv_data, &$norm_data, true, $check_junk));
+  return array($class, $class::CLASS_ID, $norm_data);
+}
+
+//returns array<ref, number>
+function config_content_match_refs(string $src) : array
+{
+  $refs = array();
+
+  if(strpos($src, '@') === false)
+    return $refs;
+
+  if(preg_match_all('~"(@[^"]+)"~', $src, $matches))
+  {
+    foreach($matches[1] as $ref)
+    {
+      if(!isset($refs[$ref]))
+        $refs[$ref] = 1;
+      ++$refs[$ref];
+    }
+  }
+  return $refs;
+}
+

scan.inc.php

@@ -0,0 +1,232 @@
+<?php
+namespace taskman;
+use Exception;
+
+class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator
+{
+  private array $base_dirs;
+
+  /*var array<string, string[]>*/
+  private array $base_dir2files = array();
+
+  private $iter_pos = 0;
+
+  function __construct(array $base_dir2files = array(), ?array $base_dirs = null)
+  {
+    foreach($base_dir2files as $dir => $files)
+      $this->base_dir2files[$dir] = $files;
+
+    if($base_dirs === null)
+      $this->base_dirs = array_keys($base_dir2files);
+    else
+      $this->base_dirs = $base_dirs;
+  }
+
+  static function makeFor(ConfigManager $mgr) : ConfigDirFiles
+  {
+    return new ConfigDirFiles([], $mgr->getGlobals()->base_dirs);
+  }
+
+  static function makeFromArtefactFiles(\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles
+  {
+    return new ConfigDirFiles($files->toMap());
+  }
+
+  function clear()
+  {
+    $this->base_dir2files = array();
+  }
+
+  function isEmpty() : bool
+  {
+    return empty($this->base_dir2files);
+  }
+
+  function count() : int
+  {
+    $total = 0;
+    foreach($this->base_dir2files as $base_dir => $files)
+      $total += count($files);
+    return $total;
+  }
+
+  function apply(callable $fn)
+  {
+    foreach($this->base_dir2files as $base_dir => $files)
+      $this->base_dir2files[$base_dir] = $fn($base_dir, $files);
+  }
+
+  function filter(callable $filter)
+  {
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      $filtered = array_filter($files, $filter);
+      //NOTE: don't want any index gaps
+      $this->base_dir2files[$base_dir] = array_values($filtered);
+    }
+  }
+
+  function forEachFile(callable $fn)
+  {
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      foreach($files as $file)
+        $fn($base_dir, $file);
+    }
+  }
+
+  function addFile(string $file, bool $unique = false)
+  {
+    $this->add(config_map_base_dir($this->base_dirs, $file, normalized: true), $file, $unique);
+  }
+
+  function add(string $base_dir, string $file, bool $unique = false)
+  {
+    if(!isset($this->base_dir2files[$base_dir]))
+      $this->base_dir2files[$base_dir] = array();
+
+    if(!$unique || !in_array($file, $this->base_dir2files[$base_dir]))
+      $this->base_dir2files[$base_dir][] = $file;
+  }
+
+  function getMap() : array
+  {
+    return $this->base_dir2files;
+  }
+
+  //returns [[base_dir, file1], [base_dir, file2], ...]
+  function getFlatArray() : array
+  {
+    $flat = [];
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      foreach($files as $file)
+        $flat[] = [$base_dir, $file];
+    }
+    return $flat;
+  }
+
+  function getAllFiles() : array
+  {
+    $all_files = [];
+    foreach($this->base_dir2files as $base_dir => $files)
+      $all_files = array_merge($all_files, $files);
+
+    return $all_files;
+  }
+   
+  //ArrayAccess interface
+  function offsetExists(mixed $offset) : bool
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    return $this->count() > $offset;
+  }
+
+  function offsetGet(mixed $offset) : mixed
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+        return $files[$offset];
+      $offset -= $n;
+    }
+    return null;
+  }
+
+  function offsetSet(mixed $offset, mixed $value) : void
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => &$files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+      {
+        $files[$offset] = $value;
+        return;
+      }
+      $offset -= $n;
+    }
+  }
+
+  function offsetUnset(mixed $offset) : void
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+      {
+        unset($files[$offset]);
+        return;
+      }
+      $offset -= $n;
+    }
+  }
+
+  //Iterator interface
+  function rewind() : void
+  {
+    $this->iter_pos = 0;
+  }
+
+  function current() : mixed 
+  {
+    return $this->offsetGet($this->iter_pos);
+  }
+
+  function key() : mixed 
+  {
+    return $this->iter_pos;
+  }
+
+  function next() : void
+  {
+    ++$this->iter_pos;
+  }
+
+  function valid() : bool 
+  {
+    return $this->offsetExists($this->iter_pos);
+  }
+}
+
+function config_scan_files(
+  iterable $base_dirs, 
+  string $ext_filter = '.conf.js', 
+  bool $verbose = false
+  ) : ConfigDirFiles
+{
+  $t = microtime(true);
+
+  $base_dir2files = [];
+  foreach($base_dirs as $base_dir)
+  {
+    $base_dir2files[$base_dir] = 
+      scan_files_rec([$base_dir], [$ext_filter]);
+  }
+
+  $result = new ConfigDirFiles($base_dir2files);
+
+  if($verbose)
+    config_log("File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec.");
+
+  return $result;
+}
+
+//NOTE: this one is obsolete
+function config_hash_changed(ConfigGlobals $globals, iterable $all_files)
+{
+  $all_crc_file = $globals->build_dir . "/configs.crc";
+  return names_hash_changed($all_crc_file, $all_files);
+}
+

task.inc.php

@@ -0,0 +1,28 @@
+<?php
+namespace taskman;
+use Exception;
+use Throwable;
+
+task('config_update_worker', function(array $args)
+{
+  if(sizeof($args) != 3)
+    throw new Exception("Config worker args not set");
+
+  $in_file = $args[0];
+  $out_file = $args[1];
+  $err_file = $args[2];
+
+  try
+  {
+    list($params, $job) = unserialize(ensure_read($in_file));
+    $result = _config_update_worker_func($params, $job);
+    ensure_write($out_file, serialize($result));
+  }
+  catch(Throwable $e)
+  {
+    //NOTE: explicitely catching all exceptions and writing to the error file 
+    //      since under Windows error file stream redirect may work unreliably 
+    file_put_contents($err_file, $e->getMessage() . "\n" . $e->getTraceAsString(), FILE_APPEND);
+    throw $e;
+  }
+});

util.inc.php

@@ -0,0 +1,135 @@
+<?php
+namespace taskman;
+use Exception;
+
+function config_get_tmp_build_path(ConfigGlobals $globals, string $file) : string
+{
+  $name = str_replace(['@', ':', "\\", "/"], ['', "-", "-", "-"], normalize_path($file));
+  $name = ltrim($name, "-");
+  return normalize_path($globals->build_dir . "/$name");
+}
+
+function config_map_base_dir(
+  array $base_dirs, string $file, 
+  bool $normalized = false, bool $strict = true
+ ) : ?string
+{
+  if(!$normalized)
+    $file = normalize_path($file);
+  foreach($base_dirs as $dir)
+    if(strpos($file, $dir) === 0)
+      return $dir;
+  if($strict)
+    throw new Exception("File '$file' is not mapped to any base dir");
+  return null;
+}
+
+function config_real_path(array $base_dirs, string $rel_path, bool $strict = true) : ?string
+{
+  foreach($base_dirs as $dir)
+    if(is_file($dir . '/' . $rel_path))
+      return $dir . '/' . $rel_path;
+  if($strict)
+    throw new Exception("No file for relative path '$rel_path'");
+  return null;
+}
+
+function config_is_file(string $filename) : bool
+{
+  return (strrpos($filename, '.conf.js') === (strlen($filename) - 8));
+}
+
+function config_file2id(string $conf_dir, string $filename) : int
+{
+  $nfilename = config_make_path($conf_dir, $filename);
+  return config_crc28($nfilename);
+}
+
+function config_file2strid(string $conf_dir, string $filename) : string
+{
+  $strid = config_make_path($conf_dir, $filename); 
+  $fname_idx = strrpos($strid, "/");
+  $fname_idx = strpos($strid, ".", $fname_idx);
+  $strid = substr($strid, 0, $fname_idx);
+  return "@".$strid;
+}
+
+function config_make_path(string $conf_dir, string $path) : string
+{
+  return ltrim(str_replace(normalize_path($conf_dir, true/*nix*/), 
+                           '', 
+                           normalize_path($path, true/*nix*/)), 
+                '/');
+}
+
+function config_crc28(string $what) : int
+{
+  return crc32($what) & 0xFFFFFFF;
+}
+
+function config_get_module_includes(\JSM_Module $cm) : array
+{
+  $includes = array();
+  //NOTE: module contains all includes (even those included from other modules)
+  foreach($cm->getIncludes() as $include => $_)
+  {
+    //maybe we should take .php includes into account as well?
+    if(str_ends_with($include, ".php"))
+      continue;
+    $includes[] = $include;
+  }
+  return $includes;
+}
+
+function config_walk_fields(object $proto, callable $callback)
+{
+  if(!method_exists($proto, 'CLASS_FIELDS_PROPS'))
+    return;
+
+  foreach($proto->CLASS_FIELDS_PROPS() as $field => $tokens)
+  {
+    $value = $proto->$field;
+    $callback($proto, $field, $value, $tokens);
+
+    if(is_object($value))
+      config_walk_fields($value, $callback);
+    else if(is_array($value))
+    {
+      foreach($value as $num => $item)
+      {
+        if(is_object($item))
+          config_walk_fields($item, $callback);
+      }
+    }
+  }
+}
+
+function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
+{
+  if(!isset($map[$file]))
+    return array($file);
+
+  $tpls = $map[$file];
+  $tpls[] = $file;
+  $res = array();
+  foreach($tpls as $tpl)
+  {
+    $content = ensure_read($tpl);
+    $content = i18n_decode_string($content);
+    if(strpos($content, $text) !== FALSE)
+      $res[] = $tpl;
+  }
+
+  return $res;
+}
+
+function config2json(object $conf, int $json_flags = 0) : string
+{
+  $arr = $conf->export(true); 
+  $arr['class'] = get_class($conf);
+  unset($arr['id']); // @phpstan-ignore-line
+  unset($arr['strid']); // @phpstan-ignore-line
+  $json = json_encode($arr, $json_flags);
+  $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json);
+  return $json;
+}