Cleaning API

CHANGELOG.md

@@ -1,3 +1,6 @@
+## v5.10.0
+- Added fmt3 bundle format - basically, fmt1 with chunk-based lz4 compression
+
 ## v4.0.2
 - Improving exception message
 

cache.inc.php

@@ -2,6 +2,101 @@
 namespace taskman;
 use Exception;
 
+class ConfigCache
+{
+  private ConfigGlobals $globals;
+
+  /** @var array<string, ConfigCacheEntry> */
+  private array $by_path = [];
+  /** @var array<int, ConfigCacheEntry> */
+  private array $by_id = [];
+  /** @var array<string, ConfigCacheEntry> */
+  private array $by_alias = [];
+
+  function __construct(ConfigGlobals $globals)
+  {
+    $this->globals = $globals;
+  }
+
+  function clear()
+  {
+    $this->by_path = [];
+    $this->by_id = [];
+    $this->by_alias = [];
+  }
+
+  function tryGetOrLoadByPath(string $path) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($path, $this->by_path))
+    {
+      $cache_file = config_get_cache_path($this->globals, $path);
+      if(!is_file($cache_file))
+        return null;
+      $ce = ConfigCacheEntry::unserialize(ensure_read($cache_file));
+      $this->by_path[$path] = $ce;
+      return $ce;
+    }
+
+    return $this->by_path[$path];
+  }
+
+  function getOrLoadByPath(string $path) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadByPath($path);
+    if($ce === null)
+      throw new Exception("Failed to find entry by path '$path'");
+    return $ce;
+  }
+
+  function tryGetOrLoadById(int $id) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($id, $this->by_id))
+    {
+      $cache_id_file = config_get_cache_id_path($this->globals, $id);
+      if(!is_file($cache_id_file))
+        return null;
+      $file = ensure_read($cache_id_file);
+      $ce = $this->tryGetOrLoadByPath($file);
+      $this->by_id[$id] = $ce;
+      return $ce;
+    }
+
+    return $this->by_id[$id];
+  }
+
+  function getOrLoadById(int $id) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadById($id);
+    if($ce === null)
+      throw new Exception("Failed to find entry by id '$id'");
+    return $ce;
+  }
+
+  function tryGetOrLoadByStrid(string $strid) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($strid, $this->by_alias))
+    {
+      $cache_strid_file = config_get_cache_strid_path($this->globals, $strid);
+      if(!is_file($cache_strid_file))
+        return null;
+      $file = ensure_read($cache_strid_file);
+      $ce = $this->tryGetOrLoadByPath($file);
+      $this->by_alias[$strid] = $ce;
+      return $ce;
+    }
+
+    return $this->by_alias[$strid];
+  }
+
+  function getOrLoadByStrid(string $strid) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadByStrid($strid);
+    if($ce === null)
+      throw new Exception("Failed to find entry by strid '$strid'");
+    return $ce;
+  }
+}
+
 /**
  * @property object $config
  * @property string $payload
@@ -146,182 +241,195 @@ class ConfigCacheEntryExtras extends \stdClass
   }
 }
 
-class ConfigFetchParams
+class ConfigCacheFileMap
 {
-  public ConfigGlobals $globals;
-  public ConfigScanResult $scanned;
-  public bool $force_stale = false;
-  public bool $verbose = false;
-  public ?int $max_workers = null;
-  public bool $touch_files_with_includes = true;
-  public bool $check_junk = false;
+  private array $file2deps = array();
 
-  function __construct(ConfigGlobals $globals, ConfigScanResult $scanned, 
-    bool $force_stale = false, bool $verbose = false, ?int $max_workers = null)
+  static function serialize(ConfigCacheFileMap $map) : string
+  {
+    $d = $map->export();
+    return serialize($d);
+  }
+
+  static function unserialize(string $str) : ?ConfigCacheFileMap
+  {
+    $d = @unserialize($str);
+    if(!is_array($d))
+      return null;
+    $map = new ConfigCacheFileMap();
+    $map->import($d);
+    return $map;
+  }
+
+  function count() : int
+  {
+    return count($this->file2deps);
+  }
+
+  function exists(string $file) : bool
+  {
+    return isset($this->file2deps[$file]);
+  }
+
+  function getAffectedFiles(string $file) : array
+  {
+    if(!isset($this->file2deps[$file]))
+      return [];
+    return array_keys($this->file2deps[$file]);
+  }
+
+  function differs(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles|array $files) : bool
+  {
+    if(!is_array($files))
+      $files = $files->getAllFiles();
+
+    $this_files = array_keys($this->file2deps);
+
+    return count($files) !== count($this_files) || 
+      count(array_intersect($files, $this_files)) != count($this_files);
+  }
+
+  function compare(array $files) : array
+  {
+    $this_files = array_keys($this->file2deps);
+
+    $added = array_diff($files, $this_files);
+    $removed = array_diff($this_files, $files);
+
+    return [$added, $removed];
+  }
+
+  function init(array $files)
+  {
+    $this->file2deps = [];
+    foreach($files as $file)
+      $this->file2deps[$file] = [];
+  }
+
+  function update(array $added, array $removed)
+  {
+    foreach($added as $file)
+    {
+      //NOTE: we preserve existing files
+      if(!isset($this->file2deps[$file]))
+        $this->file2deps[$file] = [];
+    }
+    foreach($removed as $file)
+      unset($this->file2deps[$file]);
+  }
+
+  function updateDepsForEntry(ConfigCacheEntry $entry)
+  {
+    foreach($entry->includes as $include)
+    {
+      if(isset($this->file2deps[$include]))
+        $this->file2deps[$include][$entry->file] = true;
+      else
+        $this->file2deps[$include] = [$entry->file => true];
+    }
+  }
+
+  function export() : array
+  {
+    $d = array();
+    $d[] = $this->file2deps;
+    return $d;
+  }
+
+  function import(array $d)
+  {
+    list($this->file2deps) = $d;
+  }
+}
+
+class ConfigCacheUpdateParams
+{
+  const MAX_DEFAULT_WORKERS = 5;
+  const FILES_THRESHOLD_ONE_WORKER = 100;
+
+  public ConfigGlobals $globals;
+  public ConfigDirFiles $affected_files;
+  public bool $verbose = false;
+  public bool $check_junk = true;
+  public int $max_errors_num = 15;
+  public bool $return_affected_entries = false;
+
+  function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, bool $verbose = false)
   {
     $this->globals = $globals;
-    $this->scanned = $scanned;
-    $this->force_stale = $force_stale;
+    $this->affected_files = $affected_files;
     $this->verbose = $verbose;
-    $this->max_workers = $max_workers;
   }
 
   function calcMaxWorkers() : int
   {
-    $max_workers = $this->max_workers;
-    if($max_workers === null)
-      $max_workers = $this->scanned->count() < 20 ? 1 : 5;
-    return $max_workers;
+    if($this->affected_files->count() < self::FILES_THRESHOLD_ONE_WORKER)
+      return 1;
+    return $this->globals->max_workers ?? self::MAX_DEFAULT_WORKERS;
   }
 
-  function splitFilesByChunks(int $max_workers) : array
+  function splitFilesByChunks(int $max_workers, bool $sort = true) : array
   {
-    $chunk_size = (int)ceil($this->scanned->count()/$max_workers);
-    return array_chunk($this->scanned->getFlatArray(), $chunk_size);
+    $flat = $this->affected_files->getFlatArray();
+    if(!$flat)
+      return array();
+    if($sort)
+      usort($flat, fn($a, $b) => $a[1] <=> $b[1]);
+
+    $chunk_size = (int)ceil(count($flat)/$max_workers);
+    return array_chunk($flat, $chunk_size);
   }
 
   //returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ]
-  function splitJobs() : array
+  function splitJobs(bool $sort = true) : array
   {
     $max_workers = $this->calcMaxWorkers();
     $jobs = array();
-    foreach($this->splitFilesByChunks($max_workers) as $idx => $chunk)
+    $chunks = $this->splitFilesByChunks($max_workers, $sort); 
+    foreach($chunks as $idx => $chunk)
       $jobs[] = array($idx, microtime(true), $chunk);
     return $jobs;
   }
 }
 
-class ConfigFetchResult
+function _config_cache_update(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
 {
-  /** @var ConfigCacheEntry[] */
-  public array $all = array();
-  /** @var array<int, ConfigCacheEntry> */
-  public array $by_id = array();
-  /** @var array<string, ConfigCacheEntry> */
-  public array $by_path = array();
-  /** @var array<string, ConfigCacheEntry> */
-  public array $by_alias = array();
-  /** @var array<string, string> */
-  public array $errors = array();
-  public int $stales = 0;
-  public int $corruptions = 0;
-  public int $fast_jsons = 0;
-
-  function getByAlias(string $strid) : ConfigCacheEntry
-  {
-    if(array_key_exists($strid, $this->by_alias))
-      return $this->by_alias[$strid];
-    throw new Exception("Failed to find config by alias '$strid'!");
-  }
-  
-  function getById(int $id) : ConfigCacheEntry
-  {
-    if(array_key_exists($id, $this->by_id))
-      return $this->by_id[$id];
-    throw new Exception("Failed to find config by id '$id'!");
-  }
-  
-  function getByPath(string $path) : ConfigCacheEntry
-  {
-    if(array_key_exists($path, $this->by_path))
-      return $this->by_path[$path];
-    throw new Exception("Failed to find config by path '$path'!");
-  }
-}
-
-function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
-{
-  $t = microtime(true);
-
-  $result = _config_cache_fetch($params);
-
-  if($result->errors)
-  {
-    $errors = array();
-    foreach($result->errors as $file => $error)
-    {
-      $errors[] = (count($errors) + 1) . ") Error in file '$file': $error";
-      if(count($errors) > 15)
-        break;
-    }
-
-    throw new Exception(implode("\n", $errors));
-  }
-
-  if($params->verbose)
-    config_log("Fetch from cache: " . round(microtime(true) - $t,2) . " sec.");
-
-  return $result;
-}
-
-function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
-{
-  if($params->scanned->isEmpty())
-    return new ConfigFetchResult();
-
-  //TODO: not sure if it's the best place for this one
-  $GLOBALS['CONFIG_GLOBALS'] = $params->globals;
-
-  $params->scanned->apply(function($base_dir, $files) { sort($files); return $files;});
-
-  $jobs = $params->splitJobs();
+  $jobs = $params->splitJobs(sort: true);
 
   $serial = sizeof($jobs) == 1;
 
-  $results_by_job = _config_worker_run_procs($params, $jobs, $serial);
+  $results_by_job = _config_update_worker_run_procs($params, $jobs, $serial);
 
   if(!$serial)
   {
-    //NOTE: in case of any result error try serial processing
+    //NOTE: in case result unserialize error try serial processing
     if(array_search(false, $results_by_job, true/*strict*/) !== false)
     {
       if($params->verbose)
-        config_log("Result error detected, trying serial processing...");
-      $results_by_job = _config_worker_run_procs($params, $jobs, true);
+        config_log("Corrupted result, trying serial processing...");
+      $results_by_job = _config_update_worker_run_procs(params: $params, jobs: $jobs, serial: true);
     }
   }
 
-  $t = microtime(true);
-  $result = _config_merge_fetch_results($params, $results_by_job);
-  if($params->verbose)
-    config_log("Merge results: " . round(microtime(true) - $t,2) . " sec.");
+  _config_merge_update_results($result, $results_by_job);
 
   if($params->verbose)
-    config_log("Miss(Fast)/Total: {$result->stales}($result->fast_jsons)/" . sizeof($result->all));
-
-  return $result;
+    config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
 }
 
-function config_cache_fetch_by_path(ConfigGlobals $globals, string $path, bool $force_stale = false) : ConfigCacheEntry
+function _config_merge_update_results(ConfigUpdateResult $result, array $results_by_job)
 {
-  $path = realpath($path);
-  $base_dir = config_map_base_dir($globals->base_dirs, $path);
-
-  $scanned = new ConfigScanResult();
-  $scanned->add($base_dir, $path);
-
-  $ces = config_cache_fetch(new ConfigFetchParams(
-    globals: $globals, 
-    scanned: $scanned,
-    force_stale: $force_stale
-  ));
-
-  if(empty($ces->all))
-    throw new Exception("Config not found at path '$path'");
-  return $ces->all[0];
-}
-
-function _config_merge_fetch_results(ConfigFetchParams $params, array $results_by_job) : ConfigFetchResult
-{
-  $result = new ConfigFetchResult();
-  $total_stales = 0;
   $total_fast_jsons = 0;
   
   foreach($results_by_job as $results)
   {
     foreach($results as $item) 
     {
-      list($base_dir, $file, $cache_file, $is_stale, $parser_type, $error) = $item;
+      $base_dir = $item['base_dir'];
+      $file = $item['file'];
+      $cache_file = $item['cache_file'];
+      $parser_type = $item['parser_type'];
+      $error = $item['error'];
 
       if($error !== null)
       {
@@ -331,50 +439,14 @@ function _config_merge_fetch_results(ConfigFetchParams $params, array $results_b
 
       if($parser_type === 1)
         ++$total_fast_jsons;
-
-      $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file));
-
-      //NOTE: handling any cache corruption errors
-      if($cache_entry === null)
-      {
-        $is_stale = true;
-        $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
-        ++$result->corruptions;
-      }
-
-      $includes = $cache_entry->includes;
-
-      if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes))
-      {
-        $is_stale = true;
-        $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
-        //NOTE: let's change the mtime of the file which include other files,
-        //      so that on tne next build it will be 'older' than its includes
-        //      and won't trigger rebuild
-        if($params->touch_files_with_includes)
-          touch($file);
-      }
-
-      if($is_stale)
-        ++$total_stales;
-
-      //we want results to be returned in the same order, so 
-      //we store entries by the file key and later retrieve array values
-      $result->by_path[$file] = $cache_entry;
-      $result->by_id[$cache_entry->id] = $cache_entry;
-      $result->by_alias[$cache_entry->strid] = $cache_entry;
     }
   }
 
-  $result->all = array_values($result->by_path);
-  $result->stales = $total_stales;
   $result->fast_jsons = $total_fast_jsons;
-  return $result;
 }
 
-function _config_invalidate_cache(
-  ConfigFetchParams $params, string $base_dir, string $file, 
-  string $cache_file, ?int &$parser_type = null) : ConfigCacheEntry
+function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file, 
+  ?int &$parser_type = null) : ConfigCacheEntry
 {
   list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
   if(!$conf_id)
@@ -399,6 +471,8 @@ function _config_invalidate_cache(
 
   $cache_payload_file = config_get_cache_payload_path($params->globals, $file);
 
+  $cache_file = config_get_cache_path($params->globals, $file);
+
   $cache_entry = new ConfigCacheEntry();
   $cache_entry->id = $conf_id;
   $cache_entry->strid = $conf_strid;
@@ -408,14 +482,90 @@ function _config_invalidate_cache(
   $cache_entry->payload_file = $cache_payload_file;
   //TODO: pass flag if file is actually normalized?
   $cache_entry->file = normalize_path($file);
-  $cache_entry->includes = config_get_module_includes($pres->jsm_module);
-  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
+  $cache_entry->includes = config_get_module_includes(cm: $pres->jsm_module);
   //TODO: do we really need these refs?
   $cache_entry->refs = config_content_match_refs($pres->normalized_jzon);
+  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
 
   ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
   ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
+  ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
+  ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
 
   return $cache_entry;
 }
 
+function _config_update_worker_run_procs(ConfigCacheUpdateParams $params, array $jobs, bool $serial) : array
+{
+  if($serial)
+  {
+    $results_by_job = array();
+    foreach($jobs as $job)
+      $results_by_job[] = _config_update_worker_func($params, $job, $serial);
+    return $results_by_job;
+  }
+  else
+  {
+    //initializing worker for master process anyway
+    $params->globals->initWorker(true);
+
+    $workers_args = array();
+    foreach($jobs as $job)
+      $workers_args[] = array($params, $job);
+
+    return run_background_gamectl_workers('config_update_worker', $workers_args);
+  }
+}
+
+function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job, bool $is_master_proc = false) : array
+{
+  $start_time = microtime(true);
+
+  $params->globals->initWorker($is_master_proc);
+
+  list($idx, $start_time, $chunk) = $job;
+  if($params->verbose)
+    config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
+
+  $fast_parser_num = 0;
+  $results = array();
+  foreach($chunk as $file_idx => $chunk_data)
+  {
+    list($base_dir, $file) = $chunk_data;
+
+    try
+    {
+      if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
+        config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
+
+      $cache_file = config_get_cache_path($params->globals, $file);
+
+      $parser_type = null;
+      $entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
+
+      $results[] = [
+        'base_dir' => $base_dir, 
+        'file' => $file, 
+        'cache_file' => $entry->cache_file, 
+        'parser_type' => $parser_type, 
+        'error' => null
+      ];
+    }
+    catch(\Throwable $e)
+    {
+      $results[] = [
+        'base_dir' => $base_dir, 
+        'file' => $file, 
+        'cache_file' => null,
+        'parser_type' => null,
+        'error' => $e->getMessage()
+      ];
+    }
+  }
+
+  if($params->verbose)
+    config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
+
+  return $results;
+}
+

config.inc.php

@@ -11,32 +11,371 @@ require_once(__DIR__ . '/msgpack.inc.php');
 
 class ConfigGlobals
 {
-  public array $base_dirs = array();
-  public ?string $worker_init_fn = null;
-  public string $base_class = '\ConfBase';
-  public string $build_dir;
+  public readonly array $base_dirs;
+  public readonly ?string $worker_init_fn;
+  public readonly int $workers_num;
+  public readonly string $base_class;
+  public readonly string $build_dir;
 
-  function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null)
+  function __construct(
+    array $base_dirs, 
+    string $build_dir, 
+    ?string $worker_init_fn = null,
+    int $workers_num = 1,
+    string $base_class = '\ConfBase'
+  )
   {
     $this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
     $this->build_dir = $build_dir;
     $this->worker_init_fn = $worker_init_fn;
+    $this->workers_num = $workers_num;
+    $this->base_class = $base_class;
   }
 
-  function initWorker()
+  function initWorker(bool $is_master_proc)
   {
     $GLOBALS['CONFIG_GLOBALS'] = $this;
 
     if($this->worker_init_fn !== null)
     {
       $fn = $this->worker_init_fn;
-      $fn();
+      $fn($is_master_proc);
+    }
+  }
+}
+
+enum ConfigUpdateMode : int
+{
+  case Force         = 1;
+  case DetectChanged = 2;
+  case Selected      = 3;
+}
+
+class ConfigUpdateRequest
+{
+  public ConfigUpdateMode $mode;
+  public ?ConfigDirFiles $files;
+  public ?string $result_file;
+  public ?\taskman\TaskmanFileChanges $file_changes;
+  public bool $verbose = false;
+  public bool $return_entries = false;
+
+  private function __construct() {}
+
+  static function force(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
+  {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
+    $req = new ConfigUpdateRequest();
+    $req->mode = ConfigUpdateMode::Force;
+    $req->files = $files;
+    return $req;
+  }
+
+  static function detectChanged(string $result_file, \taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
+  {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
+    $req = new ConfigUpdateRequest();
+    $req->mode = ConfigUpdateMode::DetectChanged;
+    $req->files = $files;
+    $req->result_file = $result_file;
+    return $req;
+  }
+
+  static function selected(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles $files, \taskman\TaskmanFileChanges|null $file_changes = null) : ConfigUpdateRequest
+  {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
+    $req = new ConfigUpdateRequest();
+    $req->mode = ConfigUpdateMode::Selected;
+    $req->files = $files;
+    $req->file_changes = $file_changes;
+    return $req;
+  }
+}
+
+class ConfigUpdateResult
+{
+  public ConfigDirFiles $affected_files;
+  public array $affected_entries = array();
+
+  /** @var array<string, string> */
+  public array $errors = array();
+  public int $corruptions = 0;
+  public int $fast_jsons = 0;
+
+  function __construct(ConfigCacheUpdateParams $params)
+  {
+    $this->affected_files = $params->affected_files;
+  }
+}
+
+class ConfigManager
+{
+  private ConfigGlobals $globals; 
+  private ConfigCache $cache;
+  private ?ConfigCacheFileMap $file_map;
+
+  function __construct(ConfigGlobals $globals)
+  {
+    $this->globals = $globals;
+
+    $this->cache = new ConfigCache($globals);
+    $this->file_map = null;
+  }
+
+  function getGlobals() : ConfigGlobals
+  {
+    return $this->globals;
+  }
+
+  function getCache() : ConfigCache
+  {
+    return $this->cache;
+  }
+
+  function getArtefactFilesSpec() : array
+  {
+    return [$this->globals->base_dirs, ['.js']];
+  }
+
+  function getFileMap() : ConfigCacheFileMap
+  {
+    if($this->file_map === null)
+    {
+      $this->file_map = $this->_tryLoadMap();
+      if($this->file_map === null)
+      {
+        $this->file_map = self::_makeMap($this->scanFiles(extension: '.js'));
+        $this->_saveFileMap();
+      }
+    }
+    return $this->file_map;
+  }
+
+  function update(ConfigUpdateRequest $req) : ConfigUpdateResult
+  {
+    config_log("Updating configs, mode '{$req->mode->name}'...");
+
+    if($req->files === null)
+      $req->files = $this->scanFiles(extension: '.js', verbose: $req->verbose);
+
+    $added_files = [];
+    $removed_files = [];
+    $this->_checkFileMap($req, $added_files, $removed_files);
+
+    $affected_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
+
+    //NOTE: at this point taking into account only config files
+    $affected_files->filter(fn($file) => str_ends_with($file, '.conf.js'));
+
+    config_log("Affected files: {$affected_files->count()}");
+
+    $update_params = new ConfigCacheUpdateParams(
+      globals: $this->globals, 
+      affected_files: $affected_files, 
+      verbose: $req->verbose
+    );
+    $update_result = self::_updateCache($update_params);
+
+    //let's clear internal cache once the update procedure is done
+    $this->cache->clear();
+
+    $this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
+
+    if($req->return_entries)
+    {
+      foreach($affected_files as $file)
+      {
+        $entry = $this->cache->getOrLoadByPath($file);
+        $update_result->affected_entries[] = $entry;
       }
     }
 
-  function setNormalizeBaseDirs(array $dirs)
+    return $update_result;
+  }
+
+  private function _updateCache(ConfigCacheUpdateParams $params) : ConfigUpdateResult
   {
-    $this->base_dirs = array_map(function($d) { return normalize_path($d); }, $dirs);
+    $result = new ConfigUpdateResult($params);
+
+    if($params->affected_files->isEmpty())
+      return $result;
+
+    $t = microtime(true);
+
+    _config_cache_update($result, $params);
+
+    if($result->errors)
+    {
+      $errors = array();
+      foreach($result->errors as $file => $error)
+      {
+        $errors[] = (count($errors) + 1) . ") Error in file '$file': $error";
+        if(count($errors) > $params->max_errors_num)
+          break;
+      }
+
+      throw new Exception(implode("\n", $errors));
+    }
+
+    if($params->verbose)
+      config_log("Update cache: " . round(microtime(true) - $t,2) . " sec.");
+
+    return $result;
+  }
+
+  private function _checkFileMap(ConfigUpdateRequest $req, array &$added_files, array &$removed_files)
+  {
+    $fs_cache_map = $this->getFileMap();
+
+    if($req->mode === ConfigUpdateMode::Force)
+    {
+      //let's rebuild the file map
+      $fs_cache_map->init($req->files->getAllFiles());
+      config_log("File map init: ".$fs_cache_map->count());
+    }
+    else if($req->mode === ConfigUpdateMode::DetectChanged)
+    {
+      list($added_files, $removed_files) = $fs_cache_map->compare($req->files->getAllFiles());
+      config_log("File map compare, added: ".count($added_files).", removed: ".count($removed_files));
+    }
+    else if($req->mode === ConfigUpdateMode::Selected)
+    {
+      if($req->file_changes != null)
+      {
+        foreach($req->files as $file)
+        {
+          if($req->file_changes->isCreated($file))
+            $added_files[] = $file;
+          else if($req->file_changes->isDeleted($file))
+            $removed_files[] = $file;
+        }
+      }
+    }
+  }
+
+  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
+  {
+    $fs_cache_map = $this->getFileMap();
+
+    //TODO: traverse all affected files and update file map
+    foreach($affected_files as $file)
+    {
+      $cache_entry = $this->cache->getOrLoadByPath($file);
+      $fs_cache_map->updateDepsForEntry($cache_entry);
+    }
+
+    if($req->mode == ConfigUpdateMode::Force || 
+      $affected_files->count() > 0 || 
+      $added_files || 
+      $removed_files)
+    {
+      $fs_cache_map->update($added_files, $removed_files);
+      $this->_saveFileMap();
+    }
+  }
+
+  private function _getAffectedFiles(ConfigUpdateRequest $req, array $added_files, array $removed_files) : ConfigDirFiles
+  {
+    $fs_cache_map = $this->getFileMap();
+
+    $affected_files = null;
+
+    if($req->mode === ConfigUpdateMode::Force)
+    {
+      $affected_files = $req->files;
+    }
+    else if($req->mode === ConfigUpdateMode::DetectChanged)
+    {
+      $affected_files = ConfigDirFiles::makeFor($this);
+
+      foreach($req->files->getMap() as $base_dir => $files)
+      {
+        foreach($files as $file)
+        {
+          if(need_to_regen($req->result_file, [$file]))
+          {
+            $affected_files->add($base_dir, $file);
+
+            $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+            foreach($affected_by_file as $dep)
+              $affected_files->addFile($dep, unique: true);
+          }
+        }
+      }
+
+      //if there were removed files we need to rebuild affected files
+      foreach($removed_files as $file)
+      {
+        if(!str_ends_with($file, '.conf.js'))
+        {
+          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+          foreach($affected_by_file as $dep)
+            $affected_files->addFile($dep, unique: true);
+        }
+      }
+    }
+    else if($req->mode === ConfigUpdateMode::Selected)
+    {
+      $affected_files = ConfigDirFiles::makeFor($this);
+
+      foreach($req->files as $file)
+      {
+        $affected_files->addFile($file, unique: true);
+
+        if(!str_ends_with($file, '.conf.js'))
+        {
+          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+          foreach($affected_by_file as $dep)
+            $affected_files->addFile($dep, unique: true);
+        }
+      }
+    }
+
+    return $affected_files;
+  }
+
+  function parseFile(string $file) : ConfigParseResult
+  {
+    return config_parse($this->globals->base_dirs, $file);
+  }
+
+  function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
+  {
+    return config_scan_files($this->globals->base_dirs, $extension, $verbose);
+  }
+
+  private function _getMapPath() : string
+  {
+    return $this->globals->build_dir . '/file_map.data';
+  }
+
+  private function _tryLoadMap() : ?ConfigCacheFileMap
+  {
+    if(!is_file($this->_getMapPath()))
+      return null;
+    config_log("Loading file map");
+    return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
+  }
+
+  private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
+  {
+    config_log("Creating file map");
+    $map = new ConfigCacheFileMap();
+    $map->init($files->getAllFiles());
+    return $map;
+  }
+
+  private function _saveFileMap()
+  {
+    $data = ConfigCacheFileMap::serialize($this->getFileMap()); 
+    config_log("Saving file map: " . kb($data));
+    ensure_write($this->_getMapPath(), $data);
   }
 }
 
@@ -47,12 +386,22 @@ function config_log($msg)
 
 function config_get_cache_path(ConfigGlobals $globals, string $file) : string
 {
-  return config_get_tmp_build_path($globals, $file . '.cacheb');
+  return config_get_tmp_build_path($globals, $file . '.cache');
+} 
+
+function config_get_cache_id_path(ConfigGlobals $globals, int $id) : string
+{
+  return config_get_tmp_build_path($globals, $id . '.id');
+} 
+
+function config_get_cache_strid_path(ConfigGlobals $globals, string $strid) : string
+{
+  return config_get_tmp_build_path($globals, $strid  . '.strid');
 } 
 
 function config_get_cache_payload_path(ConfigGlobals $globals, string $file) : string
 {
-  return config_get_tmp_build_path($globals, $file . '.pdata');
+  return config_get_tmp_build_path($globals, $file . '.payload');
 } 
 
 function config_load(ConfigGlobals $globals, string $conf_path, ?string $conf_dir = null) : object
@@ -120,3 +469,13 @@ function config_bench_load(ConfigGlobals $globals, string $file)
   config_log("Load: " . (microtime(true) - $t));
 }
 
+function config_changes_has_deleted_or_added(TaskmanFileChanges $file_changes, iterable $files) : bool
+{
+  foreach($files as $file)
+  {
+    if(str_ends_with($file, '.conf.js') && ($file_changes->isDeleted($file) || $file_changes->isCreated($file)))
+      return true;
+  }
+  return false;
+}
+

pack.inc.php

@@ -5,15 +5,26 @@ use Exception;
 class ConfigPackParams
 {
   /*var ConfigCacheEntry[]*/
-  public array $cache_entries;
+  public iterable $cache_entries;
   public bool $use_lz4 = false;
   public bool $use_config_refs = false;
   public int $binary_format = 1; //1,2 supported
   public ?int $version = null;
   public bool $debug = false;
 
-  function __construct(array $cache_entries, int $version, bool $use_lz4 = false, 
-    bool $use_config_refs = false, int $binary_format = 1, bool $debug = false)
+  public const EXTRA_FMT3_CHUNK_SIZE = "EXTRA_FMT3_CHUNK_SIZE";
+  public const EXTRA_FMT3_COMPRESSION_LEVEL = "EXTRA_FMT3_COMPRESSION_LEVEL";
+  public array $extras = array();
+
+  function __construct(
+    iterable $cache_entries, 
+    int $version, 
+    bool $use_lz4 = false, 
+    bool $use_config_refs = false, 
+    int $binary_format = 1, 
+    bool $debug = false,
+    array $extras = array()
+  )
   {
     $this->cache_entries = $cache_entries;
     $this->use_lz4 = $use_lz4;
@@ -21,6 +32,7 @@ class ConfigPackParams
     $this->binary_format = $binary_format;
     $this->version = $version;
     $this->debug = $debug;
+    $this->extras = $extras;
   }
 }
 
@@ -45,7 +57,18 @@ function config_pack_bundle(ConfigPackParams $params) : string
       $params->cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
-      $params->version
+      $params->version,
+    );
+  }
+  else if($params->binary_format == 3)
+  {
+    $packed_data = _config_pack_bundle_fmt3(
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version,
+      $params->extras[ConfigPackParams::EXTRA_FMT3_CHUNK_SIZE],
+      $params->extras[ConfigPackParams::EXTRA_FMT3_COMPRESSION_LEVEL],
     );
   }
   else
@@ -61,8 +84,38 @@ function config_pack_bundle(ConfigPackParams $params) : string
   return $packed_data;
 }
 
+function config_patch_bundle(ConfigPackParams $params, string $packed_data) : string
+{
+  $t = microtime(true);
+
+  $patched_data = null;
+
+  if($params->binary_format == 2)
+  {
+    $patched_data = _config_patch_bundle_fmt2(
+      $packed_data,
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version,
+    );
+  }
+  else
+    throw new Exception("Unknown binary format: {$params->binary_format}");
+
+  if($params->debug)
+    config_log("Patched entries: " . sizeof($params->cache_entries) . ", total: " . 
+      kb($patched_data) . ", format: {$params->binary_format}, lz4: " . 
+      var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
+      ", CRC: " . crc32($patched_data) . 
+      ", " . round(microtime(true) - $t,2) . " sec.");
+
+  return $patched_data;
+}
+
+//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
 function _config_pack_bundle_fmt1(
-  array $cache_entries, 
+  iterable $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
   int $version) : string
@@ -117,11 +170,14 @@ function _config_pack_bundle_fmt1(
   return $packed_data;
 }
 
+//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
+//      (each path item separated by '/' is stored as an array item)
 function _config_pack_bundle_fmt2(
-  array $cache_entries, 
+  iterable $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
-  int $version) : string
+  int $version,
+) : string
 {
   $MAP = array();
   $STRIDMAP = array();
@@ -137,20 +193,8 @@ function _config_pack_bundle_fmt2(
     $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
     $payloads_offset += $payload_size;
 
-    $strids_indices = array();
-    $strid_parts = explode('/', ltrim($entry->strid, '@'));
-    foreach($strid_parts as $strid_part)
-    {
-      if(!isset($STRIDMAP[$strid_part]))
-      {
-        $strid_index = count($STRIDLIST);
-        $STRIDLIST[] = $strid_part; 
-        $STRIDMAP[$strid_part] = $strid_index;
-        $strids_indices[] = $strid_index;
-      }
-      else
-        $strids_indices[] = $STRIDMAP[$strid_part];
-    }
+    $strids_indices = _config_encode_strid_as_indices($entry->strid, $STRIDMAP, $STRIDLIST);
+
     $strids[] = $strids_indices;
   }
 
@@ -190,6 +234,184 @@ function _config_pack_bundle_fmt2(
   return $packed_data;
 }
 
+//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
+//      This reduces overall bundle size when there are many small configs entries.
+function _config_pack_bundle_fmt3(
+  array $cache_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version,
+  int $chunk_size,
+  int $compression_level) : string
+{
+  if(!$use_lz4)
+    throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
+
+  if($compression_level < 0 || $compression_level > 12)
+    throw new Exception("LZ4 compression level must be in range [0, 12]");
+
+  $MAP = array();
+  $STRIDMAP = array();
+
+  $header = array();
+  $payloads_offset = 0;
+  $max_chunk_size = 0;
+  $chunk_offset = 0;
+  $payloads_bundle = '';
+  $payloads_buffer = '';
+  $count_entries = count($cache_entries);
+  foreach($cache_entries as $idx => $entry)
+  {
+    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
+    $payload_size = strlen($payload);
+    $payloads_buffer .= $payload;
+
+    if(isset($MAP[$entry->id]))
+      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
+    $MAP[$entry->id] = $entry->strid; 
+
+    $strid_crc = crc32($entry->strid); 
+    if(isset($STRIDMAP[$strid_crc]))
+      throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
+    $STRIDMAP[$strid_crc] = $entry->strid; 
+
+    $header[] = array(
+      $format,
+      $entry->id, 
+      crc32($entry->strid), 
+      $entry->class_id, 
+      $chunk_offset,
+      $payloads_offset,
+      $payload_size
+    );
+
+    $payloads_offset += $payload_size;
+    
+    if($payloads_offset >= $chunk_size || $idx == ($count_entries - 1))  
+    {
+      if($payloads_offset > $max_chunk_size)
+        $max_chunk_size = $payloads_offset;
+      $payloads_offset = 0;
+      $lz4_data = lz4_compress($payloads_buffer, $compression_level); 
+      $payloads_bundle .= pack("V", strlen($lz4_data));
+      $payloads_bundle .= $lz4_data;
+      $chunk_offset = strlen($payloads_bundle);
+      $payloads_buffer = '';
+    }
+  }
+
+  $header_msgpack = config_msgpack_pack($header);
+
+  $packed_data =  
+         pack("C", 3) . 
+         pack("V", $version) . 
+         pack("V", strlen($header_msgpack)) .
+         pack("V", $max_chunk_size) .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $packed_data;
+}
+
+function _config_encode_strid_as_indices(string $strid, array &$STRIDMAP, array &$STRIDLIST) : array
+{
+  $strids_indices = array();
+  $strid_parts = explode('/', ltrim($strid, '@'));
+  foreach($strid_parts as $strid_part)
+  {
+    if(!isset($STRIDMAP[$strid_part]))
+    {
+      $strid_index = count($STRIDLIST);
+      $STRIDLIST[] = $strid_part; 
+      $STRIDMAP[$strid_part] = $strid_index;
+      $strids_indices[] = $strid_index;
+    }
+    else
+      $strids_indices[] = $STRIDMAP[$strid_part];
+  }
+  return $strids_indices;
+}
+
+function _config_patch_bundle_fmt2(
+  string $packed_data,
+  array $patch_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version,
+) : string
+{
+  list($strids, $header, $_, $payloads_bundle) = 
+    _config_unpack_bundle_fmt2(packed_data: $packed_data, unpack_entries: false);
+
+  $stridmap = array_flip($strids);
+
+  foreach($patch_entries as $idx => $patch_entry)
+  {
+    list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
+
+    $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+
+    if($header_found)
+    {
+      $header_idx = key($header_found);
+      $header_entry = current($header_found);
+
+      $current_offset = $header_entry[4];
+      $current_size = $header_entry[5];
+      if($current_size >= strlen($patch_payload))
+      {                                         
+        //let's do the inline patching
+        $payloads_bundle = substr_replace($payloads_bundle, $patch_payload, $current_offset, strlen($patch_payload));
+
+        $header_entry[0] = $patch_format;
+        $header_entry[5] = strlen($patch_payload);
+        $header[$header_idx] = $header_entry; 
+      }
+      else
+      {
+        //let's add it to the end
+        $header_entry[0] = $patch_format;
+        $header_entry[4] = strlen($payloads_bundle);
+        $header_entry[5] = strlen($patch_payload);
+        $header[$header_idx] = $header_entry; 
+
+        $payloads_bundle .= $patch_payload;
+      }
+    }
+    else
+    {
+      //let's add new entry it to the end
+      $strid_indices = _config_encode_strid_as_indices($patch_entry->strid, $stridmap, $strids);
+
+      $header_entry = array(
+        $patch_format,
+        $patch_entry->id,
+        $strid_indices,
+        $patch_entry->class_id,
+        strlen($payloads_bundle),
+        strlen($patch_payload),
+      );
+
+      $header[] = $header_entry;
+      $payloads_bundle .= $patch_payload;
+    }
+  }
+
+  $strids_msgpack = config_msgpack_pack($strids);
+  $header_msgpack = config_msgpack_pack($header);
+
+  $patched_data =  
+         pack("C", 2) . 
+         pack("V", $version) . 
+         pack("V", strlen($strids_msgpack)) .
+         pack("V", strlen($header_msgpack)) .
+         $strids_msgpack .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $patched_data;
+}
+
 //format: [[class_id, [data]], ...[class_id, [data]]]
 function config_unpack_bundle(string $packed_data) : array
 {
@@ -202,7 +424,12 @@ function config_unpack_bundle(string $packed_data) : array
   }
   else if($info['format'] === 2)
   {
-    return _config_unpack_bundle_fmt2($packed_data);
+    list($_, $_, $entries) = _config_unpack_bundle_fmt2($packed_data);
+    return $entries;
+  }
+  else if($info['format'] === 3)
+  {
+    return _config_unpack_bundle_fmt3($packed_data);
   }
   else
     throw new Exception("Unknown format: {$info['format']}");
@@ -235,7 +462,7 @@ function _config_unpack_bundle_fmt1(string $packed_data) : array
   return $entries;
 }
 
-function _config_unpack_bundle_fmt2(string $packed_data) : array
+function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries = true) : array
 {
   $packed_info = substr($packed_data, 0, 1+4+4+4);
 
@@ -253,6 +480,9 @@ function _config_unpack_bundle_fmt2(string $packed_data) : array
   $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
   
   $entries = array();
+
+  if($unpack_entries)
+  {
     foreach($header as $item)
     {
       list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
@@ -261,8 +491,69 @@ function _config_unpack_bundle_fmt2(string $packed_data) : array
 
       $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
     }
+  }
 
-  return $entries;
+  return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
+}
+
+function _config_unpack_bundle_fmt3(string $packed_data): array
+{
+  if(ord($packed_data[0]) !== 3)
+  {
+    throw new Exception("Invalid config bundle format");
+  }
+
+  $offset = 1;
+  $version = unpack("V", substr($packed_data, $offset, 4))[1];
+  $offset += 4;
+  $header_len = unpack("V", substr($packed_data, $offset, 4))[1];
+  $offset += 4;
+  $max_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
+  $offset += 4;
+
+  $header_msgpack = substr($packed_data, $offset, $header_len);
+  $offset += $header_len;
+
+  $header = config_msgpack_unpack($header_msgpack);
+
+  $cache_entries = [];
+  $chunk_offset = 0;
+  $chunk_buffer = '';
+  $chunk_id = -1;
+
+  foreach ($header as $entry_data)
+  {
+    list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
+
+    if($entry_chunk_offset !== $chunk_id)
+    {
+      if($chunk_offset !== -1)
+      {
+        $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
+        $offset+=4;
+        $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
+        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+        $offset += $lz4_chunk_size;
+        $chunk_offset = $offset;
+      }
+      else
+      {
+        $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
+        $chunk_offset += 4;
+        $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
+        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+        $chunk_offset += $lz4_chunk_size;
+      }
+
+      $chunk_id = $entry_chunk_offset;
+    }
+
+    $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
+
+    $cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+  }
+
+  return $cache_entries;
 }
 
 //format: [format_id, payload_data]
@@ -307,9 +598,10 @@ function config_pack_and_write_bundle(
   /*var ConfBase[]*/
   array $configs, 
   string $file_path, 
+  int $version,
   bool $use_lz4 = true, 
   int $binary_format = 1, 
-  ?int $version = null
+  bool $debug = false
 )
 {
   $cache_entries = array();
@@ -334,7 +626,8 @@ function config_pack_and_write_bundle(
       cache_entries: $cache_entries, 
       use_lz4: $use_lz4, 
       binary_format: $binary_format, 
-      version: $version
+      version: $version,
+      debug: $debug
     )
   );
 

parse.inc.php

@@ -9,7 +9,7 @@ class ConfigParseResult
   public string $normalized_jzon = '';
   public array $parsed_arr;
   public int $parser_type;
-  public $jsm_module;
+  public \JSM_Module $jsm_module;
 }
 
 function config_parse(array $base_dirs, string $file) : ConfigParseResult

scan.inc.php

@@ -2,10 +2,40 @@
 namespace taskman;
 use Exception;
 
-class ConfigScanResult
+class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator
 {
+  private array $base_dirs;
+
   /*var array<string, string[]>*/
-  public array $base_dir2files = array();
+  private array $base_dir2files = array();
+
+  private $iter_pos = 0;
+
+  function __construct(array $base_dir2files = array(), ?array $base_dirs = null)
+  {
+    foreach($base_dir2files as $dir => $files)
+      $this->base_dir2files[$dir] = $files;
+
+    if($base_dirs === null)
+      $this->base_dirs = array_keys($base_dir2files);
+    else
+      $this->base_dirs = $base_dirs;
+  }
+
+  static function makeFor(ConfigManager $mgr) : ConfigDirFiles
+  {
+    return new ConfigDirFiles([], $mgr->getGlobals()->base_dirs);
+  }
+
+  static function makeFromArtefactFiles(\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles
+  {
+    return new ConfigDirFiles($files->toMap());
+  }
+
+  function clear()
+  {
+    $this->base_dir2files = array();
+  }
 
   function isEmpty() : bool
   {
@@ -29,7 +59,11 @@ class ConfigScanResult
   function filter(callable $filter)
   {
     foreach($this->base_dir2files as $base_dir => $files)
-      $this->base_dir2files[$base_dir] = array_filter($files, $filter);
+    {
+      $filtered = array_filter($files, $filter);
+      //NOTE: don't want any index gaps
+      $this->base_dir2files[$base_dir] = array_values($filtered);
+    }
   }
 
   function forEachFile(callable $fn)
@@ -41,14 +75,25 @@ class ConfigScanResult
     }
   }
 
-  function add(string $base_dir, string $file)
+  function addFile(string $file, bool $unique = false)
+  {
+    $this->add(config_map_base_dir($this->base_dirs, $file, normalized: true), $file, $unique);
+  }
+
+  function add(string $base_dir, string $file, bool $unique = false)
   {
     if(!isset($this->base_dir2files[$base_dir]))
       $this->base_dir2files[$base_dir] = array();
 
+    if(!$unique || !in_array($file, $this->base_dir2files[$base_dir]))
       $this->base_dir2files[$base_dir][] = $file;
   }
 
+  function getMap() : array
+  {
+    return $this->base_dir2files;
+  }
+
   //returns [[base_dir, file1], [base_dir, file2], ...]
   function getFlatArray() : array
   {
@@ -69,26 +114,119 @@ class ConfigScanResult
 
     return $all_files;
   }
+   
+  //ArrayAccess interface
+  function offsetExists(mixed $offset) : bool
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    return $this->count() > $offset;
+  }
+
+  function offsetGet(mixed $offset) : mixed
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+        return $files[$offset];
+      $offset -= $n;
+    }
+    return null;
+  }
+
+  function offsetSet(mixed $offset, mixed $value) : void
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => &$files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+      {
+        $files[$offset] = $value;
+        return;
+      }
+      $offset -= $n;
+    }
+  }
+
+  function offsetUnset(mixed $offset) : void
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+      {
+        unset($files[$offset]);
+        return;
+      }
+      $offset -= $n;
+    }
+  }
+
+  //Iterator interface
+  function rewind() : void
+  {
+    $this->iter_pos = 0;
+  }
+
+  function current() : mixed 
+  {
+    return $this->offsetGet($this->iter_pos);
+  }
+
+  function key() : mixed 
+  {
+    return $this->iter_pos;
+  }
+
+  function next() : void
+  {
+    ++$this->iter_pos;
+  }
+
+  function valid() : bool 
+  {
+    return $this->offsetExists($this->iter_pos);
+  }
 }
 
 function config_scan_files(
-  array $base_dirs, 
+  iterable $base_dirs, 
   string $ext_filter = '.conf.js', 
   bool $verbose = false
-  ) : ConfigScanResult
+  ) : ConfigDirFiles
 {
   $t = microtime(true);
 
-  $result = new ConfigScanResult();
-
+  $base_dir2files = [];
   foreach($base_dirs as $base_dir)
   {
-    $result->base_dir2files[$base_dir] = 
-      scan_files_rec(array($base_dir), array($ext_filter));
+    $base_dir2files[$base_dir] = 
+      scan_files_rec([$base_dir], [$ext_filter]);
   }
 
+  $result = new ConfigDirFiles($base_dir2files);
+
   if($verbose)
     config_log("File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec.");
 
   return $result;
 }
+
+//NOTE: this one is obsolete
+function config_hash_changed(ConfigGlobals $globals, iterable $all_files)
+{
+  $all_crc_file = $globals->build_dir . "/configs.crc";
+  return names_hash_changed($all_crc_file, $all_files);
+}
+

task.inc.php

@@ -3,7 +3,7 @@ namespace taskman;
 use Exception;
 use Throwable;
 
-task('config_worker', function(array $args)
+task('config_update_worker', function(array $args)
 {
   if(sizeof($args) != 3)
     throw new Exception("Config worker args not set");
@@ -15,7 +15,7 @@ task('config_worker', function(array $args)
   try
   {
     list($params, $job) = unserialize(ensure_read($in_file));
-    $result = _config_worker_func($params, $job);
+    $result = _config_update_worker_func($params, $job);
     ensure_write($out_file, serialize($result));
   }
   catch(Throwable $e)
@@ -26,70 +26,3 @@ task('config_worker', function(array $args)
     throw $e;
   }
 });
-
-function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $serial) : array
-{
-  if($serial)
-  {
-    $results_by_job = array();
-    foreach($jobs as $job)
-      $results_by_job[] = _config_worker_func($params, $job);
-    return $results_by_job;
-  }
-  else
-  {
-    $worker_args = array();
-    foreach($jobs as $idx => $job)
-      $worker_args[] = array($params, $job);
-
-    return run_background_gamectl_workers('config_worker', $worker_args);
-  }
-}
-
-//returns [[base_dir, file, cache_file, was_stale, parser_type, error], ...]
-function _config_worker_func(ConfigFetchParams $params, array $job) : array
-{
-  $start_time = microtime(true);
-
-  $params->globals->initWorker();
-
-  list($idx, $start_time, $chunk) = $job;
-  if($params->verbose)
-    config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
-
-  $fast_parser_num = 0;
-  $results = array();
-  foreach($chunk as $file_idx => $chunk_data)
-  {
-    list($base_dir, $file) = $chunk_data;
-
-    try
-    {
-      if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
-        config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
-
-      $cache_file = config_get_cache_path($params->globals, $file);
-
-      $parser_type = null;
-
-      $is_stale = true;
-      if(!$params->force_stale)
-        $is_stale = need_to_regen($cache_file, array($file));
-
-      if($is_stale)
-        _config_invalidate_cache($params, $base_dir, $file, $cache_file, $parser_type);
-
-      $results[] = array($base_dir, $file, $cache_file, $is_stale, $parser_type, null);
-    }
-    catch(Throwable $e)
-    {
-      $results[] = array($base_dir, $file, null, null, null, $e->getMessage());
-    }
-  }
-
-  if($params->verbose)
-    config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
-
-  return $results;
-}
-

util.inc.php

@@ -4,7 +4,7 @@ use Exception;
 
 function config_get_tmp_build_path(ConfigGlobals $globals, string $file) : string
 {
-  $name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file))));
+  $name = str_replace(['@', ':', "\\", "/"], ['', "-", "-", "-"], normalize_path($file));
   $name = ltrim($name, "-");
   return normalize_path($globals->build_dir . "/$name");
 }
@@ -70,6 +70,7 @@ function config_crc28(string $what) : int
 function config_get_module_includes(\JSM_Module $cm) : array
 {
   $includes = array();
+  //NOTE: module contains all includes (even those included from other modules)
   foreach($cm->getIncludes() as $include => $_)
   {
     //maybe we should take .php includes into account as well?
@@ -122,15 +123,13 @@ function config_includes_map_find_text_origin(array $map, string $file, string $
   return $res;
 }
 
-//TODO:
-//function conf2json(object $conf, int $json_flags = 0) : string
-//{
-//  $arr = $conf->export(true); 
-//  $arr['class'] = get_class($conf);
-//  unset($arr['id']);
-//  unset($arr['strid']);
-//  $json = json_encode($arr, $json_flags);
-//  $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json);
-//  return $json;
-//}
-
+function config2json(object $conf, int $json_flags = 0) : string
+{
+  $arr = $conf->export(true); 
+  $arr['class'] = get_class($conf);
+  unset($arr['id']); // @phpstan-ignore-line
+  unset($arr['strid']); // @phpstan-ignore-line
+  $json = json_encode($arr, $json_flags);
+  $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json);
+  return $json;
+}