CHANGELOG.md

@@ -1,6 +1,3 @@
-## v5.10.0
-- Added fmt3 bundle format - basically, fmt1 with chunk-based lz4 compression
-
 ## v4.0.2
 - Improving exception message
 

cache.inc.php

@@ -2,101 +2,6 @@
 namespace taskman;
 use Exception;
 
-class ConfigCache
-{
-  private ConfigGlobals $globals;
-
-  /** @var array<string, ConfigCacheEntry> */
-  private array $by_path = [];
-  /** @var array<int, ConfigCacheEntry> */
-  private array $by_id = [];
-  /** @var array<string, ConfigCacheEntry> */
-  private array $by_alias = [];
-
-  function __construct(ConfigGlobals $globals)
-  {
-    $this->globals = $globals;
-  }
-
-  function clear()
-  {
-    $this->by_path = [];
-    $this->by_id = [];
-    $this->by_alias = [];
-  }
-
-  function tryGetOrLoadByPath(string $path) : ?ConfigCacheEntry
-  {
-    if(!array_key_exists($path, $this->by_path))
-    {
-      $cache_file = config_get_cache_path($this->globals, $path);
-      if(!is_file($cache_file))
-        return null;
-      $ce = ConfigCacheEntry::unserialize(ensure_read($cache_file));
-      $this->by_path[$path] = $ce;
-      return $ce;
-    }
-
-    return $this->by_path[$path];
-  }
-
-  function getOrLoadByPath(string $path) : ConfigCacheEntry
-  {
-    $ce = $this->tryGetOrLoadByPath($path);
-    if($ce === null)
-      throw new Exception("Failed to find entry by path '$path'");
-    return $ce;
-  }
-
-  function tryGetOrLoadById(int $id) : ?ConfigCacheEntry
-  {
-    if(!array_key_exists($id, $this->by_id))
-    {
-      $cache_id_file = config_get_cache_id_path($this->globals, $id);
-      if(!is_file($cache_id_file))
-        return null;
-      $file = ensure_read($cache_id_file);
-      $ce = $this->tryGetOrLoadByPath($file);
-      $this->by_id[$id] = $ce;
-      return $ce;
-    }
-
-    return $this->by_id[$id];
-  }
-
-  function getOrLoadById(int $id) : ConfigCacheEntry
-  {
-    $ce = $this->tryGetOrLoadById($id);
-    if($ce === null)
-      throw new Exception("Failed to find entry by id '$id'");
-    return $ce;
-  }
-
-  function tryGetOrLoadByStrid(string $strid) : ?ConfigCacheEntry
-  {
-    if(!array_key_exists($strid, $this->by_alias))
-    {
-      $cache_strid_file = config_get_cache_strid_path($this->globals, $strid);
-      if(!is_file($cache_strid_file))
-        return null;
-      $file = ensure_read($cache_strid_file);
-      $ce = $this->tryGetOrLoadByPath($file);
-      $this->by_alias[$strid] = $ce;
-      return $ce;
-    }
-
-    return $this->by_alias[$strid];
-  }
-
-  function getOrLoadByStrid(string $strid) : ConfigCacheEntry
-  {
-    $ce = $this->tryGetOrLoadByStrid($strid);
-    if($ce === null)
-      throw new Exception("Failed to find entry by strid '$strid'");
-    return $ce;
-  }
-}
-
 /**
  * @property object $config
  * @property string $payload
@@ -241,238 +146,232 @@ class ConfigCacheEntryExtras extends \stdClass
   }
 }
 
-class ConfigCacheFileMap
+class ConfigFetchParams
 {
-  private array $file2deps = array();
-
-  static function serialize(ConfigCacheFileMap $map) : string
-  {
-    $d = $map->export();
-    return serialize($d);
-  }
-
-  static function unserialize(string $str) : ?ConfigCacheFileMap
-  {
-    $d = @unserialize($str);
-    if(!is_array($d))
-      return null;
-    $map = new ConfigCacheFileMap();
-    $map->import($d);
-    return $map;
-  }
-
-  function count() : int
-  {
-    return count($this->file2deps);
-  }
-
-  function exists(string $file) : bool
-  {
-    return isset($this->file2deps[$file]);
-  }
-
-  function getAffectedFiles(string $file) : array
-  {
-    if(!isset($this->file2deps[$file]))
-      return [];
-    return array_keys($this->file2deps[$file]);
-  }
-
-  function differs(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles|array $files) : bool
-  {
-    if(!is_array($files))
-      $files = $files->getAllFiles();
-
-    $this_files = array_keys($this->file2deps);
-
-    return count($files) !== count($this_files) || 
-      count(array_intersect($files, $this_files)) != count($this_files);
-  }
-
-  function compare(array $files) : array
-  {
-    $this_files = array_keys($this->file2deps);
-
-    $added = array_diff($files, $this_files);
-    $removed = array_diff($this_files, $files);
-
-    return [$added, $removed];
-  }
-
-  function init(array $files)
-  {
-    $this->file2deps = [];
-    foreach($files as $file)
-      $this->file2deps[$file] = [];
-  }
-
-  function update(array $added, array $removed)
-  {
-    foreach($added as $file)
-    {
-      //NOTE: we preserve existing files
-      if(!isset($this->file2deps[$file]))
-        $this->file2deps[$file] = [];
-    }
-    foreach($removed as $file)
-      unset($this->file2deps[$file]);
-  }
-
-  function updateDepsForEntry(ConfigCacheEntry $entry)
-  {
-    foreach($entry->includes as $include)
-    {
-      if(isset($this->file2deps[$include]))
-        $this->file2deps[$include][$entry->file] = true;
-      else
-        $this->file2deps[$include] = [$entry->file => true];
-    }
-  }
-
-  function export() : array
-  {
-    $d = array();
-    $d[] = $this->file2deps;
-    return $d;
-  }
-
-  function import(array $d)
-  {
-    list($this->file2deps) = $d;
-  }
-}
-
-class ConfigCacheUpdateParams
-{
-  const MAX_DEFAULT_WORKERS = 5;
-  const FILES_THRESHOLD_ONE_WORKER = 100;
-
   public ConfigGlobals $globals;
-  public ConfigDirFiles $affected_files;
+  public ConfigScanResult $scanned;
+  public bool $force_stale = false;
   public bool $verbose = false;
-  public bool $check_junk = true;
+  public ?int $max_workers = null;
-  public int $max_errors_num = 15;
+  public bool $touch_files_with_includes = true;
-  public bool $return_affected_entries = false;
 
-  function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, bool $verbose = false)
+  function __construct(ConfigGlobals $globals, ConfigScanResult $scanned, 
+    bool $force_stale = false, bool $verbose = false, ?int $max_workers = null)
   {
     $this->globals = $globals;
-    $this->affected_files = $affected_files;
+    $this->scanned = $scanned;
+    $this->force_stale = $force_stale;
     $this->verbose = $verbose;
+    $this->max_workers = $max_workers;
   }
 
   function calcMaxWorkers() : int
   {
-    if($this->affected_files->count() < self::FILES_THRESHOLD_ONE_WORKER)
+    $max_workers = $this->max_workers;
-      return 1;
+    if($max_workers === null)
-    return $this->globals->max_workers ?? self::MAX_DEFAULT_WORKERS;
+      $max_workers = $this->scanned->count() < 20 ? 1 : 5;
+    return $max_workers;
   }
 
-  function splitFilesByChunks(int $max_workers, bool $sort = true) : array
+  function splitFilesByChunks(int $max_workers) : array
   {
-    $flat = $this->affected_files->getFlatArray();
+    $chunk_size = (int)ceil($this->scanned->count()/$max_workers);
-    if(!$flat)
+    return array_chunk($this->scanned->getFlatArray(), $chunk_size);
-      return array();
-    if($sort)
-      usort($flat, fn($a, $b) => $a[1] <=> $b[1]);
-
-    $chunk_size = (int)ceil(count($flat)/$max_workers);
-    return array_chunk($flat, $chunk_size);
   }
 
   //returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ]
-  function splitJobs(bool $sort = true) : array
+  function splitJobs() : array
   {
     $max_workers = $this->calcMaxWorkers();
     $jobs = array();
-    $chunks = $this->splitFilesByChunks($max_workers, $sort); 
+    foreach($this->splitFilesByChunks($max_workers) as $idx => $chunk)
-    foreach($chunks as $idx => $chunk)
       $jobs[] = array($idx, microtime(true), $chunk);
     return $jobs;
   }
 }
 
-function _config_cache_update(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
+class ConfigFetchResult
 {
-  $jobs = $params->splitJobs(sort: true);
+  /** @var ConfigCacheEntry[] */
+  public array $all = array();
+  /** @var array<int, ConfigCacheEntry> */
+  public array $by_id = array();
+  /** @var array<string, ConfigCacheEntry> */
+  public array $by_path = array();
+  /** @var array<string, ConfigCacheEntry> */
+  public array $by_alias = array();
+  public int $stales = 0;
+  public int $corruptions = 0;
+  public int $fast_jsons = 0;
+
+  function getByAlias(string $strid) : ConfigCacheEntry
+  {
+    if(array_key_exists($strid, $this->by_alias))
+      return $this->by_alias[$strid];
+    throw new Exception("Failed to find config by alias '$strid'!");
+  }
+  
+  function getById(int $id) : ConfigCacheEntry
+  {
+    if(array_key_exists($id, $this->by_id))
+      return $this->by_id[$id];
+    throw new Exception("Failed to find config by id '$id'!");
+  }
+  
+  function getByPath(string $path) : ConfigCacheEntry
+  {
+    if(array_key_exists($path, $this->by_path))
+      return $this->by_path[$path];
+    throw new Exception("Failed to find config by path '$path'!");
+  }
+}
+
+function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
+{
+  $t = microtime(true);
+
+  $result = _config_cache_fetch($params);
+
+  if($params->verbose)
+    echo "[CFG] Fetch from cache: " . round(microtime(true) - $t,2) . " sec.\n";
+
+  return $result;
+}
+
+function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
+{
+  if($params->scanned->isEmpty())
+    return new ConfigFetchResult();
+
+  $params->scanned->apply(function($base_dir, $files) { sort($files); return $files;});
+
+  $jobs = $params->splitJobs();
 
   $serial = sizeof($jobs) == 1;
 
-  $results_by_job = _config_update_worker_run_procs($params, $jobs, $serial);
+  $results_by_job = _config_worker_run_procs($params, $jobs, $serial);
 
   if(!$serial)
   {
-    //NOTE: in case result unserialize error try serial processing
+    //NOTE: in case of any result error try serial processing
     if(array_search(false, $results_by_job, true/*strict*/) !== false)
     {
       if($params->verbose)
-        config_log("Corrupted result, trying serial processing...");
+        echo "[CFG] Result error detected, trying serial processing...\n";
-      $results_by_job = _config_update_worker_run_procs(params: $params, jobs: $jobs, serial: true);
+      $results_by_job = _config_worker_run_procs($params, $jobs, true);
     }
   }
 
-  _config_merge_update_results($result, $results_by_job);
+  $t = microtime(true);
+  $result = _config_merge_fetch_results($params, $results_by_job);
+  if($params->verbose)
+    echo "[CFG] Merge results: " . round(microtime(true) - $t,2) . " sec.\n";
 
   if($params->verbose)
-    config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
+    echo "[CFG] Miss(Fast)/Total: {$result->stales}($result->fast_jsons)/" . sizeof($result->all) . "\n";
+
+  return $result;
 }
 
-function _config_merge_update_results(ConfigUpdateResult $result, array $results_by_job)
+function config_cache_fetch_by_path(ConfigGlobals $globals, string $path, bool $force_stale = false) : ConfigCacheEntry
 {
+  $path = realpath($path);
+  $base_dir = config_map_base_dir($globals->base_dirs, $path);
+
+  $scanned = new ConfigScanResult();
+  $scanned->add($base_dir, $path);
+
+  $ces = config_cache_fetch(new ConfigFetchParams(
+    globals: $globals, 
+    scanned: $scanned,
+    force_stale: $force_stale
+  ));
+
+  if(empty($ces->all))
+    throw new Exception("Config not found at path '$path'");
+  return $ces->all[0];
+}
+
+function _config_merge_fetch_results(ConfigFetchParams $params, array $results_by_job) : ConfigFetchResult
+{
+  $result = new ConfigFetchResult();
+  $total_stales = 0;
   $total_fast_jsons = 0;
   
   foreach($results_by_job as $results)
   {
     foreach($results as $item) 
     {
-      $base_dir = $item['base_dir'];
+      list($base_dir, $file, $cache_file, $is_stale, $parser_type) = $item;
-      $file = $item['file'];
-      $cache_file = $item['cache_file'];
-      $parser_type = $item['parser_type'];
-      $error = $item['error'];
-
-      if($error !== null)
-      {
-        $result->errors[$file] = $error;
-        continue;
-      }
 
       if($parser_type === 1)
         ++$total_fast_jsons;
+
+      $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file));
+
+      //NOTE: handling any cache corruption errors
+      if($cache_entry === null)
+      {
+        $is_stale = true;
+        $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
+        ++$result->corruptions;
+      }
+
+      $includes = $cache_entry->includes;
+
+      if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes))
+      {
+        $is_stale = true;
+        $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
+        //NOTE: let's change the mtime of the file which include other files,
+        //      so that on tne next build it will be 'older' than its includes
+        //      and won't trigger rebuild
+        if($params->touch_files_with_includes)
+          touch($file);
+      }
+
+      if($is_stale)
+        ++$total_stales;
+
+      //we want results to be returned in the same order, so 
+      //we store entries by the file key and later retrieve array values
+      $result->by_path[$file] = $cache_entry;
+      $result->by_id[$cache_entry->id] = $cache_entry;
+      $result->by_alias[$cache_entry->strid] = $cache_entry;
     }
   }
 
+  $result->all = array_values($result->by_path);
+  $result->stales = $total_stales;
   $result->fast_jsons = $total_fast_jsons;
+  return $result;
 }
 
-function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file, 
+function _config_invalidate_cache(ConfigFetchParams $params, string $base_dir, string $file, string $cache_file, ?int &$parser_type = null) : ConfigCacheEntry
-  ?int &$parser_type = null) : ConfigCacheEntry
 {
   list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
   if(!$conf_id)
     throw new Exception("Bad conf id: {$conf_id}");
 
   //TODO: this is a bit ugly but kinda works for now
+  $GLOBALS['CONFIG_GLOBALS'] = $params->globals;
   $GLOBALS['CONFIG_CURRENT_FILE'] = $file; 
   $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id; 
   $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
 
   $pres = config_parse($params->globals->base_dirs, $file);
   if($pres->error !== 0)
-    throw new Exception("Parse error({$pres->error}):\n" . $pres->error_descr);
+    throw new Exception("Error({$pres->error}) while parsing {$file}:\n" . $pres->error_descr);
 
   $parser_type = $pres->parser_type;
   $parsed_arr = $pres->parsed_arr;
   //TODO: these hardcoded keys below probably shouldn't be part of data?
   $parsed_arr['id'] = $conf_id;
   $parsed_arr['strid'] = $conf_strid;
-  list($class, $class_id, $normalized_data) 
+  list($class, $class_id, $normalized_data) = config_apply_class_normalization($parsed_arr);
-    = config_apply_class_normalization($parsed_arr, $params->check_junk);
 
   $cache_payload_file = config_get_cache_payload_path($params->globals, $file);
 
-  $cache_file = config_get_cache_path($params->globals, $file);
-
   $cache_entry = new ConfigCacheEntry();
   $cache_entry->id = $conf_id;
   $cache_entry->strid = $conf_strid;
@@ -482,90 +381,14 @@ function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir,
   $cache_entry->payload_file = $cache_payload_file;
   //TODO: pass flag if file is actually normalized?
   $cache_entry->file = normalize_path($file);
-  $cache_entry->includes = config_get_module_includes(cm: $pres->jsm_module);
+  $cache_entry->includes = config_get_module_includes($pres->jsm_module);
+  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
   //TODO: do we really need these refs?
   $cache_entry->refs = config_content_match_refs($pres->normalized_jzon);
-  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
 
   ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
   ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
-  ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
-  ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
 
   return $cache_entry;
 }
 
-function _config_update_worker_run_procs(ConfigCacheUpdateParams $params, array $jobs, bool $serial) : array
-{
-  if($serial)
-  {
-    $results_by_job = array();
-    foreach($jobs as $job)
-      $results_by_job[] = _config_update_worker_func($params, $job, $serial);
-    return $results_by_job;
-  }
-  else
-  {
-    //initializing worker for master process anyway
-    $params->globals->initWorker(true);
-
-    $workers_args = array();
-    foreach($jobs as $job)
-      $workers_args[] = array($params, $job);
-
-    return run_background_gamectl_workers('config_update_worker', $workers_args);
-  }
-}
-
-function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job, bool $is_master_proc = false) : array
-{
-  $start_time = microtime(true);
-
-  $params->globals->initWorker($is_master_proc);
-
-  list($idx, $start_time, $chunk) = $job;
-  if($params->verbose)
-    config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
-
-  $fast_parser_num = 0;
-  $results = array();
-  foreach($chunk as $file_idx => $chunk_data)
-  {
-    list($base_dir, $file) = $chunk_data;
-
-    try
-    {
-      if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
-        config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
-
-      $cache_file = config_get_cache_path($params->globals, $file);
-
-      $parser_type = null;
-      $entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
-
-      $results[] = [
-        'base_dir' => $base_dir, 
-        'file' => $file, 
-        'cache_file' => $entry->cache_file, 
-        'parser_type' => $parser_type, 
-        'error' => null
-      ];
-    }
-    catch(\Throwable $e)
-    {
-      $results[] = [
-        'base_dir' => $base_dir, 
-        'file' => $file, 
-        'cache_file' => null,
-        'parser_type' => null,
-        'error' => $e->getMessage()
-      ];
-    }
-  }
-
-  if($params->verbose)
-    config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
-
-  return $results;
-}
-

config.inc.php

@@ -11,397 +11,41 @@ require_once(__DIR__ . '/msgpack.inc.php');
 
 class ConfigGlobals
 {
-  public readonly array $base_dirs;
+  public array $base_dirs = array();
-  public readonly ?string $worker_init_fn;
+  public ?string $worker_init_fn = null;
-  public readonly int $workers_num;
+  public string $base_class = '\ConfBase';
-  public readonly string $base_class;
+  public string $build_dir;
-  public readonly string $build_dir;
 
-  function __construct(
+  function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null)
-    array $base_dirs, 
-    string $build_dir, 
-    ?string $worker_init_fn = null,
-    int $workers_num = 1,
-    string $base_class = '\ConfBase'
-  )
   {
     $this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
     $this->build_dir = $build_dir;
     $this->worker_init_fn = $worker_init_fn;
-    $this->workers_num = $workers_num;
-    $this->base_class = $base_class;
   }
 
-  function initWorker(bool $is_master_proc)
+  function initWorker()
   {
-    $GLOBALS['CONFIG_GLOBALS'] = $this;
-
     if($this->worker_init_fn !== null)
     {
       $fn = $this->worker_init_fn;
-      $fn($is_master_proc);
+      $fn();
-    }
-  }
-}
-
-enum ConfigUpdateMode : int
-{
-  case Force         = 1;
-  case DetectChanged = 2;
-  case Selected      = 3;
-}
-
-class ConfigUpdateRequest
-{
-  public ConfigUpdateMode $mode;
-  public ?ConfigDirFiles $files;
-  public ?string $result_file;
-  public ?\taskman\TaskmanFileChanges $file_changes;
-  public bool $verbose = false;
-  public bool $return_entries = false;
-
-  private function __construct() {}
-
-  static function force(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
-  {
-    if($files instanceof \taskman\artefact\TaskmanDirFiles)
-      $files = ConfigDirFiles::makeFromArtefactFiles($files);
-
-    $req = new ConfigUpdateRequest();
-    $req->mode = ConfigUpdateMode::Force;
-    $req->files = $files;
-    return $req;
-  }
-
-  static function detectChanged(string $result_file, \taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
-  {
-    if($files instanceof \taskman\artefact\TaskmanDirFiles)
-      $files = ConfigDirFiles::makeFromArtefactFiles($files);
-
-    $req = new ConfigUpdateRequest();
-    $req->mode = ConfigUpdateMode::DetectChanged;
-    $req->files = $files;
-    $req->result_file = $result_file;
-    return $req;
-  }
-
-  static function selected(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles $files, \taskman\TaskmanFileChanges|null $file_changes = null) : ConfigUpdateRequest
-  {
-    if($files instanceof \taskman\artefact\TaskmanDirFiles)
-      $files = ConfigDirFiles::makeFromArtefactFiles($files);
-
-    $req = new ConfigUpdateRequest();
-    $req->mode = ConfigUpdateMode::Selected;
-    $req->files = $files;
-    $req->file_changes = $file_changes;
-    return $req;
-  }
-}
-
-class ConfigUpdateResult
-{
-  public ConfigDirFiles $affected_files;
-  public array $affected_entries = array();
-
-  /** @var array<string, string> */
-  public array $errors = array();
-  public int $corruptions = 0;
-  public int $fast_jsons = 0;
-
-  function __construct(ConfigCacheUpdateParams $params)
-  {
-    $this->affected_files = $params->affected_files;
-  }
-}
-
-class ConfigManager
-{
-  private ConfigGlobals $globals; 
-  private ConfigCache $cache;
-  private ?ConfigCacheFileMap $file_map;
-
-  function __construct(ConfigGlobals $globals)
-  {
-    $this->globals = $globals;
-
-    $this->cache = new ConfigCache($globals);
-    $this->file_map = null;
-  }
-
-  function getGlobals() : ConfigGlobals
-  {
-    return $this->globals;
-  }
-
-  function getCache() : ConfigCache
-  {
-    return $this->cache;
-  }
-
-  function getArtefactFilesSpec() : array
-  {
-    return [$this->globals->base_dirs, ['.js']];
-  }
-
-  function getFileMap() : ConfigCacheFileMap
-  {
-    if($this->file_map === null)
-    {
-      $this->file_map = $this->_tryLoadMap();
-      if($this->file_map === null)
-      {
-        $this->file_map = self::_makeMap($this->scanFiles(extension: '.js'));
-        $this->_saveFileMap();
-      }
-    }
-    return $this->file_map;
-  }
-
-  function update(ConfigUpdateRequest $req) : ConfigUpdateResult
-  {
-    config_log("Updating configs, mode '{$req->mode->name}'...");
-
-    if($req->files === null)
-      $req->files = $this->scanFiles(extension: '.js', verbose: $req->verbose);
-
-    $added_files = [];
-    $removed_files = [];
-    $this->_checkFileMap($req, $added_files, $removed_files);
-
-    $affected_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
-
-    //NOTE: at this point taking into account only config files
-    $affected_files->filter(fn($file) => str_ends_with($file, '.conf.js'));
-
-    config_log("Affected files: {$affected_files->count()}");
-
-    $update_params = new ConfigCacheUpdateParams(
-      globals: $this->globals, 
-      affected_files: $affected_files, 
-      verbose: $req->verbose
-    );
-    $update_result = self::_updateCache($update_params);
-
-    //let's clear internal cache once the update procedure is done
-    $this->cache->clear();
-
-    $this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
-
-    if($req->return_entries)
-    {
-      foreach($affected_files as $file)
-      {
-        $entry = $this->cache->getOrLoadByPath($file);
-        $update_result->affected_entries[] = $entry;
-      }
-    }
-
-    return $update_result;
-  }
-
-  private function _updateCache(ConfigCacheUpdateParams $params) : ConfigUpdateResult
-  {
-    $result = new ConfigUpdateResult($params);
-
-    if($params->affected_files->isEmpty())
-      return $result;
-
-    $t = microtime(true);
-
-    _config_cache_update($result, $params);
-
-    if($result->errors)
-    {
-      $errors = array();
-      foreach($result->errors as $file => $error)
-      {
-        $errors[] = (count($errors) + 1) . ") Error in file '$file': $error";
-        if(count($errors) > $params->max_errors_num)
-          break;
-      }
-
-      throw new Exception(implode("\n", $errors));
-    }
-
-    if($params->verbose)
-      config_log("Update cache: " . round(microtime(true) - $t,2) . " sec.");
-
-    return $result;
-  }
-
-  private function _checkFileMap(ConfigUpdateRequest $req, array &$added_files, array &$removed_files)
-  {
-    $fs_cache_map = $this->getFileMap();
-
-    if($req->mode === ConfigUpdateMode::Force)
-    {
-      //let's rebuild the file map
-      $fs_cache_map->init($req->files->getAllFiles());
-      config_log("File map init: ".$fs_cache_map->count());
-    }
-    else if($req->mode === ConfigUpdateMode::DetectChanged)
-    {
-      list($added_files, $removed_files) = $fs_cache_map->compare($req->files->getAllFiles());
-      config_log("File map compare, added: ".count($added_files).", removed: ".count($removed_files));
-    }
-    else if($req->mode === ConfigUpdateMode::Selected)
-    {
-      if($req->file_changes != null)
-      {
-        foreach($req->files as $file)
-        {
-          if($req->file_changes->isCreated($file))
-            $added_files[] = $file;
-          else if($req->file_changes->isDeleted($file))
-            $removed_files[] = $file;
-        }
-      }
     }
   }
 
-  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
+  function setNormalizeBaseDirs(array $dirs)
   {
-    $fs_cache_map = $this->getFileMap();
+    $this->base_dirs = array_map(function($d) { return normalize_path($d); }, $dirs);
-
-    //TODO: traverse all affected files and update file map
-    foreach($affected_files as $file)
-    {
-      $cache_entry = $this->cache->getOrLoadByPath($file);
-      $fs_cache_map->updateDepsForEntry($cache_entry);
-    }
-
-    if($req->mode == ConfigUpdateMode::Force || 
-      $affected_files->count() > 0 || 
-      $added_files || 
-      $removed_files)
-    {
-      $fs_cache_map->update($added_files, $removed_files);
-      $this->_saveFileMap();
-    }
   }
-
-  private function _getAffectedFiles(ConfigUpdateRequest $req, array $added_files, array $removed_files) : ConfigDirFiles
-  {
-    $fs_cache_map = $this->getFileMap();
-
-    $affected_files = null;
-
-    if($req->mode === ConfigUpdateMode::Force)
-    {
-      $affected_files = $req->files;
-    }
-    else if($req->mode === ConfigUpdateMode::DetectChanged)
-    {
-      $affected_files = ConfigDirFiles::makeFor($this);
-
-      foreach($req->files->getMap() as $base_dir => $files)
-      {
-        foreach($files as $file)
-        {
-          if(need_to_regen($req->result_file, [$file]))
-          {
-            $affected_files->add($base_dir, $file);
-
-            $affected_by_file = $fs_cache_map->getAffectedFiles($file);
-            foreach($affected_by_file as $dep)
-              $affected_files->addFile($dep, unique: true);
-          }
-        }
-      }
-
-      //if there were removed files we need to rebuild affected files
-      foreach($removed_files as $file)
-      {
-        if(!str_ends_with($file, '.conf.js'))
-        {
-          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
-          foreach($affected_by_file as $dep)
-            $affected_files->addFile($dep, unique: true);
-        }
-      }
-    }
-    else if($req->mode === ConfigUpdateMode::Selected)
-    {
-      $affected_files = ConfigDirFiles::makeFor($this);
-
-      foreach($req->files as $file)
-      {
-        $affected_files->addFile($file, unique: true);
-
-        if(!str_ends_with($file, '.conf.js'))
-        {
-          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
-          foreach($affected_by_file as $dep)
-            $affected_files->addFile($dep, unique: true);
-        }
-      }
-    }
-
-    return $affected_files;
-  }
-
-  function parseFile(string $file) : ConfigParseResult
-  {
-    return config_parse($this->globals->base_dirs, $file);
-  }
-
-  function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
-  {
-    return config_scan_files($this->globals->base_dirs, $extension, $verbose);
-  }
-
-  private function _getMapPath() : string
-  {
-    return $this->globals->build_dir . '/file_map.data';
-  }
-
-  private function _tryLoadMap() : ?ConfigCacheFileMap
-  {
-    if(!is_file($this->_getMapPath()))
-      return null;
-    config_log("Loading file map");
-    return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
-  }
-
-  private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
-  {
-    config_log("Creating file map");
-    $map = new ConfigCacheFileMap();
-    $map->init($files->getAllFiles());
-    return $map;
-  }
-
-  private function _saveFileMap()
-  {
-    $data = ConfigCacheFileMap::serialize($this->getFileMap()); 
-    config_log("Saving file map: " . kb($data));
-    ensure_write($this->_getMapPath(), $data);
-  }
-}
-
-function config_log($msg)
-{
-  echo "[CFG] $msg\n";
 }
 
 function config_get_cache_path(ConfigGlobals $globals, string $file) : string
 {
-  return config_get_tmp_build_path($globals, $file . '.cache');
+  return config_get_tmp_build_path($globals, $file . '.cacheb');
-} 
-
-function config_get_cache_id_path(ConfigGlobals $globals, int $id) : string
-{
-  return config_get_tmp_build_path($globals, $id . '.id');
-} 
-
-function config_get_cache_strid_path(ConfigGlobals $globals, string $strid) : string
-{
-  return config_get_tmp_build_path($globals, $strid  . '.strid');
 } 
 
 function config_get_cache_payload_path(ConfigGlobals $globals, string $file) : string
 {
-  return config_get_tmp_build_path($globals, $file . '.payload');
+  return config_get_tmp_build_path($globals, $file . '.pdata');
 } 
 
 function config_load(ConfigGlobals $globals, string $conf_path, ?string $conf_dir = null) : object
@@ -463,19 +107,9 @@ function config_bench_load(ConfigGlobals $globals, string $file)
   $parse_res = config_parse($globals->base_dirs, $file);
   if($parse_res->error !== 0)
     throw new Exception("Error({$parse_res->error}) while loading JSON from {$file}:\n" . $parse_res->error_descr);
-  config_log("Parse: " . (microtime(true) - $t));
+  echo "[CFG] Parse: " . (microtime(true) - $t) . "\n";
   $t = microtime(true);
   $config = config_load_from_kv_array($globals, $base_dir, $file, $parse_res->parsed_arr, $conf_id); 
-  config_log("Load: " . (microtime(true) - $t));
+  echo "[CFG] Load: " . (microtime(true) - $t) . "\n";
-}
-
-function config_changes_has_deleted_or_added(TaskmanFileChanges $file_changes, iterable $files) : bool
-{
-  foreach($files as $file)
-  {
-    if(str_ends_with($file, '.conf.js') && ($file_changes->isDeleted($file) || $file_changes->isCreated($file)))
-      return true;
-  }
-  return false;
 }
 

pack.inc.php

@@ -5,26 +5,15 @@ use Exception;
 class ConfigPackParams
 {
   /*var ConfigCacheEntry[]*/
-  public iterable $cache_entries;
+  public array $cache_entries;
   public bool $use_lz4 = false;
   public bool $use_config_refs = false;
   public int $binary_format = 1; //1,2 supported
   public ?int $version = null;
   public bool $debug = false;
 
-  public const EXTRA_FMT3_CHUNK_SIZE = "EXTRA_FMT3_CHUNK_SIZE";
+  function __construct(array $cache_entries, int $version, bool $use_lz4 = false, 
-  public const EXTRA_FMT3_COMPRESSION_LEVEL = "EXTRA_FMT3_COMPRESSION_LEVEL";
+    bool $use_config_refs = false, int $binary_format = 1, bool $debug = false)
-  public array $extras = array();
-
-  function __construct(
-    iterable $cache_entries, 
-    int $version, 
-    bool $use_lz4 = false, 
-    bool $use_config_refs = false, 
-    int $binary_format = 1, 
-    bool $debug = false,
-    array $extras = array()
-  )
   {
     $this->cache_entries = $cache_entries;
     $this->use_lz4 = $use_lz4;
@@ -32,7 +21,6 @@ class ConfigPackParams
     $this->binary_format = $binary_format;
     $this->version = $version;
     $this->debug = $debug;
-    $this->extras = $extras;
   }
 }
 
@@ -57,65 +45,24 @@ function config_pack_bundle(ConfigPackParams $params) : string
       $params->cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
-      $params->version,
+      $params->version
-    );
-  }
-  else if($params->binary_format == 3)
-  {
-    $packed_data = _config_pack_bundle_fmt3(
-      $params->cache_entries, 
-      $params->use_lz4,
-      $params->use_config_refs,
-      $params->version,
-      $params->extras[ConfigPackParams::EXTRA_FMT3_CHUNK_SIZE],
-      $params->extras[ConfigPackParams::EXTRA_FMT3_COMPRESSION_LEVEL],
     );
   }
   else
     throw new Exception("Unknown binary format: {$params->binary_format}");
 
   if($params->debug)
-    config_log("Packed entries: " . sizeof($params->cache_entries) . ", total: " . 
+    echo "[CFG] Packed entries: " . sizeof($params->cache_entries) . ", total: " . 
       kb($packed_data) . ", format: {$params->binary_format}, lz4: " . 
       var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
       ", CRC: " . crc32($packed_data) . 
-      ", " . round(microtime(true) - $t,2) . " sec.");
+      ", " . round(microtime(true) - $t,2) . " sec.\n";
 
   return $packed_data;
 }
 
-function config_patch_bundle(ConfigPackParams $params, string $packed_data) : string
-{
-  $t = microtime(true);
-
-  $patched_data = null;
-
-  if($params->binary_format == 2)
-  {
-    $patched_data = _config_patch_bundle_fmt2(
-      $packed_data,
-      $params->cache_entries, 
-      $params->use_lz4,
-      $params->use_config_refs,
-      $params->version,
-    );
-  }
-  else
-    throw new Exception("Unknown binary format: {$params->binary_format}");
-
-  if($params->debug)
-    config_log("Patched entries: " . sizeof($params->cache_entries) . ", total: " . 
-      kb($patched_data) . ", format: {$params->binary_format}, lz4: " . 
-      var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
-      ", CRC: " . crc32($patched_data) . 
-      ", " . round(microtime(true) - $t,2) . " sec.");
-
-  return $patched_data;
-}
-
-//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
 function _config_pack_bundle_fmt1(
-  iterable $cache_entries, 
+  array $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
   int $version) : string
@@ -170,14 +117,11 @@ function _config_pack_bundle_fmt1(
   return $packed_data;
 }
 
-//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
-//      (each path item separated by '/' is stored as an array item)
 function _config_pack_bundle_fmt2(
-  iterable $cache_entries, 
+  array $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
-  int $version,
+  int $version) : string
-) : string
 {
   $MAP = array();
   $STRIDMAP = array();
@@ -193,8 +137,20 @@ function _config_pack_bundle_fmt2(
     $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
     $payloads_offset += $payload_size;
 
-    $strids_indices = _config_encode_strid_as_indices($entry->strid, $STRIDMAP, $STRIDLIST);
+    $strids_indices = array();
-
+    $strid_parts = explode('/', ltrim($entry->strid, '@'));
+    foreach($strid_parts as $strid_part)
+    {
+      if(!isset($STRIDMAP[$strid_part]))
+      {
+        $strid_index = count($STRIDLIST);
+        $STRIDLIST[] = $strid_part; 
+        $STRIDMAP[$strid_part] = $strid_index;
+        $strids_indices[] = $strid_index;
+      }
+      else
+        $strids_indices[] = $STRIDMAP[$strid_part];
+    }
     $strids[] = $strids_indices;
   }
 
@@ -234,184 +190,6 @@ function _config_pack_bundle_fmt2(
   return $packed_data;
 }
 
-//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
-//      This reduces overall bundle size when there are many small configs entries.
-function _config_pack_bundle_fmt3(
-  array $cache_entries, 
-  bool $use_lz4, 
-  bool $use_config_refs, 
-  int $version,
-  int $chunk_size,
-  int $compression_level) : string
-{
-  if(!$use_lz4)
-    throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
-
-  if($compression_level < 0 || $compression_level > 12)
-    throw new Exception("LZ4 compression level must be in range [0, 12]");
-
-  $MAP = array();
-  $STRIDMAP = array();
-
-  $header = array();
-  $payloads_offset = 0;
-  $max_chunk_size = 0;
-  $chunk_offset = 0;
-  $payloads_bundle = '';
-  $payloads_buffer = '';
-  $count_entries = count($cache_entries);
-  foreach($cache_entries as $idx => $entry)
-  {
-    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
-    $payload_size = strlen($payload);
-    $payloads_buffer .= $payload;
-
-    if(isset($MAP[$entry->id]))
-      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
-    $MAP[$entry->id] = $entry->strid; 
-
-    $strid_crc = crc32($entry->strid); 
-    if(isset($STRIDMAP[$strid_crc]))
-      throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
-    $STRIDMAP[$strid_crc] = $entry->strid; 
-
-    $header[] = array(
-      $format,
-      $entry->id, 
-      crc32($entry->strid), 
-      $entry->class_id, 
-      $chunk_offset,
-      $payloads_offset,
-      $payload_size
-    );
-
-    $payloads_offset += $payload_size;
-    
-    if($payloads_offset >= $chunk_size || $idx == ($count_entries - 1))  
-    {
-      if($payloads_offset > $max_chunk_size)
-        $max_chunk_size = $payloads_offset;
-      $payloads_offset = 0;
-      $lz4_data = lz4_compress($payloads_buffer, $compression_level); 
-      $payloads_bundle .= pack("V", strlen($lz4_data));
-      $payloads_bundle .= $lz4_data;
-      $chunk_offset = strlen($payloads_bundle);
-      $payloads_buffer = '';
-    }
-  }
-
-  $header_msgpack = config_msgpack_pack($header);
-
-  $packed_data =  
-         pack("C", 3) . 
-         pack("V", $version) . 
-         pack("V", strlen($header_msgpack)) .
-         pack("V", $max_chunk_size) .
-         $header_msgpack .
-         $payloads_bundle;
-
-  return $packed_data;
-}
-
-function _config_encode_strid_as_indices(string $strid, array &$STRIDMAP, array &$STRIDLIST) : array
-{
-  $strids_indices = array();
-  $strid_parts = explode('/', ltrim($strid, '@'));
-  foreach($strid_parts as $strid_part)
-  {
-    if(!isset($STRIDMAP[$strid_part]))
-    {
-      $strid_index = count($STRIDLIST);
-      $STRIDLIST[] = $strid_part; 
-      $STRIDMAP[$strid_part] = $strid_index;
-      $strids_indices[] = $strid_index;
-    }
-    else
-      $strids_indices[] = $STRIDMAP[$strid_part];
-  }
-  return $strids_indices;
-}
-
-function _config_patch_bundle_fmt2(
-  string $packed_data,
-  array $patch_entries, 
-  bool $use_lz4, 
-  bool $use_config_refs, 
-  int $version,
-) : string
-{
-  list($strids, $header, $_, $payloads_bundle) = 
-    _config_unpack_bundle_fmt2(packed_data: $packed_data, unpack_entries: false);
-
-  $stridmap = array_flip($strids);
-
-  foreach($patch_entries as $idx => $patch_entry)
-  {
-    list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
-
-    $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
-
-    if($header_found)
-    {
-      $header_idx = key($header_found);
-      $header_entry = current($header_found);
-
-      $current_offset = $header_entry[4];
-      $current_size = $header_entry[5];
-      if($current_size >= strlen($patch_payload))
-      {                                         
-        //let's do the inline patching
-        $payloads_bundle = substr_replace($payloads_bundle, $patch_payload, $current_offset, strlen($patch_payload));
-
-        $header_entry[0] = $patch_format;
-        $header_entry[5] = strlen($patch_payload);
-        $header[$header_idx] = $header_entry; 
-      }
-      else
-      {
-        //let's add it to the end
-        $header_entry[0] = $patch_format;
-        $header_entry[4] = strlen($payloads_bundle);
-        $header_entry[5] = strlen($patch_payload);
-        $header[$header_idx] = $header_entry; 
-
-        $payloads_bundle .= $patch_payload;
-      }
-    }
-    else
-    {
-      //let's add new entry it to the end
-      $strid_indices = _config_encode_strid_as_indices($patch_entry->strid, $stridmap, $strids);
-
-      $header_entry = array(
-        $patch_format,
-        $patch_entry->id,
-        $strid_indices,
-        $patch_entry->class_id,
-        strlen($payloads_bundle),
-        strlen($patch_payload),
-      );
-
-      $header[] = $header_entry;
-      $payloads_bundle .= $patch_payload;
-    }
-  }
-
-  $strids_msgpack = config_msgpack_pack($strids);
-  $header_msgpack = config_msgpack_pack($header);
-
-  $patched_data =  
-         pack("C", 2) . 
-         pack("V", $version) . 
-         pack("V", strlen($strids_msgpack)) .
-         pack("V", strlen($header_msgpack)) .
-         $strids_msgpack .
-         $header_msgpack .
-         $payloads_bundle;
-
-  return $patched_data;
-}
-
 //format: [[class_id, [data]], ...[class_id, [data]]]
 function config_unpack_bundle(string $packed_data) : array
 {
@@ -424,12 +202,7 @@ function config_unpack_bundle(string $packed_data) : array
   }
   else if($info['format'] === 2)
   {
-    list($_, $_, $entries) = _config_unpack_bundle_fmt2($packed_data);
+    return _config_unpack_bundle_fmt2($packed_data);
-    return $entries;
-  }
-  else if($info['format'] === 3)
-  {
-    return _config_unpack_bundle_fmt3($packed_data);
   }
   else
     throw new Exception("Unknown format: {$info['format']}");
@@ -462,7 +235,7 @@ function _config_unpack_bundle_fmt1(string $packed_data) : array
   return $entries;
 }
 
-function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries = true) : array
+function _config_unpack_bundle_fmt2(string $packed_data) : array
 {
   $packed_info = substr($packed_data, 0, 1+4+4+4);
 
@@ -480,80 +253,16 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries =
   $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
   
   $entries = array();
-
+  foreach($header as $item)
-  if($unpack_entries)
   {
-    foreach($header as $item)
+    list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
-    {
-      list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
 
-      $payload = substr($payloads_bundle, $offset, $size);
+    $payload = substr($payloads_bundle, $offset, $size);
 
-      $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+    $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
-    }
   }
 
-  return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
+  return $entries;
-}
-
-function _config_unpack_bundle_fmt3(string $packed_data): array
-{
-  if(ord($packed_data[0]) !== 3)
-  {
-    throw new Exception("Invalid config bundle format");
-  }
-
-  $offset = 1;
-  $version = unpack("V", substr($packed_data, $offset, 4))[1];
-  $offset += 4;
-  $header_len = unpack("V", substr($packed_data, $offset, 4))[1];
-  $offset += 4;
-  $max_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
-  $offset += 4;
-
-  $header_msgpack = substr($packed_data, $offset, $header_len);
-  $offset += $header_len;
-
-  $header = config_msgpack_unpack($header_msgpack);
-
-  $cache_entries = [];
-  $chunk_offset = 0;
-  $chunk_buffer = '';
-  $chunk_id = -1;
-
-  foreach ($header as $entry_data)
-  {
-    list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
-
-    if($entry_chunk_offset !== $chunk_id)
-    {
-      if($chunk_offset !== -1)
-      {
-        $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
-        $offset+=4;
-        $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
-        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
-        $offset += $lz4_chunk_size;
-        $chunk_offset = $offset;
-      }
-      else
-      {
-        $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
-        $chunk_offset += 4;
-        $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
-        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
-        $chunk_offset += $lz4_chunk_size;
-      }
-
-      $chunk_id = $entry_chunk_offset;
-    }
-
-    $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
-
-    $cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
-  }
-
-  return $cache_entries;
 }
 
 //format: [format_id, payload_data]
@@ -598,10 +307,9 @@ function config_pack_and_write_bundle(
   /*var ConfBase[]*/
   array $configs, 
   string $file_path, 
-  int $version,
   bool $use_lz4 = true, 
   int $binary_format = 1, 
-  bool $debug = false
+  ?int $version = null
 )
 {
   $cache_entries = array();
@@ -626,8 +334,7 @@ function config_pack_and_write_bundle(
       cache_entries: $cache_entries, 
       use_lz4: $use_lz4, 
       binary_format: $binary_format, 
-      version: $version,
+      version: $version
-      debug: $debug
     )
   );
 

parse.inc.php

@@ -9,7 +9,7 @@ class ConfigParseResult
   public string $normalized_jzon = '';
   public array $parsed_arr;
   public int $parser_type;
-  public \JSM_Module $jsm_module;
+  public $jsm_module;
 }
 
 function config_parse(array $base_dirs, string $file) : ConfigParseResult
@@ -27,7 +27,7 @@ function config_parse(array $base_dirs, string $file) : ConfigParseResult
   catch(Exception $e)
   {
     $res->error = 1;
-    $res->error_descr = $e->getMessage() . "\n" . $e->getTraceAsString(); 
+    $res->error_descr = "File '$file':\n" . $e->getMessage() . "\n" . $e->getTraceAsString(); 
     return $res;
   }
 
@@ -41,7 +41,7 @@ function config_parse(array $base_dirs, string $file) : ConfigParseResult
   else
   {
     $res->error = $decode_res[0];
-    $res->error_descr = $decode_res[1];
+    $res->error_descr = "File '$file':\n" . $decode_res[1];
   }
 
   return $res;
@@ -51,7 +51,6 @@ function config_check_and_decode_jzon(string $json) : array
 {
   try
   {
-    $parser = 0;
     $arr = \jzon_parse($json, $parser);
     return array(0, "", $arr, $parser);
   }
@@ -143,13 +142,11 @@ function config_ensure_header(string $conf_dir, string $file, bool $force = fals
   }
 }
 
-function config_apply_class_normalization(array $kv_data, bool $check_junk = false) : array
+function config_apply_class_normalization(array $kv_data) : array
 {
   $class = $kv_data['class'];
   $norm_data = array();
-  unset($kv_data['class']);
+  call_user_func_array([$class, 'normalize'], array(&$kv_data, &$norm_data));
-  call_user_func_array([$class, 'normalize'], 
-    array(&$kv_data, &$norm_data, true, $check_junk));
   return array($class, $class::CLASS_ID, $norm_data);
 }
 

scan.inc.php

@@ -2,40 +2,10 @@
 namespace taskman;
 use Exception;
 
-class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator
+class ConfigScanResult
 {
-  private array $base_dirs;
-
   /*var array<string, string[]>*/
-  private array $base_dir2files = array();
+  public array $base_dir2files = array();
-
-  private $iter_pos = 0;
-
-  function __construct(array $base_dir2files = array(), ?array $base_dirs = null)
-  {
-    foreach($base_dir2files as $dir => $files)
-      $this->base_dir2files[$dir] = $files;
-
-    if($base_dirs === null)
-      $this->base_dirs = array_keys($base_dir2files);
-    else
-      $this->base_dirs = $base_dirs;
-  }
-
-  static function makeFor(ConfigManager $mgr) : ConfigDirFiles
-  {
-    return new ConfigDirFiles([], $mgr->getGlobals()->base_dirs);
-  }
-
-  static function makeFromArtefactFiles(\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles
-  {
-    return new ConfigDirFiles($files->toMap());
-  }
-
-  function clear()
-  {
-    $this->base_dir2files = array();
-  }
 
   function isEmpty() : bool
   {
@@ -59,11 +29,7 @@ class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator
   function filter(callable $filter)
   {
     foreach($this->base_dir2files as $base_dir => $files)
-    {
+      $this->base_dir2files[$base_dir] = array_filter($files, $filter);
-      $filtered = array_filter($files, $filter);
-      //NOTE: don't want any index gaps
-      $this->base_dir2files[$base_dir] = array_values($filtered);
-    }
   }
 
   function forEachFile(callable $fn)
@@ -75,23 +41,12 @@ class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator
     }
   }
 
-  function addFile(string $file, bool $unique = false)
+  function add(string $base_dir, string $file)
-  {
-    $this->add(config_map_base_dir($this->base_dirs, $file, normalized: true), $file, $unique);
-  }
-
-  function add(string $base_dir, string $file, bool $unique = false)
   {
     if(!isset($this->base_dir2files[$base_dir]))
       $this->base_dir2files[$base_dir] = array();
 
-    if(!$unique || !in_array($file, $this->base_dir2files[$base_dir]))
+    $this->base_dir2files[$base_dir][] = $file;
-      $this->base_dir2files[$base_dir][] = $file;
-  }
-
-  function getMap() : array
-  {
-    return $this->base_dir2files;
   }
 
   //returns [[base_dir, file1], [base_dir, file2], ...]
@@ -114,119 +69,26 @@ class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator
 
     return $all_files;
   }
-   
-  //ArrayAccess interface
-  function offsetExists(mixed $offset) : bool
-  {
-    if(!is_int($offset))
-      throw new Exception("Invalid offset");
-
-    return $this->count() > $offset;
-  }
-
-  function offsetGet(mixed $offset) : mixed
-  {
-    if(!is_int($offset))
-      throw new Exception("Invalid offset");
-
-    foreach($this->base_dir2files as $base_dir => $files)
-    {
-      $n = count($files);
-      if($offset - $n < 0)
-        return $files[$offset];
-      $offset -= $n;
-    }
-    return null;
-  }
-
-  function offsetSet(mixed $offset, mixed $value) : void
-  {
-    if(!is_int($offset))
-      throw new Exception("Invalid offset");
-
-    foreach($this->base_dir2files as $base_dir => &$files)
-    {
-      $n = count($files);
-      if($offset - $n < 0)
-      {
-        $files[$offset] = $value;
-        return;
-      }
-      $offset -= $n;
-    }
-  }
-
-  function offsetUnset(mixed $offset) : void
-  {
-    if(!is_int($offset))
-      throw new Exception("Invalid offset");
-
-    foreach($this->base_dir2files as $base_dir => $files)
-    {
-      $n = count($files);
-      if($offset - $n < 0)
-      {
-        unset($files[$offset]);
-        return;
-      }
-      $offset -= $n;
-    }
-  }
-
-  //Iterator interface
-  function rewind() : void
-  {
-    $this->iter_pos = 0;
-  }
-
-  function current() : mixed 
-  {
-    return $this->offsetGet($this->iter_pos);
-  }
-
-  function key() : mixed 
-  {
-    return $this->iter_pos;
-  }
-
-  function next() : void
-  {
-    ++$this->iter_pos;
-  }
-
-  function valid() : bool 
-  {
-    return $this->offsetExists($this->iter_pos);
-  }
 }
 
 function config_scan_files(
-  iterable $base_dirs, 
+  array $base_dirs, 
   string $ext_filter = '.conf.js', 
   bool $verbose = false
-  ) : ConfigDirFiles
+  ) : ConfigScanResult
 {
   $t = microtime(true);
 
-  $base_dir2files = [];
+  $result = new ConfigScanResult();
+
   foreach($base_dirs as $base_dir)
   {
-    $base_dir2files[$base_dir] = 
+    $result->base_dir2files[$base_dir] = 
-      scan_files_rec([$base_dir], [$ext_filter]);
+      scan_files_rec(array($base_dir), array($ext_filter));
   }
 
-  $result = new ConfigDirFiles($base_dir2files);
-
   if($verbose)
-    config_log("File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec.");
+    echo "[CFG] File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec.\n";
 
   return $result;
 }
-
-//NOTE: this one is obsolete
-function config_hash_changed(ConfigGlobals $globals, iterable $all_files)
-{
-  $all_crc_file = $globals->build_dir . "/configs.crc";
-  return names_hash_changed($all_crc_file, $all_files);
-}
-

task.inc.php

@@ -1,9 +1,8 @@
 <?php
 namespace taskman;
 use Exception;
-use Throwable;
 
-task('config_update_worker', function(array $args)
+task('config_worker', function(array $args)
 {
   if(sizeof($args) != 3)
     throw new Exception("Config worker args not set");
@@ -15,10 +14,10 @@ task('config_update_worker', function(array $args)
   try
   {
     list($params, $job) = unserialize(ensure_read($in_file));
-    $result = _config_update_worker_func($params, $job);
+    $result = _config_worker_func($params, $job);
     ensure_write($out_file, serialize($result));
   }
-  catch(Throwable $e)
+  catch(Exception $e)
   {
     //NOTE: explicitely catching all exceptions and writing to the error file 
     //      since under Windows error file stream redirect may work unreliably 
@@ -26,3 +25,70 @@ task('config_update_worker', function(array $args)
     throw $e;
   }
 });
+
+function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $serial) : array
+{
+  if($serial)
+  {
+    $results_by_job = array();
+    foreach($jobs as $job)
+      $results_by_job[] = _config_worker_func($params, $job);
+    return $results_by_job;
+  }
+  else
+  {
+    $worker_args = array();
+    foreach($jobs as $idx => $job)
+      $worker_args[] = array($params, $job);
+
+    return run_background_gamectl_workers('config_worker', $worker_args);
+  }
+}
+
+//returns [[base_dir, file, cache_file, was_stale], ...]
+function _config_worker_func(ConfigFetchParams $params, array $job) : array
+{
+  $start_time = microtime(true);
+
+  $params->globals->initWorker();
+
+  list($idx, $start_time, $chunk) = $job;
+  if($params->verbose)
+    echo "[CFG] Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)\n";
+
+  $fast_parser_num = 0;
+  $results = array();
+  foreach($chunk as $file_idx => $chunk_data)
+  {
+    try
+    {
+      list($base_dir, $file) = $chunk_data;
+
+      if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
+        echo "[CFG] Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...\n";
+
+      $cache_file = config_get_cache_path($params->globals, $file);
+
+      $parser_type = null;
+
+      $is_stale = true;
+      if(!$params->force_stale)
+        $is_stale = need_to_regen($cache_file, array($file));
+
+      if($is_stale)
+        _config_invalidate_cache($params, $base_dir, $file, $cache_file, $parser_type);
+
+      $results[] = array($base_dir, $file, $cache_file, $is_stale, $parser_type);
+    }
+    catch(Exception $e)
+    {
+      throw new Exception("Error in file '$file': " . $e->getMessage());
+    }
+  }
+
+  if($params->verbose)
+    echo "[CFG] Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)\n";
+
+  return $results;
+}
+

util.inc.php

@@ -4,7 +4,7 @@ use Exception;
 
 function config_get_tmp_build_path(ConfigGlobals $globals, string $file) : string
 {
-  $name = str_replace(['@', ':', "\\", "/"], ['', "-", "-", "-"], normalize_path($file));
+  $name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file))));
   $name = ltrim($name, "-");
   return normalize_path($globals->build_dir . "/$name");
 }
@@ -70,7 +70,6 @@ function config_crc28(string $what) : int
 function config_get_module_includes(\JSM_Module $cm) : array
 {
   $includes = array();
-  //NOTE: module contains all includes (even those included from other modules)
   foreach($cm->getIncludes() as $include => $_)
   {
     //maybe we should take .php includes into account as well?
@@ -123,13 +122,15 @@ function config_includes_map_find_text_origin(array $map, string $file, string $
   return $res;
 }
 
-function config2json(object $conf, int $json_flags = 0) : string
+//TODO:
-{
+//function conf2json(object $conf, int $json_flags = 0) : string
-  $arr = $conf->export(true); 
+//{
-  $arr['class'] = get_class($conf);
+//  $arr = $conf->export(true); 
-  unset($arr['id']); // @phpstan-ignore-line
+//  $arr['class'] = get_class($conf);
-  unset($arr['strid']); // @phpstan-ignore-line
+//  unset($arr['id']);
-  $json = json_encode($arr, $json_flags);
+//  unset($arr['strid']);
-  $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json);
+//  $json = json_encode($arr, $json_flags);
-  return $json;
+//  $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json);
-}
+//  return $json;
+//}
+