Initial commit

cache.inc.php

@@ -2,6 +2,101 @@
 namespace taskman;
 use Exception;
 
+class ConfigCache
+{
+  private ConfigGlobals $globals;
+
+  /** @var array<string, ConfigCacheEntry> */
+  private array $by_path = [];
+  /** @var array<int, ConfigCacheEntry> */
+  private array $by_id = [];
+  /** @var array<string, ConfigCacheEntry> */
+  private array $by_alias = [];
+
+  function __construct(ConfigGlobals $globals)
+  {
+    $this->globals = $globals;
+  }
+
+  function clear()
+  {
+    $this->by_path = [];
+    $this->by_id = [];
+    $this->by_alias = [];
+  }
+
+  function tryGetOrLoadByPath(string $path) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($path, $this->by_path))
+    {
+      $cache_file = config_get_cache_path($this->globals, $path);
+      if(!is_file($cache_file))
+        return null;
+      $ce = ConfigCacheEntry::unserialize(ensure_read($cache_file));
+      $this->by_path[$path] = $ce;
+      return $ce;
+    }
+
+    return $this->by_path[$path];
+  }
+
+  function getOrLoadByPath(string $path) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadByPath($path);
+    if($ce === null)
+      throw new Exception("Failed to find entry by path '$path'");
+    return $ce;
+  }
+
+  function tryGetOrLoadById(int $id) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($id, $this->by_id))
+    {
+      $cache_id_file = config_get_cache_id_path($this->globals, $id);
+      if(!is_file($cache_id_file))
+        return null;
+      $file = ensure_read($cache_id_file);
+      $ce = $this->tryGetOrLoadByPath($file);
+      $this->by_id[$id] = $ce;
+      return $ce;
+    }
+
+    return $this->by_id[$id];
+  }
+
+  function getOrLoadById(int $id) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadById($id);
+    if($ce === null)
+      throw new Exception("Failed to find entry by id '$id'");
+    return $ce;
+  }
+
+  function tryGetOrLoadByStrid(string $strid) : ?ConfigCacheEntry
+  {
+    if(!array_key_exists($strid, $this->by_alias))
+    {
+      $cache_strid_file = config_get_cache_strid_path($this->globals, $strid);
+      if(!is_file($cache_strid_file))
+        return null;
+      $file = ensure_read($cache_strid_file);
+      $ce = $this->tryGetOrLoadByPath($file);
+      $this->by_alias[$strid] = $ce;
+      return $ce;
+    }
+
+    return $this->by_alias[$strid];
+  }
+
+  function getOrLoadByStrid(string $strid) : ConfigCacheEntry
+  {
+    $ce = $this->tryGetOrLoadByStrid($strid);
+    if($ce === null)
+      throw new Exception("Failed to find entry by strid '$strid'");
+    return $ce;
+  }
+}
+
 /**
  * @property object $config
  * @property string $payload
@@ -146,22 +241,100 @@ class ConfigCacheEntryExtras extends \stdClass
   }
 }
 
-class ConfigFetchParams
+class ConfigCacheFileMap
+{
+  private array $file2deps = array();
+
+  static function serialize(ConfigCacheFileMap $map) : string
+  {
+    $d = $map->export();
+    return serialize($d);
+  }
+
+  static function unserialize(string $str) : ?ConfigCacheFileMap
+  {
+    $d = @unserialize($str);
+    if(!is_array($d))
+      return null;
+    $map = new ConfigCacheFileMap();
+    $map->import($d);
+    return $map;
+  }
+
+  function getAffectedFiles(string $file) : array
+  {
+    if(!isset($this->file2deps[$file]))
+      return [];
+    return array_keys($this->file2deps[$file]);
+  }
+
+  function update(array $files)
+  {
+    $this_files = array_keys($this->file2deps);
+
+    $added = array_diff($files, $this_files);
+    $removed = array_diff($this_files, $files);
+
+    foreach($added as $file)
+      $this->file2deps[$file] = [];
+    foreach($removed as $file)
+      unset($this->file2deps[$file]);
+
+    return [$added, $removed];
+  }
+
+  function updateDepsForEntry(ConfigCacheEntry $entry)
+  {
+    foreach($entry->includes as $include)
+    {
+      if(isset($this->file2deps[$include]))
+        $this->file2deps[$include][$entry->file] = true;
+      else
+        $this->file2deps[$include] = [$entry->file => true];
+    }
+  }
+
+  function export() : array
+  {
+    $d = array();
+    $d[] = $this->file2deps;
+    return $d;
+  }
+
+  function import(array $d)
+  {
+    list($this->file2deps) = $d;
+  }
+}
+
+class ConfigCacheUpdateResult
+{
+  public ConfigDirFiles $affected_files;
+
+  /** @var array<string, string> */
+  public array $errors = array();
+  public int $corruptions = 0;
+  public int $fast_jsons = 0;
+
+  function __construct(ConfigCacheUpdateParams $params)
+  {
+    $this->affected_files = $params->affected_files;
+  }
+}
+
+class ConfigCacheUpdateParams
 {
   public ConfigGlobals $globals;
-  public ConfigScanResult $scanned;
+  public ConfigDirFiles $affected_files;
-  public bool $force_stale = false;
   public bool $verbose = false;
   public ?int $max_workers = null;
-  public bool $touch_files_with_includes = true;
   public bool $check_junk = true;
 
-  function __construct(ConfigGlobals $globals, ConfigScanResult $scanned, 
+  function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, 
-    bool $force_stale = false, bool $verbose = false, ?int $max_workers = null)
+    bool $verbose = false, ?int $max_workers = null)
   {
     $this->globals = $globals;
-    $this->scanned = $scanned;
+    $this->affected_files = $affected_files;
-    $this->force_stale = $force_stale;
     $this->verbose = $verbose;
     $this->max_workers = $max_workers;
   }
@@ -170,13 +343,13 @@ class ConfigFetchParams
   {
     $max_workers = $this->max_workers;
     if($max_workers === null)
-      $max_workers = $this->scanned->count() < 100 ? 1 : 5;
+      $max_workers = $this->affected_files->count() < 100 ? 1 : 5;
     return $max_workers;
   }
 
   function splitFilesByChunks(int $max_workers, bool $sort = true) : array
   {
-    $flat = $this->scanned->getFlatArray();
+    $flat = $this->affected_files->getFlatArray();
     if($sort)
       usort($flat, fn($a, $b) => $a[1] <=> $b[1]);
 
@@ -196,49 +369,41 @@ class ConfigFetchParams
   }
 }
 
-class ConfigFetchResult
+function _config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
 {
-  /** @var ConfigCacheEntry[] */
+  $jobs = $params->splitJobs(sort: true);
-  public array $all = array();
-  /** @var array<int, ConfigCacheEntry> */
-  public array $by_id = array();
-  /** @var array<string, ConfigCacheEntry> */
-  public array $by_path = array();
-  /** @var array<string, ConfigCacheEntry> */
-  public array $by_alias = array();
-  /** @var array<string, string> */
-  public array $errors = array();
-  public int $stales = 0;
-  public int $corruptions = 0;
-  public int $fast_jsons = 0;
 
-  function getByAlias(string $strid) : ConfigCacheEntry
+  $serial = sizeof($jobs) == 1;
+
+  $results_by_job = _config_update_worker_run_procs($params, $jobs, $serial);
+
+  if(!$serial)
   {
-    if(array_key_exists($strid, $this->by_alias))
+    //NOTE: in case result unserialize error try serial processing
-      return $this->by_alias[$strid];
+    if(array_search(false, $results_by_job, true/*strict*/) !== false)
-    throw new Exception("Failed to find config by alias '$strid'!");
+    {
-  }
+      if($params->verbose)
-  
+        config_log("Corrupted result, trying serial processing...");
-  function getById(int $id) : ConfigCacheEntry
+      $results_by_job = _config_update_worker_run_procs(params: $params, jobs: $jobs, serial: true);
-  {
+    }
-    if(array_key_exists($id, $this->by_id))
-      return $this->by_id[$id];
-    throw new Exception("Failed to find config by id '$id'!");
-  }
-  
-  function getByPath(string $path) : ConfigCacheEntry
-  {
-    if(array_key_exists($path, $this->by_path))
-      return $this->by_path[$path];
-    throw new Exception("Failed to find config by path '$path'!");
   }
+
+  $result = _config_merge_update_results($params, $results_by_job);
+
+  if($params->verbose)
+    config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
+
+  return $result;
 }
 
-function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
+function config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
 {
+  if($params->affected_files->isEmpty())
+    return new ConfigCacheUpdateResult($params);
+
   $t = microtime(true);
 
-  $result = _config_cache_fetch($params);
+  $result = _config_cache_update($params);
 
   if($result->errors)
   {
@@ -254,74 +419,26 @@ function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
   }
 
   if($params->verbose)
-    config_log("Fetch from cache: " . round(microtime(true) - $t,2) . " sec.");
+    config_log("Update cache: " . round(microtime(true) - $t,2) . " sec.");
 
   return $result;
 }
 
-function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
+function _config_merge_update_results(ConfigCacheUpdateParams $params, array $results_by_job) : ConfigCacheUpdateResult
 {
-  if($params->scanned->isEmpty())
+  $result = new ConfigCacheUpdateResult($params);
-    return new ConfigFetchResult();
 
-  $jobs = $params->splitJobs(sort: true);
-
-  $serial = sizeof($jobs) == 1;
-
-  $results_by_job = _config_worker_run_procs($params, $jobs, $serial);
-
-  if(!$serial)
-  {
-    //NOTE: in case result unserialize error try serial processing
-    if(array_search(false, $results_by_job, true/*strict*/) !== false)
-    {
-      if($params->verbose)
-        config_log("Corrupted result, trying serial processing...");
-      $results_by_job = _config_worker_run_procs($params, $jobs, true);
-    }
-  }
-
-  $t = microtime(true);
-  $result = _config_merge_fetch_results($params, $results_by_job);
-  if($params->verbose)
-    config_log("Merge results: " . round(microtime(true) - $t,2) . " sec.");
-
-  if($params->verbose)
-    config_log("Miss(Fast)/Total: {$result->stales}($result->fast_jsons)/" . sizeof($result->all));
-
-  return $result;
-}
-
-function config_cache_fetch_by_path(ConfigGlobals $globals, string $path, bool $force_stale = false) : ConfigCacheEntry
-{
-  $path = realpath($path);
-  $base_dir = config_map_base_dir($globals->base_dirs, $path);
-
-  $scanned = new ConfigScanResult();
-  $scanned->add($base_dir, $path);
-
-  $ces = config_cache_fetch(new ConfigFetchParams(
-    globals: $globals, 
-    scanned: $scanned,
-    force_stale: $force_stale
-  ));
-
-  if(empty($ces->all))
-    throw new Exception("Config not found at path '$path'");
-  return $ces->all[0];
-}
-
-function _config_merge_fetch_results(ConfigFetchParams $params, array $results_by_job) : ConfigFetchResult
-{
-  $result = new ConfigFetchResult();
-  $total_stales = 0;
   $total_fast_jsons = 0;
   
   foreach($results_by_job as $results)
   {
     foreach($results as $item) 
     {
-      list($base_dir, $file, $cache_file, $is_stale, $parser_type, $error) = $item;
+      $base_dir = $item['base_dir'];
+      $file = $item['file'];
+      $cache_file = $item['cache_file'];
+      $parser_type = $item['parser_type'];
+      $error = $item['error'];
 
       if($error !== null)
       {
@@ -331,50 +448,16 @@ function _config_merge_fetch_results(ConfigFetchParams $params, array $results_b
 
       if($parser_type === 1)
         ++$total_fast_jsons;
-
-      $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file));
-
-      //NOTE: handling any cache corruption errors
-      if($cache_entry === null)
-      {
-        $is_stale = true;
-        $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
-        ++$result->corruptions;
-      }
-
-      $includes = $cache_entry->includes;
-
-      if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes))
-      {
-        $is_stale = true;
-        $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
-        //NOTE: let's change the mtime of the file which include other files,
-        //      so that on tne next build it will be 'older' than its includes
-        //      and won't trigger rebuild
-        if($params->touch_files_with_includes)
-          touch($file);
-      }
-
-      if($is_stale)
-        ++$total_stales;
-
-      //we want results to be returned in the same order, so 
-      //we store entries by the file key and later retrieve array values
-      $result->by_path[$file] = $cache_entry;
-      $result->by_id[$cache_entry->id] = $cache_entry;
-      $result->by_alias[$cache_entry->strid] = $cache_entry;
     }
   }
 
-  $result->all = array_values($result->by_path);
-  $result->stales = $total_stales;
   $result->fast_jsons = $total_fast_jsons;
+
   return $result;
 }
 
-function _config_invalidate_cache(
+function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file, 
-  ConfigFetchParams $params, string $base_dir, string $file, 
+  ?int &$parser_type = null) : ConfigCacheEntry
-  string $cache_file, ?int &$parser_type = null) : ConfigCacheEntry
 {
   list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
   if(!$conf_id)
@@ -399,6 +482,8 @@ function _config_invalidate_cache(
 
   $cache_payload_file = config_get_cache_payload_path($params->globals, $file);
 
+  $cache_file = config_get_cache_path($params->globals, $file);
+
   $cache_entry = new ConfigCacheEntry();
   $cache_entry->id = $conf_id;
   $cache_entry->strid = $conf_strid;
@@ -408,14 +493,90 @@ function _config_invalidate_cache(
   $cache_entry->payload_file = $cache_payload_file;
   //TODO: pass flag if file is actually normalized?
   $cache_entry->file = normalize_path($file);
-  $cache_entry->includes = config_get_module_includes($pres->jsm_module);
+  $cache_entry->includes = config_get_module_includes(cm: $pres->jsm_module);
-  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
   //TODO: do we really need these refs?
   $cache_entry->refs = config_content_match_refs($pres->normalized_jzon);
+  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
 
   ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
   ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
+  ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
+  ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
 
   return $cache_entry;
 }
 
+function _config_update_worker_run_procs(ConfigCacheUpdateParams $params, array $jobs, bool $serial) : array
+{
+  if($serial)
+  {
+    $results_by_job = array();
+    foreach($jobs as $job)
+      $results_by_job[] = _config_update_worker_func($params, $job, $serial);
+    return $results_by_job;
+  }
+  else
+  {
+    //initializing worker for master process anyway
+    $params->globals->initWorker(true);
+
+    $workers_args = array();
+    foreach($jobs as $job)
+      $workers_args[] = array($params, $job);
+
+    return run_background_gamectl_workers('config_update_worker', $workers_args);
+  }
+}
+
+function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job, bool $is_master_proc = false) : array
+{
+  $start_time = microtime(true);
+
+  $params->globals->initWorker($is_master_proc);
+
+  list($idx, $start_time, $chunk) = $job;
+  if($params->verbose)
+    config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
+
+  $fast_parser_num = 0;
+  $results = array();
+  foreach($chunk as $file_idx => $chunk_data)
+  {
+    list($base_dir, $file) = $chunk_data;
+
+    try
+    {
+      if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
+        config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
+
+      $cache_file = config_get_cache_path($params->globals, $file);
+
+      $parser_type = null;
+      $entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
+
+      $results[] = [
+        'base_dir' => $base_dir, 
+        'file' => $file, 
+        'cache_file' => $entry->cache_file, 
+        'parser_type' => $parser_type, 
+        'error' => null
+      ];
+    }
+    catch(\Throwable $e)
+    {
+      $results[] = [
+        'base_dir' => $base_dir, 
+        'file' => $file, 
+        'cache_file' => null,
+        'parser_type' => null,
+        'error' => $e->getMessage()
+      ];
+    }
+  }
+
+  if($params->verbose)
+    config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
+
+  return $results;
+}
+

config.inc.php

@@ -15,6 +15,7 @@ class ConfigGlobals
   public ?string $worker_init_fn = null;
   public string $base_class = '\ConfBase';
   public string $build_dir;
+  public ?ConfigCacheFileMap $file_map;
 
   function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null)
   {
@@ -35,6 +36,210 @@ class ConfigGlobals
   }
 }
 
+enum ConfigUpdateMode : int
+{
+  case ChangedOnly      = 1;
+  case RelativeToBundle = 2;
+  case Full             = 3;
+}
+
+class ConfigManager
+{
+  private ConfigGlobals $globals; 
+  private int $workers_num;
+
+  private ConfigCache $cache;
+
+  function __construct(ConfigGlobals $globals, int $workers_num)
+  {
+    $this->globals = $globals;
+    $this->workers_num = $workers_num;
+
+    $this->cache = new ConfigCache($globals);
+  }
+
+  function getGlobals() : ConfigGlobals
+  {
+    return $this->globals;
+  }
+
+  function getCache() : ConfigCache
+  {
+    return $this->cache;
+  }
+
+  function getArtifactFilesSpec() : array
+  {
+    return [$this->globals->base_dirs, ['.js']];
+  }
+
+  function updateCache(
+    ConfigUpdateMode $update_mode,
+    ConfigDirFiles $input_files = null,
+    ?string $result_bundle_file = null, 
+    bool $verbose = false
+   ) : ConfigCacheUpdateResult
+  {
+    if($input_files === null && $update_mode === ConfigUpdateMode::ChangedOnly)
+      throw new Exception("input_files argument is required for ChangedOnly mode");
+
+    if($input_files === null)
+      $input_files = $this->scanFiles(extension: '.js', verbose: $verbose);
+
+    $added_files = [];
+    $removed_files = [];
+    $fs_cache_map = $this->_checkFileMap(
+      $update_mode, 
+      $input_files, 
+      $added_files, 
+      $removed_files
+    );
+
+    $affected_files = $this->_getAffectedFiles(
+      $update_mode, 
+      $result_bundle_file, 
+      $fs_cache_map, 
+      $input_files, 
+      $removed_files
+    );
+
+    //NOTE: at this poine taking into account only config files
+    $affected_files->filter(fn($file) => str_ends_with($file, '.conf.js'));
+
+    $update_params = new ConfigCacheUpdateParams(
+      globals: $this->globals, 
+      affected_files: $affected_files, 
+      verbose: $verbose, 
+      max_workers: $this->workers_num
+    );
+    $update_result = config_cache_update($update_params);
+
+    $this->cache->clear();
+
+    //TODO: traverse all affected files and update file map
+    foreach($affected_files as $file)
+    {
+      $cache_entry = $this->cache->getOrLoadByPath($file);
+      $fs_cache_map->updateDepsForEntry($cache_entry);
+    }
+
+    if($affected_files->count() > 0 || $added_files || $removed_files)
+      $this->_saveMap($fs_cache_map);
+
+    return $update_result;
+  }
+
+  private function _checkFileMap(ConfigUpdateMode $update_mode, ConfigDirFiles $input_files, array &$added_files, array &$removed_files) : ConfigCacheFileMap
+  {
+    //TODO: do we need to check if cache stale somehow? 
+    $fs_cache_map = $this->_tryLoadMap();
+    //NOTE: if there's no map so far we need to create one
+    if($fs_cache_map === null)
+    {
+      $fs_cache_map = new ConfigCacheFileMap();
+
+      //let's re-use the input files
+      if($update_mode === ConfigUpdateMode::Full || $update_mode === ConfigUpdateMode::RelativeToBundle)
+        $tmp_files = $input_files;
+      else
+        $tmp_files = $this->scanFiles(extension: '.js');
+
+      $fs_cache_map->update($tmp_files->getAllFiles());
+    }
+
+    if($update_mode === ConfigUpdateMode::Full || $update_mode === ConfigUpdateMode::RelativeToBundle)
+    {
+      list($added_files, $removed_files) = $fs_cache_map->update($input_files->getAllFiles());
+      config_log("File map added: ".count($added_files).", removed: ".count($removed_files));
+    }
+
+    return $fs_cache_map;
+  }
+
+  private function _getAffectedFiles(ConfigUpdateMode $update_mode, ?string $result_bundle_file, ConfigCacheFileMap $fs_cache_map, ConfigDirFiles $input_files, array $removed_files) : ConfigDirFiles
+  {
+    $affected_files = null;
+
+    if($update_mode === ConfigUpdateMode::Full)
+    {
+      $affected_files = $input_files;
+    }
+    else if($update_mode === ConfigUpdateMode::RelativeToBundle)
+    {
+      if($result_bundle_file === null)
+        throw new Exception("result_bundle_file argument is required");
+
+      $affected_files = $this->newDirFiles();
+
+      foreach($input_files->getMap() as $base_dir => $files)
+      {
+        foreach($files as $file)
+        {
+          if(need_to_regen($result_bundle_file, [$file]))
+          {
+            $affected_files->add($base_dir, $file);
+
+            $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+            foreach($affected_by_file as $dep)
+              $affected_files->addFile($dep, unique: true);
+          }
+        }
+      }
+
+      //if there were removed files we need to rebuild affected files
+      foreach($removed_files as $file)
+      {
+        if(!str_ends_with($file, '.conf.js'))
+        {
+          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+          foreach($affected_by_file as $dep)
+            $affected_files->add(config_map_base_dir($this->globals->base_dirs, $dep), $dep, unique: true);
+        }
+        else
+        {
+          //TODO: in case config file was removed do we actually need to rebuild all configs?
+        }
+      }
+    }
+    else
+      throw new Exception("Unknown update mode: {$update_mode->value}");
+
+    return $affected_files;
+  }
+
+  function newDirFiles() : ConfigDirFiles
+  {
+    return new ConfigDirFiles([], $this->globals->base_dirs);
+  }
+
+  function parseFile(string $file) : ConfigParseResult
+  {
+    return config_parse($this->globals->base_dirs, $file);
+  }
+
+  function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
+  {
+    return config_scan_files($this->globals->base_dirs, $extension, $verbose);
+  }
+
+  private function _getMapPath() : string
+  {
+    return $this->globals->build_dir . '/file_map.data';
+  }
+
+  private function _tryLoadMap() : ?ConfigCacheFileMap
+  {
+    if(!is_file($this->_getMapPath()))
+      return null;
+    return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
+  }
+
+  private function _saveMap(ConfigCacheFileMap $map)
+  {
+    ensure_write($this->_getMapPath(), ConfigCacheFileMap::serialize($map));
+  }
+}
+
 function config_log($msg)
 {
   echo "[CFG] $msg\n";
@@ -42,12 +247,22 @@ function config_log($msg)
 
 function config_get_cache_path(ConfigGlobals $globals, string $file) : string
 {
-  return config_get_tmp_build_path($globals, $file . '.cacheb');
+  return config_get_tmp_build_path($globals, $file . '.cache');
+} 
+
+function config_get_cache_id_path(ConfigGlobals $globals, int $id) : string
+{
+  return config_get_tmp_build_path($globals, $id . '.id');
+} 
+
+function config_get_cache_strid_path(ConfigGlobals $globals, string $strid) : string
+{
+  return config_get_tmp_build_path($globals, $strid  . '.strid');
 } 
 
 function config_get_cache_payload_path(ConfigGlobals $globals, string $file) : string
 {
-  return config_get_tmp_build_path($globals, $file . '.pdata');
+  return config_get_tmp_build_path($globals, $file . '.payload');
 } 
 
 function config_load(ConfigGlobals $globals, string $conf_path, ?string $conf_dir = null) : object

pack.inc.php

@@ -5,7 +5,7 @@ use Exception;
 class ConfigPackParams
 {
   /*var ConfigCacheEntry[]*/
-  public array $cache_entries;
+  public iterable $cache_entries;
   public bool $use_lz4 = false;
   public bool $use_config_refs = false;
   public int $binary_format = 1; //1,2 supported
@@ -17,7 +17,7 @@ class ConfigPackParams
   public array $extras = array();
 
   function __construct(
-    array $cache_entries, 
+    iterable $cache_entries, 
     int $version, 
     bool $use_lz4 = false, 
     bool $use_config_refs = false, 
@@ -115,7 +115,7 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
 
 //NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
 function _config_pack_bundle_fmt1(
-  array $cache_entries, 
+  iterable $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
   int $version) : string
@@ -173,7 +173,7 @@ function _config_pack_bundle_fmt1(
 //NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
 //      (each path item separated by '/' is stored as an array item)
 function _config_pack_bundle_fmt2(
-  array $cache_entries, 
+  iterable $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
   int $version,

parse.inc.php

@@ -9,7 +9,7 @@ class ConfigParseResult
   public string $normalized_jzon = '';
   public array $parsed_arr;
   public int $parser_type;
-  public $jsm_module;
+  public \JSM_Module $jsm_module;
 }
 
 function config_parse(array $base_dirs, string $file) : ConfigParseResult

scan.inc.php

@@ -2,17 +2,24 @@
 namespace taskman;
 use Exception;
 
-class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator
+class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator
 {
+  private array $base_dirs;
+
   /*var array<string, string[]>*/
-  public array $base_dir2files = array();
+  private array $base_dir2files = array();
 
   private $iter_pos = 0;
 
-  function __construct(array $base_dir2files = array())
+  function __construct(array $base_dir2files = array(), ?array $base_dirs = null)
   {
     foreach($base_dir2files as $dir => $files)
       $this->base_dir2files[$dir] = $files;
+
+    if($base_dirs === null)
+      $this->base_dirs = array_keys($base_dir2files);
+    else
+      $this->base_dirs = $base_dirs;
   }
 
   function clear()
@@ -42,7 +49,11 @@ class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator
   function filter(callable $filter)
   {
     foreach($this->base_dir2files as $base_dir => $files)
-      $this->base_dir2files[$base_dir] = array_filter($files, $filter);
+    {
+      $filtered = array_filter($files, $filter);
+      //NOTE: don't want any index gaps
+      $this->base_dir2files[$base_dir] = array_values($filtered);
+    }
   }
 
   function forEachFile(callable $fn)
@@ -54,12 +65,23 @@ class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator
     }
   }
 
-  function add(string $base_dir, string $file)
+  function addFile(string $file, bool $unique = false)
+  {
+    $this->add(config_map_base_dir($this->base_dirs, $file, normalized: true), $file, $unique);
+  }
+
+  function add(string $base_dir, string $file, bool $unique = false)
   {
     if(!isset($this->base_dir2files[$base_dir]))
       $this->base_dir2files[$base_dir] = array();
 
-    $this->base_dir2files[$base_dir][] = $file;
+    if(!$unique || !in_array($file, $this->base_dir2files[$base_dir]))
+      $this->base_dir2files[$base_dir][] = $file;
+  }
+
+  function getMap() : array
+  {
+    return $this->base_dir2files;
   }
 
   //returns [[base_dir, file1], [base_dir, file2], ...]
@@ -172,24 +194,26 @@ function config_scan_files(
   iterable $base_dirs, 
   string $ext_filter = '.conf.js', 
   bool $verbose = false
-  ) : ConfigScanResult
+  ) : ConfigDirFiles
 {
   $t = microtime(true);
 
-  $result = new ConfigScanResult();
+  $base_dir2files = [];
-
   foreach($base_dirs as $base_dir)
   {
-    $result->base_dir2files[$base_dir] = 
+    $base_dir2files[$base_dir] = 
-      scan_files_rec(array($base_dir), array($ext_filter));
+      scan_files_rec([$base_dir], [$ext_filter]);
   }
 
+  $result = new ConfigDirFiles($base_dir2files);
+
   if($verbose)
     config_log("File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec.");
 
   return $result;
 }
 
+//NOTE: this one is obsolete
 function config_hash_changed(ConfigGlobals $globals, iterable $all_files)
 {
   $all_crc_file = $globals->build_dir . "/configs.crc";

task.inc.php

@@ -3,7 +3,7 @@ namespace taskman;
 use Exception;
 use Throwable;
 
-task('config_worker', function(array $args)
+task('config_update_worker', function(array $args)
 {
   if(sizeof($args) != 3)
     throw new Exception("Config worker args not set");
@@ -15,7 +15,7 @@ task('config_worker', function(array $args)
   try
   {
     list($params, $job) = unserialize(ensure_read($in_file));
-    $result = _config_worker_func($params, $job);
+    $result = _config_update_worker_func($params, $job);
     ensure_write($out_file, serialize($result));
   }
   catch(Throwable $e)
@@ -26,73 +26,3 @@ task('config_worker', function(array $args)
     throw $e;
   }
 });
-
-function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $serial) : array
-{
-  if($serial)
-  {
-    $results_by_job = array();
-    foreach($jobs as $job)
-      $results_by_job[] = _config_worker_func($params, $job, $serial);
-    return $results_by_job;
-  }
-  else
-  {
-    //initializing worker for master process anyway
-    $params->globals->initWorker(true);
-
-    $workers_args = array();
-    foreach($jobs as $job)
-      $workers_args[] = array($params, $job);
-
-    return run_background_gamectl_workers('config_worker', $workers_args);
-  }
-}
-
-//returns [[base_dir, file, cache_file, was_stale, parser_type, error], ...]
-function _config_worker_func(ConfigFetchParams $params, array $job, bool $is_master_proc = false) : array
-{
-  $start_time = microtime(true);
-
-  $params->globals->initWorker($is_master_proc);
-
-  list($idx, $start_time, $chunk) = $job;
-  if($params->verbose)
-    config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
-
-  $fast_parser_num = 0;
-  $results = array();
-  foreach($chunk as $file_idx => $chunk_data)
-  {
-    list($base_dir, $file) = $chunk_data;
-
-    try
-    {
-      if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
-        config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
-
-      $cache_file = config_get_cache_path($params->globals, $file);
-
-      $parser_type = null;
-
-      $is_stale = true;
-      if(!$params->force_stale)
-        $is_stale = need_to_regen($cache_file, array($file));
-
-      if($is_stale)
-        _config_invalidate_cache($params, $base_dir, $file, $cache_file, $parser_type);
-
-      $results[] = array($base_dir, $file, $cache_file, $is_stale, $parser_type, null);
-    }
-    catch(Throwable $e)
-    {
-      $results[] = array($base_dir, $file, null, null, null, $e->getMessage());
-    }
-  }
-
-  if($params->verbose)
-    config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
-
-  return $results;
-}
-

util.inc.php

@@ -4,7 +4,7 @@ use Exception;
 
 function config_get_tmp_build_path(ConfigGlobals $globals, string $file) : string
 {
-  $name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file))));
+  $name = str_replace(['@', ':', "\\", "/"], ['', "-", "-", "-"], normalize_path($file));
   $name = ltrim($name, "-");
   return normalize_path($globals->build_dir . "/$name");
 }
@@ -70,6 +70,7 @@ function config_crc28(string $what) : int
 function config_get_module_includes(\JSM_Module $cm) : array
 {
   $includes = array();
+  //NOTE: module contains all includes (even those included from other modules)
   foreach($cm->getIncludes() as $include => $_)
   {
     //maybe we should take .php includes into account as well?