@foo/bar refs are now taken into account during affected files detection

cache.inc.php

@@ -95,6 +95,13 @@ class ConfigCache
       throw new Exception("Failed to find entry by strid '$strid'");
     return $ce;
   }
+
+  function update(ConfigCacheEntry $ce)
+  {
+    $this->by_path[$ce->file] = $ce;
+    $this->by_id[$ce->id] = $ce;
+    $this->by_alias[$ce->strid] = $ce;
+  }
 }
 
 /**
@@ -261,6 +268,16 @@ class ConfigCacheFileMap
     return $map;
   }
 
+  function count() : int
+  {
+    return count($this->file2deps);
+  }
+
+  function exists(string $file) : bool
+  {
+    return isset($this->file2deps[$file]);
+  }
+
   function getAffectedFiles(string $file) : array
   {
     if(!isset($this->file2deps[$file]))
@@ -268,8 +285,11 @@ class ConfigCacheFileMap
     return array_keys($this->file2deps[$file]);
   }
 
-  function differs(array $files) : bool
+  function differs(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles|array $files) : bool
   {
+    if(!is_array($files))
+      $files = $files->getAllFiles();
+
     $this_files = array_keys($this->file2deps);
 
     return count($files) !== count($this_files) || 
@@ -305,14 +325,24 @@ class ConfigCacheFileMap
       unset($this->file2deps[$file]);
   }
 
-  function updateDepsForEntry(ConfigCacheEntry $entry)
+  function updateDepsForEntry(array $base_dirs, ConfigCacheEntry $entry)
   {
-    foreach($entry->includes as $include)
+    //1. add includes
+    $all_refs = $entry->includes;
+
+    //2. add refs
+    foreach($entry->refs as $ref => $_)
     {
-      if(isset($this->file2deps[$include]))
-        $this->file2deps[$include][$entry->file] = true;
+      $file = config_real_path($base_dirs, substr($ref, 1) . '.conf.js');
+      $all_refs[] = $file;
+    }
+
+    foreach($all_refs as $ref_file)
+    {
+      if(isset($this->file2deps[$ref_file]))
+        $this->file2deps[$ref_file][$entry->file] = true;
       else
-        $this->file2deps[$include] = [$entry->file => true];
+        $this->file2deps[$ref_file] = [$entry->file => true];
     }
   }
 
@@ -329,22 +359,6 @@ class ConfigCacheFileMap
   }
 }
 
-class ConfigCacheUpdateResult
-{
-  public ConfigDirFiles $affected_files;
-  public array $affected_entries = array();
-
-  /** @var array<string, string> */
-  public array $errors = array();
-  public int $corruptions = 0;
-  public int $fast_jsons = 0;
-
-  function __construct(ConfigCacheUpdateParams $params)
-  {
-    $this->affected_files = $params->affected_files;
-  }
-}
-
 class ConfigCacheUpdateParams
 {
   const MAX_DEFAULT_WORKERS = 5;
@@ -353,25 +367,21 @@ class ConfigCacheUpdateParams
   public ConfigGlobals $globals;
   public ConfigDirFiles $affected_files;
   public bool $verbose = false;
-  public ?int $max_workers = null;
-  public bool $check_junk = true;
   public int $max_errors_num = 15;
   public bool $return_affected_entries = false;
 
-  function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, 
-    bool $verbose = false, ?int $max_workers = null)
+  function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, bool $verbose = false)
   {
     $this->globals = $globals;
     $this->affected_files = $affected_files;
     $this->verbose = $verbose;
-    $this->max_workers = $max_workers;
   }
 
   function calcMaxWorkers() : int
   {
     if($this->affected_files->count() < self::FILES_THRESHOLD_ONE_WORKER)
       return 1;
-    return $this->max_workers ?? self::MAX_DEFAULT_WORKERS;
+    return $this->globals->max_workers ?? self::MAX_DEFAULT_WORKERS;
   }
 
   function splitFilesByChunks(int $max_workers, bool $sort = true) : array
@@ -398,7 +408,7 @@ class ConfigCacheUpdateParams
   }
 }
 
-function _config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
+function _config_cache_update(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
 {
   $jobs = $params->splitJobs(sort: true);
 
@@ -417,18 +427,14 @@ function _config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpda
     }
   }
 
-  $result = _config_merge_update_results($params, $results_by_job);
+  _config_merge_update_results($result, $results_by_job);
 
   if($params->verbose)
     config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
-
-  return $result;
 }
 
-function _config_merge_update_results(ConfigCacheUpdateParams $params, array $results_by_job) : ConfigCacheUpdateResult
+function _config_merge_update_results(ConfigUpdateResult $result, array $results_by_job)
 {
-  $result = new ConfigCacheUpdateResult($params);
-
   $total_fast_jsons = 0;
   
   foreach($results_by_job as $results)
@@ -453,11 +459,9 @@ function _config_merge_update_results(ConfigCacheUpdateParams $params, array $re
   }
 
   $result->fast_jsons = $total_fast_jsons;
-
-  return $result;
 }
 
-function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file, 
+function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $file, string $cache_file, 
   ?int &$parser_type = null) : ConfigCacheEntry
 {
   list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
@@ -469,7 +473,7 @@ function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir,
   $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id; 
   $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
 
-  $pres = config_parse($params->globals->base_dirs, $file);
+  $pres = config_parse($globals->base_dirs, $file);
   if($pres->error !== 0)
     throw new Exception("Parse error({$pres->error}):\n" . $pres->error_descr);
 
@@ -479,11 +483,9 @@ function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir,
   $parsed_arr['id'] = $conf_id;
   $parsed_arr['strid'] = $conf_strid;
   list($class, $class_id, $normalized_data) 
-    = config_apply_class_normalization($parsed_arr, $params->check_junk);
+    = config_apply_class_normalization($parsed_arr, check_junk: true);
 
-  $cache_payload_file = config_get_cache_payload_path($params->globals, $file);
-
-  $cache_file = config_get_cache_path($params->globals, $file);
+  $cache_payload_file = config_get_cache_payload_path($globals, $file);
 
   $cache_entry = new ConfigCacheEntry();
   $cache_entry->id = $conf_id;
@@ -501,8 +503,8 @@ function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir,
 
   ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
   ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
-  ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
-  ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
+  ensure_write(config_get_cache_id_path($globals, $conf_id), $cache_entry->file);
+  ensure_write(config_get_cache_strid_path($globals, $conf_strid), $cache_entry->file);
 
   return $cache_entry;
 }
@@ -553,12 +555,12 @@ function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job,
       $cache_file = config_get_cache_path($params->globals, $file);
 
       $parser_type = null;
-      $entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
+      $entry = _config_update_cache_entry($params->globals, $base_dir, $file, $cache_file, $parser_type);
 
       $results[] = [
         'base_dir' => $base_dir, 
         'file' => $file, 
-        'cache_file' => $entry->cache_file, 
+        'cache_file' => $cache_file, 
         'parser_type' => $parser_type, 
         'error' => null
       ];

config.inc.php

@@ -11,16 +11,30 @@ require_once(__DIR__ . '/msgpack.inc.php');
 
 class ConfigGlobals
 {
-  public array $base_dirs = array();
-  public ?string $worker_init_fn = null;
-  public string $base_class = '\ConfBase';
-  public string $build_dir;
+  public readonly array $base_dirs;
+  //NOTE: it's a string since is serialized
+  public readonly ?string $worker_init_fn;
+  public readonly int $workers_num;
+  public readonly string $base_class;
+  public readonly string $build_dir;
+  //NOTE: it's a string since is serialized
+  public readonly ?string $files_filter; 
 
-  function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null)
+  function __construct(
+    array $base_dirs, 
+    string $build_dir, 
+    ?string $worker_init_fn = null,
+    int $workers_num = 1,
+    string $base_class = '\ConfBase',
+    ?string $files_filter = null
+  )
   {
     $this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
     $this->build_dir = $build_dir;
     $this->worker_init_fn = $worker_init_fn;
+    $this->workers_num = $workers_num;
+    $this->base_class = $base_class;
+    $this->files_filter = $files_filter;
   }
 
   function initWorker(bool $is_master_proc)
@@ -35,59 +49,105 @@ class ConfigGlobals
   }
 }
 
+enum ConfigGetMode : int
+{
+  case ForceUpdate   = 1;
+  case Auto          = 2;
+  case CacheOnly     = 3;
+}
+
 enum ConfigUpdateMode : int
 {
   case Force         = 1;
   case DetectChanged = 2;
-  case Patch         = 3;
+  case Selected      = 3;
 }
 
 class ConfigUpdateRequest
 {
   public ConfigUpdateMode $mode;
   public ?ConfigDirFiles $files;
-  public ?string $result_file;
+  public ?\taskman\TaskmanFileChanges $file_changes;
+  public bool $verbose = false;
+  public bool $return_affected = false;
 
   private function __construct() {}
 
-  static function force(?ConfigDirFiles $files = null) : ConfigUpdateRequest
+  static function force(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
   {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
     $req = new ConfigUpdateRequest();
     $req->mode = ConfigUpdateMode::Force;
     $req->files = $files;
     return $req;
   }
 
-  static function detectChanged(string $result_file, ?ConfigDirFiles $files = null) : ConfigUpdateRequest
+  static function detectChanged(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
   {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
     $req = new ConfigUpdateRequest();
     $req->mode = ConfigUpdateMode::DetectChanged;
     $req->files = $files;
-    $req->result_file = $result_file;
     return $req;
   }
 
-  static function patch(ConfigDirFiles $files) : ConfigUpdateRequest
+  static function selected(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles $files, \taskman\TaskmanFileChanges|null $file_changes = null) : ConfigUpdateRequest
   {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
     $req = new ConfigUpdateRequest();
-    $req->mode = ConfigUpdateMode::Patch;
+    $req->mode = ConfigUpdateMode::Selected;
     $req->files = $files;
+    $req->file_changes = $file_changes;
     return $req;
   }
 }
 
+class ConfigUpdateResult
+{
+  public ConfigUpdateRequest $request;
+
+  public ConfigDirFiles $affected_files;
+  public array $affected_entries = array();
+
+  public array $added_files = array();
+  public array $removed_files = array();
+
+  /** @var array<string, string> */
+  public array $errors = array();
+  public int $corruptions = 0;
+  public int $fast_jsons = 0;
+
+  function isEmpty() : bool
+  {
+    return count($this->affected_files) == 0 && 
+      count($this->added_files) == 0 && 
+      count($this->removed_files) == 0;
+  }
+
+  function isPatchPossible() : bool
+  {
+    return ($this->request->mode == ConfigUpdateMode::Selected ||
+            $this->request->mode === ConfigUpdateMode::DetectChanged) && 
+            count(array_filter($this->added_files, fn($f) => config_is_file($f))) == 0 &&
+            count(array_filter($this->removed_files, fn($f) => config_is_file($f))) == 0;
+  }
+}
+
 class ConfigManager
 {
   private ConfigGlobals $globals; 
-  private int $workers_num;
-
   private ConfigCache $cache;
   private ?ConfigCacheFileMap $file_map;
 
-  function __construct(ConfigGlobals $globals, int $workers_num)
+  function __construct(ConfigGlobals $globals)
   {
     $this->globals = $globals;
-    $this->workers_num = $workers_num;
 
     $this->cache = new ConfigCache($globals);
     $this->file_map = null;
@@ -103,7 +163,7 @@ class ConfigManager
     return $this->cache;
   }
 
-  function getArtifactFilesSpec() : array
+  function getArtefactFilesSpec() : array
   {
     return [$this->globals->base_dirs, ['.js']];
   }
@@ -115,63 +175,124 @@ class ConfigManager
       $this->file_map = $this->_tryLoadMap();
       if($this->file_map === null)
       {
-        $this->file_map = self::_makeMap($files ?? $this->scanFiles(extension: '.js'));
+        $this->file_map = self::_makeMap($this->scanFiles(extension: '.js'));
         $this->_saveFileMap();
       }
     }
     return $this->file_map;
   }
 
-  function updateCache(ConfigUpdateRequest $req, bool $return_entries = false, bool $verbose = false) : ConfigCacheUpdateResult
+  function fetchAll(bool $force = false) : \Generator
   {
-    config_log("Updating cache, mode '{$req->mode->name}'...");
+    $result = $this->updateAll($force);
+    $filtered = $this->filterConfigFiles($result->request->files);
+    return $this->iterateCache($filtered);
+  }
+
+  function fetchByPath(string $path) : ConfigCacheEntry
+  {
+    $path = realpath($path);
+    $dir_files = ConfigDirFiles::makeFor($this);
+    $dir_files->addFile($path);
+    $request = ConfigUpdateRequest::selected($dir_files);
+    $request->return_affected = true;
+    $result = $this->update($request);
+    return $result->affected_entries[0];
+  }
+
+  function iterateCache(iterable $files) : \Generator
+  {
+    foreach($files as $file)
+    {
+      $ce = $this->cache->getOrLoadByPath($file);
+      yield $ce;
+    }
+  }
+
+  function update(ConfigUpdateRequest $req) : ConfigUpdateResult
+  {
+    config_log("Updating configs, mode '{$req->mode->name}'...");
 
     if($req->files === null)
-      $req->files = $this->scanFiles(extension: '.js', verbose: $verbose);
+      $req->files = $this->scanFiles(extension: '.js', verbose: $req->verbose);
 
     $added_files = [];
     $removed_files = [];
     $this->_checkFileMap($req, $added_files, $removed_files);
 
-    $affected_files = $this->_getAffectedFiles($req, $removed_files);
+    $affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
+    $affected_conf_files = $this->filterConfigFiles($affected_all_files);
 
-    //NOTE: at this poine taking into account only config files
-    $affected_files->filter(fn($file) => str_ends_with($file, '.conf.js'));
+    config_log("Affected configs: {$affected_conf_files->count()} (requested: {$req->files->count()})");
 
-    config_log("Affected files: {$affected_files->count()}");
+    $update_result = new ConfigUpdateResult();
+    $update_result->request = $req;
+    $update_result->affected_files = $affected_conf_files;
+    $update_result->added_files = $added_files;
+    $update_result->removed_files = $removed_files;
 
     $update_params = new ConfigCacheUpdateParams(
       globals: $this->globals, 
-      affected_files: $affected_files, 
-      verbose: $verbose, 
-      max_workers: $this->workers_num
+      affected_files: $affected_conf_files, 
+      verbose: $req->verbose
     );
-    $update_result = self::_updateCache($update_params);
+    self::_updateCache($update_result, $update_params);
 
+    //let's clear internal cache once the update procedure is done
     $this->cache->clear();
 
-    $this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
+    $this->_updateFileMap($req, $affected_conf_files, $added_files, $removed_files);
 
-    if($return_entries)
+    if($req->return_affected)
     {
-      foreach($affected_files as $file)
+      foreach($affected_conf_files as $file)
       {
         $entry = $this->cache->getOrLoadByPath($file);
         $update_result->affected_entries[] = $entry;
       }
     }
 
+    touch($this->_getLastUpdateFile());
+
     return $update_result;
   }
 
-  private function _updateCache(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
+  function filterConfigFiles(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles 
+  {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
+    $filtered = $this->globals->files_filter != null ? 
+      call_user_func($this->globals->files_filter, $files) : 
+      new ConfigDirFiles($files->getMap());
+    
+    $filtered->filter(fn($file) => config_is_file($file));
+    return $filtered;
+  }
+
+  function updateAll(bool $force = false) : ConfigUpdateResult
+  {
+    $files = $this->scanFiles(extension: '.js');
+    $result = $this->update($force ? 
+      ConfigUpdateRequest::force($files) : 
+      ConfigUpdateRequest::detectChanged($files)
+    );
+    return $result;
+  }
+
+  function _getLastUpdateFile() : string
+  {
+    return $this->globals->build_dir . '/update.last';
+  }
+
+  private function _updateCache(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
   {
     if($params->affected_files->isEmpty())
-      return new ConfigCacheUpdateResult($params);
+      return;
 
     $t = microtime(true);
 
-    $result = _config_cache_update($params);
+    _config_cache_update($result, $params);
 
     if($result->errors)
     {
@@ -188,8 +309,6 @@ class ConfigManager
 
     if($params->verbose)
       config_log("Update cache: " . round(microtime(true) - $t,2) . " sec.");
-
-    return $result;
   }
 
   private function _checkFileMap(ConfigUpdateRequest $req, array &$added_files, array &$removed_files)
@@ -198,39 +317,52 @@ class ConfigManager
 
     if($req->mode === ConfigUpdateMode::Force)
     {
-      $added_files = $req->files->getAllFiles();
       //let's rebuild the file map
-      $fs_cache_map->init($added_files);
-      config_log("File map init: ".count($added_files));
+      $fs_cache_map->init($req->files->getAllFiles());
+      config_log("File map init: ".$fs_cache_map->count());
     }
     else if($req->mode === ConfigUpdateMode::DetectChanged)
     {
       list($added_files, $removed_files) = $fs_cache_map->compare($req->files->getAllFiles());
       config_log("File map compare, added: ".count($added_files).", removed: ".count($removed_files));
     }
+    else if($req->mode === ConfigUpdateMode::Selected)
+    {
+      if($req->file_changes != null)
+      {
+        foreach($req->files as $file)
+        {
+          if($req->file_changes->isCreated($file))
+            $added_files[] = $file;
+          else if($req->file_changes->isDeleted($file))
+            $removed_files[] = $file;
+        }
+      }
+    }
   }
 
-  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
+  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_conf_files, array $added_files, array $removed_files)
   {
     $fs_cache_map = $this->getFileMap();
 
-    //TODO: traverse all affected files and update file map
-    foreach($affected_files as $file)
+    //TODO: traverse all affected config files and update file map
+    foreach($affected_conf_files as $file)
     {
       $cache_entry = $this->cache->getOrLoadByPath($file);
-      $fs_cache_map->updateDepsForEntry($cache_entry);
+      $fs_cache_map->updateDepsForEntry($this->globals->base_dirs, $cache_entry);
     }
 
-    if($affected_files->count() > 0 || $added_files || $removed_files)
+    if($req->mode == ConfigUpdateMode::Force || 
+      $affected_conf_files->count() > 0 || 
+      $added_files || 
+      $removed_files)
     {
-      //in case of Force map was already cleared and initialized
-      if($req->mode != ConfigUpdateMode::Force)
-        $fs_cache_map->update($added_files, $removed_files);
+      $fs_cache_map->update($added_files, $removed_files);
       $this->_saveFileMap();
     }
   }
 
-  private function _getAffectedFiles(ConfigUpdateRequest $req, array $removed_files) : ConfigDirFiles
+  private function _getAffectedFiles(ConfigUpdateRequest $req, array $added_files, array $removed_files) : ConfigDirFiles
   {
     $fs_cache_map = $this->getFileMap();
 
@@ -238,17 +370,19 @@ class ConfigManager
 
     if($req->mode === ConfigUpdateMode::Force)
     {
-      $affected_files = $req->files;
+      $affected_files = new ConfigDirFiles($req->files->getMap());
     }
     else if($req->mode === ConfigUpdateMode::DetectChanged)
     {
       $affected_files = ConfigDirFiles::makeFor($this);
 
+      $last_run_file = $this->_getLastUpdateFile();
+
       foreach($req->files->getMap() as $base_dir => $files)
       {
         foreach($files as $file)
         {
-          if(need_to_regen($req->result_file, [$file]))
+          if(need_to_regen($last_run_file, [$file]))
           {
             $affected_files->add($base_dir, $file);
 
@@ -262,32 +396,21 @@ class ConfigManager
       //if there were removed files we need to rebuild affected files
       foreach($removed_files as $file)
       {
-        if(!str_ends_with($file, '.conf.js'))
-        {
-          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
-          foreach($affected_by_file as $dep)
-            $affected_files->addFile($dep, unique: true);
-        }
-        else
-        {
-          //TODO: in case config file was removed do we actually need to rebuild all configs?
-        }
+        $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+        foreach($affected_by_file as $dep)
+          $affected_files->addFile($dep, unique: true);
       }
     }
-    else if($req->mode === ConfigUpdateMode::Patch)
+    else if($req->mode === ConfigUpdateMode::Selected)
     {
       $affected_files = ConfigDirFiles::makeFor($this);
 
       foreach($req->files as $file)
       {
         $affected_files->addFile($file, unique: true);
-
-        if(!str_ends_with($file, '.conf.js'))
-        {
-          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
-          foreach($affected_by_file as $dep)
-            $affected_files->addFile($dep, unique: true);
-        }
+        $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+        foreach($affected_by_file as $dep)
+          $affected_files->addFile($dep, unique: true);
       }
     }
 
@@ -313,8 +436,11 @@ class ConfigManager
   {
     if(!is_file($this->_getMapPath()))
       return null;
-    config_log("Loading file map");
-    return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
+    $t = microtime(true);
+    $data = ensure_read($this->_getMapPath());
+    $map = ConfigCacheFileMap::unserialize($data);
+    config_log("Loaded file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
+    return $map;
   }
 
   private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
@@ -327,9 +453,10 @@ class ConfigManager
 
   private function _saveFileMap()
   {
+    $t = microtime(true);
     $data = ConfigCacheFileMap::serialize($this->getFileMap()); 
-    config_log("Saving file map: " . kb($data));
     ensure_write($this->_getMapPath(), $data);
+    config_log("Saved file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
   }
 }
 

pack.inc.php

@@ -42,10 +42,12 @@ function config_pack_bundle(ConfigPackParams $params) : string
 
   $packed_data = null;
 
+   $cache_entries = iterator_to_array($params->cache_entries);
+
   if($params->binary_format == 1)
   {
     $packed_data = _config_pack_bundle_fmt1(
-      $params->cache_entries, 
+      $cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
       $params->version
@@ -54,7 +56,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
   else if($params->binary_format == 2)
   {
     $packed_data = _config_pack_bundle_fmt2(
-      $params->cache_entries, 
+      $cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
       $params->version,
@@ -63,7 +65,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
   else if($params->binary_format == 3)
   {
     $packed_data = _config_pack_bundle_fmt3(
-      $params->cache_entries, 
+      $cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
       $params->version,
@@ -75,7 +77,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
     throw new Exception("Unknown binary format: {$params->binary_format}");
 
   if($params->debug)
-    config_log("Packed entries: " . sizeof($params->cache_entries) . ", total: " . 
+    config_log("Packed entries: " . count($cache_entries) . ", total: " . 
       kb($packed_data) . ", format: {$params->binary_format}, lz4: " . 
       var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
       ", CRC: " . crc32($packed_data) . 
@@ -100,6 +102,16 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
       $params->version,
     );
   }
+  else if($params->binary_format == 3)
+  {                  
+    $patched_data = _config_patch_bundle_fmt3(
+      $packed_data,
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version,
+    );
+  }
   else
     throw new Exception("Unknown binary format: {$params->binary_format}");
 
@@ -115,7 +127,7 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
 
 //NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
 function _config_pack_bundle_fmt1(
-  iterable $cache_entries, 
+  array $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
   int $version) : string
@@ -173,7 +185,7 @@ function _config_pack_bundle_fmt1(
 //NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
 //      (each path item separated by '/' is stored as an array item)
 function _config_pack_bundle_fmt2(
-  iterable $cache_entries, 
+  array $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
   int $version,
@@ -234,7 +246,7 @@ function _config_pack_bundle_fmt2(
   return $packed_data;
 }
 
-//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
+//NOTE: Much like fmt1, but configs entries are grouped into sizeable chunks, with each chunk lz4-compressed.
 //      This reduces overall bundle size when there are many small configs entries.
 function _config_pack_bundle_fmt3(
   array $cache_entries, 
@@ -349,12 +361,12 @@ function _config_patch_bundle_fmt2(
   {
     list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
 
-    $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+    $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
 
-    if($header_found)
+    if($headers_found)
     {
-      $header_idx = key($header_found);
-      $header_entry = current($header_found);
+      $header_idx = key($headers_found);
+      $header_entry = current($headers_found);
 
       $current_offset = $header_entry[4];
       $current_size = $header_entry[5];
@@ -412,6 +424,97 @@ function _config_patch_bundle_fmt2(
   return $patched_data;
 }
 
+function _config_patch_bundle_fmt3(
+  string $packed_data,
+  array $patch_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version,
+) : string
+{
+  if(!$use_lz4)
+    throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
+
+  list($header, $entries, $chunks_offset, $max_chunk_size) = 
+    _config_unpack_bundle_fmt3(packed_data: $packed_data, unpack_entries: false);
+
+  $payloads_bundle = substr($packed_data, $chunks_offset);
+
+  foreach($patch_entries as $idx => $patch_entry)
+  {
+    list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
+
+    $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+
+    if($headers_found)
+    {
+      $header_idx = key($headers_found);
+      $header_entry = current($headers_found);
+
+      $chunk_offset = $header_entry[4];
+      $payloads_offset = $header_entry[5]; //within chunk
+      $payloads_size = $header_entry[6];
+
+      $chunk_entries = array_filter($header, fn($item) => $item[4] == $chunk_offset); 
+
+      //chunk contains only one patch entry
+      $patch_payload_lz4 = lz4_compress($patch_payload, 9); 
+      $patched_chunk = pack("V", strlen($patch_payload_lz4));
+      $patched_chunk .= $patch_payload_lz4;
+      if(strlen($patch_payload) > $max_chunk_size)
+        $max_chunk_size = strlen($patch_payload);
+
+      //TODO: make a better generic algorithm for patching the whole chunk
+      //special case for single entry 
+      if(count($chunk_entries) == 1 && $payloads_offset == 0)
+      {
+        $lz4_chunk_size = unpack("V", substr($payloads_bundle, $chunk_offset, 4))[1];
+
+        //we can just replace chunk in place if the size of a patched chunk is less or equal
+        if(strlen($patch_payload_lz4) <= $lz4_chunk_size)
+        {
+          $payloads_bundle = substr_replace($payloads_bundle, $patched_chunk, $chunk_offset, strlen($patched_chunk));
+          $header_entry[6] = strlen($patch_payload);
+        }
+        //just append to the end
+        else
+        {
+          $header_entry[4] = strlen($payloads_bundle); //chunk offset
+          $header_entry[5] = 0;
+          $header_entry[6] = strlen($patch_payload);
+
+          $payloads_bundle .= $patched_chunk;
+        }
+      }
+      //just append to the end
+      else
+      {
+        $header_entry[4] = strlen($payloads_bundle); //chunk offset
+        $header_entry[5] = 0;
+        $header_entry[6] = strlen($patch_payload);
+
+        $payloads_bundle .= $patched_chunk;
+      }
+
+      $header[$header_idx] = $header_entry; 
+    }
+    else
+      throw new Exception("Patched entry {$patch_entry->id} not found in config bundle");
+  }
+
+  $header_msgpack = config_msgpack_pack($header);
+
+  $packed_data =  
+         pack("C", 3) . 
+         pack("V", $version) . 
+         pack("V", strlen($header_msgpack)) .
+         pack("V", $max_chunk_size) .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $packed_data;
+}
+
 //format: [[class_id, [data]], ...[class_id, [data]]]
 function config_unpack_bundle(string $packed_data) : array
 {
@@ -429,7 +532,8 @@ function config_unpack_bundle(string $packed_data) : array
   }
   else if($info['format'] === 3)
   {
-    return _config_unpack_bundle_fmt3($packed_data);
+    list($_, $entries) = _config_unpack_bundle_fmt3($packed_data);
+    return $entries;
   }
   else
     throw new Exception("Unknown format: {$info['format']}");
@@ -496,12 +600,10 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries =
   return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
 }
 
-function _config_unpack_bundle_fmt3(string $packed_data): array
+function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries = true): array
 {
   if(ord($packed_data[0]) !== 3)
-  {
     throw new Exception("Invalid config bundle format");
-  }
 
   $offset = 1;
   $version = unpack("V", substr($packed_data, $offset, 4))[1];
@@ -513,6 +615,7 @@ function _config_unpack_bundle_fmt3(string $packed_data): array
 
   $header_msgpack = substr($packed_data, $offset, $header_len);
   $offset += $header_len;
+  $chunks_offset = $offset;
 
   $header = config_msgpack_unpack($header_msgpack);
 
@@ -521,39 +624,45 @@ function _config_unpack_bundle_fmt3(string $packed_data): array
   $chunk_buffer = '';
   $chunk_id = -1;
 
-  foreach ($header as $entry_data)
+  foreach($header as $entry_data)
   {
     list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
 
-    if($entry_chunk_offset !== $chunk_id)
+    $unpacked_payload = null;
+
+    if($unpack_entries)
     {
-      if($chunk_offset !== -1)
+      if($entry_chunk_offset !== $chunk_id)
       {
-        $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
-        $offset+=4;
-        $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
-        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
-        $offset += $lz4_chunk_size;
-        $chunk_offset = $offset;
-      }
-      else
-      {
-        $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
-        $chunk_offset += 4;
-        $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
-        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
-        $chunk_offset += $lz4_chunk_size;
+        if($chunk_offset !== -1)
+        {
+          $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
+          $offset+=4;
+          $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
+          $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+          $offset += $lz4_chunk_size;
+          $chunk_offset = $offset;
+        }
+        else
+        {
+          $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
+          $chunk_offset += 4;
+          $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
+          $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+          $chunk_offset += $lz4_chunk_size;
+        }
+
+        $chunk_id = $entry_chunk_offset;
       }
 
-      $chunk_id = $entry_chunk_offset;
+      $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
+      $unpacked_payload = _config_unpack_payload($format, $payload);
     }
 
-    $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
-
-    $cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+    $cache_entries[$id] = array($class_id, $unpacked_payload); 
   }
 
-  return $cache_entries;
+  return array($header, $cache_entries, $chunks_offset, $max_chunk_size);
 }
 
 //format: [format_id, payload_data]

util.inc.php

@@ -28,15 +28,17 @@ function config_real_path(array $base_dirs, string $rel_path, bool $strict = tru
 {
   foreach($base_dirs as $dir)
     if(is_file($dir . '/' . $rel_path))
-      return $dir . '/' . $rel_path;
+      return normalize_path($dir . '/' . $rel_path);
+
   if($strict)
     throw new Exception("No file for relative path '$rel_path'");
+
   return null;
 }
 
 function config_is_file(string $filename) : bool
 {
-  return (strrpos($filename, '.conf.js') === (strlen($filename) - 8));
+  return str_ends_with($filename, '.conf.js');
 }
 
 function config_file2id(string $conf_dir, string $filename) : int
@@ -104,6 +106,7 @@ function config_walk_fields(object $proto, callable $callback)
   }
 }
 
+//TODO: doesn't really belong here
 function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
 {
   if(!isset($map[$file]))