cache.inc.php

@@ -95,13 +95,6 @@ class ConfigCache
       throw new Exception("Failed to find entry by strid '$strid'");
     return $ce;
   }
-
-  function update(ConfigCacheEntry $ce)
-  {
-    $this->by_path[$ce->file] = $ce;
-    $this->by_id[$ce->id] = $ce;
-    $this->by_alias[$ce->strid] = $ce;
-  }
 }
 
 /**
@@ -268,16 +261,6 @@ class ConfigCacheFileMap
     return $map;
   }
 
-  function count() : int
-  {
-    return count($this->file2deps);
-  }
-
-  function exists(string $file) : bool
-  {
-    return isset($this->file2deps[$file]);
-  }
-
   function getAffectedFiles(string $file) : array
   {
     if(!isset($this->file2deps[$file]))
@@ -285,11 +268,8 @@ class ConfigCacheFileMap
     return array_keys($this->file2deps[$file]);
   }
 
-  function differs(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles|array $files) : bool
+  function differs(array $files) : bool
   {
-    if(!is_array($files))
-      $files = $files->getAllFiles();
-
     $this_files = array_keys($this->file2deps);
 
     return count($files) !== count($this_files) || 
@@ -325,24 +305,14 @@ class ConfigCacheFileMap
       unset($this->file2deps[$file]);
   }
 
-  function updateDepsForEntry(array $base_dirs, ConfigCacheEntry $entry)
+  function updateDepsForEntry(ConfigCacheEntry $entry)
   {
-    //1. add includes
+    foreach($entry->includes as $include)
-    $all_refs = $entry->includes;
-
-    //2. add refs
-    foreach($entry->refs as $ref => $_)
     {
-      $file = config_real_path($base_dirs, substr($ref, 1) . '.conf.js');
+      if(isset($this->file2deps[$include]))
-      $all_refs[] = $file;
+        $this->file2deps[$include][$entry->file] = true;
-    }
-
-    foreach($all_refs as $ref_file)
-    {
-      if(isset($this->file2deps[$ref_file]))
-        $this->file2deps[$ref_file][$entry->file] = true;
       else
-        $this->file2deps[$ref_file] = [$entry->file => true];
+        $this->file2deps[$include] = [$entry->file => true];
     }
   }
 
@@ -359,6 +329,22 @@ class ConfigCacheFileMap
   }
 }
 
+class ConfigCacheUpdateResult
+{
+  public ConfigDirFiles $affected_files;
+  public array $affected_entries = array();
+
+  /** @var array<string, string> */
+  public array $errors = array();
+  public int $corruptions = 0;
+  public int $fast_jsons = 0;
+
+  function __construct(ConfigCacheUpdateParams $params)
+  {
+    $this->affected_files = $params->affected_files;
+  }
+}
+
 class ConfigCacheUpdateParams
 {
   const MAX_DEFAULT_WORKERS = 5;
@@ -367,21 +353,25 @@ class ConfigCacheUpdateParams
   public ConfigGlobals $globals;
   public ConfigDirFiles $affected_files;
   public bool $verbose = false;
+  public ?int $max_workers = null;
+  public bool $check_junk = true;
   public int $max_errors_num = 15;
   public bool $return_affected_entries = false;
 
-  function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, bool $verbose = false)
+  function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, 
+    bool $verbose = false, ?int $max_workers = null)
   {
     $this->globals = $globals;
     $this->affected_files = $affected_files;
     $this->verbose = $verbose;
+    $this->max_workers = $max_workers;
   }
 
   function calcMaxWorkers() : int
   {
     if($this->affected_files->count() < self::FILES_THRESHOLD_ONE_WORKER)
       return 1;
-    return $this->globals->max_workers ?? self::MAX_DEFAULT_WORKERS;
+    return $this->max_workers ?? self::MAX_DEFAULT_WORKERS;
   }
 
   function splitFilesByChunks(int $max_workers, bool $sort = true) : array
@@ -408,7 +398,7 @@ class ConfigCacheUpdateParams
   }
 }
 
-function _config_cache_update(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
+function _config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
 {
   $jobs = $params->splitJobs(sort: true);
 
@@ -427,14 +417,18 @@ function _config_cache_update(ConfigUpdateResult $result, ConfigCacheUpdateParam
     }
   }
 
-  _config_merge_update_results($result, $results_by_job);
+  $result = _config_merge_update_results($params, $results_by_job);
 
   if($params->verbose)
     config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
+
+  return $result;
 }
 
-function _config_merge_update_results(ConfigUpdateResult $result, array $results_by_job)
+function _config_merge_update_results(ConfigCacheUpdateParams $params, array $results_by_job) : ConfigCacheUpdateResult
 {
+  $result = new ConfigCacheUpdateResult($params);
+
   $total_fast_jsons = 0;
   
   foreach($results_by_job as $results)
@@ -459,9 +453,11 @@ function _config_merge_update_results(ConfigUpdateResult $result, array $results
   }
 
   $result->fast_jsons = $total_fast_jsons;
+
+  return $result;
 }
 
-function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $file, string $cache_file, 
+function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file, 
   ?int &$parser_type = null) : ConfigCacheEntry
 {
   list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
@@ -473,7 +469,7 @@ function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $f
   $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id; 
   $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
 
-  $pres = config_parse($globals->base_dirs, $file);
+  $pres = config_parse($params->globals->base_dirs, $file);
   if($pres->error !== 0)
     throw new Exception("Parse error({$pres->error}):\n" . $pres->error_descr);
 
@@ -483,9 +479,11 @@ function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $f
   $parsed_arr['id'] = $conf_id;
   $parsed_arr['strid'] = $conf_strid;
   list($class, $class_id, $normalized_data) 
-    = config_apply_class_normalization($parsed_arr, check_junk: true);
+    = config_apply_class_normalization($parsed_arr, $params->check_junk);
 
-  $cache_payload_file = config_get_cache_payload_path($globals, $file);
+  $cache_payload_file = config_get_cache_payload_path($params->globals, $file);
+
+  $cache_file = config_get_cache_path($params->globals, $file);
 
   $cache_entry = new ConfigCacheEntry();
   $cache_entry->id = $conf_id;
@@ -503,8 +501,8 @@ function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $f
 
   ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
   ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
-  ensure_write(config_get_cache_id_path($globals, $conf_id), $cache_entry->file);
+  ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
-  ensure_write(config_get_cache_strid_path($globals, $conf_strid), $cache_entry->file);
+  ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
 
   return $cache_entry;
 }
@@ -555,12 +553,12 @@ function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job,
       $cache_file = config_get_cache_path($params->globals, $file);
 
       $parser_type = null;
-      $entry = _config_update_cache_entry($params->globals, $base_dir, $file, $cache_file, $parser_type);
+      $entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
 
       $results[] = [
         'base_dir' => $base_dir, 
         'file' => $file, 
-        'cache_file' => $cache_file, 
+        'cache_file' => $entry->cache_file, 
         'parser_type' => $parser_type, 
         'error' => null
       ];

config.inc.php

@@ -11,30 +11,16 @@ require_once(__DIR__ . '/msgpack.inc.php');
 
 class ConfigGlobals
 {
-  public readonly array $base_dirs;
+  public array $base_dirs = array();
-  //NOTE: it's a string since is serialized
+  public ?string $worker_init_fn = null;
-  public readonly ?string $worker_init_fn;
+  public string $base_class = '\ConfBase';
-  public readonly int $workers_num;
+  public string $build_dir;
-  public readonly string $base_class;
-  public readonly string $build_dir;
-  //NOTE: it's a string since is serialized
-  public readonly ?string $files_filter; 
 
-  function __construct(
+  function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null)
-    array $base_dirs, 
-    string $build_dir, 
-    ?string $worker_init_fn = null,
-    int $workers_num = 1,
-    string $base_class = '\ConfBase',
-    ?string $files_filter = null
-  )
   {
     $this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
     $this->build_dir = $build_dir;
     $this->worker_init_fn = $worker_init_fn;
-    $this->workers_num = $workers_num;
-    $this->base_class = $base_class;
-    $this->files_filter = $files_filter;
   }
 
   function initWorker(bool $is_master_proc)
@@ -49,105 +35,59 @@ class ConfigGlobals
   }
 }
 
-enum ConfigGetMode : int
-{
-  case ForceUpdate   = 1;
-  case Auto          = 2;
-  case CacheOnly     = 3;
-}
-
 enum ConfigUpdateMode : int
 {
   case Force         = 1;
   case DetectChanged = 2;
-  case Selected      = 3;
+  case Patch         = 3;
 }
 
 class ConfigUpdateRequest
 {
   public ConfigUpdateMode $mode;
   public ?ConfigDirFiles $files;
-  public ?\taskman\TaskmanFileChanges $file_changes;
+  public ?string $result_file;
-  public bool $verbose = false;
-  public bool $return_affected = false;
 
   private function __construct() {}
 
-  static function force(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
+  static function force(?ConfigDirFiles $files = null) : ConfigUpdateRequest
   {
-    if($files instanceof \taskman\artefact\TaskmanDirFiles)
-      $files = ConfigDirFiles::makeFromArtefactFiles($files);
-
     $req = new ConfigUpdateRequest();
     $req->mode = ConfigUpdateMode::Force;
     $req->files = $files;
     return $req;
   }
 
-  static function detectChanged(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
+  static function detectChanged(string $result_file, ?ConfigDirFiles $files = null) : ConfigUpdateRequest
   {
-    if($files instanceof \taskman\artefact\TaskmanDirFiles)
-      $files = ConfigDirFiles::makeFromArtefactFiles($files);
-
     $req = new ConfigUpdateRequest();
     $req->mode = ConfigUpdateMode::DetectChanged;
     $req->files = $files;
+    $req->result_file = $result_file;
     return $req;
   }
 
-  static function selected(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles $files, \taskman\TaskmanFileChanges|null $file_changes = null) : ConfigUpdateRequest
+  static function patch(ConfigDirFiles $files) : ConfigUpdateRequest
   {
-    if($files instanceof \taskman\artefact\TaskmanDirFiles)
-      $files = ConfigDirFiles::makeFromArtefactFiles($files);
-
     $req = new ConfigUpdateRequest();
-    $req->mode = ConfigUpdateMode::Selected;
+    $req->mode = ConfigUpdateMode::Patch;
     $req->files = $files;
-    $req->file_changes = $file_changes;
     return $req;
   }
 }
 
-class ConfigUpdateResult
-{
-  public ConfigUpdateRequest $request;
-
-  public ConfigDirFiles $affected_files;
-  public array $affected_entries = array();
-
-  public array $added_files = array();
-  public array $removed_files = array();
-
-  /** @var array<string, string> */
-  public array $errors = array();
-  public int $corruptions = 0;
-  public int $fast_jsons = 0;
-
-  function isEmpty() : bool
-  {
-    return count($this->affected_files) == 0 && 
-      count($this->added_files) == 0 && 
-      count($this->removed_files) == 0;
-  }
-
-  function isPatchPossible() : bool
-  {
-    return ($this->request->mode == ConfigUpdateMode::Selected ||
-            $this->request->mode === ConfigUpdateMode::DetectChanged) && 
-            count(array_filter($this->added_files, fn($f) => config_is_file($f))) == 0 &&
-            count(array_filter($this->removed_files, fn($f) => config_is_file($f))) == 0;
-  }
-}
-
 class ConfigManager
 {
   private ConfigGlobals $globals; 
+  private int $workers_num;
+
   private ConfigCache $cache;
   private ?ConfigCacheFileMap $file_map;
 
-  function __construct(ConfigGlobals $globals)
+  function __construct(ConfigGlobals $globals, int $workers_num)
   {
     $this->globals = $globals;
+    $this->workers_num = $workers_num;
 
     $this->cache = new ConfigCache($globals);
     $this->file_map = null;
@@ -163,7 +103,7 @@ class ConfigManager
     return $this->cache;
   }
 
-  function getArtefactFilesSpec() : array
+  function getArtifactFilesSpec() : array
   {
     return [$this->globals->base_dirs, ['.js']];
   }
@@ -175,124 +115,63 @@ class ConfigManager
       $this->file_map = $this->_tryLoadMap();
       if($this->file_map === null)
       {
-        $this->file_map = self::_makeMap($this->scanFiles(extension: '.js'));
+        $this->file_map = self::_makeMap($files ?? $this->scanFiles(extension: '.js'));
         $this->_saveFileMap();
       }
     }
     return $this->file_map;
   }
 
-  function fetchAll(bool $force = false) : \Generator
+  function updateCache(ConfigUpdateRequest $req, bool $return_entries = false, bool $verbose = false) : ConfigCacheUpdateResult
   {
-    $result = $this->updateAll($force);
+    config_log("Updating cache, mode '{$req->mode->name}'...");
-    $filtered = $this->filterConfigFiles($result->request->files);
-    return $this->iterateCache($filtered);
-  }
-
-  function fetchByPath(string $path) : ConfigCacheEntry
-  {
-    $path = realpath($path);
-    $dir_files = ConfigDirFiles::makeFor($this);
-    $dir_files->addFile($path);
-    $request = ConfigUpdateRequest::selected($dir_files);
-    $request->return_affected = true;
-    $result = $this->update($request);
-    return $result->affected_entries[0];
-  }
-
-  function iterateCache(iterable $files) : \Generator
-  {
-    foreach($files as $file)
-    {
-      $ce = $this->cache->getOrLoadByPath($file);
-      yield $ce;
-    }
-  }
-
-  function update(ConfigUpdateRequest $req) : ConfigUpdateResult
-  {
-    config_log("Updating configs, mode '{$req->mode->name}'...");
 
     if($req->files === null)
-      $req->files = $this->scanFiles(extension: '.js', verbose: $req->verbose);
+      $req->files = $this->scanFiles(extension: '.js', verbose: $verbose);
 
     $added_files = [];
     $removed_files = [];
     $this->_checkFileMap($req, $added_files, $removed_files);
 
-    $affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
+    $affected_files = $this->_getAffectedFiles($req, $removed_files);
-    $affected_conf_files = $this->filterConfigFiles($affected_all_files);
 
-    config_log("Affected configs: {$affected_conf_files->count()} (requested: {$req->files->count()})");
+    //NOTE: at this poine taking into account only config files
+    $affected_files->filter(fn($file) => str_ends_with($file, '.conf.js'));
 
-    $update_result = new ConfigUpdateResult();
+    config_log("Affected files: {$affected_files->count()}");
-    $update_result->request = $req;
-    $update_result->affected_files = $affected_conf_files;
-    $update_result->added_files = $added_files;
-    $update_result->removed_files = $removed_files;
 
     $update_params = new ConfigCacheUpdateParams(
       globals: $this->globals, 
-      affected_files: $affected_conf_files, 
+      affected_files: $affected_files, 
-      verbose: $req->verbose
+      verbose: $verbose, 
+      max_workers: $this->workers_num
     );
-    self::_updateCache($update_result, $update_params);
+    $update_result = self::_updateCache($update_params);
 
-    //let's clear internal cache once the update procedure is done
     $this->cache->clear();
 
-    $this->_updateFileMap($req, $affected_conf_files, $added_files, $removed_files);
+    $this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
 
-    if($req->return_affected)
+    if($return_entries)
     {
-      foreach($affected_conf_files as $file)
+      foreach($affected_files as $file)
       {
         $entry = $this->cache->getOrLoadByPath($file);
         $update_result->affected_entries[] = $entry;
       }
     }
 
-    touch($this->_getLastUpdateFile());
-
     return $update_result;
   }
 
-  function filterConfigFiles(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles 
+  private function _updateCache(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
-  {
-    if($files instanceof \taskman\artefact\TaskmanDirFiles)
-      $files = ConfigDirFiles::makeFromArtefactFiles($files);
-
-    $filtered = $this->globals->files_filter != null ? 
-      call_user_func($this->globals->files_filter, $files) : 
-      new ConfigDirFiles($files->getMap());
-    
-    $filtered->filter(fn($file) => config_is_file($file));
-    return $filtered;
-  }
-
-  function updateAll(bool $force = false) : ConfigUpdateResult
-  {
-    $files = $this->scanFiles(extension: '.js');
-    $result = $this->update($force ? 
-      ConfigUpdateRequest::force($files) : 
-      ConfigUpdateRequest::detectChanged($files)
-    );
-    return $result;
-  }
-
-  function _getLastUpdateFile() : string
-  {
-    return $this->globals->build_dir . '/update.last';
-  }
-
-  private function _updateCache(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
   {
     if($params->affected_files->isEmpty())
-      return;
+      return new ConfigCacheUpdateResult($params);
 
     $t = microtime(true);
 
-    _config_cache_update($result, $params);
+    $result = _config_cache_update($params);
 
     if($result->errors)
     {
@@ -309,6 +188,8 @@ class ConfigManager
 
     if($params->verbose)
       config_log("Update cache: " . round(microtime(true) - $t,2) . " sec.");
+
+    return $result;
   }
 
   private function _checkFileMap(ConfigUpdateRequest $req, array &$added_files, array &$removed_files)
@@ -317,52 +198,39 @@ class ConfigManager
 
     if($req->mode === ConfigUpdateMode::Force)
     {
+      $added_files = $req->files->getAllFiles();
       //let's rebuild the file map
-      $fs_cache_map->init($req->files->getAllFiles());
+      $fs_cache_map->init($added_files);
-      config_log("File map init: ".$fs_cache_map->count());
+      config_log("File map init: ".count($added_files));
     }
     else if($req->mode === ConfigUpdateMode::DetectChanged)
     {
       list($added_files, $removed_files) = $fs_cache_map->compare($req->files->getAllFiles());
       config_log("File map compare, added: ".count($added_files).", removed: ".count($removed_files));
     }
-    else if($req->mode === ConfigUpdateMode::Selected)
-    {
-      if($req->file_changes != null)
-      {
-        foreach($req->files as $file)
-        {
-          if($req->file_changes->isCreated($file))
-            $added_files[] = $file;
-          else if($req->file_changes->isDeleted($file))
-            $removed_files[] = $file;
-        }
-      }
-    }
   }
 
-  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_conf_files, array $added_files, array $removed_files)
+  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
   {
     $fs_cache_map = $this->getFileMap();
 
-    //TODO: traverse all affected config files and update file map
+    //TODO: traverse all affected files and update file map
-    foreach($affected_conf_files as $file)
+    foreach($affected_files as $file)
     {
       $cache_entry = $this->cache->getOrLoadByPath($file);
-      $fs_cache_map->updateDepsForEntry($this->globals->base_dirs, $cache_entry);
+      $fs_cache_map->updateDepsForEntry($cache_entry);
     }
 
-    if($req->mode == ConfigUpdateMode::Force || 
+    if($affected_files->count() > 0 || $added_files || $removed_files)
-      $affected_conf_files->count() > 0 || 
-      $added_files || 
-      $removed_files)
     {
-      $fs_cache_map->update($added_files, $removed_files);
+      //in case of Force map was already cleared and initialized
+      if($req->mode != ConfigUpdateMode::Force)
+        $fs_cache_map->update($added_files, $removed_files);
       $this->_saveFileMap();
     }
   }
 
-  private function _getAffectedFiles(ConfigUpdateRequest $req, array $added_files, array $removed_files) : ConfigDirFiles
+  private function _getAffectedFiles(ConfigUpdateRequest $req, array $removed_files) : ConfigDirFiles
   {
     $fs_cache_map = $this->getFileMap();
 
@@ -370,19 +238,17 @@ class ConfigManager
 
     if($req->mode === ConfigUpdateMode::Force)
     {
-      $affected_files = new ConfigDirFiles($req->files->getMap());
+      $affected_files = $req->files;
     }
     else if($req->mode === ConfigUpdateMode::DetectChanged)
     {
       $affected_files = ConfigDirFiles::makeFor($this);
 
-      $last_run_file = $this->_getLastUpdateFile();
-
       foreach($req->files->getMap() as $base_dir => $files)
       {
         foreach($files as $file)
         {
-          if(need_to_regen($last_run_file, [$file]))
+          if(need_to_regen($req->result_file, [$file]))
           {
             $affected_files->add($base_dir, $file);
 
@@ -396,21 +262,32 @@ class ConfigManager
       //if there were removed files we need to rebuild affected files
       foreach($removed_files as $file)
       {
-        $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+        if(!str_ends_with($file, '.conf.js'))
-        foreach($affected_by_file as $dep)
+        {
-          $affected_files->addFile($dep, unique: true);
+          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+          foreach($affected_by_file as $dep)
+            $affected_files->addFile($dep, unique: true);
+        }
+        else
+        {
+          //TODO: in case config file was removed do we actually need to rebuild all configs?
+        }
       }
     }
-    else if($req->mode === ConfigUpdateMode::Selected)
+    else if($req->mode === ConfigUpdateMode::Patch)
     {
       $affected_files = ConfigDirFiles::makeFor($this);
 
       foreach($req->files as $file)
       {
         $affected_files->addFile($file, unique: true);
-        $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+
-        foreach($affected_by_file as $dep)
+        if(!str_ends_with($file, '.conf.js'))
-          $affected_files->addFile($dep, unique: true);
+        {
+          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+          foreach($affected_by_file as $dep)
+            $affected_files->addFile($dep, unique: true);
+        }
       }
     }
 
@@ -436,11 +313,8 @@ class ConfigManager
   {
     if(!is_file($this->_getMapPath()))
       return null;
-    $t = microtime(true);
+    config_log("Loading file map");
-    $data = ensure_read($this->_getMapPath());
+    return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
-    $map = ConfigCacheFileMap::unserialize($data);
-    config_log("Loaded file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
-    return $map;
   }
 
   private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
@@ -453,10 +327,9 @@ class ConfigManager
 
   private function _saveFileMap()
   {
-    $t = microtime(true);
     $data = ConfigCacheFileMap::serialize($this->getFileMap()); 
+    config_log("Saving file map: " . kb($data));
     ensure_write($this->_getMapPath(), $data);
-    config_log("Saved file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
   }
 }
 

pack.inc.php

@@ -42,12 +42,10 @@ function config_pack_bundle(ConfigPackParams $params) : string
 
   $packed_data = null;
 
-   $cache_entries = iterator_to_array($params->cache_entries);
-
   if($params->binary_format == 1)
   {
     $packed_data = _config_pack_bundle_fmt1(
-      $cache_entries, 
+      $params->cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
       $params->version
@@ -56,7 +54,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
   else if($params->binary_format == 2)
   {
     $packed_data = _config_pack_bundle_fmt2(
-      $cache_entries, 
+      $params->cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
       $params->version,
@@ -65,7 +63,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
   else if($params->binary_format == 3)
   {
     $packed_data = _config_pack_bundle_fmt3(
-      $cache_entries, 
+      $params->cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
       $params->version,
@@ -77,7 +75,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
     throw new Exception("Unknown binary format: {$params->binary_format}");
 
   if($params->debug)
-    config_log("Packed entries: " . count($cache_entries) . ", total: " . 
+    config_log("Packed entries: " . sizeof($params->cache_entries) . ", total: " . 
       kb($packed_data) . ", format: {$params->binary_format}, lz4: " . 
       var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
       ", CRC: " . crc32($packed_data) . 
@@ -102,16 +100,6 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
       $params->version,
     );
   }
-  else if($params->binary_format == 3)
-  {                  
-    $patched_data = _config_patch_bundle_fmt3(
-      $packed_data,
-      $params->cache_entries, 
-      $params->use_lz4,
-      $params->use_config_refs,
-      $params->version,
-    );
-  }
   else
     throw new Exception("Unknown binary format: {$params->binary_format}");
 
@@ -127,7 +115,7 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
 
 //NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
 function _config_pack_bundle_fmt1(
-  array $cache_entries, 
+  iterable $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
   int $version) : string
@@ -185,7 +173,7 @@ function _config_pack_bundle_fmt1(
 //NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
 //      (each path item separated by '/' is stored as an array item)
 function _config_pack_bundle_fmt2(
-  array $cache_entries, 
+  iterable $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
   int $version,
@@ -246,7 +234,7 @@ function _config_pack_bundle_fmt2(
   return $packed_data;
 }
 
-//NOTE: Much like fmt1, but configs entries are grouped into sizeable chunks, with each chunk lz4-compressed.
+//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
 //      This reduces overall bundle size when there are many small configs entries.
 function _config_pack_bundle_fmt3(
   array $cache_entries, 
@@ -299,7 +287,7 @@ function _config_pack_bundle_fmt3(
 
     $payloads_offset += $payload_size;
     
-    if($payloads_offset >= $chunk_size || $idx == ($count_entries - 1))
+    if($payloads_offset >= $chunk_size || $idx == ($count_entries - 1))  
     {
       if($payloads_offset > $max_chunk_size)
         $max_chunk_size = $payloads_offset;
@@ -361,12 +349,12 @@ function _config_patch_bundle_fmt2(
   {
     list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
 
-    $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+    $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
 
-    if($headers_found)
+    if($header_found)
     {
-      $header_idx = key($headers_found);
+      $header_idx = key($header_found);
-      $header_entry = current($headers_found);
+      $header_entry = current($header_found);
 
       $current_offset = $header_entry[4];
       $current_size = $header_entry[5];
@@ -424,97 +412,6 @@ function _config_patch_bundle_fmt2(
   return $patched_data;
 }
 
-function _config_patch_bundle_fmt3(
-  string $packed_data,
-  array $patch_entries, 
-  bool $use_lz4, 
-  bool $use_config_refs, 
-  int $version,
-) : string
-{
-  if(!$use_lz4)
-    throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
-
-  list($header, $entries, $chunks_offset, $max_chunk_size) = 
-    _config_unpack_bundle_fmt3(packed_data: $packed_data, unpack_entries: false);
-
-  $payloads_bundle = substr($packed_data, $chunks_offset);
-
-  foreach($patch_entries as $idx => $patch_entry)
-  {
-    list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
-
-    $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
-
-    if($headers_found)
-    {
-      $header_idx = key($headers_found);
-      $header_entry = current($headers_found);
-
-      $chunk_offset = $header_entry[4];
-      $payloads_offset = $header_entry[5]; //within chunk
-      $payloads_size = $header_entry[6];
-
-      $chunk_entries = array_filter($header, fn($item) => $item[4] == $chunk_offset); 
-
-      //chunk contains only one patch entry
-      $patch_payload_lz4 = lz4_compress($patch_payload, 9); 
-      $patched_chunk = pack("V", strlen($patch_payload_lz4));
-      $patched_chunk .= $patch_payload_lz4;
-      if(strlen($patch_payload) > $max_chunk_size)
-        $max_chunk_size = strlen($patch_payload);
-
-      //TODO: make a better generic algorithm for patching the whole chunk
-      //special case for single entry 
-      if(count($chunk_entries) == 1 && $payloads_offset == 0)
-      {
-        $lz4_chunk_size = unpack("V", substr($payloads_bundle, $chunk_offset, 4))[1];
-
-        //we can just replace chunk in place if the size of a patched chunk is less or equal
-        if(strlen($patch_payload_lz4) <= $lz4_chunk_size)
-        {
-          $payloads_bundle = substr_replace($payloads_bundle, $patched_chunk, $chunk_offset, strlen($patched_chunk));
-          $header_entry[6] = strlen($patch_payload);
-        }
-        //just append to the end
-        else
-        {
-          $header_entry[4] = strlen($payloads_bundle); //chunk offset
-          $header_entry[5] = 0;
-          $header_entry[6] = strlen($patch_payload);
-
-          $payloads_bundle .= $patched_chunk;
-        }
-      }
-      //just append to the end
-      else
-      {
-        $header_entry[4] = strlen($payloads_bundle); //chunk offset
-        $header_entry[5] = 0;
-        $header_entry[6] = strlen($patch_payload);
-
-        $payloads_bundle .= $patched_chunk;
-      }
-
-      $header[$header_idx] = $header_entry; 
-    }
-    else
-      throw new Exception("Patched entry {$patch_entry->id} not found in config bundle");
-  }
-
-  $header_msgpack = config_msgpack_pack($header);
-
-  $packed_data =  
-         pack("C", 3) . 
-         pack("V", $version) . 
-         pack("V", strlen($header_msgpack)) .
-         pack("V", $max_chunk_size) .
-         $header_msgpack .
-         $payloads_bundle;
-
-  return $packed_data;
-}
-
 //format: [[class_id, [data]], ...[class_id, [data]]]
 function config_unpack_bundle(string $packed_data) : array
 {
@@ -532,8 +429,7 @@ function config_unpack_bundle(string $packed_data) : array
   }
   else if($info['format'] === 3)
   {
-    list($_, $entries) = _config_unpack_bundle_fmt3($packed_data);
+    return _config_unpack_bundle_fmt3($packed_data);
-    return $entries;
   }
   else
     throw new Exception("Unknown format: {$info['format']}");
@@ -600,10 +496,12 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries =
   return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
 }
 
-function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries = true): array
+function _config_unpack_bundle_fmt3(string $packed_data): array
 {
   if(ord($packed_data[0]) !== 3)
+  {
     throw new Exception("Invalid config bundle format");
+  }
 
   $offset = 1;
   $version = unpack("V", substr($packed_data, $offset, 4))[1];
@@ -615,7 +513,6 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
 
   $header_msgpack = substr($packed_data, $offset, $header_len);
   $offset += $header_len;
-  $chunks_offset = $offset;
 
   $header = config_msgpack_unpack($header_msgpack);
 
@@ -624,45 +521,39 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
   $chunk_buffer = '';
   $chunk_id = -1;
 
-  foreach($header as $entry_data)
+  foreach ($header as $entry_data)
   {
     list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
 
-    $unpacked_payload = null;
+    if($entry_chunk_offset !== $chunk_id)
-
-    if($unpack_entries)
     {
-      if($entry_chunk_offset !== $chunk_id)
+      if($chunk_offset !== -1)
       {
-        if($chunk_offset !== -1)
+        $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
-        {
+        $offset+=4;
-          $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
+        $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
-          $offset+=4;
+        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
-          $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
+        $offset += $lz4_chunk_size;
-          $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+        $chunk_offset = $offset;
-          $offset += $lz4_chunk_size;
+      }
-          $chunk_offset = $offset;
+      else
-        }
+      {
-        else
+        $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
-        {
+        $chunk_offset += 4;
-          $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
+        $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
-          $chunk_offset += 4;
+        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
-          $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
+        $chunk_offset += $lz4_chunk_size;
-          $chunk_buffer = lz4_uncompress($lz4_chunk_data);
-          $chunk_offset += $lz4_chunk_size;
-        }
-
-        $chunk_id = $entry_chunk_offset;
       }
 
-      $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
+      $chunk_id = $entry_chunk_offset;
-      $unpacked_payload = _config_unpack_payload($format, $payload);
     }
 
-    $cache_entries[$id] = array($class_id, $unpacked_payload); 
+    $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
+
+    $cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
   }
 
-  return array($header, $cache_entries, $chunks_offset, $max_chunk_size);
+  return $cache_entries;
 }
 
 //format: [format_id, payload_data]

util.inc.php

@@ -28,17 +28,15 @@ function config_real_path(array $base_dirs, string $rel_path, bool $strict = tru
 {
   foreach($base_dirs as $dir)
     if(is_file($dir . '/' . $rel_path))
-      return normalize_path($dir . '/' . $rel_path);
+      return $dir . '/' . $rel_path;
-
   if($strict)
     throw new Exception("No file for relative path '$rel_path'");
-
   return null;
 }
 
 function config_is_file(string $filename) : bool
 {
-  return str_ends_with($filename, '.conf.js');
+  return (strrpos($filename, '.conf.js') === (strlen($filename) - 8));
 }
 
 function config_file2id(string $conf_dir, string $filename) : int
@@ -106,7 +104,6 @@ function config_walk_fields(object $proto, callable $callback)
   }
 }
 
-//TODO: doesn't really belong here
 function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
 {
   if(!isset($map[$file]))