@foo/bar refs are now taken into account during affected files detection

cache.inc.php

@@ -325,14 +325,24 @@ class ConfigCacheFileMap
       unset($this->file2deps[$file]);
   }
 
-  function updateDepsForEntry(ConfigCacheEntry $entry)
+  function updateDepsForEntry(array $base_dirs, ConfigCacheEntry $entry)
   {
-    foreach($entry->includes as $include)
+    //1. add includes
+    $all_refs = $entry->includes;
+
+    //2. add refs
+    foreach($entry->refs as $ref => $_)
     {
-      if(isset($this->file2deps[$include]))
+      $file = config_real_path($base_dirs, substr($ref, 1) . '.conf.js');
-        $this->file2deps[$include][$entry->file] = true;
+      $all_refs[] = $file;
+    }
+
+    foreach($all_refs as $ref_file)
+    {
+      if(isset($this->file2deps[$ref_file]))
+        $this->file2deps[$ref_file][$entry->file] = true;
       else
-        $this->file2deps[$include] = [$entry->file => true];
+        $this->file2deps[$ref_file] = [$entry->file => true];
     }
   }
 

config.inc.php

@@ -18,7 +18,7 @@ class ConfigGlobals
   public readonly string $base_class;
   public readonly string $build_dir;
   //NOTE: it's a string since is serialized
-  public readonly ?string $files_overrider; 
+  public readonly ?string $files_filter; 
 
   function __construct(
     array $base_dirs, 
@@ -26,7 +26,7 @@ class ConfigGlobals
     ?string $worker_init_fn = null,
     int $workers_num = 1,
     string $base_class = '\ConfBase',
-    ?string $files_overrider = null
+    ?string $files_filter = null
   )
   {
     $this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
@@ -34,7 +34,7 @@ class ConfigGlobals
     $this->worker_init_fn = $worker_init_fn;
     $this->workers_num = $workers_num;
     $this->base_class = $base_class;
-    $this->files_overrider = $files_overrider;
+    $this->files_filter = $files_filter;
   }
 
   function initWorker(bool $is_master_proc)
@@ -123,6 +123,13 @@ class ConfigUpdateResult
   public int $corruptions = 0;
   public int $fast_jsons = 0;
 
+  function isEmpty() : bool
+  {
+    return count($this->affected_files) == 0 && 
+      count($this->added_files) == 0 && 
+      count($this->removed_files) == 0;
+  }
+
   function isPatchPossible() : bool
   {
     return ($this->request->mode == ConfigUpdateMode::Selected ||
@@ -178,7 +185,8 @@ class ConfigManager
   function fetchAll(bool $force = false) : \Generator
   {
     $result = $this->updateAll($force);
-    return $this->iterateCache($result->request->files);
+    $filtered = $this->filterConfigFiles($result->request->files);
+    return $this->iterateCache($filtered);
   }
 
   function fetchByPath(string $path) : ConfigCacheEntry
@@ -196,11 +204,8 @@ class ConfigManager
   {
     foreach($files as $file)
     {
-      if(config_is_file($file))
+      $ce = $this->cache->getOrLoadByPath($file);
-      {
+      yield $ce;
-        $ce = $this->cache->getOrLoadByPath($file);
-        yield $ce;
-      }
     }
   }
 
@@ -215,22 +220,20 @@ class ConfigManager
     $removed_files = [];
     $this->_checkFileMap($req, $added_files, $removed_files);
 
-    $affected_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
+    $affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
+    $affected_conf_files = $this->filterConfigFiles($affected_all_files);
 
-    //NOTE: at this point taking into account only config files
+    config_log("Affected configs: {$affected_conf_files->count()} (requested: {$req->files->count()})");
-    $affected_files->filter(fn($file) => config_is_file($file));
-
-    config_log("Affected files: {$affected_files->count()}");
 
     $update_result = new ConfigUpdateResult();
     $update_result->request = $req;
-    $update_result->affected_files = $affected_files;
+    $update_result->affected_files = $affected_conf_files;
     $update_result->added_files = $added_files;
     $update_result->removed_files = $removed_files;
 
     $update_params = new ConfigCacheUpdateParams(
       globals: $this->globals, 
-      affected_files: $affected_files, 
+      affected_files: $affected_conf_files, 
       verbose: $req->verbose
     );
     self::_updateCache($update_result, $update_params);
@@ -238,11 +241,11 @@ class ConfigManager
     //let's clear internal cache once the update procedure is done
     $this->cache->clear();
 
-    $this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
+    $this->_updateFileMap($req, $affected_conf_files, $added_files, $removed_files);
 
     if($req->return_affected)
     {
-      foreach($affected_files as $file)
+      foreach($affected_conf_files as $file)
       {
         $entry = $this->cache->getOrLoadByPath($file);
         $update_result->affected_entries[] = $entry;
@@ -254,6 +257,19 @@ class ConfigManager
     return $update_result;
   }
 
+  function filterConfigFiles(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles 
+  {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
+    $filtered = $this->globals->files_filter != null ? 
+      call_user_func($this->globals->files_filter, $files) : 
+      new ConfigDirFiles($files->getMap());
+    
+    $filtered->filter(fn($file) => config_is_file($file));
+    return $filtered;
+  }
+
   function updateAll(bool $force = false) : ConfigUpdateResult
   {
     $files = $this->scanFiles(extension: '.js');
@@ -325,19 +341,19 @@ class ConfigManager
     }
   }
 
-  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
+  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_conf_files, array $added_files, array $removed_files)
   {
     $fs_cache_map = $this->getFileMap();
 
-    //TODO: traverse all affected files and update file map
+    //TODO: traverse all affected config files and update file map
-    foreach($affected_files as $file)
+    foreach($affected_conf_files as $file)
     {
       $cache_entry = $this->cache->getOrLoadByPath($file);
-      $fs_cache_map->updateDepsForEntry($cache_entry);
+      $fs_cache_map->updateDepsForEntry($this->globals->base_dirs, $cache_entry);
     }
 
     if($req->mode == ConfigUpdateMode::Force || 
-      $affected_files->count() > 0 || 
+      $affected_conf_files->count() > 0 || 
       $added_files || 
       $removed_files)
     {
@@ -354,7 +370,7 @@ class ConfigManager
 
     if($req->mode === ConfigUpdateMode::Force)
     {
-      $affected_files = $req->files;
+      $affected_files = new ConfigDirFiles($req->files->getMap());
     }
     else if($req->mode === ConfigUpdateMode::DetectChanged)
     {
@@ -380,12 +396,9 @@ class ConfigManager
       //if there were removed files we need to rebuild affected files
       foreach($removed_files as $file)
       {
-        if(!config_is_file($file))
+        $affected_by_file = $fs_cache_map->getAffectedFiles($file);
-        {
+        foreach($affected_by_file as $dep)
-          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
+          $affected_files->addFile($dep, unique: true);
-          foreach($affected_by_file as $dep)
-            $affected_files->addFile($dep, unique: true);
-        }
       }
     }
     else if($req->mode === ConfigUpdateMode::Selected)
@@ -395,13 +408,9 @@ class ConfigManager
       foreach($req->files as $file)
       {
         $affected_files->addFile($file, unique: true);
-
+        $affected_by_file = $fs_cache_map->getAffectedFiles($file);
-        if(!config_is_file($file))
+        foreach($affected_by_file as $dep)
-        {
+          $affected_files->addFile($dep, unique: true);
-          $affected_by_file = $fs_cache_map->getAffectedFiles($file);
-          foreach($affected_by_file as $dep)
-            $affected_files->addFile($dep, unique: true);
-        }
       }
     }
 
@@ -415,10 +424,7 @@ class ConfigManager
 
   function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
   {
-    $res = config_scan_files($this->globals->base_dirs, $extension, $verbose);
+    return config_scan_files($this->globals->base_dirs, $extension, $verbose);
-    if($this->globals->files_overrider != null)
-      $res = call_user_func($this->globals->files_overrider, $res);
-    return $res;
   }
 
   private function _getMapPath() : string
@@ -430,8 +436,11 @@ class ConfigManager
   {
     if(!is_file($this->_getMapPath()))
       return null;
-    config_log("Loading file map");
+    $t = microtime(true);
-    return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
+    $data = ensure_read($this->_getMapPath());
+    $map = ConfigCacheFileMap::unserialize($data);
+    config_log("Loaded file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
+    return $map;
   }
 
   private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
@@ -444,9 +453,10 @@ class ConfigManager
 
   private function _saveFileMap()
   {
+    $t = microtime(true);
     $data = ConfigCacheFileMap::serialize($this->getFileMap()); 
-    config_log("Saving file map: " . kb($data));
     ensure_write($this->_getMapPath(), $data);
+    config_log("Saved file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
   }
 }
 

pack.inc.php

@@ -102,6 +102,16 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
       $params->version,
     );
   }
+  else if($params->binary_format == 3)
+  {                  
+    $patched_data = _config_patch_bundle_fmt3(
+      $packed_data,
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version,
+    );
+  }
   else
     throw new Exception("Unknown binary format: {$params->binary_format}");
 
@@ -236,7 +246,7 @@ function _config_pack_bundle_fmt2(
   return $packed_data;
 }
 
-//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
+//NOTE: Much like fmt1, but configs entries are grouped into sizeable chunks, with each chunk lz4-compressed.
 //      This reduces overall bundle size when there are many small configs entries.
 function _config_pack_bundle_fmt3(
   array $cache_entries, 
@@ -351,12 +361,12 @@ function _config_patch_bundle_fmt2(
   {
     list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
 
-    $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+    $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
 
-    if($header_found)
+    if($headers_found)
     {
-      $header_idx = key($header_found);
+      $header_idx = key($headers_found);
-      $header_entry = current($header_found);
+      $header_entry = current($headers_found);
 
       $current_offset = $header_entry[4];
       $current_size = $header_entry[5];
@@ -414,6 +424,97 @@ function _config_patch_bundle_fmt2(
   return $patched_data;
 }
 
+function _config_patch_bundle_fmt3(
+  string $packed_data,
+  array $patch_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version,
+) : string
+{
+  if(!$use_lz4)
+    throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
+
+  list($header, $entries, $chunks_offset, $max_chunk_size) = 
+    _config_unpack_bundle_fmt3(packed_data: $packed_data, unpack_entries: false);
+
+  $payloads_bundle = substr($packed_data, $chunks_offset);
+
+  foreach($patch_entries as $idx => $patch_entry)
+  {
+    list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
+
+    $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+
+    if($headers_found)
+    {
+      $header_idx = key($headers_found);
+      $header_entry = current($headers_found);
+
+      $chunk_offset = $header_entry[4];
+      $payloads_offset = $header_entry[5]; //within chunk
+      $payloads_size = $header_entry[6];
+
+      $chunk_entries = array_filter($header, fn($item) => $item[4] == $chunk_offset); 
+
+      //chunk contains only one patch entry
+      $patch_payload_lz4 = lz4_compress($patch_payload, 9); 
+      $patched_chunk = pack("V", strlen($patch_payload_lz4));
+      $patched_chunk .= $patch_payload_lz4;
+      if(strlen($patch_payload) > $max_chunk_size)
+        $max_chunk_size = strlen($patch_payload);
+
+      //TODO: make a better generic algorithm for patching the whole chunk
+      //special case for single entry 
+      if(count($chunk_entries) == 1 && $payloads_offset == 0)
+      {
+        $lz4_chunk_size = unpack("V", substr($payloads_bundle, $chunk_offset, 4))[1];
+
+        //we can just replace chunk in place if the size of a patched chunk is less or equal
+        if(strlen($patch_payload_lz4) <= $lz4_chunk_size)
+        {
+          $payloads_bundle = substr_replace($payloads_bundle, $patched_chunk, $chunk_offset, strlen($patched_chunk));
+          $header_entry[6] = strlen($patch_payload);
+        }
+        //just append to the end
+        else
+        {
+          $header_entry[4] = strlen($payloads_bundle); //chunk offset
+          $header_entry[5] = 0;
+          $header_entry[6] = strlen($patch_payload);
+
+          $payloads_bundle .= $patched_chunk;
+        }
+      }
+      //just append to the end
+      else
+      {
+        $header_entry[4] = strlen($payloads_bundle); //chunk offset
+        $header_entry[5] = 0;
+        $header_entry[6] = strlen($patch_payload);
+
+        $payloads_bundle .= $patched_chunk;
+      }
+
+      $header[$header_idx] = $header_entry; 
+    }
+    else
+      throw new Exception("Patched entry {$patch_entry->id} not found in config bundle");
+  }
+
+  $header_msgpack = config_msgpack_pack($header);
+
+  $packed_data =  
+         pack("C", 3) . 
+         pack("V", $version) . 
+         pack("V", strlen($header_msgpack)) .
+         pack("V", $max_chunk_size) .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $packed_data;
+}
+
 //format: [[class_id, [data]], ...[class_id, [data]]]
 function config_unpack_bundle(string $packed_data) : array
 {
@@ -431,7 +532,8 @@ function config_unpack_bundle(string $packed_data) : array
   }
   else if($info['format'] === 3)
   {
-    return _config_unpack_bundle_fmt3($packed_data);
+    list($_, $entries) = _config_unpack_bundle_fmt3($packed_data);
+    return $entries;
   }
   else
     throw new Exception("Unknown format: {$info['format']}");
@@ -498,12 +600,10 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries =
   return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
 }
 
-function _config_unpack_bundle_fmt3(string $packed_data): array
+function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries = true): array
 {
   if(ord($packed_data[0]) !== 3)
-  {
     throw new Exception("Invalid config bundle format");
-  }
 
   $offset = 1;
   $version = unpack("V", substr($packed_data, $offset, 4))[1];
@@ -515,6 +615,7 @@ function _config_unpack_bundle_fmt3(string $packed_data): array
 
   $header_msgpack = substr($packed_data, $offset, $header_len);
   $offset += $header_len;
+  $chunks_offset = $offset;
 
   $header = config_msgpack_unpack($header_msgpack);
 
@@ -523,39 +624,45 @@ function _config_unpack_bundle_fmt3(string $packed_data): array
   $chunk_buffer = '';
   $chunk_id = -1;
 
-  foreach ($header as $entry_data)
+  foreach($header as $entry_data)
   {
     list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
 
-    if($entry_chunk_offset !== $chunk_id)
+    $unpacked_payload = null;
+
+    if($unpack_entries)
     {
-      if($chunk_offset !== -1)
+      if($entry_chunk_offset !== $chunk_id)
       {
-        $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
+        if($chunk_offset !== -1)
-        $offset+=4;
+        {
-        $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
+          $lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
-        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+          $offset+=4;
-        $offset += $lz4_chunk_size;
+          $lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
-        $chunk_offset = $offset;
+          $chunk_buffer = lz4_uncompress($lz4_chunk_data);
-      }
+          $offset += $lz4_chunk_size;
-      else
+          $chunk_offset = $offset;
-      {
+        }
-        $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
+        else
-        $chunk_offset += 4;
+        {
-        $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
+          $lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
-        $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+          $chunk_offset += 4;
-        $chunk_offset += $lz4_chunk_size;
+          $lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
+          $chunk_buffer = lz4_uncompress($lz4_chunk_data);
+          $chunk_offset += $lz4_chunk_size;
+        }
+
+        $chunk_id = $entry_chunk_offset;
       }
 
-      $chunk_id = $entry_chunk_offset;
+      $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
+      $unpacked_payload = _config_unpack_payload($format, $payload);
     }
 
-    $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
+    $cache_entries[$id] = array($class_id, $unpacked_payload); 
-
-    $cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
   }
 
-  return $cache_entries;
+  return array($header, $cache_entries, $chunks_offset, $max_chunk_size);
 }
 
 //format: [format_id, payload_data]

util.inc.php

@@ -28,9 +28,11 @@ function config_real_path(array $base_dirs, string $rel_path, bool $strict = tru
 {
   foreach($base_dirs as $dir)
     if(is_file($dir . '/' . $rel_path))
-      return $dir . '/' . $rel_path;
+      return normalize_path($dir . '/' . $rel_path);
+
   if($strict)
     throw new Exception("No file for relative path '$rel_path'");
+
   return null;
 }
 
@@ -104,6 +106,7 @@ function config_walk_fields(object $proto, callable $callback)
   }
 }
 
+//TODO: doesn't really belong here
 function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
 {
   if(!isset($map[$file]))