cache.inc.php

@@ -325,24 +325,14 @@ class ConfigCacheFileMap
       unset($this->file2deps[$file]);
   }
 
-  function updateDepsForEntry(array $base_dirs, ConfigCacheEntry $entry)
+  function updateDepsForEntry(ConfigCacheEntry $entry)
   {
-    //1. add includes
+    foreach($entry->includes as $include)
-    $all_refs = $entry->includes;
-
-    //2. add refs
-    foreach($entry->refs as $ref => $_)
     {
-      $file = config_real_path($base_dirs, substr($ref, 1) . '.conf.js');
+      if(isset($this->file2deps[$include]))
-      $all_refs[] = $file;
+        $this->file2deps[$include][$entry->file] = true;
-    }
-
-    foreach($all_refs as $ref_file)
-    {
-      if(isset($this->file2deps[$ref_file]))
-        $this->file2deps[$ref_file][$entry->file] = true;
       else
-        $this->file2deps[$ref_file] = [$entry->file => true];
+        $this->file2deps[$include] = [$entry->file => true];
     }
   }
 

config.inc.php

@@ -18,7 +18,7 @@ class ConfigGlobals
   public readonly string $base_class;
   public readonly string $build_dir;
   //NOTE: it's a string since is serialized
-  public readonly ?string $files_filter; 
+  public readonly ?string $files_overrider; 
 
   function __construct(
     array $base_dirs, 
@@ -26,7 +26,7 @@ class ConfigGlobals
     ?string $worker_init_fn = null,
     int $workers_num = 1,
     string $base_class = '\ConfBase',
-    ?string $files_filter = null
+    ?string $files_overrider = null
   )
   {
     $this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
@@ -34,7 +34,7 @@ class ConfigGlobals
     $this->worker_init_fn = $worker_init_fn;
     $this->workers_num = $workers_num;
     $this->base_class = $base_class;
-    $this->files_filter = $files_filter;
+    $this->files_overrider = $files_overrider;
   }
 
   function initWorker(bool $is_master_proc)
@@ -123,13 +123,6 @@ class ConfigUpdateResult
   public int $corruptions = 0;
   public int $fast_jsons = 0;
 
-  function isEmpty() : bool
-  {
-    return count($this->affected_files) == 0 && 
-      count($this->added_files) == 0 && 
-      count($this->removed_files) == 0;
-  }
-
   function isPatchPossible() : bool
   {
     return ($this->request->mode == ConfigUpdateMode::Selected ||
@@ -185,8 +178,7 @@ class ConfigManager
   function fetchAll(bool $force = false) : \Generator
   {
     $result = $this->updateAll($force);
-    $filtered = $this->filterConfigFiles($result->request->files);
+    return $this->iterateCache($result->request->files);
-    return $this->iterateCache($filtered);
   }
 
   function fetchByPath(string $path) : ConfigCacheEntry
@@ -203,11 +195,14 @@ class ConfigManager
   function iterateCache(iterable $files) : \Generator
   {
     foreach($files as $file)
+    {
+      if(config_is_file($file))
       {
         $ce = $this->cache->getOrLoadByPath($file);
         yield $ce;
       }
     }
+  }
 
   function update(ConfigUpdateRequest $req) : ConfigUpdateResult
   {
@@ -220,20 +215,22 @@ class ConfigManager
     $removed_files = [];
     $this->_checkFileMap($req, $added_files, $removed_files);
 
-    $affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
+    $affected_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
-    $affected_conf_files = $this->filterConfigFiles($affected_all_files);
 
-    config_log("Affected configs: {$affected_conf_files->count()} (requested: {$req->files->count()})");
+    //NOTE: at this point taking into account only config files
+    $affected_files->filter(fn($file) => config_is_file($file));
+
+    config_log("Affected files: {$affected_files->count()}");
 
     $update_result = new ConfigUpdateResult();
     $update_result->request = $req;
-    $update_result->affected_files = $affected_conf_files;
+    $update_result->affected_files = $affected_files;
     $update_result->added_files = $added_files;
     $update_result->removed_files = $removed_files;
 
     $update_params = new ConfigCacheUpdateParams(
       globals: $this->globals, 
-      affected_files: $affected_conf_files, 
+      affected_files: $affected_files, 
       verbose: $req->verbose
     );
     self::_updateCache($update_result, $update_params);
@@ -241,11 +238,11 @@ class ConfigManager
     //let's clear internal cache once the update procedure is done
     $this->cache->clear();
 
-    $this->_updateFileMap($req, $affected_conf_files, $added_files, $removed_files);
+    $this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
 
     if($req->return_affected)
     {
-      foreach($affected_conf_files as $file)
+      foreach($affected_files as $file)
       {
         $entry = $this->cache->getOrLoadByPath($file);
         $update_result->affected_entries[] = $entry;
@@ -257,19 +254,6 @@ class ConfigManager
     return $update_result;
   }
 
-  function filterConfigFiles(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles 
-  {
-    if($files instanceof \taskman\artefact\TaskmanDirFiles)
-      $files = ConfigDirFiles::makeFromArtefactFiles($files);
-
-    $filtered = $this->globals->files_filter != null ? 
-      call_user_func($this->globals->files_filter, $files) : 
-      new ConfigDirFiles($files->getMap());
-    
-    $filtered->filter(fn($file) => config_is_file($file));
-    return $filtered;
-  }
-
   function updateAll(bool $force = false) : ConfigUpdateResult
   {
     $files = $this->scanFiles(extension: '.js');
@@ -341,19 +325,19 @@ class ConfigManager
     }
   }
 
-  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_conf_files, array $added_files, array $removed_files)
+  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
   {
     $fs_cache_map = $this->getFileMap();
 
-    //TODO: traverse all affected config files and update file map
+    //TODO: traverse all affected files and update file map
-    foreach($affected_conf_files as $file)
+    foreach($affected_files as $file)
     {
       $cache_entry = $this->cache->getOrLoadByPath($file);
-      $fs_cache_map->updateDepsForEntry($this->globals->base_dirs, $cache_entry);
+      $fs_cache_map->updateDepsForEntry($cache_entry);
     }
 
     if($req->mode == ConfigUpdateMode::Force || 
-      $affected_conf_files->count() > 0 || 
+      $affected_files->count() > 0 || 
       $added_files || 
       $removed_files)
     {
@@ -370,7 +354,7 @@ class ConfigManager
 
     if($req->mode === ConfigUpdateMode::Force)
     {
-      $affected_files = new ConfigDirFiles($req->files->getMap());
+      $affected_files = $req->files;
     }
     else if($req->mode === ConfigUpdateMode::DetectChanged)
     {
@@ -395,12 +379,15 @@ class ConfigManager
 
       //if there were removed files we need to rebuild affected files
       foreach($removed_files as $file)
+      {
+        if(!config_is_file($file))
         {
           $affected_by_file = $fs_cache_map->getAffectedFiles($file);
           foreach($affected_by_file as $dep)
             $affected_files->addFile($dep, unique: true);
         }
       }
+    }
     else if($req->mode === ConfigUpdateMode::Selected)
     {
       $affected_files = ConfigDirFiles::makeFor($this);
@@ -408,11 +395,15 @@ class ConfigManager
       foreach($req->files as $file)
       {
         $affected_files->addFile($file, unique: true);
+
+        if(!config_is_file($file))
+        {
           $affected_by_file = $fs_cache_map->getAffectedFiles($file);
           foreach($affected_by_file as $dep)
             $affected_files->addFile($dep, unique: true);
         }
       }
+    }
 
     return $affected_files;
   }
@@ -424,7 +415,10 @@ class ConfigManager
 
   function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
   {
-    return config_scan_files($this->globals->base_dirs, $extension, $verbose);
+    $res = config_scan_files($this->globals->base_dirs, $extension, $verbose);
+    if($this->globals->files_overrider != null)
+      $res = call_user_func($this->globals->files_overrider, $res);
+    return $res;
   }
 
   private function _getMapPath() : string
@@ -436,11 +430,8 @@ class ConfigManager
   {
     if(!is_file($this->_getMapPath()))
       return null;
-    $t = microtime(true);
+    config_log("Loading file map");
-    $data = ensure_read($this->_getMapPath());
+    return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
-    $map = ConfigCacheFileMap::unserialize($data);
-    config_log("Loaded file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
-    return $map;
   }
 
   private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
@@ -453,10 +444,9 @@ class ConfigManager
 
   private function _saveFileMap()
   {
-    $t = microtime(true);
     $data = ConfigCacheFileMap::serialize($this->getFileMap()); 
+    config_log("Saving file map: " . kb($data));
     ensure_write($this->_getMapPath(), $data);
-    config_log("Saved file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
   }
 }
 

pack.inc.php

@@ -102,16 +102,6 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
       $params->version,
     );
   }
-  else if($params->binary_format == 3)
-  {                  
-    $patched_data = _config_patch_bundle_fmt3(
-      $packed_data,
-      $params->cache_entries, 
-      $params->use_lz4,
-      $params->use_config_refs,
-      $params->version,
-    );
-  }
   else
     throw new Exception("Unknown binary format: {$params->binary_format}");
 
@@ -246,7 +236,7 @@ function _config_pack_bundle_fmt2(
   return $packed_data;
 }
 
-//NOTE: Much like fmt1, but configs entries are grouped into sizeable chunks, with each chunk lz4-compressed.
+//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
 //      This reduces overall bundle size when there are many small configs entries.
 function _config_pack_bundle_fmt3(
   array $cache_entries, 
@@ -361,12 +351,12 @@ function _config_patch_bundle_fmt2(
   {
     list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
 
-    $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+    $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
 
-    if($headers_found)
+    if($header_found)
     {
-      $header_idx = key($headers_found);
+      $header_idx = key($header_found);
-      $header_entry = current($headers_found);
+      $header_entry = current($header_found);
 
       $current_offset = $header_entry[4];
       $current_size = $header_entry[5];
@@ -424,97 +414,6 @@ function _config_patch_bundle_fmt2(
   return $patched_data;
 }
 
-function _config_patch_bundle_fmt3(
-  string $packed_data,
-  array $patch_entries, 
-  bool $use_lz4, 
-  bool $use_config_refs, 
-  int $version,
-) : string
-{
-  if(!$use_lz4)
-    throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
-
-  list($header, $entries, $chunks_offset, $max_chunk_size) = 
-    _config_unpack_bundle_fmt3(packed_data: $packed_data, unpack_entries: false);
-
-  $payloads_bundle = substr($packed_data, $chunks_offset);
-
-  foreach($patch_entries as $idx => $patch_entry)
-  {
-    list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
-
-    $headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
-
-    if($headers_found)
-    {
-      $header_idx = key($headers_found);
-      $header_entry = current($headers_found);
-
-      $chunk_offset = $header_entry[4];
-      $payloads_offset = $header_entry[5]; //within chunk
-      $payloads_size = $header_entry[6];
-
-      $chunk_entries = array_filter($header, fn($item) => $item[4] == $chunk_offset); 
-
-      //chunk contains only one patch entry
-      $patch_payload_lz4 = lz4_compress($patch_payload, 9); 
-      $patched_chunk = pack("V", strlen($patch_payload_lz4));
-      $patched_chunk .= $patch_payload_lz4;
-      if(strlen($patch_payload) > $max_chunk_size)
-        $max_chunk_size = strlen($patch_payload);
-
-      //TODO: make a better generic algorithm for patching the whole chunk
-      //special case for single entry 
-      if(count($chunk_entries) == 1 && $payloads_offset == 0)
-      {
-        $lz4_chunk_size = unpack("V", substr($payloads_bundle, $chunk_offset, 4))[1];
-
-        //we can just replace chunk in place if the size of a patched chunk is less or equal
-        if(strlen($patch_payload_lz4) <= $lz4_chunk_size)
-        {
-          $payloads_bundle = substr_replace($payloads_bundle, $patched_chunk, $chunk_offset, strlen($patched_chunk));
-          $header_entry[6] = strlen($patch_payload);
-        }
-        //just append to the end
-        else
-        {
-          $header_entry[4] = strlen($payloads_bundle); //chunk offset
-          $header_entry[5] = 0;
-          $header_entry[6] = strlen($patch_payload);
-
-          $payloads_bundle .= $patched_chunk;
-        }
-      }
-      //just append to the end
-      else
-      {
-        $header_entry[4] = strlen($payloads_bundle); //chunk offset
-        $header_entry[5] = 0;
-        $header_entry[6] = strlen($patch_payload);
-
-        $payloads_bundle .= $patched_chunk;
-      }
-
-      $header[$header_idx] = $header_entry; 
-    }
-    else
-      throw new Exception("Patched entry {$patch_entry->id} not found in config bundle");
-  }
-
-  $header_msgpack = config_msgpack_pack($header);
-
-  $packed_data =  
-         pack("C", 3) . 
-         pack("V", $version) . 
-         pack("V", strlen($header_msgpack)) .
-         pack("V", $max_chunk_size) .
-         $header_msgpack .
-         $payloads_bundle;
-
-  return $packed_data;
-}
-
 //format: [[class_id, [data]], ...[class_id, [data]]]
 function config_unpack_bundle(string $packed_data) : array
 {
@@ -532,8 +431,7 @@ function config_unpack_bundle(string $packed_data) : array
   }
   else if($info['format'] === 3)
   {
-    list($_, $entries) = _config_unpack_bundle_fmt3($packed_data);
+    return _config_unpack_bundle_fmt3($packed_data);
-    return $entries;
   }
   else
     throw new Exception("Unknown format: {$info['format']}");
@@ -600,10 +498,12 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries =
   return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
 }
 
-function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries = true): array
+function _config_unpack_bundle_fmt3(string $packed_data): array
 {
   if(ord($packed_data[0]) !== 3)
+  {
     throw new Exception("Invalid config bundle format");
+  }
 
   $offset = 1;
   $version = unpack("V", substr($packed_data, $offset, 4))[1];
@@ -615,7 +515,6 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
 
   $header_msgpack = substr($packed_data, $offset, $header_len);
   $offset += $header_len;
-  $chunks_offset = $offset;
 
   $header = config_msgpack_unpack($header_msgpack);
 
@@ -624,14 +523,10 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
   $chunk_buffer = '';
   $chunk_id = -1;
 
-  foreach($header as $entry_data)
+  foreach ($header as $entry_data)
   {
     list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
 
-    $unpacked_payload = null;
-
-    if($unpack_entries)
-    {
     if($entry_chunk_offset !== $chunk_id)
     {
       if($chunk_offset !== -1)
@@ -656,13 +551,11 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
     }
 
     $payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
-      $unpacked_payload = _config_unpack_payload($format, $payload);
+
+    $cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
   }
 
-    $cache_entries[$id] = array($class_id, $unpacked_payload); 
+  return $cache_entries;
-  }
-
-  return array($header, $cache_entries, $chunks_offset, $max_chunk_size);
 }
 
 //format: [format_id, payload_data]

util.inc.php

@@ -28,11 +28,9 @@ function config_real_path(array $base_dirs, string $rel_path, bool $strict = tru
 {
   foreach($base_dirs as $dir)
     if(is_file($dir . '/' . $rel_path))
-      return normalize_path($dir . '/' . $rel_path);
+      return $dir . '/' . $rel_path;
-
   if($strict)
     throw new Exception("No file for relative path '$rel_path'");
-
   return null;
 }
 
@@ -106,7 +104,6 @@ function config_walk_fields(object $proto, callable $callback)
   }
 }
 
-//TODO: doesn't really belong here
 function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
 {
   if(!isset($map[$file]))