Improving config files filtering logic

config.inc.php

@@ -18,7 +18,7 @@ class ConfigGlobals
   public readonly string $base_class;
   public readonly string $build_dir;
   //NOTE: it's a string since is serialized
-  public readonly ?string $files_overrider; 
+  public readonly ?string $files_filter; 
 
   function __construct(
     array $base_dirs, 
@@ -26,7 +26,7 @@ class ConfigGlobals
     ?string $worker_init_fn = null,
     int $workers_num = 1,
     string $base_class = '\ConfBase',
-    ?string $files_overrider = null
+    ?string $files_filter = null
   )
   {
     $this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
@@ -34,7 +34,7 @@ class ConfigGlobals
     $this->worker_init_fn = $worker_init_fn;
     $this->workers_num = $workers_num;
     $this->base_class = $base_class;
-    $this->files_overrider = $files_overrider;
+    $this->files_filter = $files_filter;
   }
 
   function initWorker(bool $is_master_proc)
@@ -183,7 +183,8 @@ class ConfigManager
   function fetchAll(bool $force = false) : \Generator
   {
     $result = $this->updateAll($force);
-    return $this->iterateCache($result->request->files);
+    $filtered = $this->filterConfigFiles($result->request->files);
+    return $this->iterateCache($filtered);
   }
 
   function fetchByPath(string $path) : ConfigCacheEntry
@@ -201,11 +202,8 @@ class ConfigManager
   {
     foreach($files as $file)
     {
-      if(config_is_file($file))
+      $ce = $this->cache->getOrLoadByPath($file);
-      {
+      yield $ce;
-        $ce = $this->cache->getOrLoadByPath($file);
-        yield $ce;
-      }
     }
   }
 
@@ -220,22 +218,20 @@ class ConfigManager
     $removed_files = [];
     $this->_checkFileMap($req, $added_files, $removed_files);
 
-    $affected_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
+    $affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
+    $affected_conf_files = $this->filterConfigFiles($affected_all_files);
 
-    //NOTE: at this point taking into account only config files
+    config_log("Affected configs: {$affected_conf_files->count()}/{$req->files->count()}");
-    $affected_files->filter(fn($file) => config_is_file($file));
-
-    config_log("Affected files: {$affected_files->count()}/{$req->files->count()}");
 
     $update_result = new ConfigUpdateResult();
     $update_result->request = $req;
-    $update_result->affected_files = $affected_files;
+    $update_result->affected_files = $affected_conf_files;
     $update_result->added_files = $added_files;
     $update_result->removed_files = $removed_files;
 
     $update_params = new ConfigCacheUpdateParams(
       globals: $this->globals, 
-      affected_files: $affected_files, 
+      affected_files: $affected_conf_files, 
       verbose: $req->verbose
     );
     self::_updateCache($update_result, $update_params);
@@ -243,11 +239,11 @@ class ConfigManager
     //let's clear internal cache once the update procedure is done
     $this->cache->clear();
 
-    $this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
+    $this->_updateFileMap($req, $affected_conf_files, $added_files, $removed_files);
 
     if($req->return_affected)
     {
-      foreach($affected_files as $file)
+      foreach($affected_conf_files as $file)
       {
         $entry = $this->cache->getOrLoadByPath($file);
         $update_result->affected_entries[] = $entry;
@@ -259,6 +255,19 @@ class ConfigManager
     return $update_result;
   }
 
+  function filterConfigFiles(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles 
+  {
+    if($files instanceof \taskman\artefact\TaskmanDirFiles)
+      $files = ConfigDirFiles::makeFromArtefactFiles($files);
+
+    $filtered = $this->globals->files_filter != null ? 
+      call_user_func($this->globals->files_filter, $files) : 
+      new ConfigDirFiles($files->getMap());
+    
+    $filtered->filter(fn($file) => config_is_file($file));
+    return $filtered;
+  }
+
   function updateAll(bool $force = false) : ConfigUpdateResult
   {
     $files = $this->scanFiles(extension: '.js');
@@ -330,19 +339,19 @@ class ConfigManager
     }
   }
 
-  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
+  private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_conf_files, array $added_files, array $removed_files)
   {
     $fs_cache_map = $this->getFileMap();
 
-    //TODO: traverse all affected files and update file map
+    //TODO: traverse all affected config files and update file map
-    foreach($affected_files as $file)
+    foreach($affected_conf_files as $file)
     {
       $cache_entry = $this->cache->getOrLoadByPath($file);
       $fs_cache_map->updateDepsForEntry($cache_entry);
     }
 
     if($req->mode == ConfigUpdateMode::Force || 
-      $affected_files->count() > 0 || 
+      $affected_conf_files->count() > 0 || 
       $added_files || 
       $removed_files)
     {
@@ -420,10 +429,7 @@ class ConfigManager
 
   function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
   {
-    $res = config_scan_files($this->globals->base_dirs, $extension, $verbose);
+    return config_scan_files($this->globals->base_dirs, $extension, $verbose);
-    if($this->globals->files_overrider != null)
-      $res = call_user_func($this->globals->files_overrider, $res);
-    return $res;
   }
 
   private function _getMapPath() : string
@@ -435,8 +441,11 @@ class ConfigManager
   {
     if(!is_file($this->_getMapPath()))
       return null;
-    config_log("Loading file map");
+    $t = microtime(true);
-    return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
+    $data = ensure_read($this->_getMapPath());
+    $map = ConfigCacheFileMap::unserialize($data);
+    config_log("Loaded file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
+    return $map;
   }
 
   private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
@@ -449,9 +458,10 @@ class ConfigManager
 
   private function _saveFileMap()
   {
+    $t = microtime(true);
     $data = ConfigCacheFileMap::serialize($this->getFileMap()); 
-    config_log("Saving file map: " . kb($data));
     ensure_write($this->_getMapPath(), $data);
+    config_log("Saved file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
   }
 }