From fc0aa1636e3905659fb0c19305acf5436bd25ec8 Mon Sep 17 00:00:00 2001 From: Pavel Shevaev Date: Thu, 17 Apr 2025 17:12:23 +0300 Subject: [PATCH] Internal refactoring related to file map cache --- cache.inc.php | 21 ++++++++---- config.inc.php | 86 +++++++++++++++++++++++++++++--------------------- scan.inc.php | 5 +++ 3 files changed, 70 insertions(+), 42 deletions(-) diff --git a/cache.inc.php b/cache.inc.php index 041223c..a97f93f 100644 --- a/cache.inc.php +++ b/cache.inc.php @@ -286,12 +286,19 @@ class ConfigCacheFileMap return [$added, $removed]; } + function init(array $files) + { + $this->file2deps = []; + foreach($files as $file) + $this->file2deps[$file] = []; + } + function update(array $added, array $removed) { foreach($added as $file) { - //TODO: are we sure about this check? - if(isset($this->file2deps[$file])) + //NOTE: we preserve existing files + if(!isset($this->file2deps[$file])) $this->file2deps[$file] = []; } foreach($removed as $file) @@ -339,6 +346,9 @@ class ConfigCacheUpdateResult class ConfigCacheUpdateParams { + const MAX_DEFAULT_WORKERS = 5; + const FILES_THRESHOLD_ONE_WORKER = 100; + public ConfigGlobals $globals; public ConfigDirFiles $affected_files; public bool $verbose = false; @@ -356,10 +366,9 @@ class ConfigCacheUpdateParams function calcMaxWorkers() : int { - $max_workers = $this->max_workers; - if($max_workers === null) - $max_workers = $this->affected_files->count() < 100 ? 1 : 5; - return $max_workers; + if($this->affected_files->count() < self::FILES_THRESHOLD_ONE_WORKER) + return 1; + return $this->max_workers ?? self::MAX_DEFAULT_WORKERS; } function splitFilesByChunks(int $max_workers, bool $sort = true) : array diff --git a/config.inc.php b/config.inc.php index 52542c0..68e314f 100644 --- a/config.inc.php +++ b/config.inc.php @@ -37,9 +37,9 @@ class ConfigGlobals enum ConfigUpdateMode : int { - case All = 1; - case AllChanged = 2; - case Patch = 3; + case Force = 1; + case DetectChanged = 2; + case Patch = 3; } class ConfigUpdateRequest @@ -50,29 +50,29 @@ class ConfigUpdateRequest private function __construct() {} - static function all(?ConfigDirFiles $files = null) : ConfigUpdateRequest + static function force(?ConfigDirFiles $files = null) : ConfigUpdateRequest { - $update = new ConfigUpdateRequest(); - $update->mode = ConfigUpdateMode::All; - $update->files = $files; - return $update; + $req = new ConfigUpdateRequest(); + $req->mode = ConfigUpdateMode::Force; + $req->files = $files; + return $req; } - static function allChanged(string $result_file, ?ConfigDirFiles $files = null) : ConfigUpdateRequest + static function detectChanged(string $result_file, ?ConfigDirFiles $files = null) : ConfigUpdateRequest { - $update = new ConfigUpdateRequest(); - $update->mode = ConfigUpdateMode::Patch; - $update->files = $files; - $update->result_file = $result_file; - return $update; + $req = new ConfigUpdateRequest(); + $req->mode = ConfigUpdateMode::DetectChanged; + $req->files = $files; + $req->result_file = $result_file; + return $req; } static function patch(ConfigDirFiles $files) : ConfigUpdateRequest { - $update = new ConfigUpdateRequest(); - $update->mode = ConfigUpdateMode::Patch; - $update->files = $files; - return $update; + $req = new ConfigUpdateRequest(); + $req->mode = ConfigUpdateMode::Patch; + $req->files = $files; + return $req; } } @@ -128,7 +128,7 @@ class ConfigManager function updateCache(ConfigUpdateRequest $req, bool $verbose = false) : ConfigCacheUpdateResult { - config_log("Updating cache, mode {$req->mode->value}..."); + config_log("Updating cache, mode '{$req->mode->name}'..."); if($req->files === null) $req->files = $this->scanFiles(extension: '.js', verbose: $verbose); @@ -154,6 +154,31 @@ class ConfigManager $this->cache->clear(); + $this->_updateFileMap($req, $affected_files, $added_files, $removed_files); + + return $update_result; + } + + private function _checkFileMap(ConfigUpdateRequest $req, array &$added_files, array &$removed_files) + { + $fs_cache_map = $this->getFileMap(); + + if($req->mode === ConfigUpdateMode::Force) + { + $added_files = $req->files->getAllFiles(); + //let's rebuild the file map + $fs_cache_map->init($added_files); + config_log("File map init: ".count($added_files)); + } + else if($req->mode === ConfigUpdateMode::DetectChanged) + { + list($added_files, $removed_files) = $fs_cache_map->compare($req->files->getAllFiles()); + config_log("File map compare, added: ".count($added_files).", removed: ".count($removed_files)); + } + } + + private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files) + { $fs_cache_map = $this->getFileMap(); //TODO: traverse all affected files and update file map @@ -165,22 +190,11 @@ class ConfigManager if($affected_files->count() > 0 || $added_files || $removed_files) { - $fs_cache_map->update($added_files, $removed_files); + //in case of Force map was already cleared and initialized + if($req->mode != ConfigUpdateMode::Force) + $fs_cache_map->update($added_files, $removed_files); $this->_saveFileMap(); } - - return $update_result; - } - - private function _checkFileMap(ConfigUpdateRequest $req, array &$added_files, array &$removed_files) - { - $fs_cache_map = $this->_getFileMap($req->mode != ConfigUpdateMode::Patch ? $req->files : null); - - if($req->mode === ConfigUpdateMode::All || $req->mode === ConfigUpdateMode::AllChanged) - { - list($added_files, $removed_files) = $fs_cache_map->compare($req->files->getAllFiles()); - config_log("File map added: ".count($added_files).", removed: ".count($removed_files)); - } } private function _getAffectedFiles(ConfigUpdateRequest $req, array $removed_files) : ConfigDirFiles @@ -189,11 +203,11 @@ class ConfigManager $affected_files = null; - if($req->mode === ConfigUpdateMode::All) + if($req->mode === ConfigUpdateMode::Force) { $affected_files = $req->files; } - else if($req->mode === ConfigUpdateMode::AllChanged) + else if($req->mode === ConfigUpdateMode::DetectChanged) { $affected_files = ConfigDirFiles::makeFor($this); @@ -273,7 +287,7 @@ class ConfigManager { config_log("Creating file map"); $map = new ConfigCacheFileMap(); - $map->update($files->getAllFiles(), []); + $map->init($files->getAllFiles()); return $map; } diff --git a/scan.inc.php b/scan.inc.php index bc16376..01d99e9 100644 --- a/scan.inc.php +++ b/scan.inc.php @@ -27,6 +27,11 @@ class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator return new ConfigDirFiles([], $mgr->getGlobals()->base_dirs); } + static function makeFromArtefactFiles(\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles + { + return new ConfigDirFiles($files->toMap()); + } + function clear() { $this->base_dir2files = array();