diff --git a/cache.inc.php b/cache.inc.php index 5f716de..27869fe 100644 --- a/cache.inc.php +++ b/cache.inc.php @@ -2,6 +2,101 @@ namespace taskman; use Exception; +class ConfigCache +{ + private ConfigGlobals $globals; + + /** @var array */ + private array $by_path = []; + /** @var array */ + private array $by_id = []; + /** @var array */ + private array $by_alias = []; + + function __construct(ConfigGlobals $globals) + { + $this->globals = $globals; + } + + function clear() + { + $this->by_path = []; + $this->by_id = []; + $this->by_alias = []; + } + + function tryGetOrLoadByPath(string $path) : ?ConfigCacheEntry + { + if(!array_key_exists($path, $this->by_path)) + { + $cache_file = config_get_cache_path($this->globals, $path); + if(!is_file($cache_file)) + return null; + $ce = ConfigCacheEntry::unserialize(ensure_read($cache_file)); + $this->by_path[$path] = $ce; + return $ce; + } + + return $this->by_path[$path]; + } + + function getOrLoadByPath(string $path) : ConfigCacheEntry + { + $ce = $this->tryGetOrLoadByPath($path); + if($ce === null) + throw new Exception("Failed to find entry by path '$path'"); + return $ce; + } + + function tryGetOrLoadById(int $id) : ?ConfigCacheEntry + { + if(!array_key_exists($id, $this->by_id)) + { + $cache_id_file = config_get_cache_id_path($this->globals, $id); + if(!is_file($cache_id_file)) + return null; + $file = ensure_read($cache_id_file); + $ce = $this->tryGetOrLoadByPath($file); + $this->by_id[$id] = $ce; + return $ce; + } + + return $this->by_id[$id]; + } + + function getOrLoadById(int $id) : ConfigCacheEntry + { + $ce = $this->tryGetOrLoadById($id); + if($ce === null) + throw new Exception("Failed to find entry by id '$id'"); + return $ce; + } + + function tryGetOrLoadByStrid(string $strid) : ?ConfigCacheEntry + { + if(!array_key_exists($strid, $this->by_alias)) + { + $cache_strid_file = config_get_cache_strid_path($this->globals, $strid); + if(!is_file($cache_strid_file)) + return null; + $file = ensure_read($cache_strid_file); + $ce = $this->tryGetOrLoadByPath($file); + $this->by_alias[$strid] = $ce; + return $ce; + } + + return $this->by_alias[$strid]; + } + + function getOrLoadByStrid(string $strid) : ConfigCacheEntry + { + $ce = $this->tryGetOrLoadByStrid($strid); + if($ce === null) + throw new Exception("Failed to find entry by strid '$strid'"); + return $ce; + } +} + /** * @property object $config * @property string $payload @@ -146,22 +241,100 @@ class ConfigCacheEntryExtras extends \stdClass } } -class ConfigFetchParams +class ConfigCacheFileMap +{ + private array $file2deps = array(); + + static function serialize(ConfigCacheFileMap $map) : string + { + $d = $map->export(); + return serialize($d); + } + + static function unserialize(string $str) : ?ConfigCacheFileMap + { + $d = @unserialize($str); + if(!is_array($d)) + return null; + $map = new ConfigCacheFileMap(); + $map->import($d); + return $map; + } + + function getAffectedFiles(string $file) : array + { + if(!isset($this->file2deps[$file])) + return []; + return array_keys($this->file2deps[$file]); + } + + function update(array $files) + { + $this_files = array_keys($this->file2deps); + + $added = array_diff($files, $this_files); + $removed = array_diff($this_files, $files); + + foreach($added as $file) + $this->file2deps[$file] = []; + foreach($removed as $file) + unset($this->file2deps[$file]); + + return [$added, $removed]; + } + + function updateDepsForEntry(ConfigCacheEntry $entry) + { + foreach($entry->includes as $include) + { + if(isset($this->file2deps[$include])) + $this->file2deps[$include][$entry->file] = true; + else + $this->file2deps[$include] = [$entry->file => true]; + } + } + + function export() : array + { + $d = array(); + $d[] = $this->file2deps; + return $d; + } + + function import(array $d) + { + list($this->file2deps) = $d; + } +} + +class ConfigCacheUpdateResult +{ + public ConfigDirFiles $affected_files; + + /** @var array */ + public array $errors = array(); + public int $corruptions = 0; + public int $fast_jsons = 0; + + function __construct(ConfigCacheUpdateParams $params) + { + $this->affected_files = $params->affected_files; + } +} + +class ConfigCacheUpdateParams { public ConfigGlobals $globals; - public ConfigScanResult $scanned; - public bool $force_stale = false; + public ConfigDirFiles $affected_files; public bool $verbose = false; public ?int $max_workers = null; - public bool $touch_files_with_includes = true; public bool $check_junk = true; - function __construct(ConfigGlobals $globals, ConfigScanResult $scanned, - bool $force_stale = false, bool $verbose = false, ?int $max_workers = null) + function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, + bool $verbose = false, ?int $max_workers = null) { $this->globals = $globals; - $this->scanned = $scanned; - $this->force_stale = $force_stale; + $this->affected_files = $affected_files; $this->verbose = $verbose; $this->max_workers = $max_workers; } @@ -170,13 +343,13 @@ class ConfigFetchParams { $max_workers = $this->max_workers; if($max_workers === null) - $max_workers = $this->scanned->count() < 100 ? 1 : 5; + $max_workers = $this->affected_files->count() < 100 ? 1 : 5; return $max_workers; } function splitFilesByChunks(int $max_workers, bool $sort = true) : array { - $flat = $this->scanned->getFlatArray(); + $flat = $this->affected_files->getFlatArray(); if($sort) usort($flat, fn($a, $b) => $a[1] <=> $b[1]); @@ -196,49 +369,41 @@ class ConfigFetchParams } } -class ConfigFetchResult +function _config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult { - /** @var ConfigCacheEntry[] */ - public array $all = array(); - /** @var array */ - public array $by_id = array(); - /** @var array */ - public array $by_path = array(); - /** @var array */ - public array $by_alias = array(); - /** @var array */ - public array $errors = array(); - public int $stales = 0; - public int $corruptions = 0; - public int $fast_jsons = 0; + $jobs = $params->splitJobs(sort: true); - function getByAlias(string $strid) : ConfigCacheEntry + $serial = sizeof($jobs) == 1; + + $results_by_job = _config_update_worker_run_procs($params, $jobs, $serial); + + if(!$serial) { - if(array_key_exists($strid, $this->by_alias)) - return $this->by_alias[$strid]; - throw new Exception("Failed to find config by alias '$strid'!"); - } - - function getById(int $id) : ConfigCacheEntry - { - if(array_key_exists($id, $this->by_id)) - return $this->by_id[$id]; - throw new Exception("Failed to find config by id '$id'!"); - } - - function getByPath(string $path) : ConfigCacheEntry - { - if(array_key_exists($path, $this->by_path)) - return $this->by_path[$path]; - throw new Exception("Failed to find config by path '$path'!"); + //NOTE: in case result unserialize error try serial processing + if(array_search(false, $results_by_job, true/*strict*/) !== false) + { + if($params->verbose) + config_log("Corrupted result, trying serial processing..."); + $results_by_job = _config_update_worker_run_procs(params: $params, jobs: $jobs, serial: true); + } } + + $result = _config_merge_update_results($params, $results_by_job); + + if($params->verbose) + config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})"); + + return $result; } -function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult +function config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult { + if($params->affected_files->isEmpty()) + return new ConfigCacheUpdateResult($params); + $t = microtime(true); - $result = _config_cache_fetch($params); + $result = _config_cache_update($params); if($result->errors) { @@ -254,74 +419,26 @@ function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult } if($params->verbose) - config_log("Fetch from cache: " . round(microtime(true) - $t,2) . " sec."); + config_log("Update cache: " . round(microtime(true) - $t,2) . " sec."); return $result; } -function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult +function _config_merge_update_results(ConfigCacheUpdateParams $params, array $results_by_job) : ConfigCacheUpdateResult { - if($params->scanned->isEmpty()) - return new ConfigFetchResult(); + $result = new ConfigCacheUpdateResult($params); - $jobs = $params->splitJobs(sort: true); - - $serial = sizeof($jobs) == 1; - - $results_by_job = _config_worker_run_procs($params, $jobs, $serial); - - if(!$serial) - { - //NOTE: in case result unserialize error try serial processing - if(array_search(false, $results_by_job, true/*strict*/) !== false) - { - if($params->verbose) - config_log("Corrupted result, trying serial processing..."); - $results_by_job = _config_worker_run_procs($params, $jobs, true); - } - } - - $t = microtime(true); - $result = _config_merge_fetch_results($params, $results_by_job); - if($params->verbose) - config_log("Merge results: " . round(microtime(true) - $t,2) . " sec."); - - if($params->verbose) - config_log("Miss(Fast)/Total: {$result->stales}($result->fast_jsons)/" . sizeof($result->all)); - - return $result; -} - -function config_cache_fetch_by_path(ConfigGlobals $globals, string $path, bool $force_stale = false) : ConfigCacheEntry -{ - $path = realpath($path); - $base_dir = config_map_base_dir($globals->base_dirs, $path); - - $scanned = new ConfigScanResult(); - $scanned->add($base_dir, $path); - - $ces = config_cache_fetch(new ConfigFetchParams( - globals: $globals, - scanned: $scanned, - force_stale: $force_stale - )); - - if(empty($ces->all)) - throw new Exception("Config not found at path '$path'"); - return $ces->all[0]; -} - -function _config_merge_fetch_results(ConfigFetchParams $params, array $results_by_job) : ConfigFetchResult -{ - $result = new ConfigFetchResult(); - $total_stales = 0; $total_fast_jsons = 0; foreach($results_by_job as $results) { foreach($results as $item) { - list($base_dir, $file, $cache_file, $is_stale, $parser_type, $error) = $item; + $base_dir = $item['base_dir']; + $file = $item['file']; + $cache_file = $item['cache_file']; + $parser_type = $item['parser_type']; + $error = $item['error']; if($error !== null) { @@ -331,50 +448,16 @@ function _config_merge_fetch_results(ConfigFetchParams $params, array $results_b if($parser_type === 1) ++$total_fast_jsons; - - $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file)); - - //NOTE: handling any cache corruption errors - if($cache_entry === null) - { - $is_stale = true; - $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file); - ++$result->corruptions; - } - - $includes = $cache_entry->includes; - - if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes)) - { - $is_stale = true; - $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file); - //NOTE: let's change the mtime of the file which include other files, - // so that on tne next build it will be 'older' than its includes - // and won't trigger rebuild - if($params->touch_files_with_includes) - touch($file); - } - - if($is_stale) - ++$total_stales; - - //we want results to be returned in the same order, so - //we store entries by the file key and later retrieve array values - $result->by_path[$file] = $cache_entry; - $result->by_id[$cache_entry->id] = $cache_entry; - $result->by_alias[$cache_entry->strid] = $cache_entry; } } - $result->all = array_values($result->by_path); - $result->stales = $total_stales; $result->fast_jsons = $total_fast_jsons; + return $result; } -function _config_invalidate_cache( - ConfigFetchParams $params, string $base_dir, string $file, - string $cache_file, ?int &$parser_type = null) : ConfigCacheEntry +function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file, + ?int &$parser_type = null) : ConfigCacheEntry { list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file); if(!$conf_id) @@ -399,6 +482,8 @@ function _config_invalidate_cache( $cache_payload_file = config_get_cache_payload_path($params->globals, $file); + $cache_file = config_get_cache_path($params->globals, $file); + $cache_entry = new ConfigCacheEntry(); $cache_entry->id = $conf_id; $cache_entry->strid = $conf_strid; @@ -408,14 +493,90 @@ function _config_invalidate_cache( $cache_entry->payload_file = $cache_payload_file; //TODO: pass flag if file is actually normalized? $cache_entry->file = normalize_path($file); - $cache_entry->includes = config_get_module_includes($pres->jsm_module); - $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS']; + $cache_entry->includes = config_get_module_includes(cm: $pres->jsm_module); //TODO: do we really need these refs? $cache_entry->refs = config_content_match_refs($pres->normalized_jzon); + $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS']; ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry)); ensure_write($cache_payload_file, config_msgpack_pack($normalized_data)); + ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file); + ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file); return $cache_entry; } +function _config_update_worker_run_procs(ConfigCacheUpdateParams $params, array $jobs, bool $serial) : array +{ + if($serial) + { + $results_by_job = array(); + foreach($jobs as $job) + $results_by_job[] = _config_update_worker_func($params, $job, $serial); + return $results_by_job; + } + else + { + //initializing worker for master process anyway + $params->globals->initWorker(true); + + $workers_args = array(); + foreach($jobs as $job) + $workers_args[] = array($params, $job); + + return run_background_gamectl_workers('config_update_worker', $workers_args); + } +} + +function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job, bool $is_master_proc = false) : array +{ + $start_time = microtime(true); + + $params->globals->initWorker($is_master_proc); + + list($idx, $start_time, $chunk) = $job; + if($params->verbose) + config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)"); + + $fast_parser_num = 0; + $results = array(); + foreach($chunk as $file_idx => $chunk_data) + { + list($base_dir, $file) = $chunk_data; + + try + { + if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0) + config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ..."); + + $cache_file = config_get_cache_path($params->globals, $file); + + $parser_type = null; + $entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type); + + $results[] = [ + 'base_dir' => $base_dir, + 'file' => $file, + 'cache_file' => $entry->cache_file, + 'parser_type' => $parser_type, + 'error' => null + ]; + } + catch(\Throwable $e) + { + $results[] = [ + 'base_dir' => $base_dir, + 'file' => $file, + 'cache_file' => null, + 'parser_type' => null, + 'error' => $e->getMessage() + ]; + } + } + + if($params->verbose) + config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)"); + + return $results; +} + diff --git a/config.inc.php b/config.inc.php index 9acb45c..d754d55 100644 --- a/config.inc.php +++ b/config.inc.php @@ -15,6 +15,7 @@ class ConfigGlobals public ?string $worker_init_fn = null; public string $base_class = '\ConfBase'; public string $build_dir; + public ?ConfigCacheFileMap $file_map; function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null) { @@ -35,6 +36,210 @@ class ConfigGlobals } } +enum ConfigUpdateMode : int +{ + case ChangedOnly = 1; + case RelativeToBundle = 2; + case Full = 3; +} + +class ConfigManager +{ + private ConfigGlobals $globals; + private int $workers_num; + + private ConfigCache $cache; + + function __construct(ConfigGlobals $globals, int $workers_num) + { + $this->globals = $globals; + $this->workers_num = $workers_num; + + $this->cache = new ConfigCache($globals); + } + + function getGlobals() : ConfigGlobals + { + return $this->globals; + } + + function getCache() : ConfigCache + { + return $this->cache; + } + + function getArtifactFilesSpec() : array + { + return [$this->globals->base_dirs, ['.js']]; + } + + function updateCache( + ConfigUpdateMode $update_mode, + ConfigDirFiles $input_files = null, + ?string $result_bundle_file = null, + bool $verbose = false + ) : ConfigCacheUpdateResult + { + if($input_files === null && $update_mode === ConfigUpdateMode::ChangedOnly) + throw new Exception("input_files argument is required for ChangedOnly mode"); + + if($input_files === null) + $input_files = $this->scanFiles(extension: '.js', verbose: $verbose); + + $added_files = []; + $removed_files = []; + $fs_cache_map = $this->_checkFileMap( + $update_mode, + $input_files, + $added_files, + $removed_files + ); + + $affected_files = $this->_getAffectedFiles( + $update_mode, + $result_bundle_file, + $fs_cache_map, + $input_files, + $removed_files + ); + + //NOTE: at this poine taking into account only config files + $affected_files->filter(fn($file) => str_ends_with($file, '.conf.js')); + + $update_params = new ConfigCacheUpdateParams( + globals: $this->globals, + affected_files: $affected_files, + verbose: $verbose, + max_workers: $this->workers_num + ); + $update_result = config_cache_update($update_params); + + $this->cache->clear(); + + //TODO: traverse all affected files and update file map + foreach($affected_files as $file) + { + $cache_entry = $this->cache->getOrLoadByPath($file); + $fs_cache_map->updateDepsForEntry($cache_entry); + } + + if($affected_files->count() > 0 || $added_files || $removed_files) + $this->_saveMap($fs_cache_map); + + return $update_result; + } + + private function _checkFileMap(ConfigUpdateMode $update_mode, ConfigDirFiles $input_files, array &$added_files, array &$removed_files) : ConfigCacheFileMap + { + //TODO: do we need to check if cache stale somehow? + $fs_cache_map = $this->_tryLoadMap(); + //NOTE: if there's no map so far we need to create one + if($fs_cache_map === null) + { + $fs_cache_map = new ConfigCacheFileMap(); + + //let's re-use the input files + if($update_mode === ConfigUpdateMode::Full || $update_mode === ConfigUpdateMode::RelativeToBundle) + $tmp_files = $input_files; + else + $tmp_files = $this->scanFiles(extension: '.js'); + + $fs_cache_map->update($tmp_files->getAllFiles()); + } + + if($update_mode === ConfigUpdateMode::Full || $update_mode === ConfigUpdateMode::RelativeToBundle) + { + list($added_files, $removed_files) = $fs_cache_map->update($input_files->getAllFiles()); + config_log("File map added: ".count($added_files).", removed: ".count($removed_files)); + } + + return $fs_cache_map; + } + + private function _getAffectedFiles(ConfigUpdateMode $update_mode, ?string $result_bundle_file, ConfigCacheFileMap $fs_cache_map, ConfigDirFiles $input_files, array $removed_files) : ConfigDirFiles + { + $affected_files = null; + + if($update_mode === ConfigUpdateMode::Full) + { + $affected_files = $input_files; + } + else if($update_mode === ConfigUpdateMode::RelativeToBundle) + { + if($result_bundle_file === null) + throw new Exception("result_bundle_file argument is required"); + + $affected_files = $this->newDirFiles(); + + foreach($input_files->getMap() as $base_dir => $files) + { + foreach($files as $file) + { + if(need_to_regen($result_bundle_file, [$file])) + { + $affected_files->add($base_dir, $file); + + $affected_by_file = $fs_cache_map->getAffectedFiles($file); + foreach($affected_by_file as $dep) + $affected_files->addFile($dep, unique: true); + } + } + } + + //if there were removed files we need to rebuild affected files + foreach($removed_files as $file) + { + if(!str_ends_with($file, '.conf.js')) + { + $affected_by_file = $fs_cache_map->getAffectedFiles($file); + foreach($affected_by_file as $dep) + $affected_files->add(config_map_base_dir($this->globals->base_dirs, $dep), $dep, unique: true); + } + else + { + //TODO: in case config file was removed do we actually need to rebuild all configs? + } + } + } + else + throw new Exception("Unknown update mode: {$update_mode->value}"); + + return $affected_files; + } + + function newDirFiles() : ConfigDirFiles + { + return new ConfigDirFiles([], $this->globals->base_dirs); + } + + function parseFile(string $file) : ConfigParseResult + { + return config_parse($this->globals->base_dirs, $file); + } + + function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles + { + return config_scan_files($this->globals->base_dirs, $extension, $verbose); + } + + private function _getMapPath() : string + { + return $this->globals->build_dir . '/file_map.data'; + } + + private function _tryLoadMap() : ?ConfigCacheFileMap + { + if(!is_file($this->_getMapPath())) + return null; + return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath())); + } + + private function _saveMap(ConfigCacheFileMap $map) + { + ensure_write($this->_getMapPath(), ConfigCacheFileMap::serialize($map)); + } +} + function config_log($msg) { echo "[CFG] $msg\n"; @@ -42,12 +247,22 @@ function config_log($msg) function config_get_cache_path(ConfigGlobals $globals, string $file) : string { - return config_get_tmp_build_path($globals, $file . '.cacheb'); + return config_get_tmp_build_path($globals, $file . '.cache'); +} + +function config_get_cache_id_path(ConfigGlobals $globals, int $id) : string +{ + return config_get_tmp_build_path($globals, $id . '.id'); +} + +function config_get_cache_strid_path(ConfigGlobals $globals, string $strid) : string +{ + return config_get_tmp_build_path($globals, $strid . '.strid'); } function config_get_cache_payload_path(ConfigGlobals $globals, string $file) : string { - return config_get_tmp_build_path($globals, $file . '.pdata'); + return config_get_tmp_build_path($globals, $file . '.payload'); } function config_load(ConfigGlobals $globals, string $conf_path, ?string $conf_dir = null) : object diff --git a/pack.inc.php b/pack.inc.php index 373cb4f..bb0278b 100644 --- a/pack.inc.php +++ b/pack.inc.php @@ -5,7 +5,7 @@ use Exception; class ConfigPackParams { /*var ConfigCacheEntry[]*/ - public array $cache_entries; + public iterable $cache_entries; public bool $use_lz4 = false; public bool $use_config_refs = false; public int $binary_format = 1; //1,2 supported @@ -17,7 +17,7 @@ class ConfigPackParams public array $extras = array(); function __construct( - array $cache_entries, + iterable $cache_entries, int $version, bool $use_lz4 = false, bool $use_config_refs = false, @@ -115,7 +115,7 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st //NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) function _config_pack_bundle_fmt1( - array $cache_entries, + iterable $cache_entries, bool $use_lz4, bool $use_config_refs, int $version) : string @@ -173,7 +173,7 @@ function _config_pack_bundle_fmt1( //NOTE: strids are stored as lookup strings, and actually an array of lookup string indices // (each path item separated by '/' is stored as an array item) function _config_pack_bundle_fmt2( - array $cache_entries, + iterable $cache_entries, bool $use_lz4, bool $use_config_refs, int $version, diff --git a/parse.inc.php b/parse.inc.php index 5bb3f77..254e2e4 100644 --- a/parse.inc.php +++ b/parse.inc.php @@ -9,7 +9,7 @@ class ConfigParseResult public string $normalized_jzon = ''; public array $parsed_arr; public int $parser_type; - public $jsm_module; + public \JSM_Module $jsm_module; } function config_parse(array $base_dirs, string $file) : ConfigParseResult diff --git a/scan.inc.php b/scan.inc.php index 8e9d853..90f502f 100644 --- a/scan.inc.php +++ b/scan.inc.php @@ -2,17 +2,24 @@ namespace taskman; use Exception; -class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator +class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator { + private array $base_dirs; + /*var array*/ - public array $base_dir2files = array(); + private array $base_dir2files = array(); private $iter_pos = 0; - function __construct(array $base_dir2files = array()) + function __construct(array $base_dir2files = array(), ?array $base_dirs = null) { foreach($base_dir2files as $dir => $files) $this->base_dir2files[$dir] = $files; + + if($base_dirs === null) + $this->base_dirs = array_keys($base_dir2files); + else + $this->base_dirs = $base_dirs; } function clear() @@ -42,7 +49,11 @@ class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator function filter(callable $filter) { foreach($this->base_dir2files as $base_dir => $files) - $this->base_dir2files[$base_dir] = array_filter($files, $filter); + { + $filtered = array_filter($files, $filter); + //NOTE: don't want any index gaps + $this->base_dir2files[$base_dir] = array_values($filtered); + } } function forEachFile(callable $fn) @@ -54,12 +65,23 @@ class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator } } - function add(string $base_dir, string $file) + function addFile(string $file, bool $unique = false) + { + $this->add(config_map_base_dir($this->base_dirs, $file, normalized: true), $file, $unique); + } + + function add(string $base_dir, string $file, bool $unique = false) { if(!isset($this->base_dir2files[$base_dir])) $this->base_dir2files[$base_dir] = array(); - $this->base_dir2files[$base_dir][] = $file; + if(!$unique || !in_array($file, $this->base_dir2files[$base_dir])) + $this->base_dir2files[$base_dir][] = $file; + } + + function getMap() : array + { + return $this->base_dir2files; } //returns [[base_dir, file1], [base_dir, file2], ...] @@ -172,24 +194,26 @@ function config_scan_files( iterable $base_dirs, string $ext_filter = '.conf.js', bool $verbose = false - ) : ConfigScanResult + ) : ConfigDirFiles { $t = microtime(true); - $result = new ConfigScanResult(); - + $base_dir2files = []; foreach($base_dirs as $base_dir) { - $result->base_dir2files[$base_dir] = - scan_files_rec(array($base_dir), array($ext_filter)); + $base_dir2files[$base_dir] = + scan_files_rec([$base_dir], [$ext_filter]); } + $result = new ConfigDirFiles($base_dir2files); + if($verbose) config_log("File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec."); return $result; } +//NOTE: this one is obsolete function config_hash_changed(ConfigGlobals $globals, iterable $all_files) { $all_crc_file = $globals->build_dir . "/configs.crc"; diff --git a/task.inc.php b/task.inc.php index d3e078f..673ee50 100644 --- a/task.inc.php +++ b/task.inc.php @@ -3,7 +3,7 @@ namespace taskman; use Exception; use Throwable; -task('config_worker', function(array $args) +task('config_update_worker', function(array $args) { if(sizeof($args) != 3) throw new Exception("Config worker args not set"); @@ -15,7 +15,7 @@ task('config_worker', function(array $args) try { list($params, $job) = unserialize(ensure_read($in_file)); - $result = _config_worker_func($params, $job); + $result = _config_update_worker_func($params, $job); ensure_write($out_file, serialize($result)); } catch(Throwable $e) @@ -26,73 +26,3 @@ task('config_worker', function(array $args) throw $e; } }); - -function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $serial) : array -{ - if($serial) - { - $results_by_job = array(); - foreach($jobs as $job) - $results_by_job[] = _config_worker_func($params, $job, $serial); - return $results_by_job; - } - else - { - //initializing worker for master process anyway - $params->globals->initWorker(true); - - $workers_args = array(); - foreach($jobs as $job) - $workers_args[] = array($params, $job); - - return run_background_gamectl_workers('config_worker', $workers_args); - } -} - -//returns [[base_dir, file, cache_file, was_stale, parser_type, error], ...] -function _config_worker_func(ConfigFetchParams $params, array $job, bool $is_master_proc = false) : array -{ - $start_time = microtime(true); - - $params->globals->initWorker($is_master_proc); - - list($idx, $start_time, $chunk) = $job; - if($params->verbose) - config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)"); - - $fast_parser_num = 0; - $results = array(); - foreach($chunk as $file_idx => $chunk_data) - { - list($base_dir, $file) = $chunk_data; - - try - { - if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0) - config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ..."); - - $cache_file = config_get_cache_path($params->globals, $file); - - $parser_type = null; - - $is_stale = true; - if(!$params->force_stale) - $is_stale = need_to_regen($cache_file, array($file)); - - if($is_stale) - _config_invalidate_cache($params, $base_dir, $file, $cache_file, $parser_type); - - $results[] = array($base_dir, $file, $cache_file, $is_stale, $parser_type, null); - } - catch(Throwable $e) - { - $results[] = array($base_dir, $file, null, null, null, $e->getMessage()); - } - } - - if($params->verbose) - config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)"); - - return $results; -} - diff --git a/util.inc.php b/util.inc.php index 16608df..5846e99 100644 --- a/util.inc.php +++ b/util.inc.php @@ -4,7 +4,7 @@ use Exception; function config_get_tmp_build_path(ConfigGlobals $globals, string $file) : string { - $name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file)))); + $name = str_replace(['@', ':', "\\", "/"], ['', "-", "-", "-"], normalize_path($file)); $name = ltrim($name, "-"); return normalize_path($globals->build_dir . "/$name"); } @@ -70,6 +70,7 @@ function config_crc28(string $what) : int function config_get_module_includes(\JSM_Module $cm) : array { $includes = array(); + //NOTE: module contains all includes (even those included from other modules) foreach($cm->getIncludes() as $include => $_) { //maybe we should take .php includes into account as well?