diff --git a/cache.inc.php b/cache.inc.php index d2b3ad1..551e914 100644 --- a/cache.inc.php +++ b/cache.inc.php @@ -170,22 +170,27 @@ class ConfigFetchParams { $max_workers = $this->max_workers; if($max_workers === null) - $max_workers = $this->scanned->count() < 20 ? 1 : 5; + $max_workers = $this->scanned->count() < 100 ? 1 : 5; return $max_workers; } - function splitFilesByChunks(int $max_workers) : array + function splitFilesByChunks(int $max_workers, bool $sort = true) : array { - $chunk_size = (int)ceil($this->scanned->count()/$max_workers); - return array_chunk($this->scanned->getFlatArray(), $chunk_size); + $flat = $this->scanned->getFlatArray(); + if($sort) + usort($flat, fn($a, $b) => $a[1] <=> $b[1]); + + $chunk_size = (int)ceil(count($flat)/$max_workers); + return array_chunk($flat, $chunk_size); } //returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ] - function splitJobs() : array + function splitJobs(bool $sort = true) : array { $max_workers = $this->calcMaxWorkers(); $jobs = array(); - foreach($this->splitFilesByChunks($max_workers) as $idx => $chunk) + $chunks = $this->splitFilesByChunks($max_workers, $sort); + foreach($chunks as $idx => $chunk) $jobs[] = array($idx, microtime(true), $chunk); return $jobs; } @@ -259,12 +264,7 @@ function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult if($params->scanned->isEmpty()) return new ConfigFetchResult(); - //TODO: not sure if it's the best place for this one - $GLOBALS['CONFIG_GLOBALS'] = $params->globals; - - $params->scanned->apply(function($base_dir, $files) { sort($files); return $files;}); - - $jobs = $params->splitJobs(); + $jobs = $params->splitJobs(sort: true); $serial = sizeof($jobs) == 1; diff --git a/config.inc.php b/config.inc.php index c50fae5..9acb45c 100644 --- a/config.inc.php +++ b/config.inc.php @@ -23,21 +23,16 @@ class ConfigGlobals $this->worker_init_fn = $worker_init_fn; } - function initWorker() + function initWorker(bool $is_master_proc) { $GLOBALS['CONFIG_GLOBALS'] = $this; if($this->worker_init_fn !== null) { $fn = $this->worker_init_fn; - $fn(); + $fn($is_master_proc); } } - - function setNormalizeBaseDirs(array $dirs) - { - $this->base_dirs = array_map(function($d) { return normalize_path($d); }, $dirs); - } } function config_log($msg) diff --git a/pack.inc.php b/pack.inc.php index 2c41b49..a8035ab 100644 --- a/pack.inc.php +++ b/pack.inc.php @@ -61,6 +61,7 @@ function config_pack_bundle(ConfigPackParams $params) : string return $packed_data; } +//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) function _config_pack_bundle_fmt1( array $cache_entries, bool $use_lz4, @@ -117,6 +118,7 @@ function _config_pack_bundle_fmt1( return $packed_data; } +//NOTE: strids are stored as lookup strings function _config_pack_bundle_fmt2( array $cache_entries, bool $use_lz4, diff --git a/task.inc.php b/task.inc.php index 0506cbb..c46fa85 100644 --- a/task.inc.php +++ b/task.inc.php @@ -33,7 +33,7 @@ function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $ { $results_by_job = array(); foreach($jobs as $job) - $results_by_job[] = _config_worker_func($params, $job); + $results_by_job[] = _config_worker_func($params, $job, $serial); return $results_by_job; } else @@ -47,11 +47,11 @@ function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $ } //returns [[base_dir, file, cache_file, was_stale, parser_type, error], ...] -function _config_worker_func(ConfigFetchParams $params, array $job) : array +function _config_worker_func(ConfigFetchParams $params, array $job, bool $is_master_proc = false) : array { $start_time = microtime(true); - $params->globals->initWorker(); + $params->globals->initWorker($is_master_proc); list($idx, $start_time, $chunk) = $job; if($params->verbose)