bit
/
taskman_config
25
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Improving initialization of workers; Adding sorting when splitting config jobs; Minor improvements here and there
Publish PHP Package / docker (push) Successful in 7s Details

This commit is contained in:
Pavel Shevaev 2025-02-27 01:35:16 +03:00
parent 0842ba416f
commit 63c2c46cc7
4 changed files with 19 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
cache.inc.php
View File

@ -170,22 +170,27 @@ class ConfigFetchParams
{
$max_workers = $this->max_workers;
if($max_workers === null)
$max_workers = $this->scanned->count() < 20 ? 1 : 5;
$max_workers = $this->scanned->count() < 100 ? 1 : 5;
return $max_workers;
}
function splitFilesByChunks(int $max_workers) : array
function splitFilesByChunks(int $max_workers, bool $sort = true) : array
{
$chunk_size = (int)ceil($this->scanned->count()/$max_workers);
return array_chunk($this->scanned->getFlatArray(), $chunk_size);
$flat = $this->scanned->getFlatArray();
if($sort)
usort($flat, fn($a, $b) => $a[1] <=> $b[1]);
$chunk_size = (int)ceil(count($flat)/$max_workers);
return array_chunk($flat, $chunk_size);
}
//returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ]
function splitJobs() : array
function splitJobs(bool $sort = true) : array
{
$max_workers = $this->calcMaxWorkers();
$jobs = array();
foreach($this->splitFilesByChunks($max_workers) as $idx => $chunk)
$chunks = $this->splitFilesByChunks($max_workers, $sort);
foreach($chunks as $idx => $chunk)
$jobs[] = array($idx, microtime(true), $chunk);
return $jobs;
}
@ -259,12 +264,7 @@ function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
if($params->scanned->isEmpty())
return new ConfigFetchResult();
//TODO: not sure if it's the best place for this one
$GLOBALS['CONFIG_GLOBALS'] = $params->globals;
$params->scanned->apply(function($base_dir, $files) { sort($files); return $files;});
$jobs = $params->splitJobs();
$jobs = $params->splitJobs(sort: true);
$serial = sizeof($jobs) == 1;

9
config.inc.php
View File

@ -23,21 +23,16 @@ class ConfigGlobals
$this->worker_init_fn = $worker_init_fn;
}
function initWorker()
function initWorker(bool $is_master_proc)
{
$GLOBALS['CONFIG_GLOBALS'] = $this;
if($this->worker_init_fn !== null)
{
$fn = $this->worker_init_fn;
$fn();
$fn($is_master_proc);
}
}
function setNormalizeBaseDirs(array $dirs)
{
$this->base_dirs = array_map(function($d) { return normalize_path($d); }, $dirs);
}
}
function config_log($msg)

2
pack.inc.php
View File

@ -61,6 +61,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
return $packed_data;
}
//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build)
function _config_pack_bundle_fmt1(
array $cache_entries,
bool $use_lz4,
@ -117,6 +118,7 @@ function _config_pack_bundle_fmt1(
return $packed_data;
}
//NOTE: strids are stored as lookup strings
function _config_pack_bundle_fmt2(
array $cache_entries,
bool $use_lz4,

6
task.inc.php
View File

@ -33,7 +33,7 @@ function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $
{
$results_by_job = array();
foreach($jobs as $job)
$results_by_job[] = _config_worker_func($params, $job);
$results_by_job[] = _config_worker_func($params, $job, $serial);
return $results_by_job;
}
else
@ -47,11 +47,11 @@ function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $
}
//returns [[base_dir, file, cache_file, was_stale, parser_type, error], ...]
function _config_worker_func(ConfigFetchParams $params, array $job) : array
function _config_worker_func(ConfigFetchParams $params, array $job, bool $is_master_proc = false) : array
{
$start_time = microtime(true);
$params->globals->initWorker();
$params->globals->initWorker($is_master_proc);
list($idx, $start_time, $chunk) = $job;
if($params->verbose)