Initial commit
This commit is contained in:
Pavel Shevaev
parent
6fe8a0dff5
commit
1b25beb4ed
439
cache.inc.php
439
cache.inc.php
|
|
@ -2,6 +2,101 @@
|
|||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
class ConfigCache
|
||||
{
|
||||
private ConfigGlobals $globals;
|
||||
|
||||
/** @var array<string, ConfigCacheEntry> */
|
||||
private array $by_path = [];
|
||||
/** @var array<int, ConfigCacheEntry> */
|
||||
private array $by_id = [];
|
||||
/** @var array<string, ConfigCacheEntry> */
|
||||
private array $by_alias = [];
|
||||
|
||||
function __construct(ConfigGlobals $globals)
|
||||
{
|
||||
$this->globals = $globals;
|
||||
}
|
||||
|
||||
function clear()
|
||||
{
|
||||
$this->by_path = [];
|
||||
$this->by_id = [];
|
||||
$this->by_alias = [];
|
||||
}
|
||||
|
||||
function tryGetOrLoadByPath(string $path) : ?ConfigCacheEntry
|
||||
{
|
||||
if(!array_key_exists($path, $this->by_path))
|
||||
{
|
||||
$cache_file = config_get_cache_path($this->globals, $path);
|
||||
if(!is_file($cache_file))
|
||||
return null;
|
||||
$ce = ConfigCacheEntry::unserialize(ensure_read($cache_file));
|
||||
$this->by_path[$path] = $ce;
|
||||
return $ce;
|
||||
}
|
||||
|
||||
return $this->by_path[$path];
|
||||
}
|
||||
|
||||
function getOrLoadByPath(string $path) : ConfigCacheEntry
|
||||
{
|
||||
$ce = $this->tryGetOrLoadByPath($path);
|
||||
if($ce === null)
|
||||
throw new Exception("Failed to find entry by path '$path'");
|
||||
return $ce;
|
||||
}
|
||||
|
||||
function tryGetOrLoadById(int $id) : ?ConfigCacheEntry
|
||||
{
|
||||
if(!array_key_exists($id, $this->by_id))
|
||||
{
|
||||
$cache_id_file = config_get_cache_id_path($this->globals, $id);
|
||||
if(!is_file($cache_id_file))
|
||||
return null;
|
||||
$file = ensure_read($cache_id_file);
|
||||
$ce = $this->tryGetOrLoadByPath($file);
|
||||
$this->by_id[$id] = $ce;
|
||||
return $ce;
|
||||
}
|
||||
|
||||
return $this->by_id[$id];
|
||||
}
|
||||
|
||||
function getOrLoadById(int $id) : ConfigCacheEntry
|
||||
{
|
||||
$ce = $this->tryGetOrLoadById($id);
|
||||
if($ce === null)
|
||||
throw new Exception("Failed to find entry by id '$id'");
|
||||
return $ce;
|
||||
}
|
||||
|
||||
function tryGetOrLoadByStrid(string $strid) : ?ConfigCacheEntry
|
||||
{
|
||||
if(!array_key_exists($strid, $this->by_alias))
|
||||
{
|
||||
$cache_strid_file = config_get_cache_strid_path($this->globals, $strid);
|
||||
if(!is_file($cache_strid_file))
|
||||
return null;
|
||||
$file = ensure_read($cache_strid_file);
|
||||
$ce = $this->tryGetOrLoadByPath($file);
|
||||
$this->by_alias[$strid] = $ce;
|
||||
return $ce;
|
||||
}
|
||||
|
||||
return $this->by_alias[$strid];
|
||||
}
|
||||
|
||||
function getOrLoadByStrid(string $strid) : ConfigCacheEntry
|
||||
{
|
||||
$ce = $this->tryGetOrLoadByStrid($strid);
|
||||
if($ce === null)
|
||||
throw new Exception("Failed to find entry by strid '$strid'");
|
||||
return $ce;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @property object $config
|
||||
* @property string $payload
|
||||
|
|
@ -146,22 +241,100 @@ class ConfigCacheEntryExtras extends \stdClass
|
|||
}
|
||||
}
|
||||
|
||||
class ConfigFetchParams
|
||||
class ConfigCacheFileMap
|
||||
{
|
||||
private array $file2deps = array();
|
||||
|
||||
static function serialize(ConfigCacheFileMap $map) : string
|
||||
{
|
||||
$d = $map->export();
|
||||
return serialize($d);
|
||||
}
|
||||
|
||||
static function unserialize(string $str) : ?ConfigCacheFileMap
|
||||
{
|
||||
$d = @unserialize($str);
|
||||
if(!is_array($d))
|
||||
return null;
|
||||
$map = new ConfigCacheFileMap();
|
||||
$map->import($d);
|
||||
return $map;
|
||||
}
|
||||
|
||||
function getAffectedFiles(string $file) : array
|
||||
{
|
||||
if(!isset($this->file2deps[$file]))
|
||||
return [];
|
||||
return array_keys($this->file2deps[$file]);
|
||||
}
|
||||
|
||||
function update(array $files)
|
||||
{
|
||||
$this_files = array_keys($this->file2deps);
|
||||
|
||||
$added = array_diff($files, $this_files);
|
||||
$removed = array_diff($this_files, $files);
|
||||
|
||||
foreach($added as $file)
|
||||
$this->file2deps[$file] = [];
|
||||
foreach($removed as $file)
|
||||
unset($this->file2deps[$file]);
|
||||
|
||||
return [$added, $removed];
|
||||
}
|
||||
|
||||
function updateDepsForEntry(ConfigCacheEntry $entry)
|
||||
{
|
||||
foreach($entry->includes as $include)
|
||||
{
|
||||
if(isset($this->file2deps[$include]))
|
||||
$this->file2deps[$include][$entry->file] = true;
|
||||
else
|
||||
$this->file2deps[$include] = [$entry->file => true];
|
||||
}
|
||||
}
|
||||
|
||||
function export() : array
|
||||
{
|
||||
$d = array();
|
||||
$d[] = $this->file2deps;
|
||||
return $d;
|
||||
}
|
||||
|
||||
function import(array $d)
|
||||
{
|
||||
list($this->file2deps) = $d;
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigCacheUpdateResult
|
||||
{
|
||||
public ConfigDirFiles $affected_files;
|
||||
|
||||
/** @var array<string, string> */
|
||||
public array $errors = array();
|
||||
public int $corruptions = 0;
|
||||
public int $fast_jsons = 0;
|
||||
|
||||
function __construct(ConfigCacheUpdateParams $params)
|
||||
{
|
||||
$this->affected_files = $params->affected_files;
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigCacheUpdateParams
|
||||
{
|
||||
public ConfigGlobals $globals;
|
||||
public ConfigScanResult $scanned;
|
||||
public bool $force_stale = false;
|
||||
public ConfigDirFiles $affected_files;
|
||||
public bool $verbose = false;
|
||||
public ?int $max_workers = null;
|
||||
public bool $touch_files_with_includes = true;
|
||||
public bool $check_junk = true;
|
||||
|
||||
function __construct(ConfigGlobals $globals, ConfigScanResult $scanned,
|
||||
bool $force_stale = false, bool $verbose = false, ?int $max_workers = null)
|
||||
function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files,
|
||||
bool $verbose = false, ?int $max_workers = null)
|
||||
{
|
||||
$this->globals = $globals;
|
||||
$this->scanned = $scanned;
|
||||
$this->force_stale = $force_stale;
|
||||
$this->affected_files = $affected_files;
|
||||
$this->verbose = $verbose;
|
||||
$this->max_workers = $max_workers;
|
||||
}
|
||||
|
|
@ -170,13 +343,13 @@ class ConfigFetchParams
|
|||
{
|
||||
$max_workers = $this->max_workers;
|
||||
if($max_workers === null)
|
||||
$max_workers = $this->scanned->count() < 100 ? 1 : 5;
|
||||
$max_workers = $this->affected_files->count() < 100 ? 1 : 5;
|
||||
return $max_workers;
|
||||
}
|
||||
|
||||
function splitFilesByChunks(int $max_workers, bool $sort = true) : array
|
||||
{
|
||||
$flat = $this->scanned->getFlatArray();
|
||||
$flat = $this->affected_files->getFlatArray();
|
||||
if($sort)
|
||||
usort($flat, fn($a, $b) => $a[1] <=> $b[1]);
|
||||
|
||||
|
|
@ -196,49 +369,41 @@ class ConfigFetchParams
|
|||
}
|
||||
}
|
||||
|
||||
class ConfigFetchResult
|
||||
function _config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
|
||||
{
|
||||
/** @var ConfigCacheEntry[] */
|
||||
public array $all = array();
|
||||
/** @var array<int, ConfigCacheEntry> */
|
||||
public array $by_id = array();
|
||||
/** @var array<string, ConfigCacheEntry> */
|
||||
public array $by_path = array();
|
||||
/** @var array<string, ConfigCacheEntry> */
|
||||
public array $by_alias = array();
|
||||
/** @var array<string, string> */
|
||||
public array $errors = array();
|
||||
public int $stales = 0;
|
||||
public int $corruptions = 0;
|
||||
public int $fast_jsons = 0;
|
||||
$jobs = $params->splitJobs(sort: true);
|
||||
|
||||
function getByAlias(string $strid) : ConfigCacheEntry
|
||||
$serial = sizeof($jobs) == 1;
|
||||
|
||||
$results_by_job = _config_update_worker_run_procs($params, $jobs, $serial);
|
||||
|
||||
if(!$serial)
|
||||
{
|
||||
if(array_key_exists($strid, $this->by_alias))
|
||||
return $this->by_alias[$strid];
|
||||
throw new Exception("Failed to find config by alias '$strid'!");
|
||||
}
|
||||
|
||||
function getById(int $id) : ConfigCacheEntry
|
||||
{
|
||||
if(array_key_exists($id, $this->by_id))
|
||||
return $this->by_id[$id];
|
||||
throw new Exception("Failed to find config by id '$id'!");
|
||||
}
|
||||
|
||||
function getByPath(string $path) : ConfigCacheEntry
|
||||
{
|
||||
if(array_key_exists($path, $this->by_path))
|
||||
return $this->by_path[$path];
|
||||
throw new Exception("Failed to find config by path '$path'!");
|
||||
//NOTE: in case result unserialize error try serial processing
|
||||
if(array_search(false, $results_by_job, true/*strict*/) !== false)
|
||||
{
|
||||
if($params->verbose)
|
||||
config_log("Corrupted result, trying serial processing...");
|
||||
$results_by_job = _config_update_worker_run_procs(params: $params, jobs: $jobs, serial: true);
|
||||
}
|
||||
}
|
||||
|
||||
$result = _config_merge_update_results($params, $results_by_job);
|
||||
|
||||
if($params->verbose)
|
||||
config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
|
||||
function config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
|
||||
{
|
||||
if($params->affected_files->isEmpty())
|
||||
return new ConfigCacheUpdateResult($params);
|
||||
|
||||
$t = microtime(true);
|
||||
|
||||
$result = _config_cache_fetch($params);
|
||||
$result = _config_cache_update($params);
|
||||
|
||||
if($result->errors)
|
||||
{
|
||||
|
|
@ -254,74 +419,26 @@ function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
|
|||
}
|
||||
|
||||
if($params->verbose)
|
||||
config_log("Fetch from cache: " . round(microtime(true) - $t,2) . " sec.");
|
||||
config_log("Update cache: " . round(microtime(true) - $t,2) . " sec.");
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
|
||||
function _config_merge_update_results(ConfigCacheUpdateParams $params, array $results_by_job) : ConfigCacheUpdateResult
|
||||
{
|
||||
if($params->scanned->isEmpty())
|
||||
return new ConfigFetchResult();
|
||||
$result = new ConfigCacheUpdateResult($params);
|
||||
|
||||
$jobs = $params->splitJobs(sort: true);
|
||||
|
||||
$serial = sizeof($jobs) == 1;
|
||||
|
||||
$results_by_job = _config_worker_run_procs($params, $jobs, $serial);
|
||||
|
||||
if(!$serial)
|
||||
{
|
||||
//NOTE: in case result unserialize error try serial processing
|
||||
if(array_search(false, $results_by_job, true/*strict*/) !== false)
|
||||
{
|
||||
if($params->verbose)
|
||||
config_log("Corrupted result, trying serial processing...");
|
||||
$results_by_job = _config_worker_run_procs($params, $jobs, true);
|
||||
}
|
||||
}
|
||||
|
||||
$t = microtime(true);
|
||||
$result = _config_merge_fetch_results($params, $results_by_job);
|
||||
if($params->verbose)
|
||||
config_log("Merge results: " . round(microtime(true) - $t,2) . " sec.");
|
||||
|
||||
if($params->verbose)
|
||||
config_log("Miss(Fast)/Total: {$result->stales}($result->fast_jsons)/" . sizeof($result->all));
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function config_cache_fetch_by_path(ConfigGlobals $globals, string $path, bool $force_stale = false) : ConfigCacheEntry
|
||||
{
|
||||
$path = realpath($path);
|
||||
$base_dir = config_map_base_dir($globals->base_dirs, $path);
|
||||
|
||||
$scanned = new ConfigScanResult();
|
||||
$scanned->add($base_dir, $path);
|
||||
|
||||
$ces = config_cache_fetch(new ConfigFetchParams(
|
||||
globals: $globals,
|
||||
scanned: $scanned,
|
||||
force_stale: $force_stale
|
||||
));
|
||||
|
||||
if(empty($ces->all))
|
||||
throw new Exception("Config not found at path '$path'");
|
||||
return $ces->all[0];
|
||||
}
|
||||
|
||||
function _config_merge_fetch_results(ConfigFetchParams $params, array $results_by_job) : ConfigFetchResult
|
||||
{
|
||||
$result = new ConfigFetchResult();
|
||||
$total_stales = 0;
|
||||
$total_fast_jsons = 0;
|
||||
|
||||
foreach($results_by_job as $results)
|
||||
{
|
||||
foreach($results as $item)
|
||||
{
|
||||
list($base_dir, $file, $cache_file, $is_stale, $parser_type, $error) = $item;
|
||||
$base_dir = $item['base_dir'];
|
||||
$file = $item['file'];
|
||||
$cache_file = $item['cache_file'];
|
||||
$parser_type = $item['parser_type'];
|
||||
$error = $item['error'];
|
||||
|
||||
if($error !== null)
|
||||
{
|
||||
|
|
@ -331,50 +448,16 @@ function _config_merge_fetch_results(ConfigFetchParams $params, array $results_b
|
|||
|
||||
if($parser_type === 1)
|
||||
++$total_fast_jsons;
|
||||
|
||||
$cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file));
|
||||
|
||||
//NOTE: handling any cache corruption errors
|
||||
if($cache_entry === null)
|
||||
{
|
||||
$is_stale = true;
|
||||
$cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
|
||||
++$result->corruptions;
|
||||
}
|
||||
|
||||
$includes = $cache_entry->includes;
|
||||
|
||||
if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes))
|
||||
{
|
||||
$is_stale = true;
|
||||
$cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
|
||||
//NOTE: let's change the mtime of the file which include other files,
|
||||
// so that on tne next build it will be 'older' than its includes
|
||||
// and won't trigger rebuild
|
||||
if($params->touch_files_with_includes)
|
||||
touch($file);
|
||||
}
|
||||
|
||||
if($is_stale)
|
||||
++$total_stales;
|
||||
|
||||
//we want results to be returned in the same order, so
|
||||
//we store entries by the file key and later retrieve array values
|
||||
$result->by_path[$file] = $cache_entry;
|
||||
$result->by_id[$cache_entry->id] = $cache_entry;
|
||||
$result->by_alias[$cache_entry->strid] = $cache_entry;
|
||||
}
|
||||
}
|
||||
|
||||
$result->all = array_values($result->by_path);
|
||||
$result->stales = $total_stales;
|
||||
$result->fast_jsons = $total_fast_jsons;
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function _config_invalidate_cache(
|
||||
ConfigFetchParams $params, string $base_dir, string $file,
|
||||
string $cache_file, ?int &$parser_type = null) : ConfigCacheEntry
|
||||
function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file,
|
||||
?int &$parser_type = null) : ConfigCacheEntry
|
||||
{
|
||||
list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
|
||||
if(!$conf_id)
|
||||
|
|
@ -399,6 +482,8 @@ function _config_invalidate_cache(
|
|||
|
||||
$cache_payload_file = config_get_cache_payload_path($params->globals, $file);
|
||||
|
||||
$cache_file = config_get_cache_path($params->globals, $file);
|
||||
|
||||
$cache_entry = new ConfigCacheEntry();
|
||||
$cache_entry->id = $conf_id;
|
||||
$cache_entry->strid = $conf_strid;
|
||||
|
|
@ -408,14 +493,90 @@ function _config_invalidate_cache(
|
|||
$cache_entry->payload_file = $cache_payload_file;
|
||||
//TODO: pass flag if file is actually normalized?
|
||||
$cache_entry->file = normalize_path($file);
|
||||
$cache_entry->includes = config_get_module_includes($pres->jsm_module);
|
||||
$cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
|
||||
$cache_entry->includes = config_get_module_includes(cm: $pres->jsm_module);
|
||||
//TODO: do we really need these refs?
|
||||
$cache_entry->refs = config_content_match_refs($pres->normalized_jzon);
|
||||
$cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
|
||||
|
||||
ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
|
||||
ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
|
||||
ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
|
||||
ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
|
||||
|
||||
return $cache_entry;
|
||||
}
|
||||
|
||||
function _config_update_worker_run_procs(ConfigCacheUpdateParams $params, array $jobs, bool $serial) : array
|
||||
{
|
||||
if($serial)
|
||||
{
|
||||
$results_by_job = array();
|
||||
foreach($jobs as $job)
|
||||
$results_by_job[] = _config_update_worker_func($params, $job, $serial);
|
||||
return $results_by_job;
|
||||
}
|
||||
else
|
||||
{
|
||||
//initializing worker for master process anyway
|
||||
$params->globals->initWorker(true);
|
||||
|
||||
$workers_args = array();
|
||||
foreach($jobs as $job)
|
||||
$workers_args[] = array($params, $job);
|
||||
|
||||
return run_background_gamectl_workers('config_update_worker', $workers_args);
|
||||
}
|
||||
}
|
||||
|
||||
function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job, bool $is_master_proc = false) : array
|
||||
{
|
||||
$start_time = microtime(true);
|
||||
|
||||
$params->globals->initWorker($is_master_proc);
|
||||
|
||||
list($idx, $start_time, $chunk) = $job;
|
||||
if($params->verbose)
|
||||
config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
|
||||
|
||||
$fast_parser_num = 0;
|
||||
$results = array();
|
||||
foreach($chunk as $file_idx => $chunk_data)
|
||||
{
|
||||
list($base_dir, $file) = $chunk_data;
|
||||
|
||||
try
|
||||
{
|
||||
if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
|
||||
config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
|
||||
|
||||
$cache_file = config_get_cache_path($params->globals, $file);
|
||||
|
||||
$parser_type = null;
|
||||
$entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
|
||||
|
||||
$results[] = [
|
||||
'base_dir' => $base_dir,
|
||||
'file' => $file,
|
||||
'cache_file' => $entry->cache_file,
|
||||
'parser_type' => $parser_type,
|
||||
'error' => null
|
||||
];
|
||||
}
|
||||
catch(\Throwable $e)
|
||||
{
|
||||
$results[] = [
|
||||
'base_dir' => $base_dir,
|
||||
'file' => $file,
|
||||
'cache_file' => null,
|
||||
'parser_type' => null,
|
||||
'error' => $e->getMessage()
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
if($params->verbose)
|
||||
config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
|
|
|
|||
219
config.inc.php
219
config.inc.php
|
|
@ -15,6 +15,7 @@ class ConfigGlobals
|
|||
public ?string $worker_init_fn = null;
|
||||
public string $base_class = '\ConfBase';
|
||||
public string $build_dir;
|
||||
public ?ConfigCacheFileMap $file_map;
|
||||
|
||||
function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null)
|
||||
{
|
||||
|
|
@ -35,6 +36,210 @@ class ConfigGlobals
|
|||
}
|
||||
}
|
||||
|
||||
enum ConfigUpdateMode : int
|
||||
{
|
||||
case ChangedOnly = 1;
|
||||
case RelativeToBundle = 2;
|
||||
case Full = 3;
|
||||
}
|
||||
|
||||
class ConfigManager
|
||||
{
|
||||
private ConfigGlobals $globals;
|
||||
private int $workers_num;
|
||||
|
||||
private ConfigCache $cache;
|
||||
|
||||
function __construct(ConfigGlobals $globals, int $workers_num)
|
||||
{
|
||||
$this->globals = $globals;
|
||||
$this->workers_num = $workers_num;
|
||||
|
||||
$this->cache = new ConfigCache($globals);
|
||||
}
|
||||
|
||||
function getGlobals() : ConfigGlobals
|
||||
{
|
||||
return $this->globals;
|
||||
}
|
||||
|
||||
function getCache() : ConfigCache
|
||||
{
|
||||
return $this->cache;
|
||||
}
|
||||
|
||||
function getArtifactFilesSpec() : array
|
||||
{
|
||||
return [$this->globals->base_dirs, ['.js']];
|
||||
}
|
||||
|
||||
function updateCache(
|
||||
ConfigUpdateMode $update_mode,
|
||||
ConfigDirFiles $input_files = null,
|
||||
?string $result_bundle_file = null,
|
||||
bool $verbose = false
|
||||
) : ConfigCacheUpdateResult
|
||||
{
|
||||
if($input_files === null && $update_mode === ConfigUpdateMode::ChangedOnly)
|
||||
throw new Exception("input_files argument is required for ChangedOnly mode");
|
||||
|
||||
if($input_files === null)
|
||||
$input_files = $this->scanFiles(extension: '.js', verbose: $verbose);
|
||||
|
||||
$added_files = [];
|
||||
$removed_files = [];
|
||||
$fs_cache_map = $this->_checkFileMap(
|
||||
$update_mode,
|
||||
$input_files,
|
||||
$added_files,
|
||||
$removed_files
|
||||
);
|
||||
|
||||
$affected_files = $this->_getAffectedFiles(
|
||||
$update_mode,
|
||||
$result_bundle_file,
|
||||
$fs_cache_map,
|
||||
$input_files,
|
||||
$removed_files
|
||||
);
|
||||
|
||||
//NOTE: at this poine taking into account only config files
|
||||
$affected_files->filter(fn($file) => str_ends_with($file, '.conf.js'));
|
||||
|
||||
$update_params = new ConfigCacheUpdateParams(
|
||||
globals: $this->globals,
|
||||
affected_files: $affected_files,
|
||||
verbose: $verbose,
|
||||
max_workers: $this->workers_num
|
||||
);
|
||||
$update_result = config_cache_update($update_params);
|
||||
|
||||
$this->cache->clear();
|
||||
|
||||
//TODO: traverse all affected files and update file map
|
||||
foreach($affected_files as $file)
|
||||
{
|
||||
$cache_entry = $this->cache->getOrLoadByPath($file);
|
||||
$fs_cache_map->updateDepsForEntry($cache_entry);
|
||||
}
|
||||
|
||||
if($affected_files->count() > 0 || $added_files || $removed_files)
|
||||
$this->_saveMap($fs_cache_map);
|
||||
|
||||
return $update_result;
|
||||
}
|
||||
|
||||
private function _checkFileMap(ConfigUpdateMode $update_mode, ConfigDirFiles $input_files, array &$added_files, array &$removed_files) : ConfigCacheFileMap
|
||||
{
|
||||
//TODO: do we need to check if cache stale somehow?
|
||||
$fs_cache_map = $this->_tryLoadMap();
|
||||
//NOTE: if there's no map so far we need to create one
|
||||
if($fs_cache_map === null)
|
||||
{
|
||||
$fs_cache_map = new ConfigCacheFileMap();
|
||||
|
||||
//let's re-use the input files
|
||||
if($update_mode === ConfigUpdateMode::Full || $update_mode === ConfigUpdateMode::RelativeToBundle)
|
||||
$tmp_files = $input_files;
|
||||
else
|
||||
$tmp_files = $this->scanFiles(extension: '.js');
|
||||
|
||||
$fs_cache_map->update($tmp_files->getAllFiles());
|
||||
}
|
||||
|
||||
if($update_mode === ConfigUpdateMode::Full || $update_mode === ConfigUpdateMode::RelativeToBundle)
|
||||
{
|
||||
list($added_files, $removed_files) = $fs_cache_map->update($input_files->getAllFiles());
|
||||
config_log("File map added: ".count($added_files).", removed: ".count($removed_files));
|
||||
}
|
||||
|
||||
return $fs_cache_map;
|
||||
}
|
||||
|
||||
private function _getAffectedFiles(ConfigUpdateMode $update_mode, ?string $result_bundle_file, ConfigCacheFileMap $fs_cache_map, ConfigDirFiles $input_files, array $removed_files) : ConfigDirFiles
|
||||
{
|
||||
$affected_files = null;
|
||||
|
||||
if($update_mode === ConfigUpdateMode::Full)
|
||||
{
|
||||
$affected_files = $input_files;
|
||||
}
|
||||
else if($update_mode === ConfigUpdateMode::RelativeToBundle)
|
||||
{
|
||||
if($result_bundle_file === null)
|
||||
throw new Exception("result_bundle_file argument is required");
|
||||
|
||||
$affected_files = $this->newDirFiles();
|
||||
|
||||
foreach($input_files->getMap() as $base_dir => $files)
|
||||
{
|
||||
foreach($files as $file)
|
||||
{
|
||||
if(need_to_regen($result_bundle_file, [$file]))
|
||||
{
|
||||
$affected_files->add($base_dir, $file);
|
||||
|
||||
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
||||
foreach($affected_by_file as $dep)
|
||||
$affected_files->addFile($dep, unique: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//if there were removed files we need to rebuild affected files
|
||||
foreach($removed_files as $file)
|
||||
{
|
||||
if(!str_ends_with($file, '.conf.js'))
|
||||
{
|
||||
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
||||
foreach($affected_by_file as $dep)
|
||||
$affected_files->add(config_map_base_dir($this->globals->base_dirs, $dep), $dep, unique: true);
|
||||
}
|
||||
else
|
||||
{
|
||||
//TODO: in case config file was removed do we actually need to rebuild all configs?
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
throw new Exception("Unknown update mode: {$update_mode->value}");
|
||||
|
||||
return $affected_files;
|
||||
}
|
||||
|
||||
function newDirFiles() : ConfigDirFiles
|
||||
{
|
||||
return new ConfigDirFiles([], $this->globals->base_dirs);
|
||||
}
|
||||
|
||||
function parseFile(string $file) : ConfigParseResult
|
||||
{
|
||||
return config_parse($this->globals->base_dirs, $file);
|
||||
}
|
||||
|
||||
function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
|
||||
{
|
||||
return config_scan_files($this->globals->base_dirs, $extension, $verbose);
|
||||
}
|
||||
|
||||
private function _getMapPath() : string
|
||||
{
|
||||
return $this->globals->build_dir . '/file_map.data';
|
||||
}
|
||||
|
||||
private function _tryLoadMap() : ?ConfigCacheFileMap
|
||||
{
|
||||
if(!is_file($this->_getMapPath()))
|
||||
return null;
|
||||
return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
|
||||
}
|
||||
|
||||
private function _saveMap(ConfigCacheFileMap $map)
|
||||
{
|
||||
ensure_write($this->_getMapPath(), ConfigCacheFileMap::serialize($map));
|
||||
}
|
||||
}
|
||||
|
||||
function config_log($msg)
|
||||
{
|
||||
echo "[CFG] $msg\n";
|
||||
|
|
@ -42,12 +247,22 @@ function config_log($msg)
|
|||
|
||||
function config_get_cache_path(ConfigGlobals $globals, string $file) : string
|
||||
{
|
||||
return config_get_tmp_build_path($globals, $file . '.cacheb');
|
||||
return config_get_tmp_build_path($globals, $file . '.cache');
|
||||
}
|
||||
|
||||
function config_get_cache_id_path(ConfigGlobals $globals, int $id) : string
|
||||
{
|
||||
return config_get_tmp_build_path($globals, $id . '.id');
|
||||
}
|
||||
|
||||
function config_get_cache_strid_path(ConfigGlobals $globals, string $strid) : string
|
||||
{
|
||||
return config_get_tmp_build_path($globals, $strid . '.strid');
|
||||
}
|
||||
|
||||
function config_get_cache_payload_path(ConfigGlobals $globals, string $file) : string
|
||||
{
|
||||
return config_get_tmp_build_path($globals, $file . '.pdata');
|
||||
return config_get_tmp_build_path($globals, $file . '.payload');
|
||||
}
|
||||
|
||||
function config_load(ConfigGlobals $globals, string $conf_path, ?string $conf_dir = null) : object
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use Exception;
|
|||
class ConfigPackParams
|
||||
{
|
||||
/*var ConfigCacheEntry[]*/
|
||||
public array $cache_entries;
|
||||
public iterable $cache_entries;
|
||||
public bool $use_lz4 = false;
|
||||
public bool $use_config_refs = false;
|
||||
public int $binary_format = 1; //1,2 supported
|
||||
|
|
@ -17,7 +17,7 @@ class ConfigPackParams
|
|||
public array $extras = array();
|
||||
|
||||
function __construct(
|
||||
array $cache_entries,
|
||||
iterable $cache_entries,
|
||||
int $version,
|
||||
bool $use_lz4 = false,
|
||||
bool $use_config_refs = false,
|
||||
|
|
@ -115,7 +115,7 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
|
|||
|
||||
//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build)
|
||||
function _config_pack_bundle_fmt1(
|
||||
array $cache_entries,
|
||||
iterable $cache_entries,
|
||||
bool $use_lz4,
|
||||
bool $use_config_refs,
|
||||
int $version) : string
|
||||
|
|
@ -173,7 +173,7 @@ function _config_pack_bundle_fmt1(
|
|||
//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
|
||||
// (each path item separated by '/' is stored as an array item)
|
||||
function _config_pack_bundle_fmt2(
|
||||
array $cache_entries,
|
||||
iterable $cache_entries,
|
||||
bool $use_lz4,
|
||||
bool $use_config_refs,
|
||||
int $version,
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ class ConfigParseResult
|
|||
public string $normalized_jzon = '';
|
||||
public array $parsed_arr;
|
||||
public int $parser_type;
|
||||
public $jsm_module;
|
||||
public \JSM_Module $jsm_module;
|
||||
}
|
||||
|
||||
function config_parse(array $base_dirs, string $file) : ConfigParseResult
|
||||
|
|
|
|||
46
scan.inc.php
46
scan.inc.php
|
|
@ -2,17 +2,24 @@
|
|||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator
|
||||
class ConfigDirFiles implements \ArrayAccess, \Countable, \Iterator
|
||||
{
|
||||
private array $base_dirs;
|
||||
|
||||
/*var array<string, string[]>*/
|
||||
public array $base_dir2files = array();
|
||||
private array $base_dir2files = array();
|
||||
|
||||
private $iter_pos = 0;
|
||||
|
||||
function __construct(array $base_dir2files = array())
|
||||
function __construct(array $base_dir2files = array(), ?array $base_dirs = null)
|
||||
{
|
||||
foreach($base_dir2files as $dir => $files)
|
||||
$this->base_dir2files[$dir] = $files;
|
||||
|
||||
if($base_dirs === null)
|
||||
$this->base_dirs = array_keys($base_dir2files);
|
||||
else
|
||||
$this->base_dirs = $base_dirs;
|
||||
}
|
||||
|
||||
function clear()
|
||||
|
|
@ -42,7 +49,11 @@ class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator
|
|||
function filter(callable $filter)
|
||||
{
|
||||
foreach($this->base_dir2files as $base_dir => $files)
|
||||
$this->base_dir2files[$base_dir] = array_filter($files, $filter);
|
||||
{
|
||||
$filtered = array_filter($files, $filter);
|
||||
//NOTE: don't want any index gaps
|
||||
$this->base_dir2files[$base_dir] = array_values($filtered);
|
||||
}
|
||||
}
|
||||
|
||||
function forEachFile(callable $fn)
|
||||
|
|
@ -54,12 +65,23 @@ class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator
|
|||
}
|
||||
}
|
||||
|
||||
function add(string $base_dir, string $file)
|
||||
function addFile(string $file, bool $unique = false)
|
||||
{
|
||||
$this->add(config_map_base_dir($this->base_dirs, $file, normalized: true), $file, $unique);
|
||||
}
|
||||
|
||||
function add(string $base_dir, string $file, bool $unique = false)
|
||||
{
|
||||
if(!isset($this->base_dir2files[$base_dir]))
|
||||
$this->base_dir2files[$base_dir] = array();
|
||||
|
||||
$this->base_dir2files[$base_dir][] = $file;
|
||||
if(!$unique || !in_array($file, $this->base_dir2files[$base_dir]))
|
||||
$this->base_dir2files[$base_dir][] = $file;
|
||||
}
|
||||
|
||||
function getMap() : array
|
||||
{
|
||||
return $this->base_dir2files;
|
||||
}
|
||||
|
||||
//returns [[base_dir, file1], [base_dir, file2], ...]
|
||||
|
|
@ -172,24 +194,26 @@ function config_scan_files(
|
|||
iterable $base_dirs,
|
||||
string $ext_filter = '.conf.js',
|
||||
bool $verbose = false
|
||||
) : ConfigScanResult
|
||||
) : ConfigDirFiles
|
||||
{
|
||||
$t = microtime(true);
|
||||
|
||||
$result = new ConfigScanResult();
|
||||
|
||||
$base_dir2files = [];
|
||||
foreach($base_dirs as $base_dir)
|
||||
{
|
||||
$result->base_dir2files[$base_dir] =
|
||||
scan_files_rec(array($base_dir), array($ext_filter));
|
||||
$base_dir2files[$base_dir] =
|
||||
scan_files_rec([$base_dir], [$ext_filter]);
|
||||
}
|
||||
|
||||
$result = new ConfigDirFiles($base_dir2files);
|
||||
|
||||
if($verbose)
|
||||
config_log("File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec.");
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
//NOTE: this one is obsolete
|
||||
function config_hash_changed(ConfigGlobals $globals, iterable $all_files)
|
||||
{
|
||||
$all_crc_file = $globals->build_dir . "/configs.crc";
|
||||
|
|
|
|||
74
task.inc.php
74
task.inc.php
|
|
@ -3,7 +3,7 @@ namespace taskman;
|
|||
use Exception;
|
||||
use Throwable;
|
||||
|
||||
task('config_worker', function(array $args)
|
||||
task('config_update_worker', function(array $args)
|
||||
{
|
||||
if(sizeof($args) != 3)
|
||||
throw new Exception("Config worker args not set");
|
||||
|
|
@ -15,7 +15,7 @@ task('config_worker', function(array $args)
|
|||
try
|
||||
{
|
||||
list($params, $job) = unserialize(ensure_read($in_file));
|
||||
$result = _config_worker_func($params, $job);
|
||||
$result = _config_update_worker_func($params, $job);
|
||||
ensure_write($out_file, serialize($result));
|
||||
}
|
||||
catch(Throwable $e)
|
||||
|
|
@ -26,73 +26,3 @@ task('config_worker', function(array $args)
|
|||
throw $e;
|
||||
}
|
||||
});
|
||||
|
||||
function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $serial) : array
|
||||
{
|
||||
if($serial)
|
||||
{
|
||||
$results_by_job = array();
|
||||
foreach($jobs as $job)
|
||||
$results_by_job[] = _config_worker_func($params, $job, $serial);
|
||||
return $results_by_job;
|
||||
}
|
||||
else
|
||||
{
|
||||
//initializing worker for master process anyway
|
||||
$params->globals->initWorker(true);
|
||||
|
||||
$workers_args = array();
|
||||
foreach($jobs as $job)
|
||||
$workers_args[] = array($params, $job);
|
||||
|
||||
return run_background_gamectl_workers('config_worker', $workers_args);
|
||||
}
|
||||
}
|
||||
|
||||
//returns [[base_dir, file, cache_file, was_stale, parser_type, error], ...]
|
||||
function _config_worker_func(ConfigFetchParams $params, array $job, bool $is_master_proc = false) : array
|
||||
{
|
||||
$start_time = microtime(true);
|
||||
|
||||
$params->globals->initWorker($is_master_proc);
|
||||
|
||||
list($idx, $start_time, $chunk) = $job;
|
||||
if($params->verbose)
|
||||
config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
|
||||
|
||||
$fast_parser_num = 0;
|
||||
$results = array();
|
||||
foreach($chunk as $file_idx => $chunk_data)
|
||||
{
|
||||
list($base_dir, $file) = $chunk_data;
|
||||
|
||||
try
|
||||
{
|
||||
if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
|
||||
config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
|
||||
|
||||
$cache_file = config_get_cache_path($params->globals, $file);
|
||||
|
||||
$parser_type = null;
|
||||
|
||||
$is_stale = true;
|
||||
if(!$params->force_stale)
|
||||
$is_stale = need_to_regen($cache_file, array($file));
|
||||
|
||||
if($is_stale)
|
||||
_config_invalidate_cache($params, $base_dir, $file, $cache_file, $parser_type);
|
||||
|
||||
$results[] = array($base_dir, $file, $cache_file, $is_stale, $parser_type, null);
|
||||
}
|
||||
catch(Throwable $e)
|
||||
{
|
||||
$results[] = array($base_dir, $file, null, null, null, $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
if($params->verbose)
|
||||
config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use Exception;
|
|||
|
||||
function config_get_tmp_build_path(ConfigGlobals $globals, string $file) : string
|
||||
{
|
||||
$name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file))));
|
||||
$name = str_replace(['@', ':', "\\", "/"], ['', "-", "-", "-"], normalize_path($file));
|
||||
$name = ltrim($name, "-");
|
||||
return normalize_path($globals->build_dir . "/$name");
|
||||
}
|
||||
|
|
@ -70,6 +70,7 @@ function config_crc28(string $what) : int
|
|||
function config_get_module_includes(\JSM_Module $cm) : array
|
||||
{
|
||||
$includes = array();
|
||||
//NOTE: module contains all includes (even those included from other modules)
|
||||
foreach($cm->getIncludes() as $include => $_)
|
||||
{
|
||||
//maybe we should take .php includes into account as well?
|
||||
|
|
|
|||
Loading…
Reference in New Issue