584 lines
15 KiB
PHP
584 lines
15 KiB
PHP
<?php
|
|
namespace taskman;
|
|
use Exception;
|
|
|
|
class ConfigCache
|
|
{
|
|
private ConfigGlobals $globals;
|
|
|
|
/** @var array<string, ConfigCacheEntry> */
|
|
private array $by_path = [];
|
|
/** @var array<int, ConfigCacheEntry> */
|
|
private array $by_id = [];
|
|
/** @var array<string, ConfigCacheEntry> */
|
|
private array $by_alias = [];
|
|
|
|
function __construct(ConfigGlobals $globals)
|
|
{
|
|
$this->globals = $globals;
|
|
}
|
|
|
|
function clear()
|
|
{
|
|
$this->by_path = [];
|
|
$this->by_id = [];
|
|
$this->by_alias = [];
|
|
}
|
|
|
|
function tryGetOrLoadByPath(string $path) : ?ConfigCacheEntry
|
|
{
|
|
if(!array_key_exists($path, $this->by_path))
|
|
{
|
|
$cache_file = config_get_cache_path($this->globals, $path);
|
|
if(!is_file($cache_file))
|
|
return null;
|
|
$ce = ConfigCacheEntry::unserialize(ensure_read($cache_file));
|
|
$this->by_path[$path] = $ce;
|
|
return $ce;
|
|
}
|
|
|
|
return $this->by_path[$path];
|
|
}
|
|
|
|
function getOrLoadByPath(string $path) : ConfigCacheEntry
|
|
{
|
|
$ce = $this->tryGetOrLoadByPath($path);
|
|
if($ce === null)
|
|
throw new Exception("Failed to find entry by path '$path'");
|
|
return $ce;
|
|
}
|
|
|
|
function tryGetOrLoadById(int $id) : ?ConfigCacheEntry
|
|
{
|
|
if(!array_key_exists($id, $this->by_id))
|
|
{
|
|
$cache_id_file = config_get_cache_id_path($this->globals, $id);
|
|
if(!is_file($cache_id_file))
|
|
return null;
|
|
$file = ensure_read($cache_id_file);
|
|
$ce = $this->tryGetOrLoadByPath($file);
|
|
$this->by_id[$id] = $ce;
|
|
return $ce;
|
|
}
|
|
|
|
return $this->by_id[$id];
|
|
}
|
|
|
|
function getOrLoadById(int $id) : ConfigCacheEntry
|
|
{
|
|
$ce = $this->tryGetOrLoadById($id);
|
|
if($ce === null)
|
|
throw new Exception("Failed to find entry by id '$id'");
|
|
return $ce;
|
|
}
|
|
|
|
function tryGetOrLoadByStrid(string $strid) : ?ConfigCacheEntry
|
|
{
|
|
if(!array_key_exists($strid, $this->by_alias))
|
|
{
|
|
$cache_strid_file = config_get_cache_strid_path($this->globals, $strid);
|
|
if(!is_file($cache_strid_file))
|
|
return null;
|
|
$file = ensure_read($cache_strid_file);
|
|
$ce = $this->tryGetOrLoadByPath($file);
|
|
$this->by_alias[$strid] = $ce;
|
|
return $ce;
|
|
}
|
|
|
|
return $this->by_alias[$strid];
|
|
}
|
|
|
|
function getOrLoadByStrid(string $strid) : ConfigCacheEntry
|
|
{
|
|
$ce = $this->tryGetOrLoadByStrid($strid);
|
|
if($ce === null)
|
|
throw new Exception("Failed to find entry by strid '$strid'");
|
|
return $ce;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @property object $config
|
|
* @property string $payload
|
|
*/
|
|
class ConfigCacheEntry
|
|
{
|
|
const FMT_BINARY = 0;
|
|
const FMT_LZ4 = 1;
|
|
const FMT_FILE_REF = 2;
|
|
|
|
public string $class;
|
|
public int $class_id;
|
|
public int $id;
|
|
public string $strid;
|
|
public string $cache_file;
|
|
//NOTE: actual payload is stored in a separate file for faster incremental retrievals
|
|
public string $payload_file;
|
|
public string $file;
|
|
public array $includes = array();
|
|
public array $refs = array();
|
|
public ?ConfigCacheEntryExtras $extras;
|
|
|
|
public $_config;
|
|
public $_payload;
|
|
|
|
static function serialize(ConfigCacheEntry $ce) : string
|
|
{
|
|
$d = $ce->export();
|
|
return serialize($d);
|
|
}
|
|
|
|
static function unserialize(string $str) : ?ConfigCacheEntry
|
|
{
|
|
$d = @unserialize($str);
|
|
if(!is_array($d))
|
|
return null;
|
|
$ce = new ConfigCacheEntry();
|
|
$ce->import($d);
|
|
return $ce;
|
|
}
|
|
|
|
function __construct()
|
|
{
|
|
$this->extras = ConfigCacheEntryExtras::create();
|
|
}
|
|
|
|
function __get(string $name)
|
|
{
|
|
if($name === "config")
|
|
{
|
|
if($this->_config === null)
|
|
{
|
|
$klass = $this->class;
|
|
$data = config_msgpack_unpack($this->payload);
|
|
$this->_config = new $klass();
|
|
$this->_config->import($data);
|
|
}
|
|
return $this->_config;
|
|
}
|
|
else if($name === "payload")
|
|
{
|
|
if($this->_payload !== null)
|
|
return $this->_payload;
|
|
//if payload not set directly not storing it
|
|
return ensure_read($this->payload_file);
|
|
}
|
|
else
|
|
throw new Exception("No such property '$name'");
|
|
}
|
|
|
|
function __set(string $name, $v)
|
|
{
|
|
if($name === "config")
|
|
$this->_config = $v;
|
|
else if($name === "payload")
|
|
$this->_payload = $v;
|
|
else
|
|
throw new Exception("No such property '$name'");
|
|
}
|
|
|
|
function export() : array
|
|
{
|
|
$d = array();
|
|
$d[] = $this->class;
|
|
$d[] = $this->class_id;
|
|
$d[] = $this->id;
|
|
$d[] = $this->strid;
|
|
$d[] = $this->cache_file;
|
|
$d[] = $this->payload_file;
|
|
$d[] = $this->file;
|
|
$d[] = $this->includes;
|
|
$d[] = $this->refs;
|
|
$d[] = $this->extras->export();
|
|
return $d;
|
|
}
|
|
|
|
function import(array $d)
|
|
{
|
|
$extras = array();
|
|
|
|
list(
|
|
$this->class,
|
|
$this->class_id,
|
|
$this->id,
|
|
$this->strid,
|
|
$this->cache_file,
|
|
$this->payload_file,
|
|
$this->file,
|
|
$this->includes,
|
|
$this->refs,
|
|
$extras,
|
|
) = $d;
|
|
|
|
$this->extras->import($extras);
|
|
}
|
|
}
|
|
|
|
class ConfigCacheEntryExtras extends \stdClass
|
|
{
|
|
private static string $klass = ConfigCacheEntryExtras::class;
|
|
|
|
static function init(string $project_specific_klass)
|
|
{
|
|
self::$klass = $project_specific_klass;
|
|
}
|
|
|
|
static function create() : object
|
|
{
|
|
return new self::$klass();
|
|
}
|
|
|
|
function export() : array
|
|
{
|
|
$as_array = get_object_vars($this);
|
|
return $as_array;
|
|
}
|
|
|
|
function import(array $as_array)
|
|
{
|
|
foreach($as_array as $field_name => $field_value)
|
|
$this->$field_name = $field_value;
|
|
}
|
|
}
|
|
|
|
class ConfigCacheFileMap
|
|
{
|
|
private array $file2deps = array();
|
|
|
|
static function serialize(ConfigCacheFileMap $map) : string
|
|
{
|
|
$d = $map->export();
|
|
return serialize($d);
|
|
}
|
|
|
|
static function unserialize(string $str) : ?ConfigCacheFileMap
|
|
{
|
|
$d = @unserialize($str);
|
|
if(!is_array($d))
|
|
return null;
|
|
$map = new ConfigCacheFileMap();
|
|
$map->import($d);
|
|
return $map;
|
|
}
|
|
|
|
function getAffectedFiles(string $file) : array
|
|
{
|
|
if(!isset($this->file2deps[$file]))
|
|
return [];
|
|
return array_keys($this->file2deps[$file]);
|
|
}
|
|
|
|
function differs(array $files) : bool
|
|
{
|
|
$this_files = array_keys($this->file2deps);
|
|
|
|
return count($files) !== count($this_files) ||
|
|
count(array_intersect($files, $this_files)) != count($this_files);
|
|
}
|
|
|
|
function compare(array $files) : array
|
|
{
|
|
$this_files = array_keys($this->file2deps);
|
|
|
|
$added = array_diff($files, $this_files);
|
|
$removed = array_diff($this_files, $files);
|
|
|
|
return [$added, $removed];
|
|
}
|
|
|
|
function init(array $files)
|
|
{
|
|
$this->file2deps = [];
|
|
foreach($files as $file)
|
|
$this->file2deps[$file] = [];
|
|
}
|
|
|
|
function update(array $added, array $removed)
|
|
{
|
|
foreach($added as $file)
|
|
{
|
|
//NOTE: we preserve existing files
|
|
if(!isset($this->file2deps[$file]))
|
|
$this->file2deps[$file] = [];
|
|
}
|
|
foreach($removed as $file)
|
|
unset($this->file2deps[$file]);
|
|
}
|
|
|
|
function updateDepsForEntry(ConfigCacheEntry $entry)
|
|
{
|
|
foreach($entry->includes as $include)
|
|
{
|
|
if(isset($this->file2deps[$include]))
|
|
$this->file2deps[$include][$entry->file] = true;
|
|
else
|
|
$this->file2deps[$include] = [$entry->file => true];
|
|
}
|
|
}
|
|
|
|
function export() : array
|
|
{
|
|
$d = array();
|
|
$d[] = $this->file2deps;
|
|
return $d;
|
|
}
|
|
|
|
function import(array $d)
|
|
{
|
|
list($this->file2deps) = $d;
|
|
}
|
|
}
|
|
|
|
class ConfigCacheUpdateResult
|
|
{
|
|
public ConfigDirFiles $affected_files;
|
|
public array $affected_entries = array();
|
|
|
|
/** @var array<string, string> */
|
|
public array $errors = array();
|
|
public int $corruptions = 0;
|
|
public int $fast_jsons = 0;
|
|
|
|
function __construct(ConfigCacheUpdateParams $params)
|
|
{
|
|
$this->affected_files = $params->affected_files;
|
|
}
|
|
}
|
|
|
|
class ConfigCacheUpdateParams
|
|
{
|
|
const MAX_DEFAULT_WORKERS = 5;
|
|
const FILES_THRESHOLD_ONE_WORKER = 100;
|
|
|
|
public ConfigGlobals $globals;
|
|
public ConfigDirFiles $affected_files;
|
|
public bool $verbose = false;
|
|
public ?int $max_workers = null;
|
|
public bool $check_junk = true;
|
|
public int $max_errors_num = 15;
|
|
public bool $return_affected_entries = false;
|
|
|
|
function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files,
|
|
bool $verbose = false, ?int $max_workers = null)
|
|
{
|
|
$this->globals = $globals;
|
|
$this->affected_files = $affected_files;
|
|
$this->verbose = $verbose;
|
|
$this->max_workers = $max_workers;
|
|
}
|
|
|
|
function calcMaxWorkers() : int
|
|
{
|
|
if($this->affected_files->count() < self::FILES_THRESHOLD_ONE_WORKER)
|
|
return 1;
|
|
return $this->max_workers ?? self::MAX_DEFAULT_WORKERS;
|
|
}
|
|
|
|
function splitFilesByChunks(int $max_workers, bool $sort = true) : array
|
|
{
|
|
$flat = $this->affected_files->getFlatArray();
|
|
if(!$flat)
|
|
return array();
|
|
if($sort)
|
|
usort($flat, fn($a, $b) => $a[1] <=> $b[1]);
|
|
|
|
$chunk_size = (int)ceil(count($flat)/$max_workers);
|
|
return array_chunk($flat, $chunk_size);
|
|
}
|
|
|
|
//returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ]
|
|
function splitJobs(bool $sort = true) : array
|
|
{
|
|
$max_workers = $this->calcMaxWorkers();
|
|
$jobs = array();
|
|
$chunks = $this->splitFilesByChunks($max_workers, $sort);
|
|
foreach($chunks as $idx => $chunk)
|
|
$jobs[] = array($idx, microtime(true), $chunk);
|
|
return $jobs;
|
|
}
|
|
}
|
|
|
|
function _config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
|
|
{
|
|
$jobs = $params->splitJobs(sort: true);
|
|
|
|
$serial = sizeof($jobs) == 1;
|
|
|
|
$results_by_job = _config_update_worker_run_procs($params, $jobs, $serial);
|
|
|
|
if(!$serial)
|
|
{
|
|
//NOTE: in case result unserialize error try serial processing
|
|
if(array_search(false, $results_by_job, true/*strict*/) !== false)
|
|
{
|
|
if($params->verbose)
|
|
config_log("Corrupted result, trying serial processing...");
|
|
$results_by_job = _config_update_worker_run_procs(params: $params, jobs: $jobs, serial: true);
|
|
}
|
|
}
|
|
|
|
$result = _config_merge_update_results($params, $results_by_job);
|
|
|
|
if($params->verbose)
|
|
config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
|
|
|
|
return $result;
|
|
}
|
|
|
|
function _config_merge_update_results(ConfigCacheUpdateParams $params, array $results_by_job) : ConfigCacheUpdateResult
|
|
{
|
|
$result = new ConfigCacheUpdateResult($params);
|
|
|
|
$total_fast_jsons = 0;
|
|
|
|
foreach($results_by_job as $results)
|
|
{
|
|
foreach($results as $item)
|
|
{
|
|
$base_dir = $item['base_dir'];
|
|
$file = $item['file'];
|
|
$cache_file = $item['cache_file'];
|
|
$parser_type = $item['parser_type'];
|
|
$error = $item['error'];
|
|
|
|
if($error !== null)
|
|
{
|
|
$result->errors[$file] = $error;
|
|
continue;
|
|
}
|
|
|
|
if($parser_type === 1)
|
|
++$total_fast_jsons;
|
|
}
|
|
}
|
|
|
|
$result->fast_jsons = $total_fast_jsons;
|
|
|
|
return $result;
|
|
}
|
|
|
|
function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file,
|
|
?int &$parser_type = null) : ConfigCacheEntry
|
|
{
|
|
list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
|
|
if(!$conf_id)
|
|
throw new Exception("Bad conf id: {$conf_id}");
|
|
|
|
//TODO: this is a bit ugly but kinda works for now
|
|
$GLOBALS['CONFIG_CURRENT_FILE'] = $file;
|
|
$GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id;
|
|
$GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
|
|
|
|
$pres = config_parse($params->globals->base_dirs, $file);
|
|
if($pres->error !== 0)
|
|
throw new Exception("Parse error({$pres->error}):\n" . $pres->error_descr);
|
|
|
|
$parser_type = $pres->parser_type;
|
|
$parsed_arr = $pres->parsed_arr;
|
|
//TODO: these hardcoded keys below probably shouldn't be part of data?
|
|
$parsed_arr['id'] = $conf_id;
|
|
$parsed_arr['strid'] = $conf_strid;
|
|
list($class, $class_id, $normalized_data)
|
|
= config_apply_class_normalization($parsed_arr, $params->check_junk);
|
|
|
|
$cache_payload_file = config_get_cache_payload_path($params->globals, $file);
|
|
|
|
$cache_file = config_get_cache_path($params->globals, $file);
|
|
|
|
$cache_entry = new ConfigCacheEntry();
|
|
$cache_entry->id = $conf_id;
|
|
$cache_entry->strid = $conf_strid;
|
|
$cache_entry->cache_file = $cache_file;
|
|
$cache_entry->class = $class;
|
|
$cache_entry->class_id = $class_id;
|
|
$cache_entry->payload_file = $cache_payload_file;
|
|
//TODO: pass flag if file is actually normalized?
|
|
$cache_entry->file = normalize_path($file);
|
|
$cache_entry->includes = config_get_module_includes(cm: $pres->jsm_module);
|
|
//TODO: do we really need these refs?
|
|
$cache_entry->refs = config_content_match_refs($pres->normalized_jzon);
|
|
$cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
|
|
|
|
ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
|
|
ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
|
|
ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
|
|
ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
|
|
|
|
return $cache_entry;
|
|
}
|
|
|
|
function _config_update_worker_run_procs(ConfigCacheUpdateParams $params, array $jobs, bool $serial) : array
|
|
{
|
|
if($serial)
|
|
{
|
|
$results_by_job = array();
|
|
foreach($jobs as $job)
|
|
$results_by_job[] = _config_update_worker_func($params, $job, $serial);
|
|
return $results_by_job;
|
|
}
|
|
else
|
|
{
|
|
//initializing worker for master process anyway
|
|
$params->globals->initWorker(true);
|
|
|
|
$workers_args = array();
|
|
foreach($jobs as $job)
|
|
$workers_args[] = array($params, $job);
|
|
|
|
return run_background_gamectl_workers('config_update_worker', $workers_args);
|
|
}
|
|
}
|
|
|
|
function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job, bool $is_master_proc = false) : array
|
|
{
|
|
$start_time = microtime(true);
|
|
|
|
$params->globals->initWorker($is_master_proc);
|
|
|
|
list($idx, $start_time, $chunk) = $job;
|
|
if($params->verbose)
|
|
config_log("Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)");
|
|
|
|
$fast_parser_num = 0;
|
|
$results = array();
|
|
foreach($chunk as $file_idx => $chunk_data)
|
|
{
|
|
list($base_dir, $file) = $chunk_data;
|
|
|
|
try
|
|
{
|
|
if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
|
|
config_log("Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...");
|
|
|
|
$cache_file = config_get_cache_path($params->globals, $file);
|
|
|
|
$parser_type = null;
|
|
$entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
|
|
|
|
$results[] = [
|
|
'base_dir' => $base_dir,
|
|
'file' => $file,
|
|
'cache_file' => $entry->cache_file,
|
|
'parser_type' => $parser_type,
|
|
'error' => null
|
|
];
|
|
}
|
|
catch(\Throwable $e)
|
|
{
|
|
$results[] = [
|
|
'base_dir' => $base_dir,
|
|
'file' => $file,
|
|
'cache_file' => null,
|
|
'parser_type' => null,
|
|
'error' => $e->getMessage()
|
|
];
|
|
}
|
|
}
|
|
|
|
if($params->verbose)
|
|
config_log("Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)");
|
|
|
|
return $results;
|
|
}
|
|
|