Compare commits
No commits in common. "master" and "6.x" have entirely different histories.
|
|
@ -95,13 +95,6 @@ class ConfigCache
|
|||
throw new Exception("Failed to find entry by strid '$strid'");
|
||||
return $ce;
|
||||
}
|
||||
|
||||
function update(ConfigCacheEntry $ce)
|
||||
{
|
||||
$this->by_path[$ce->file] = $ce;
|
||||
$this->by_id[$ce->id] = $ce;
|
||||
$this->by_alias[$ce->strid] = $ce;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -268,16 +261,6 @@ class ConfigCacheFileMap
|
|||
return $map;
|
||||
}
|
||||
|
||||
function count() : int
|
||||
{
|
||||
return count($this->file2deps);
|
||||
}
|
||||
|
||||
function exists(string $file) : bool
|
||||
{
|
||||
return isset($this->file2deps[$file]);
|
||||
}
|
||||
|
||||
function getAffectedFiles(string $file) : array
|
||||
{
|
||||
if(!isset($this->file2deps[$file]))
|
||||
|
|
@ -285,11 +268,8 @@ class ConfigCacheFileMap
|
|||
return array_keys($this->file2deps[$file]);
|
||||
}
|
||||
|
||||
function differs(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles|array $files) : bool
|
||||
function differs(array $files) : bool
|
||||
{
|
||||
if(!is_array($files))
|
||||
$files = $files->getAllFiles();
|
||||
|
||||
$this_files = array_keys($this->file2deps);
|
||||
|
||||
return count($files) !== count($this_files) ||
|
||||
|
|
@ -325,24 +305,14 @@ class ConfigCacheFileMap
|
|||
unset($this->file2deps[$file]);
|
||||
}
|
||||
|
||||
function updateDepsForEntry(array $base_dirs, ConfigCacheEntry $entry)
|
||||
function updateDepsForEntry(ConfigCacheEntry $entry)
|
||||
{
|
||||
//1. add includes
|
||||
$all_refs = $entry->includes;
|
||||
|
||||
//2. add refs
|
||||
foreach($entry->refs as $ref => $_)
|
||||
foreach($entry->includes as $include)
|
||||
{
|
||||
$file = config_real_path($base_dirs, substr($ref, 1) . '.conf.js');
|
||||
$all_refs[] = $file;
|
||||
}
|
||||
|
||||
foreach($all_refs as $ref_file)
|
||||
{
|
||||
if(isset($this->file2deps[$ref_file]))
|
||||
$this->file2deps[$ref_file][$entry->file] = true;
|
||||
if(isset($this->file2deps[$include]))
|
||||
$this->file2deps[$include][$entry->file] = true;
|
||||
else
|
||||
$this->file2deps[$ref_file] = [$entry->file => true];
|
||||
$this->file2deps[$include] = [$entry->file => true];
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -359,6 +329,22 @@ class ConfigCacheFileMap
|
|||
}
|
||||
}
|
||||
|
||||
class ConfigCacheUpdateResult
|
||||
{
|
||||
public ConfigDirFiles $affected_files;
|
||||
public array $affected_entries = array();
|
||||
|
||||
/** @var array<string, string> */
|
||||
public array $errors = array();
|
||||
public int $corruptions = 0;
|
||||
public int $fast_jsons = 0;
|
||||
|
||||
function __construct(ConfigCacheUpdateParams $params)
|
||||
{
|
||||
$this->affected_files = $params->affected_files;
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigCacheUpdateParams
|
||||
{
|
||||
const MAX_DEFAULT_WORKERS = 5;
|
||||
|
|
@ -367,21 +353,25 @@ class ConfigCacheUpdateParams
|
|||
public ConfigGlobals $globals;
|
||||
public ConfigDirFiles $affected_files;
|
||||
public bool $verbose = false;
|
||||
public ?int $max_workers = null;
|
||||
public bool $check_junk = true;
|
||||
public int $max_errors_num = 15;
|
||||
public bool $return_affected_entries = false;
|
||||
|
||||
function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files, bool $verbose = false)
|
||||
function __construct(ConfigGlobals $globals, ConfigDirFiles $affected_files,
|
||||
bool $verbose = false, ?int $max_workers = null)
|
||||
{
|
||||
$this->globals = $globals;
|
||||
$this->affected_files = $affected_files;
|
||||
$this->verbose = $verbose;
|
||||
$this->max_workers = $max_workers;
|
||||
}
|
||||
|
||||
function calcMaxWorkers() : int
|
||||
{
|
||||
if($this->affected_files->count() < self::FILES_THRESHOLD_ONE_WORKER)
|
||||
return 1;
|
||||
return $this->globals->max_workers ?? self::MAX_DEFAULT_WORKERS;
|
||||
return $this->max_workers ?? self::MAX_DEFAULT_WORKERS;
|
||||
}
|
||||
|
||||
function splitFilesByChunks(int $max_workers, bool $sort = true) : array
|
||||
|
|
@ -408,7 +398,7 @@ class ConfigCacheUpdateParams
|
|||
}
|
||||
}
|
||||
|
||||
function _config_cache_update(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
|
||||
function _config_cache_update(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
|
||||
{
|
||||
$jobs = $params->splitJobs(sort: true);
|
||||
|
||||
|
|
@ -427,14 +417,18 @@ function _config_cache_update(ConfigUpdateResult $result, ConfigCacheUpdateParam
|
|||
}
|
||||
}
|
||||
|
||||
_config_merge_update_results($result, $results_by_job);
|
||||
$result = _config_merge_update_results($params, $results_by_job);
|
||||
|
||||
if($params->verbose)
|
||||
config_log("Miss(Fast): {$result->affected_files->count()}({$result->fast_jsons})");
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function _config_merge_update_results(ConfigUpdateResult $result, array $results_by_job)
|
||||
function _config_merge_update_results(ConfigCacheUpdateParams $params, array $results_by_job) : ConfigCacheUpdateResult
|
||||
{
|
||||
$result = new ConfigCacheUpdateResult($params);
|
||||
|
||||
$total_fast_jsons = 0;
|
||||
|
||||
foreach($results_by_job as $results)
|
||||
|
|
@ -459,9 +453,11 @@ function _config_merge_update_results(ConfigUpdateResult $result, array $results
|
|||
}
|
||||
|
||||
$result->fast_jsons = $total_fast_jsons;
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $file, string $cache_file,
|
||||
function _config_update_cache_entry(ConfigCacheUpdateParams $params, $base_dir, string $file,
|
||||
?int &$parser_type = null) : ConfigCacheEntry
|
||||
{
|
||||
list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
|
||||
|
|
@ -473,7 +469,7 @@ function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $f
|
|||
$GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id;
|
||||
$GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
|
||||
|
||||
$pres = config_parse($globals->base_dirs, $file);
|
||||
$pres = config_parse($params->globals->base_dirs, $file);
|
||||
if($pres->error !== 0)
|
||||
throw new Exception("Parse error({$pres->error}):\n" . $pres->error_descr);
|
||||
|
||||
|
|
@ -483,9 +479,11 @@ function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $f
|
|||
$parsed_arr['id'] = $conf_id;
|
||||
$parsed_arr['strid'] = $conf_strid;
|
||||
list($class, $class_id, $normalized_data)
|
||||
= config_apply_class_normalization($parsed_arr, check_junk: true);
|
||||
= config_apply_class_normalization($parsed_arr, $params->check_junk);
|
||||
|
||||
$cache_payload_file = config_get_cache_payload_path($globals, $file);
|
||||
$cache_payload_file = config_get_cache_payload_path($params->globals, $file);
|
||||
|
||||
$cache_file = config_get_cache_path($params->globals, $file);
|
||||
|
||||
$cache_entry = new ConfigCacheEntry();
|
||||
$cache_entry->id = $conf_id;
|
||||
|
|
@ -503,8 +501,8 @@ function _config_update_cache_entry(ConfigGlobals $globals, $base_dir, string $f
|
|||
|
||||
ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
|
||||
ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
|
||||
ensure_write(config_get_cache_id_path($globals, $conf_id), $cache_entry->file);
|
||||
ensure_write(config_get_cache_strid_path($globals, $conf_strid), $cache_entry->file);
|
||||
ensure_write(config_get_cache_id_path($params->globals, $conf_id), $cache_entry->file);
|
||||
ensure_write(config_get_cache_strid_path($params->globals, $conf_strid), $cache_entry->file);
|
||||
|
||||
return $cache_entry;
|
||||
}
|
||||
|
|
@ -555,12 +553,12 @@ function _config_update_worker_func(ConfigCacheUpdateParams $params, array $job,
|
|||
$cache_file = config_get_cache_path($params->globals, $file);
|
||||
|
||||
$parser_type = null;
|
||||
$entry = _config_update_cache_entry($params->globals, $base_dir, $file, $cache_file, $parser_type);
|
||||
$entry = _config_update_cache_entry($params, $base_dir, $file, $parser_type);
|
||||
|
||||
$results[] = [
|
||||
'base_dir' => $base_dir,
|
||||
'file' => $file,
|
||||
'cache_file' => $cache_file,
|
||||
'cache_file' => $entry->cache_file,
|
||||
'parser_type' => $parser_type,
|
||||
'error' => null
|
||||
];
|
||||
|
|
|
|||
271
config.inc.php
271
config.inc.php
|
|
@ -11,30 +11,16 @@ require_once(__DIR__ . '/msgpack.inc.php');
|
|||
|
||||
class ConfigGlobals
|
||||
{
|
||||
public readonly array $base_dirs;
|
||||
//NOTE: it's a string since is serialized
|
||||
public readonly ?string $worker_init_fn;
|
||||
public readonly int $workers_num;
|
||||
public readonly string $base_class;
|
||||
public readonly string $build_dir;
|
||||
//NOTE: it's a string since is serialized
|
||||
public readonly ?string $files_filter;
|
||||
public array $base_dirs = array();
|
||||
public ?string $worker_init_fn = null;
|
||||
public string $base_class = '\ConfBase';
|
||||
public string $build_dir;
|
||||
|
||||
function __construct(
|
||||
array $base_dirs,
|
||||
string $build_dir,
|
||||
?string $worker_init_fn = null,
|
||||
int $workers_num = 1,
|
||||
string $base_class = '\ConfBase',
|
||||
?string $files_filter = null
|
||||
)
|
||||
function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null)
|
||||
{
|
||||
$this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
|
||||
$this->build_dir = $build_dir;
|
||||
$this->worker_init_fn = $worker_init_fn;
|
||||
$this->workers_num = $workers_num;
|
||||
$this->base_class = $base_class;
|
||||
$this->files_filter = $files_filter;
|
||||
}
|
||||
|
||||
function initWorker(bool $is_master_proc)
|
||||
|
|
@ -49,105 +35,59 @@ class ConfigGlobals
|
|||
}
|
||||
}
|
||||
|
||||
enum ConfigGetMode : int
|
||||
{
|
||||
case ForceUpdate = 1;
|
||||
case Auto = 2;
|
||||
case CacheOnly = 3;
|
||||
}
|
||||
|
||||
enum ConfigUpdateMode : int
|
||||
{
|
||||
case Force = 1;
|
||||
case DetectChanged = 2;
|
||||
case Selected = 3;
|
||||
case Patch = 3;
|
||||
}
|
||||
|
||||
class ConfigUpdateRequest
|
||||
{
|
||||
public ConfigUpdateMode $mode;
|
||||
public ?ConfigDirFiles $files;
|
||||
public ?\taskman\TaskmanFileChanges $file_changes;
|
||||
public bool $verbose = false;
|
||||
public bool $return_affected = false;
|
||||
public ?string $result_file;
|
||||
|
||||
private function __construct() {}
|
||||
|
||||
static function force(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
|
||||
static function force(?ConfigDirFiles $files = null) : ConfigUpdateRequest
|
||||
{
|
||||
if($files instanceof \taskman\artefact\TaskmanDirFiles)
|
||||
$files = ConfigDirFiles::makeFromArtefactFiles($files);
|
||||
|
||||
$req = new ConfigUpdateRequest();
|
||||
$req->mode = ConfigUpdateMode::Force;
|
||||
$req->files = $files;
|
||||
return $req;
|
||||
}
|
||||
|
||||
static function detectChanged(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
|
||||
static function detectChanged(string $result_file, ?ConfigDirFiles $files = null) : ConfigUpdateRequest
|
||||
{
|
||||
if($files instanceof \taskman\artefact\TaskmanDirFiles)
|
||||
$files = ConfigDirFiles::makeFromArtefactFiles($files);
|
||||
|
||||
$req = new ConfigUpdateRequest();
|
||||
$req->mode = ConfigUpdateMode::DetectChanged;
|
||||
$req->files = $files;
|
||||
$req->result_file = $result_file;
|
||||
return $req;
|
||||
}
|
||||
|
||||
static function selected(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles $files, \taskman\TaskmanFileChanges|null $file_changes = null) : ConfigUpdateRequest
|
||||
static function patch(ConfigDirFiles $files) : ConfigUpdateRequest
|
||||
{
|
||||
if($files instanceof \taskman\artefact\TaskmanDirFiles)
|
||||
$files = ConfigDirFiles::makeFromArtefactFiles($files);
|
||||
|
||||
$req = new ConfigUpdateRequest();
|
||||
$req->mode = ConfigUpdateMode::Selected;
|
||||
$req->mode = ConfigUpdateMode::Patch;
|
||||
$req->files = $files;
|
||||
$req->file_changes = $file_changes;
|
||||
return $req;
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigUpdateResult
|
||||
{
|
||||
public ConfigUpdateRequest $request;
|
||||
|
||||
public ConfigDirFiles $affected_files;
|
||||
public array $affected_entries = array();
|
||||
|
||||
public array $added_files = array();
|
||||
public array $removed_files = array();
|
||||
|
||||
/** @var array<string, string> */
|
||||
public array $errors = array();
|
||||
public int $corruptions = 0;
|
||||
public int $fast_jsons = 0;
|
||||
|
||||
function isEmpty() : bool
|
||||
{
|
||||
return count($this->affected_files) == 0 &&
|
||||
count($this->added_files) == 0 &&
|
||||
count($this->removed_files) == 0;
|
||||
}
|
||||
|
||||
function isPatchPossible() : bool
|
||||
{
|
||||
return ($this->request->mode == ConfigUpdateMode::Selected ||
|
||||
$this->request->mode === ConfigUpdateMode::DetectChanged) &&
|
||||
count(array_filter($this->added_files, fn($f) => config_is_file($f))) == 0 &&
|
||||
count(array_filter($this->removed_files, fn($f) => config_is_file($f))) == 0;
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigManager
|
||||
{
|
||||
private ConfigGlobals $globals;
|
||||
private int $workers_num;
|
||||
|
||||
private ConfigCache $cache;
|
||||
private ?ConfigCacheFileMap $file_map;
|
||||
|
||||
function __construct(ConfigGlobals $globals)
|
||||
function __construct(ConfigGlobals $globals, int $workers_num)
|
||||
{
|
||||
$this->globals = $globals;
|
||||
$this->workers_num = $workers_num;
|
||||
|
||||
$this->cache = new ConfigCache($globals);
|
||||
$this->file_map = null;
|
||||
|
|
@ -163,7 +103,7 @@ class ConfigManager
|
|||
return $this->cache;
|
||||
}
|
||||
|
||||
function getArtefactFilesSpec() : array
|
||||
function getArtifactFilesSpec() : array
|
||||
{
|
||||
return [$this->globals->base_dirs, ['.js']];
|
||||
}
|
||||
|
|
@ -175,124 +115,63 @@ class ConfigManager
|
|||
$this->file_map = $this->_tryLoadMap();
|
||||
if($this->file_map === null)
|
||||
{
|
||||
$this->file_map = self::_makeMap($this->scanFiles(extension: '.js'));
|
||||
$this->file_map = self::_makeMap($files ?? $this->scanFiles(extension: '.js'));
|
||||
$this->_saveFileMap();
|
||||
}
|
||||
}
|
||||
return $this->file_map;
|
||||
}
|
||||
|
||||
function fetchAll(bool $force = false) : \Generator
|
||||
function updateCache(ConfigUpdateRequest $req, bool $return_entries = false, bool $verbose = false) : ConfigCacheUpdateResult
|
||||
{
|
||||
$result = $this->updateAll($force);
|
||||
$filtered = $this->filterConfigFiles($result->request->files);
|
||||
return $this->iterateCache($filtered);
|
||||
}
|
||||
|
||||
function fetchByPath(string $path) : ConfigCacheEntry
|
||||
{
|
||||
$path = realpath($path);
|
||||
$dir_files = ConfigDirFiles::makeFor($this);
|
||||
$dir_files->addFile($path);
|
||||
$request = ConfigUpdateRequest::selected($dir_files);
|
||||
$request->return_affected = true;
|
||||
$result = $this->update($request);
|
||||
return $result->affected_entries[0];
|
||||
}
|
||||
|
||||
function iterateCache(iterable $files) : \Generator
|
||||
{
|
||||
foreach($files as $file)
|
||||
{
|
||||
$ce = $this->cache->getOrLoadByPath($file);
|
||||
yield $ce;
|
||||
}
|
||||
}
|
||||
|
||||
function update(ConfigUpdateRequest $req) : ConfigUpdateResult
|
||||
{
|
||||
config_log("Updating configs, mode '{$req->mode->name}'...");
|
||||
config_log("Updating cache, mode '{$req->mode->name}'...");
|
||||
|
||||
if($req->files === null)
|
||||
$req->files = $this->scanFiles(extension: '.js', verbose: $req->verbose);
|
||||
$req->files = $this->scanFiles(extension: '.js', verbose: $verbose);
|
||||
|
||||
$added_files = [];
|
||||
$removed_files = [];
|
||||
$this->_checkFileMap($req, $added_files, $removed_files);
|
||||
|
||||
$affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
|
||||
$affected_conf_files = $this->filterConfigFiles($affected_all_files);
|
||||
$affected_files = $this->_getAffectedFiles($req, $removed_files);
|
||||
|
||||
config_log("Affected configs: {$affected_conf_files->count()} (requested: {$req->files->count()})");
|
||||
//NOTE: at this poine taking into account only config files
|
||||
$affected_files->filter(fn($file) => str_ends_with($file, '.conf.js'));
|
||||
|
||||
$update_result = new ConfigUpdateResult();
|
||||
$update_result->request = $req;
|
||||
$update_result->affected_files = $affected_conf_files;
|
||||
$update_result->added_files = $added_files;
|
||||
$update_result->removed_files = $removed_files;
|
||||
config_log("Affected files: {$affected_files->count()}");
|
||||
|
||||
$update_params = new ConfigCacheUpdateParams(
|
||||
globals: $this->globals,
|
||||
affected_files: $affected_conf_files,
|
||||
verbose: $req->verbose
|
||||
affected_files: $affected_files,
|
||||
verbose: $verbose,
|
||||
max_workers: $this->workers_num
|
||||
);
|
||||
self::_updateCache($update_result, $update_params);
|
||||
$update_result = self::_updateCache($update_params);
|
||||
|
||||
//let's clear internal cache once the update procedure is done
|
||||
$this->cache->clear();
|
||||
|
||||
$this->_updateFileMap($req, $affected_conf_files, $added_files, $removed_files);
|
||||
$this->_updateFileMap($req, $affected_files, $added_files, $removed_files);
|
||||
|
||||
if($req->return_affected)
|
||||
if($return_entries)
|
||||
{
|
||||
foreach($affected_conf_files as $file)
|
||||
foreach($affected_files as $file)
|
||||
{
|
||||
$entry = $this->cache->getOrLoadByPath($file);
|
||||
$update_result->affected_entries[] = $entry;
|
||||
}
|
||||
}
|
||||
|
||||
touch($this->_getLastUpdateFile());
|
||||
|
||||
return $update_result;
|
||||
}
|
||||
|
||||
function filterConfigFiles(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles
|
||||
{
|
||||
if($files instanceof \taskman\artefact\TaskmanDirFiles)
|
||||
$files = ConfigDirFiles::makeFromArtefactFiles($files);
|
||||
|
||||
$filtered = $this->globals->files_filter != null ?
|
||||
call_user_func($this->globals->files_filter, $files) :
|
||||
new ConfigDirFiles($files->getMap());
|
||||
|
||||
$filtered->filter(fn($file) => config_is_file($file));
|
||||
return $filtered;
|
||||
}
|
||||
|
||||
function updateAll(bool $force = false) : ConfigUpdateResult
|
||||
{
|
||||
$files = $this->scanFiles(extension: '.js');
|
||||
$result = $this->update($force ?
|
||||
ConfigUpdateRequest::force($files) :
|
||||
ConfigUpdateRequest::detectChanged($files)
|
||||
);
|
||||
return $result;
|
||||
}
|
||||
|
||||
function _getLastUpdateFile() : string
|
||||
{
|
||||
return $this->globals->build_dir . '/update.last';
|
||||
}
|
||||
|
||||
private function _updateCache(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
|
||||
private function _updateCache(ConfigCacheUpdateParams $params) : ConfigCacheUpdateResult
|
||||
{
|
||||
if($params->affected_files->isEmpty())
|
||||
return;
|
||||
return new ConfigCacheUpdateResult($params);
|
||||
|
||||
$t = microtime(true);
|
||||
|
||||
_config_cache_update($result, $params);
|
||||
$result = _config_cache_update($params);
|
||||
|
||||
if($result->errors)
|
||||
{
|
||||
|
|
@ -309,6 +188,8 @@ class ConfigManager
|
|||
|
||||
if($params->verbose)
|
||||
config_log("Update cache: " . round(microtime(true) - $t,2) . " sec.");
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
private function _checkFileMap(ConfigUpdateRequest $req, array &$added_files, array &$removed_files)
|
||||
|
|
@ -317,52 +198,39 @@ class ConfigManager
|
|||
|
||||
if($req->mode === ConfigUpdateMode::Force)
|
||||
{
|
||||
$added_files = $req->files->getAllFiles();
|
||||
//let's rebuild the file map
|
||||
$fs_cache_map->init($req->files->getAllFiles());
|
||||
config_log("File map init: ".$fs_cache_map->count());
|
||||
$fs_cache_map->init($added_files);
|
||||
config_log("File map init: ".count($added_files));
|
||||
}
|
||||
else if($req->mode === ConfigUpdateMode::DetectChanged)
|
||||
{
|
||||
list($added_files, $removed_files) = $fs_cache_map->compare($req->files->getAllFiles());
|
||||
config_log("File map compare, added: ".count($added_files).", removed: ".count($removed_files));
|
||||
}
|
||||
else if($req->mode === ConfigUpdateMode::Selected)
|
||||
{
|
||||
if($req->file_changes != null)
|
||||
{
|
||||
foreach($req->files as $file)
|
||||
{
|
||||
if($req->file_changes->isCreated($file))
|
||||
$added_files[] = $file;
|
||||
else if($req->file_changes->isDeleted($file))
|
||||
$removed_files[] = $file;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_conf_files, array $added_files, array $removed_files)
|
||||
private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_files, array $added_files, array $removed_files)
|
||||
{
|
||||
$fs_cache_map = $this->getFileMap();
|
||||
|
||||
//TODO: traverse all affected config files and update file map
|
||||
foreach($affected_conf_files as $file)
|
||||
//TODO: traverse all affected files and update file map
|
||||
foreach($affected_files as $file)
|
||||
{
|
||||
$cache_entry = $this->cache->getOrLoadByPath($file);
|
||||
$fs_cache_map->updateDepsForEntry($this->globals->base_dirs, $cache_entry);
|
||||
$fs_cache_map->updateDepsForEntry($cache_entry);
|
||||
}
|
||||
|
||||
if($req->mode == ConfigUpdateMode::Force ||
|
||||
$affected_conf_files->count() > 0 ||
|
||||
$added_files ||
|
||||
$removed_files)
|
||||
if($affected_files->count() > 0 || $added_files || $removed_files)
|
||||
{
|
||||
$fs_cache_map->update($added_files, $removed_files);
|
||||
//in case of Force map was already cleared and initialized
|
||||
if($req->mode != ConfigUpdateMode::Force)
|
||||
$fs_cache_map->update($added_files, $removed_files);
|
||||
$this->_saveFileMap();
|
||||
}
|
||||
}
|
||||
|
||||
private function _getAffectedFiles(ConfigUpdateRequest $req, array $added_files, array $removed_files) : ConfigDirFiles
|
||||
private function _getAffectedFiles(ConfigUpdateRequest $req, array $removed_files) : ConfigDirFiles
|
||||
{
|
||||
$fs_cache_map = $this->getFileMap();
|
||||
|
||||
|
|
@ -370,19 +238,17 @@ class ConfigManager
|
|||
|
||||
if($req->mode === ConfigUpdateMode::Force)
|
||||
{
|
||||
$affected_files = new ConfigDirFiles($req->files->getMap());
|
||||
$affected_files = $req->files;
|
||||
}
|
||||
else if($req->mode === ConfigUpdateMode::DetectChanged)
|
||||
{
|
||||
$affected_files = ConfigDirFiles::makeFor($this);
|
||||
|
||||
$last_run_file = $this->_getLastUpdateFile();
|
||||
|
||||
foreach($req->files->getMap() as $base_dir => $files)
|
||||
{
|
||||
foreach($files as $file)
|
||||
{
|
||||
if(need_to_regen($last_run_file, [$file]))
|
||||
if(need_to_regen($req->result_file, [$file]))
|
||||
{
|
||||
$affected_files->add($base_dir, $file);
|
||||
|
||||
|
|
@ -396,21 +262,32 @@ class ConfigManager
|
|||
//if there were removed files we need to rebuild affected files
|
||||
foreach($removed_files as $file)
|
||||
{
|
||||
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
||||
foreach($affected_by_file as $dep)
|
||||
$affected_files->addFile($dep, unique: true);
|
||||
if(!str_ends_with($file, '.conf.js'))
|
||||
{
|
||||
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
||||
foreach($affected_by_file as $dep)
|
||||
$affected_files->addFile($dep, unique: true);
|
||||
}
|
||||
else
|
||||
{
|
||||
//TODO: in case config file was removed do we actually need to rebuild all configs?
|
||||
}
|
||||
}
|
||||
}
|
||||
else if($req->mode === ConfigUpdateMode::Selected)
|
||||
else if($req->mode === ConfigUpdateMode::Patch)
|
||||
{
|
||||
$affected_files = ConfigDirFiles::makeFor($this);
|
||||
|
||||
foreach($req->files as $file)
|
||||
{
|
||||
$affected_files->addFile($file, unique: true);
|
||||
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
||||
foreach($affected_by_file as $dep)
|
||||
$affected_files->addFile($dep, unique: true);
|
||||
|
||||
if(!str_ends_with($file, '.conf.js'))
|
||||
{
|
||||
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
||||
foreach($affected_by_file as $dep)
|
||||
$affected_files->addFile($dep, unique: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -436,11 +313,8 @@ class ConfigManager
|
|||
{
|
||||
if(!is_file($this->_getMapPath()))
|
||||
return null;
|
||||
$t = microtime(true);
|
||||
$data = ensure_read($this->_getMapPath());
|
||||
$map = ConfigCacheFileMap::unserialize($data);
|
||||
config_log("Loaded file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
|
||||
return $map;
|
||||
config_log("Loading file map");
|
||||
return ConfigCacheFileMap::unserialize(ensure_read($this->_getMapPath()));
|
||||
}
|
||||
|
||||
private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
|
||||
|
|
@ -453,10 +327,9 @@ class ConfigManager
|
|||
|
||||
private function _saveFileMap()
|
||||
{
|
||||
$t = microtime(true);
|
||||
$data = ConfigCacheFileMap::serialize($this->getFileMap());
|
||||
config_log("Saving file map: " . kb($data));
|
||||
ensure_write($this->_getMapPath(), $data);
|
||||
config_log("Saved file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
183
pack.inc.php
183
pack.inc.php
|
|
@ -42,12 +42,10 @@ function config_pack_bundle(ConfigPackParams $params) : string
|
|||
|
||||
$packed_data = null;
|
||||
|
||||
$cache_entries = iterator_to_array($params->cache_entries);
|
||||
|
||||
if($params->binary_format == 1)
|
||||
{
|
||||
$packed_data = _config_pack_bundle_fmt1(
|
||||
$cache_entries,
|
||||
$params->cache_entries,
|
||||
$params->use_lz4,
|
||||
$params->use_config_refs,
|
||||
$params->version
|
||||
|
|
@ -56,7 +54,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
|
|||
else if($params->binary_format == 2)
|
||||
{
|
||||
$packed_data = _config_pack_bundle_fmt2(
|
||||
$cache_entries,
|
||||
$params->cache_entries,
|
||||
$params->use_lz4,
|
||||
$params->use_config_refs,
|
||||
$params->version,
|
||||
|
|
@ -65,7 +63,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
|
|||
else if($params->binary_format == 3)
|
||||
{
|
||||
$packed_data = _config_pack_bundle_fmt3(
|
||||
$cache_entries,
|
||||
$params->cache_entries,
|
||||
$params->use_lz4,
|
||||
$params->use_config_refs,
|
||||
$params->version,
|
||||
|
|
@ -77,7 +75,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
|
|||
throw new Exception("Unknown binary format: {$params->binary_format}");
|
||||
|
||||
if($params->debug)
|
||||
config_log("Packed entries: " . count($cache_entries) . ", total: " .
|
||||
config_log("Packed entries: " . sizeof($params->cache_entries) . ", total: " .
|
||||
kb($packed_data) . ", format: {$params->binary_format}, lz4: " .
|
||||
var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) .
|
||||
", CRC: " . crc32($packed_data) .
|
||||
|
|
@ -102,16 +100,6 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
|
|||
$params->version,
|
||||
);
|
||||
}
|
||||
else if($params->binary_format == 3)
|
||||
{
|
||||
$patched_data = _config_patch_bundle_fmt3(
|
||||
$packed_data,
|
||||
$params->cache_entries,
|
||||
$params->use_lz4,
|
||||
$params->use_config_refs,
|
||||
$params->version,
|
||||
);
|
||||
}
|
||||
else
|
||||
throw new Exception("Unknown binary format: {$params->binary_format}");
|
||||
|
||||
|
|
@ -127,7 +115,7 @@ function config_patch_bundle(ConfigPackParams $params, string $packed_data) : st
|
|||
|
||||
//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build)
|
||||
function _config_pack_bundle_fmt1(
|
||||
array $cache_entries,
|
||||
iterable $cache_entries,
|
||||
bool $use_lz4,
|
||||
bool $use_config_refs,
|
||||
int $version) : string
|
||||
|
|
@ -185,7 +173,7 @@ function _config_pack_bundle_fmt1(
|
|||
//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
|
||||
// (each path item separated by '/' is stored as an array item)
|
||||
function _config_pack_bundle_fmt2(
|
||||
array $cache_entries,
|
||||
iterable $cache_entries,
|
||||
bool $use_lz4,
|
||||
bool $use_config_refs,
|
||||
int $version,
|
||||
|
|
@ -246,7 +234,7 @@ function _config_pack_bundle_fmt2(
|
|||
return $packed_data;
|
||||
}
|
||||
|
||||
//NOTE: Much like fmt1, but configs entries are grouped into sizeable chunks, with each chunk lz4-compressed.
|
||||
//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
|
||||
// This reduces overall bundle size when there are many small configs entries.
|
||||
function _config_pack_bundle_fmt3(
|
||||
array $cache_entries,
|
||||
|
|
@ -361,12 +349,12 @@ function _config_patch_bundle_fmt2(
|
|||
{
|
||||
list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
|
||||
|
||||
$headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
|
||||
$header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
|
||||
|
||||
if($headers_found)
|
||||
if($header_found)
|
||||
{
|
||||
$header_idx = key($headers_found);
|
||||
$header_entry = current($headers_found);
|
||||
$header_idx = key($header_found);
|
||||
$header_entry = current($header_found);
|
||||
|
||||
$current_offset = $header_entry[4];
|
||||
$current_size = $header_entry[5];
|
||||
|
|
@ -424,97 +412,6 @@ function _config_patch_bundle_fmt2(
|
|||
return $patched_data;
|
||||
}
|
||||
|
||||
function _config_patch_bundle_fmt3(
|
||||
string $packed_data,
|
||||
array $patch_entries,
|
||||
bool $use_lz4,
|
||||
bool $use_config_refs,
|
||||
int $version,
|
||||
) : string
|
||||
{
|
||||
if(!$use_lz4)
|
||||
throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
|
||||
|
||||
list($header, $entries, $chunks_offset, $max_chunk_size) =
|
||||
_config_unpack_bundle_fmt3(packed_data: $packed_data, unpack_entries: false);
|
||||
|
||||
$payloads_bundle = substr($packed_data, $chunks_offset);
|
||||
|
||||
foreach($patch_entries as $idx => $patch_entry)
|
||||
{
|
||||
list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
|
||||
|
||||
$headers_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
|
||||
|
||||
if($headers_found)
|
||||
{
|
||||
$header_idx = key($headers_found);
|
||||
$header_entry = current($headers_found);
|
||||
|
||||
$chunk_offset = $header_entry[4];
|
||||
$payloads_offset = $header_entry[5]; //within chunk
|
||||
$payloads_size = $header_entry[6];
|
||||
|
||||
$chunk_entries = array_filter($header, fn($item) => $item[4] == $chunk_offset);
|
||||
|
||||
//chunk contains only one patch entry
|
||||
$patch_payload_lz4 = lz4_compress($patch_payload, 9);
|
||||
$patched_chunk = pack("V", strlen($patch_payload_lz4));
|
||||
$patched_chunk .= $patch_payload_lz4;
|
||||
if(strlen($patch_payload) > $max_chunk_size)
|
||||
$max_chunk_size = strlen($patch_payload);
|
||||
|
||||
//TODO: make a better generic algorithm for patching the whole chunk
|
||||
//special case for single entry
|
||||
if(count($chunk_entries) == 1 && $payloads_offset == 0)
|
||||
{
|
||||
$lz4_chunk_size = unpack("V", substr($payloads_bundle, $chunk_offset, 4))[1];
|
||||
|
||||
//we can just replace chunk in place if the size of a patched chunk is less or equal
|
||||
if(strlen($patch_payload_lz4) <= $lz4_chunk_size)
|
||||
{
|
||||
$payloads_bundle = substr_replace($payloads_bundle, $patched_chunk, $chunk_offset, strlen($patched_chunk));
|
||||
$header_entry[6] = strlen($patch_payload);
|
||||
}
|
||||
//just append to the end
|
||||
else
|
||||
{
|
||||
$header_entry[4] = strlen($payloads_bundle); //chunk offset
|
||||
$header_entry[5] = 0;
|
||||
$header_entry[6] = strlen($patch_payload);
|
||||
|
||||
$payloads_bundle .= $patched_chunk;
|
||||
}
|
||||
}
|
||||
//just append to the end
|
||||
else
|
||||
{
|
||||
$header_entry[4] = strlen($payloads_bundle); //chunk offset
|
||||
$header_entry[5] = 0;
|
||||
$header_entry[6] = strlen($patch_payload);
|
||||
|
||||
$payloads_bundle .= $patched_chunk;
|
||||
}
|
||||
|
||||
$header[$header_idx] = $header_entry;
|
||||
}
|
||||
else
|
||||
throw new Exception("Patched entry {$patch_entry->id} not found in config bundle");
|
||||
}
|
||||
|
||||
$header_msgpack = config_msgpack_pack($header);
|
||||
|
||||
$packed_data =
|
||||
pack("C", 3) .
|
||||
pack("V", $version) .
|
||||
pack("V", strlen($header_msgpack)) .
|
||||
pack("V", $max_chunk_size) .
|
||||
$header_msgpack .
|
||||
$payloads_bundle;
|
||||
|
||||
return $packed_data;
|
||||
}
|
||||
|
||||
//format: [[class_id, [data]], ...[class_id, [data]]]
|
||||
function config_unpack_bundle(string $packed_data) : array
|
||||
{
|
||||
|
|
@ -532,8 +429,7 @@ function config_unpack_bundle(string $packed_data) : array
|
|||
}
|
||||
else if($info['format'] === 3)
|
||||
{
|
||||
list($_, $entries) = _config_unpack_bundle_fmt3($packed_data);
|
||||
return $entries;
|
||||
return _config_unpack_bundle_fmt3($packed_data);
|
||||
}
|
||||
else
|
||||
throw new Exception("Unknown format: {$info['format']}");
|
||||
|
|
@ -600,10 +496,12 @@ function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries =
|
|||
return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
|
||||
}
|
||||
|
||||
function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries = true): array
|
||||
function _config_unpack_bundle_fmt3(string $packed_data): array
|
||||
{
|
||||
if(ord($packed_data[0]) !== 3)
|
||||
{
|
||||
throw new Exception("Invalid config bundle format");
|
||||
}
|
||||
|
||||
$offset = 1;
|
||||
$version = unpack("V", substr($packed_data, $offset, 4))[1];
|
||||
|
|
@ -615,7 +513,6 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
|
|||
|
||||
$header_msgpack = substr($packed_data, $offset, $header_len);
|
||||
$offset += $header_len;
|
||||
$chunks_offset = $offset;
|
||||
|
||||
$header = config_msgpack_unpack($header_msgpack);
|
||||
|
||||
|
|
@ -624,45 +521,39 @@ function _config_unpack_bundle_fmt3(string $packed_data, bool $unpack_entries =
|
|||
$chunk_buffer = '';
|
||||
$chunk_id = -1;
|
||||
|
||||
foreach($header as $entry_data)
|
||||
foreach ($header as $entry_data)
|
||||
{
|
||||
list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
|
||||
|
||||
$unpacked_payload = null;
|
||||
|
||||
if($unpack_entries)
|
||||
if($entry_chunk_offset !== $chunk_id)
|
||||
{
|
||||
if($entry_chunk_offset !== $chunk_id)
|
||||
if($chunk_offset !== -1)
|
||||
{
|
||||
if($chunk_offset !== -1)
|
||||
{
|
||||
$lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
|
||||
$offset+=4;
|
||||
$lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
|
||||
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
|
||||
$offset += $lz4_chunk_size;
|
||||
$chunk_offset = $offset;
|
||||
}
|
||||
else
|
||||
{
|
||||
$lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
|
||||
$chunk_offset += 4;
|
||||
$lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
|
||||
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
|
||||
$chunk_offset += $lz4_chunk_size;
|
||||
}
|
||||
|
||||
$chunk_id = $entry_chunk_offset;
|
||||
$lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
|
||||
$offset+=4;
|
||||
$lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
|
||||
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
|
||||
$offset += $lz4_chunk_size;
|
||||
$chunk_offset = $offset;
|
||||
}
|
||||
else
|
||||
{
|
||||
$lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
|
||||
$chunk_offset += 4;
|
||||
$lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
|
||||
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
|
||||
$chunk_offset += $lz4_chunk_size;
|
||||
}
|
||||
|
||||
$payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
|
||||
$unpacked_payload = _config_unpack_payload($format, $payload);
|
||||
$chunk_id = $entry_chunk_offset;
|
||||
}
|
||||
|
||||
$cache_entries[$id] = array($class_id, $unpacked_payload);
|
||||
$payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
|
||||
|
||||
$cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
|
||||
}
|
||||
|
||||
return array($header, $cache_entries, $chunks_offset, $max_chunk_size);
|
||||
return $cache_entries;
|
||||
}
|
||||
|
||||
//format: [format_id, payload_data]
|
||||
|
|
|
|||
|
|
@ -28,17 +28,15 @@ function config_real_path(array $base_dirs, string $rel_path, bool $strict = tru
|
|||
{
|
||||
foreach($base_dirs as $dir)
|
||||
if(is_file($dir . '/' . $rel_path))
|
||||
return normalize_path($dir . '/' . $rel_path);
|
||||
|
||||
return $dir . '/' . $rel_path;
|
||||
if($strict)
|
||||
throw new Exception("No file for relative path '$rel_path'");
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function config_is_file(string $filename) : bool
|
||||
{
|
||||
return str_ends_with($filename, '.conf.js');
|
||||
return (strrpos($filename, '.conf.js') === (strlen($filename) - 8));
|
||||
}
|
||||
|
||||
function config_file2id(string $conf_dir, string $filename) : int
|
||||
|
|
@ -106,7 +104,6 @@ function config_walk_fields(object $proto, callable $callback)
|
|||
}
|
||||
}
|
||||
|
||||
//TODO: doesn't really belong here
|
||||
function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
|
||||
{
|
||||
if(!isset($map[$file]))
|
||||
|
|
|
|||
Loading…
Reference in New Issue