553 lines
16 KiB
PHP
553 lines
16 KiB
PHP
<?php
|
|
namespace taskman;
|
|
use Exception;
|
|
|
|
require_once(__DIR__ . '/task.inc.php');
|
|
require_once(__DIR__ . '/scan.inc.php');
|
|
require_once(__DIR__ . '/cache.inc.php');
|
|
require_once(__DIR__ . '/pack.inc.php');
|
|
require_once(__DIR__ . '/util.inc.php');
|
|
require_once(__DIR__ . '/msgpack.inc.php');
|
|
|
|
class ConfigGlobals
|
|
{
|
|
public readonly array $base_dirs;
|
|
//NOTE: it's a string since is serialized
|
|
public readonly ?string $worker_init_fn;
|
|
public readonly int $workers_num;
|
|
public readonly string $base_class;
|
|
public readonly string $build_dir;
|
|
//NOTE: it's a string since is serialized
|
|
public readonly ?string $files_filter;
|
|
|
|
function __construct(
|
|
array $base_dirs,
|
|
string $build_dir,
|
|
?string $worker_init_fn = null,
|
|
int $workers_num = 1,
|
|
string $base_class = '\ConfBase',
|
|
?string $files_filter = null
|
|
)
|
|
{
|
|
$this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs);
|
|
$this->build_dir = $build_dir;
|
|
$this->worker_init_fn = $worker_init_fn;
|
|
$this->workers_num = $workers_num;
|
|
$this->base_class = $base_class;
|
|
$this->files_filter = $files_filter;
|
|
}
|
|
|
|
function initWorker(bool $is_master_proc)
|
|
{
|
|
$GLOBALS['CONFIG_GLOBALS'] = $this;
|
|
|
|
if($this->worker_init_fn !== null)
|
|
{
|
|
$fn = $this->worker_init_fn;
|
|
$fn($is_master_proc);
|
|
}
|
|
}
|
|
}
|
|
|
|
enum ConfigGetMode : int
|
|
{
|
|
case ForceUpdate = 1;
|
|
case Auto = 2;
|
|
case CacheOnly = 3;
|
|
}
|
|
|
|
enum ConfigUpdateMode : int
|
|
{
|
|
case Force = 1;
|
|
case DetectChanged = 2;
|
|
case Selected = 3;
|
|
}
|
|
|
|
class ConfigUpdateRequest
|
|
{
|
|
public ConfigUpdateMode $mode;
|
|
public ?ConfigDirFiles $files;
|
|
public ?\taskman\TaskmanFileChanges $file_changes;
|
|
public bool $verbose = false;
|
|
public bool $return_affected = false;
|
|
|
|
private function __construct() {}
|
|
|
|
static function force(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
|
|
{
|
|
if($files instanceof \taskman\artefact\TaskmanDirFiles)
|
|
$files = ConfigDirFiles::makeFromArtefactFiles($files);
|
|
|
|
$req = new ConfigUpdateRequest();
|
|
$req->mode = ConfigUpdateMode::Force;
|
|
$req->files = $files;
|
|
return $req;
|
|
}
|
|
|
|
static function detectChanged(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles|null $files = null) : ConfigUpdateRequest
|
|
{
|
|
if($files instanceof \taskman\artefact\TaskmanDirFiles)
|
|
$files = ConfigDirFiles::makeFromArtefactFiles($files);
|
|
|
|
$req = new ConfigUpdateRequest();
|
|
$req->mode = ConfigUpdateMode::DetectChanged;
|
|
$req->files = $files;
|
|
return $req;
|
|
}
|
|
|
|
static function selected(\taskman\artefact\TaskmanDirFiles|ConfigDirFiles $files, \taskman\TaskmanFileChanges|null $file_changes = null) : ConfigUpdateRequest
|
|
{
|
|
if($files instanceof \taskman\artefact\TaskmanDirFiles)
|
|
$files = ConfigDirFiles::makeFromArtefactFiles($files);
|
|
|
|
$req = new ConfigUpdateRequest();
|
|
$req->mode = ConfigUpdateMode::Selected;
|
|
$req->files = $files;
|
|
$req->file_changes = $file_changes;
|
|
return $req;
|
|
}
|
|
}
|
|
|
|
class ConfigUpdateResult
|
|
{
|
|
public ConfigUpdateRequest $request;
|
|
|
|
public ConfigDirFiles $affected_files;
|
|
public array $affected_entries = array();
|
|
|
|
public array $added_files = array();
|
|
public array $removed_files = array();
|
|
|
|
/** @var array<string, string> */
|
|
public array $errors = array();
|
|
public int $corruptions = 0;
|
|
public int $fast_jsons = 0;
|
|
|
|
function isEmpty() : bool
|
|
{
|
|
return count($this->affected_files) == 0 &&
|
|
count($this->added_files) == 0 &&
|
|
count($this->removed_files) == 0;
|
|
}
|
|
|
|
function isPatchPossible() : bool
|
|
{
|
|
return ($this->request->mode == ConfigUpdateMode::Selected ||
|
|
$this->request->mode === ConfigUpdateMode::DetectChanged) &&
|
|
count(array_filter($this->added_files, fn($f) => config_is_file($f))) == 0 &&
|
|
count(array_filter($this->removed_files, fn($f) => config_is_file($f))) == 0;
|
|
}
|
|
}
|
|
|
|
class ConfigManager
|
|
{
|
|
private ConfigGlobals $globals;
|
|
private ConfigCache $cache;
|
|
private ?ConfigCacheFileMap $file_map;
|
|
|
|
function __construct(ConfigGlobals $globals)
|
|
{
|
|
$this->globals = $globals;
|
|
|
|
$this->cache = new ConfigCache($globals);
|
|
$this->file_map = null;
|
|
}
|
|
|
|
function getGlobals() : ConfigGlobals
|
|
{
|
|
return $this->globals;
|
|
}
|
|
|
|
function getCache() : ConfigCache
|
|
{
|
|
return $this->cache;
|
|
}
|
|
|
|
function getArtefactFilesSpec() : array
|
|
{
|
|
return [$this->globals->base_dirs, ['.js']];
|
|
}
|
|
|
|
function getFileMap() : ConfigCacheFileMap
|
|
{
|
|
if($this->file_map === null)
|
|
{
|
|
$this->file_map = $this->_tryLoadMap();
|
|
if($this->file_map === null)
|
|
{
|
|
$this->file_map = self::_makeMap($this->scanFiles(extension: '.js'));
|
|
$this->_saveFileMap();
|
|
}
|
|
}
|
|
return $this->file_map;
|
|
}
|
|
|
|
function fetchAll(bool $force = false) : \Generator
|
|
{
|
|
$result = $this->updateAll($force);
|
|
$filtered = $this->filterConfigFiles($result->request->files);
|
|
return $this->iterateCache($filtered);
|
|
}
|
|
|
|
function fetchByPath(string $path) : ConfigCacheEntry
|
|
{
|
|
$path = realpath($path);
|
|
$dir_files = ConfigDirFiles::makeFor($this);
|
|
$dir_files->addFile($path);
|
|
$request = ConfigUpdateRequest::selected($dir_files);
|
|
$request->return_affected = true;
|
|
$result = $this->update($request);
|
|
return $result->affected_entries[0];
|
|
}
|
|
|
|
function iterateCache(iterable $files) : \Generator
|
|
{
|
|
foreach($files as $file)
|
|
{
|
|
$ce = $this->cache->getOrLoadByPath($file);
|
|
yield $ce;
|
|
}
|
|
}
|
|
|
|
function update(ConfigUpdateRequest $req) : ConfigUpdateResult
|
|
{
|
|
config_log("Updating configs, mode '{$req->mode->name}'...");
|
|
|
|
if($req->files === null)
|
|
$req->files = $this->scanFiles(extension: '.js', verbose: $req->verbose);
|
|
|
|
$added_files = [];
|
|
$removed_files = [];
|
|
$this->_checkFileMap($req, $added_files, $removed_files);
|
|
|
|
$affected_all_files = $this->_getAffectedFiles($req, $added_files, $removed_files);
|
|
$affected_conf_files = $this->filterConfigFiles($affected_all_files);
|
|
|
|
config_log("Affected configs: {$affected_conf_files->count()} (requested: {$req->files->count()})");
|
|
|
|
$update_result = new ConfigUpdateResult();
|
|
$update_result->request = $req;
|
|
$update_result->affected_files = $affected_conf_files;
|
|
$update_result->added_files = $added_files;
|
|
$update_result->removed_files = $removed_files;
|
|
|
|
$update_params = new ConfigCacheUpdateParams(
|
|
globals: $this->globals,
|
|
affected_files: $affected_conf_files,
|
|
verbose: $req->verbose
|
|
);
|
|
self::_updateCache($update_result, $update_params);
|
|
|
|
//let's clear internal cache once the update procedure is done
|
|
$this->cache->clear();
|
|
|
|
$this->_updateFileMap($req, $affected_conf_files, $added_files, $removed_files);
|
|
|
|
if($req->return_affected)
|
|
{
|
|
foreach($affected_conf_files as $file)
|
|
{
|
|
$entry = $this->cache->getOrLoadByPath($file);
|
|
$update_result->affected_entries[] = $entry;
|
|
}
|
|
}
|
|
|
|
touch($this->_getLastUpdateFile());
|
|
|
|
return $update_result;
|
|
}
|
|
|
|
function filterConfigFiles(ConfigDirFiles|\taskman\artefact\TaskmanDirFiles $files) : ConfigDirFiles
|
|
{
|
|
if($files instanceof \taskman\artefact\TaskmanDirFiles)
|
|
$files = ConfigDirFiles::makeFromArtefactFiles($files);
|
|
|
|
$filtered = $this->globals->files_filter != null ?
|
|
call_user_func($this->globals->files_filter, $files) :
|
|
new ConfigDirFiles($files->getMap());
|
|
|
|
$filtered->filter(fn($file) => config_is_file($file));
|
|
return $filtered;
|
|
}
|
|
|
|
function updateAll(bool $force = false) : ConfigUpdateResult
|
|
{
|
|
$files = $this->scanFiles(extension: '.js');
|
|
$result = $this->update($force ?
|
|
ConfigUpdateRequest::force($files) :
|
|
ConfigUpdateRequest::detectChanged($files)
|
|
);
|
|
return $result;
|
|
}
|
|
|
|
function _getLastUpdateFile() : string
|
|
{
|
|
return $this->globals->build_dir . '/update.last';
|
|
}
|
|
|
|
private function _updateCache(ConfigUpdateResult $result, ConfigCacheUpdateParams $params)
|
|
{
|
|
if($params->affected_files->isEmpty())
|
|
return;
|
|
|
|
$t = microtime(true);
|
|
|
|
_config_cache_update($result, $params);
|
|
|
|
if($result->errors)
|
|
{
|
|
$errors = array();
|
|
foreach($result->errors as $file => $error)
|
|
{
|
|
$errors[] = (count($errors) + 1) . ") Error in file '$file': $error";
|
|
if(count($errors) > $params->max_errors_num)
|
|
break;
|
|
}
|
|
|
|
throw new Exception(implode("\n", $errors));
|
|
}
|
|
|
|
if($params->verbose)
|
|
config_log("Update cache: " . round(microtime(true) - $t,2) . " sec.");
|
|
}
|
|
|
|
private function _checkFileMap(ConfigUpdateRequest $req, array &$added_files, array &$removed_files)
|
|
{
|
|
$fs_cache_map = $this->getFileMap();
|
|
|
|
if($req->mode === ConfigUpdateMode::Force)
|
|
{
|
|
//let's rebuild the file map
|
|
$fs_cache_map->init($req->files->getAllFiles());
|
|
config_log("File map init: ".$fs_cache_map->count());
|
|
}
|
|
else if($req->mode === ConfigUpdateMode::DetectChanged)
|
|
{
|
|
list($added_files, $removed_files) = $fs_cache_map->compare($req->files->getAllFiles());
|
|
config_log("File map compare, added: ".count($added_files).", removed: ".count($removed_files));
|
|
}
|
|
else if($req->mode === ConfigUpdateMode::Selected)
|
|
{
|
|
if($req->file_changes != null)
|
|
{
|
|
foreach($req->files as $file)
|
|
{
|
|
if($req->file_changes->isCreated($file))
|
|
$added_files[] = $file;
|
|
else if($req->file_changes->isDeleted($file))
|
|
$removed_files[] = $file;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private function _updateFileMap(ConfigUpdateRequest $req, ConfigDirFiles $affected_conf_files, array $added_files, array $removed_files)
|
|
{
|
|
$fs_cache_map = $this->getFileMap();
|
|
|
|
//TODO: traverse all affected config files and update file map
|
|
foreach($affected_conf_files as $file)
|
|
{
|
|
$cache_entry = $this->cache->getOrLoadByPath($file);
|
|
$fs_cache_map->updateDepsForEntry($this->globals->base_dirs, $cache_entry);
|
|
}
|
|
|
|
if($req->mode == ConfigUpdateMode::Force ||
|
|
$affected_conf_files->count() > 0 ||
|
|
$added_files ||
|
|
$removed_files)
|
|
{
|
|
$fs_cache_map->update($added_files, $removed_files);
|
|
$this->_saveFileMap();
|
|
}
|
|
}
|
|
|
|
private function _getAffectedFiles(ConfigUpdateRequest $req, array $added_files, array $removed_files) : ConfigDirFiles
|
|
{
|
|
$fs_cache_map = $this->getFileMap();
|
|
|
|
$affected_files = null;
|
|
|
|
if($req->mode === ConfigUpdateMode::Force)
|
|
{
|
|
$affected_files = new ConfigDirFiles($req->files->getMap());
|
|
}
|
|
else if($req->mode === ConfigUpdateMode::DetectChanged)
|
|
{
|
|
$affected_files = ConfigDirFiles::makeFor($this);
|
|
|
|
$last_run_file = $this->_getLastUpdateFile();
|
|
|
|
foreach($req->files->getMap() as $base_dir => $files)
|
|
{
|
|
foreach($files as $file)
|
|
{
|
|
if(need_to_regen($last_run_file, [$file]))
|
|
{
|
|
$affected_files->add($base_dir, $file);
|
|
|
|
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
|
foreach($affected_by_file as $dep)
|
|
$affected_files->addFile($dep, unique: true);
|
|
}
|
|
}
|
|
}
|
|
|
|
//if there were removed files we need to rebuild affected files
|
|
foreach($removed_files as $file)
|
|
{
|
|
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
|
foreach($affected_by_file as $dep)
|
|
$affected_files->addFile($dep, unique: true);
|
|
}
|
|
}
|
|
else if($req->mode === ConfigUpdateMode::Selected)
|
|
{
|
|
$affected_files = ConfigDirFiles::makeFor($this);
|
|
|
|
foreach($req->files as $file)
|
|
{
|
|
$affected_files->addFile($file, unique: true);
|
|
$affected_by_file = $fs_cache_map->getAffectedFiles($file);
|
|
foreach($affected_by_file as $dep)
|
|
$affected_files->addFile($dep, unique: true);
|
|
}
|
|
}
|
|
|
|
return $affected_files;
|
|
}
|
|
|
|
function parseFile(string $file) : ConfigParseResult
|
|
{
|
|
return config_parse($this->globals->base_dirs, $file);
|
|
}
|
|
|
|
function scanFiles(string $extension = '.conf.js', bool $verbose = false) : ConfigDirFiles
|
|
{
|
|
return config_scan_files($this->globals->base_dirs, $extension, $verbose);
|
|
}
|
|
|
|
private function _getMapPath() : string
|
|
{
|
|
return $this->globals->build_dir . '/file_map.data';
|
|
}
|
|
|
|
private function _tryLoadMap() : ?ConfigCacheFileMap
|
|
{
|
|
if(!is_file($this->_getMapPath()))
|
|
return null;
|
|
$t = microtime(true);
|
|
$data = ensure_read($this->_getMapPath());
|
|
$map = ConfigCacheFileMap::unserialize($data);
|
|
config_log("Loaded file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
|
|
return $map;
|
|
}
|
|
|
|
private static function _makeMap(ConfigDirFiles $files) : ConfigCacheFileMap
|
|
{
|
|
config_log("Creating file map");
|
|
$map = new ConfigCacheFileMap();
|
|
$map->init($files->getAllFiles());
|
|
return $map;
|
|
}
|
|
|
|
private function _saveFileMap()
|
|
{
|
|
$t = microtime(true);
|
|
$data = ConfigCacheFileMap::serialize($this->getFileMap());
|
|
ensure_write($this->_getMapPath(), $data);
|
|
config_log("Saved file map: " . kb($data) . ", " . round(microtime(true) - $t,2) . " sec.");
|
|
}
|
|
}
|
|
|
|
function config_log($msg)
|
|
{
|
|
echo "[CFG] $msg\n";
|
|
}
|
|
|
|
function config_get_cache_path(ConfigGlobals $globals, string $file) : string
|
|
{
|
|
return config_get_tmp_build_path($globals, $file . '.cache');
|
|
}
|
|
|
|
function config_get_cache_id_path(ConfigGlobals $globals, int $id) : string
|
|
{
|
|
return config_get_tmp_build_path($globals, $id . '.id');
|
|
}
|
|
|
|
function config_get_cache_strid_path(ConfigGlobals $globals, string $strid) : string
|
|
{
|
|
return config_get_tmp_build_path($globals, $strid . '.strid');
|
|
}
|
|
|
|
function config_get_cache_payload_path(ConfigGlobals $globals, string $file) : string
|
|
{
|
|
return config_get_tmp_build_path($globals, $file . '.payload');
|
|
}
|
|
|
|
function config_load(ConfigGlobals $globals, string $conf_path, ?string $conf_dir = null) : object
|
|
{
|
|
$conf_dir ??= config_map_base_dir($globals->base_dirs, $conf_path);
|
|
|
|
list($conf_id, $_) = config_ensure_header($conf_dir, $conf_path);
|
|
if(!$conf_id)
|
|
throw new Exception("Bad conf id: {$conf_id}");
|
|
|
|
$pres = config_parse(array($conf_dir), $conf_path);
|
|
if($pres->error !== 0)
|
|
throw new Exception("Error({$pres->error}) while loading JSON from {$conf_path}:\n" . $pres->error_descr);
|
|
|
|
return config_load_from_kv_array($globals, $conf_dir, $conf_path, $pres->parsed_arr, $conf_id);
|
|
}
|
|
|
|
function config_load_from_kv_array(
|
|
ConfigGlobals $globals, string $conf_dir, string $file,
|
|
array $arr, ?int $id = null) : object
|
|
{
|
|
if(!isset($arr['class']) || !isset($arr['class'][0]))
|
|
throw new Exception("Class is not set in file '$file'.");
|
|
|
|
$id ??= config_file2id($conf_dir, $file);
|
|
|
|
$cnf = null;
|
|
try
|
|
{
|
|
list($klass, $class_id, $norm_arr) = config_apply_class_normalization($arr);
|
|
|
|
if(!class_exists($klass))
|
|
throw new Exception("No such class '$klass'");
|
|
$cnf = new $klass;
|
|
if(!is_a($cnf, $globals->base_class))
|
|
throw new Exception("'$klass' is not subclass of '".ltrim($globals->base_class, '\\')."'");
|
|
|
|
$cnf->import($norm_arr);
|
|
}
|
|
catch(Exception $e)
|
|
{
|
|
throw new Exception($e->getMessage() . " in file '{$file}'"/* . $e->getTraceAsString()*/);
|
|
}
|
|
|
|
$cnf->id = $id;
|
|
$cnf->strid = config_file2strid($conf_dir, $file);
|
|
|
|
return $cnf;
|
|
}
|
|
|
|
function config_bench_load(ConfigGlobals $globals, string $file)
|
|
{
|
|
$base_dir = config_map_base_dir($globals->base_dirs, $file);
|
|
|
|
list($conf_id, $_) = config_ensure_header($base_dir, $file);
|
|
if(!$conf_id)
|
|
throw new Exception("Bad conf id: {$conf_id}");
|
|
$t = microtime(true);
|
|
$parse_res = config_parse($globals->base_dirs, $file);
|
|
if($parse_res->error !== 0)
|
|
throw new Exception("Error({$parse_res->error}) while loading JSON from {$file}:\n" . $parse_res->error_descr);
|
|
config_log("Parse: " . (microtime(true) - $t));
|
|
$t = microtime(true);
|
|
$config = config_load_from_kv_array($globals, $base_dir, $file, $parse_res->parsed_arr, $conf_id);
|
|
config_log("Load: " . (microtime(true) - $t));
|
|
}
|
|
|