API and internals overhaul, optimizing parsing and import, still work in progress
Publish PHP Package / docker (push) Successful in 6s
Details
Publish PHP Package / docker (push) Successful in 6s
Details
This commit is contained in:
Pavel Shevaev
parent
d422998ebf
commit
00f17cc1a8
|
|
@ -0,0 +1,394 @@
|
|||
<?php
|
||||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
/**
|
||||
* @property object $config
|
||||
* @property string $payload
|
||||
*/
|
||||
class ConfigCacheEntry
|
||||
{
|
||||
const FMT_BINARY = 0;
|
||||
const FMT_LZ4 = 1;
|
||||
const FMT_FILE_REF = 2;
|
||||
|
||||
public string $class;
|
||||
public int $class_id;
|
||||
public int $id;
|
||||
public string $strid;
|
||||
public string $cache_file;
|
||||
//NOTE: actual payload is stored in a separate file for faster incremental retrievals
|
||||
public string $payload_file;
|
||||
public string $file;
|
||||
public array $includes = array();
|
||||
public array $refs = array();
|
||||
public ?ConfigCacheEntryExtras $extras;
|
||||
|
||||
public $_config;
|
||||
public $_payload;
|
||||
|
||||
static function serialize(ConfigCacheEntry $ce) : string
|
||||
{
|
||||
$d = $ce->export();
|
||||
return serialize($d);
|
||||
}
|
||||
|
||||
static function unserialize(string $str) : ?ConfigCacheEntry
|
||||
{
|
||||
$d = @unserialize($str);
|
||||
if(!is_array($d))
|
||||
return null;
|
||||
$ce = new ConfigCacheEntry();
|
||||
$ce->import($d);
|
||||
return $ce;
|
||||
}
|
||||
|
||||
function __construct()
|
||||
{
|
||||
$this->extras = ConfigCacheEntryExtras::create();
|
||||
}
|
||||
|
||||
function __get(string $name)
|
||||
{
|
||||
if($name === "config")
|
||||
{
|
||||
if($this->_config === null)
|
||||
{
|
||||
$klass = $this->class;
|
||||
$data = config_msgpack_unpack($this->payload);
|
||||
$this->_config = new $klass();
|
||||
$this->_config->import($data);
|
||||
}
|
||||
return $this->_config;
|
||||
}
|
||||
else if($name === "payload")
|
||||
{
|
||||
if($this->_payload !== null)
|
||||
return $this->_payload;
|
||||
//if payload not set directly not storing it
|
||||
return ensure_read($this->payload_file);
|
||||
}
|
||||
else
|
||||
throw new Exception("No such property '$name'");
|
||||
}
|
||||
|
||||
function __set(string $name, $v)
|
||||
{
|
||||
if($name === "config")
|
||||
$this->_config = $v;
|
||||
else if($name === "payload")
|
||||
$this->_payload = $v;
|
||||
else
|
||||
throw new Exception("No such property '$name'");
|
||||
}
|
||||
|
||||
function export() : array
|
||||
{
|
||||
$d = array();
|
||||
$d[] = $this->class;
|
||||
$d[] = $this->class_id;
|
||||
$d[] = $this->id;
|
||||
$d[] = $this->strid;
|
||||
$d[] = $this->cache_file;
|
||||
$d[] = $this->payload_file;
|
||||
$d[] = $this->file;
|
||||
$d[] = $this->includes;
|
||||
$d[] = $this->refs;
|
||||
$d[] = $this->extras->export();
|
||||
return $d;
|
||||
}
|
||||
|
||||
function import(array $d)
|
||||
{
|
||||
$extras = array();
|
||||
|
||||
list(
|
||||
$this->class,
|
||||
$this->class_id,
|
||||
$this->id,
|
||||
$this->strid,
|
||||
$this->cache_file,
|
||||
$this->payload_file,
|
||||
$this->file,
|
||||
$this->includes,
|
||||
$this->refs,
|
||||
$extras,
|
||||
) = $d;
|
||||
|
||||
$this->extras->import($extras);
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigCacheEntryExtras extends \stdClass
|
||||
{
|
||||
private static string $klass = ConfigCacheEntryExtras::class;
|
||||
|
||||
static function init(string $project_specific_klass)
|
||||
{
|
||||
self::$klass = $project_specific_klass;
|
||||
}
|
||||
|
||||
static function create() : object
|
||||
{
|
||||
return new self::$klass();
|
||||
}
|
||||
|
||||
function export() : array
|
||||
{
|
||||
$as_array = get_object_vars($this);
|
||||
return $as_array;
|
||||
}
|
||||
|
||||
function import(array $as_array)
|
||||
{
|
||||
foreach($as_array as $field_name => $field_value)
|
||||
$this->$field_name = $field_value;
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigFetchParams
|
||||
{
|
||||
public ConfigGlobals $globals;
|
||||
public ConfigScanResult $scanned;
|
||||
public bool $force_stale = false;
|
||||
public bool $verbose = false;
|
||||
public ?int $max_workers = null;
|
||||
public bool $touch_files_with_includes = true;
|
||||
|
||||
function __construct(ConfigGlobals $globals, ConfigScanResult $scanned,
|
||||
bool $force_stale = false, bool $verbose = false, ?int $max_workers = null)
|
||||
{
|
||||
$this->globals = $globals;
|
||||
$this->scanned = $scanned;
|
||||
$this->force_stale = $force_stale;
|
||||
$this->verbose = $verbose;
|
||||
$this->max_workers = $max_workers;
|
||||
}
|
||||
|
||||
function calcMaxWorkers() : int
|
||||
{
|
||||
$max_workers = $this->max_workers;
|
||||
if($max_workers === null)
|
||||
$max_workers = $this->scanned->count() < 20 ? 1 : 5;
|
||||
return $max_workers;
|
||||
}
|
||||
|
||||
function splitFilesByChunks(int $max_workers) : array
|
||||
{
|
||||
$chunk_size = (int)ceil($this->scanned->count()/$max_workers);
|
||||
return array_chunk($this->scanned->getFlatArray(), $chunk_size);
|
||||
}
|
||||
|
||||
//returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ]
|
||||
function splitJobs() : array
|
||||
{
|
||||
$max_workers = $this->calcMaxWorkers();
|
||||
$jobs = array();
|
||||
foreach($this->splitFilesByChunks($max_workers) as $idx => $chunk)
|
||||
$jobs[] = array($idx, microtime(true), $chunk);
|
||||
return $jobs;
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigFetchResult
|
||||
{
|
||||
/** @var ConfigCacheEntry[] */
|
||||
public array $all = array();
|
||||
/** @var array<int, ConfigCacheEntry> */
|
||||
public array $by_id = array();
|
||||
/** @var array<string, ConfigCacheEntry> */
|
||||
public array $by_path = array();
|
||||
/** @var array<string, ConfigCacheEntry> */
|
||||
public array $by_alias = array();
|
||||
public int $stales = 0;
|
||||
public int $corruptions = 0;
|
||||
public int $fast_jsons = 0;
|
||||
|
||||
function getByAlias(string $strid) : ConfigCacheEntry
|
||||
{
|
||||
if(array_key_exists($strid, $this->by_alias))
|
||||
return $this->by_alias[$strid];
|
||||
throw new Exception("Failed to find config by alias '$strid'!");
|
||||
}
|
||||
|
||||
function getById(int $id) : ConfigCacheEntry
|
||||
{
|
||||
if(array_key_exists($id, $this->by_id))
|
||||
return $this->by_id[$id];
|
||||
throw new Exception("Failed to find config by id '$id'!");
|
||||
}
|
||||
|
||||
function getByPath(string $path) : ConfigCacheEntry
|
||||
{
|
||||
if(array_key_exists($path, $this->by_path))
|
||||
return $this->by_path[$path];
|
||||
throw new Exception("Failed to find config by path '$path'!");
|
||||
}
|
||||
}
|
||||
|
||||
function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
|
||||
{
|
||||
$t = microtime(true);
|
||||
|
||||
$result = _config_cache_fetch($params);
|
||||
|
||||
if($params->verbose)
|
||||
echo "[CFG] Fetch from cache: " . round(microtime(true) - $t,2) . " sec.\n";
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
|
||||
{
|
||||
if($params->scanned->isEmpty())
|
||||
return new ConfigFetchResult();
|
||||
|
||||
$params->scanned->apply(function($base_dir, $files) { sort($files); return $files;});
|
||||
|
||||
$jobs = $params->splitJobs();
|
||||
|
||||
$serial = sizeof($jobs) == 1;
|
||||
|
||||
$results_by_job = _config_worker_run_procs($params, $jobs, $serial);
|
||||
|
||||
if(!$serial)
|
||||
{
|
||||
//NOTE: in case of any result error try serial processing
|
||||
if(array_search(false, $results_by_job, true/*strict*/) !== false)
|
||||
{
|
||||
if($params->verbose)
|
||||
echo "[CFG] Result error detected, trying serial processing...\n";
|
||||
$results_by_job = _config_worker_run_procs($params, $jobs, true);
|
||||
}
|
||||
}
|
||||
|
||||
$t = microtime(true);
|
||||
$result = _config_merge_fetch_results($params, $results_by_job);
|
||||
if($params->verbose)
|
||||
echo "[CFG] Merge results: " . round(microtime(true) - $t,2) . " sec.\n";
|
||||
|
||||
if($params->verbose)
|
||||
echo "[CFG] Miss(Fast)/Total: {$result->stales}($result->fast_jsons)/" . sizeof($result->all) . "\n";
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function config_cache_fetch_by_path(ConfigGlobals $globals, string $path, bool $force_stale = false) : ConfigCacheEntry
|
||||
{
|
||||
$path = realpath($path);
|
||||
$base_dir = config_map_base_dir($globals->base_dirs, $path);
|
||||
|
||||
$scanned = new ConfigScanResult();
|
||||
$scanned->add($base_dir, $path);
|
||||
|
||||
$ces = config_cache_fetch(new ConfigFetchParams(
|
||||
globals: $globals,
|
||||
scanned: $scanned,
|
||||
force_stale: $force_stale
|
||||
));
|
||||
|
||||
if(empty($ces->all))
|
||||
throw new Exception("Config not found at path '$path'");
|
||||
return $ces->all[0];
|
||||
}
|
||||
|
||||
function _config_merge_fetch_results(ConfigFetchParams $params, array $results_by_job) : ConfigFetchResult
|
||||
{
|
||||
$result = new ConfigFetchResult();
|
||||
$total_stales = 0;
|
||||
$total_fast_jsons = 0;
|
||||
|
||||
foreach($results_by_job as $results)
|
||||
{
|
||||
foreach($results as $item)
|
||||
{
|
||||
list($base_dir, $file, $cache_file, $is_stale, $parser_type) = $item;
|
||||
|
||||
if($parser_type === 1)
|
||||
++$total_fast_jsons;
|
||||
|
||||
$cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file));
|
||||
|
||||
//NOTE: handling any cache corruption errors
|
||||
if($cache_entry === null)
|
||||
{
|
||||
$is_stale = true;
|
||||
$cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
|
||||
++$result->corruptions;
|
||||
}
|
||||
|
||||
$includes = $cache_entry->includes;
|
||||
|
||||
if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes))
|
||||
{
|
||||
$is_stale = true;
|
||||
$cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file);
|
||||
//NOTE: let's change the mtime of the file which include other files,
|
||||
// so that on tne next build it will be 'older' than its includes
|
||||
// and won't trigger rebuild
|
||||
if($params->touch_files_with_includes)
|
||||
touch($file);
|
||||
}
|
||||
|
||||
if($is_stale)
|
||||
++$total_stales;
|
||||
|
||||
//we want results to be returned in the same order, so
|
||||
//we store entries by the file key and later retrieve array values
|
||||
$result->by_path[$file] = $cache_entry;
|
||||
$result->by_id[$cache_entry->id] = $cache_entry;
|
||||
$result->by_alias[$cache_entry->strid] = $cache_entry;
|
||||
}
|
||||
}
|
||||
|
||||
$result->all = array_values($result->by_path);
|
||||
$result->stales = $total_stales;
|
||||
$result->fast_jsons = $total_fast_jsons;
|
||||
return $result;
|
||||
}
|
||||
|
||||
function _config_invalidate_cache(ConfigFetchParams $params, string $base_dir, string $file, string $cache_file, ?int &$parser_type = null) : ConfigCacheEntry
|
||||
{
|
||||
list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file);
|
||||
if(!$conf_id)
|
||||
throw new Exception("Bad conf id: {$conf_id}");
|
||||
|
||||
//TODO: this is a bit ugly but kinda works for now
|
||||
$GLOBALS['CONFIG_GLOBALS'] = $params->globals;
|
||||
$GLOBALS['CONFIG_CURRENT_FILE'] = $file;
|
||||
$GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id;
|
||||
$GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
|
||||
|
||||
$pres = config_parse($params->globals->base_dirs, $file);
|
||||
if($pres->error !== 0)
|
||||
throw new Exception("Error({$pres->error}) while parsing {$file}:\n" . $pres->error_descr);
|
||||
|
||||
$parser_type = $pres->parser_type;
|
||||
$parsed_arr = $pres->parsed_arr;
|
||||
//TODO: these hardcoded keys below probably shouldn't be part of data?
|
||||
$parsed_arr['id'] = $conf_id;
|
||||
$parsed_arr['strid'] = $conf_strid;
|
||||
list($class, $class_id, $normalized_data) = config_apply_class_normalization($parsed_arr);
|
||||
|
||||
$cache_payload_file = config_get_cache_payload_path($params->globals, $file);
|
||||
|
||||
$cache_entry = new ConfigCacheEntry();
|
||||
$cache_entry->id = $conf_id;
|
||||
$cache_entry->strid = $conf_strid;
|
||||
$cache_entry->cache_file = $cache_file;
|
||||
$cache_entry->class = $class;
|
||||
$cache_entry->class_id = $class_id;
|
||||
$cache_entry->payload_file = $cache_payload_file;
|
||||
//TODO: pass flag if file is actually normalized?
|
||||
$cache_entry->file = normalize_path($file);
|
||||
$cache_entry->includes = config_get_module_includes($pres->jsm_module);
|
||||
$cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
|
||||
//TODO: do we really need these refs?
|
||||
$cache_entry->refs = config_content_match_refs($pres->normalized_jzon);
|
||||
|
||||
ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
|
||||
ensure_write($cache_payload_file, config_msgpack_pack($normalized_data));
|
||||
|
||||
return $cache_entry;
|
||||
}
|
||||
|
||||
1138
config.inc.php
1138
config.inc.php
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,36 @@
|
|||
<?php
|
||||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
if(function_exists('msgpack_pack'))
|
||||
{
|
||||
function config_msgpack_pack(array $data) : string
|
||||
{
|
||||
$prev = ini_set('msgpack.use_str8_serialization', '0');
|
||||
$res = msgpack_pack($data);
|
||||
if($prev != '0')
|
||||
ini_set('msgpack.use_str8_serialization', $prev);
|
||||
return $res;
|
||||
}
|
||||
|
||||
function config_msgpack_unpack(string $data) : array
|
||||
{
|
||||
return msgpack_unpack($data);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
function config_msgpack_pack(array $data) : string
|
||||
{
|
||||
include_once(__DIR__ . '/msgpack/msgpack_custom.inc.php');
|
||||
|
||||
$packer = new \MessagePackCustom();
|
||||
return $packer->pack($data);
|
||||
}
|
||||
|
||||
function config_msgpack_unpack(string $data) : array
|
||||
{
|
||||
return \MessagePack\MessagePack::unpack($data);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,343 @@
|
|||
<?php
|
||||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
class ConfigPackParams
|
||||
{
|
||||
/*var ConfigCacheEntry[]*/
|
||||
public array $cache_entries;
|
||||
public bool $use_lz4 = false;
|
||||
public bool $use_config_refs = false;
|
||||
public int $binary_format = 1; //1,2 supported
|
||||
public ?int $version = null;
|
||||
public bool $debug = false;
|
||||
|
||||
function __construct(array $cache_entries, int $version, bool $use_lz4 = false,
|
||||
bool $use_config_refs = false, int $binary_format = 1, bool $debug = false)
|
||||
{
|
||||
$this->cache_entries = $cache_entries;
|
||||
$this->use_lz4 = $use_lz4;
|
||||
$this->use_config_refs = $use_config_refs;
|
||||
$this->binary_format = $binary_format;
|
||||
$this->version = $version;
|
||||
$this->debug = $debug;
|
||||
}
|
||||
}
|
||||
|
||||
function config_pack_bundle(ConfigPackParams $params) : string
|
||||
{
|
||||
$t = microtime(true);
|
||||
|
||||
$packed_data = null;
|
||||
|
||||
if($params->binary_format == 1)
|
||||
{
|
||||
$packed_data = _config_pack_bundle_fmt1(
|
||||
$params->cache_entries,
|
||||
$params->use_lz4,
|
||||
$params->use_config_refs,
|
||||
$params->version
|
||||
);
|
||||
}
|
||||
else if($params->binary_format == 2)
|
||||
{
|
||||
$packed_data = _config_pack_bundle_fmt2(
|
||||
$params->cache_entries,
|
||||
$params->use_lz4,
|
||||
$params->use_config_refs,
|
||||
$params->version
|
||||
);
|
||||
}
|
||||
else
|
||||
throw new Exception("Unknown binary format: {$params->binary_format}");
|
||||
|
||||
if($params->debug)
|
||||
echo "[CFG] Packed entries: " . sizeof($params->cache_entries) . ", total: " .
|
||||
kb($packed_data) . ", format: {$params->binary_format}, lz4: " .
|
||||
var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) .
|
||||
", CRC: " . crc32($packed_data) .
|
||||
", " . round(microtime(true) - $t,2) . " sec.\n";
|
||||
|
||||
return $packed_data;
|
||||
}
|
||||
|
||||
function _config_pack_bundle_fmt1(
|
||||
array $cache_entries,
|
||||
bool $use_lz4,
|
||||
bool $use_config_refs,
|
||||
int $version) : string
|
||||
{
|
||||
$MAP = array();
|
||||
$STRIDMAP = array();
|
||||
|
||||
$payloads = array();
|
||||
$payloads_offset = 0;
|
||||
foreach($cache_entries as $entry)
|
||||
{
|
||||
list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
|
||||
$payload_size = strlen($payload);
|
||||
$payloads[] = array($payloads_offset, $payload, $format, $payload_size);
|
||||
$payloads_offset += $payload_size;
|
||||
}
|
||||
|
||||
$header = array();
|
||||
foreach($cache_entries as $idx => $entry)
|
||||
{
|
||||
if(isset($MAP[$entry->id]))
|
||||
throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
|
||||
$MAP[$entry->id] = $entry->strid;
|
||||
|
||||
$strid_crc = crc32($entry->strid);
|
||||
if(isset($STRIDMAP[$strid_crc]))
|
||||
throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
|
||||
$STRIDMAP[$strid_crc] = $entry->strid;
|
||||
|
||||
$header[] = array(
|
||||
$payloads[$idx][2],
|
||||
$entry->id,
|
||||
crc32($entry->strid),
|
||||
$entry->class_id,
|
||||
$payloads[$idx][0],
|
||||
$payloads[$idx][3]
|
||||
);
|
||||
}
|
||||
|
||||
$header_msgpack = config_msgpack_pack($header);
|
||||
$payloads_bundle = '';
|
||||
foreach($payloads as $item)
|
||||
$payloads_bundle .= $item[1];
|
||||
|
||||
$packed_data =
|
||||
pack("C", 1) .
|
||||
pack("V", $version) .
|
||||
pack("V", strlen($header_msgpack)) .
|
||||
$header_msgpack .
|
||||
$payloads_bundle;
|
||||
|
||||
return $packed_data;
|
||||
}
|
||||
|
||||
function _config_pack_bundle_fmt2(
|
||||
array $cache_entries,
|
||||
bool $use_lz4,
|
||||
bool $use_config_refs,
|
||||
int $version) : string
|
||||
{
|
||||
$MAP = array();
|
||||
$STRIDMAP = array();
|
||||
$STRIDLIST = array();
|
||||
|
||||
$payloads = array();
|
||||
$strids = array();
|
||||
$payloads_offset = 0;
|
||||
foreach($cache_entries as $entry)
|
||||
{
|
||||
list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
|
||||
$payload_size = strlen($payload);
|
||||
$payloads[] = array($payloads_offset, $payload, $format, $payload_size);
|
||||
$payloads_offset += $payload_size;
|
||||
|
||||
$strids_indices = array();
|
||||
$strid_parts = explode('/', ltrim($entry->strid, '@'));
|
||||
foreach($strid_parts as $strid_part)
|
||||
{
|
||||
if(!isset($STRIDMAP[$strid_part]))
|
||||
{
|
||||
$strid_index = count($STRIDLIST);
|
||||
$STRIDLIST[] = $strid_part;
|
||||
$STRIDMAP[$strid_part] = $strid_index;
|
||||
$strids_indices[] = $strid_index;
|
||||
}
|
||||
else
|
||||
$strids_indices[] = $STRIDMAP[$strid_part];
|
||||
}
|
||||
$strids[] = $strids_indices;
|
||||
}
|
||||
|
||||
$header = array();
|
||||
foreach($cache_entries as $idx => $entry)
|
||||
{
|
||||
if(isset($MAP[$entry->id]))
|
||||
throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
|
||||
$MAP[$entry->id] = $entry->strid;
|
||||
|
||||
$header[] = array(
|
||||
$payloads[$idx][2], //format
|
||||
$entry->id,
|
||||
$strids[$idx], //strid as a lookup indices
|
||||
$entry->class_id,
|
||||
$payloads[$idx][0], //offset
|
||||
$payloads[$idx][3] //size
|
||||
);
|
||||
}
|
||||
|
||||
$strids_msgpack = config_msgpack_pack($STRIDLIST);
|
||||
|
||||
$header_msgpack = config_msgpack_pack($header);
|
||||
$payloads_bundle = '';
|
||||
foreach($payloads as $item)
|
||||
$payloads_bundle .= $item[1];
|
||||
|
||||
$packed_data =
|
||||
pack("C", 2) .
|
||||
pack("V", $version) .
|
||||
pack("V", strlen($strids_msgpack)) .
|
||||
pack("V", strlen($header_msgpack)) .
|
||||
$strids_msgpack .
|
||||
$header_msgpack .
|
||||
$payloads_bundle;
|
||||
|
||||
return $packed_data;
|
||||
}
|
||||
|
||||
//format: [[class_id, [data]], ...[class_id, [data]]]
|
||||
function config_unpack_bundle(string $packed_data) : array
|
||||
{
|
||||
$packed_info = substr($packed_data, 0, 1);
|
||||
$info = unpack('Cformat', $packed_info);
|
||||
|
||||
if($info['format'] === 1)
|
||||
{
|
||||
return _config_unpack_bundle_fmt1($packed_data);
|
||||
}
|
||||
else if($info['format'] === 2)
|
||||
{
|
||||
return _config_unpack_bundle_fmt2($packed_data);
|
||||
}
|
||||
else
|
||||
throw new Exception("Unknown format: {$info['format']}");
|
||||
}
|
||||
|
||||
function _config_unpack_bundle_fmt1(string $packed_data) : array
|
||||
{
|
||||
$packed_info = substr($packed_data, 0, 1+4+4);
|
||||
|
||||
$info = unpack('Cformat/Vversion/Vheader_len', $packed_info);
|
||||
|
||||
if($info['format'] !== 1)
|
||||
throw new Exception("Unknown format: {$info['format']}");
|
||||
|
||||
$header_msgpack = substr($packed_data, 1+4+4, $info['header_len']);
|
||||
$header = config_msgpack_unpack($header_msgpack);
|
||||
|
||||
$payloads_bundle = substr($packed_data, 1+4+4+$info['header_len']);
|
||||
|
||||
$entries = array();
|
||||
foreach($header as $item)
|
||||
{
|
||||
list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
|
||||
|
||||
$payload = substr($payloads_bundle, $offset, $size);
|
||||
|
||||
$entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
|
||||
}
|
||||
|
||||
return $entries;
|
||||
}
|
||||
|
||||
function _config_unpack_bundle_fmt2(string $packed_data) : array
|
||||
{
|
||||
$packed_info = substr($packed_data, 0, 1+4+4+4);
|
||||
|
||||
$info = unpack('Cformat/Vversion/Vstrids_len/Vheader_len', $packed_info);
|
||||
|
||||
if($info['format'] !== 2)
|
||||
throw new Exception("Unknown format: {$info['format']}");
|
||||
|
||||
$strids_msgpack = substr($packed_data, 1+4+4+4, $info['strids_len']);
|
||||
$strids = config_msgpack_unpack($strids_msgpack);
|
||||
|
||||
$header_msgpack = substr($packed_data, 1+4+4+4+$info['strids_len'], $info['header_len']);
|
||||
$header = config_msgpack_unpack($header_msgpack);
|
||||
|
||||
$payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
|
||||
|
||||
$entries = array();
|
||||
foreach($header as $item)
|
||||
{
|
||||
list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
|
||||
|
||||
$payload = substr($payloads_bundle, $offset, $size);
|
||||
|
||||
$entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
|
||||
}
|
||||
|
||||
return $entries;
|
||||
}
|
||||
|
||||
//format: [format_id, payload_data]
|
||||
function _config_get_payload(ConfigCacheEntry $ce, bool $use_lz4, bool $use_config_refs) : array
|
||||
{
|
||||
$format = ConfigCacheEntry::FMT_BINARY;
|
||||
$payload = null;
|
||||
|
||||
if($use_config_refs && $ce->payload_file)
|
||||
{
|
||||
$format = ConfigCacheEntry::FMT_FILE_REF;
|
||||
$payload = $ce->payload_file;
|
||||
}
|
||||
else
|
||||
{
|
||||
$payload = $ce->payload;
|
||||
if($use_lz4 && strlen($payload) > 512)
|
||||
{
|
||||
$format = ConfigCacheEntry::FMT_LZ4;
|
||||
$payload = lz4_compress($payload, 9);
|
||||
}
|
||||
}
|
||||
|
||||
return array($format, $payload);
|
||||
}
|
||||
|
||||
function _config_unpack_payload(int $format, string $payload) : array
|
||||
{
|
||||
$msg_packed = null;
|
||||
if($format === ConfigCacheEntry::FMT_LZ4)
|
||||
$msg_packed = lz4_uncompress($payload);
|
||||
else if($format === ConfigCacheEntry::FMT_BINARY)
|
||||
$msg_packed = $payload;
|
||||
else if($format === ConfigCacheEntry::FMT_FILE_REF)
|
||||
$msg_packed = ensure_read($payload);
|
||||
else
|
||||
throw new Exception("Bad format: $format");
|
||||
return config_msgpack_unpack($msg_packed);
|
||||
}
|
||||
|
||||
function config_pack_and_write_bundle(
|
||||
/*var ConfBase[]*/
|
||||
array $configs,
|
||||
string $file_path,
|
||||
bool $use_lz4 = true,
|
||||
int $binary_format = 1,
|
||||
?int $version = null
|
||||
)
|
||||
{
|
||||
$cache_entries = array();
|
||||
foreach($configs as $conf)
|
||||
{
|
||||
$payload = config_msgpack_pack($conf->export());
|
||||
|
||||
//creating fake cache entries
|
||||
$entry = new ConfigCacheEntry();
|
||||
$entry->id = $conf->id;
|
||||
$entry->strid = $conf->strid;
|
||||
$entry->class_id = $conf->getClassId();
|
||||
$entry->class = get_class($conf);
|
||||
$entry->payload = $payload;
|
||||
$entry->config = $conf;
|
||||
|
||||
$cache_entries[] = $entry;
|
||||
}
|
||||
|
||||
$packed_data = config_pack_bundle(
|
||||
new ConfigPackParams(
|
||||
cache_entries: $cache_entries,
|
||||
use_lz4: $use_lz4,
|
||||
binary_format: $binary_format,
|
||||
version: $version
|
||||
)
|
||||
);
|
||||
|
||||
ensure_write($file_path, $packed_data);
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
<?php
|
||||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
class ConfigParseResult
|
||||
{
|
||||
public int $error = 0;
|
||||
public string $error_descr;
|
||||
public string $normalized_jzon = '';
|
||||
public array $parsed_arr;
|
||||
public int $parser_type;
|
||||
public $jsm_module;
|
||||
}
|
||||
|
||||
function config_parse(array $base_dirs, string $file) : ConfigParseResult
|
||||
{
|
||||
$res = new ConfigParseResult();
|
||||
|
||||
$normalized_jzon = '';
|
||||
try
|
||||
{
|
||||
$jsm = new \JSM($base_dirs, $file);
|
||||
list($normalized_jzon, $jsm_module) = $jsm->process();
|
||||
$res->normalized_jzon = $normalized_jzon;
|
||||
$res->jsm_module = $jsm_module;
|
||||
}
|
||||
catch(Exception $e)
|
||||
{
|
||||
$res->error = 1;
|
||||
$res->error_descr = "File '$file':\n" . $e->getMessage() . "\n" . $e->getTraceAsString();
|
||||
return $res;
|
||||
}
|
||||
|
||||
$decode_res = config_check_and_decode_jzon($normalized_jzon);
|
||||
|
||||
if($decode_res[0] === 0)
|
||||
{
|
||||
$res->parsed_arr = $decode_res[2];
|
||||
$res->parser_type = $decode_res[3];
|
||||
}
|
||||
else
|
||||
{
|
||||
$res->error = $decode_res[0];
|
||||
$res->error_descr = "File '$file':\n" . $decode_res[1];
|
||||
}
|
||||
|
||||
return $res;
|
||||
}
|
||||
|
||||
function config_check_and_decode_jzon(string $json) : array
|
||||
{
|
||||
try
|
||||
{
|
||||
$arr = \jzon_parse($json, $parser);
|
||||
return array(0, "", $arr, $parser);
|
||||
}
|
||||
catch(Exception $e)
|
||||
{
|
||||
return array(1, $e->getMessage(), array(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
function config_get_header(string $file, ?int &$conf_id, ?string &$strid, bool $use_cache = true) : bool
|
||||
{
|
||||
static $cache = array();
|
||||
if($use_cache && isset($cache[$file]))
|
||||
{
|
||||
list($conf_id, $strid) = $cache[$file];
|
||||
return true;
|
||||
}
|
||||
|
||||
$h = fopen($file, "r");
|
||||
$line = fgets($h, 256);
|
||||
fclose($h);
|
||||
|
||||
//TODO: rewrite it without regex usage
|
||||
if(preg_match('~\{\s*/\*\s*proto_id\s*=\s*(\d+)\s*;\s*alias\s*=\s*([^\s]+)~', $line, $matches))
|
||||
{
|
||||
$conf_id = (int)$matches[1];
|
||||
$strid = $matches[2];
|
||||
if($use_cache)
|
||||
$cache[$file] = array($conf_id, $strid);
|
||||
return true;
|
||||
}
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
function config_set_header(string $contents, int $conf_id, string $strid, bool &$is_success) : string
|
||||
{
|
||||
$contents = preg_replace('~\s*\{~', "{ /* proto_id = {$conf_id} ; alias = {$strid} */", $contents, 1/*limit*/, $count);
|
||||
$is_success = $count == 1;
|
||||
return $contents;
|
||||
}
|
||||
|
||||
function config_replace_header(string $contents) : string
|
||||
{
|
||||
return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents);
|
||||
}
|
||||
|
||||
//returns [proto_id, alias]
|
||||
function config_ensure_header(string $conf_dir, string $file, bool $force = false) : array
|
||||
{
|
||||
if(config_get_header($file, $curr_proto_id, $curr_alias, !$force))
|
||||
{
|
||||
//TODO: add 'normalized' argument when we are sure both path arguments are normalized
|
||||
$alias = config_file2strid($conf_dir, $file);
|
||||
if($force)
|
||||
$curr_proto_id = config_file2id($conf_dir, $file);
|
||||
|
||||
//NOTE: keeping current proto id intact if not forced
|
||||
if($force || $curr_alias !== $alias)
|
||||
{
|
||||
$lines = file($file);
|
||||
//TODO: why not using config_set_header(..) ?
|
||||
$lines[0] = preg_replace(
|
||||
'~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{ /* proto_id = {$curr_proto_id} ; alias = {$alias} */",
|
||||
$lines[0],
|
||||
1/*limit*/,
|
||||
$count);
|
||||
if($count != 1)
|
||||
throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
|
||||
ensure_write_if_differs($file, join("", $lines));
|
||||
}
|
||||
|
||||
return array($curr_proto_id, $alias);
|
||||
}
|
||||
else
|
||||
{
|
||||
$conf_id = config_file2id($conf_dir, $file);
|
||||
$alias = config_file2strid($conf_dir, $file);
|
||||
|
||||
$lines = file($file);
|
||||
|
||||
$is_success = false;
|
||||
$lines[0] = config_set_header($lines[0], $conf_id, $alias, $is_success);
|
||||
if(!$is_success)
|
||||
throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
|
||||
ensure_write($file, join("", $lines));
|
||||
|
||||
return array($conf_id, $alias);
|
||||
}
|
||||
}
|
||||
|
||||
function config_apply_class_normalization(array $kv_data) : array
|
||||
{
|
||||
$class = $kv_data['class'];
|
||||
$norm_data = array();
|
||||
call_user_func_array([$class, 'normalize'], array(&$kv_data, &$norm_data));
|
||||
return array($class, $class::CLASS_ID, $norm_data);
|
||||
}
|
||||
|
||||
//returns array<ref, number>
|
||||
function config_content_match_refs(string $src) : array
|
||||
{
|
||||
$refs = array();
|
||||
|
||||
if(strpos($src, '@') === false)
|
||||
return $refs;
|
||||
|
||||
if(preg_match_all('~"(@[^"]+)"~', $src, $matches))
|
||||
{
|
||||
foreach($matches[1] as $ref)
|
||||
{
|
||||
if(!isset($refs[$ref]))
|
||||
$refs[$ref] = 1;
|
||||
++$refs[$ref];
|
||||
}
|
||||
}
|
||||
return $refs;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
<?php
|
||||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
class ConfigScanResult
|
||||
{
|
||||
/*var array<string, string[]>*/
|
||||
public array $base_dir2files = array();
|
||||
|
||||
function isEmpty() : bool
|
||||
{
|
||||
return empty($this->base_dir2files);
|
||||
}
|
||||
|
||||
function count() : int
|
||||
{
|
||||
$total = 0;
|
||||
foreach($this->base_dir2files as $base_dir => $files)
|
||||
$total += count($files);
|
||||
return $total;
|
||||
}
|
||||
|
||||
function apply(callable $fn)
|
||||
{
|
||||
foreach($this->base_dir2files as $base_dir => $files)
|
||||
$this->base_dir2files[$base_dir] = $fn($base_dir, $files);
|
||||
}
|
||||
|
||||
function filter(callable $filter)
|
||||
{
|
||||
foreach($this->base_dir2files as $base_dir => $files)
|
||||
$this->base_dir2files[$base_dir] = array_filter($files, $filter);
|
||||
}
|
||||
|
||||
function forEachFile(callable $fn)
|
||||
{
|
||||
foreach($this->base_dir2files as $base_dir => $files)
|
||||
{
|
||||
foreach($files as $file)
|
||||
$fn($base_dir, $file);
|
||||
}
|
||||
}
|
||||
|
||||
function add(string $base_dir, string $file)
|
||||
{
|
||||
if(!isset($this->base_dir2files[$base_dir]))
|
||||
$this->base_dir2files[$base_dir] = array();
|
||||
|
||||
$this->base_dir2files[$base_dir][] = $file;
|
||||
}
|
||||
|
||||
//returns [[base_dir, file1], [base_dir, file2], ...]
|
||||
function getFlatArray() : array
|
||||
{
|
||||
$flat = [];
|
||||
foreach($this->base_dir2files as $base_dir => $files)
|
||||
{
|
||||
foreach($files as $file)
|
||||
$flat[] = [$base_dir, $file];
|
||||
}
|
||||
return $flat;
|
||||
}
|
||||
|
||||
function getAllFiles() : array
|
||||
{
|
||||
$all_files = [];
|
||||
foreach($this->base_dir2files as $base_dir => $files)
|
||||
$all_files = array_merge($all_files, $files);
|
||||
|
||||
return $all_files;
|
||||
}
|
||||
}
|
||||
|
||||
function config_scan_files(
|
||||
array $base_dirs,
|
||||
string $ext_filter = '.conf.js',
|
||||
bool $verbose = false
|
||||
) : ConfigScanResult
|
||||
{
|
||||
$t = microtime(true);
|
||||
|
||||
$result = new ConfigScanResult();
|
||||
|
||||
foreach($base_dirs as $base_dir)
|
||||
{
|
||||
$result->base_dir2files[$base_dir] =
|
||||
scan_files_rec(array($base_dir), array($ext_filter));
|
||||
}
|
||||
|
||||
if($verbose)
|
||||
echo "[CFG] File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec.\n";
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
<?php
|
||||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
task('config_worker', function(array $args)
|
||||
{
|
||||
if(sizeof($args) != 3)
|
||||
throw new Exception("Config worker args not set");
|
||||
|
||||
$in_file = $args[0];
|
||||
$out_file = $args[1];
|
||||
$err_file = $args[2];
|
||||
|
||||
try
|
||||
{
|
||||
list($params, $job) = unserialize(ensure_read($in_file));
|
||||
$result = _config_worker_func($params, $job);
|
||||
ensure_write($out_file, serialize($result));
|
||||
}
|
||||
catch(Exception $e)
|
||||
{
|
||||
//NOTE: explicitely catching all exceptions and writing to the error file
|
||||
// since under Windows error file stream redirect may work unreliably
|
||||
file_put_contents($err_file, $e->getMessage() . "\n" . $e->getTraceAsString(), FILE_APPEND);
|
||||
throw $e;
|
||||
}
|
||||
});
|
||||
|
||||
function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $serial) : array
|
||||
{
|
||||
if($serial)
|
||||
{
|
||||
$results_by_job = array();
|
||||
foreach($jobs as $job)
|
||||
$results_by_job[] = _config_worker_func($params, $job);
|
||||
return $results_by_job;
|
||||
}
|
||||
else
|
||||
{
|
||||
$worker_args = array();
|
||||
foreach($jobs as $idx => $job)
|
||||
$worker_args[] = array($params, $job);
|
||||
|
||||
return run_background_gamectl_workers('config_worker', $worker_args);
|
||||
}
|
||||
}
|
||||
|
||||
//returns [[base_dir, file, cache_file, was_stale], ...]
|
||||
function _config_worker_func(ConfigFetchParams $params, array $job) : array
|
||||
{
|
||||
$start_time = microtime(true);
|
||||
|
||||
$params->globals->initWorker();
|
||||
|
||||
list($idx, $start_time, $chunk) = $job;
|
||||
if($params->verbose)
|
||||
echo "[CFG] Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)\n";
|
||||
|
||||
$fast_parser_num = 0;
|
||||
$results = array();
|
||||
foreach($chunk as $file_idx => $chunk_data)
|
||||
{
|
||||
try
|
||||
{
|
||||
list($base_dir, $file) = $chunk_data;
|
||||
|
||||
if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0)
|
||||
echo "[CFG] Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...\n";
|
||||
|
||||
$cache_file = config_get_cache_path($params->globals, $file);
|
||||
|
||||
$parser_type = null;
|
||||
|
||||
$is_stale = true;
|
||||
if(!$params->force_stale)
|
||||
$is_stale = need_to_regen($cache_file, array($file));
|
||||
|
||||
if($is_stale)
|
||||
_config_invalidate_cache($params, $base_dir, $file, $cache_file, $parser_type);
|
||||
|
||||
$results[] = array($base_dir, $file, $cache_file, $is_stale, $parser_type);
|
||||
}
|
||||
catch(Exception $e)
|
||||
{
|
||||
throw new Exception("Error in file '$file': " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
if($params->verbose)
|
||||
echo "[CFG] Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)\n";
|
||||
|
||||
return $results;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
<?php
|
||||
namespace taskman;
|
||||
use Exception;
|
||||
|
||||
function config_get_tmp_build_path(ConfigGlobals $globals, string $file) : string
|
||||
{
|
||||
$name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file))));
|
||||
$name = ltrim($name, "-");
|
||||
return normalize_path($globals->build_dir . "/$name");
|
||||
}
|
||||
|
||||
function config_map_base_dir(
|
||||
array $base_dirs, string $file,
|
||||
bool $normalized = false, bool $strict = true
|
||||
) : ?string
|
||||
{
|
||||
if(!$normalized)
|
||||
$file = normalize_path($file);
|
||||
foreach($base_dirs as $dir)
|
||||
if(strpos($file, $dir) === 0)
|
||||
return $dir;
|
||||
if($strict)
|
||||
throw new Exception("File '$file' is not mapped to any base dir");
|
||||
return null;
|
||||
}
|
||||
|
||||
function config_real_path(array $base_dirs, string $rel_path, bool $strict = true) : ?string
|
||||
{
|
||||
foreach($base_dirs as $dir)
|
||||
if(is_file($dir . '/' . $rel_path))
|
||||
return $dir . '/' . $rel_path;
|
||||
if($strict)
|
||||
throw new Exception("No file for relative path '$rel_path'");
|
||||
return null;
|
||||
}
|
||||
|
||||
function config_is_file(string $filename) : bool
|
||||
{
|
||||
return (strrpos($filename, '.conf.js') === (strlen($filename) - 8));
|
||||
}
|
||||
|
||||
function config_file2id(string $conf_dir, string $filename) : int
|
||||
{
|
||||
$nfilename = config_make_path($conf_dir, $filename);
|
||||
return config_crc28($nfilename);
|
||||
}
|
||||
|
||||
function config_file2strid(string $conf_dir, string $filename) : string
|
||||
{
|
||||
$strid = config_make_path($conf_dir, $filename);
|
||||
$fname_idx = strrpos($strid, "/");
|
||||
$fname_idx = strpos($strid, ".", $fname_idx);
|
||||
$strid = substr($strid, 0, $fname_idx);
|
||||
return "@".$strid;
|
||||
}
|
||||
|
||||
function config_make_path(string $conf_dir, string $path) : string
|
||||
{
|
||||
return ltrim(str_replace(normalize_path($conf_dir, true/*nix*/),
|
||||
'',
|
||||
normalize_path($path, true/*nix*/)),
|
||||
'/');
|
||||
}
|
||||
|
||||
function config_crc28(string $what) : int
|
||||
{
|
||||
return crc32($what) & 0xFFFFFFF;
|
||||
}
|
||||
|
||||
function config_get_module_includes(\JSM_Module $cm) : array
|
||||
{
|
||||
$includes = array();
|
||||
foreach($cm->getIncludes() as $include => $_)
|
||||
{
|
||||
//maybe we should take .php includes into account as well?
|
||||
if(str_ends_with($include, ".php"))
|
||||
continue;
|
||||
$includes[] = $include;
|
||||
}
|
||||
return $includes;
|
||||
}
|
||||
|
||||
function config_walk_fields(object $proto, callable $callback)
|
||||
{
|
||||
if(!method_exists($proto, 'CLASS_FIELDS_PROPS'))
|
||||
return;
|
||||
|
||||
foreach($proto->CLASS_FIELDS_PROPS() as $field => $tokens)
|
||||
{
|
||||
$value = $proto->$field;
|
||||
$callback($proto, $field, $value, $tokens);
|
||||
|
||||
if(is_object($value))
|
||||
config_walk_fields($value, $callback);
|
||||
else if(is_array($value))
|
||||
{
|
||||
foreach($value as $num => $item)
|
||||
{
|
||||
if(is_object($item))
|
||||
config_walk_fields($item, $callback);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
|
||||
{
|
||||
if(!isset($map[$file]))
|
||||
return array($file);
|
||||
|
||||
$tpls = $map[$file];
|
||||
$tpls[] = $file;
|
||||
$res = array();
|
||||
foreach($tpls as $tpl)
|
||||
{
|
||||
$content = ensure_read($tpl);
|
||||
$content = i18n_decode_string($content);
|
||||
if(strpos($content, $text) !== FALSE)
|
||||
$res[] = $tpl;
|
||||
}
|
||||
|
||||
return $res;
|
||||
}
|
||||
|
||||
//TODO:
|
||||
//function conf2json(object $conf, int $json_flags = 0) : string
|
||||
//{
|
||||
// $arr = $conf->export(true);
|
||||
// $arr['class'] = get_class($conf);
|
||||
// unset($arr['id']);
|
||||
// unset($arr['strid']);
|
||||
// $json = json_encode($arr, $json_flags);
|
||||
// $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json);
|
||||
// return $json;
|
||||
//}
|
||||
|
||||
Loading…
Reference in New Issue