diff --git a/cache.inc.php b/cache.inc.php new file mode 100644 index 0000000..f7e4ba9 --- /dev/null +++ b/cache.inc.php @@ -0,0 +1,394 @@ +export(); + return serialize($d); + } + + static function unserialize(string $str) : ?ConfigCacheEntry + { + $d = @unserialize($str); + if(!is_array($d)) + return null; + $ce = new ConfigCacheEntry(); + $ce->import($d); + return $ce; + } + + function __construct() + { + $this->extras = ConfigCacheEntryExtras::create(); + } + + function __get(string $name) + { + if($name === "config") + { + if($this->_config === null) + { + $klass = $this->class; + $data = config_msgpack_unpack($this->payload); + $this->_config = new $klass(); + $this->_config->import($data); + } + return $this->_config; + } + else if($name === "payload") + { + if($this->_payload !== null) + return $this->_payload; + //if payload not set directly not storing it + return ensure_read($this->payload_file); + } + else + throw new Exception("No such property '$name'"); + } + + function __set(string $name, $v) + { + if($name === "config") + $this->_config = $v; + else if($name === "payload") + $this->_payload = $v; + else + throw new Exception("No such property '$name'"); + } + + function export() : array + { + $d = array(); + $d[] = $this->class; + $d[] = $this->class_id; + $d[] = $this->id; + $d[] = $this->strid; + $d[] = $this->cache_file; + $d[] = $this->payload_file; + $d[] = $this->file; + $d[] = $this->includes; + $d[] = $this->refs; + $d[] = $this->extras->export(); + return $d; + } + + function import(array $d) + { + $extras = array(); + + list( + $this->class, + $this->class_id, + $this->id, + $this->strid, + $this->cache_file, + $this->payload_file, + $this->file, + $this->includes, + $this->refs, + $extras, + ) = $d; + + $this->extras->import($extras); + } +} + +class ConfigCacheEntryExtras extends \stdClass +{ + private static string $klass = ConfigCacheEntryExtras::class; + + static function init(string $project_specific_klass) + { + self::$klass = $project_specific_klass; + } + + static function create() : object + { + return new self::$klass(); + } + + function export() : array + { + $as_array = get_object_vars($this); + return $as_array; + } + + function import(array $as_array) + { + foreach($as_array as $field_name => $field_value) + $this->$field_name = $field_value; + } +} + +class ConfigFetchParams +{ + public ConfigGlobals $globals; + public ConfigScanResult $scanned; + public bool $force_stale = false; + public bool $verbose = false; + public ?int $max_workers = null; + public bool $touch_files_with_includes = true; + + function __construct(ConfigGlobals $globals, ConfigScanResult $scanned, + bool $force_stale = false, bool $verbose = false, ?int $max_workers = null) + { + $this->globals = $globals; + $this->scanned = $scanned; + $this->force_stale = $force_stale; + $this->verbose = $verbose; + $this->max_workers = $max_workers; + } + + function calcMaxWorkers() : int + { + $max_workers = $this->max_workers; + if($max_workers === null) + $max_workers = $this->scanned->count() < 20 ? 1 : 5; + return $max_workers; + } + + function splitFilesByChunks(int $max_workers) : array + { + $chunk_size = (int)ceil($this->scanned->count()/$max_workers); + return array_chunk($this->scanned->getFlatArray(), $chunk_size); + } + + //returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ] + function splitJobs() : array + { + $max_workers = $this->calcMaxWorkers(); + $jobs = array(); + foreach($this->splitFilesByChunks($max_workers) as $idx => $chunk) + $jobs[] = array($idx, microtime(true), $chunk); + return $jobs; + } +} + +class ConfigFetchResult +{ + /** @var ConfigCacheEntry[] */ + public array $all = array(); + /** @var array */ + public array $by_id = array(); + /** @var array */ + public array $by_path = array(); + /** @var array */ + public array $by_alias = array(); + public int $stales = 0; + public int $corruptions = 0; + public int $fast_jsons = 0; + + function getByAlias(string $strid) : ConfigCacheEntry + { + if(array_key_exists($strid, $this->by_alias)) + return $this->by_alias[$strid]; + throw new Exception("Failed to find config by alias '$strid'!"); + } + + function getById(int $id) : ConfigCacheEntry + { + if(array_key_exists($id, $this->by_id)) + return $this->by_id[$id]; + throw new Exception("Failed to find config by id '$id'!"); + } + + function getByPath(string $path) : ConfigCacheEntry + { + if(array_key_exists($path, $this->by_path)) + return $this->by_path[$path]; + throw new Exception("Failed to find config by path '$path'!"); + } +} + +function config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult +{ + $t = microtime(true); + + $result = _config_cache_fetch($params); + + if($params->verbose) + echo "[CFG] Fetch from cache: " . round(microtime(true) - $t,2) . " sec.\n"; + + return $result; +} + +function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult +{ + if($params->scanned->isEmpty()) + return new ConfigFetchResult(); + + $params->scanned->apply(function($base_dir, $files) { sort($files); return $files;}); + + $jobs = $params->splitJobs(); + + $serial = sizeof($jobs) == 1; + + $results_by_job = _config_worker_run_procs($params, $jobs, $serial); + + if(!$serial) + { + //NOTE: in case of any result error try serial processing + if(array_search(false, $results_by_job, true/*strict*/) !== false) + { + if($params->verbose) + echo "[CFG] Result error detected, trying serial processing...\n"; + $results_by_job = _config_worker_run_procs($params, $jobs, true); + } + } + + $t = microtime(true); + $result = _config_merge_fetch_results($params, $results_by_job); + if($params->verbose) + echo "[CFG] Merge results: " . round(microtime(true) - $t,2) . " sec.\n"; + + if($params->verbose) + echo "[CFG] Miss(Fast)/Total: {$result->stales}($result->fast_jsons)/" . sizeof($result->all) . "\n"; + + return $result; +} + +function config_cache_fetch_by_path(ConfigGlobals $globals, string $path, bool $force_stale = false) : ConfigCacheEntry +{ + $path = realpath($path); + $base_dir = config_map_base_dir($globals->base_dirs, $path); + + $scanned = new ConfigScanResult(); + $scanned->add($base_dir, $path); + + $ces = config_cache_fetch(new ConfigFetchParams( + globals: $globals, + scanned: $scanned, + force_stale: $force_stale + )); + + if(empty($ces->all)) + throw new Exception("Config not found at path '$path'"); + return $ces->all[0]; +} + +function _config_merge_fetch_results(ConfigFetchParams $params, array $results_by_job) : ConfigFetchResult +{ + $result = new ConfigFetchResult(); + $total_stales = 0; + $total_fast_jsons = 0; + + foreach($results_by_job as $results) + { + foreach($results as $item) + { + list($base_dir, $file, $cache_file, $is_stale, $parser_type) = $item; + + if($parser_type === 1) + ++$total_fast_jsons; + + $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file)); + + //NOTE: handling any cache corruption errors + if($cache_entry === null) + { + $is_stale = true; + $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file); + ++$result->corruptions; + } + + $includes = $cache_entry->includes; + + if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes)) + { + $is_stale = true; + $cache_entry = _config_invalidate_cache($params, $base_dir, $file, $cache_file); + //NOTE: let's change the mtime of the file which include other files, + // so that on tne next build it will be 'older' than its includes + // and won't trigger rebuild + if($params->touch_files_with_includes) + touch($file); + } + + if($is_stale) + ++$total_stales; + + //we want results to be returned in the same order, so + //we store entries by the file key and later retrieve array values + $result->by_path[$file] = $cache_entry; + $result->by_id[$cache_entry->id] = $cache_entry; + $result->by_alias[$cache_entry->strid] = $cache_entry; + } + } + + $result->all = array_values($result->by_path); + $result->stales = $total_stales; + $result->fast_jsons = $total_fast_jsons; + return $result; +} + +function _config_invalidate_cache(ConfigFetchParams $params, string $base_dir, string $file, string $cache_file, ?int &$parser_type = null) : ConfigCacheEntry +{ + list($conf_id, $conf_strid) = config_ensure_header($base_dir, $file); + if(!$conf_id) + throw new Exception("Bad conf id: {$conf_id}"); + + //TODO: this is a bit ugly but kinda works for now + $GLOBALS['CONFIG_GLOBALS'] = $params->globals; + $GLOBALS['CONFIG_CURRENT_FILE'] = $file; + $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id; + $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create(); + + $pres = config_parse($params->globals->base_dirs, $file); + if($pres->error !== 0) + throw new Exception("Error({$pres->error}) while parsing {$file}:\n" . $pres->error_descr); + + $parser_type = $pres->parser_type; + $parsed_arr = $pres->parsed_arr; + //TODO: these hardcoded keys below probably shouldn't be part of data? + $parsed_arr['id'] = $conf_id; + $parsed_arr['strid'] = $conf_strid; + list($class, $class_id, $normalized_data) = config_apply_class_normalization($parsed_arr); + + $cache_payload_file = config_get_cache_payload_path($params->globals, $file); + + $cache_entry = new ConfigCacheEntry(); + $cache_entry->id = $conf_id; + $cache_entry->strid = $conf_strid; + $cache_entry->cache_file = $cache_file; + $cache_entry->class = $class; + $cache_entry->class_id = $class_id; + $cache_entry->payload_file = $cache_payload_file; + //TODO: pass flag if file is actually normalized? + $cache_entry->file = normalize_path($file); + $cache_entry->includes = config_get_module_includes($pres->jsm_module); + $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS']; + //TODO: do we really need these refs? + $cache_entry->refs = config_content_match_refs($pres->normalized_jzon); + + ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry)); + ensure_write($cache_payload_file, config_msgpack_pack($normalized_data)); + + return $cache_entry; +} + diff --git a/config.inc.php b/config.inc.php index 145f8c3..467b07a 100644 --- a/config.inc.php +++ b/config.inc.php @@ -2,917 +2,56 @@ namespace taskman; use Exception; -$GLOBALS['CONFIG_BASE_DIRS'] = array(); -$GLOBALS['CONFIG_INIT_WORKER_FUNC'] = null; -$GLOBALS['CONFIG_FILTER_FN'] = null; -$GLOBALS['CONFIG_BASE_CLASS'] = '\ConfBase'; +require_once(__DIR__ . '/task.inc.php'); +require_once(__DIR__ . '/scan.inc.php'); +require_once(__DIR__ . '/cache.inc.php'); +require_once(__DIR__ . '/pack.inc.php'); +require_once(__DIR__ . '/util.inc.php'); +require_once(__DIR__ . '/msgpack.inc.php'); -task('config_worker', function(array $args) +class ConfigGlobals { - if(sizeof($args) != 3) - throw new Exception("Config worker args not set"); + public array $base_dirs = array(); + public ?string $worker_init_fn = null; + public string $base_class = '\ConfBase'; + public string $build_dir; - $in_file = $args[0]; - $out_file = $args[1]; - $err_file = $args[2]; - - try + function __construct(array $base_dirs, string $build_dir, ?string $worker_init_fn = null) { - list($job, $force, $verbose) = unserialize(ensure_read($in_file)); - $result = _config_worker_func($job, $force, $verbose); - ensure_write($out_file, serialize($result)); - } - catch(Exception $e) - { - //NOTE: explicitely catching all exceptions and writing to the error file - // since under Windows error file stream redirect may work unreliably - file_put_contents($err_file, $e->getMessage() . "\n" . $e->getTraceAsString(), FILE_APPEND); - throw $e; - } -}); - -function config_set_base_dirs(array $dirs) -{ - global $CONFIG_BASE_DIRS; - $CONFIG_BASE_DIRS = array_map(function($d) { return normalize_path($d); }, $dirs); -} - -function config_base_dirs() : array -{ - global $CONFIG_BASE_DIRS; - return $CONFIG_BASE_DIRS; -} - -function config_map_base_dir(string $file, bool $normalized = false, bool $strict = true, ?array $dirs = null) : ?string -{ - if(!is_array($dirs)) - $dirs = config_base_dirs(); - if(!$normalized) - $file = normalize_path($file); - foreach($dirs as $dir) - if(strpos($file, $dir) === 0) - return $dir; - if($strict) - throw new Exception("File '$file' is not mapped to any base dir"); - return null; -} - -function config_real_path(string $rel_path, bool $strict = true, ?array $dirs = null) : ?string -{ - if(!is_array($dirs)) - $dirs = config_base_dirs(); - foreach($dirs as $dir) - if(is_file($dir . '/' . $rel_path)) - return $dir . '/' . $rel_path; - if($strict) - throw new Exception("No file for relative path '$rel_path'"); - return null; -} - -function config_build_dir() : string -{ - global $GAME_ROOT; - return "$GAME_ROOT/build/tmp/"; -} - -function config_set_worker_init_fn(callable $fn) -{ - global $CONFIG_INIT_WORKER_FUNC; - $CONFIG_INIT_WORKER_FUNC = $fn; -} - -function config_scan_files() : array -{ - $files = scan_files_rec(config_base_dirs(), array('conf.js')); - return config_filter_files($files); -} - -function config_filter_files(array $files) : array -{ - global $CONFIG_FILTER_FN; - if($CONFIG_FILTER_FN != null) - $files = call_user_func($CONFIG_FILTER_FN, $files); - return $files; -} - -function config_set_files_filter_fn(callable $fn) -{ - global $CONFIG_FILTER_FN; - $CONFIG_FILTER_FN = $fn; -} - -function config_pack_bundle( - array $cache_entries, - bool $use_lz4 = false, - bool $use_config_refs = false, - int $binary_format = 1, - ?int $version = null, - bool $debug = true -) : string -{ - global $GAME_ROOT; - - if(is_null($version)) - $version = game_version_code(); - - $packed_data = null; - - if($binary_format == 1) - { - $packed_data = _config_pack_bundle_fmt1( - $cache_entries, - $use_lz4, - $use_config_refs, - $version - ); - } - else if($binary_format == 2) - { - $packed_data = _config_pack_bundle_fmt2( - $cache_entries, - $use_lz4, - $use_config_refs, - $version - ); - } - else - throw new Exception("Unknown binary format: $binary_format"); - - if($debug) - echo "CONF.BUNDLE: entries " . sizeof($cache_entries) . "; total " . kb($packed_data) . - "; format $binary_format; lz4 $use_lz4; refs $use_config_refs; CRC " . crc32($packed_data) . "\n"; - - return $packed_data; -} - -function _config_pack_bundle_fmt1(array $cache_entries, bool $use_lz4, bool $use_config_refs, int $version) : string -{ - $MAP = array(); - $STRIDMAP = array(); - - $payloads = array(); - $payloads_offset = 0; - foreach($cache_entries as $entry) - { - list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs); - $payload_size = strlen($payload); - $payloads[] = array($payloads_offset, $payload, $format, $payload_size); - $payloads_offset += $payload_size; + $this->base_dirs = array_map(fn($path) => normalize_path($path), $base_dirs); + $this->build_dir = $build_dir; + $this->worker_init_fn = $worker_init_fn; } - $header = array(); - foreach($cache_entries as $idx => $entry) - { - if(isset($MAP[$entry->id])) - throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'"); - $MAP[$entry->id] = $entry->strid; - - $strid_crc = crc32($entry->strid); - if(isset($STRIDMAP[$strid_crc])) - throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'"); - $STRIDMAP[$strid_crc] = $entry->strid; - - $header[] = array( - $payloads[$idx][2], - $entry->id, - crc32($entry->strid), - $entry->class_id, - $payloads[$idx][0], - $payloads[$idx][3] - ); - } - - $header_msgpack = config_msgpack_pack($header); - $payloads_bundle = ''; - foreach($payloads as $item) - $payloads_bundle .= $item[1]; - - $packed_data = - pack("C", 1) . - pack("V", $version) . - pack("V", strlen($header_msgpack)) . - $header_msgpack . - $payloads_bundle; - - return $packed_data; -} - -function _config_pack_bundle_fmt2(array $cache_entries, bool $use_lz4, bool $use_config_refs, int $version) : string -{ - $MAP = array(); - $STRIDMAP = array(); - $STRIDLIST = array(); - - $payloads = array(); - $strids = array(); - $payloads_offset = 0; - foreach($cache_entries as $entry) + function initWorker() { - list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs); - $payload_size = strlen($payload); - $payloads[] = array($payloads_offset, $payload, $format, $payload_size); - $payloads_offset += $payload_size; - - $strids_indices = array(); - $strid_parts = explode('/', ltrim($entry->strid, '@')); - foreach($strid_parts as $strid_part) + if($this->worker_init_fn !== null) { - if(!isset($STRIDMAP[$strid_part])) - { - $strid_index = count($STRIDLIST); - $STRIDLIST[] = $strid_part; - $STRIDMAP[$strid_part] = $strid_index; - $strids_indices[] = $strid_index; - } - else - $strids_indices[] = $STRIDMAP[$strid_part]; - } - $strids[] = $strids_indices; - } - - $header = array(); - foreach($cache_entries as $idx => $entry) - { - if(isset($MAP[$entry->id])) - throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'"); - $MAP[$entry->id] = $entry->strid; - - $header[] = array( - $payloads[$idx][2], //format - $entry->id, - $strids[$idx], //strid as a lookup indices - $entry->class_id, - $payloads[$idx][0], //offset - $payloads[$idx][3] //size - ); - } - - $strids_msgpack = config_msgpack_pack($STRIDLIST); - - $header_msgpack = config_msgpack_pack($header); - $payloads_bundle = ''; - foreach($payloads as $item) - $payloads_bundle .= $item[1]; - - $packed_data = - pack("C", 2) . - pack("V", $version) . - pack("V", strlen($strids_msgpack)) . - pack("V", strlen($header_msgpack)) . - $strids_msgpack . - $header_msgpack . - $payloads_bundle; - - return $packed_data; -} - -function config_unpack_bundle(string $packed_data) : array -{ - $packed_info = substr($packed_data, 0, 1); - $info = unpack('Cformat', $packed_info); - - if($info['format'] === 1) - { - return _config_unpack_bundle_fmt1($packed_data); - } - else if($info['format'] === 2) - { - return _config_unpack_bundle_fmt2($packed_data); - } - else - throw new Exception("Unknown format: {$info['format']}"); -} - -function _config_unpack_bundle_fmt1(string $packed_data) : array -{ - $packed_info = substr($packed_data, 0, 1+4+4); - - $info = unpack('Cformat/Vversion/Vheader_len', $packed_info); - - if($info['format'] !== 1) - throw new Exception("Unknown format: {$info['format']}"); - - $header_msgpack = substr($packed_data, 1+4+4, $info['header_len']); - $header = config_msgpack_unpack($header_msgpack); - - $payloads_bundle = substr($packed_data, 1+4+4+$info['header_len']); - - $entries = array(); - foreach($header as $item) - { - list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; - - $payload = substr($payloads_bundle, $offset, $size); - - $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); - } - - return $entries; -} - -function _config_unpack_bundle_fmt2(string $packed_data) : array -{ - $packed_info = substr($packed_data, 0, 1+4+4+4); - - $info = unpack('Cformat/Vversion/Vstrids_len/Vheader_len', $packed_info); - - if($info['format'] !== 2) - throw new Exception("Unknown format: {$info['format']}"); - - $strids_msgpack = substr($packed_data, 1+4+4+4, $info['strids_len']); - $strids = config_msgpack_unpack($strids_msgpack); - - $header_msgpack = substr($packed_data, 1+4+4+4+$info['strids_len'], $info['header_len']); - $header = config_msgpack_unpack($header_msgpack); - - $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']); - - $entries = array(); - foreach($header as $item) - { - list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; - - $payload = substr($payloads_bundle, $offset, $size); - - $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); - } - - return $entries; -} - -function _config_get_payload(ConfigCacheEntry $ce, bool $use_lz4, bool $use_config_refs) : array -{ - $format = ConfigCacheEntry::FMT_BINARY; - $payload = null; - - if($use_config_refs && $ce->payload_file) - { - $format = ConfigCacheEntry::FMT_FILE_REF; - $payload = $ce->payload_file; - } - else - { - $payload = $ce->payload; - if($use_lz4 && strlen($payload) > 512) - { - $format = ConfigCacheEntry::FMT_LZ4; - $payload = lz4_compress($payload, 9); + $fn = $this->worker_init_fn; + $fn(); } } - return array($format, $payload); -} - -function _config_unpack_payload(int $format, string $payload) : array -{ - $msg_packed = null; - if($format === ConfigCacheEntry::FMT_LZ4) - $msg_packed = lz4_uncompress($payload); - else if($format === ConfigCacheEntry::FMT_BINARY) - $msg_packed = $payload; - else if($format === ConfigCacheEntry::FMT_FILE_REF) - $msg_packed = ensure_read($payload); - else - throw new Exception("Bad format: $format"); - return config_msgpack_unpack($msg_packed); -} - -function config_make_standalone_ext_bundle( - array $configs, - string $file_path, - bool $use_lz4 = true, - int $binary_format = 1, - ?int $version = null -) -{ - $cache_entries = array(); - foreach($configs as $conf) + function setNormalizeBaseDirs(array $dirs) { - $payload = config_msgpack_pack($conf->export()); - - //creating fake cache entries - $entry = new ConfigCacheEntry(); - $entry->id = $conf->id; - $entry->strid = $conf->strid; - $entry->class_id = $conf->getClassId(); - $entry->class = get_class($conf); - $entry->payload = $payload; - $entry->config = $conf; - - $cache_entries[] = $entry; - } - - $packed_data = config_pack_bundle($cache_entries, $use_lz4, false, $binary_format, $version); - - ensure_write($file_path, $packed_data); -} - -function config_bench_load(string $file) -{ - $base_dir = config_map_base_dir($file); - - list($conf_id, $_) = config_ensure_header($base_dir, $file); - if(!$conf_id) - throw new Exception("Bad conf id: {$conf_id}"); - $t = microtime(true); - $parse_res = config_parse(config_base_dirs(), $file); - if($parse_res->error !== 0) - throw new Exception("Error({$parse_res->error}) while loading JSON from {$file}:\n" . $parse_res->error_descr); - echo "PARSE: " . (microtime(true) - $t) . "\n"; - $t = microtime(true); - $config = config_load_ex($base_dir, $file, $parse_res->parsed_arr, $conf_id); - echo "LOAD: " . (microtime(true) - $t) . "\n"; -} - -function config_get_tmp_build_path(string $file) : string -{ - $name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file)))); - $name = ltrim($name, "-"); - return normalize_path(config_build_dir() . "/$name"); -} - -class ConfigFetchResult -{ - public array $all = array(); - public array $by_id = array(); - public array $by_path = array(); - public array $by_alias = array(); -} - -class ConfigFetchParams -{ - public array $files = array(); - public bool $force_stale = false; - public bool $verbose = false; - public ?int $max_workers = null; - public bool $touch_files_with_includes = true; -} - -function config_fetch(ConfigFetchParams $params) : ConfigFetchResult -{ - if(!$params->files) - return new ConfigFetchResult(); - - $max_workers = $params->max_workers; - if($max_workers === null) - $max_workers = sizeof($params->files) < 20 ? 1 : 4; - - $chunk_size = (int)ceil(sizeof($params->files)/$max_workers); - $jobs = array(); - foreach(array_chunk($params->files, $chunk_size) as $idx => $chunk_files) - $jobs[] = array($idx, $chunk_files); - - $results_by_job = null; - - $serial = $max_workers == 1; - - if(!$serial) - { - $results_by_job = _config_worker_run_procs($jobs, $params->force_stale, $params->verbose); - //in case of any result error try serial processing - if(array_search(false, $results_by_job, true/*strict*/) !== false) - { - if($params->verbose) - echo "Result error detected, trying serial processing...\n"; - $serial = true; - } - } - - if($serial) - { - $results_by_job = array(); - foreach($jobs as $job) - $results_by_job[] = _config_worker_func($job, $params->force_stale, $params->verbose); - } - - list($result, $total_stales) = _config_fetch_cache_ex($results_by_job, $params->touch_files_with_includes); - - if($params->verbose) - echo "Miss/All: $total_stales/" . sizeof($result->all) . "\n"; - - return $result; -} - -//TODO: deprecated function, kept for BC -function config_fetch_ex( - array $files, - bool $force_stale = false, - bool $verbose = false, - //TODO: removing this will break BC, keepint it for a while - $not_used = null, - ?int $max_workers = null -) : ConfigFetchResult -{ - $params = new ConfigFetchParams(); - $params->files = $files; - $params->force_stale = $force_stale; - $params->verbose = $verbose; - $params->max_workers = $max_workers; - - return config_fetch($params); -} - -function _config_fetch_cache_ex(array $results_by_job, bool $touch_files_with_includes = true) : array -{ - $result = new ConfigFetchResult(); - $total_stales = 0; - - foreach($results_by_job as $results) - { - foreach($results as $file => $item) - { - list($cache_file, $is_stale) = $item; - - $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file)); - //NOTE: handling any cache corruption errors - if($cache_entry === null) - { - $is_stale = true; - $cache_entry = _config_invalidate_cache($file, $cache_file); - } - - $includes = $cache_entry->includes; - - if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes)) - { - $is_stale = true; - $cache_entry = _config_invalidate_cache($file, $cache_file); - //NOTE: let's change the mtime of the file which include other files, - // so that on tne next build it will be 'older' than its includes - // and won't trigger rebuild - if($touch_files_with_includes) - touch($file); - } - - if($is_stale) - ++$total_stales; - - //we want results to be returned in the same order, so - //we store entries by the file key and later retrieve array values - $result->by_path[$file] = $cache_entry; - $result->by_id[$cache_entry->id] = $cache_entry; - $result->by_alias[$cache_entry->strid] = $cache_entry; - } - } - - $result->all = array_values($result->by_path); - return array($result, $total_stales); -} - -function _config_worker_run_procs(array $jobs, bool $force, bool $verbose) : array -{ - $worker_args = array(); - foreach($jobs as $idx => $job) - $worker_args[] = array($job, $force, $verbose); - - return run_background_gamectl_workers('config_worker', $worker_args); -} - -function _config_worker_func(array $job, bool $force, bool $verbose) : array -{ - global $CONFIG_INIT_WORKER_FUNC; - if(is_callable($CONFIG_INIT_WORKER_FUNC)) - $CONFIG_INIT_WORKER_FUNC(); - - $start_time = microtime(true); - list($idx, $files) = $job; - if($verbose) - echo "Worker $idx (" . sizeof($files) . ") started\n"; - $results = array(); - foreach($files as $file_idx => $file) - { - if($verbose && $file_idx > 0 && ($file_idx % 500) == 0) - echo "Worker $idx progress: " . round($file_idx / sizeof($files) * 100) . "% ...\n"; - - $cache_file = config_get_cache_path($file); - - $is_stale = true; - if(!$force) - $is_stale = need_to_regen($cache_file, array($file)); - - if($is_stale) - _config_invalidate_cache($file, $cache_file); - - $results[$file] = array($cache_file, $is_stale); - } - if($verbose) - echo "Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)\n"; - return $results; -} - -function _config_invalidate_cache(string $file, string $cache_file) : ConfigCacheEntry -{ - $cache_payload_file = config_get_cache_payload_path($file); - - //TODO: pass it from above? - $base_dir = config_map_base_dir($file); - - list($conf_id, $_) = config_ensure_header($base_dir, $file); - if(!$conf_id) - throw new Exception("Bad conf id: {$conf_id}"); - - $GLOBALS['CONFIG_CURRENT_FILE'] = $file; - $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id; - $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create(); - - $pres = config_parse(config_base_dirs(), $file); - if($pres->error !== 0) - throw new Exception("Error({$pres->error}) while loading JSON in {$file}:\n" . $pres->error_descr); - - $includes = config_get_module_includes($pres->jsm_module); - - $config = config_load_ex($base_dir, $file, $pres->parsed_arr, $conf_id); - $payload_data = config_msgpack_pack($config->export()); - - $cache_entry = new ConfigCacheEntry(); - $cache_entry->id = $config->id; - $cache_entry->config = $config; - $cache_entry->strid = $config->strid; - $cache_entry->cache_file = $cache_file; - $cache_entry->class = get_class($config); - $cache_entry->class_id = $config->getClassId(); - $cache_entry->payload_file = $cache_payload_file; - $cache_entry->file = normalize_path($file); - $cache_entry->includes = $includes; - $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS']; - $cache_entry->refs = config_extract_refs($pres->normalized_jzon); - - ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry)); - ensure_write($cache_payload_file, $payload_data); - - return $cache_entry; -} - -function config_find_by_alias(ConfigFetchResult $cache_entries, string $strid) : ConfigCacheEntry -{ - if(array_key_exists($strid, $cache_entries->by_alias)) - return $cache_entries->by_alias[$strid]; - throw new Exception("Failed to find config by alias '$strid'!"); -} - -function config_find_by_id(ConfigFetchResult $cache_entries, int $id) : ConfigCacheEntry -{ - if(array_key_exists($id, $cache_entries->by_id)) - return $cache_entries->by_id[$id]; - throw new Exception("Failed to find config by id '$id'!"); -} - -function config_find_by_path(ConfigFetchResult $cache_entries, string $path) : ConfigCacheEntry -{ - if(array_key_exists($path, $cache_entries->by_path)) - return $cache_entries->by_path[$path]; - throw new Exception("Failed to find config by path '$path'!"); -} - -function config_fetch_by_path(string $path, bool $force_stale = false) : ConfigCacheEntry -{ - $ces = config_fetch_ex(array($path), $force_stale); - if(empty($ces->all)) - throw new Exception("Config not found at path '$path'"); - return $ces->all[0]; -} - -function config_fetch_all(bool $force_stale = false) : ConfigFetchResult -{ - return config_fetch_ex(config_scan_files(), $force_stale); -} - -class ConfigCacheEntryExtras extends \stdClass -{ - private static string $klass = ConfigCacheEntryExtras::class; - - static function init(string $project_specific_klass) - { - self::$klass = $project_specific_klass; - } - - static function create() : object - { - return new self::$klass(); - } - - function export() : array - { - $as_array = get_object_vars($this); - return $as_array; - } - - function import(array $as_array) - { - foreach($as_array as $field_name => $field_value) - $this->$field_name = $field_value; + $this->base_dirs = array_map(function($d) { return normalize_path($d); }, $dirs); } } -/** - * @property object $config - * @property string $payload - */ -class ConfigCacheEntry +function config_get_cache_path(ConfigGlobals $globals, string $file) : string { - const FMT_BINARY = 0; - const FMT_LZ4 = 1; - const FMT_FILE_REF = 2; - - public string $class; - public int $class_id; - public int $id; - public string $strid; - public string $cache_file; - //NOTE: actual payload is stored in a separate file for faster incremental retrievals - public string $payload_file; - public string $file; - public array $includes = array(); - public array $refs = array(); - public object $extras; - - public $_config; - public $_payload; - - static function serialize(ConfigCacheEntry $ce) : string - { - $d = $ce->export(); - return serialize($d); - } - - static function unserialize(string $str) : ?ConfigCacheEntry - { - $d = @unserialize($str); - if(!is_array($d)) - return null; - $ce = new ConfigCacheEntry(); - $ce->import($d); - return $ce; - } - - function __construct() - { - $this->extras = ConfigCacheEntryExtras::create(); - } - - function __get(string $name) - { - if($name === "config") - { - if($this->_config === null) - { - $klass = $this->class; - $data = config_msgpack_unpack($this->payload); - $this->_config = new $klass($data); - } - return $this->_config; - } - else if($name === "payload") - { - if($this->_payload !== null) - return $this->_payload; - //if payload not set directly not storing it - return ensure_read($this->payload_file); - } - else - throw new Exception("No such property '$name'"); - } - - function __set(string $name, $v) - { - if($name === "config") - $this->_config = $v; - else if($name === "payload") - $this->_payload = $v; - else - throw new Exception("No such property '$name'"); - } - - function export() : array - { - $d = array(); - $d[] = $this->class; - $d[] = $this->class_id; - $d[] = $this->id; - $d[] = $this->strid; - $d[] = $this->cache_file; - $d[] = $this->payload_file; - $d[] = $this->file; - $d[] = $this->includes; - $d[] = $this->refs; - $d[] = $this->extras->export(); - return $d; - } - - function import(array $d) - { - $extras = array(); - - list( - $this->class, - $this->class_id, - $this->id, - $this->strid, - $this->cache_file, - $this->payload_file, - $this->file, - $this->includes, - $this->refs, - $extras, - ) = $d; - - $this->extras->import($extras); - } -} - -function config_get_cache_path(string $file) : string -{ - return config_get_tmp_build_path($file . '.cacheb'); + return config_get_tmp_build_path($globals, $file . '.cacheb'); } -function config_get_cache_payload_path(string $file) : string +function config_get_cache_payload_path(ConfigGlobals $globals, string $file) : string { - return config_get_tmp_build_path($file . '.pdata'); + return config_get_tmp_build_path($globals, $file . '.pdata'); } -function config_extract_refs(string $src, bool $as_map = true) : array +function config_load(ConfigGlobals $globals, string $conf_path, ?string $conf_dir = null) : object { - $refs = array(); - if(preg_match_all('~"(@[^"]+)"~', $src, $matches)) - { - foreach($matches[1] as $ref) - { - if(!isset($refs[$ref])) - $refs[$ref] = 1; - ++$refs[$ref]; - } - } - return $as_map ? $refs : array_keys($refs); -} + $conf_dir ??= config_map_base_dir($globals->base_dirs, $conf_path); -function config_get_header(string $file, ?int &$conf_id, ?string &$strid) : bool -{ - $h = fopen($file, "r"); - $line = fgets($h, 256); - fclose($h); - - if(preg_match('~\{\s*/\*\s*proto_id\s*=\s*(\d+)\s*;\s*alias\s*=\s*([^\s]+)~', $line, $matches)) - { - $conf_id = (int)$matches[1]; - $strid = $matches[2]; - return true; - } - else - return false; -} - -function config_set_header(string $contents, int $conf_id, string $strid, bool &$is_success) : string -{ - $contents = preg_replace('~\s*\{~', "{ /* proto_id = {$conf_id} ; alias = {$strid} */", $contents, 1/*limit*/, $count); - $is_success = $count == 1; - return $contents; -} - -function config_extract_header(string $contents) : string -{ - return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents); -} - -function config_ensure_header(string $conf_dir, string $file, bool $force = false) : array -{ - if(config_get_header($file, $curr_proto_id, $curr_alias)) - { - $alias = config_file2strid($conf_dir, $file); - if($force) - $curr_proto_id = config_file2id($conf_dir, $file); - - //NOTE: keeping current proto id intact if not forced - if($force || $curr_alias !== $alias) - { - $lines = file($file); - //TODO: why not using config_set_header(..) ? - $lines[0] = preg_replace('~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{ /* proto_id = {$curr_proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count); - if($count != 1) - throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); - ensure_write_if_differs($file, join("", $lines)); - } - - return array($curr_proto_id, $alias); - } - else - { - $conf_id = config_file2id($conf_dir, $file); - $alias = config_file2strid($conf_dir, $file); - - $lines = file($file); - - $is_success = false; - $lines[0] = config_set_header($lines[0], $conf_id, $alias, $is_success); - if(!$is_success) - throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); - ensure_write($file, join("", $lines)); - - return array($conf_id, $alias); - } -} - -function config_load(string $conf_dir, string $conf_path) : object -{ list($conf_id, $_) = config_ensure_header($conf_dir, $conf_path); if(!$conf_id) throw new Exception("Bad conf id: {$conf_id}"); @@ -921,97 +60,30 @@ function config_load(string $conf_dir, string $conf_path) : object if($pres->error !== 0) throw new Exception("Error({$pres->error}) while loading JSON from {$conf_path}:\n" . $pres->error_descr); - return config_load_ex($conf_dir, $conf_path, $pres->parsed_arr, $conf_id); + return config_load_from_kv_array($globals, $conf_dir, $conf_path, $pres->parsed_arr, $conf_id); } -function conf2json($conf, int $json_flags = 0) -{ - $arr = $conf->export(true); - $arr['class'] = get_class($conf); - unset($arr['id']); - unset($arr['strid']); - $json = json_encode($arr, $json_flags); - $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json); - return $json; -} - -class ConfigParseResult -{ - public int $error = 0; - public string $error_descr; - public string $normalized_jzon = ''; - public array $parsed_arr; - public $jsm_module; -} - -function config_parse(array $base_dirs, string $file) : ConfigParseResult -{ - $res = new ConfigParseResult(); - - $normalized_jzon = ''; - try - { - $jsm = new \JSM($base_dirs, $file); - list($normalized_jzon, $jsm_module) = $jsm->process(); - $res->normalized_jzon = $normalized_jzon; - $res->jsm_module = $jsm_module; - } - catch(Exception $e) - { - $res->error = 1; - $res->error_descr = "File '$file':\n" . $e->getMessage() . "\n" . $e->getTraceAsString(); - return $res; - } - - $decode_res = config_check_and_decode_jzon($normalized_jzon); - - if($decode_res[0] === 0) - { - $res->parsed_arr = $decode_res[2]; - } - else - { - $res->error = $decode_res[0]; - $res->error_descr = "File '$file':\n" . $decode_res[1]; - } - - return $res; -} - -function config_check_and_decode_jzon(string $json) : array -{ - try - { - $arr = \jzon_parse($json); - return array(0, "", $arr); - } - catch(Exception $e) - { - return array(1, $e->getMessage(), array()); - } -} - -function config_load_ex(string $conf_dir, string $file, array $arr, ?int $id = null) : object +function config_load_from_kv_array( + ConfigGlobals $globals, string $conf_dir, string $file, + array $arr, ?int $id = null) : object { if(!isset($arr['class']) || !isset($arr['class'][0])) throw new Exception("Class is not set in file '$file'."); - $klass = $arr['class']; - unset($arr['class']); - - if($id === null) - $id = config_file2id($conf_dir, $file); + $id ??= config_file2id($conf_dir, $file); $cnf = null; try { + list($klass, $class_id, $norm_arr) = config_apply_class_normalization($arr); + if(!class_exists($klass)) throw new Exception("No such class '$klass'"); $cnf = new $klass; - if(!is_a($cnf, $GLOBALS['CONFIG_BASE_CLASS'])) - throw new Exception("'$klass' is not subclass of '".ltrim($GLOBALS['CONFIG_BASE_CLASS'], '\\')."'"); + if(!is_a($cnf, $globals->base_class)) + throw new Exception("'$klass' is not subclass of '".ltrim($globals->base_class, '\\')."'"); - $cnf->import($arr, true); + $cnf->import($norm_arr); } catch(Exception $e) { @@ -1024,128 +96,20 @@ function config_load_ex(string $conf_dir, string $file, array $arr, ?int $id = n return $cnf; } -function config_is_file(string $filename) : bool +function config_bench_load(ConfigGlobals $globals, string $file) { - return (strrpos($filename, '.conf.js') === (strlen($filename) - 8)); + $base_dir = config_map_base_dir($globals->base_dirs, $file); + + list($conf_id, $_) = config_ensure_header($base_dir, $file); + if(!$conf_id) + throw new Exception("Bad conf id: {$conf_id}"); + $t = microtime(true); + $parse_res = config_parse($globals->base_dirs, $file); + if($parse_res->error !== 0) + throw new Exception("Error({$parse_res->error}) while loading JSON from {$file}:\n" . $parse_res->error_descr); + echo "[CFG] Parse: " . (microtime(true) - $t) . "\n"; + $t = microtime(true); + $config = config_load_from_kv_array($globals, $base_dir, $file, $parse_res->parsed_arr, $conf_id); + echo "[CFG] Load: " . (microtime(true) - $t) . "\n"; } -function config_file2id(string $conf_dir, string $filename) : int -{ - $nfilename = config_make_path($conf_dir, $filename); - return config_crc28($nfilename); -} - -function config_file2strid(string $conf_dir, string $filename) : string -{ - $strid = config_make_path($conf_dir, $filename); - $fname_idx = strrpos($strid, "/"); - $fname_idx = strpos($strid, ".", $fname_idx); - $strid = substr($strid, 0, $fname_idx); - return "@".$strid; -} - -function config_make_path(string $conf_dir, string $path) : string -{ - return ltrim(str_replace(normalize_path($conf_dir, true/*nix*/), - '', - normalize_path($path, true/*nix*/)), - '/'); -} - -function config_crc28(string $what) : int -{ - return crc32($what) & 0xFFFFFFF; -} - -function config_get_module_includes(\JSM_Module $cm) : array -{ - $includes = array(); - foreach($cm->getIncludes() as $include => $_) - { - //maybe we should take .php includes into account as well? - if(config_str_ends_with($include, ".php")) - continue; - $includes[] = $include; - } - return $includes; -} - -function config_includes_map_find_text_origin(array $map, string $file, string $text) : array -{ - if(!isset($map[$file])) - return array($file); - - $tpls = $map[$file]; - $tpls[] = $file; - $res = array(); - foreach($tpls as $tpl) - { - $content = ensure_read($tpl); - $content = i18n_decode_string($content); - if(strpos($content, $text) !== FALSE) - $res[] = $tpl; - } - - return $res; -} - -function config_str_ends_with(string $haystack, string $needle) : bool -{ - // search forward starting from end minus needle length characters - return $needle === "" || (($temp = strlen($haystack) - strlen($needle)) >= 0 && strpos($haystack, $needle, $temp) !== false); -} - -function config_walk_fields($proto, callable $callback) -{ - if(!method_exists($proto, 'CLASS_FIELDS_PROPS')) - return; - - foreach($proto->CLASS_FIELDS_PROPS() as $field => $tokens) - { - $value = $proto->$field; - $callback($proto, $field, $value, $tokens); - - if(is_object($value)) - config_walk_fields($value, $callback); - else if(is_array($value)) - { - foreach($value as $num => $item) - { - if(is_object($item)) - config_walk_fields($item, $callback); - } - } - } -} - -if(function_exists('msgpack_pack')) -{ - function config_msgpack_pack(array $data) : string - { - $prev = ini_set('msgpack.use_str8_serialization', '0'); - $res = msgpack_pack($data); - if($prev != '0') - ini_set('msgpack.use_str8_serialization', $prev); - return $res; - } - - function config_msgpack_unpack(string $data) : array - { - return msgpack_unpack($data); - } -} -else -{ - function config_msgpack_pack(array $data) : string - { - include_once(__DIR__ . '/msgpack/msgpack_custom.inc.php'); - - $packer = new \MessagePackCustom(); - return $packer->pack($data); - } - - function config_msgpack_unpack(string $data) : array - { - return \MessagePack\MessagePack::unpack($data); - } -} diff --git a/msgpack.inc.php b/msgpack.inc.php new file mode 100644 index 0000000..6cf4b0b --- /dev/null +++ b/msgpack.inc.php @@ -0,0 +1,36 @@ +pack($data); + } + + function config_msgpack_unpack(string $data) : array + { + return \MessagePack\MessagePack::unpack($data); + } +} + diff --git a/pack.inc.php b/pack.inc.php new file mode 100644 index 0000000..f494abe --- /dev/null +++ b/pack.inc.php @@ -0,0 +1,343 @@ +cache_entries = $cache_entries; + $this->use_lz4 = $use_lz4; + $this->use_config_refs = $use_config_refs; + $this->binary_format = $binary_format; + $this->version = $version; + $this->debug = $debug; + } +} + +function config_pack_bundle(ConfigPackParams $params) : string +{ + $t = microtime(true); + + $packed_data = null; + + if($params->binary_format == 1) + { + $packed_data = _config_pack_bundle_fmt1( + $params->cache_entries, + $params->use_lz4, + $params->use_config_refs, + $params->version + ); + } + else if($params->binary_format == 2) + { + $packed_data = _config_pack_bundle_fmt2( + $params->cache_entries, + $params->use_lz4, + $params->use_config_refs, + $params->version + ); + } + else + throw new Exception("Unknown binary format: {$params->binary_format}"); + + if($params->debug) + echo "[CFG] Packed entries: " . sizeof($params->cache_entries) . ", total: " . + kb($packed_data) . ", format: {$params->binary_format}, lz4: " . + var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . + ", CRC: " . crc32($packed_data) . + ", " . round(microtime(true) - $t,2) . " sec.\n"; + + return $packed_data; +} + +function _config_pack_bundle_fmt1( + array $cache_entries, + bool $use_lz4, + bool $use_config_refs, + int $version) : string +{ + $MAP = array(); + $STRIDMAP = array(); + + $payloads = array(); + $payloads_offset = 0; + foreach($cache_entries as $entry) + { + list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs); + $payload_size = strlen($payload); + $payloads[] = array($payloads_offset, $payload, $format, $payload_size); + $payloads_offset += $payload_size; + } + + $header = array(); + foreach($cache_entries as $idx => $entry) + { + if(isset($MAP[$entry->id])) + throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'"); + $MAP[$entry->id] = $entry->strid; + + $strid_crc = crc32($entry->strid); + if(isset($STRIDMAP[$strid_crc])) + throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'"); + $STRIDMAP[$strid_crc] = $entry->strid; + + $header[] = array( + $payloads[$idx][2], + $entry->id, + crc32($entry->strid), + $entry->class_id, + $payloads[$idx][0], + $payloads[$idx][3] + ); + } + + $header_msgpack = config_msgpack_pack($header); + $payloads_bundle = ''; + foreach($payloads as $item) + $payloads_bundle .= $item[1]; + + $packed_data = + pack("C", 1) . + pack("V", $version) . + pack("V", strlen($header_msgpack)) . + $header_msgpack . + $payloads_bundle; + + return $packed_data; +} + +function _config_pack_bundle_fmt2( + array $cache_entries, + bool $use_lz4, + bool $use_config_refs, + int $version) : string +{ + $MAP = array(); + $STRIDMAP = array(); + $STRIDLIST = array(); + + $payloads = array(); + $strids = array(); + $payloads_offset = 0; + foreach($cache_entries as $entry) + { + list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs); + $payload_size = strlen($payload); + $payloads[] = array($payloads_offset, $payload, $format, $payload_size); + $payloads_offset += $payload_size; + + $strids_indices = array(); + $strid_parts = explode('/', ltrim($entry->strid, '@')); + foreach($strid_parts as $strid_part) + { + if(!isset($STRIDMAP[$strid_part])) + { + $strid_index = count($STRIDLIST); + $STRIDLIST[] = $strid_part; + $STRIDMAP[$strid_part] = $strid_index; + $strids_indices[] = $strid_index; + } + else + $strids_indices[] = $STRIDMAP[$strid_part]; + } + $strids[] = $strids_indices; + } + + $header = array(); + foreach($cache_entries as $idx => $entry) + { + if(isset($MAP[$entry->id])) + throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'"); + $MAP[$entry->id] = $entry->strid; + + $header[] = array( + $payloads[$idx][2], //format + $entry->id, + $strids[$idx], //strid as a lookup indices + $entry->class_id, + $payloads[$idx][0], //offset + $payloads[$idx][3] //size + ); + } + + $strids_msgpack = config_msgpack_pack($STRIDLIST); + + $header_msgpack = config_msgpack_pack($header); + $payloads_bundle = ''; + foreach($payloads as $item) + $payloads_bundle .= $item[1]; + + $packed_data = + pack("C", 2) . + pack("V", $version) . + pack("V", strlen($strids_msgpack)) . + pack("V", strlen($header_msgpack)) . + $strids_msgpack . + $header_msgpack . + $payloads_bundle; + + return $packed_data; +} + +//format: [[class_id, [data]], ...[class_id, [data]]] +function config_unpack_bundle(string $packed_data) : array +{ + $packed_info = substr($packed_data, 0, 1); + $info = unpack('Cformat', $packed_info); + + if($info['format'] === 1) + { + return _config_unpack_bundle_fmt1($packed_data); + } + else if($info['format'] === 2) + { + return _config_unpack_bundle_fmt2($packed_data); + } + else + throw new Exception("Unknown format: {$info['format']}"); +} + +function _config_unpack_bundle_fmt1(string $packed_data) : array +{ + $packed_info = substr($packed_data, 0, 1+4+4); + + $info = unpack('Cformat/Vversion/Vheader_len', $packed_info); + + if($info['format'] !== 1) + throw new Exception("Unknown format: {$info['format']}"); + + $header_msgpack = substr($packed_data, 1+4+4, $info['header_len']); + $header = config_msgpack_unpack($header_msgpack); + + $payloads_bundle = substr($packed_data, 1+4+4+$info['header_len']); + + $entries = array(); + foreach($header as $item) + { + list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; + + $payload = substr($payloads_bundle, $offset, $size); + + $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); + } + + return $entries; +} + +function _config_unpack_bundle_fmt2(string $packed_data) : array +{ + $packed_info = substr($packed_data, 0, 1+4+4+4); + + $info = unpack('Cformat/Vversion/Vstrids_len/Vheader_len', $packed_info); + + if($info['format'] !== 2) + throw new Exception("Unknown format: {$info['format']}"); + + $strids_msgpack = substr($packed_data, 1+4+4+4, $info['strids_len']); + $strids = config_msgpack_unpack($strids_msgpack); + + $header_msgpack = substr($packed_data, 1+4+4+4+$info['strids_len'], $info['header_len']); + $header = config_msgpack_unpack($header_msgpack); + + $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']); + + $entries = array(); + foreach($header as $item) + { + list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; + + $payload = substr($payloads_bundle, $offset, $size); + + $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); + } + + return $entries; +} + +//format: [format_id, payload_data] +function _config_get_payload(ConfigCacheEntry $ce, bool $use_lz4, bool $use_config_refs) : array +{ + $format = ConfigCacheEntry::FMT_BINARY; + $payload = null; + + if($use_config_refs && $ce->payload_file) + { + $format = ConfigCacheEntry::FMT_FILE_REF; + $payload = $ce->payload_file; + } + else + { + $payload = $ce->payload; + if($use_lz4 && strlen($payload) > 512) + { + $format = ConfigCacheEntry::FMT_LZ4; + $payload = lz4_compress($payload, 9); + } + } + + return array($format, $payload); +} + +function _config_unpack_payload(int $format, string $payload) : array +{ + $msg_packed = null; + if($format === ConfigCacheEntry::FMT_LZ4) + $msg_packed = lz4_uncompress($payload); + else if($format === ConfigCacheEntry::FMT_BINARY) + $msg_packed = $payload; + else if($format === ConfigCacheEntry::FMT_FILE_REF) + $msg_packed = ensure_read($payload); + else + throw new Exception("Bad format: $format"); + return config_msgpack_unpack($msg_packed); +} + +function config_pack_and_write_bundle( + /*var ConfBase[]*/ + array $configs, + string $file_path, + bool $use_lz4 = true, + int $binary_format = 1, + ?int $version = null +) +{ + $cache_entries = array(); + foreach($configs as $conf) + { + $payload = config_msgpack_pack($conf->export()); + + //creating fake cache entries + $entry = new ConfigCacheEntry(); + $entry->id = $conf->id; + $entry->strid = $conf->strid; + $entry->class_id = $conf->getClassId(); + $entry->class = get_class($conf); + $entry->payload = $payload; + $entry->config = $conf; + + $cache_entries[] = $entry; + } + + $packed_data = config_pack_bundle( + new ConfigPackParams( + cache_entries: $cache_entries, + use_lz4: $use_lz4, + binary_format: $binary_format, + version: $version + ) + ); + + ensure_write($file_path, $packed_data); +} + diff --git a/parse.inc.php b/parse.inc.php new file mode 100644 index 0000000..8d539c5 --- /dev/null +++ b/parse.inc.php @@ -0,0 +1,172 @@ +process(); + $res->normalized_jzon = $normalized_jzon; + $res->jsm_module = $jsm_module; + } + catch(Exception $e) + { + $res->error = 1; + $res->error_descr = "File '$file':\n" . $e->getMessage() . "\n" . $e->getTraceAsString(); + return $res; + } + + $decode_res = config_check_and_decode_jzon($normalized_jzon); + + if($decode_res[0] === 0) + { + $res->parsed_arr = $decode_res[2]; + $res->parser_type = $decode_res[3]; + } + else + { + $res->error = $decode_res[0]; + $res->error_descr = "File '$file':\n" . $decode_res[1]; + } + + return $res; +} + +function config_check_and_decode_jzon(string $json) : array +{ + try + { + $arr = \jzon_parse($json, $parser); + return array(0, "", $arr, $parser); + } + catch(Exception $e) + { + return array(1, $e->getMessage(), array(), 0); + } +} + +function config_get_header(string $file, ?int &$conf_id, ?string &$strid, bool $use_cache = true) : bool +{ + static $cache = array(); + if($use_cache && isset($cache[$file])) + { + list($conf_id, $strid) = $cache[$file]; + return true; + } + + $h = fopen($file, "r"); + $line = fgets($h, 256); + fclose($h); + + //TODO: rewrite it without regex usage + if(preg_match('~\{\s*/\*\s*proto_id\s*=\s*(\d+)\s*;\s*alias\s*=\s*([^\s]+)~', $line, $matches)) + { + $conf_id = (int)$matches[1]; + $strid = $matches[2]; + if($use_cache) + $cache[$file] = array($conf_id, $strid); + return true; + } + else + return false; +} + +function config_set_header(string $contents, int $conf_id, string $strid, bool &$is_success) : string +{ + $contents = preg_replace('~\s*\{~', "{ /* proto_id = {$conf_id} ; alias = {$strid} */", $contents, 1/*limit*/, $count); + $is_success = $count == 1; + return $contents; +} + +function config_replace_header(string $contents) : string +{ + return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents); +} + +//returns [proto_id, alias] +function config_ensure_header(string $conf_dir, string $file, bool $force = false) : array +{ + if(config_get_header($file, $curr_proto_id, $curr_alias, !$force)) + { + //TODO: add 'normalized' argument when we are sure both path arguments are normalized + $alias = config_file2strid($conf_dir, $file); + if($force) + $curr_proto_id = config_file2id($conf_dir, $file); + + //NOTE: keeping current proto id intact if not forced + if($force || $curr_alias !== $alias) + { + $lines = file($file); + //TODO: why not using config_set_header(..) ? + $lines[0] = preg_replace( + '~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{ /* proto_id = {$curr_proto_id} ; alias = {$alias} */", + $lines[0], + 1/*limit*/, + $count); + if($count != 1) + throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); + ensure_write_if_differs($file, join("", $lines)); + } + + return array($curr_proto_id, $alias); + } + else + { + $conf_id = config_file2id($conf_dir, $file); + $alias = config_file2strid($conf_dir, $file); + + $lines = file($file); + + $is_success = false; + $lines[0] = config_set_header($lines[0], $conf_id, $alias, $is_success); + if(!$is_success) + throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); + ensure_write($file, join("", $lines)); + + return array($conf_id, $alias); + } +} + +function config_apply_class_normalization(array $kv_data) : array +{ + $class = $kv_data['class']; + $norm_data = array(); + call_user_func_array([$class, 'normalize'], array(&$kv_data, &$norm_data)); + return array($class, $class::CLASS_ID, $norm_data); +} + +//returns array +function config_content_match_refs(string $src) : array +{ + $refs = array(); + + if(strpos($src, '@') === false) + return $refs; + + if(preg_match_all('~"(@[^"]+)"~', $src, $matches)) + { + foreach($matches[1] as $ref) + { + if(!isset($refs[$ref])) + $refs[$ref] = 1; + ++$refs[$ref]; + } + } + return $refs; +} + diff --git a/scan.inc.php b/scan.inc.php new file mode 100644 index 0000000..458da2e --- /dev/null +++ b/scan.inc.php @@ -0,0 +1,94 @@ +*/ + public array $base_dir2files = array(); + + function isEmpty() : bool + { + return empty($this->base_dir2files); + } + + function count() : int + { + $total = 0; + foreach($this->base_dir2files as $base_dir => $files) + $total += count($files); + return $total; + } + + function apply(callable $fn) + { + foreach($this->base_dir2files as $base_dir => $files) + $this->base_dir2files[$base_dir] = $fn($base_dir, $files); + } + + function filter(callable $filter) + { + foreach($this->base_dir2files as $base_dir => $files) + $this->base_dir2files[$base_dir] = array_filter($files, $filter); + } + + function forEachFile(callable $fn) + { + foreach($this->base_dir2files as $base_dir => $files) + { + foreach($files as $file) + $fn($base_dir, $file); + } + } + + function add(string $base_dir, string $file) + { + if(!isset($this->base_dir2files[$base_dir])) + $this->base_dir2files[$base_dir] = array(); + + $this->base_dir2files[$base_dir][] = $file; + } + + //returns [[base_dir, file1], [base_dir, file2], ...] + function getFlatArray() : array + { + $flat = []; + foreach($this->base_dir2files as $base_dir => $files) + { + foreach($files as $file) + $flat[] = [$base_dir, $file]; + } + return $flat; + } + + function getAllFiles() : array + { + $all_files = []; + foreach($this->base_dir2files as $base_dir => $files) + $all_files = array_merge($all_files, $files); + + return $all_files; + } +} + +function config_scan_files( + array $base_dirs, + string $ext_filter = '.conf.js', + bool $verbose = false + ) : ConfigScanResult +{ + $t = microtime(true); + + $result = new ConfigScanResult(); + + foreach($base_dirs as $base_dir) + { + $result->base_dir2files[$base_dir] = + scan_files_rec(array($base_dir), array($ext_filter)); + } + + if($verbose) + echo "[CFG] File scan: {$result->count()}, done " . round(microtime(true) - $t,2) . " sec.\n"; + + return $result; +} diff --git a/task.inc.php b/task.inc.php new file mode 100644 index 0000000..0c5a176 --- /dev/null +++ b/task.inc.php @@ -0,0 +1,94 @@ +getMessage() . "\n" . $e->getTraceAsString(), FILE_APPEND); + throw $e; + } +}); + +function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $serial) : array +{ + if($serial) + { + $results_by_job = array(); + foreach($jobs as $job) + $results_by_job[] = _config_worker_func($params, $job); + return $results_by_job; + } + else + { + $worker_args = array(); + foreach($jobs as $idx => $job) + $worker_args[] = array($params, $job); + + return run_background_gamectl_workers('config_worker', $worker_args); + } +} + +//returns [[base_dir, file, cache_file, was_stale], ...] +function _config_worker_func(ConfigFetchParams $params, array $job) : array +{ + $start_time = microtime(true); + + $params->globals->initWorker(); + + list($idx, $start_time, $chunk) = $job; + if($params->verbose) + echo "[CFG] Worker $idx (" . sizeof($chunk) . ") started (".round(microtime(true)-$start_time, 2)." sec)\n"; + + $fast_parser_num = 0; + $results = array(); + foreach($chunk as $file_idx => $chunk_data) + { + try + { + list($base_dir, $file) = $chunk_data; + + if($params->verbose && $file_idx > 0 && ($file_idx % 500) == 0) + echo "[CFG] Worker $idx progress: " . round($file_idx / sizeof($chunk) * 100) . "% ...\n"; + + $cache_file = config_get_cache_path($params->globals, $file); + + $parser_type = null; + + $is_stale = true; + if(!$params->force_stale) + $is_stale = need_to_regen($cache_file, array($file)); + + if($is_stale) + _config_invalidate_cache($params, $base_dir, $file, $cache_file, $parser_type); + + $results[] = array($base_dir, $file, $cache_file, $is_stale, $parser_type); + } + catch(Exception $e) + { + throw new Exception("Error in file '$file': " . $e->getMessage()); + } + } + + if($params->verbose) + echo "[CFG] Worker $idx done (".round(microtime(true)-$start_time, 2)." sec)\n"; + + return $results; +} + diff --git a/util.inc.php b/util.inc.php new file mode 100644 index 0000000..0eac5d1 --- /dev/null +++ b/util.inc.php @@ -0,0 +1,136 @@ +build_dir . "/$name"); +} + +function config_map_base_dir( + array $base_dirs, string $file, + bool $normalized = false, bool $strict = true + ) : ?string +{ + if(!$normalized) + $file = normalize_path($file); + foreach($base_dirs as $dir) + if(strpos($file, $dir) === 0) + return $dir; + if($strict) + throw new Exception("File '$file' is not mapped to any base dir"); + return null; +} + +function config_real_path(array $base_dirs, string $rel_path, bool $strict = true) : ?string +{ + foreach($base_dirs as $dir) + if(is_file($dir . '/' . $rel_path)) + return $dir . '/' . $rel_path; + if($strict) + throw new Exception("No file for relative path '$rel_path'"); + return null; +} + +function config_is_file(string $filename) : bool +{ + return (strrpos($filename, '.conf.js') === (strlen($filename) - 8)); +} + +function config_file2id(string $conf_dir, string $filename) : int +{ + $nfilename = config_make_path($conf_dir, $filename); + return config_crc28($nfilename); +} + +function config_file2strid(string $conf_dir, string $filename) : string +{ + $strid = config_make_path($conf_dir, $filename); + $fname_idx = strrpos($strid, "/"); + $fname_idx = strpos($strid, ".", $fname_idx); + $strid = substr($strid, 0, $fname_idx); + return "@".$strid; +} + +function config_make_path(string $conf_dir, string $path) : string +{ + return ltrim(str_replace(normalize_path($conf_dir, true/*nix*/), + '', + normalize_path($path, true/*nix*/)), + '/'); +} + +function config_crc28(string $what) : int +{ + return crc32($what) & 0xFFFFFFF; +} + +function config_get_module_includes(\JSM_Module $cm) : array +{ + $includes = array(); + foreach($cm->getIncludes() as $include => $_) + { + //maybe we should take .php includes into account as well? + if(str_ends_with($include, ".php")) + continue; + $includes[] = $include; + } + return $includes; +} + +function config_walk_fields(object $proto, callable $callback) +{ + if(!method_exists($proto, 'CLASS_FIELDS_PROPS')) + return; + + foreach($proto->CLASS_FIELDS_PROPS() as $field => $tokens) + { + $value = $proto->$field; + $callback($proto, $field, $value, $tokens); + + if(is_object($value)) + config_walk_fields($value, $callback); + else if(is_array($value)) + { + foreach($value as $num => $item) + { + if(is_object($item)) + config_walk_fields($item, $callback); + } + } + } +} + +function config_includes_map_find_text_origin(array $map, string $file, string $text) : array +{ + if(!isset($map[$file])) + return array($file); + + $tpls = $map[$file]; + $tpls[] = $file; + $res = array(); + foreach($tpls as $tpl) + { + $content = ensure_read($tpl); + $content = i18n_decode_string($content); + if(strpos($content, $text) !== FALSE) + $res[] = $tpl; + } + + return $res; +} + +//TODO: +//function conf2json(object $conf, int $json_flags = 0) : string +//{ +// $arr = $conf->export(true); +// $arr['class'] = get_class($conf); +// unset($arr['id']); +// unset($arr['strid']); +// $json = json_encode($arr, $json_flags); +// $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json); +// return $json; +//} +