From cccabe3c485c8d43a44a5f4d082c864447691290 Mon Sep 17 00:00:00 2001 From: Pavel Shevaev Date: Mon, 16 May 2022 14:21:18 +0300 Subject: [PATCH] first commit --- README.md | 0 config.inc.php | 910 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 910 insertions(+) create mode 100644 README.md create mode 100644 config.inc.php diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/config.inc.php b/config.inc.php new file mode 100644 index 0000000..a7caf3c --- /dev/null +++ b/config.inc.php @@ -0,0 +1,910 @@ +getMessage() . "\n" . $e->getTraceAsString(), FILE_APPEND); + throw $e; + } +}); + +function config_base_dir() +{ + global $GAME_ROOT; + return "$GAME_ROOT/unity/Assets/Configs/"; +} + +function config_build_dir() +{ + global $GAME_ROOT; + return "$GAME_ROOT/build/tmp/"; +} + +function config_set_global_deps(array $deps) +{ + global $CONFIG_GLOBAL_DEPS; + $CONFIG_GLOBAL_DEPS = array(); + foreach($deps as $d) + { + if(is_file($d)) + $CONFIG_GLOBAL_DEPS[] = normalize_path($d); + } +} + +function config_set_worker_init_fn($fn) +{ + global $CONFIG_INIT_WORKER_FUNC; + $CONFIG_INIT_WORKER_FUNC = $fn; +} + +function config_get_global_deps() +{ + global $CONFIG_GLOBAL_DEPS; + return $CONFIG_GLOBAL_DEPS; +} + +function config_get_build_ext_dir() +{ + global $GAME_ROOT; + return "$GAME_ROOT/build/ext_config/"; +} + +function config_get_bundle_ext_path() +{ + return config_get_build_ext_dir() . "/configs/" . get("CONFIGS_BUNDLE_NAME"); +} + +function config_scan_files() +{ + return scan_files_rec(array(config_base_dir()), array('conf.js')); +} + +function config_pack_bundle(array $cache_entries, $use_lz4 = false, $use_config_refs = false) +{ + global $GAME_ROOT; + + $binary_format = 1; + $version = game_version_code(); + + $MAP = array(); + $STRIDMAP = array(); + + $payloads = array(); + $payloads_offset = 0; + foreach($cache_entries as $entry) + { + list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs); + $payload_size = strlen($payload); + $payloads[] = array($payloads_offset, $payload, $format, $payload_size); + $payloads_offset += $payload_size; + } + + $header = array(); + foreach($cache_entries as $idx => $entry) + { + if(isset($MAP[$entry->id])) + throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'"); + $MAP[$entry->id] = $entry->strid; + + $strid_crc = crc32($entry->strid); + if(isset($STRIDMAP[$strid_crc])) + throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'"); + $STRIDMAP[$strid_crc] = $entry->strid; + + $header[] = array( + $payloads[$idx][2], + $entry->id, + crc32($entry->strid), + $entry->class_id, + $payloads[$idx][0], + $payloads[$idx][3] + ); + } + + $header_msgpack = config_msgpack_pack($header); + $payloads_bundle = ''; + foreach($payloads as $item) + $payloads_bundle .= $item[1]; + + $packed_data = + pack("C", $binary_format) . + pack("V", $version) . + pack("V", strlen($header_msgpack)) . + $header_msgpack . + $payloads_bundle; + + echo "CONF.BUNDLE: entries " . sizeof($cache_entries) . "; total " . kb($packed_data) . + "; lz4 $use_lz4; refs $use_config_refs; CRC " . crc32($packed_data) . "\n"; + + return $packed_data; +} + +function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs) +{ + $format = ConfigCacheEntry::FMT_BINARY; + $payload = null; + + if($use_config_refs && $ce->payload_file) + { + $format = ConfigCacheEntry::FMT_FILE_REF; + $payload = $ce->payload_file; + } + else + { + $payload = $ce->payload; + if($use_lz4 && strlen($payload) > 512) + { + $format = ConfigCacheEntry::FMT_LZ4; + $payload = lz4_compress($payload); + } + } + + return array($format, $payload); +} + +function config_sync_build_dir_with_prod() +{ + global $GAME_ROOT; + + $build_ext_dir = config_get_build_ext_dir(); + ensure_mkdir($build_ext_dir); + + ensure_sync($build_ext_dir, "$GAME_ROOT/unity/Assets/Resources/ext_config/"); +} + +function config_make_standalone_ext_bundle(array $configs, $file_path) +{ + $cache_entries = array(); + foreach($configs as $conf) + { + $payload = config_msgpack_pack($conf->export()); + + //creating fake cache entries + $entry = new ConfigCacheEntry(); + $entry->id = $conf->id; + $entry->strid = $conf->strid; + $entry->class_id = $conf->getClassId(); + $entry->class = get_class($conf); + $entry->payload = $payload; + $entry->config = $conf; + + $cache_entries[] = $entry; + } + + $packed_data = config_pack_bundle($cache_entries, get('USE_LZ4_CONFIGS')); + + ensure_write($file_path, $packed_data); +} + +function config_filter_files(array $files) +{ + $configs = array(); + foreach($files as $file) + { + if(strpos($file, '.conf.js') === false || + strpos($file, '.bhl') !== false + ) + continue; + + $configs[] = $file; + } + return $configs; +} + +function config_bench_load($file) +{ + list($proto_id, $_) = config_ensure_header(config_base_dir(), $file); + if(!$proto_id) + throw new Exception("Bad proto_id: {$proto_id}"); + $t = microtime(true); + $parse_res = config_parse(config_base_dir(), $file); + if($parse_res->error !== 0) + throw new Exception("Error({$parse_res->error}) while loading JSON from {$file}:\n" . $parse_res->error_descr); + echo "PARSE: " . (microtime(true) - $t) . "\n"; + $t = microtime(true); + $config = config_load_ex(config_base_dir(), $file, $parse_res->parsed_arr, $proto_id); + echo "LOAD: " . (microtime(true) - $t) . "\n"; +} + +function config_get_tmp_build_path($file) +{ + $name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file)))); + $name = ltrim($name, "-"); + return normalize_path(config_build_dir() . "/$name"); +} + +function config_get_includes_map_path() +{ + return config_build_dir() . "/includes.map"; +} + +function config_load_includes_map($file = null) +{ + $file = $file ? $file : config_get_includes_map_path(); + + $includes_map = array(array(), array()); + + if(is_file($file)) + { + $tmp_map = @unserialize(ensure_read($file)); + if(is_array($tmp_map)) + $includes_map = $tmp_map; + } + + return $includes_map; +} + +function config_save_includes_map(array $includes_map, $file = null) +{ + $file = $file ? $file : config_get_includes_map_path(); + + ensure_write($file, serialize($includes_map)); +} + +function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $includes_map_file = null, $max_workers = null) +{ + if(!$files) + return array(); + + if($max_workers === null) + $max_workers = sizeof($files) < 20 ? 1 : 4; + + $includes_map_file = $includes_map_file ? $includes_map_file : config_get_includes_map_path(); + $includes_map = config_load_includes_map($includes_map_file); + + $chunk_size = ceil(sizeof($files)/$max_workers); + $jobs = array(); + foreach(array_chunk($files, $chunk_size) as $idx => $chunk_files) + $jobs[] = array($idx, $chunk_files); + + $results_by_job = null; + + $serial = $max_workers == 1; + + if(!$serial) + { + $results_by_job = _config_worker_run_procs($jobs, $includes_map_file, $force_stale, $verbose); + //in case of any result error try serial processing + if(array_search(false, $results_by_job, true/*strict*/) !== false) + { + if($verbose) + echo "Result error detected, trying serial processing...\n"; + $serial = true; + } + } + + if($serial) + { + $results_by_job = array(); + foreach($jobs as $job) + $results_by_job[] = _config_worker_func($job, $includes_map, $force_stale, $verbose); + } + + $total_stales = 0; + $cache_entries = array(); + foreach($results_by_job as $results) + { + foreach($results as $file => $item) + { + list($cache_file, $is_stale) = $item; + + $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file)); + //NOTE: handling any cache corruption errors + if($cache_entry === null) + { + $cache_entry = _config_invalidate_cache($file, $cache_file); + $is_stale = true; + } + + if($is_stale) + ++$total_stales; + + $includes_map[$file] = $cache_entry->includes; + + //we want results to be returned in the same order, so + //we store entries by the file key and later retrieve array values + $cache_entries[$file] = $cache_entry; + } + } + + if($verbose) + echo "Miss/Hit: $total_stales/" . sizeof($cache_entries) . "\n"; + + config_save_includes_map($includes_map, $includes_map_file); + + return array_values($cache_entries); +} + +function _config_worker_run_procs(array $jobs, $includes_map_file, $force, $verbose) +{ + $worker_args = array(); + foreach($jobs as $idx => $job) + $worker_args[] = array($job, $includes_map_file, $force, $verbose); + + return run_background_gamectl_workers('config_worker', $worker_args); +} + +function _config_worker_func(array $job, array $includes_map, $force, $verbose) +{ + global $CONFIG_INIT_WORKER_FUNC; + if(is_callable($CONFIG_INIT_WORKER_FUNC)) + $CONFIG_INIT_WORKER_FUNC(); + + list($idx, $files) = $job; + if($verbose) + echo "Worker $idx (" . sizeof($files) . ") started\n"; + $results = array(); + foreach($files as $file_idx => $file) + { + if($verbose && $file_idx > 0 && ($file_idx % 500) == 0) + echo "Worker $idx progress: " . round($file_idx / sizeof($files) * 100) . "% ...\n"; + + $cache_file = config_get_cache_path($file); + + $is_stale = true; + if(!$force) + { + $file_deps = array($file); + if(isset($includes_map[$file])) + $file_deps = array_merge($file_deps, $includes_map[$file]); + $is_stale = need_to_regen($cache_file, $file_deps); + } + + if($is_stale) + _config_invalidate_cache($file, $cache_file); + + $results[$file] = array($cache_file, $is_stale); + } + if($verbose) + echo "Worker $idx done\n"; + return $results; +} + +function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry +{ + $cache_payload_file = config_get_cache_payload_path($file); + + list($proto_id, $_) = config_ensure_header(config_base_dir(), $file); + if(!$proto_id) + throw new Exception("Bad proto_id: {$proto_id}"); + + $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $proto_id; + $GLOBALS['CONFIG_PREFAB_DEPS'] = array(); + $GLOBALS['CONFIG_BHL_FUNCS'] = array(); + + $pres = config_parse(config_base_dir(), $file); + if($pres->error !== 0) + throw new Exception("Error({$pres->error}) while loading JSON from {$file}:\n" . $pres->error_descr); + + $includes = config_get_module_includes($pres->jsm_module); + + $config = config_load_ex(config_base_dir(), $file, $pres->parsed_arr, $proto_id); + $payload_data = config_msgpack_pack($config->export()); + + $cache_entry = new ConfigCacheEntry(); + $cache_entry->id = $config->id; + $cache_entry->config = $config; + $cache_entry->strid = $config->strid; + $cache_entry->cache_file = $cache_file; + $cache_entry->class = get_class($config); + $cache_entry->class_id = $config->getClassId(); + $cache_entry->payload_file = $cache_payload_file; + $cache_entry->file = normalize_path($file); + $cache_entry->includes = $includes; + + ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry)); + ensure_write($cache_payload_file, $payload_data); + + return $cache_entry; +} + +function config_find_by_alias(array $cache_entries, $strid) : ConfigCacheEntry +{ + foreach($cache_entries as $ce) + if($ce->strid === $strid) + return $ce; + return null; +} + +function config_find_by_id(array $cache_entries, $id) : ConfigCacheEntry +{ + foreach($cache_entries as $ce) + if($ce->id === $id) + return $ce; + return null; +} + +function config_find_by_path(array $cache_entries, $path) : ConfigCacheEntry +{ + $path = normalize_path($path); + foreach($cache_entries as $ce) + if($ce->file === $path) + return $ce; + return null; +} + +function config_fetch_by_path(string $path, $force_stale = false, $use_cache = true, $includes_map_file = null) : ConfigCacheEntry +{ + $ces = config_fetch_ex(array($path), $force_stale, $verbose = false, $includes_map_file, null, $use_cache); + if(!$ces) + throw new Exception("Config not found at path '$path'"); + return $ces[0]; +} + +function config_fetch_all($force_stale = false, $use_cache = true, $includes_map_file = null) +{ + return config_fetch_ex(config_scan_files(), $force_stale, $verbose = false, $includes_map_file, null, $use_cache); +} + +class ConfigCacheEntry +{ + const FMT_BINARY = 0; + const FMT_LZ4 = 1; + const FMT_FILE_REF = 2; + + public $class; + public $class_id; + public $id; + public $strid; + public $cache_file; + //NOTE: actual payload is stored in a separate file for faster incremental retrievals + public $payload_file; + public $file; + public $includes = array(); + //TODO: do we need these? + public $refs = array(); + public $prefab_deps = array(); + public $bhl_funcs = array(); + + public $_config; + public $_payload; + + static function serialize($ce) + { + $d = $ce->export(); + return serialize($d); + } + + static function unserialize($str) + { + $d = @unserialize($str); + if(!is_array($d)) + return null; + $ce = new ConfigCacheEntry(); + $ce->import($d); + return $ce; + } + + function __get($name) + { + if($name === "config") + { + if($this->_config === null) + { + $klass = $this->class; + $data = config_msgpack_unpack($this->payload); + $this->_config = new $klass($data); + } + return $this->_config; + } + else if($name === "payload") + { + if($this->_payload !== null) + return $this->_payload; + //if payload not set directly not storing it + return ensure_read($this->payload_file); + } + else + throw new Exception("No such property '$name'"); + } + + function __set($name, $v) + { + if($name === "config") + $this->_config = $v; + else if($name === "payload") + $this->_payload = $v; + else + throw new Exception("No such property '$name'"); + } + + function export() + { + $d = array(); + $d[] = $this->class; + $d[] = $this->class_id; + $d[] = $this->id; + $d[] = $this->strid; + $d[] = $this->cache_file; + $d[] = $this->payload_file; + $d[] = $this->file; + $d[] = $this->includes; + //TODO: do we need these? + $d[] = $this->refs; + $d[] = $this->prefab_deps; + $d[] = $this->bhl_funcs; + return $d; + } + + function import(array $d) + { + list( + $this->class, + $this->class_id, + $this->id, + $this->strid, + $this->cache_file, + $this->payload_file, + $this->file, + $this->includes, + //TODO: do we need these? + $this->refs, + $this->prefab_deps, + $this->bhl_funcs, + ) = $d; + } +} + +function config_get_cache_path($file) +{ + return config_get_tmp_build_path($file . '.cacheb'); +} + +function config_get_cache_payload_path($file) +{ + return config_get_tmp_build_path($file . '.pdata'); +} + +function config_get_id($conf_dir, $file_path) +{ + return config_file2id($conf_dir, $file_path); +} + +function config_get_strid($conf_dir, $file_path) +{ + return config_file2strid($conf_dir, $file_path); +} + +function config_extract_refs($src, $as_map = true) +{ + $refs = array(); + if(preg_match_all('~"(@[^"]+)"~', $src, $matches)) + { + foreach($matches[1] as $ref) + { + if(!isset($refs[$ref])) + $refs[$ref] = 1; + ++$refs[$ref]; + } + } + return $as_map ? $refs : array_keys($refs); +} + +function config_get_header($file, &$proto_id, &$alias) +{ + $h = fopen($file, "r"); + $line = fgets($h, 256); + fclose($h); + + if(preg_match('~\{\s*/\*\s*proto_id\s*=\s*(\d+)\s*;\s*alias\s*=\s*([^\s]+)~', $line, $matches)) + { + $proto_id = (int)$matches[1]; + $alias = $matches[2]; + return true; + } + else + return false; +} + +function config_extract_header($contents) +{ + return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents); +} + +function config_ensure_header($conf_dir, $file, $force = false) +{ + if(config_get_header($file, $curr_proto_id, $curr_alias)) + { + $alias = config_get_strid($conf_dir, $file); + if($force) + $curr_proto_id = config_get_id($conf_dir, $file); + + //NOTE: keeping current proto id intact if not forced + if($force || $curr_alias !== $alias) + { + $lines = file($file); + $lines[0] = preg_replace('~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{ /* proto_id = {$curr_proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count); + if($count != 1) + throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); + ensure_write($file, join("", $lines)); + } + + return array($curr_proto_id, $alias); + } + else + { + $proto_id = config_get_id($conf_dir, $file); + $alias = config_get_strid($conf_dir, $file); + + $lines = file($file); + $lines[0] = preg_replace('~\s*\{~', "{ /* proto_id = {$proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count); + if($count != 1) + throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); + ensure_write($file, join("", $lines)); + + return array($proto_id, $alias); + } +} + +function config_load($conf_dir, $conf_path) +{ + list($proto_id, $_) = config_ensure_header($conf_dir, $conf_path); + if(!$proto_id) + throw new Exception("Bad proto_id: {$proto_id}"); + + $pres = config_parse($conf_dir, $conf_path); + if($pres->error !== 0) + throw new Exception("Error({$pres->error}) while loading JSON from {$conf_path}:\n" . $pres->error_descr); + + return config_load_ex($conf_dir, $conf_path, $pres->parsed_arr, $proto_id); +} + +function conf2json($conf, $json_flags = 0) +{ + $arr = $conf->export(true); + $arr['class'] = get_class($conf); + unset($arr['id']); + unset($arr['strid']); + $json = json_encode($arr, $json_flags); + $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json); + return $json; +} + +class ConfigParseResult +{ + public $error = 0; + public $error_descr; + public $normalized_jzon = ''; + public $parsed_arr; + public $jsm_module; +} + +function config_parse($conf_dir, $file) : ConfigParseResult +{ + $res = new ConfigParseResult(); + + $jsm = new \JSM($conf_dir, $file); + + $normalized_jzon = ''; + try + { + list($normalized_jzon, $jsm_module) = $jsm->process(); + $res->normalized_jzon = $normalized_jzon; + $res->jsm_module = $jsm_module; + } + catch(Exception $e) + { + $res->error = 1; + $res->error_descr = "File '$file':\n" . $e->getMessage() . "\n" . $e->getTraceAsString(); + return $res; + } + + $decode_res = config_check_and_decode_jzon($normalized_jzon); + + if($decode_res[0] === 0) + { + $res->parsed_arr = $decode_res[2]; + } + else + { + $res->error = $decode_res[0]; + $res->error_descr = "File '$file':\n" . $decode_res[1]; + } + + return $res; +} + +function config_check_and_decode_jzon($json) +{ + try + { + $arr = jzon_parse($json); + return array(0, "", $arr); + } + catch(Exception $e) + { + return array(1, $e->getMessage(), array()); + } +} + +function config_load_ex($conf_dir, $file, array $arr, $id = null) +{ + if(!isset($arr['class']) || !isset($arr['class'][0])) + throw new Exception("Class is not set. File '$file'."); + + $klass = $arr['class']; + unset($arr['class']); + + if($id === null) + $id = config_file2id($conf_dir, $file); + + $cnf = null; + try + { + $cnf = new $klass; + $cnf->import($arr, true); + } + catch(Exception $e) + { + throw new Exception($e->getMessage() . ". File '{$file}'"/* . $e->getTraceAsString()*/); + } + + $cnf->id = $id; + $cnf->strid = config_file2strid($conf_dir, $file); + + return $cnf; +} + +function config_is_file($filename) +{ + return (strrpos($filename, '.conf.js') === (strlen($filename) - 8)); +} + +function config_file2id($conf_dir, $filename) +{ + $nfilename = config_make_path($conf_dir, $filename); + return config_crc28($nfilename); +} + +function config_file2strid($conf_dir, $filename) +{ + $strid = config_make_path($conf_dir, $filename); + $fname_idx = strrpos($strid, "/"); + $fname_idx = strpos($strid, ".", $fname_idx); + $strid = substr($strid, 0, $fname_idx); + return "@".$strid; +} + +function config_make_path($conf_dir, $path) +{ + return ltrim(str_replace(normalize_path($conf_dir, true/*nix*/), + '', + normalize_path($path, true/*nix*/)), + '/'); +} + +function config_str2id($str) +{ + if(strpos($str, '@') === 0) + $str = substr($str, 1) . '.conf.js'; + return config_crc28($str); +} + +function config_crc28($what) +{ + return crc32($what) & 0xFFFFFFF; +} + +function config_get_module_includes(\JSM_Module $cm) +{ + $includes = array(); + foreach($cm->getIncludes() as $include => $_) + { + //maybe we should take .php includes into account as well? + if(config_str_ends_with($include, ".php")) + continue; + $includes[] = $include; + } + return $includes; +} + +function config_includes_map_find_text_origin(array $map, $file, $text) +{ + if(!isset($map[$file])) + return array($file); + + $tpls = $map[$file]; + $tpls[] = $file; + $res = array(); + foreach($tpls as $tpl) + { + $content = ensure_read($tpl); + $content = i18n_decode_string($content); + if(strpos($content, $text) !== FALSE) + $res[] = $tpl; + } + + return $res; +} + +function config_str_replace_in_files($subj, $repl, array $files) +{ + foreach($files as $file) + { + $contents = ensure_read($file); + $new_contents = str_replace($subj, $repl, $contents); + if($new_contents !== $contents) + ensure_write($file, $new_contents); + } +} + +function config_str_ends_with($haystack, $needle) +{ + // search forward starting from end minus needle length characters + return $needle === "" || (($temp = strlen($haystack) - strlen($needle)) >= 0 && strpos($haystack, $needle, $temp) !== false); +} + +function config_walk_fields($proto, $callback) +{ + if(!method_exists($proto, 'CLASS_FIELDS_PROPS')) + return; + + foreach($proto->CLASS_FIELDS_PROPS() as $field => $tokens) + { + $value = $proto->$field; + $callback($proto, $field, $value, $tokens); + + if(is_object($value)) + config_walk_fields($value, $callback); + else if(is_array($value)) + { + foreach($value as $num => $item) + { + if(is_object($item)) + config_walk_fields($item, $callback); + } + } + } +} + +if(function_exists('msgpack_pack')) +{ + function config_msgpack_pack($data) + { + $prev = ini_set('msgpack.use_str8_serialization', '0'); + $res = msgpack_pack($data); + if($prev != '0') + ini_set('msgpack.use_str8_serialization', $prev); + return $res; + } + + function config_msgpack_unpack($data) + { + return msgpack_unpack($data); + } +} +else +{ + function config_msgpack_pack($data) + { + include_once(__DIR__ . '/../utils/msgpack_custom.inc.php'); + + $packer = new MessagePackCustom(); + return $packer->pack($data); + } + + function config_msgpack_unpack($data) + { + return MessagePack\MessagePack::unpack($data); + } +}