getMessage() . "\n" . $e->getTraceAsString(), FILE_APPEND); throw $e; } }); function config_base_dir() { global $GAME_ROOT; return "$GAME_ROOT/unity/Assets/Configs/"; } function config_build_dir() { global $GAME_ROOT; return "$GAME_ROOT/build/tmp/"; } function config_set_global_deps(array $deps) { global $CONFIG_GLOBAL_DEPS; $CONFIG_GLOBAL_DEPS = array(); foreach($deps as $d) { if(is_file($d)) $CONFIG_GLOBAL_DEPS[] = normalize_path($d); } } function config_set_worker_init_fn($fn) { global $CONFIG_INIT_WORKER_FUNC; $CONFIG_INIT_WORKER_FUNC = $fn; } function config_get_global_deps() { global $CONFIG_GLOBAL_DEPS; return $CONFIG_GLOBAL_DEPS; } function config_get_build_ext_dir() { global $GAME_ROOT; return "$GAME_ROOT/build/ext_config/"; } function config_get_bundle_ext_path() { return config_get_build_ext_dir() . "/configs/" . get("CONFIGS_BUNDLE_NAME"); } function config_scan_files() { return scan_files_rec(array(config_base_dir()), array('conf.js')); } function config_pack_bundle(array $cache_entries, $use_lz4 = false, $use_config_refs = false) { global $GAME_ROOT; $binary_format = 1; $version = game_version_code(); $MAP = array(); $STRIDMAP = array(); $payloads = array(); $payloads_offset = 0; foreach($cache_entries as $entry) { list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs); $payload_size = strlen($payload); $payloads[] = array($payloads_offset, $payload, $format, $payload_size); $payloads_offset += $payload_size; } $header = array(); foreach($cache_entries as $idx => $entry) { if(isset($MAP[$entry->id])) throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'"); $MAP[$entry->id] = $entry->strid; $strid_crc = crc32($entry->strid); if(isset($STRIDMAP[$strid_crc])) throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'"); $STRIDMAP[$strid_crc] = $entry->strid; $header[] = array( $payloads[$idx][2], $entry->id, crc32($entry->strid), $entry->class_id, $payloads[$idx][0], $payloads[$idx][3] ); } $header_msgpack = config_msgpack_pack($header); $payloads_bundle = ''; foreach($payloads as $item) $payloads_bundle .= $item[1]; $packed_data = pack("C", $binary_format) . pack("V", $version) . pack("V", strlen($header_msgpack)) . $header_msgpack . $payloads_bundle; echo "CONF.BUNDLE: entries " . sizeof($cache_entries) . "; total " . kb($packed_data) . "; lz4 $use_lz4; refs $use_config_refs; CRC " . crc32($packed_data) . "\n"; return $packed_data; } function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs) { $format = ConfigCacheEntry::FMT_BINARY; $payload = null; if($use_config_refs && $ce->payload_file) { $format = ConfigCacheEntry::FMT_FILE_REF; $payload = $ce->payload_file; } else { $payload = $ce->payload; if($use_lz4 && strlen($payload) > 512) { $format = ConfigCacheEntry::FMT_LZ4; $payload = lz4_compress($payload); } } return array($format, $payload); } function config_sync_build_dir_with_prod() { global $GAME_ROOT; $build_ext_dir = config_get_build_ext_dir(); ensure_mkdir($build_ext_dir); ensure_sync($build_ext_dir, "$GAME_ROOT/unity/Assets/Resources/ext_config/"); } function config_make_standalone_ext_bundle(array $configs, $file_path) { $cache_entries = array(); foreach($configs as $conf) { $payload = config_msgpack_pack($conf->export()); //creating fake cache entries $entry = new ConfigCacheEntry(); $entry->id = $conf->id; $entry->strid = $conf->strid; $entry->class_id = $conf->getClassId(); $entry->class = get_class($conf); $entry->payload = $payload; $entry->config = $conf; $cache_entries[] = $entry; } $packed_data = config_pack_bundle($cache_entries, get('USE_LZ4_CONFIGS')); ensure_write($file_path, $packed_data); } function config_filter_files(array $files) { $configs = array(); foreach($files as $file) { if(strpos($file, '.conf.js') === false || strpos($file, '.bhl') !== false ) continue; $configs[] = $file; } return $configs; } function config_bench_load($file) { list($proto_id, $_) = config_ensure_header(config_base_dir(), $file); if(!$proto_id) throw new Exception("Bad proto_id: {$proto_id}"); $t = microtime(true); $parse_res = config_parse(config_base_dir(), $file); if($parse_res->error !== 0) throw new Exception("Error({$parse_res->error}) while loading JSON from {$file}:\n" . $parse_res->error_descr); echo "PARSE: " . (microtime(true) - $t) . "\n"; $t = microtime(true); $config = config_load_ex(config_base_dir(), $file, $parse_res->parsed_arr, $proto_id); echo "LOAD: " . (microtime(true) - $t) . "\n"; } function config_get_tmp_build_path($file) { $name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file)))); $name = ltrim($name, "-"); return normalize_path(config_build_dir() . "/$name"); } function config_get_includes_map_path() { return config_build_dir() . "/includes.map"; } function config_load_includes_map($file = null) { $file = $file ? $file : config_get_includes_map_path(); $includes_map = array(array(), array()); if(is_file($file)) { $tmp_map = @unserialize(ensure_read($file)); if(is_array($tmp_map)) $includes_map = $tmp_map; } return $includes_map; } function config_save_includes_map(array $includes_map, $file = null) { $file = $file ? $file : config_get_includes_map_path(); ensure_write($file, serialize($includes_map)); } function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $includes_map_file = null, $max_workers = null) { if(!$files) return array(); if($max_workers === null) $max_workers = sizeof($files) < 20 ? 1 : 4; $includes_map_file = $includes_map_file ? $includes_map_file : config_get_includes_map_path(); $includes_map = config_load_includes_map($includes_map_file); $chunk_size = ceil(sizeof($files)/$max_workers); $jobs = array(); foreach(array_chunk($files, $chunk_size) as $idx => $chunk_files) $jobs[] = array($idx, $chunk_files); $results_by_job = null; $serial = $max_workers == 1; if(!$serial) { $results_by_job = _config_worker_run_procs($jobs, $includes_map_file, $force_stale, $verbose); //in case of any result error try serial processing if(array_search(false, $results_by_job, true/*strict*/) !== false) { if($verbose) echo "Result error detected, trying serial processing...\n"; $serial = true; } } if($serial) { $results_by_job = array(); foreach($jobs as $job) $results_by_job[] = _config_worker_func($job, $includes_map, $force_stale, $verbose); } $total_stales = 0; $cache_entries = array(); foreach($results_by_job as $results) { foreach($results as $file => $item) { list($cache_file, $is_stale) = $item; $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file)); //NOTE: handling any cache corruption errors if($cache_entry === null) { $cache_entry = _config_invalidate_cache($file, $cache_file); $is_stale = true; } if($is_stale) ++$total_stales; $includes_map[$file] = $cache_entry->includes; //we want results to be returned in the same order, so //we store entries by the file key and later retrieve array values $cache_entries[$file] = $cache_entry; } } if($verbose) echo "Miss/Hit: $total_stales/" . sizeof($cache_entries) . "\n"; config_save_includes_map($includes_map, $includes_map_file); return array_values($cache_entries); } function _config_worker_run_procs(array $jobs, $includes_map_file, $force, $verbose) { $worker_args = array(); foreach($jobs as $idx => $job) $worker_args[] = array($job, $includes_map_file, $force, $verbose); return run_background_gamectl_workers('config_worker', $worker_args); } function _config_worker_func(array $job, array $includes_map, $force, $verbose) { global $CONFIG_INIT_WORKER_FUNC; if(is_callable($CONFIG_INIT_WORKER_FUNC)) $CONFIG_INIT_WORKER_FUNC(); list($idx, $files) = $job; if($verbose) echo "Worker $idx (" . sizeof($files) . ") started\n"; $results = array(); foreach($files as $file_idx => $file) { if($verbose && $file_idx > 0 && ($file_idx % 500) == 0) echo "Worker $idx progress: " . round($file_idx / sizeof($files) * 100) . "% ...\n"; $cache_file = config_get_cache_path($file); $is_stale = true; if(!$force) { $file_deps = array($file); if(isset($includes_map[$file])) $file_deps = array_merge($file_deps, $includes_map[$file]); $is_stale = need_to_regen($cache_file, $file_deps); } if($is_stale) _config_invalidate_cache($file, $cache_file); $results[$file] = array($cache_file, $is_stale); } if($verbose) echo "Worker $idx done\n"; return $results; } function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry { $cache_payload_file = config_get_cache_payload_path($file); list($proto_id, $_) = config_ensure_header(config_base_dir(), $file); if(!$proto_id) throw new Exception("Bad proto_id: {$proto_id}"); $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $proto_id; $GLOBALS['CONFIG_PREFAB_DEPS'] = array(); $GLOBALS['CONFIG_BHL_FUNCS'] = array(); $pres = config_parse(config_base_dir(), $file); if($pres->error !== 0) throw new Exception("Error({$pres->error}) while loading JSON from {$file}:\n" . $pres->error_descr); $includes = config_get_module_includes($pres->jsm_module); $config = config_load_ex(config_base_dir(), $file, $pres->parsed_arr, $proto_id); $payload_data = config_msgpack_pack($config->export()); $cache_entry = new ConfigCacheEntry(); $cache_entry->id = $config->id; $cache_entry->config = $config; $cache_entry->strid = $config->strid; $cache_entry->cache_file = $cache_file; $cache_entry->class = get_class($config); $cache_entry->class_id = $config->getClassId(); $cache_entry->payload_file = $cache_payload_file; $cache_entry->file = normalize_path($file); $cache_entry->includes = $includes; ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry)); ensure_write($cache_payload_file, $payload_data); return $cache_entry; } function config_find_by_alias(array $cache_entries, $strid) : ConfigCacheEntry { foreach($cache_entries as $ce) if($ce->strid === $strid) return $ce; return null; } function config_find_by_id(array $cache_entries, $id) : ConfigCacheEntry { foreach($cache_entries as $ce) if($ce->id === $id) return $ce; return null; } function config_find_by_path(array $cache_entries, $path) : ConfigCacheEntry { $path = normalize_path($path); foreach($cache_entries as $ce) if($ce->file === $path) return $ce; return null; } function config_fetch_by_path(string $path, $force_stale = false, $use_cache = true, $includes_map_file = null) : ConfigCacheEntry { $ces = config_fetch_ex(array($path), $force_stale, $verbose = false, $includes_map_file, null, $use_cache); if(!$ces) throw new Exception("Config not found at path '$path'"); return $ces[0]; } function config_fetch_all($force_stale = false, $use_cache = true, $includes_map_file = null) { return config_fetch_ex(config_scan_files(), $force_stale, $verbose = false, $includes_map_file, null, $use_cache); } class ConfigCacheEntry { const FMT_BINARY = 0; const FMT_LZ4 = 1; const FMT_FILE_REF = 2; public $class; public $class_id; public $id; public $strid; public $cache_file; //NOTE: actual payload is stored in a separate file for faster incremental retrievals public $payload_file; public $file; public $includes = array(); //TODO: do we need these? public $refs = array(); public $prefab_deps = array(); public $bhl_funcs = array(); public $_config; public $_payload; static function serialize($ce) { $d = $ce->export(); return serialize($d); } static function unserialize($str) { $d = @unserialize($str); if(!is_array($d)) return null; $ce = new ConfigCacheEntry(); $ce->import($d); return $ce; } function __get($name) { if($name === "config") { if($this->_config === null) { $klass = $this->class; $data = config_msgpack_unpack($this->payload); $this->_config = new $klass($data); } return $this->_config; } else if($name === "payload") { if($this->_payload !== null) return $this->_payload; //if payload not set directly not storing it return ensure_read($this->payload_file); } else throw new Exception("No such property '$name'"); } function __set($name, $v) { if($name === "config") $this->_config = $v; else if($name === "payload") $this->_payload = $v; else throw new Exception("No such property '$name'"); } function export() { $d = array(); $d[] = $this->class; $d[] = $this->class_id; $d[] = $this->id; $d[] = $this->strid; $d[] = $this->cache_file; $d[] = $this->payload_file; $d[] = $this->file; $d[] = $this->includes; //TODO: do we need these? $d[] = $this->refs; $d[] = $this->prefab_deps; $d[] = $this->bhl_funcs; return $d; } function import(array $d) { list( $this->class, $this->class_id, $this->id, $this->strid, $this->cache_file, $this->payload_file, $this->file, $this->includes, //TODO: do we need these? $this->refs, $this->prefab_deps, $this->bhl_funcs, ) = $d; } } function config_get_cache_path($file) { return config_get_tmp_build_path($file . '.cacheb'); } function config_get_cache_payload_path($file) { return config_get_tmp_build_path($file . '.pdata'); } function config_get_id($conf_dir, $file_path) { return config_file2id($conf_dir, $file_path); } function config_get_strid($conf_dir, $file_path) { return config_file2strid($conf_dir, $file_path); } function config_extract_refs($src, $as_map = true) { $refs = array(); if(preg_match_all('~"(@[^"]+)"~', $src, $matches)) { foreach($matches[1] as $ref) { if(!isset($refs[$ref])) $refs[$ref] = 1; ++$refs[$ref]; } } return $as_map ? $refs : array_keys($refs); } function config_get_header($file, &$proto_id, &$alias) { $h = fopen($file, "r"); $line = fgets($h, 256); fclose($h); if(preg_match('~\{\s*/\*\s*proto_id\s*=\s*(\d+)\s*;\s*alias\s*=\s*([^\s]+)~', $line, $matches)) { $proto_id = (int)$matches[1]; $alias = $matches[2]; return true; } else return false; } function config_extract_header($contents) { return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents); } function config_ensure_header($conf_dir, $file, $force = false) { if(config_get_header($file, $curr_proto_id, $curr_alias)) { $alias = config_get_strid($conf_dir, $file); if($force) $curr_proto_id = config_get_id($conf_dir, $file); //NOTE: keeping current proto id intact if not forced if($force || $curr_alias !== $alias) { $lines = file($file); $lines[0] = preg_replace('~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{ /* proto_id = {$curr_proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count); if($count != 1) throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); ensure_write($file, join("", $lines)); } return array($curr_proto_id, $alias); } else { $proto_id = config_get_id($conf_dir, $file); $alias = config_get_strid($conf_dir, $file); $lines = file($file); $lines[0] = preg_replace('~\s*\{~', "{ /* proto_id = {$proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count); if($count != 1) throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); ensure_write($file, join("", $lines)); return array($proto_id, $alias); } } function config_load($conf_dir, $conf_path) { list($proto_id, $_) = config_ensure_header($conf_dir, $conf_path); if(!$proto_id) throw new Exception("Bad proto_id: {$proto_id}"); $pres = config_parse($conf_dir, $conf_path); if($pres->error !== 0) throw new Exception("Error({$pres->error}) while loading JSON from {$conf_path}:\n" . $pres->error_descr); return config_load_ex($conf_dir, $conf_path, $pres->parsed_arr, $proto_id); } function conf2json($conf, $json_flags = 0) { $arr = $conf->export(true); $arr['class'] = get_class($conf); unset($arr['id']); unset($arr['strid']); $json = json_encode($arr, $json_flags); $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json); return $json; } class ConfigParseResult { public $error = 0; public $error_descr; public $normalized_jzon = ''; public $parsed_arr; public $jsm_module; } function config_parse($conf_dir, $file) : ConfigParseResult { $res = new ConfigParseResult(); $jsm = new \JSM($conf_dir, $file); $normalized_jzon = ''; try { list($normalized_jzon, $jsm_module) = $jsm->process(); $res->normalized_jzon = $normalized_jzon; $res->jsm_module = $jsm_module; } catch(Exception $e) { $res->error = 1; $res->error_descr = "File '$file':\n" . $e->getMessage() . "\n" . $e->getTraceAsString(); return $res; } $decode_res = config_check_and_decode_jzon($normalized_jzon); if($decode_res[0] === 0) { $res->parsed_arr = $decode_res[2]; } else { $res->error = $decode_res[0]; $res->error_descr = "File '$file':\n" . $decode_res[1]; } return $res; } function config_check_and_decode_jzon($json) { try { $arr = \jzon_parse($json); return array(0, "", $arr); } catch(Exception $e) { return array(1, $e->getMessage(), array()); } } function config_load_ex($conf_dir, $file, array $arr, $id = null) { if(!isset($arr['class']) || !isset($arr['class'][0])) throw new Exception("Class is not set. File '$file'."); $klass = $arr['class']; unset($arr['class']); if($id === null) $id = config_file2id($conf_dir, $file); $cnf = null; try { $cnf = new $klass; $cnf->import($arr, true); } catch(Exception $e) { throw new Exception($e->getMessage() . ". File '{$file}'"/* . $e->getTraceAsString()*/); } $cnf->id = $id; $cnf->strid = config_file2strid($conf_dir, $file); return $cnf; } function config_is_file($filename) { return (strrpos($filename, '.conf.js') === (strlen($filename) - 8)); } function config_file2id($conf_dir, $filename) { $nfilename = config_make_path($conf_dir, $filename); return config_crc28($nfilename); } function config_file2strid($conf_dir, $filename) { $strid = config_make_path($conf_dir, $filename); $fname_idx = strrpos($strid, "/"); $fname_idx = strpos($strid, ".", $fname_idx); $strid = substr($strid, 0, $fname_idx); return "@".$strid; } function config_make_path($conf_dir, $path) { return ltrim(str_replace(normalize_path($conf_dir, true/*nix*/), '', normalize_path($path, true/*nix*/)), '/'); } function config_str2id($str) { if(strpos($str, '@') === 0) $str = substr($str, 1) . '.conf.js'; return config_crc28($str); } function config_crc28($what) { return crc32($what) & 0xFFFFFFF; } function config_get_module_includes(\JSM_Module $cm) { $includes = array(); foreach($cm->getIncludes() as $include => $_) { //maybe we should take .php includes into account as well? if(config_str_ends_with($include, ".php")) continue; $includes[] = $include; } return $includes; } function config_includes_map_find_text_origin(array $map, $file, $text) { if(!isset($map[$file])) return array($file); $tpls = $map[$file]; $tpls[] = $file; $res = array(); foreach($tpls as $tpl) { $content = ensure_read($tpl); $content = i18n_decode_string($content); if(strpos($content, $text) !== FALSE) $res[] = $tpl; } return $res; } function config_str_replace_in_files($subj, $repl, array $files) { foreach($files as $file) { $contents = ensure_read($file); $new_contents = str_replace($subj, $repl, $contents); if($new_contents !== $contents) ensure_write($file, $new_contents); } } function config_str_ends_with($haystack, $needle) { // search forward starting from end minus needle length characters return $needle === "" || (($temp = strlen($haystack) - strlen($needle)) >= 0 && strpos($haystack, $needle, $temp) !== false); } function config_walk_fields($proto, $callback) { if(!method_exists($proto, 'CLASS_FIELDS_PROPS')) return; foreach($proto->CLASS_FIELDS_PROPS() as $field => $tokens) { $value = $proto->$field; $callback($proto, $field, $value, $tokens); if(is_object($value)) config_walk_fields($value, $callback); else if(is_array($value)) { foreach($value as $num => $item) { if(is_object($item)) config_walk_fields($item, $callback); } } } } if(function_exists('msgpack_pack')) { function config_msgpack_pack($data) { $prev = ini_set('msgpack.use_str8_serialization', '0'); $res = msgpack_pack($data); if($prev != '0') ini_set('msgpack.use_str8_serialization', $prev); return $res; } function config_msgpack_unpack($data) { return msgpack_unpack($data); } } else { function config_msgpack_pack($data) { include_once(__DIR__ . '/msgpack/msgpack_custom.inc.php'); $packer = new \MessagePackCustom(); return $packer->pack($data); } function config_msgpack_unpack($data) { return \MessagePack\MessagePack::unpack($data); } }