diff --git a/config.inc.php b/config.inc.php index 4fb2d75..aa6e788 100644 --- a/config.inc.php +++ b/config.inc.php @@ -31,7 +31,10 @@ task('config_worker', function(array $args) function config_base_dir() { - return get("UNITY_ASSETS_DIR") . "/Configs"; + if(is("CONFIGS_BASE_DIR")) + return get("CONFIGS_BASE_DIR"); + else + return get("UNITY_ASSETS_DIR") . "/Configs"; } function config_build_dir() @@ -139,6 +142,32 @@ function config_pack_bundle(array $cache_entries, $use_lz4 = false, $use_config_ return $packed_data; } +function config_unpack_bundle($packed_data) +{ + $packed_info = substr($packed_data, 0, 1+4+4); + $info = unpack('Cformat/Vversion/Vheader_len', $packed_info); + + if($info['format'] !== 1) + throw new Exception("Unknown format: {$info['format']}"); + + $header_msgpack = substr($packed_data, 1+4+4, $info['header_len']); + $header = config_msgpack_unpack($header_msgpack); + + $payloads_bundle = substr($packed_data, 1+4+4 + $info['header_len']); + + $entries = array(); + foreach($header as $item) + { + list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; + + $payload = substr($payloads_bundle, $offset, $size); + + $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); + } + + return $entries; +} + function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs) { $format = ConfigCacheEntry::FMT_BINARY; @@ -155,21 +184,25 @@ function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs) if($use_lz4 && strlen($payload) > 512) { $format = ConfigCacheEntry::FMT_LZ4; - $payload = lz4_compress($payload); + $payload = lz4_compress($payload, 9); } } return array($format, $payload); } -function config_sync_build_dir_with_prod() +function _config_unpack_payload($format, $payload) { - global $GAME_ROOT; - - $build_ext_dir = config_get_build_ext_dir(); - ensure_mkdir($build_ext_dir); - - ensure_sync($build_ext_dir, get("UNITY_ASSETS_DIR")."/Resources/ext_config/"); + $msg_packed = null; + if($format === ConfigCacheEntry::FMT_LZ4) + $msg_packed = lz4_uncompress($payload); + else if($format === ConfigCacheEntry::FMT_BINARY) + $msg_packed = $payload; + else if($format === ConfigCacheEntry::FMT_FILE_REF) + $msg_packed = ensure_read($payload); + else + throw new Exception("Bad format: $format"); + return config_msgpack_unpack($msg_packed); } function config_make_standalone_ext_bundle(array $configs, $file_path) @@ -196,21 +229,6 @@ function config_make_standalone_ext_bundle(array $configs, $file_path) ensure_write($file_path, $packed_data); } -function config_filter_files(array $files) -{ - $configs = array(); - foreach($files as $file) - { - if(strpos($file, '.conf.js') === false || - strpos($file, '.bhl') !== false - ) - continue; - - $configs[] = $file; - } - return $configs; -} - function config_bench_load($file) { list($proto_id, $_) = config_ensure_header(config_base_dir(), $file); @@ -261,10 +279,24 @@ function config_save_includes_map(array $includes_map, $file = null) ensure_write($file, serialize($includes_map)); } -function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $includes_map_file = null, $max_workers = null) +class ConfigFetchResult +{ + public $all = array(); + public $by_id = array(); + public $by_path = array(); + public $by_alias = array(); +} + +function config_fetch_ex( + array $files, + $force_stale = false, + $verbose = false, + $includes_map_file = null, + $max_workers = null +) { if(!$files) - return array(); + return new ConfigFetchResult(); if($max_workers === null) $max_workers = sizeof($files) < 20 ? 1 : 4; @@ -295,13 +327,25 @@ function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $ if($serial) { - $results_by_job = array(); + $results_by_job = array(); foreach($jobs as $job) $results_by_job[] = _config_worker_func($job, $includes_map, $force_stale, $verbose); } + list($result, $total_stales) = _config_fetch_cache_ex($results_by_job); + + if($verbose) + echo "Miss/Hit: $total_stales/" . sizeof($result->all) . "\n"; + + config_save_includes_map($includes_map, $includes_map_file); + + return $result; +} + +function _config_fetch_cache_ex($results_by_job) +{ + $result = new ConfigFetchResult(); $total_stales = 0; - $cache_entries = array(); foreach($results_by_job as $results) { foreach($results as $file => $item) @@ -312,27 +356,32 @@ function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $ //NOTE: handling any cache corruption errors if($cache_entry === null) { - $cache_entry = _config_invalidate_cache($file, $cache_file); $is_stale = true; + $cache_entry = _config_invalidate_cache($file, $cache_file); + } + + $includes = $cache_entry->includes; + $includes_map[$file] = $includes; + + if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes)) + { + $is_stale = true; + $cache_entry = _config_invalidate_cache($file, $cache_file); } if($is_stale) ++$total_stales; - $includes_map[$file] = $cache_entry->includes; - //we want results to be returned in the same order, so //we store entries by the file key and later retrieve array values - $cache_entries[$file] = $cache_entry; + $result->by_path[$file] = $cache_entry; + $result->by_id[$cache_entry->id] = $cache_entry; + $result->by_alias[$cache_entry->strid] = $cache_entry; } } - if($verbose) - echo "Miss/Hit: $total_stales/" . sizeof($cache_entries) . "\n"; - - config_save_includes_map($includes_map, $includes_map_file); - - return array_values($cache_entries); + $result->all = array_values($result->by_path); + return array($result, $total_stales); } function _config_worker_run_procs(array $jobs, $includes_map_file, $force, $verbose) @@ -389,12 +438,12 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry throw new Exception("Bad proto_id: {$proto_id}"); $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $proto_id; - $GLOBALS['CONFIG_PREFAB_DEPS'] = array(); + $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create(); $GLOBALS['CONFIG_BHL_FUNCS'] = array(); $pres = config_parse(config_base_dir(), $file); if($pres->error !== 0) - throw new Exception("Error({$pres->error}) while loading JSON in file '{$file}':\n" . $pres->error_descr); + throw new Exception("Error({$pres->error}) while loading JSON in {$file}:\n" . $pres->error_descr); $includes = config_get_module_includes($pres->jsm_module); @@ -411,6 +460,8 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry $cache_entry->payload_file = $cache_payload_file; $cache_entry->file = normalize_path($file); $cache_entry->includes = $includes; + $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS']; + $cache_entry->refs = config_extract_refs($pres->normalized_jzon); ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry)); ensure_write($cache_payload_file, $payload_data); @@ -418,28 +469,27 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry return $cache_entry; } -function config_find_by_alias(array $cache_entries, $strid) : ConfigCacheEntry +function config_find_by_alias(ConfigFetchResult $cache_entries, $strid) : ConfigCacheEntry { - foreach($cache_entries as $ce) - if($ce->strid === $strid) - return $ce; + if(array_key_exists($strid, $cache_entries->by_alias)) + return $cache_entries->by_alias[$strid]; + throw new Exception("Failed to find config by alias '$strid'!"); return null; } -function config_find_by_id(array $cache_entries, $id) : ConfigCacheEntry +function config_find_by_id(ConfigFetchResult $cache_entries, $id) : ConfigCacheEntry { - foreach($cache_entries as $ce) - if($ce->id === $id) - return $ce; + if(array_key_exists($id, $cache_entries->by_id)) + return $cache_entries->by_id[$id]; + throw new Exception("Failed to find config by id '$id'!"); return null; } -function config_find_by_path(array $cache_entries, $path) : ConfigCacheEntry +function config_find_by_path(ConfigFetchResult $cache_entries, $path) : ConfigCacheEntry { - $path = normalize_path($path); - foreach($cache_entries as $ce) - if($ce->file === $path) - return $ce; + if(array_key_exists($path, $cache_entries->by_path)) + return $cache_entries->by_path[$path]; + throw new Exception("Failed to find config by path '$path'!"); return null; } @@ -448,7 +498,7 @@ function config_fetch_by_path(string $path, $force_stale = false, $use_cache = t $ces = config_fetch_ex(array($path), $force_stale, $verbose = false, $includes_map_file, null, $use_cache); if(!$ces) throw new Exception("Config not found at path '$path'"); - return $ces[0]; + return $ces->all[0]; } function config_fetch_all($force_stale = false, $use_cache = true, $includes_map_file = null) @@ -456,6 +506,33 @@ function config_fetch_all($force_stale = false, $use_cache = true, $includes_map return config_fetch_ex(config_scan_files(), $force_stale, $verbose = false, $includes_map_file, null, $use_cache); } +class ConfigCacheEntryExtras +{ + private static $klass = ConfigCacheEntryExtras::class; + + static function init($project_specific_klass) + { + self::$klass = $project_specific_klass; + } + + static function create() + { + return new self::$klass(); + } + + function export() + { + $as_array = get_object_vars($this); + return $as_array; + } + + function import($as_array) + { + foreach($as_array as $field_name => $field_value) + $this->$field_name = $field_value; + } +} + class ConfigCacheEntry { const FMT_BINARY = 0; @@ -471,10 +548,8 @@ class ConfigCacheEntry public $payload_file; public $file; public $includes = array(); - //TODO: do we need these? public $refs = array(); - public $prefab_deps = array(); - public $bhl_funcs = array(); + public $extras; public $_config; public $_payload; @@ -495,6 +570,11 @@ class ConfigCacheEntry return $ce; } + function __construct() + { + $this->extras = ConfigCacheEntryExtras::create(); + } + function __get($name) { if($name === "config") @@ -539,15 +619,15 @@ class ConfigCacheEntry $d[] = $this->payload_file; $d[] = $this->file; $d[] = $this->includes; - //TODO: do we need these? $d[] = $this->refs; - $d[] = $this->prefab_deps; - $d[] = $this->bhl_funcs; + $d[] = $this->extras->export(); return $d; } function import(array $d) { + $extras = array(); + list( $this->class, $this->class_id, @@ -557,11 +637,11 @@ class ConfigCacheEntry $this->payload_file, $this->file, $this->includes, - //TODO: do we need these? $this->refs, - $this->prefab_deps, - $this->bhl_funcs, + $extras, ) = $d; + + $this->extras->import($extras); } } @@ -616,6 +696,13 @@ function config_get_header($file, &$proto_id, &$alias) return false; } +function config_set_header($contents, $proto_id, $alias, &$is_success) +{ + $contents = preg_replace('~\s*\{~', "{ /* proto_id = {$proto_id} ; alias = {$alias} */", $contents, 1/*limit*/, $count); + $is_success = $count == 1; + return $contents; +} + function config_extract_header($contents) { return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents); @@ -633,10 +720,11 @@ function config_ensure_header($conf_dir, $file, $force = false) if($force || $curr_alias !== $alias) { $lines = file($file); + //TODO: why not using config_set_header(..) ? $lines[0] = preg_replace('~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{ /* proto_id = {$curr_proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count); if($count != 1) throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); - ensure_write($file, join("", $lines)); + ensure_write_if_differs($file, join("", $lines)); } return array($curr_proto_id, $alias); @@ -647,8 +735,9 @@ function config_ensure_header($conf_dir, $file, $force = false) $alias = config_get_strid($conf_dir, $file); $lines = file($file); - $lines[0] = preg_replace('~\s*\{~', "{ /* proto_id = {$proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count); - if($count != 1) + + $lines[0] = config_set_header($lines[0], $proto_id, $alias, $is_success); + if(!$is_success) throw new Exception("Could not set header for '$file' in line: {$lines[0]}"); ensure_write($file, join("", $lines)); @@ -693,11 +782,11 @@ function config_parse($conf_dir, $file) : ConfigParseResult { $res = new ConfigParseResult(); - $jsm = new \JSM($conf_dir, $file); $normalized_jzon = ''; try { + $jsm = new \JSM($conf_dir, $file); list($normalized_jzon, $jsm_module) = $jsm->process(); $res->normalized_jzon = $normalized_jzon; $res->jsm_module = $jsm_module;