Making code more robust and less specific
This commit is contained in:
Pavel Shevaev
parent
1d757658ff
commit
63e05c1c73
219
config.inc.php
219
config.inc.php
|
|
@ -31,6 +31,9 @@ task('config_worker', function(array $args)
|
|||
|
||||
function config_base_dir()
|
||||
{
|
||||
if(is("CONFIGS_BASE_DIR"))
|
||||
return get("CONFIGS_BASE_DIR");
|
||||
else
|
||||
return get("UNITY_ASSETS_DIR") . "/Configs";
|
||||
}
|
||||
|
||||
|
|
@ -139,6 +142,32 @@ function config_pack_bundle(array $cache_entries, $use_lz4 = false, $use_config_
|
|||
return $packed_data;
|
||||
}
|
||||
|
||||
function config_unpack_bundle($packed_data)
|
||||
{
|
||||
$packed_info = substr($packed_data, 0, 1+4+4);
|
||||
$info = unpack('Cformat/Vversion/Vheader_len', $packed_info);
|
||||
|
||||
if($info['format'] !== 1)
|
||||
throw new Exception("Unknown format: {$info['format']}");
|
||||
|
||||
$header_msgpack = substr($packed_data, 1+4+4, $info['header_len']);
|
||||
$header = config_msgpack_unpack($header_msgpack);
|
||||
|
||||
$payloads_bundle = substr($packed_data, 1+4+4 + $info['header_len']);
|
||||
|
||||
$entries = array();
|
||||
foreach($header as $item)
|
||||
{
|
||||
list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
|
||||
|
||||
$payload = substr($payloads_bundle, $offset, $size);
|
||||
|
||||
$entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
|
||||
}
|
||||
|
||||
return $entries;
|
||||
}
|
||||
|
||||
function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs)
|
||||
{
|
||||
$format = ConfigCacheEntry::FMT_BINARY;
|
||||
|
|
@ -155,21 +184,25 @@ function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs)
|
|||
if($use_lz4 && strlen($payload) > 512)
|
||||
{
|
||||
$format = ConfigCacheEntry::FMT_LZ4;
|
||||
$payload = lz4_compress($payload);
|
||||
$payload = lz4_compress($payload, 9);
|
||||
}
|
||||
}
|
||||
|
||||
return array($format, $payload);
|
||||
}
|
||||
|
||||
function config_sync_build_dir_with_prod()
|
||||
function _config_unpack_payload($format, $payload)
|
||||
{
|
||||
global $GAME_ROOT;
|
||||
|
||||
$build_ext_dir = config_get_build_ext_dir();
|
||||
ensure_mkdir($build_ext_dir);
|
||||
|
||||
ensure_sync($build_ext_dir, get("UNITY_ASSETS_DIR")."/Resources/ext_config/");
|
||||
$msg_packed = null;
|
||||
if($format === ConfigCacheEntry::FMT_LZ4)
|
||||
$msg_packed = lz4_uncompress($payload);
|
||||
else if($format === ConfigCacheEntry::FMT_BINARY)
|
||||
$msg_packed = $payload;
|
||||
else if($format === ConfigCacheEntry::FMT_FILE_REF)
|
||||
$msg_packed = ensure_read($payload);
|
||||
else
|
||||
throw new Exception("Bad format: $format");
|
||||
return config_msgpack_unpack($msg_packed);
|
||||
}
|
||||
|
||||
function config_make_standalone_ext_bundle(array $configs, $file_path)
|
||||
|
|
@ -196,21 +229,6 @@ function config_make_standalone_ext_bundle(array $configs, $file_path)
|
|||
ensure_write($file_path, $packed_data);
|
||||
}
|
||||
|
||||
function config_filter_files(array $files)
|
||||
{
|
||||
$configs = array();
|
||||
foreach($files as $file)
|
||||
{
|
||||
if(strpos($file, '.conf.js') === false ||
|
||||
strpos($file, '.bhl') !== false
|
||||
)
|
||||
continue;
|
||||
|
||||
$configs[] = $file;
|
||||
}
|
||||
return $configs;
|
||||
}
|
||||
|
||||
function config_bench_load($file)
|
||||
{
|
||||
list($proto_id, $_) = config_ensure_header(config_base_dir(), $file);
|
||||
|
|
@ -261,10 +279,24 @@ function config_save_includes_map(array $includes_map, $file = null)
|
|||
ensure_write($file, serialize($includes_map));
|
||||
}
|
||||
|
||||
function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $includes_map_file = null, $max_workers = null)
|
||||
class ConfigFetchResult
|
||||
{
|
||||
public $all = array();
|
||||
public $by_id = array();
|
||||
public $by_path = array();
|
||||
public $by_alias = array();
|
||||
}
|
||||
|
||||
function config_fetch_ex(
|
||||
array $files,
|
||||
$force_stale = false,
|
||||
$verbose = false,
|
||||
$includes_map_file = null,
|
||||
$max_workers = null
|
||||
)
|
||||
{
|
||||
if(!$files)
|
||||
return array();
|
||||
return new ConfigFetchResult();
|
||||
|
||||
if($max_workers === null)
|
||||
$max_workers = sizeof($files) < 20 ? 1 : 4;
|
||||
|
|
@ -300,8 +332,20 @@ function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $
|
|||
$results_by_job[] = _config_worker_func($job, $includes_map, $force_stale, $verbose);
|
||||
}
|
||||
|
||||
list($result, $total_stales) = _config_fetch_cache_ex($results_by_job);
|
||||
|
||||
if($verbose)
|
||||
echo "Miss/Hit: $total_stales/" . sizeof($result->all) . "\n";
|
||||
|
||||
config_save_includes_map($includes_map, $includes_map_file);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
function _config_fetch_cache_ex($results_by_job)
|
||||
{
|
||||
$result = new ConfigFetchResult();
|
||||
$total_stales = 0;
|
||||
$cache_entries = array();
|
||||
foreach($results_by_job as $results)
|
||||
{
|
||||
foreach($results as $file => $item)
|
||||
|
|
@ -312,27 +356,32 @@ function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $
|
|||
//NOTE: handling any cache corruption errors
|
||||
if($cache_entry === null)
|
||||
{
|
||||
$cache_entry = _config_invalidate_cache($file, $cache_file);
|
||||
$is_stale = true;
|
||||
$cache_entry = _config_invalidate_cache($file, $cache_file);
|
||||
}
|
||||
|
||||
$includes = $cache_entry->includes;
|
||||
$includes_map[$file] = $includes;
|
||||
|
||||
if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes))
|
||||
{
|
||||
$is_stale = true;
|
||||
$cache_entry = _config_invalidate_cache($file, $cache_file);
|
||||
}
|
||||
|
||||
if($is_stale)
|
||||
++$total_stales;
|
||||
|
||||
$includes_map[$file] = $cache_entry->includes;
|
||||
|
||||
//we want results to be returned in the same order, so
|
||||
//we store entries by the file key and later retrieve array values
|
||||
$cache_entries[$file] = $cache_entry;
|
||||
$result->by_path[$file] = $cache_entry;
|
||||
$result->by_id[$cache_entry->id] = $cache_entry;
|
||||
$result->by_alias[$cache_entry->strid] = $cache_entry;
|
||||
}
|
||||
}
|
||||
|
||||
if($verbose)
|
||||
echo "Miss/Hit: $total_stales/" . sizeof($cache_entries) . "\n";
|
||||
|
||||
config_save_includes_map($includes_map, $includes_map_file);
|
||||
|
||||
return array_values($cache_entries);
|
||||
$result->all = array_values($result->by_path);
|
||||
return array($result, $total_stales);
|
||||
}
|
||||
|
||||
function _config_worker_run_procs(array $jobs, $includes_map_file, $force, $verbose)
|
||||
|
|
@ -389,12 +438,12 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry
|
|||
throw new Exception("Bad proto_id: {$proto_id}");
|
||||
|
||||
$GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $proto_id;
|
||||
$GLOBALS['CONFIG_PREFAB_DEPS'] = array();
|
||||
$GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
|
||||
$GLOBALS['CONFIG_BHL_FUNCS'] = array();
|
||||
|
||||
$pres = config_parse(config_base_dir(), $file);
|
||||
if($pres->error !== 0)
|
||||
throw new Exception("Error({$pres->error}) while loading JSON in file '{$file}':\n" . $pres->error_descr);
|
||||
throw new Exception("Error({$pres->error}) while loading JSON in {$file}:\n" . $pres->error_descr);
|
||||
|
||||
$includes = config_get_module_includes($pres->jsm_module);
|
||||
|
||||
|
|
@ -411,6 +460,8 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry
|
|||
$cache_entry->payload_file = $cache_payload_file;
|
||||
$cache_entry->file = normalize_path($file);
|
||||
$cache_entry->includes = $includes;
|
||||
$cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
|
||||
$cache_entry->refs = config_extract_refs($pres->normalized_jzon);
|
||||
|
||||
ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
|
||||
ensure_write($cache_payload_file, $payload_data);
|
||||
|
|
@ -418,28 +469,27 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry
|
|||
return $cache_entry;
|
||||
}
|
||||
|
||||
function config_find_by_alias(array $cache_entries, $strid) : ConfigCacheEntry
|
||||
function config_find_by_alias(ConfigFetchResult $cache_entries, $strid) : ConfigCacheEntry
|
||||
{
|
||||
foreach($cache_entries as $ce)
|
||||
if($ce->strid === $strid)
|
||||
return $ce;
|
||||
if(array_key_exists($strid, $cache_entries->by_alias))
|
||||
return $cache_entries->by_alias[$strid];
|
||||
throw new Exception("Failed to find config by alias '$strid'!");
|
||||
return null;
|
||||
}
|
||||
|
||||
function config_find_by_id(array $cache_entries, $id) : ConfigCacheEntry
|
||||
function config_find_by_id(ConfigFetchResult $cache_entries, $id) : ConfigCacheEntry
|
||||
{
|
||||
foreach($cache_entries as $ce)
|
||||
if($ce->id === $id)
|
||||
return $ce;
|
||||
if(array_key_exists($id, $cache_entries->by_id))
|
||||
return $cache_entries->by_id[$id];
|
||||
throw new Exception("Failed to find config by id '$id'!");
|
||||
return null;
|
||||
}
|
||||
|
||||
function config_find_by_path(array $cache_entries, $path) : ConfigCacheEntry
|
||||
function config_find_by_path(ConfigFetchResult $cache_entries, $path) : ConfigCacheEntry
|
||||
{
|
||||
$path = normalize_path($path);
|
||||
foreach($cache_entries as $ce)
|
||||
if($ce->file === $path)
|
||||
return $ce;
|
||||
if(array_key_exists($path, $cache_entries->by_path))
|
||||
return $cache_entries->by_path[$path];
|
||||
throw new Exception("Failed to find config by path '$path'!");
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -448,7 +498,7 @@ function config_fetch_by_path(string $path, $force_stale = false, $use_cache = t
|
|||
$ces = config_fetch_ex(array($path), $force_stale, $verbose = false, $includes_map_file, null, $use_cache);
|
||||
if(!$ces)
|
||||
throw new Exception("Config not found at path '$path'");
|
||||
return $ces[0];
|
||||
return $ces->all[0];
|
||||
}
|
||||
|
||||
function config_fetch_all($force_stale = false, $use_cache = true, $includes_map_file = null)
|
||||
|
|
@ -456,6 +506,33 @@ function config_fetch_all($force_stale = false, $use_cache = true, $includes_map
|
|||
return config_fetch_ex(config_scan_files(), $force_stale, $verbose = false, $includes_map_file, null, $use_cache);
|
||||
}
|
||||
|
||||
class ConfigCacheEntryExtras
|
||||
{
|
||||
private static $klass = ConfigCacheEntryExtras::class;
|
||||
|
||||
static function init($project_specific_klass)
|
||||
{
|
||||
self::$klass = $project_specific_klass;
|
||||
}
|
||||
|
||||
static function create()
|
||||
{
|
||||
return new self::$klass();
|
||||
}
|
||||
|
||||
function export()
|
||||
{
|
||||
$as_array = get_object_vars($this);
|
||||
return $as_array;
|
||||
}
|
||||
|
||||
function import($as_array)
|
||||
{
|
||||
foreach($as_array as $field_name => $field_value)
|
||||
$this->$field_name = $field_value;
|
||||
}
|
||||
}
|
||||
|
||||
class ConfigCacheEntry
|
||||
{
|
||||
const FMT_BINARY = 0;
|
||||
|
|
@ -471,10 +548,8 @@ class ConfigCacheEntry
|
|||
public $payload_file;
|
||||
public $file;
|
||||
public $includes = array();
|
||||
//TODO: do we need these?
|
||||
public $refs = array();
|
||||
public $prefab_deps = array();
|
||||
public $bhl_funcs = array();
|
||||
public $extras;
|
||||
|
||||
public $_config;
|
||||
public $_payload;
|
||||
|
|
@ -495,6 +570,11 @@ class ConfigCacheEntry
|
|||
return $ce;
|
||||
}
|
||||
|
||||
function __construct()
|
||||
{
|
||||
$this->extras = ConfigCacheEntryExtras::create();
|
||||
}
|
||||
|
||||
function __get($name)
|
||||
{
|
||||
if($name === "config")
|
||||
|
|
@ -539,15 +619,15 @@ class ConfigCacheEntry
|
|||
$d[] = $this->payload_file;
|
||||
$d[] = $this->file;
|
||||
$d[] = $this->includes;
|
||||
//TODO: do we need these?
|
||||
$d[] = $this->refs;
|
||||
$d[] = $this->prefab_deps;
|
||||
$d[] = $this->bhl_funcs;
|
||||
$d[] = $this->extras->export();
|
||||
return $d;
|
||||
}
|
||||
|
||||
function import(array $d)
|
||||
{
|
||||
$extras = array();
|
||||
|
||||
list(
|
||||
$this->class,
|
||||
$this->class_id,
|
||||
|
|
@ -557,11 +637,11 @@ class ConfigCacheEntry
|
|||
$this->payload_file,
|
||||
$this->file,
|
||||
$this->includes,
|
||||
//TODO: do we need these?
|
||||
$this->refs,
|
||||
$this->prefab_deps,
|
||||
$this->bhl_funcs,
|
||||
$extras,
|
||||
) = $d;
|
||||
|
||||
$this->extras->import($extras);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -616,6 +696,13 @@ function config_get_header($file, &$proto_id, &$alias)
|
|||
return false;
|
||||
}
|
||||
|
||||
function config_set_header($contents, $proto_id, $alias, &$is_success)
|
||||
{
|
||||
$contents = preg_replace('~\s*\{~', "{ /* proto_id = {$proto_id} ; alias = {$alias} */", $contents, 1/*limit*/, $count);
|
||||
$is_success = $count == 1;
|
||||
return $contents;
|
||||
}
|
||||
|
||||
function config_extract_header($contents)
|
||||
{
|
||||
return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents);
|
||||
|
|
@ -633,10 +720,11 @@ function config_ensure_header($conf_dir, $file, $force = false)
|
|||
if($force || $curr_alias !== $alias)
|
||||
{
|
||||
$lines = file($file);
|
||||
//TODO: why not using config_set_header(..) ?
|
||||
$lines[0] = preg_replace('~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{ /* proto_id = {$curr_proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count);
|
||||
if($count != 1)
|
||||
throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
|
||||
ensure_write($file, join("", $lines));
|
||||
ensure_write_if_differs($file, join("", $lines));
|
||||
}
|
||||
|
||||
return array($curr_proto_id, $alias);
|
||||
|
|
@ -647,8 +735,9 @@ function config_ensure_header($conf_dir, $file, $force = false)
|
|||
$alias = config_get_strid($conf_dir, $file);
|
||||
|
||||
$lines = file($file);
|
||||
$lines[0] = preg_replace('~\s*\{~', "{ /* proto_id = {$proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count);
|
||||
if($count != 1)
|
||||
|
||||
$lines[0] = config_set_header($lines[0], $proto_id, $alias, $is_success);
|
||||
if(!$is_success)
|
||||
throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
|
||||
ensure_write($file, join("", $lines));
|
||||
|
||||
|
|
@ -693,11 +782,11 @@ function config_parse($conf_dir, $file) : ConfigParseResult
|
|||
{
|
||||
$res = new ConfigParseResult();
|
||||
|
||||
$jsm = new \JSM($conf_dir, $file);
|
||||
|
||||
$normalized_jzon = '';
|
||||
try
|
||||
{
|
||||
$jsm = new \JSM($conf_dir, $file);
|
||||
list($normalized_jzon, $jsm_module) = $jsm->process();
|
||||
$res->normalized_jzon = $normalized_jzon;
|
||||
$res->jsm_module = $jsm_module;
|
||||
|
|
|
|||
Loading…
Reference in New Issue