taskman_config/config.inc.php

911 lines
22 KiB
PHP
Raw Normal View History

2022-05-16 14:21:18 +03:00
<?php
namespace taskman;
$GLOBALS['CONFIG_GLOBAL_DEPS'] = array();
$GLOBALS['CONFIG_INIT_WORKER_FUNC'] = null;
task('config_worker', function(array $args)
{
if(sizeof($args) != 3)
throw new Exception("Config worker args not set");
$in_file = $args[0];
$out_file = $args[1];
$err_file = $args[2];
try
{
list($job, $includes_map_file, $force, $verbose) = unserialize(ensure_read($in_file));
$result = _config_worker_func($job, config_load_includes_map($includes_map_file), $force, $verbose);
ensure_write($out_file, serialize($result));
}
catch(Exception $e)
{
//NOTE: explicitely catching all exceptions and writing to the error file
// since under Windows error file stream redirect may work unreliably
file_put_contents($err_file, $e->getMessage() . "\n" . $e->getTraceAsString(), FILE_APPEND);
throw $e;
}
});
function config_base_dir()
{
global $GAME_ROOT;
return "$GAME_ROOT/unity/Assets/Configs/";
}
function config_build_dir()
{
global $GAME_ROOT;
return "$GAME_ROOT/build/tmp/";
}
function config_set_global_deps(array $deps)
{
global $CONFIG_GLOBAL_DEPS;
$CONFIG_GLOBAL_DEPS = array();
foreach($deps as $d)
{
if(is_file($d))
$CONFIG_GLOBAL_DEPS[] = normalize_path($d);
}
}
function config_set_worker_init_fn($fn)
{
global $CONFIG_INIT_WORKER_FUNC;
$CONFIG_INIT_WORKER_FUNC = $fn;
}
function config_get_global_deps()
{
global $CONFIG_GLOBAL_DEPS;
return $CONFIG_GLOBAL_DEPS;
}
function config_get_build_ext_dir()
{
global $GAME_ROOT;
return "$GAME_ROOT/build/ext_config/";
}
function config_get_bundle_ext_path()
{
return config_get_build_ext_dir() . "/configs/" . get("CONFIGS_BUNDLE_NAME");
}
function config_scan_files()
{
return scan_files_rec(array(config_base_dir()), array('conf.js'));
}
function config_pack_bundle(array $cache_entries, $use_lz4 = false, $use_config_refs = false)
{
global $GAME_ROOT;
$binary_format = 1;
$version = game_version_code();
$MAP = array();
$STRIDMAP = array();
$payloads = array();
$payloads_offset = 0;
foreach($cache_entries as $entry)
{
list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
$payload_size = strlen($payload);
$payloads[] = array($payloads_offset, $payload, $format, $payload_size);
$payloads_offset += $payload_size;
}
$header = array();
foreach($cache_entries as $idx => $entry)
{
if(isset($MAP[$entry->id]))
throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
$MAP[$entry->id] = $entry->strid;
$strid_crc = crc32($entry->strid);
if(isset($STRIDMAP[$strid_crc]))
throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
$STRIDMAP[$strid_crc] = $entry->strid;
$header[] = array(
$payloads[$idx][2],
$entry->id,
crc32($entry->strid),
$entry->class_id,
$payloads[$idx][0],
$payloads[$idx][3]
);
}
$header_msgpack = config_msgpack_pack($header);
$payloads_bundle = '';
foreach($payloads as $item)
$payloads_bundle .= $item[1];
$packed_data =
pack("C", $binary_format) .
pack("V", $version) .
pack("V", strlen($header_msgpack)) .
$header_msgpack .
$payloads_bundle;
echo "CONF.BUNDLE: entries " . sizeof($cache_entries) . "; total " . kb($packed_data) .
"; lz4 $use_lz4; refs $use_config_refs; CRC " . crc32($packed_data) . "\n";
return $packed_data;
}
function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs)
{
$format = ConfigCacheEntry::FMT_BINARY;
$payload = null;
if($use_config_refs && $ce->payload_file)
{
$format = ConfigCacheEntry::FMT_FILE_REF;
$payload = $ce->payload_file;
}
else
{
$payload = $ce->payload;
if($use_lz4 && strlen($payload) > 512)
{
$format = ConfigCacheEntry::FMT_LZ4;
$payload = lz4_compress($payload);
}
}
return array($format, $payload);
}
function config_sync_build_dir_with_prod()
{
global $GAME_ROOT;
$build_ext_dir = config_get_build_ext_dir();
ensure_mkdir($build_ext_dir);
ensure_sync($build_ext_dir, "$GAME_ROOT/unity/Assets/Resources/ext_config/");
}
function config_make_standalone_ext_bundle(array $configs, $file_path)
{
$cache_entries = array();
foreach($configs as $conf)
{
$payload = config_msgpack_pack($conf->export());
//creating fake cache entries
$entry = new ConfigCacheEntry();
$entry->id = $conf->id;
$entry->strid = $conf->strid;
$entry->class_id = $conf->getClassId();
$entry->class = get_class($conf);
$entry->payload = $payload;
$entry->config = $conf;
$cache_entries[] = $entry;
}
$packed_data = config_pack_bundle($cache_entries, get('USE_LZ4_CONFIGS'));
ensure_write($file_path, $packed_data);
}
function config_filter_files(array $files)
{
$configs = array();
foreach($files as $file)
{
if(strpos($file, '.conf.js') === false ||
strpos($file, '.bhl') !== false
)
continue;
$configs[] = $file;
}
return $configs;
}
function config_bench_load($file)
{
list($proto_id, $_) = config_ensure_header(config_base_dir(), $file);
if(!$proto_id)
throw new Exception("Bad proto_id: {$proto_id}");
$t = microtime(true);
$parse_res = config_parse(config_base_dir(), $file);
if($parse_res->error !== 0)
throw new Exception("Error({$parse_res->error}) while loading JSON from {$file}:\n" . $parse_res->error_descr);
echo "PARSE: " . (microtime(true) - $t) . "\n";
$t = microtime(true);
$config = config_load_ex(config_base_dir(), $file, $parse_res->parsed_arr, $proto_id);
echo "LOAD: " . (microtime(true) - $t) . "\n";
}
function config_get_tmp_build_path($file)
{
$name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file))));
$name = ltrim($name, "-");
return normalize_path(config_build_dir() . "/$name");
}
function config_get_includes_map_path()
{
return config_build_dir() . "/includes.map";
}
function config_load_includes_map($file = null)
{
$file = $file ? $file : config_get_includes_map_path();
$includes_map = array(array(), array());
if(is_file($file))
{
$tmp_map = @unserialize(ensure_read($file));
if(is_array($tmp_map))
$includes_map = $tmp_map;
}
return $includes_map;
}
function config_save_includes_map(array $includes_map, $file = null)
{
$file = $file ? $file : config_get_includes_map_path();
ensure_write($file, serialize($includes_map));
}
function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $includes_map_file = null, $max_workers = null)
{
if(!$files)
return array();
if($max_workers === null)
$max_workers = sizeof($files) < 20 ? 1 : 4;
$includes_map_file = $includes_map_file ? $includes_map_file : config_get_includes_map_path();
$includes_map = config_load_includes_map($includes_map_file);
$chunk_size = ceil(sizeof($files)/$max_workers);
$jobs = array();
foreach(array_chunk($files, $chunk_size) as $idx => $chunk_files)
$jobs[] = array($idx, $chunk_files);
$results_by_job = null;
$serial = $max_workers == 1;
if(!$serial)
{
$results_by_job = _config_worker_run_procs($jobs, $includes_map_file, $force_stale, $verbose);
//in case of any result error try serial processing
if(array_search(false, $results_by_job, true/*strict*/) !== false)
{
if($verbose)
echo "Result error detected, trying serial processing...\n";
$serial = true;
}
}
if($serial)
{
$results_by_job = array();
foreach($jobs as $job)
$results_by_job[] = _config_worker_func($job, $includes_map, $force_stale, $verbose);
}
$total_stales = 0;
$cache_entries = array();
foreach($results_by_job as $results)
{
foreach($results as $file => $item)
{
list($cache_file, $is_stale) = $item;
$cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file));
//NOTE: handling any cache corruption errors
if($cache_entry === null)
{
$cache_entry = _config_invalidate_cache($file, $cache_file);
$is_stale = true;
}
if($is_stale)
++$total_stales;
$includes_map[$file] = $cache_entry->includes;
//we want results to be returned in the same order, so
//we store entries by the file key and later retrieve array values
$cache_entries[$file] = $cache_entry;
}
}
if($verbose)
echo "Miss/Hit: $total_stales/" . sizeof($cache_entries) . "\n";
config_save_includes_map($includes_map, $includes_map_file);
return array_values($cache_entries);
}
function _config_worker_run_procs(array $jobs, $includes_map_file, $force, $verbose)
{
$worker_args = array();
foreach($jobs as $idx => $job)
$worker_args[] = array($job, $includes_map_file, $force, $verbose);
return run_background_gamectl_workers('config_worker', $worker_args);
}
function _config_worker_func(array $job, array $includes_map, $force, $verbose)
{
global $CONFIG_INIT_WORKER_FUNC;
if(is_callable($CONFIG_INIT_WORKER_FUNC))
$CONFIG_INIT_WORKER_FUNC();
list($idx, $files) = $job;
if($verbose)
echo "Worker $idx (" . sizeof($files) . ") started\n";
$results = array();
foreach($files as $file_idx => $file)
{
if($verbose && $file_idx > 0 && ($file_idx % 500) == 0)
echo "Worker $idx progress: " . round($file_idx / sizeof($files) * 100) . "% ...\n";
$cache_file = config_get_cache_path($file);
$is_stale = true;
if(!$force)
{
$file_deps = array($file);
if(isset($includes_map[$file]))
$file_deps = array_merge($file_deps, $includes_map[$file]);
$is_stale = need_to_regen($cache_file, $file_deps);
}
if($is_stale)
_config_invalidate_cache($file, $cache_file);
$results[$file] = array($cache_file, $is_stale);
}
if($verbose)
echo "Worker $idx done\n";
return $results;
}
function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry
{
$cache_payload_file = config_get_cache_payload_path($file);
list($proto_id, $_) = config_ensure_header(config_base_dir(), $file);
if(!$proto_id)
throw new Exception("Bad proto_id: {$proto_id}");
$GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $proto_id;
$GLOBALS['CONFIG_PREFAB_DEPS'] = array();
$GLOBALS['CONFIG_BHL_FUNCS'] = array();
$pres = config_parse(config_base_dir(), $file);
if($pres->error !== 0)
throw new Exception("Error({$pres->error}) while loading JSON from {$file}:\n" . $pres->error_descr);
$includes = config_get_module_includes($pres->jsm_module);
$config = config_load_ex(config_base_dir(), $file, $pres->parsed_arr, $proto_id);
$payload_data = config_msgpack_pack($config->export());
$cache_entry = new ConfigCacheEntry();
$cache_entry->id = $config->id;
$cache_entry->config = $config;
$cache_entry->strid = $config->strid;
$cache_entry->cache_file = $cache_file;
$cache_entry->class = get_class($config);
$cache_entry->class_id = $config->getClassId();
$cache_entry->payload_file = $cache_payload_file;
$cache_entry->file = normalize_path($file);
$cache_entry->includes = $includes;
ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
ensure_write($cache_payload_file, $payload_data);
return $cache_entry;
}
function config_find_by_alias(array $cache_entries, $strid) : ConfigCacheEntry
{
foreach($cache_entries as $ce)
if($ce->strid === $strid)
return $ce;
return null;
}
function config_find_by_id(array $cache_entries, $id) : ConfigCacheEntry
{
foreach($cache_entries as $ce)
if($ce->id === $id)
return $ce;
return null;
}
function config_find_by_path(array $cache_entries, $path) : ConfigCacheEntry
{
$path = normalize_path($path);
foreach($cache_entries as $ce)
if($ce->file === $path)
return $ce;
return null;
}
function config_fetch_by_path(string $path, $force_stale = false, $use_cache = true, $includes_map_file = null) : ConfigCacheEntry
{
$ces = config_fetch_ex(array($path), $force_stale, $verbose = false, $includes_map_file, null, $use_cache);
if(!$ces)
throw new Exception("Config not found at path '$path'");
return $ces[0];
}
function config_fetch_all($force_stale = false, $use_cache = true, $includes_map_file = null)
{
return config_fetch_ex(config_scan_files(), $force_stale, $verbose = false, $includes_map_file, null, $use_cache);
}
class ConfigCacheEntry
{
const FMT_BINARY = 0;
const FMT_LZ4 = 1;
const FMT_FILE_REF = 2;
public $class;
public $class_id;
public $id;
public $strid;
public $cache_file;
//NOTE: actual payload is stored in a separate file for faster incremental retrievals
public $payload_file;
public $file;
public $includes = array();
//TODO: do we need these?
public $refs = array();
public $prefab_deps = array();
public $bhl_funcs = array();
public $_config;
public $_payload;
static function serialize($ce)
{
$d = $ce->export();
return serialize($d);
}
static function unserialize($str)
{
$d = @unserialize($str);
if(!is_array($d))
return null;
$ce = new ConfigCacheEntry();
$ce->import($d);
return $ce;
}
function __get($name)
{
if($name === "config")
{
if($this->_config === null)
{
$klass = $this->class;
$data = config_msgpack_unpack($this->payload);
$this->_config = new $klass($data);
}
return $this->_config;
}
else if($name === "payload")
{
if($this->_payload !== null)
return $this->_payload;
//if payload not set directly not storing it
return ensure_read($this->payload_file);
}
else
throw new Exception("No such property '$name'");
}
function __set($name, $v)
{
if($name === "config")
$this->_config = $v;
else if($name === "payload")
$this->_payload = $v;
else
throw new Exception("No such property '$name'");
}
function export()
{
$d = array();
$d[] = $this->class;
$d[] = $this->class_id;
$d[] = $this->id;
$d[] = $this->strid;
$d[] = $this->cache_file;
$d[] = $this->payload_file;
$d[] = $this->file;
$d[] = $this->includes;
//TODO: do we need these?
$d[] = $this->refs;
$d[] = $this->prefab_deps;
$d[] = $this->bhl_funcs;
return $d;
}
function import(array $d)
{
list(
$this->class,
$this->class_id,
$this->id,
$this->strid,
$this->cache_file,
$this->payload_file,
$this->file,
$this->includes,
//TODO: do we need these?
$this->refs,
$this->prefab_deps,
$this->bhl_funcs,
) = $d;
}
}
function config_get_cache_path($file)
{
return config_get_tmp_build_path($file . '.cacheb');
}
function config_get_cache_payload_path($file)
{
return config_get_tmp_build_path($file . '.pdata');
}
function config_get_id($conf_dir, $file_path)
{
return config_file2id($conf_dir, $file_path);
}
function config_get_strid($conf_dir, $file_path)
{
return config_file2strid($conf_dir, $file_path);
}
function config_extract_refs($src, $as_map = true)
{
$refs = array();
if(preg_match_all('~"(@[^"]+)"~', $src, $matches))
{
foreach($matches[1] as $ref)
{
if(!isset($refs[$ref]))
$refs[$ref] = 1;
++$refs[$ref];
}
}
return $as_map ? $refs : array_keys($refs);
}
function config_get_header($file, &$proto_id, &$alias)
{
$h = fopen($file, "r");
$line = fgets($h, 256);
fclose($h);
if(preg_match('~\{\s*/\*\s*proto_id\s*=\s*(\d+)\s*;\s*alias\s*=\s*([^\s]+)~', $line, $matches))
{
$proto_id = (int)$matches[1];
$alias = $matches[2];
return true;
}
else
return false;
}
function config_extract_header($contents)
{
return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents);
}
function config_ensure_header($conf_dir, $file, $force = false)
{
if(config_get_header($file, $curr_proto_id, $curr_alias))
{
$alias = config_get_strid($conf_dir, $file);
if($force)
$curr_proto_id = config_get_id($conf_dir, $file);
//NOTE: keeping current proto id intact if not forced
if($force || $curr_alias !== $alias)
{
$lines = file($file);
$lines[0] = preg_replace('~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{ /* proto_id = {$curr_proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count);
if($count != 1)
throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
ensure_write($file, join("", $lines));
}
return array($curr_proto_id, $alias);
}
else
{
$proto_id = config_get_id($conf_dir, $file);
$alias = config_get_strid($conf_dir, $file);
$lines = file($file);
$lines[0] = preg_replace('~\s*\{~', "{ /* proto_id = {$proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count);
if($count != 1)
throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
ensure_write($file, join("", $lines));
return array($proto_id, $alias);
}
}
function config_load($conf_dir, $conf_path)
{
list($proto_id, $_) = config_ensure_header($conf_dir, $conf_path);
if(!$proto_id)
throw new Exception("Bad proto_id: {$proto_id}");
$pres = config_parse($conf_dir, $conf_path);
if($pres->error !== 0)
throw new Exception("Error({$pres->error}) while loading JSON from {$conf_path}:\n" . $pres->error_descr);
return config_load_ex($conf_dir, $conf_path, $pres->parsed_arr, $proto_id);
}
function conf2json($conf, $json_flags = 0)
{
$arr = $conf->export(true);
$arr['class'] = get_class($conf);
unset($arr['id']);
unset($arr['strid']);
$json = json_encode($arr, $json_flags);
$json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json);
return $json;
}
class ConfigParseResult
{
public $error = 0;
public $error_descr;
public $normalized_jzon = '';
public $parsed_arr;
public $jsm_module;
}
function config_parse($conf_dir, $file) : ConfigParseResult
{
$res = new ConfigParseResult();
$jsm = new \JSM($conf_dir, $file);
$normalized_jzon = '';
try
{
list($normalized_jzon, $jsm_module) = $jsm->process();
$res->normalized_jzon = $normalized_jzon;
$res->jsm_module = $jsm_module;
}
catch(Exception $e)
{
$res->error = 1;
$res->error_descr = "File '$file':\n" . $e->getMessage() . "\n" . $e->getTraceAsString();
return $res;
}
$decode_res = config_check_and_decode_jzon($normalized_jzon);
if($decode_res[0] === 0)
{
$res->parsed_arr = $decode_res[2];
}
else
{
$res->error = $decode_res[0];
$res->error_descr = "File '$file':\n" . $decode_res[1];
}
return $res;
}
function config_check_and_decode_jzon($json)
{
try
{
2022-05-16 17:29:16 +03:00
$arr = \jzon_parse($json);
2022-05-16 14:21:18 +03:00
return array(0, "", $arr);
}
catch(Exception $e)
{
return array(1, $e->getMessage(), array());
}
}
function config_load_ex($conf_dir, $file, array $arr, $id = null)
{
if(!isset($arr['class']) || !isset($arr['class'][0]))
throw new Exception("Class is not set. File '$file'.");
$klass = $arr['class'];
unset($arr['class']);
if($id === null)
$id = config_file2id($conf_dir, $file);
$cnf = null;
try
{
$cnf = new $klass;
$cnf->import($arr, true);
}
catch(Exception $e)
{
throw new Exception($e->getMessage() . ". File '{$file}'"/* . $e->getTraceAsString()*/);
}
$cnf->id = $id;
$cnf->strid = config_file2strid($conf_dir, $file);
return $cnf;
}
function config_is_file($filename)
{
return (strrpos($filename, '.conf.js') === (strlen($filename) - 8));
}
function config_file2id($conf_dir, $filename)
{
$nfilename = config_make_path($conf_dir, $filename);
return config_crc28($nfilename);
}
function config_file2strid($conf_dir, $filename)
{
$strid = config_make_path($conf_dir, $filename);
$fname_idx = strrpos($strid, "/");
$fname_idx = strpos($strid, ".", $fname_idx);
$strid = substr($strid, 0, $fname_idx);
return "@".$strid;
}
function config_make_path($conf_dir, $path)
{
return ltrim(str_replace(normalize_path($conf_dir, true/*nix*/),
'',
normalize_path($path, true/*nix*/)),
'/');
}
function config_str2id($str)
{
if(strpos($str, '@') === 0)
$str = substr($str, 1) . '.conf.js';
return config_crc28($str);
}
function config_crc28($what)
{
return crc32($what) & 0xFFFFFFF;
}
function config_get_module_includes(\JSM_Module $cm)
{
$includes = array();
foreach($cm->getIncludes() as $include => $_)
{
//maybe we should take .php includes into account as well?
if(config_str_ends_with($include, ".php"))
continue;
$includes[] = $include;
}
return $includes;
}
function config_includes_map_find_text_origin(array $map, $file, $text)
{
if(!isset($map[$file]))
return array($file);
$tpls = $map[$file];
$tpls[] = $file;
$res = array();
foreach($tpls as $tpl)
{
$content = ensure_read($tpl);
$content = i18n_decode_string($content);
if(strpos($content, $text) !== FALSE)
$res[] = $tpl;
}
return $res;
}
function config_str_replace_in_files($subj, $repl, array $files)
{
foreach($files as $file)
{
$contents = ensure_read($file);
$new_contents = str_replace($subj, $repl, $contents);
if($new_contents !== $contents)
ensure_write($file, $new_contents);
}
}
function config_str_ends_with($haystack, $needle)
{
// search forward starting from end minus needle length characters
return $needle === "" || (($temp = strlen($haystack) - strlen($needle)) >= 0 && strpos($haystack, $needle, $temp) !== false);
}
function config_walk_fields($proto, $callback)
{
if(!method_exists($proto, 'CLASS_FIELDS_PROPS'))
return;
foreach($proto->CLASS_FIELDS_PROPS() as $field => $tokens)
{
$value = $proto->$field;
$callback($proto, $field, $value, $tokens);
if(is_object($value))
config_walk_fields($value, $callback);
else if(is_array($value))
{
foreach($value as $num => $item)
{
if(is_object($item))
config_walk_fields($item, $callback);
}
}
}
}
if(function_exists('msgpack_pack'))
{
function config_msgpack_pack($data)
{
$prev = ini_set('msgpack.use_str8_serialization', '0');
$res = msgpack_pack($data);
if($prev != '0')
ini_set('msgpack.use_str8_serialization', $prev);
return $res;
}
function config_msgpack_unpack($data)
{
return msgpack_unpack($data);
}
}
else
{
function config_msgpack_pack($data)
{
2022-05-16 17:52:12 +03:00
include_once(__DIR__ . '/msgpack_custom.inc.php');
2022-05-16 14:21:18 +03:00
2022-05-16 17:52:12 +03:00
$packer = new \MessagePackCustom();
2022-05-16 14:21:18 +03:00
return $packer->pack($data);
}
function config_msgpack_unpack($data)
{
return MessagePack\MessagePack::unpack($data);
}
}