Making code more robust and less specific

config.inc.php

@@ -31,7 +31,10 @@ task('config_worker', function(array $args)
 
 function config_base_dir()
 {
-  return get("UNITY_ASSETS_DIR") . "/Configs";
+  if(is("CONFIGS_BASE_DIR"))
+    return get("CONFIGS_BASE_DIR");
+  else
+    return get("UNITY_ASSETS_DIR") . "/Configs";
 }
 
 function config_build_dir()
@@ -139,6 +142,32 @@ function config_pack_bundle(array $cache_entries, $use_lz4 = false, $use_config_
   return $packed_data;
 }
 
+function config_unpack_bundle($packed_data)
+{
+  $packed_info = substr($packed_data, 0, 1+4+4);
+  $info = unpack('Cformat/Vversion/Vheader_len', $packed_info);
+
+  if($info['format'] !== 1)
+    throw new Exception("Unknown format: {$info['format']}");
+
+  $header_msgpack = substr($packed_data, 1+4+4, $info['header_len']); 
+  $header = config_msgpack_unpack($header_msgpack);
+
+  $payloads_bundle = substr($packed_data, 1+4+4 + $info['header_len']);
+  
+  $entries = array();
+  foreach($header as $item)
+  {
+    list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
+
+    $payload = substr($payloads_bundle, $offset, $size);
+
+    $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+  }
+
+  return $entries;
+}
+
 function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs)
 {
   $format = ConfigCacheEntry::FMT_BINARY;
@@ -155,21 +184,25 @@ function _config_get_payload(ConfigCacheEntry $ce, $use_lz4, $use_config_refs)
     if($use_lz4 && strlen($payload) > 512)
     {
       $format = ConfigCacheEntry::FMT_LZ4;
-      $payload = lz4_compress($payload);
+      $payload = lz4_compress($payload, 9);
     }
   }
 
   return array($format, $payload);
 }
 
-function config_sync_build_dir_with_prod()
+function _config_unpack_payload($format, $payload)
 {
-  global $GAME_ROOT;
+  $msg_packed = null;
-
+  if($format === ConfigCacheEntry::FMT_LZ4)
-  $build_ext_dir = config_get_build_ext_dir(); 
+    $msg_packed = lz4_uncompress($payload);
-  ensure_mkdir($build_ext_dir);
+  else if($format === ConfigCacheEntry::FMT_BINARY)
-
+    $msg_packed = $payload;
-  ensure_sync($build_ext_dir, get("UNITY_ASSETS_DIR")."/Resources/ext_config/");
+  else if($format === ConfigCacheEntry::FMT_FILE_REF)
+    $msg_packed = ensure_read($payload);
+  else
+    throw new Exception("Bad format: $format");
+  return config_msgpack_unpack($msg_packed);
 }
 
 function config_make_standalone_ext_bundle(array $configs, $file_path)
@@ -196,21 +229,6 @@ function config_make_standalone_ext_bundle(array $configs, $file_path)
   ensure_write($file_path, $packed_data);
 }
 
-function config_filter_files(array $files)
-{
-  $configs = array();
-  foreach($files as $file)
-  {
-    if(strpos($file, '.conf.js') === false || 
-       strpos($file, '.bhl') !== false
-      )
-      continue;
-
-    $configs[] = $file;
-  }
-  return $configs;
-}
-
 function config_bench_load($file)
 {
   list($proto_id, $_) = config_ensure_header(config_base_dir(), $file);
@@ -261,10 +279,24 @@ function config_save_includes_map(array $includes_map, $file = null)
   ensure_write($file, serialize($includes_map));
 }
 
-function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $includes_map_file = null, $max_workers = null)
+class ConfigFetchResult
+{
+  public $all = array();
+  public $by_id = array();
+  public $by_path = array();
+  public $by_alias = array();
+}
+
+function config_fetch_ex(
+  array $files, 
+  $force_stale = false, 
+  $verbose = false, 
+  $includes_map_file = null, 
+  $max_workers = null
+)
 {
   if(!$files)
-    return array();
+    return new ConfigFetchResult();
 
   if($max_workers === null)
     $max_workers = sizeof($files) < 20 ? 1 : 4;
@@ -295,13 +327,25 @@ function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $
 
   if($serial)
   {
-  $results_by_job = array();
+    $results_by_job = array();
     foreach($jobs as $job)
       $results_by_job[] = _config_worker_func($job, $includes_map, $force_stale, $verbose);
   }
 
+  list($result, $total_stales) = _config_fetch_cache_ex($results_by_job);
+
+  if($verbose)
+    echo "Miss/Hit: $total_stales/" . sizeof($result->all) . "\n";
+
+  config_save_includes_map($includes_map, $includes_map_file);
+
+  return $result;
+}
+
+function _config_fetch_cache_ex($results_by_job)
+{
+  $result = new ConfigFetchResult();
   $total_stales = 0;
-  $cache_entries = array();
   foreach($results_by_job as $results)
   {
     foreach($results as $file => $item) 
@@ -312,27 +356,32 @@ function config_fetch_ex(array $files, $force_stale = false, $verbose = false, $
       //NOTE: handling any cache corruption errors
       if($cache_entry === null)
       {
-        $cache_entry = _config_invalidate_cache($file, $cache_file);
         $is_stale = true;
+        $cache_entry = _config_invalidate_cache($file, $cache_file);
+      }
+
+      $includes = $cache_entry->includes;
+      $includes_map[$file] = $includes;
+
+      if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes))
+      {
+        $is_stale = true;
+        $cache_entry = _config_invalidate_cache($file, $cache_file);
       }
 
       if($is_stale)
         ++$total_stales;
 
-      $includes_map[$file] = $cache_entry->includes;
-
       //we want results to be returned in the same order, so 
       //we store entries by the file key and later retrieve array values
-      $cache_entries[$file] = $cache_entry;
+      $result->by_path[$file] = $cache_entry;
+      $result->by_id[$cache_entry->id] = $cache_entry;
+      $result->by_alias[$cache_entry->strid] = $cache_entry;
     }
   }
 
-  if($verbose)
+  $result->all = array_values($result->by_path);
-    echo "Miss/Hit: $total_stales/" . sizeof($cache_entries) . "\n";
+  return array($result, $total_stales);
-
-  config_save_includes_map($includes_map, $includes_map_file);
-
-  return array_values($cache_entries);
 }
 
 function _config_worker_run_procs(array $jobs, $includes_map_file, $force, $verbose)
@@ -389,12 +438,12 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry
     throw new Exception("Bad proto_id: {$proto_id}");
 
   $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $proto_id; 
-  $GLOBALS['CONFIG_PREFAB_DEPS'] = array();
+  $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();
   $GLOBALS['CONFIG_BHL_FUNCS'] = array();
 
   $pres = config_parse(config_base_dir(), $file);
   if($pres->error !== 0)
-    throw new Exception("Error({$pres->error}) while loading JSON in file '{$file}':\n" . $pres->error_descr);
+    throw new Exception("Error({$pres->error}) while loading JSON in {$file}:\n" . $pres->error_descr);
 
   $includes = config_get_module_includes($pres->jsm_module);
 
@@ -411,6 +460,8 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry
   $cache_entry->payload_file = $cache_payload_file;
   $cache_entry->file = normalize_path($file);
   $cache_entry->includes = $includes;
+  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
+  $cache_entry->refs = config_extract_refs($pres->normalized_jzon);
 
   ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
   ensure_write($cache_payload_file, $payload_data);
@@ -418,28 +469,27 @@ function _config_invalidate_cache($file, $cache_file) : ConfigCacheEntry
   return $cache_entry;
 }
 
-function config_find_by_alias(array $cache_entries, $strid) : ConfigCacheEntry
+function config_find_by_alias(ConfigFetchResult $cache_entries, $strid) : ConfigCacheEntry
 {
-  foreach($cache_entries as $ce)
+  if(array_key_exists($strid, $cache_entries->by_alias))
-    if($ce->strid === $strid)
+    return $cache_entries->by_alias[$strid];
-      return $ce;
+  throw new Exception("Failed to find config by alias '$strid'!");
   return null;
 }
 
-function config_find_by_id(array $cache_entries, $id) : ConfigCacheEntry
+function config_find_by_id(ConfigFetchResult $cache_entries, $id) : ConfigCacheEntry
 {
-  foreach($cache_entries as $ce)
+  if(array_key_exists($id, $cache_entries->by_id))
-    if($ce->id === $id)
+    return $cache_entries->by_id[$id];
-      return $ce;
+  throw new Exception("Failed to find config by id '$id'!");
   return null;
 }
 
-function config_find_by_path(array $cache_entries, $path) : ConfigCacheEntry
+function config_find_by_path(ConfigFetchResult $cache_entries, $path) : ConfigCacheEntry
 {
-  $path = normalize_path($path);
+  if(array_key_exists($path, $cache_entries->by_path))
-  foreach($cache_entries as $ce)
+    return $cache_entries->by_path[$path];
-    if($ce->file === $path)
+  throw new Exception("Failed to find config by path '$path'!");
-      return $ce;
   return null;
 }
 
@@ -448,7 +498,7 @@ function config_fetch_by_path(string $path, $force_stale = false, $use_cache = t
   $ces = config_fetch_ex(array($path), $force_stale, $verbose = false, $includes_map_file, null, $use_cache);
   if(!$ces)
     throw new Exception("Config not found at path '$path'");
-  return $ces[0];
+  return $ces->all[0];
 }
 
 function config_fetch_all($force_stale = false, $use_cache = true, $includes_map_file = null)
@@ -456,6 +506,33 @@ function config_fetch_all($force_stale = false, $use_cache = true, $includes_map
   return config_fetch_ex(config_scan_files(), $force_stale, $verbose = false, $includes_map_file, null, $use_cache);
 }
 
+class ConfigCacheEntryExtras
+{
+  private static $klass = ConfigCacheEntryExtras::class;
+
+  static function init($project_specific_klass)
+  {
+    self::$klass = $project_specific_klass;
+  }
+
+  static function create()
+  {
+    return new self::$klass();
+  }
+
+  function export()
+  {
+    $as_array = get_object_vars($this);
+    return $as_array;
+  }
+
+  function import($as_array)
+  {
+    foreach($as_array as $field_name => $field_value)
+      $this->$field_name = $field_value;
+  }
+}
+
 class ConfigCacheEntry
 {
   const FMT_BINARY     = 0;
@@ -471,10 +548,8 @@ class ConfigCacheEntry
   public $payload_file;
   public $file;
   public $includes = array();
-  //TODO: do we need these?
   public $refs = array();
-  public $prefab_deps = array();
+  public $extras;
-  public $bhl_funcs = array();
 
   public $_config;
   public $_payload;
@@ -495,6 +570,11 @@ class ConfigCacheEntry
     return $ce;
   }
 
+  function __construct()
+  { 
+    $this->extras = ConfigCacheEntryExtras::create();
+  }
+
   function __get($name)
   {
     if($name === "config")
@@ -539,15 +619,15 @@ class ConfigCacheEntry
     $d[] = $this->payload_file;
     $d[] = $this->file;
     $d[] = $this->includes;
-    //TODO: do we need these?
     $d[] = $this->refs;
-    $d[] = $this->prefab_deps;
+    $d[] = $this->extras->export();
-    $d[] = $this->bhl_funcs;
     return $d;
   }
 
   function import(array $d)
   {
+    $extras = array();
+
     list(
       $this->class,
       $this->class_id,
@@ -557,11 +637,11 @@ class ConfigCacheEntry
       $this->payload_file,
       $this->file,
       $this->includes,
-      //TODO: do we need these?
       $this->refs,
-      $this->prefab_deps,
+      $extras,
-      $this->bhl_funcs,
     ) = $d;
+
+    $this->extras->import($extras);
   }
 }
 
@@ -616,6 +696,13 @@ function config_get_header($file, &$proto_id, &$alias)
     return false;
 }
 
+function config_set_header($contents, $proto_id, $alias, &$is_success)
+{
+  $contents = preg_replace('~\s*\{~', "{  /* proto_id = {$proto_id} ; alias = {$alias} */", $contents, 1/*limit*/, $count);
+  $is_success = $count == 1;
+  return $contents;
+}
+
 function config_extract_header($contents)
 {
   return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents);
@@ -633,10 +720,11 @@ function config_ensure_header($conf_dir, $file, $force = false)
     if($force || $curr_alias !== $alias)
     {
       $lines = file($file);
+      //TODO: why not using config_set_header(..) ?
       $lines[0] = preg_replace('~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{  /* proto_id = {$curr_proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count);
       if($count != 1)
         throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
-      ensure_write($file, join("", $lines));
+      ensure_write_if_differs($file, join("", $lines));
     }
 
     return array($curr_proto_id, $alias);
@@ -647,8 +735,9 @@ function config_ensure_header($conf_dir, $file, $force = false)
     $alias = config_get_strid($conf_dir, $file);
 
     $lines = file($file);
-    $lines[0] = preg_replace('~\s*\{~', "{  /* proto_id = {$proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count);
+
-    if($count != 1)
+    $lines[0] = config_set_header($lines[0], $proto_id, $alias, $is_success);
+    if(!$is_success)
       throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
     ensure_write($file, join("", $lines));
 
@@ -693,11 +782,11 @@ function config_parse($conf_dir, $file) : ConfigParseResult
 {
   $res = new ConfigParseResult();
 
-  $jsm = new \JSM($conf_dir, $file);
 
   $normalized_jzon = '';
   try
   {
+    $jsm = new \JSM($conf_dir, $file);
     list($normalized_jzon, $jsm_module) = $jsm->process();
     $res->normalized_jzon = $normalized_jzon;
     $res->jsm_module = $jsm_module;