taskman_config/config.inc.php

<?php
namespace taskman;
use Exception;

$GLOBALS['CONFIG_BASE_DIRS'] = array();
$GLOBALS['CONFIG_INIT_WORKER_FUNC'] = null;
$GLOBALS['CONFIG_FILTER_FN'] = null;
$GLOBALS['CONFIG_BASE_CLASS'] = '\ConfBase';

task('config_worker', function(array $args)
{
  if(sizeof($args) != 3)
    throw new Exception("Config worker args not set");

  $in_file = $args[0];
  $out_file = $args[1];
  $err_file = $args[2];

  try
  {
    list($job, $includes_map_file, $force, $verbose) = unserialize(ensure_read($in_file));
    $result = _config_worker_func($job, config_load_includes_map($includes_map_file), $force, $verbose);
    ensure_write($out_file, serialize($result));
  }
  catch(Exception $e)
  {
    //NOTE: explicitely catching all exceptions and writing to the error file 
    //      since under Windows error file stream redirect may work unreliably 
    file_put_contents($err_file, $e->getMessage() . "\n" . $e->getTraceAsString(), FILE_APPEND);
    throw $e;
  }
});

function config_set_base_dirs(array $dirs)
{
  global $CONFIG_BASE_DIRS;
  $CONFIG_BASE_DIRS = array_map(function($d) { return normalize_path($d); }, $dirs);
}

function config_base_dirs() : array
{
  global $CONFIG_BASE_DIRS;
  return $CONFIG_BASE_DIRS;
}

function config_map_base_dir(string $file, bool $normalized = false, bool $strict = true, ?array $dirs = null) : ?string
{
  if(!is_array($dirs))
    $dirs = config_base_dirs();
  if(!$normalized)
    $file = normalize_path($file);
  foreach($dirs as $dir)
    if(strpos($file, $dir) === 0)
      return $dir;
  if($strict)
    throw new Exception("File '$file' is not mapped to any base dir");
  return null;
}

function config_real_path(string $rel_path, bool $strict = true, ?array $dirs = null) : ?string
{
  if(!is_array($dirs))
    $dirs = config_base_dirs();
  foreach($dirs as $dir)
    if(is_file($dir . '/' . $rel_path))
      return $dir . '/' . $rel_path;
  if($strict)
    throw new Exception("No file for relative path '$rel_path'");
  return null;
}

function config_build_dir() : string
{
  global $GAME_ROOT;
  return "$GAME_ROOT/build/tmp/";
}

function config_set_worker_init_fn(callable $fn)
{
  global $CONFIG_INIT_WORKER_FUNC;
  $CONFIG_INIT_WORKER_FUNC = $fn;
}

function config_scan_files() : array
{
  $files = scan_files_rec(config_base_dirs(), array('conf.js'));
  return config_filter_files($files);
}

function config_filter_files(array $files) : array
{
  global $CONFIG_FILTER_FN;
  if($CONFIG_FILTER_FN != null)
    $files = call_user_func($CONFIG_FILTER_FN, $files); 
  return $files;
}

function config_set_files_filter_fn(callable $fn)
{
  global $CONFIG_FILTER_FN;
  $CONFIG_FILTER_FN = $fn;
}

function config_pack_bundle(
  array $cache_entries, 
  bool $use_lz4 = false, 
  bool $use_config_refs = false,
  int $binary_format = 1,
  ?int $version = null
) : string
{
  global $GAME_ROOT;

  if(is_null($version))
    $version = game_version_code();

  $packed_data = null;

  if($binary_format == 1)
  {
    $packed_data = _config_pack_bundle_fmt1(
      $cache_entries, 
      $use_lz4,
      $use_config_refs,
      $version
    );
  }
  else if($binary_format == 2)
  {
    $packed_data = _config_pack_bundle_fmt2(
      $cache_entries, 
      $use_lz4,
      $use_config_refs,
      $version
    );
  }
  else
    throw new Exception("Unknown binary format: $binary_format");

  echo "CONF.BUNDLE: entries " . sizeof($cache_entries) . "; total " . kb($packed_data) . 
      "; format $binary_format; lz4 $use_lz4; refs $use_config_refs; CRC " . crc32($packed_data) . "\n";

  return $packed_data;
}

function _config_pack_bundle_fmt1(
  array $cache_entries, 
  bool $use_lz4 = false, 
  bool $use_config_refs = false,
  int $version) : string
{
  $MAP = array();
  $STRIDMAP = array();

  $payloads = array();
  $payloads_offset = 0;
  foreach($cache_entries as $entry)
  {
    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
    $payload_size = strlen($payload);
    $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
    $payloads_offset += $payload_size;
  }

  $header = array();
  foreach($cache_entries as $idx => $entry)
  { 
    if(isset($MAP[$entry->id]))
      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
    $MAP[$entry->id] = $entry->strid; 

    $strid_crc = crc32($entry->strid); 
    if(isset($STRIDMAP[$strid_crc]))
      throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
    $STRIDMAP[$strid_crc] = $entry->strid; 

    $header[] = array(
      $payloads[$idx][2],
      $entry->id, 
      crc32($entry->strid), 
      $entry->class_id, 
      $payloads[$idx][0],
      $payloads[$idx][3]
    );
  }

  $header_msgpack = config_msgpack_pack($header);
  $payloads_bundle = '';
  foreach($payloads as $item)
    $payloads_bundle .= $item[1];

  $packed_data =  
         pack("C", 1) . 
         pack("V", $version) . 
         pack("V", strlen($header_msgpack)) .
         $header_msgpack .
         $payloads_bundle;

  return $packed_data;
}

function _config_pack_bundle_fmt2(
  array $cache_entries, 
  bool $use_lz4 = false, 
  bool $use_config_refs = false,
  int $version) : string
{
  $MAP = array();
  $STRIDMAP = array();
  $STRIDLIST = array();

  $payloads = array();
  $strids = array();
  $payloads_offset = 0;
  foreach($cache_entries as $entry)
  {
    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
    $payload_size = strlen($payload);
    $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
    $payloads_offset += $payload_size;

    $strids_indices = array();
    $strid_parts = explode('/', ltrim($entry->strid, '@'));
    foreach($strid_parts as $strid_part)
    {
      if(!isset($STRIDMAP[$strid_part]))
      {
        $strid_index = count($STRIDLIST);
        $STRIDLIST[] = $strid_part; 
        $STRIDMAP[$strid_part] = $strid_index;
        $strids_indices[] = $strid_index;
      }
      else
        $strids_indices[] = $STRIDMAP[$strid_part];
    }
    $strids[] = $strids_indices;
  }

  $header = array();
  foreach($cache_entries as $idx => $entry)
  { 
    if(isset($MAP[$entry->id]))
      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
    $MAP[$entry->id] = $entry->strid; 

    $header[] = array(
      $payloads[$idx][2], //format
      $entry->id, 
      $strids[$idx], //strid as a lookup indices 
      $entry->class_id, 
      $payloads[$idx][0], //offset
      $payloads[$idx][3]  //size
    );
  }

  $strids_msgpack = config_msgpack_pack($STRIDLIST);

  $header_msgpack = config_msgpack_pack($header);
  $payloads_bundle = '';
  foreach($payloads as $item)
    $payloads_bundle .= $item[1];

  $packed_data =  
         pack("C", 2) . 
         pack("V", $version) . 
         pack("V", strlen($strids_msgpack)) .
         pack("V", strlen($header_msgpack)) .
         $strids_msgpack .
         $header_msgpack .
         $payloads_bundle;

  return $packed_data;
}

function config_unpack_bundle(string $packed_data) : array
{
  $packed_info = substr($packed_data, 0, 1);
  $info = unpack('Cformat', $packed_info);

  if($info['format'] === 1)
  {
    return _config_unpack_bundle_fmt1($packed_data);
  }
  else if($info['format'] === 2)
  {
    return _config_unpack_bundle_fmt2($packed_data);
  }
  else
    throw new Exception("Unknown format: {$info['format']}");
}

function _config_unpack_bundle_fmt1(string $packed_data) : array
{
  $packed_info = substr($packed_data, 0, 1+4+4);

  $info = unpack('Cformat/Vversion/Vheader_len', $packed_info);

  if($info['format'] !== 1)
    throw new Exception("Unknown format: {$info['format']}");

  $header_msgpack = substr($packed_data, 1+4+4, $info['header_len']); 
  $header = config_msgpack_unpack($header_msgpack);

  $payloads_bundle = substr($packed_data, 1+4+4+$info['header_len']);
  
  $entries = array();
  foreach($header as $item)
  {
    list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;

    $payload = substr($payloads_bundle, $offset, $size);

    $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
  }

  return $entries;
}

function _config_unpack_bundle_fmt2(string $packed_data) : array
{
  $packed_info = substr($packed_data, 0, 1+4+4+4);

  $info = unpack('Cformat/Vversion/Vstrids_len/Vheader_len', $packed_info);

  if($info['format'] !== 2)
    throw new Exception("Unknown format: {$info['format']}");

  $strids_msgpack = substr($packed_data, 1+4+4+4, $info['strids_len']); 
  $strids = config_msgpack_unpack($strids_msgpack);

  $header_msgpack = substr($packed_data, 1+4+4+4+$info['strids_len'], $info['header_len']); 
  $header = config_msgpack_unpack($header_msgpack);

  $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
  
  $entries = array();
  foreach($header as $item)
  {
    list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;

    $payload = substr($payloads_bundle, $offset, $size);

    $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
  }

  return $entries;
}

function _config_get_payload(ConfigCacheEntry $ce, bool $use_lz4, bool $use_config_refs) : array
{
  $format = ConfigCacheEntry::FMT_BINARY;
  $payload = null;

  if($use_config_refs && $ce->payload_file)
  {
    $format = ConfigCacheEntry::FMT_FILE_REF;
    $payload = $ce->payload_file;
  }
  else
  {
    $payload = $ce->payload;
    if($use_lz4 && strlen($payload) > 512)
    {
      $format = ConfigCacheEntry::FMT_LZ4;
      $payload = lz4_compress($payload, 9);
    }
  }

  return array($format, $payload);
}

function _config_unpack_payload(int $format, string $payload) : array
{
  $msg_packed = null;
  if($format === ConfigCacheEntry::FMT_LZ4)
    $msg_packed = lz4_uncompress($payload);
  else if($format === ConfigCacheEntry::FMT_BINARY)
    $msg_packed = $payload;
  else if($format === ConfigCacheEntry::FMT_FILE_REF)
    $msg_packed = ensure_read($payload);
  else
    throw new Exception("Bad format: $format");
  return config_msgpack_unpack($msg_packed);
}

function config_make_standalone_ext_bundle(
  array $configs, 
  string $file_path, 
  bool $use_lz4 = true, 
  int $binary_format = 1, 
  ?int $version = null
)
{
  $cache_entries = array();
  foreach($configs as $conf)
  {
    $payload = config_msgpack_pack($conf->export());

    //creating fake cache entries
    $entry = new ConfigCacheEntry();
    $entry->id = $conf->id;
    $entry->strid = $conf->strid;
    $entry->class_id = $conf->getClassId();
    $entry->class = get_class($conf);
    $entry->payload = $payload;
    $entry->config = $conf;

    $cache_entries[] = $entry;
  }

  $packed_data = config_pack_bundle($cache_entries, $use_lz4, false, $binary_format, $version);

  ensure_write($file_path, $packed_data);
}

function config_bench_load(string $file)
{
  $base_dir = config_map_base_dir($file);

  list($conf_id, $_) = config_ensure_header($base_dir, $file);
  if(!$conf_id)
    throw new Exception("Bad conf id: {$conf_id}");
  $t = microtime(true);
  $parse_res = config_parse(config_base_dirs(), $file);
  if($parse_res->error !== 0)
    throw new Exception("Error({$parse_res->error}) while loading JSON from {$file}:\n" . $parse_res->error_descr);
  echo "PARSE: " . (microtime(true) - $t) . "\n";
  $t = microtime(true);
  $config = config_load_ex($base_dir, $file, $parse_res->parsed_arr, $conf_id); 
  echo "LOAD: " . (microtime(true) - $t) . "\n";
}

function config_get_tmp_build_path(string $file) : string
{
  $name = str_replace(":", "-", str_replace("\\", "-", str_replace("/", "-", normalize_path($file))));
  $name = ltrim($name, "-");
  return normalize_path(config_build_dir() . "/$name");
}

function config_get_includes_map_path() : string 
{
  return config_build_dir() . "/includes.map";
}

function config_load_includes_map(?string $file = null) : array
{
  $file = $file ? $file : config_get_includes_map_path();

  $includes_map = array(array(), array());

  if(is_file($file))
  {
    $tmp_map = @unserialize(ensure_read($file));
    if(is_array($tmp_map))
      $includes_map = $tmp_map;
  }

  return $includes_map;
}

function config_save_includes_map(array $includes_map, ?string $file = null)
{
  $file = $file ? $file : config_get_includes_map_path();

  ensure_write($file, serialize($includes_map));
}

class ConfigFetchResult
{
  public array $all = array();
  public array $by_id = array();
  public array $by_path = array();
  public array $by_alias = array();
}

function config_fetch_ex(
  array $files, 
  bool $force_stale = false, 
  bool $verbose = false, 
  ?string $includes_map_file = null, 
  ?int $max_workers = null
) : ConfigFetchResult
{
  if(!$files)
    return new ConfigFetchResult();

  if($max_workers === null)
    $max_workers = sizeof($files) < 20 ? 1 : 4;

  $includes_map_file = $includes_map_file ? $includes_map_file : config_get_includes_map_path();
  $includes_map = config_load_includes_map($includes_map_file);

  $chunk_size = (int)ceil(sizeof($files)/$max_workers);
  $jobs = array();
  foreach(array_chunk($files, $chunk_size) as $idx => $chunk_files)
    $jobs[] = array($idx, $chunk_files);

  $results_by_job = null;

  $serial = $max_workers == 1;

  if(!$serial)
  {
    $results_by_job = _config_worker_run_procs($jobs, $includes_map_file, $force_stale, $verbose);
    //in case of any result error try serial processing
    if(array_search(false, $results_by_job, true/*strict*/) !== false)
    {
      if($verbose)
        echo "Result error detected, trying serial processing...\n";
      $serial = true;
    }
  }

  if($serial)
  {
    $results_by_job = array();
    foreach($jobs as $job)
      $results_by_job[] = _config_worker_func($job, $includes_map, $force_stale, $verbose);
  }

  list($result, $total_stales) = _config_fetch_cache_ex($results_by_job);

  if($verbose)
    echo "Miss/All: $total_stales/" . sizeof($result->all) . "\n";

  config_save_includes_map($includes_map, $includes_map_file);

  return $result;
}

function _config_fetch_cache_ex(array $results_by_job) : array
{
  $result = new ConfigFetchResult();
  $total_stales = 0;
  foreach($results_by_job as $results)
  {
    foreach($results as $file => $item) 
    {
      list($cache_file, $is_stale) = $item;

      $cache_entry = ConfigCacheEntry::unserialize(ensure_read($cache_file));
      //NOTE: handling any cache corruption errors
      if($cache_entry === null)
      {
        $is_stale = true;
        $cache_entry = _config_invalidate_cache($file, $cache_file);
      }

      $includes = $cache_entry->includes;
      $includes_map[$file] = $includes;

      if(!$is_stale && count($includes) > 0 && need_to_regen($file, $includes))
      {
        $is_stale = true;
        $cache_entry = _config_invalidate_cache($file, $cache_file);
        //NOTE: let's change the mtime of the file which include other files,
        //      so that on tne next build it will be 'older' than its includes
        //      and won't trigger rebuild
        touch($file);
      }

      if($is_stale)
        ++$total_stales;

      //we want results to be returned in the same order, so 
      //we store entries by the file key and later retrieve array values
      $result->by_path[$file] = $cache_entry;
      $result->by_id[$cache_entry->id] = $cache_entry;
      $result->by_alias[$cache_entry->strid] = $cache_entry;
    }
  }

  $result->all = array_values($result->by_path);
  return array($result, $total_stales);
}

function _config_worker_run_procs(array $jobs, string $includes_map_file, bool $force, bool $verbose) : array
{
  $worker_args = array();
  foreach($jobs as $idx => $job)
    $worker_args[] = array($job, $includes_map_file, $force, $verbose);

  return run_background_gamectl_workers('config_worker', $worker_args);
}

function _config_worker_func(array $job, array $includes_map, bool $force, bool $verbose) : array
{
  global $CONFIG_INIT_WORKER_FUNC;
  if(is_callable($CONFIG_INIT_WORKER_FUNC))
    $CONFIG_INIT_WORKER_FUNC();

  list($idx, $files) = $job;
  if($verbose)
    echo "Worker $idx (" . sizeof($files) . ") started\n";
  $results = array();
  foreach($files as $file_idx => $file)
  {
    if($verbose && $file_idx > 0 && ($file_idx % 500) == 0)
      echo "Worker $idx progress: " . round($file_idx / sizeof($files) * 100) . "% ...\n";

    $cache_file = config_get_cache_path($file);

    $is_stale = true;
    if(!$force)
    {
      $file_deps = array($file);
      if(isset($includes_map[$file]))
        $file_deps = array_merge($file_deps, $includes_map[$file]);
      $is_stale = need_to_regen($cache_file, $file_deps);
    }

    if($is_stale)
      _config_invalidate_cache($file, $cache_file);

    $results[$file] = array($cache_file, $is_stale);
  }
  if($verbose)
    echo "Worker $idx done\n";
  return $results;
}

function _config_invalidate_cache(string $file, string $cache_file) : ConfigCacheEntry
{
  $cache_payload_file = config_get_cache_payload_path($file);

  //TODO: pass it from above?
  $base_dir = config_map_base_dir($file);

  list($conf_id, $_) = config_ensure_header($base_dir, $file);
  if(!$conf_id)
    throw new Exception("Bad conf id: {$conf_id}");

  $GLOBALS['CONFIG_CURRENT_FILE'] = $file; 
  $GLOBALS['CONFIG_CURRENT_PROTO_ID'] = $conf_id; 
  $GLOBALS['CONFIG_EXTRAS'] = ConfigCacheEntryExtras::create();

  $pres = config_parse(config_base_dirs(), $file);
  if($pres->error !== 0)
    throw new Exception("Error({$pres->error}) while loading JSON in {$file}:\n" . $pres->error_descr);

  $includes = config_get_module_includes($pres->jsm_module);

  $config = config_load_ex($base_dir, $file, $pres->parsed_arr, $conf_id); 
  $payload_data = config_msgpack_pack($config->export());

  $cache_entry = new ConfigCacheEntry();
  $cache_entry->id = $config->id;
  $cache_entry->config = $config;
  $cache_entry->strid = $config->strid;
  $cache_entry->cache_file = $cache_file;
  $cache_entry->class = get_class($config);
  $cache_entry->class_id = $config->getClassId();
  $cache_entry->payload_file = $cache_payload_file;
  $cache_entry->file = normalize_path($file);
  $cache_entry->includes = $includes;
  $cache_entry->extras = $GLOBALS['CONFIG_EXTRAS'];
  $cache_entry->refs = config_extract_refs($pres->normalized_jzon);

  ensure_write($cache_file, ConfigCacheEntry::serialize($cache_entry));
  ensure_write($cache_payload_file, $payload_data);

  return $cache_entry;
}

function config_find_by_alias(ConfigFetchResult $cache_entries, string $strid) : ConfigCacheEntry
{
  if(array_key_exists($strid, $cache_entries->by_alias))
    return $cache_entries->by_alias[$strid];
  throw new Exception("Failed to find config by alias '$strid'!");
}

function config_find_by_id(ConfigFetchResult $cache_entries, int $id) : ConfigCacheEntry
{
  if(array_key_exists($id, $cache_entries->by_id))
    return $cache_entries->by_id[$id];
  throw new Exception("Failed to find config by id '$id'!");
}

function config_find_by_path(ConfigFetchResult $cache_entries, string $path) : ConfigCacheEntry
{
  if(array_key_exists($path, $cache_entries->by_path))
    return $cache_entries->by_path[$path];
  throw new Exception("Failed to find config by path '$path'!");
}

function config_fetch_by_path(string $path, bool $force_stale = false) : ConfigCacheEntry
{
  $ces = config_fetch_ex(array($path), $force_stale);
  if(empty($ces->all))
    throw new Exception("Config not found at path '$path'");
  return $ces->all[0];
}

function config_fetch_all(bool $force_stale = false) : ConfigFetchResult
{
  return config_fetch_ex(config_scan_files(), $force_stale);
}

class ConfigCacheEntryExtras
{
  private static string $klass = ConfigCacheEntryExtras::class;

  static function init(string $project_specific_klass)
  {
    self::$klass = $project_specific_klass;
  }

  static function create() : object
  {
    return new self::$klass();
  }

  function export() : array
  {
    $as_array = get_object_vars($this);
    return $as_array;
  }

  function import(array $as_array)
  {
    foreach($as_array as $field_name => $field_value)
      $this->$field_name = $field_value;
  }
}

/**
 * @property object $config
 * @property string $payload
 */
class ConfigCacheEntry
{
  const FMT_BINARY     = 0;
  const FMT_LZ4        = 1;
  const FMT_FILE_REF   = 2;

  public string $class;
  public int $class_id;
  public int $id;
  public string $strid;
  public string $cache_file;
  //NOTE: actual payload is stored in a separate file for faster incremental retrievals
  public string $payload_file;
  public string $file;
  public array $includes = array();
  public array $refs = array();
  public object $extras;

  public $_config;
  public $_payload;

  static function serialize(ConfigCacheEntry $ce) : string
  {
    $d = $ce->export();
    return serialize($d);
  }

  static function unserialize(string $str) : ?ConfigCacheEntry
  {
    $d = @unserialize($str);
    if(!is_array($d))
      return null;
    $ce = new ConfigCacheEntry();
    $ce->import($d);
    return $ce;
  }

  function __construct()
  { 
    $this->extras = ConfigCacheEntryExtras::create();
  }

  function __get(string $name)
  {
    if($name === "config")
    {
      if($this->_config === null)
      {
        $klass = $this->class;
        $data = config_msgpack_unpack($this->payload);
        $this->_config = new $klass($data);
      }
      return $this->_config;
    }
    else if($name === "payload")
    {
      if($this->_payload !== null)
        return $this->_payload;
      //if payload not set directly not storing it
      return ensure_read($this->payload_file);
    }
    else
      throw new Exception("No such property '$name'");
  }

  function __set(string $name, $v)
  {
    if($name === "config")
      $this->_config = $v;
    else if($name === "payload")
      $this->_payload = $v;
    else
      throw new Exception("No such property '$name'");
  }

  function export() : array
  {
    $d = array();
    $d[] = $this->class;
    $d[] = $this->class_id;
    $d[] = $this->id;
    $d[] = $this->strid;
    $d[] = $this->cache_file;
    $d[] = $this->payload_file;
    $d[] = $this->file;
    $d[] = $this->includes;
    $d[] = $this->refs;
    $d[] = $this->extras->export();
    return $d;
  }

  function import(array $d)
  {
    $extras = array();

    list(
      $this->class,
      $this->class_id,
      $this->id,
      $this->strid,
      $this->cache_file,
      $this->payload_file,
      $this->file,
      $this->includes,
      $this->refs,
      $extras,
    ) = $d;

    $this->extras->import($extras);
  }
}

function config_get_cache_path(string $file) : string
{
  return config_get_tmp_build_path($file . '.cacheb');
} 

function config_get_cache_payload_path(string $file) : string
{
  return config_get_tmp_build_path($file . '.pdata');
} 

function config_extract_refs(string $src, bool $as_map = true) : array
{
  $refs = array();
  if(preg_match_all('~"(@[^"]+)"~', $src, $matches))
  {
    foreach($matches[1] as $ref)
    {
      if(!isset($refs[$ref]))
        $refs[$ref] = 1;
      ++$refs[$ref];
    }
  }
  return $as_map ? $refs : array_keys($refs);
}

function config_get_header(string $file, ?int &$conf_id, ?string &$strid) : bool
{
  $h = fopen($file, "r");
  $line = fgets($h, 256);
  fclose($h);

  if(preg_match('~\{\s*/\*\s*proto_id\s*=\s*(\d+)\s*;\s*alias\s*=\s*([^\s]+)~', $line, $matches))
  {
    $conf_id = (int)$matches[1];
    $strid = $matches[2];
    return true;
  }
  else
    return false;
}

function config_set_header(string $contents, int $conf_id, string $strid, bool &$is_success) : string
{
  $contents = preg_replace('~\s*\{~', "{  /* proto_id = {$conf_id} ; alias = {$strid} */", $contents, 1/*limit*/, $count);
  $is_success = $count == 1;
  return $contents;
}

function config_extract_header(string $contents) : string
{
  return preg_replace('~(\s*\{)\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', '$1', $contents);
}

function config_ensure_header(string $conf_dir, string $file, bool $force = false) : array
{
  if(config_get_header($file, $curr_proto_id, $curr_alias))
  {
    $alias = config_file2strid($conf_dir, $file);
    if($force)
      $curr_proto_id = config_file2id($conf_dir, $file);

    //NOTE: keeping current proto id intact if not forced
    if($force || $curr_alias !== $alias)
    {
      $lines = file($file);
      //TODO: why not using config_set_header(..) ?
      $lines[0] = preg_replace('~\s*\{\s*/\*\s*proto_id\s*=\s*\d+\s*;\s*alias\s*=\s*[^\s]+\s*\*/~', "{  /* proto_id = {$curr_proto_id} ; alias = {$alias} */", $lines[0], 1/*limit*/, $count);
      if($count != 1)
        throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
      ensure_write_if_differs($file, join("", $lines));
    }

    return array($curr_proto_id, $alias);
  }
  else
  {
    $conf_id = config_file2id($conf_dir, $file);
    $alias = config_file2strid($conf_dir, $file);

    $lines = file($file);

    $lines[0] = config_set_header($lines[0], $conf_id, $alias, $is_success);
    if(!$is_success)
      throw new Exception("Could not set header for '$file' in line: {$lines[0]}");
    ensure_write($file, join("", $lines));

    return array($conf_id, $alias);
  }
}

function config_load(string $conf_dir, string $conf_path) : object
{
  list($conf_id, $_) = config_ensure_header($conf_dir, $conf_path);
  if(!$conf_id)
    throw new Exception("Bad conf id: {$conf_id}");

  $pres = config_parse(array($conf_dir), $conf_path);
  if($pres->error !== 0)
    throw new Exception("Error({$pres->error}) while loading JSON from {$conf_path}:\n" . $pres->error_descr);

  return config_load_ex($conf_dir, $conf_path, $pres->parsed_arr, $conf_id); 
}

function conf2json($conf, int $json_flags = 0)
{
  $arr = $conf->export(true); 
  $arr['class'] = get_class($conf);
  unset($arr['id']);
  unset($arr['strid']);
  $json = json_encode($arr, $json_flags);
  $json = str_replace(array('\\\\', '\\n', '\/'), array('\\', "\n", '/'), $json);
  return $json;
}

class ConfigParseResult
{
  public int $error = 0;
  public string $error_descr;
  public string $normalized_jzon = '';
  public array $parsed_arr;
  public $jsm_module;
}

function config_parse(array $base_dirs, string $file) : ConfigParseResult
{
  $res = new ConfigParseResult();

  $normalized_jzon = '';
  try
  {
    $jsm = new \JSM($base_dirs, $file);
    list($normalized_jzon, $jsm_module) = $jsm->process();
    $res->normalized_jzon = $normalized_jzon;
    $res->jsm_module = $jsm_module;
  }
  catch(Exception $e)
  {
    $res->error = 1;
    $res->error_descr = "File '$file':\n" . $e->getMessage() . "\n" . $e->getTraceAsString(); 
    return $res;
  }

  $decode_res = config_check_and_decode_jzon($normalized_jzon);
  
  if($decode_res[0] === 0)
  {
    $res->parsed_arr = $decode_res[2];
  }
  else
  {
    $res->error = $decode_res[0];
    $res->error_descr = "File '$file':\n" . $decode_res[1];
  }

  return $res;
}

function config_check_and_decode_jzon(string $json) : array
{
  try
  {
    $arr = \jzon_parse($json);
    return array(0, "", $arr);
  }
  catch(Exception $e)
  {
    return array(1, $e->getMessage(), array());
  }
}

function config_load_ex(string $conf_dir, string $file, array $arr, ?int $id = null) : object
{
  if(!isset($arr['class']) || !isset($arr['class'][0]))
    throw new Exception("Class is not set in file '$file'.");

  $klass = $arr['class'];
  unset($arr['class']);
  
  if($id === null)
    $id = config_file2id($conf_dir, $file);  

  $cnf = null;
  try
  {
    if(!class_exists($klass))
      throw new Exception("No such class '$klass'");
    $cnf = new $klass;   
    if(!is_a($cnf, $GLOBALS['CONFIG_BASE_CLASS']))
      throw new Exception("Class '$klass' must inherit from '{$GLOBALS['CONFIG_BASE_CLASS']}'");

    $cnf->import($arr, true);
  }
  catch(Exception $e)
  {
    throw new Exception($e->getMessage() . " in file '{$file}'"/* . $e->getTraceAsString()*/);
  }
  
  $cnf->id = $id;
  $cnf->strid = config_file2strid($conf_dir, $file);

  return $cnf;
}

function config_is_file(string $filename) : bool
{
  return (strrpos($filename, '.conf.js') === (strlen($filename) - 8));
}

function config_file2id(string $conf_dir, string $filename) : int
{
  $nfilename = config_make_path($conf_dir, $filename);
  return config_crc28($nfilename);
}

function config_file2strid(string $conf_dir, string $filename) : string
{
  $strid = config_make_path($conf_dir, $filename); 
  $fname_idx = strrpos($strid, "/");
  $fname_idx = strpos($strid, ".", $fname_idx);
  $strid = substr($strid, 0, $fname_idx);
  return "@".$strid;
}

function config_make_path(string $conf_dir, string $path) : string
{
  return ltrim(str_replace(normalize_path($conf_dir, true/*nix*/), 
                           '', 
                           normalize_path($path, true/*nix*/)), 
                '/');
}

function config_crc28(string $what) : int
{
  return crc32($what) & 0xFFFFFFF;
}

function config_get_module_includes(\JSM_Module $cm) : array
{
  $includes = array();
  foreach($cm->getIncludes() as $include => $_)
  {
    //maybe we should take .php includes into account as well?
    if(config_str_ends_with($include, ".php"))
      continue;
    $includes[] = $include;
  }
  return $includes;
}

function config_includes_map_find_text_origin(array $map, string $file, string $text) : array
{
  if(!isset($map[$file]))
    return array($file);

  $tpls = $map[$file];
  $tpls[] = $file;
  $res = array();
  foreach($tpls as $tpl)
  {
    $content = ensure_read($tpl);
    $content = i18n_decode_string($content);
    if(strpos($content, $text) !== FALSE)
      $res[] = $tpl;
  }

  return $res;
}

function config_str_ends_with(string $haystack, string $needle) : bool
{
  // search forward starting from end minus needle length characters
  return $needle === "" || (($temp = strlen($haystack) - strlen($needle)) >= 0 && strpos($haystack, $needle, $temp) !== false);
}

function config_walk_fields($proto, callable $callback)
{
  if(!method_exists($proto, 'CLASS_FIELDS_PROPS'))
    return;

  foreach($proto->CLASS_FIELDS_PROPS() as $field => $tokens)
  {
    $value = $proto->$field;
    $callback($proto, $field, $value, $tokens);

    if(is_object($value))
      config_walk_fields($value, $callback);
    else if(is_array($value))
    {
      foreach($value as $num => $item)
      {
        if(is_object($item))
          config_walk_fields($item, $callback);
      }
    }
  }
}

if(function_exists('msgpack_pack'))
{
  function config_msgpack_pack(array $data) : string
  {
    $prev = ini_set('msgpack.use_str8_serialization', '0');
    $res =  msgpack_pack($data);
    if($prev != '0')
      ini_set('msgpack.use_str8_serialization', $prev);
    return $res;
  }

  function config_msgpack_unpack(string $data) : array
  {
    return msgpack_unpack($data);
  }
}
else
{
  function config_msgpack_pack(array $data) : string
  {
    include_once(__DIR__ . '/msgpack/msgpack_custom.inc.php');

    $packer = new \MessagePackCustom(); 
    return $packer->pack($data);
  }

  function config_msgpack_unpack(string $data) : array
  {
    return \MessagePack\MessagePack::unpack($data);
  }
}