Support for configs bundle patching WIP; Migrating to iterable from array where it makes sense; Adding config_hash_changed(..)

pack.inc.php

@@ -12,8 +12,14 @@ class ConfigPackParams
   public ?int $version = null;
   public bool $debug = false;
 
-  function __construct(array $cache_entries, int $version, bool $use_lz4 = false, 
+  function __construct(
-    bool $use_config_refs = false, int $binary_format = 1, bool $debug = false)
+    array $cache_entries, 
+    int $version, 
+    bool $use_lz4 = false, 
+    bool $use_config_refs = false, 
+    int $binary_format = 1, 
+    bool $debug = false
+  )
   {
     $this->cache_entries = $cache_entries;
     $this->use_lz4 = $use_lz4;
@@ -45,7 +51,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
       $params->cache_entries, 
       $params->use_lz4,
       $params->use_config_refs,
-      $params->version
+      $params->version,
     );
   }
   else
@@ -61,6 +67,35 @@ function config_pack_bundle(ConfigPackParams $params) : string
   return $packed_data;
 }
 
+function config_patch_bundle(ConfigPackParams $params, string $packed_data) : string
+{
+  $t = microtime(true);
+
+  $patched_data = null;
+
+  if($params->binary_format == 2)
+  {
+    $patched_data = _config_patch_bundle_fmt2(
+      $packed_data,
+      $params->cache_entries, 
+      $params->use_lz4,
+      $params->use_config_refs,
+      $params->version,
+    );
+  }
+  else
+    throw new Exception("Unknown binary format: {$params->binary_format}");
+
+  if($params->debug)
+    config_log("Patched entries: " . sizeof($params->cache_entries) . ", total: " . 
+      kb($patched_data) . ", format: {$params->binary_format}, lz4: " . 
+      var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
+      ", CRC: " . crc32($patched_data) . 
+      ", " . round(microtime(true) - $t,2) . " sec.");
+
+  return $patched_data;
+}
+
 //NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
 function _config_pack_bundle_fmt1(
   array $cache_entries, 
@@ -118,12 +153,14 @@ function _config_pack_bundle_fmt1(
   return $packed_data;
 }
 
-//NOTE: strids are stored as lookup strings
+//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
+//      (each path item separated by '/' is stored as an array item)
 function _config_pack_bundle_fmt2(
   array $cache_entries, 
   bool $use_lz4, 
   bool $use_config_refs, 
-  int $version) : string
+  int $version,
+) : string
 {
   $MAP = array();
   $STRIDMAP = array();
@@ -139,20 +176,8 @@ function _config_pack_bundle_fmt2(
     $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
     $payloads_offset += $payload_size;
 
-    $strids_indices = array();
+    $strids_indices = _config_encode_strid_as_indices($entry->strid, $STRIDMAP, $STRIDLIST);
-    $strid_parts = explode('/', ltrim($entry->strid, '@'));
+
-    foreach($strid_parts as $strid_part)
-    {
-      if(!isset($STRIDMAP[$strid_part]))
-      {
-        $strid_index = count($STRIDLIST);
-        $STRIDLIST[] = $strid_part; 
-        $STRIDMAP[$strid_part] = $strid_index;
-        $strids_indices[] = $strid_index;
-      }
-      else
-        $strids_indices[] = $STRIDMAP[$strid_part];
-    }
     $strids[] = $strids_indices;
   }
 
@@ -192,6 +217,105 @@ function _config_pack_bundle_fmt2(
   return $packed_data;
 }
 
+function _config_encode_strid_as_indices(string $strid, array &$STRIDMAP, array &$STRIDLIST) : array
+{
+  $strids_indices = array();
+  $strid_parts = explode('/', ltrim($strid, '@'));
+  foreach($strid_parts as $strid_part)
+  {
+    if(!isset($STRIDMAP[$strid_part]))
+    {
+      $strid_index = count($STRIDLIST);
+      $STRIDLIST[] = $strid_part; 
+      $STRIDMAP[$strid_part] = $strid_index;
+      $strids_indices[] = $strid_index;
+    }
+    else
+      $strids_indices[] = $STRIDMAP[$strid_part];
+  }
+  return $strids_indices;
+}
+
+function _config_patch_bundle_fmt2(
+  string $packed_data,
+  array $patch_entries, 
+  bool $use_lz4, 
+  bool $use_config_refs, 
+  int $version,
+) : string
+{
+  list($strids, $header, $_, $payloads_bundle) = 
+    _config_unpack_bundle_fmt2(packed_data: $packed_data, unpack_entries: false);
+
+  $stridmap = array_flip($strids);
+
+  foreach($patch_entries as $idx => $patch_entry)
+  {
+    list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
+
+    $header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id); 
+
+    if($header_found)
+    {
+      $header_idx = key($header_found);
+      $header_entry = current($header_found);
+
+      $current_offset = $header_entry[4];
+      $current_size = $header_entry[5];
+      if($current_size >= strlen($patch_payload))
+      {                                         
+        //let's do the inline patching
+        $payloads_bundle = substr_replace($payloads_bundle, $patch_payload, $current_offset, strlen($patch_payload));
+
+        $header_entry[0] = $patch_format;
+        $header_entry[5] = strlen($patch_payload);
+        $header[$header_idx] = $header_entry; 
+      }
+      else
+      {
+        //let's add it to the end
+        $header_entry[0] = $patch_format;
+        $header_entry[4] = strlen($payloads_bundle);
+        $header_entry[5] = strlen($patch_payload);
+        $header[$header_idx] = $header_entry; 
+
+        $payloads_bundle .= $patch_payload;
+      }
+    }
+    else
+    {
+      //let's add new entry it to the end
+      $strid_indices = _config_encode_strid_as_indices($patch_entry->strid, $stridmap, $strids);
+
+      $header_entry = array(
+        $patch_format,
+        $patch_entry->id,
+        $strid_indices,
+        $patch_entry->class_id,
+        strlen($payloads_bundle),
+        strlen($patch_payload),
+      );
+
+      $header[] = $header_entry;
+      $payloads_bundle .= $patch_payload;
+    }
+  }
+
+  $strids_msgpack = config_msgpack_pack($strids);
+  $header_msgpack = config_msgpack_pack($header);
+
+  $patched_data =  
+         pack("C", 2) . 
+         pack("V", $version) . 
+         pack("V", strlen($strids_msgpack)) .
+         pack("V", strlen($header_msgpack)) .
+         $strids_msgpack .
+         $header_msgpack .
+         $payloads_bundle;
+
+  return $patched_data;
+}
+
 //format: [[class_id, [data]], ...[class_id, [data]]]
 function config_unpack_bundle(string $packed_data) : array
 {
@@ -204,7 +328,8 @@ function config_unpack_bundle(string $packed_data) : array
   }
   else if($info['format'] === 2)
   {
-    return _config_unpack_bundle_fmt2($packed_data);
+    list($_, $_, $entries) = _config_unpack_bundle_fmt2($packed_data);
+    return $entries;
   }
   else
     throw new Exception("Unknown format: {$info['format']}");
@@ -237,7 +362,7 @@ function _config_unpack_bundle_fmt1(string $packed_data) : array
   return $entries;
 }
 
-function _config_unpack_bundle_fmt2(string $packed_data) : array
+function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries = true) : array
 {
   $packed_info = substr($packed_data, 0, 1+4+4+4);
 
@@ -255,16 +380,20 @@ function _config_unpack_bundle_fmt2(string $packed_data) : array
   $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
   
   $entries = array();
-  foreach($header as $item)
+
+  if($unpack_entries)
   {
-    list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
+    foreach($header as $item)
+    {
+      list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
 
-    $payload = substr($payloads_bundle, $offset, $size);
+      $payload = substr($payloads_bundle, $offset, $size);
 
-    $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+      $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
+    }
   }
 
-  return $entries;
+  return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
 }
 
 //format: [format_id, payload_data]

scan.inc.php

@@ -2,11 +2,24 @@
 namespace taskman;
 use Exception;
 
-class ConfigScanResult
+class ConfigScanResult implements \ArrayAccess, \Countable, \Iterator
 {
   /*var array<string, string[]>*/
   public array $base_dir2files = array();
 
+  private $iter_pos = 0;
+
+  function __construct(array $base_dir2files = array())
+  {
+    foreach($base_dir2files as $dir => $files)
+      $this->base_dir2files[$dir] = $files;
+  }
+
+  function clear()
+  {
+    $this->base_dir2files = array();
+  }
+
   function isEmpty() : bool
   {
     return empty($this->base_dir2files);
@@ -69,10 +82,94 @@ class ConfigScanResult
 
     return $all_files;
   }
+   
+  //ArrayAccess interface
+  function offsetExists(mixed $offset) : bool
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    return $this->count() > $offset;
+  }
+
+  function offsetGet(mixed $offset) : mixed
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+        return $files[$offset];
+      $offset -= $n;
+    }
+    return null;
+  }
+
+  function offsetSet(mixed $offset, mixed $value) : void
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => &$files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+      {
+        $files[$offset] = $value;
+        return;
+      }
+      $offset -= $n;
+    }
+  }
+
+  function offsetUnset(mixed $offset) : void
+  {
+    if(!is_int($offset))
+      throw new Exception("Invalid offset");
+
+    foreach($this->base_dir2files as $base_dir => $files)
+    {
+      $n = count($files);
+      if($offset - $n < 0)
+      {
+        unset($files[$offset]);
+        return;
+      }
+      $offset -= $n;
+    }
+  }
+
+  //Iterator interface
+  function rewind() : void
+  {
+    $this->iter_pos = 0;
+  }
+
+  function current() : mixed 
+  {
+    return $this->offsetGet($this->iter_pos);
+  }
+
+  function key() : mixed 
+  {
+    return $this->iter_pos;
+  }
+
+  function next() : void
+  {
+    ++$this->iter_pos;
+  }
+
+  function valid() : bool 
+  {
+    return $this->offsetExists($this->iter_pos);
+  }
 }
 
 function config_scan_files(
-  array $base_dirs, 
+  iterable $base_dirs, 
   string $ext_filter = '.conf.js', 
   bool $verbose = false
   ) : ConfigScanResult
@@ -92,3 +189,10 @@ function config_scan_files(
 
   return $result;
 }
+
+function config_hash_changed(ConfigGlobals $globals, iterable $all_files)
+{
+  $all_crc_file = $globals->build_dir . "/configs.crc";
+  return names_hash_changed($all_crc_file, $all_files);
+}
+