taskman_config/pack.inc.php

<?php
namespace taskman;
use Exception;

class ConfigPackParams
{
  /*var ConfigCacheEntry[]*/
  public array $cache_entries;
  public bool $use_lz4 = false;
  public bool $use_config_refs = false;
  public int $binary_format = 1; //1,2 supported
  public ?int $version = null;
  public bool $debug = false;

  function __construct(array $cache_entries, int $version, bool $use_lz4 = false, 
    bool $use_config_refs = false, int $binary_format = 1, bool $debug = false)
  {
    $this->cache_entries = $cache_entries;
    $this->use_lz4 = $use_lz4;
    $this->use_config_refs = $use_config_refs;
    $this->binary_format = $binary_format;
    $this->version = $version;
    $this->debug = $debug;
  }
}

function config_pack_bundle(ConfigPackParams $params) : string
{
  $t = microtime(true);

  $packed_data = null;

  if($params->binary_format == 1)
  {
    $packed_data = _config_pack_bundle_fmt1(
      $params->cache_entries, 
      $params->use_lz4,
      $params->use_config_refs,
      $params->version
    );
  }
  else if($params->binary_format == 2)
  {
    $packed_data = _config_pack_bundle_fmt2(
      $params->cache_entries, 
      $params->use_lz4,
      $params->use_config_refs,
      $params->version
    );
  }
  else
    throw new Exception("Unknown binary format: {$params->binary_format}");

  if($params->debug)
    config_log("Packed entries: " . sizeof($params->cache_entries) . ", total: " . 
      kb($packed_data) . ", format: {$params->binary_format}, lz4: " . 
      var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . 
      ", CRC: " . crc32($packed_data) . 
      ", " . round(microtime(true) - $t,2) . " sec.");

  return $packed_data;
}

//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
function _config_pack_bundle_fmt1(
  array $cache_entries, 
  bool $use_lz4, 
  bool $use_config_refs, 
  int $version) : string
{
  $MAP = array();
  $STRIDMAP = array();

  $payloads = array();
  $payloads_offset = 0;
  foreach($cache_entries as $entry)
  {
    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
    $payload_size = strlen($payload);
    $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
    $payloads_offset += $payload_size;
  }

  $header = array();
  foreach($cache_entries as $idx => $entry)
  { 
    if(isset($MAP[$entry->id]))
      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
    $MAP[$entry->id] = $entry->strid; 

    $strid_crc = crc32($entry->strid); 
    if(isset($STRIDMAP[$strid_crc]))
      throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
    $STRIDMAP[$strid_crc] = $entry->strid; 

    $header[] = array(
      $payloads[$idx][2],
      $entry->id, 
      crc32($entry->strid), 
      $entry->class_id, 
      $payloads[$idx][0],
      $payloads[$idx][3]
    );
  }

  $header_msgpack = config_msgpack_pack($header);
  $payloads_bundle = '';
  foreach($payloads as $item)
    $payloads_bundle .= $item[1];

  $packed_data =  
         pack("C", 1) . 
         pack("V", $version) . 
         pack("V", strlen($header_msgpack)) .
         $header_msgpack .
         $payloads_bundle;

  return $packed_data;
}

//NOTE: strids are stored as lookup strings
function _config_pack_bundle_fmt2(
  array $cache_entries, 
  bool $use_lz4, 
  bool $use_config_refs, 
  int $version) : string
{
  $MAP = array();
  $STRIDMAP = array();
  $STRIDLIST = array();

  $payloads = array();
  $strids = array();
  $payloads_offset = 0;
  foreach($cache_entries as $entry)
  {
    list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
    $payload_size = strlen($payload);
    $payloads[] = array($payloads_offset, $payload, $format, $payload_size);
    $payloads_offset += $payload_size;

    $strids_indices = array();
    $strid_parts = explode('/', ltrim($entry->strid, '@'));
    foreach($strid_parts as $strid_part)
    {
      if(!isset($STRIDMAP[$strid_part]))
      {
        $strid_index = count($STRIDLIST);
        $STRIDLIST[] = $strid_part; 
        $STRIDMAP[$strid_part] = $strid_index;
        $strids_indices[] = $strid_index;
      }
      else
        $strids_indices[] = $STRIDMAP[$strid_part];
    }
    $strids[] = $strids_indices;
  }

  $header = array();
  foreach($cache_entries as $idx => $entry)
  { 
    if(isset($MAP[$entry->id]))
      throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
    $MAP[$entry->id] = $entry->strid; 

    $header[] = array(
      $payloads[$idx][2], //format
      $entry->id, 
      $strids[$idx], //strid as a lookup indices 
      $entry->class_id, 
      $payloads[$idx][0], //offset
      $payloads[$idx][3]  //size
    );
  }

  $strids_msgpack = config_msgpack_pack($STRIDLIST);

  $header_msgpack = config_msgpack_pack($header);
  $payloads_bundle = '';
  foreach($payloads as $item)
    $payloads_bundle .= $item[1];

  $packed_data =  
         pack("C", 2) . 
         pack("V", $version) . 
         pack("V", strlen($strids_msgpack)) .
         pack("V", strlen($header_msgpack)) .
         $strids_msgpack .
         $header_msgpack .
         $payloads_bundle;

  return $packed_data;
}

//format: [[class_id, [data]], ...[class_id, [data]]]
function config_unpack_bundle(string $packed_data) : array
{
  $packed_info = substr($packed_data, 0, 1);
  $info = unpack('Cformat', $packed_info);

  if($info['format'] === 1)
  {
    return _config_unpack_bundle_fmt1($packed_data);
  }
  else if($info['format'] === 2)
  {
    return _config_unpack_bundle_fmt2($packed_data);
  }
  else
    throw new Exception("Unknown format: {$info['format']}");
}

function _config_unpack_bundle_fmt1(string $packed_data) : array
{
  $packed_info = substr($packed_data, 0, 1+4+4);

  $info = unpack('Cformat/Vversion/Vheader_len', $packed_info);

  if($info['format'] !== 1)
    throw new Exception("Unknown format: {$info['format']}");

  $header_msgpack = substr($packed_data, 1+4+4, $info['header_len']); 
  $header = config_msgpack_unpack($header_msgpack);

  $payloads_bundle = substr($packed_data, 1+4+4+$info['header_len']);
  
  $entries = array();
  foreach($header as $item)
  {
    list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;

    $payload = substr($payloads_bundle, $offset, $size);

    $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
  }

  return $entries;
}

function _config_unpack_bundle_fmt2(string $packed_data) : array
{
  $packed_info = substr($packed_data, 0, 1+4+4+4);

  $info = unpack('Cformat/Vversion/Vstrids_len/Vheader_len', $packed_info);

  if($info['format'] !== 2)
    throw new Exception("Unknown format: {$info['format']}");

  $strids_msgpack = substr($packed_data, 1+4+4+4, $info['strids_len']); 
  $strids = config_msgpack_unpack($strids_msgpack);

  $header_msgpack = substr($packed_data, 1+4+4+4+$info['strids_len'], $info['header_len']); 
  $header = config_msgpack_unpack($header_msgpack);

  $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
  
  $entries = array();
  foreach($header as $item)
  {
    list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;

    $payload = substr($payloads_bundle, $offset, $size);

    $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); 
  }

  return $entries;
}

//format: [format_id, payload_data]
function _config_get_payload(ConfigCacheEntry $ce, bool $use_lz4, bool $use_config_refs) : array
{
  $format = ConfigCacheEntry::FMT_BINARY;
  $payload = null;

  if($use_config_refs && $ce->payload_file)
  {
    $format = ConfigCacheEntry::FMT_FILE_REF;
    $payload = $ce->payload_file;
  }
  else
  {
    $payload = $ce->payload;
    if($use_lz4 && strlen($payload) > 512)
    {
      $format = ConfigCacheEntry::FMT_LZ4;
      $payload = lz4_compress($payload, 9);
    }
  }

  return array($format, $payload);
}

function _config_unpack_payload(int $format, string $payload) : array
{
  $msg_packed = null;
  if($format === ConfigCacheEntry::FMT_LZ4)
    $msg_packed = lz4_uncompress($payload);
  else if($format === ConfigCacheEntry::FMT_BINARY)
    $msg_packed = $payload;
  else if($format === ConfigCacheEntry::FMT_FILE_REF)
    $msg_packed = ensure_read($payload);
  else
    throw new Exception("Bad format: $format");
  return config_msgpack_unpack($msg_packed);
}

function config_pack_and_write_bundle(
  /*var ConfBase[]*/
  array $configs, 
  string $file_path, 
  int $version,
  bool $use_lz4 = true, 
  int $binary_format = 1, 
  bool $debug = false
)
{
  $cache_entries = array();
  foreach($configs as $conf)
  {
    $payload = config_msgpack_pack($conf->export());

    //creating fake cache entries
    $entry = new ConfigCacheEntry();
    $entry->id = $conf->id;
    $entry->strid = $conf->strid;
    $entry->class_id = $conf->getClassId();
    $entry->class = get_class($conf);
    $entry->payload = $payload;
    $entry->config = $conf;

    $cache_entries[] = $entry;
  }

  $packed_data = config_pack_bundle(
    new ConfigPackParams(
      cache_entries: $cache_entries, 
      use_lz4: $use_lz4, 
      binary_format: $binary_format, 
      version: $version,
      debug: $debug
    )
  );

  ensure_write($file_path, $packed_data);
}