637 lines
18 KiB
PHP
637 lines
18 KiB
PHP
<?php
|
|
namespace taskman;
|
|
use Exception;
|
|
|
|
class ConfigPackParams
|
|
{
|
|
/*var ConfigCacheEntry[]*/
|
|
public array $cache_entries;
|
|
public bool $use_lz4 = false;
|
|
public bool $use_config_refs = false;
|
|
public int $binary_format = 1; //1,2 supported
|
|
public ?int $version = null;
|
|
public bool $debug = false;
|
|
|
|
public const string EXTRA_FMT3_CHUNK_SIZE = "EXTRA_FMT3_CHUNK_SIZE";
|
|
public const string EXTRA_FMT3_COMPRESSION_LEVEL = "EXTRA_FMT3_COMPRESSION_LEVEL";
|
|
public array $extras = array();
|
|
|
|
function __construct(
|
|
array $cache_entries,
|
|
int $version,
|
|
bool $use_lz4 = false,
|
|
bool $use_config_refs = false,
|
|
int $binary_format = 1,
|
|
bool $debug = false,
|
|
array $extras = array()
|
|
)
|
|
{
|
|
$this->cache_entries = $cache_entries;
|
|
$this->use_lz4 = $use_lz4;
|
|
$this->use_config_refs = $use_config_refs;
|
|
$this->binary_format = $binary_format;
|
|
$this->version = $version;
|
|
$this->debug = $debug;
|
|
$this->extras = $extras;
|
|
}
|
|
}
|
|
|
|
function config_pack_bundle(ConfigPackParams $params) : string
|
|
{
|
|
$t = microtime(true);
|
|
|
|
$packed_data = null;
|
|
|
|
if($params->binary_format == 1)
|
|
{
|
|
$packed_data = _config_pack_bundle_fmt1(
|
|
$params->cache_entries,
|
|
$params->use_lz4,
|
|
$params->use_config_refs,
|
|
$params->version
|
|
);
|
|
}
|
|
else if($params->binary_format == 2)
|
|
{
|
|
$packed_data = _config_pack_bundle_fmt2(
|
|
$params->cache_entries,
|
|
$params->use_lz4,
|
|
$params->use_config_refs,
|
|
$params->version,
|
|
);
|
|
}
|
|
else if($params->binary_format == 3)
|
|
{
|
|
$packed_data = _config_pack_bundle_fmt3(
|
|
$params->cache_entries,
|
|
$params->use_lz4,
|
|
$params->use_config_refs,
|
|
$params->version,
|
|
$params->extras[ConfigPackParams::EXTRA_FMT3_CHUNK_SIZE],
|
|
$params->extras[ConfigPackParams::EXTRA_FMT3_COMPRESSION_LEVEL],
|
|
);
|
|
}
|
|
else
|
|
throw new Exception("Unknown binary format: {$params->binary_format}");
|
|
|
|
if($params->debug)
|
|
config_log("Packed entries: " . sizeof($params->cache_entries) . ", total: " .
|
|
kb($packed_data) . ", format: {$params->binary_format}, lz4: " .
|
|
var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) .
|
|
", CRC: " . crc32($packed_data) .
|
|
", " . round(microtime(true) - $t,2) . " sec.");
|
|
|
|
return $packed_data;
|
|
}
|
|
|
|
function config_patch_bundle(ConfigPackParams $params, string $packed_data) : string
|
|
{
|
|
$t = microtime(true);
|
|
|
|
$patched_data = null;
|
|
|
|
if($params->binary_format == 2)
|
|
{
|
|
$patched_data = _config_patch_bundle_fmt2(
|
|
$packed_data,
|
|
$params->cache_entries,
|
|
$params->use_lz4,
|
|
$params->use_config_refs,
|
|
$params->version,
|
|
);
|
|
}
|
|
else
|
|
throw new Exception("Unknown binary format: {$params->binary_format}");
|
|
|
|
if($params->debug)
|
|
config_log("Patched entries: " . sizeof($params->cache_entries) . ", total: " .
|
|
kb($patched_data) . ", format: {$params->binary_format}, lz4: " .
|
|
var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) .
|
|
", CRC: " . crc32($patched_data) .
|
|
", " . round(microtime(true) - $t,2) . " sec.");
|
|
|
|
return $patched_data;
|
|
}
|
|
|
|
//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build)
|
|
function _config_pack_bundle_fmt1(
|
|
array $cache_entries,
|
|
bool $use_lz4,
|
|
bool $use_config_refs,
|
|
int $version) : string
|
|
{
|
|
$MAP = array();
|
|
$STRIDMAP = array();
|
|
|
|
$payloads = array();
|
|
$payloads_offset = 0;
|
|
foreach($cache_entries as $entry)
|
|
{
|
|
list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
|
|
$payload_size = strlen($payload);
|
|
$payloads[] = array($payloads_offset, $payload, $format, $payload_size);
|
|
$payloads_offset += $payload_size;
|
|
}
|
|
|
|
$header = array();
|
|
foreach($cache_entries as $idx => $entry)
|
|
{
|
|
if(isset($MAP[$entry->id]))
|
|
throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
|
|
$MAP[$entry->id] = $entry->strid;
|
|
|
|
$strid_crc = crc32($entry->strid);
|
|
if(isset($STRIDMAP[$strid_crc]))
|
|
throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
|
|
$STRIDMAP[$strid_crc] = $entry->strid;
|
|
|
|
$header[] = array(
|
|
$payloads[$idx][2],
|
|
$entry->id,
|
|
crc32($entry->strid),
|
|
$entry->class_id,
|
|
$payloads[$idx][0],
|
|
$payloads[$idx][3]
|
|
);
|
|
}
|
|
|
|
$header_msgpack = config_msgpack_pack($header);
|
|
$payloads_bundle = '';
|
|
foreach($payloads as $item)
|
|
$payloads_bundle .= $item[1];
|
|
|
|
$packed_data =
|
|
pack("C", 1) .
|
|
pack("V", $version) .
|
|
pack("V", strlen($header_msgpack)) .
|
|
$header_msgpack .
|
|
$payloads_bundle;
|
|
|
|
return $packed_data;
|
|
}
|
|
|
|
//NOTE: strids are stored as lookup strings, and actually an array of lookup string indices
|
|
// (each path item separated by '/' is stored as an array item)
|
|
function _config_pack_bundle_fmt2(
|
|
array $cache_entries,
|
|
bool $use_lz4,
|
|
bool $use_config_refs,
|
|
int $version,
|
|
) : string
|
|
{
|
|
$MAP = array();
|
|
$STRIDMAP = array();
|
|
$STRIDLIST = array();
|
|
|
|
$payloads = array();
|
|
$strids = array();
|
|
$payloads_offset = 0;
|
|
foreach($cache_entries as $entry)
|
|
{
|
|
list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
|
|
$payload_size = strlen($payload);
|
|
$payloads[] = array($payloads_offset, $payload, $format, $payload_size);
|
|
$payloads_offset += $payload_size;
|
|
|
|
$strids_indices = _config_encode_strid_as_indices($entry->strid, $STRIDMAP, $STRIDLIST);
|
|
|
|
$strids[] = $strids_indices;
|
|
}
|
|
|
|
$header = array();
|
|
foreach($cache_entries as $idx => $entry)
|
|
{
|
|
if(isset($MAP[$entry->id]))
|
|
throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
|
|
$MAP[$entry->id] = $entry->strid;
|
|
|
|
$header[] = array(
|
|
$payloads[$idx][2], //format
|
|
$entry->id,
|
|
$strids[$idx], //strid as a lookup indices
|
|
$entry->class_id,
|
|
$payloads[$idx][0], //offset
|
|
$payloads[$idx][3] //size
|
|
);
|
|
}
|
|
|
|
$strids_msgpack = config_msgpack_pack($STRIDLIST);
|
|
|
|
$header_msgpack = config_msgpack_pack($header);
|
|
$payloads_bundle = '';
|
|
foreach($payloads as $item)
|
|
$payloads_bundle .= $item[1];
|
|
|
|
$packed_data =
|
|
pack("C", 2) .
|
|
pack("V", $version) .
|
|
pack("V", strlen($strids_msgpack)) .
|
|
pack("V", strlen($header_msgpack)) .
|
|
$strids_msgpack .
|
|
$header_msgpack .
|
|
$payloads_bundle;
|
|
|
|
return $packed_data;
|
|
}
|
|
|
|
//NOTE: Much like fmt1, but configs entries are grouped into sizeable chuncks, with each chunk lz4-compressed.
|
|
// This reduces overall bundle size when there are many small configs entries.
|
|
function _config_pack_bundle_fmt3(
|
|
array $cache_entries,
|
|
bool $use_lz4,
|
|
bool $use_config_refs,
|
|
int $version,
|
|
int $chunk_size,
|
|
int $compression_level) : string
|
|
{
|
|
if(!$use_lz4)
|
|
throw new Exception("Config bundle FMT3 is only available with LZ4 enabled");
|
|
|
|
if($compression_level < 0 || $compression_level > 12)
|
|
throw new Exception("LZ4 compression level must be in range [0, 12]");
|
|
|
|
$MAP = array();
|
|
$STRIDMAP = array();
|
|
|
|
$header = array();
|
|
$payloads_offset = 0;
|
|
$max_chunk_size = 0;
|
|
$chunk_offset = 0;
|
|
$payloads_bundle = '';
|
|
$payloads_buffer = '';
|
|
$count_entries = count($cache_entries);
|
|
foreach($cache_entries as $idx => $entry)
|
|
{
|
|
list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs);
|
|
$payload_size = strlen($payload);
|
|
$payloads_buffer .= $payload;
|
|
|
|
if(isset($MAP[$entry->id]))
|
|
throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'");
|
|
$MAP[$entry->id] = $entry->strid;
|
|
|
|
$strid_crc = crc32($entry->strid);
|
|
if(isset($STRIDMAP[$strid_crc]))
|
|
throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'");
|
|
$STRIDMAP[$strid_crc] = $entry->strid;
|
|
|
|
$header[] = array(
|
|
$format,
|
|
$entry->id,
|
|
crc32($entry->strid),
|
|
$entry->class_id,
|
|
$chunk_offset,
|
|
$payloads_offset,
|
|
$payload_size
|
|
);
|
|
|
|
$payloads_offset += $payload_size;
|
|
|
|
if($payloads_offset >= $chunk_size || $idx == ($count_entries - 1))
|
|
{
|
|
if($payloads_offset > $max_chunk_size)
|
|
$max_chunk_size = $payloads_offset;
|
|
$payloads_offset = 0;
|
|
$lz4_data = lz4_compress($payloads_buffer, $compression_level);
|
|
$payloads_bundle .= pack("V", strlen($lz4_data));
|
|
$payloads_bundle .= $lz4_data;
|
|
$chunk_offset = strlen($payloads_bundle);
|
|
$payloads_buffer = '';
|
|
}
|
|
}
|
|
|
|
$header_msgpack = config_msgpack_pack($header);
|
|
|
|
$packed_data =
|
|
pack("C", 3) .
|
|
pack("V", $version) .
|
|
pack("V", strlen($header_msgpack)) .
|
|
pack("V", $max_chunk_size) .
|
|
$header_msgpack .
|
|
$payloads_bundle;
|
|
|
|
return $packed_data;
|
|
}
|
|
|
|
function _config_encode_strid_as_indices(string $strid, array &$STRIDMAP, array &$STRIDLIST) : array
|
|
{
|
|
$strids_indices = array();
|
|
$strid_parts = explode('/', ltrim($strid, '@'));
|
|
foreach($strid_parts as $strid_part)
|
|
{
|
|
if(!isset($STRIDMAP[$strid_part]))
|
|
{
|
|
$strid_index = count($STRIDLIST);
|
|
$STRIDLIST[] = $strid_part;
|
|
$STRIDMAP[$strid_part] = $strid_index;
|
|
$strids_indices[] = $strid_index;
|
|
}
|
|
else
|
|
$strids_indices[] = $STRIDMAP[$strid_part];
|
|
}
|
|
return $strids_indices;
|
|
}
|
|
|
|
function _config_patch_bundle_fmt2(
|
|
string $packed_data,
|
|
array $patch_entries,
|
|
bool $use_lz4,
|
|
bool $use_config_refs,
|
|
int $version,
|
|
) : string
|
|
{
|
|
list($strids, $header, $_, $payloads_bundle) =
|
|
_config_unpack_bundle_fmt2(packed_data: $packed_data, unpack_entries: false);
|
|
|
|
$stridmap = array_flip($strids);
|
|
|
|
foreach($patch_entries as $idx => $patch_entry)
|
|
{
|
|
list($patch_format, $patch_payload) = _config_get_payload($patch_entry, $use_lz4, $use_config_refs);
|
|
|
|
$header_found = array_filter($header, fn($item) => $item[1] == $patch_entry->id);
|
|
|
|
if($header_found)
|
|
{
|
|
$header_idx = key($header_found);
|
|
$header_entry = current($header_found);
|
|
|
|
$current_offset = $header_entry[4];
|
|
$current_size = $header_entry[5];
|
|
if($current_size >= strlen($patch_payload))
|
|
{
|
|
//let's do the inline patching
|
|
$payloads_bundle = substr_replace($payloads_bundle, $patch_payload, $current_offset, strlen($patch_payload));
|
|
|
|
$header_entry[0] = $patch_format;
|
|
$header_entry[5] = strlen($patch_payload);
|
|
$header[$header_idx] = $header_entry;
|
|
}
|
|
else
|
|
{
|
|
//let's add it to the end
|
|
$header_entry[0] = $patch_format;
|
|
$header_entry[4] = strlen($payloads_bundle);
|
|
$header_entry[5] = strlen($patch_payload);
|
|
$header[$header_idx] = $header_entry;
|
|
|
|
$payloads_bundle .= $patch_payload;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//let's add new entry it to the end
|
|
$strid_indices = _config_encode_strid_as_indices($patch_entry->strid, $stridmap, $strids);
|
|
|
|
$header_entry = array(
|
|
$patch_format,
|
|
$patch_entry->id,
|
|
$strid_indices,
|
|
$patch_entry->class_id,
|
|
strlen($payloads_bundle),
|
|
strlen($patch_payload),
|
|
);
|
|
|
|
$header[] = $header_entry;
|
|
$payloads_bundle .= $patch_payload;
|
|
}
|
|
}
|
|
|
|
$strids_msgpack = config_msgpack_pack($strids);
|
|
$header_msgpack = config_msgpack_pack($header);
|
|
|
|
$patched_data =
|
|
pack("C", 2) .
|
|
pack("V", $version) .
|
|
pack("V", strlen($strids_msgpack)) .
|
|
pack("V", strlen($header_msgpack)) .
|
|
$strids_msgpack .
|
|
$header_msgpack .
|
|
$payloads_bundle;
|
|
|
|
return $patched_data;
|
|
}
|
|
|
|
//format: [[class_id, [data]], ...[class_id, [data]]]
|
|
function config_unpack_bundle(string $packed_data) : array
|
|
{
|
|
$packed_info = substr($packed_data, 0, 1);
|
|
$info = unpack('Cformat', $packed_info);
|
|
|
|
if($info['format'] === 1)
|
|
{
|
|
return _config_unpack_bundle_fmt1($packed_data);
|
|
}
|
|
else if($info['format'] === 2)
|
|
{
|
|
list($_, $_, $entries) = _config_unpack_bundle_fmt2($packed_data);
|
|
return $entries;
|
|
}
|
|
else if($info['format'] === 3)
|
|
{
|
|
return _config_unpack_bundle_fmt3($packed_data);
|
|
}
|
|
else
|
|
throw new Exception("Unknown format: {$info['format']}");
|
|
}
|
|
|
|
function _config_unpack_bundle_fmt1(string $packed_data) : array
|
|
{
|
|
$packed_info = substr($packed_data, 0, 1+4+4);
|
|
|
|
$info = unpack('Cformat/Vversion/Vheader_len', $packed_info);
|
|
|
|
if($info['format'] !== 1)
|
|
throw new Exception("Unknown format: {$info['format']}");
|
|
|
|
$header_msgpack = substr($packed_data, 1+4+4, $info['header_len']);
|
|
$header = config_msgpack_unpack($header_msgpack);
|
|
|
|
$payloads_bundle = substr($packed_data, 1+4+4+$info['header_len']);
|
|
|
|
$entries = array();
|
|
foreach($header as $item)
|
|
{
|
|
list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
|
|
|
|
$payload = substr($payloads_bundle, $offset, $size);
|
|
|
|
$entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
|
|
}
|
|
|
|
return $entries;
|
|
}
|
|
|
|
function _config_unpack_bundle_fmt2(string $packed_data, bool $unpack_entries = true) : array
|
|
{
|
|
$packed_info = substr($packed_data, 0, 1+4+4+4);
|
|
|
|
$info = unpack('Cformat/Vversion/Vstrids_len/Vheader_len', $packed_info);
|
|
|
|
if($info['format'] !== 2)
|
|
throw new Exception("Unknown format: {$info['format']}");
|
|
|
|
$strids_msgpack = substr($packed_data, 1+4+4+4, $info['strids_len']);
|
|
$strids = config_msgpack_unpack($strids_msgpack);
|
|
|
|
$header_msgpack = substr($packed_data, 1+4+4+4+$info['strids_len'], $info['header_len']);
|
|
$header = config_msgpack_unpack($header_msgpack);
|
|
|
|
$payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']);
|
|
|
|
$entries = array();
|
|
|
|
if($unpack_entries)
|
|
{
|
|
foreach($header as $item)
|
|
{
|
|
list($format, $id, $strid_crc, $class_id, $offset, $size) = $item;
|
|
|
|
$payload = substr($payloads_bundle, $offset, $size);
|
|
|
|
$entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
|
|
}
|
|
}
|
|
|
|
return array($strids, $header, $entries, $unpack_entries ? null : $payloads_bundle);
|
|
}
|
|
|
|
function _config_unpack_bundle_fmt3(string $packed_data): array
|
|
{
|
|
if(ord($packed_data[0]) !== 3)
|
|
{
|
|
throw new Exception("Invalid config bundle format");
|
|
}
|
|
|
|
$offset = 1;
|
|
$version = unpack("V", substr($packed_data, $offset, 4))[1];
|
|
$offset += 4;
|
|
$header_len = unpack("V", substr($packed_data, $offset, 4))[1];
|
|
$offset += 4;
|
|
$max_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
|
|
$offset += 4;
|
|
|
|
$header_msgpack = substr($packed_data, $offset, $header_len);
|
|
$offset += $header_len;
|
|
|
|
$header = config_msgpack_unpack($header_msgpack);
|
|
|
|
$cache_entries = [];
|
|
$chunk_offset = 0;
|
|
$chunk_buffer = '';
|
|
$chunk_id = -1;
|
|
|
|
foreach ($header as $entry_data)
|
|
{
|
|
list($format, $id, $strid_crc, $class_id, $entry_chunk_offset, $payload_offset_within_chunk, $payload_size) = $entry_data;
|
|
|
|
if($entry_chunk_offset !== $chunk_id)
|
|
{
|
|
if($chunk_offset !== -1)
|
|
{
|
|
$lz4_chunk_size = unpack("V", substr($packed_data, $offset, 4))[1];
|
|
$offset+=4;
|
|
$lz4_chunk_data = substr($packed_data, $offset, $lz4_chunk_size);
|
|
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
|
|
$offset += $lz4_chunk_size;
|
|
$chunk_offset = $offset;
|
|
}
|
|
else
|
|
{
|
|
$lz4_chunk_size = unpack("V", substr($packed_data, $chunk_offset, 4))[1];
|
|
$chunk_offset += 4;
|
|
$lz4_chunk_data = substr($packed_data, $chunk_offset, $lz4_chunk_size);
|
|
$chunk_buffer = lz4_uncompress($lz4_chunk_data);
|
|
$chunk_offset += $lz4_chunk_size;
|
|
}
|
|
|
|
$chunk_id = $entry_chunk_offset;
|
|
}
|
|
|
|
$payload = substr($chunk_buffer, $payload_offset_within_chunk, $payload_size);
|
|
|
|
$cache_entries[$id] = array($class_id, _config_unpack_payload($format, $payload));
|
|
}
|
|
|
|
return $cache_entries;
|
|
}
|
|
|
|
//format: [format_id, payload_data]
|
|
function _config_get_payload(ConfigCacheEntry $ce, bool $use_lz4, bool $use_config_refs) : array
|
|
{
|
|
$format = ConfigCacheEntry::FMT_BINARY;
|
|
$payload = null;
|
|
|
|
if($use_config_refs && $ce->payload_file)
|
|
{
|
|
$format = ConfigCacheEntry::FMT_FILE_REF;
|
|
$payload = $ce->payload_file;
|
|
}
|
|
else
|
|
{
|
|
$payload = $ce->payload;
|
|
if($use_lz4 && strlen($payload) > 512)
|
|
{
|
|
$format = ConfigCacheEntry::FMT_LZ4;
|
|
$payload = lz4_compress($payload, 9);
|
|
}
|
|
}
|
|
|
|
return array($format, $payload);
|
|
}
|
|
|
|
function _config_unpack_payload(int $format, string $payload) : array
|
|
{
|
|
$msg_packed = null;
|
|
if($format === ConfigCacheEntry::FMT_LZ4)
|
|
$msg_packed = lz4_uncompress($payload);
|
|
else if($format === ConfigCacheEntry::FMT_BINARY)
|
|
$msg_packed = $payload;
|
|
else if($format === ConfigCacheEntry::FMT_FILE_REF)
|
|
$msg_packed = ensure_read($payload);
|
|
else
|
|
throw new Exception("Bad format: $format");
|
|
return config_msgpack_unpack($msg_packed);
|
|
}
|
|
|
|
function config_pack_and_write_bundle(
|
|
/*var ConfBase[]*/
|
|
array $configs,
|
|
string $file_path,
|
|
int $version,
|
|
bool $use_lz4 = true,
|
|
int $binary_format = 1,
|
|
bool $debug = false
|
|
)
|
|
{
|
|
$cache_entries = array();
|
|
foreach($configs as $conf)
|
|
{
|
|
$payload = config_msgpack_pack($conf->export());
|
|
|
|
//creating fake cache entries
|
|
$entry = new ConfigCacheEntry();
|
|
$entry->id = $conf->id;
|
|
$entry->strid = $conf->strid;
|
|
$entry->class_id = $conf->getClassId();
|
|
$entry->class = get_class($conf);
|
|
$entry->payload = $payload;
|
|
$entry->config = $conf;
|
|
|
|
$cache_entries[] = $entry;
|
|
}
|
|
|
|
$packed_data = config_pack_bundle(
|
|
new ConfigPackParams(
|
|
cache_entries: $cache_entries,
|
|
use_lz4: $use_lz4,
|
|
binary_format: $binary_format,
|
|
version: $version,
|
|
debug: $debug
|
|
)
|
|
);
|
|
|
|
ensure_write($file_path, $packed_data);
|
|
}
|
|
|