cache_entries = $cache_entries; $this->use_lz4 = $use_lz4; $this->use_config_refs = $use_config_refs; $this->binary_format = $binary_format; $this->version = $version; $this->debug = $debug; } } function config_pack_bundle(ConfigPackParams $params) : string { $t = microtime(true); $packed_data = null; if($params->binary_format == 1) { $packed_data = _config_pack_bundle_fmt1( $params->cache_entries, $params->use_lz4, $params->use_config_refs, $params->version ); } else if($params->binary_format == 2) { $packed_data = _config_pack_bundle_fmt2( $params->cache_entries, $params->use_lz4, $params->use_config_refs, $params->version ); } else throw new Exception("Unknown binary format: {$params->binary_format}"); if($params->debug) config_log("Packed entries: " . sizeof($params->cache_entries) . ", total: " . kb($packed_data) . ", format: {$params->binary_format}, lz4: " . var_export($params->use_lz4, true) . ", refs: " . var_export($params->use_config_refs, true) . ", CRC: " . crc32($packed_data) . ", " . round(microtime(true) - $t,2) . " sec."); return $packed_data; } //NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) function _config_pack_bundle_fmt1( array $cache_entries, bool $use_lz4, bool $use_config_refs, int $version) : string { $MAP = array(); $STRIDMAP = array(); $payloads = array(); $payloads_offset = 0; foreach($cache_entries as $entry) { list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs); $payload_size = strlen($payload); $payloads[] = array($payloads_offset, $payload, $format, $payload_size); $payloads_offset += $payload_size; } $header = array(); foreach($cache_entries as $idx => $entry) { if(isset($MAP[$entry->id])) throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'"); $MAP[$entry->id] = $entry->strid; $strid_crc = crc32($entry->strid); if(isset($STRIDMAP[$strid_crc])) throw new Exception("Duplicating config str id crc for '{$entry->strid}' conflicts with '{$STRIDMAP[$strid_crc]}'"); $STRIDMAP[$strid_crc] = $entry->strid; $header[] = array( $payloads[$idx][2], $entry->id, crc32($entry->strid), $entry->class_id, $payloads[$idx][0], $payloads[$idx][3] ); } $header_msgpack = config_msgpack_pack($header); $payloads_bundle = ''; foreach($payloads as $item) $payloads_bundle .= $item[1]; $packed_data = pack("C", 1) . pack("V", $version) . pack("V", strlen($header_msgpack)) . $header_msgpack . $payloads_bundle; return $packed_data; } //NOTE: strids are stored as lookup strings function _config_pack_bundle_fmt2( array $cache_entries, bool $use_lz4, bool $use_config_refs, int $version) : string { $MAP = array(); $STRIDMAP = array(); $STRIDLIST = array(); $payloads = array(); $strids = array(); $payloads_offset = 0; foreach($cache_entries as $entry) { list($format, $payload) = _config_get_payload($entry, $use_lz4, $use_config_refs); $payload_size = strlen($payload); $payloads[] = array($payloads_offset, $payload, $format, $payload_size); $payloads_offset += $payload_size; $strids_indices = array(); $strid_parts = explode('/', ltrim($entry->strid, '@')); foreach($strid_parts as $strid_part) { if(!isset($STRIDMAP[$strid_part])) { $strid_index = count($STRIDLIST); $STRIDLIST[] = $strid_part; $STRIDMAP[$strid_part] = $strid_index; $strids_indices[] = $strid_index; } else $strids_indices[] = $STRIDMAP[$strid_part]; } $strids[] = $strids_indices; } $header = array(); foreach($cache_entries as $idx => $entry) { if(isset($MAP[$entry->id])) throw new Exception("Duplicating config id for '{$entry->strid}' conflicts with '{$MAP[$entry->id]}'"); $MAP[$entry->id] = $entry->strid; $header[] = array( $payloads[$idx][2], //format $entry->id, $strids[$idx], //strid as a lookup indices $entry->class_id, $payloads[$idx][0], //offset $payloads[$idx][3] //size ); } $strids_msgpack = config_msgpack_pack($STRIDLIST); $header_msgpack = config_msgpack_pack($header); $payloads_bundle = ''; foreach($payloads as $item) $payloads_bundle .= $item[1]; $packed_data = pack("C", 2) . pack("V", $version) . pack("V", strlen($strids_msgpack)) . pack("V", strlen($header_msgpack)) . $strids_msgpack . $header_msgpack . $payloads_bundle; return $packed_data; } //format: [[class_id, [data]], ...[class_id, [data]]] function config_unpack_bundle(string $packed_data) : array { $packed_info = substr($packed_data, 0, 1); $info = unpack('Cformat', $packed_info); if($info['format'] === 1) { return _config_unpack_bundle_fmt1($packed_data); } else if($info['format'] === 2) { return _config_unpack_bundle_fmt2($packed_data); } else throw new Exception("Unknown format: {$info['format']}"); } function _config_unpack_bundle_fmt1(string $packed_data) : array { $packed_info = substr($packed_data, 0, 1+4+4); $info = unpack('Cformat/Vversion/Vheader_len', $packed_info); if($info['format'] !== 1) throw new Exception("Unknown format: {$info['format']}"); $header_msgpack = substr($packed_data, 1+4+4, $info['header_len']); $header = config_msgpack_unpack($header_msgpack); $payloads_bundle = substr($packed_data, 1+4+4+$info['header_len']); $entries = array(); foreach($header as $item) { list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; $payload = substr($payloads_bundle, $offset, $size); $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); } return $entries; } function _config_unpack_bundle_fmt2(string $packed_data) : array { $packed_info = substr($packed_data, 0, 1+4+4+4); $info = unpack('Cformat/Vversion/Vstrids_len/Vheader_len', $packed_info); if($info['format'] !== 2) throw new Exception("Unknown format: {$info['format']}"); $strids_msgpack = substr($packed_data, 1+4+4+4, $info['strids_len']); $strids = config_msgpack_unpack($strids_msgpack); $header_msgpack = substr($packed_data, 1+4+4+4+$info['strids_len'], $info['header_len']); $header = config_msgpack_unpack($header_msgpack); $payloads_bundle = substr($packed_data, 1+4+4+4+$info['strids_len']+$info['header_len']); $entries = array(); foreach($header as $item) { list($format, $id, $strid_crc, $class_id, $offset, $size) = $item; $payload = substr($payloads_bundle, $offset, $size); $entries[$id] = array($class_id, _config_unpack_payload($format, $payload)); } return $entries; } //format: [format_id, payload_data] function _config_get_payload(ConfigCacheEntry $ce, bool $use_lz4, bool $use_config_refs) : array { $format = ConfigCacheEntry::FMT_BINARY; $payload = null; if($use_config_refs && $ce->payload_file) { $format = ConfigCacheEntry::FMT_FILE_REF; $payload = $ce->payload_file; } else { $payload = $ce->payload; if($use_lz4 && strlen($payload) > 512) { $format = ConfigCacheEntry::FMT_LZ4; $payload = lz4_compress($payload, 9); } } return array($format, $payload); } function _config_unpack_payload(int $format, string $payload) : array { $msg_packed = null; if($format === ConfigCacheEntry::FMT_LZ4) $msg_packed = lz4_uncompress($payload); else if($format === ConfigCacheEntry::FMT_BINARY) $msg_packed = $payload; else if($format === ConfigCacheEntry::FMT_FILE_REF) $msg_packed = ensure_read($payload); else throw new Exception("Bad format: $format"); return config_msgpack_unpack($msg_packed); } function config_pack_and_write_bundle( /*var ConfBase[]*/ array $configs, string $file_path, int $version, bool $use_lz4 = true, int $binary_format = 1, bool $debug = false ) { $cache_entries = array(); foreach($configs as $conf) { $payload = config_msgpack_pack($conf->export()); //creating fake cache entries $entry = new ConfigCacheEntry(); $entry->id = $conf->id; $entry->strid = $conf->strid; $entry->class_id = $conf->getClassId(); $entry->class = get_class($conf); $entry->payload = $payload; $entry->config = $conf; $cache_entries[] = $entry; } $packed_data = config_pack_bundle( new ConfigPackParams( cache_entries: $cache_entries, use_lz4: $use_lz4, binary_format: $binary_format, version: $version, debug: $debug ) ); ensure_write($file_path, $packed_data); }