Improving initialization of workers; Adding sorting when splitting config jobs; Minor improvements here and there

cache.inc.php

@@ -170,22 +170,27 @@ class ConfigFetchParams
   {
     $max_workers = $this->max_workers;
     if($max_workers === null)
-      $max_workers = $this->scanned->count() < 20 ? 1 : 5;
+      $max_workers = $this->scanned->count() < 100 ? 1 : 5;
     return $max_workers;
   }
 
-  function splitFilesByChunks(int $max_workers) : array
+  function splitFilesByChunks(int $max_workers, bool $sort = true) : array
   {
-    $chunk_size = (int)ceil($this->scanned->count()/$max_workers);
+    $flat = $this->scanned->getFlatArray();
-    return array_chunk($this->scanned->getFlatArray(), $chunk_size);
+    if($sort)
+      usort($flat, fn($a, $b) => $a[1] <=> $b[1]);
+
+    $chunk_size = (int)ceil(count($flat)/$max_workers);
+    return array_chunk($flat, $chunk_size);
   }
 
   //returns [[idx, time, [[[base_dir, file1], [base_dir, file2], ..]]], ]
-  function splitJobs() : array
+  function splitJobs(bool $sort = true) : array
   {
     $max_workers = $this->calcMaxWorkers();
     $jobs = array();
-    foreach($this->splitFilesByChunks($max_workers) as $idx => $chunk)
+    $chunks = $this->splitFilesByChunks($max_workers, $sort); 
+    foreach($chunks as $idx => $chunk)
       $jobs[] = array($idx, microtime(true), $chunk);
     return $jobs;
   }
@@ -259,12 +264,7 @@ function _config_cache_fetch(ConfigFetchParams $params) : ConfigFetchResult
   if($params->scanned->isEmpty())
     return new ConfigFetchResult();
 
-  //TODO: not sure if it's the best place for this one
+  $jobs = $params->splitJobs(sort: true);
-  $GLOBALS['CONFIG_GLOBALS'] = $params->globals;
-
-  $params->scanned->apply(function($base_dir, $files) { sort($files); return $files;});
-
-  $jobs = $params->splitJobs();
 
   $serial = sizeof($jobs) == 1;
 

config.inc.php

@@ -23,21 +23,16 @@ class ConfigGlobals
     $this->worker_init_fn = $worker_init_fn;
   }
 
-  function initWorker()
+  function initWorker(bool $is_master_proc)
   {
     $GLOBALS['CONFIG_GLOBALS'] = $this;
 
     if($this->worker_init_fn !== null)
     {
       $fn = $this->worker_init_fn;
-      $fn();
+      $fn($is_master_proc);
     }
   }
-
-  function setNormalizeBaseDirs(array $dirs)
-  {
-    $this->base_dirs = array_map(function($d) { return normalize_path($d); }, $dirs);
-  }
 }
 
 function config_log($msg)

pack.inc.php

@@ -61,6 +61,7 @@ function config_pack_bundle(ConfigPackParams $params) : string
   return $packed_data;
 }
 
+//NOTE: strids are stored as CRCs, potential collision may happen (error will be raised during build) 
 function _config_pack_bundle_fmt1(
   array $cache_entries, 
   bool $use_lz4, 
@@ -117,6 +118,7 @@ function _config_pack_bundle_fmt1(
   return $packed_data;
 }
 
+//NOTE: strids are stored as lookup strings
 function _config_pack_bundle_fmt2(
   array $cache_entries, 
   bool $use_lz4, 

task.inc.php

@@ -33,7 +33,7 @@ function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $
   {
     $results_by_job = array();
     foreach($jobs as $job)
-      $results_by_job[] = _config_worker_func($params, $job);
+      $results_by_job[] = _config_worker_func($params, $job, $serial);
     return $results_by_job;
   }
   else
@@ -47,11 +47,11 @@ function _config_worker_run_procs(ConfigFetchParams $params, array $jobs, bool $
 }
 
 //returns [[base_dir, file, cache_file, was_stale, parser_type, error], ...]
-function _config_worker_func(ConfigFetchParams $params, array $job) : array
+function _config_worker_func(ConfigFetchParams $params, array $job, bool $is_master_proc = false) : array
 {
   $start_time = microtime(true);
 
-  $params->globals->initWorker();
+  $params->globals->initWorker($is_master_proc);
 
   list($idx, $start_time, $chunk) = $job;
   if($params->verbose)