Adding support for RAW messages posting

atf.inc.php

@@ -114,17 +114,17 @@ function _try_lz4_replay(string $txt)
   return "4:" . base64_encode(lz4_compress(base64_decode($txt)));
 }
 
-function log($msg)
+function log(string $msg)
 {
   echo date("Y-m-d H:i:s") . " " . $msg . "\n";
 }
 
-function err($msg)
+function err(string $msg)
 {
   taskman\stderr(date("Y-m-d H:i:s") . " " . $msg . "\n");
 }
 
-function _retry($max_tries, $func)
+function _retry(int $max_tries, callable $func, int $sleep = 1)
 {
   for($i=0;$i<$max_tries;++$i)
   {
@@ -133,12 +133,12 @@ function _retry($max_tries, $func)
       $func();
       return;
     }
-    catch(Exception $e)
+    catch(\Throwable $e)
     {
       if(($i+1) == $max_tries)
         throw $e;
     }
-    sleep(1);
+    sleep($sleep);
   }
 }
 

device.inc.php

@@ -278,11 +278,26 @@ function device_cputop_async(string $atf_host, string $device) : Amp\Promise
   });
 }
 
-function device_temperature_async(string $atf_host, string $device) : Amp\Promise
+function device_cpu_freq_min_async(string $atf_host, string $device, int $core_index = 0) : Amp\Promise
 {
-  return Amp\call(function() use($atf_host, $device) {
+  return device_cpu_measure_async($atf_host, $device, "cpuinfo_min_freq", $core_index); 
+}
+
+function device_cpu_freq_max_async(string $atf_host, string $device, int $core_index = 0) : Amp\Promise
+{
+  return device_cpu_measure_async($atf_host, $device, "cpuinfo_max_freq", $core_index); 
+}
+
+function device_cpu_freq_cur_async(string $atf_host, string $device, int $core_index = 0) : Amp\Promise
+{
+  return device_cpu_measure_async($atf_host, $device, "scaling_cur_freq", $core_index); 
+}
+
+function device_cpu_measure_async(string $atf_host, string $device, string $metric, int $core_index) : Amp\Promise
+{
+  return Amp\call(function() use($atf_host, $device, $metric, $core_index) {
     //NOTE: on current devices 16 thermal zone are responsible for GPU temperature probing
-    list($code, $lines) = yield host_exec_async($atf_host, "%{adb}% -s $device shell cat /sys/class/thermal/thermal_zone16/temp", DEPLOY_OPT_ERR_OK, 1);
+    list($code, $lines) = yield host_exec_async($atf_host, "%{adb}% -s $device shell cat /sys/devices/system/cpu/cpu$core_index/cpufreq/$metric", DEPLOY_OPT_ERR_OK, 1);
 
     if($code !== 0)
       return 0;
@@ -298,3 +313,52 @@ function device_temperature_async(string $atf_host, string $device) : Amp\Promis
     return 0;
   });
 }
+
+function device_temperature_async(string $atf_host, string $device) : Amp\Promise
+{
+  return Amp\call(function() use($atf_host, $device) {
+    list($code, $lines) = yield host_exec_async($atf_host, "%{adb}% -s $device shell dumpsys thermalservice", DEPLOY_OPT_ERR_OK, 1);
+
+    if($code !== 0)
+      return array();
+
+    return device_parse_soc_temps($lines);
+  });
+}
+
+function device_parse_soc_temps($thermal_output_lines) 
+{
+  $cpu_temp = null;
+  $gpu_temp = null;
+
+  $hal_section_found = false;
+
+  foreach($thermal_output_lines as $line) 
+  {
+    if(!$hal_section_found)
+    {
+      if(strpos($line, 'Current temperatures from HAL:') !== false)
+        $hal_section_found = true;
+      continue;
+    }
+
+    if(strpos($line, 'mType=0') !== false) //Type 0 indicates CPI
+      $cpu_temp = device_parse_thermal_line($line);
+
+    if(strpos($line, 'mType=1') !== false) //Type 1 indicates GPU
+      $gpu_temp = device_parse_thermal_line($line); 
+
+    if($cpu_temp && $gpu_temp)
+      break;
+  }
+
+  return [
+    'cpu_temp' => $cpu_temp,
+    'gpu_temp' => $gpu_temp
+  ];
+}
+
+function device_parse_thermal_line($line)
+{
+  return preg_match('/mValue=([^,]+)/', $line, $matches) ? floatval($matches[1]) : null; 
+}

im.inc.php

@@ -8,6 +8,7 @@ interface IMChan
 {
   function post(string $message, array $fields = array()) : array;
   function updatePost(string $thread_id, array $fields = array());
+  function postFile(string $title, string $mime_type, string $file_path, array $fields = array()) : array;
   function postPNG(string $title, string $png_file, array $fields = array()) : array;
   function getPermalink(string $id) : string;
 }
@@ -24,6 +25,11 @@ class NullMessenger implements IMChan
     return array('ok' => true);
   }
 
+  function postFile(string $title, string $mime_type, string $file_path, array $fields = array()) : array
+  {
+    return array('ok' => true);
+  }
+
   function postPNG(string $title, string $png_file, array $fields = array()) : array
   {
     return array('ok' => true);
@@ -60,6 +66,11 @@ class MattermostChan implements IMChan
     return mm_update($this->server, $this->token, $id, $fields);
   }
 
+  function postFile(string $title, string $mime_type, string $file_path, array $fields = array()) : array
+  {
+    return mm_post_file($this->server, $this->token, $this->chan, $title, $mime_type, $file_path, $fields);
+  }
+
   function postPNG(string $title, string $png_file, array $fields = array()) : array
   {
     return mm_post_png($this->server, $this->token, $this->chan, $title, $png_file, $fields);
@@ -122,7 +133,49 @@ function mm_update(string $server, string $token, string $id, array $fields)
   return json_decode($result, true);
 }
 
-function mm_post_png(string $server, string $token, string $chan, string $title, string $png_file, $msg_fields = array())
+function mm_post_file(
+  string $server, string $token, string $chan, 
+  string $title, string $mime_type, string $file_path, 
+  $msg_fields = array()
+)
+{
+  $headers = array(
+    'Content-Type: multipart/form-data',
+    'Authorization: Bearer ' . $token
+  );
+
+  $file = new \CurlFile(realpath($file_path), $mime_type, basename($file_path));
+
+  $fields = array();
+  $fields['channel_id'] = $chan;
+  $fields['files'] =  $file;
+
+  $ch = curl_init("$server/api/v4/files");
+  curl_setopt($ch, CURLOPT_POST, 1);
+  curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
+  curl_setopt($ch, CURLOPT_POSTFIELDS, $fields);
+  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+
+  $result = curl_exec($ch);
+  $status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+  curl_close($ch);
+
+  if($status != 201)
+    throw new Exception("File failed with status: " . $status .", result: " . var_export($result, true));
+
+  $result = json_decode($result, true);
+  $file_id = $result['file_infos'][0]['id'];
+
+  $msg_fields['file_ids'] = array($file_id);
+
+  return mm_post($server, $token, $chan, $title, $msg_fields);
+}
+
+function mm_post_png(
+  string $server, string $token, string $chan, 
+  string $title, string $png_file, 
+  $msg_fields = array()
+)
 {
   $headers = array(
     'Content-Type: multipart/form-data',

plan.inc.php

@@ -16,6 +16,8 @@ class Plan
   const BOGUS_REINSTALL_APP_EVERY_N      = 2;
   const IM_THREAD_UPDATE_INTERVAL        = 5;
 
+  const MaxImTextPostLen = 3000;
+
   public Session $session;
 
   public string $name;
@@ -251,7 +253,17 @@ class Plan
 
       while(!$this->isOver())
       {
+        try
+        {
+          //let's not exit the testing just because we couldn't update IM thread for some reason
+          _retry(3, function() {
             $this->_updateMessengerThread(false);
+          });
+        }
+        catch(\Throwable $e) 
+        {
+          err("Task #{$task->run_id} IM thread update error: {$e} *{$task->device}*...");
+        }
 
         yield Amp\delay((int)($this->session->conf->sleep_time*1000));
 
@@ -288,7 +300,10 @@ class Plan
         }
       }
 
+      _retry(3, function() {
           $this->_updateMessengerThread(true);
+        }
+      );
     });
   }
 
@@ -320,6 +335,7 @@ class Plan
         $task->getCmd(), 
         $task->getCmdArgs()
       );
+      $task->ext_cmd_start_time = time();
     }
     catch(Exception $e)
     {
@@ -537,7 +553,7 @@ class Plan
 
   static function _printToShellExtItem(Task $task, array $item)
   {
-    $shell_msg = _trim($item['message'], 3000);
+    $shell_msg = _trim($item['message'], self::MaxImTextPostLen);
     if(Task::isProblemCode($item['error']))
       $shell_msg = "[PRB] Code:{$item['error']}, $shell_msg";
     $shell_msg = "(".round($item['time'],1)."s) {$shell_msg} *{$task->device}*";
@@ -547,7 +563,7 @@ class Plan
 
   function _postToMessengerExtStatusItem(Task $task, array $item)
   {
-    $orig_msg = _trim($item['message'], 3000);
+    $orig_msg = _trim($item['message'], self::MaxImTextPostLen);
     $mm_msg = $orig_msg;
 
     if($item['error'] == Task::CODE_EXCEPTION)
@@ -567,6 +583,30 @@ class Plan
     return $this->session->conf->im->post($msg, $args);
   }
 
+  function postFileData(string $title, string $mime_type, string $file_data, string $file_extension, array $args = array()) : bool
+  {
+    $args['root_id'] = $this->im_thread_id;
+
+    if($file_data)
+    {
+      $tmp_file = tempnam(sys_get_temp_dir(), 'tmp_upload') . $file_extension;
+      taskman\ensure_write($tmp_file, $file_data);
+      try
+      {
+        $this->session->conf->im->postFile($title, $mime_type, $tmp_file, $args);
+        return true;
+      }
+      catch(Exception $e)
+      {}
+      finally
+      {
+        taskman\ensure_rm($tmp_file);
+      }
+    }
+
+    return false;
+  }
+
   function _analyzeExtStatusItemAsync(Task $task, array $item) : Amp\Promise
   {
     return Amp\call(function() use($task, $item) {
@@ -602,6 +642,10 @@ class Plan
     {
       $this->_postToMessengerExtStatusItem($task, $item);
     }
+    else if($item['error'] == Task::CODE_RAW)
+    {
+      $this->_postRawDataToMessenger($task, $item['message']);
+    }
     else if($msg_type === '[WRN]')
     {
       $task->addStatusCode(Task::CODE_WARN);
@@ -636,9 +680,21 @@ class Plan
     });
   }
 
+  function _postRawDataToMessenger(Task $task, string $txt)
+  {
+    if(strlen($txt) > self::MaxImTextPostLen)
+      $this->postFileData("Raw data: *{$task->device}*", 'text/plain', $txt, '.txt');
+    else
+      $this->post("Raw data: \n```\n$txt\n```\n *{$task->device}*");
+  }
+
   function _postReplayToMessenger(Task $task, $repl_txt)
   {
     $repl_txt = _try_lz4_replay($repl_txt);
+
+    if(strlen($repl_txt) > self::MaxImTextPostLen)
+      $this->postFileData("Replay File: *{$task->device}*", 'text/plain', $repl_txt, '.txt');
+    else
       $this->post("Last Replay: \n```\n$repl_txt\n```\n *{$task->device}*");
   }
 

session.inc.php

@@ -51,6 +51,7 @@ class SessionConfig
 class Session
 {
   const RUN_SLEEP = 3;
+  const IGNORE_DEVICE_TTL = 300;
 
   public string $name;
   public SessionConfig $conf;
@@ -111,18 +112,30 @@ class Session
 
   function getDevices() : array
   {
-    $devices = array_diff(
+    $this->_checkIgnoredDevices();
-      $this->device_pool->get(), 
+
-      $this->ignored_devices
+    $connected_devices = $this->device_pool->get();
-    );
+    $actual_devices = array_diff($connected_devices, array_keys($this->ignored_devices));
-    return $devices;
+
+    return $actual_devices;
+  }
+
+  function _checkIgnoredDevices()
+  {
+    //let's remove from ignored devices devices which were ignored for some time 
+    $devices = array_keys($this->ignored_devices);
+    foreach($devices as $device)
+    {
+      if(time() - $this->ignored_devices[$device] > self::IGNORE_DEVICE_TTL)
+        unset($this->ignored_devices[$device]);
+    }
   }
 
   function ignoreDevice(string $device, string $reason)
   {
     err("Ignoring device *$device*: $reason");
 
-    $this->ignored_devices[] = $device;
+    $this->ignored_devices[$device] = time();
   }
 
   function incBogusDevice(string $device)

task.inc.php

@@ -26,6 +26,7 @@ class Task
   const CODE_MESSAGE   = 0;
   const CODE_EXCEPTION = 1;
   const CODE_WARN      = 2;
+  const CODE_RAW       = 3;
   const CODE_NSTART    = 124;
   const CODE_STUCK     = 125;
   const CODE_GONE      = 126;
@@ -42,6 +43,8 @@ class Task
         return "exception";
       case self::CODE_WARN:
         return "warning";
+      case self::CODE_RAW:
+        return "raw";
       case self::CODE_STUCK:
         return "stuck";
       case self::CODE_NSTART:
@@ -79,6 +82,7 @@ class Task
 
   public float $start_time = 0;
   public float $reset_time = 0;
+  public float $ext_cmd_start_time = 0;
 
   public float $last_progress = 0;
   public int $last_done_arg_idx = -1;