bit
/
taskman_atf
26
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preparing new major release

This commit is contained in:
Pavel Shevaev 2023-11-07 15:42:40 +03:00
parent 6fac6409e5
commit c4cf2ea195
10 changed files with 2282 additions and 1591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
tags

1774
atf.inc.php
View File

File diff suppressed because it is too large Load Diff

12
composer.json
View File

@ -1,12 +1,16 @@
{ {
"name": "bit/taskman_atf", "name": "bit/taskman_atf",
"description": "android test farm", "description": "Automation Test Farm routines",
"homepage": "https://git.bit5.ru/composer/taskman_atf", "homepage": "https://git.bit5.ru/composer/taskman_atf",
"require": { "require": {
"php": ">=7.4" "php": ">=7.4"
}, },
"autoload": { "autoload": {
"classmap": ["atf.inc.php"], "files": ["atf.inc.php"],
"classmap": ["slack.inc.php"] "classmap" : [
"session.inc.php", "plan.inc.php",
"tasks.inc.php", "device.inc.php",
"im.inc.php", "stats.inc.php"
]
} }
} }

161
device.inc.php Normal file
View File

@ -0,0 +1,161 @@
<?php
namespace ATF;
use Exception;
interface IDevicePool
{
function get() : array;
}
class ApkInstaller
{
const ST_IN_PROGRESS = 1;
const ST_INSTALLED = 2;
private string $atf_host;
private ?string $apk_name;
private int $max_installs_in_progress;
private array $installs = array();
private bool $override_external_files;
private array $external_files = array();
function __construct(
string $atf_host,
int $max_installs_in_progress,
?string $apk_name,
bool $override_external_files,
array $external_files
)
{
$this->atf_host = $atf_host;
$this->max_installs_in_progress = $max_installs_in_progress;
$this->apk_name = $apk_name;
$this->override_external_files = $override_external_files;
$this->external_files = $external_files;
}
function isInstalled(string $device)
{
return isset($this->installs[$device]) && $this->installs[$device] == self::ST_INSTALLED;
}
function forgetInstall(string $device)
{
unset($this->installs[$device]);
}
function countInstallsInProgress()
{
$c = 0;
foreach($this->installs as $status)
if($status == self::ST_IN_PROGRESS)
++$c;
return $c;
}
function installAsync(string $device, bool $force = false) : Amp\Promise
{
return Amp\call(function() use($device, $force) {
while($this->countInstallsInProgress() >= $this->max_installs_in_progress)
yield Amp\delay(1000);
if($force)
unset($this->installs[$device]);
if(isset($this->installs[$device]))
return;
$this->installs[$device] = self::ST_IN_PROGRESS;
try
{
yield host_exec_async($this->atf_host, "%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK);
if($this->apk_name !== null)
{
yield host_exec_async($this->atf_host, "%{adb}% -s $device uninstall %{package_id}%", DEPLOY_OPT_ERR_OK, 30);
yield host_exec_async($this->atf_host, "%{adb}% -s $device install -r %{dir}%/{$this->apk_name}", 0, 300);
//delete overrides only if there's a new apk file uploaded
yield del_external_files_async($this->atf_host, $device, $this->external_files);
}
if($this->override_external_files)
yield put_external_files_async($this->atf_host, $device, $this->external_files);
$this->installs[$device] = self::ST_INSTALLED;
}
catch(Exception $e)
{
err("Error during install *$device*: " . $e->getMessage());
unset($this->installs[$device]);
throw $e;
}
});
}
function getSize()
{
list($status, $lines) = host_exec($this->atf_host, "ls -sd -- %{dir}%/{$this->apk_name}", DEPLOY_OPT_ERR_OK|DEPLOY_OPT_SILENT);
if($status != 0)
return 0;
$items = explode(' ', $lines[0]);
//ls -s returns size in blocks where each block is 512 bytes long
return intval($items[0])*512;
}
}
class AdbDevicePool implements IDevicePool
{
private string $atf_host;
function __construct(string $atf_host)
{
$this->atf_host = $atf_host;
}
function get() : array
{
return get_devices($this->atf_host);
}
}
class FixedDevicePool implements IDevicePool
{
private array $devices;
function __construct(array $devices)
{
$this->devices = $devices;
}
function get() : array
{
return $this->devices;
}
}
class CachedDevices implements IDevicePool
{
private IDevicePool $provider;
private ?array $cached = null;
private int $last_cache_time;
private int $keep_cache_time;
function __construct(IDevicePool $provider, int $keep_cache_time)
{
$this->provider = $provider;
$this->keep_cache_time = $keep_cache_time;
}
function get() : array
{
if($this->cached === null || (time() - $this->last_cache_time) > $this->keep_cache_time)
{
$this->cached = $this->provider->get();
$this->last_cache_time = time();
}
return $this->cached;
}
}

183
im.inc.php Normal file
View File

@ -0,0 +1,183 @@
<?php
namespace ATF;
use Exception;
interface IMChan
{
function post(string $message, array $fields = array()) : array;
function updatePost(string $thread_id, array $fields = array());
function postPNG(string $title, string $png_file, array $fields = array()) : array;
function getPermalink(string $id) : string;
}
class NullMessenger implements IMChan
{
function post(string $message, array $fields = array()) : array
{
return array('ok' => true, 'id' => 1);
}
function updatePost(string $id, array $fields = array()) : array
{
return array('ok' => true);
}
function postPNG(string $title, string $png_file, array $fields = array()) : array
{
return array('ok' => true);
}
function getPermalink(string $id) : string
{
return "https://fake/$id";
}
}
class MattermostChan implements IMChan
{
private string $server;
private string $token;
private string $team;
private string $chan;
function __construct(string $server, string $token, string $team, string $chan)
{
$this->server = $server;
$this->token = $token;
$this->team = $team;
$this->chan = $chan;
}
function post(string $message, array $fields = array()) : array
{
return mm_post($this->server, $this->token, $this->chan, $message, $fields);
}
function updatePost(string $id, array $fields = array()) : array
{
return mm_update($this->server, $this->token, $id, $fields);
}
function postPNG(string $title, string $png_file, array $fields = array()) : array
{
return mm_post_png($this->server, $this->token, $this->chan, $title, $png_file, $fields);
}
function getPermalink(string $id) : string
{
return mm_get_permalink($this->server, $this->team, $id);
}
}
function mm_post(string $server, string $token, string $chan, string $message, $fields = array())
{
$ch = _atf_mm_post_curl($server, $token, $chan, $message, $fields);
$result = curl_exec($ch);
curl_close($ch);
$json = json_decode($result, true);
if(!$json)
return array('ok' => false, 'id' => 0);
return array('ok' => true, 'id' => $json['id']);
}
function mm_post_async(string $server, string $token, string $chan, string $message, $fields = array()) : Amp\Promise
{
return Amp\call(function() use($server, $token, $chan, $message, $fields) {
$ch = _atf_mm_post_curl($server, $token, $chan, $message, $fields);
$result = yield _atf_multi_curl_async($ch, true);
curl_close($ch);
$json = json_decode($result, true);
if(!$json)
return array('ok' => false, 'id' => 0);
return array('ok' => true, 'id' => $json['id']);
});
}
function mm_update(string $server, string $token, string $id, array $fields)
{
$ch = curl_init("$server/api/v4/posts/$id");
$headers = array(
'Content-Type: application/json',
'Authorization: Bearer ' . $token
);
$fields["post_id"] = $id;
$fields["id"] = $id;
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT');
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($fields));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$result = curl_exec($ch);
curl_close($ch);
return json_decode($result, true);
}
function mm_post_png(string $server, string $token, string $chan, string $title, string $png_file, $msg_fields = array())
{
$headers = array(
'Content-Type: multipart/form-data',
'Authorization: Bearer ' . $token
);
$file = new \CurlFile(realpath($png_file), 'image/png', basename($png_file));
$fields = array();
$fields['channel_id'] = $chan;
$fields['files'] = $file;
$ch = curl_init("$server/api/v4/files");
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
$result = curl_exec($ch);
$status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
if($status != 201)
throw new Exception("File failed with status: " . $status .", result: " . $result);
$result = json_decode($result, true);
$file_id = $result['file_infos'][0]['id'];
$msg_fields['file_ids'] = array($file_id);
return mm_post($server, $token, $chan, $title, $msg_fields);
}
function mm_get_permalink(string $server, string $team, string $msg_id)
{
return "$server/$team/pl/$msg_id";
}
function _atf_mm_post_curl(string $server, string $token, string $chan, string $message, $fields = array())
{
$ch = curl_init("$server/api/v4/posts");
$headers = array(
'Content-Type: application/json',
'Authorization: Bearer ' . $token
);
if(!isset($fields['channel_id']))
$fields['channel_id'] = $chan;
$fields['message'] = $message;
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($fields));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
return $ch;
}

706
plan.inc.php Normal file
View File

@ -0,0 +1,706 @@
<?php
namespace ATF;
use Exception;
use Amp;
use taskman;
class Plan
{
const EXCEPTIONS_THRESHOLD = 5;
const STUCK_THRESHOLD = 50;
const GONE_THRESHOLD = 20;
const HUNG_THRESHOLD = 20;
const BOGUS_DEVICE_COUNT_THRESHOLD = 6;
const BOGUS_REINSTALL_APP_EVERY_N = 2;
const IM_THREAD_UPDATE_INTERVAL = 5;
public Session $session;
public string $name;
public array $tasks = array();
public string $im_thread_id = '';
public int $im_thread_last_update = 0;
public int $start_time;
//execution history of all tasks
//array of tuples [code, msg]
public array $status_codes = array();
function __construct(string $name)
{
$this->name = $name;
}
function getTitle() : string
{
$issues_summary = '';
foreach($this->getProblemsHistogram() as $code => $count)
{
if($code == Task::CODE_GONE)
$issues_summary .= " gones:$count";
else if($code == Task::CODE_NSTART)
$issues_summary .= " nstrts:$count";
else if($code == Task::CODE_STUCK)
$issues_summary .= " stucks:$count";
else if($code == Task::CODE_HUNG)
$issues_summary .= " hungs:$count";
else if($code == Task::CODE_EXCEPTION)
$issues_summary .= " excepts:$count";
else if($code == Task::CODE_WARN)
$issues_summary .= " warns:$count";
}
$running_count = $this->countRunningTasks();
$over_count = $this->countFinishedTasks();
$predicted_total_duration = $this->getPredictedTotalDuration();
$progress_txt = ' tasks:' . ($running_count > 0 ? $running_count . '|' : '') . $over_count . '/' . sizeof($this->tasks) .
' (' . round($this->getProgress()*100, 2) . '%) (' .
round($this->getDuration()/60, 1) . '/' . ($predicted_total_duration > 0 ? round($predicted_total_duration/60, 1) : '?') .
' min)';
return "({$this->session->id}) Plan '{$this->name}'" . $issues_summary . $progress_txt;
}
function getDuration() : int
{
return time() - $this->start_time;
}
function getPredictedTotalDuration()
{
if($this->getDuration() <= 0)
return 0;
if($this->getProgress() <= 0)
return 0;
return $this->getDuration() / $this->getProgress();
}
function _createMessengerThread()
{
if($this->im_thread_id)
return;
for($i=0;$i<5;++$i)
{
$resp = $this->session->conf->im->post($this->getTitle(), $this->_getMessengerThreadProps());
if(isset($resp['ok']))
{
$this->im_thread_id = $resp['id'];
return;
}
sleep(1);
}
throw new Exception("Could not create IM thread");
}
function _updateMessengerThread(bool $force)
{
if(!$force && (time() - $this->im_thread_last_update) < self::IM_THREAD_UPDATE_INTERVAL)
return;
$this->im_thread_last_update = time();
if(!$this->im_thread_id)
return;
$this->session->conf->im->updatePost($this->im_thread_id, $this->_getMessengerThreadProps());
}
function _getMessengerThreadProps() : array
{
return ['props' => ['attachments' => [
[
'text' => '* ' . $this->getTitle(),
'color' => $this->_getThreadColor()
]
]]];
}
function addStatusCode($code, $msg = '')
{
$this->status_codes[] = array($code, $msg);
}
function getProblemsHistogram()
{
$histogram = array();
foreach($this->status_codes as $item)
{
list($code, $_) = $item;
if(!Task::isProblemCode($code))
continue;
if(!isset($histogram[$code]))
$histogram[$code] = 0;
++$histogram[$code];
}
return $histogram;
}
function hasStatusCode($code) : bool
{
return $this->countStatusCode($code) > 0;
}
function countStatusCode($code) : int
{
$c = 0;
foreach($this->status_codes as $item)
{
if($item[0] === $code)
$c++;
}
return $c;
}
function getStats() : Stats
{
$stats = new Stats();
$stats->gones = $this->countStatusCode(Task::CODE_GONE);
$stats->hungs = $this->countStatusCode(Task::CODE_HUNG);
$stats->stucks = $this->countStatusCode(Task::CODE_STUCK);
$stats->nostarts = $this->countStatusCode(Task::CODE_NSTART);
$stats->excepts = $this->countStatusCode(Task::CODE_EXCEPTION);
$stats->warns = $this->countStatusCode(Task::CODE_WARN);
return $stats;
}
function _getThreadColor() : string
{
if(!$this->session->getDevices())
return "#000000";
$stats = $this->getStats();
return $stats->getColor($this->isOver());
}
function addTask(Task $task)
{
$task->plan = $this;
$this->tasks[] = $task;
}
function isOver() : bool
{
if($this->hasTooManyFatalProblems())
return true;
foreach($this->tasks as $task)
{
if(!$task->isDone())
return false;
}
return true;
}
function hasTooManyFatalProblems() : bool
{
return
($this->countStatusCode(Task::CODE_EXCEPTION) >= self::EXCEPTIONS_THRESHOLD) ||
($this->countStatusCode(Task::CODE_STUCK) >= self::STUCK_THRESHOLD) ||
($this->countStatusCode(Task::CODE_GONE) >= self::GONE_THRESHOLD) ||
($this->countStatusCode(Task::CODE_HUNG) >= self::HUNG_THRESHOLD)
;
}
function getProgress() : float
{
$total_progress = 0;
foreach($this->tasks as $task)
$total_progress += $task->getProgress();
return $total_progress / sizeof($this->tasks);
}
function countRunningTasks() : int
{
$running_count = 0;
foreach($this->tasks as $task)
{
if($task->device)
$running_count++;
}
return $running_count;
}
function countFinishedTasks() : int
{
$done_count = 0;
foreach($this->tasks as $task)
{
if($task->isDone())
$done_count++;
}
return $done_count;
}
function runTaskAsync(Task $task) : Amp\Promise
{
return Amp\call(function() use($task) {
yield $this->_startAsync($task);
while(!$this->isOver())
{
$this->_updateMessengerThread(false);
yield Amp\delay((int)($this->session->conf->sleep_time*1000));
log("Task #{$task->run_id} {$task->cmd}(".sizeof($task->args).") progress {$task->getProgress()}, last ext time {$task->last_ext_status_item_time} *{$task->device}*...");
$check_error = yield $this->_tryCheckExtStatusAsync($task, /*attempts*/3, /*timeout*/20);
if($check_error !== null)
{
//something wrong with device, let's ignore the device and reschedule the task
$this->_ignoreDevice($task->device, $check_error);
$task->reschedule();
break;
}
if(!$task->isDone())
yield $this->_checkHealthAsync($task);
if($task->hasFatalProblem())
{
yield $this->_processFatalProblemAsync($task);
break;
}
else if($task->isDone())
{
//let's reset bogus stats for this device
$this->session->resetBogusDeviceCount($task->device);
//let's free the device
$task->device = null;
break;
}
}
$this->_updateMessengerThread(true);
});
}
function _onStart()
{
$this->start_time = time();
log("Starting " . $this->getTitle() . "...");
$this->_createMessengerThread();
}
function _startAsync(Task $task) : Amp\Promise
{
//let's create a Slack thread synchronously if it's not present yet
if(!$this->im_thread_id)
$this->_onStart();
return Amp\call(function() use($task) {
try
{
log("Preparing *{$task->device}*");
yield $this->session->apk_installer->installAsync($task->device);
yield start_ext_cmd_on_device_async(
$this->session->conf->atf_host,
$task->device,
$task->getCmd(),
$task->getCmdArgs()
);
}
catch(Exception $e)
{
$this->_ignoreDevice($task->device, $e->getMessage());
$task->reschedule();
return false;
}
$task->start();
return true;
});
}
function _ignoreDevice($device, $reason)
{
$this->session->ignoreDevice($device, $reason);
$this->post("Ignoring device *$device*: $reason");
}
function _processFatalProblemAsync(Task $task) : Amp\Promise
{
return Amp\call(function() use($task) {
$fatal_msg = "Fatal problem ({$task->getLastFatalProblem()} - ".Task::code2string($task->getLastFatalProblem())."), attempt:{$task->attempts} *{$task->device}*";
err("[FTL] $fatal_msg");
$this->post($fatal_msg);
$this->_postScreenToMessenger($task);
if($task->getLastFatalProblem() == Task::CODE_EXCEPTION)
yield $this->_postLastReplayToMessengerAsync($task);
else if($task->getLastFatalProblem() == Task::CODE_GONE ||
$task->getLastFatalProblem() == Task::CODE_HUNG)
{
$app_log = get_logcat_unity($this->session->conf->atf_host, $task->device, 300);
$app_log = _atf_trim_start($app_log, 2000);
$this->post("Last logs: \n```\n$app_log\n```\n *{$task->device}*");
}
if($this->session->conf->error_pause)
$this->_ignoreDevice($task->device, "Ignored for error investigation");
//Let's reinstall application on this device if it's failing too often
else
$this->_incAndCheckBogusDevice($task->device);
$task->onFatalProblem();
//let's free the device anyway, it's up to Task::onFatalProblem() to reschedule it or not
$task->device = null;
});
}
function _tryCheckExtStatusAsync(Task $task, int $attempts, int $timeout) : Amp\Promise
{
return Amp\call(function() use($task, $attempts, $timeout) {
$last_error = null;
for($i=0;$i<$attempts;++$i)
{
try
{
yield $this->_checkExtStatusAsync($task, $timeout);
break;
}
catch(Exception $e)
{
$last_error = $e->getMessage();
continue;
}
}
return $last_error;
});
}
function _checkExtStatusAsync(Task $task, int $timeout) : Amp\Promise
{
return Amp\call(function() use($task, $timeout) {
$ext_status = yield get_ext_status_async($this->session->conf->atf_host, $task->device, $timeout);
if(!is_array($ext_status))
return;
$new_items = $this->_getExtStatusItemsSince($task->last_ext_status_item_time, $ext_status);
if(!$new_items)
return;
$task->last_ext_status_item_time = end($new_items)['time'];
foreach($new_items as $item)
yield $this->_analyzeExtStatusItemAsync($task, $item);
});
}
function _getExtInputRequestsSince($id, array $all_reqs) : array
{
$new_reqs = array();
foreach($all_reqs as $req)
{
if($req['id'] > $id)
$new_reqs[] = $req;
}
return $new_reqs;
}
function _getExtStatusItemsSince($time, array $ext_status) : array
{
$new_items = array();
foreach($ext_status['entries'] as $item)
{
if($item['time'] > $time)
$new_items[] = $item;
}
return $new_items;
}
function _checkIfAppRunning(Task $task) : Amp\Promise
{
return Amp\call(function() use($task) {
list($status, $_) =
yield host_exec_async($this->session->conf->atf_host, "%{adb}% -s {$task->device} shell pidof %{package_id}%", DEPLOY_OPT_ERR_OK);
return $status == 0;
});
}
function _reportGone(Task $task)
{
//check it application hasn't even started
if($task->last_alive_check_time === $task->reset_time)
{
$task->addStatusCode(Task::CODE_NSTART);
$this->session->trySendStatsEvent($task, 'nstart');
err("[NFD] No app started after {$this->session->conf->dead_threshold} seconds *{$task->device}*");
}
else
{
$task->addStatusCode(Task::CODE_GONE);
$this->session->trySendStatsEvent($task, 'gone');
err("[GNE] App is gone after {$this->session->conf->dead_threshold} seconds *{$task->device}*");
}
}
function _checkHealthAsync(Task $task) : Amp\Promise
{
return Amp\call(function() use($task) {
$not_alive_time = microtime(true) - $task->last_alive_check_time;
$stuck_time = microtime(true) - $task->last_stuck_check_time;
if($not_alive_time > $this->session->conf->dead_threshold)
{
if(yield $this->_checkIfAppRunning($task))
{
$task->addStatusCode(Task::CODE_HUNG);
$this->session->trySendStatsEvent($task, 'hung');
err("[HNG] No activity for {$this->session->conf->dead_threshold} seconds *{$task->device}*");
}
else
$this->_reportGone($task);
$this->_reportErrorFromLogcatToMessenger($task, 1000);
}
else if($stuck_time > $this->session->conf->stuck_threshold)
{
if(yield $this->_checkIfAppRunning($task))
{
$task->addStatusCode(Task::CODE_STUCK);
$this->session->trySendStatsEvent($task, 'stuck');
err("[STK] Stuck for {$this->session->conf->stuck_threshold} seconds *{$task->device}*");
}
else
$this->_reportGone($task);
}
});
}
function _incAndCheckBogusDevice(string $device)
{
$count = $this->session->incBogusDevice($device);
if($count > self::BOGUS_DEVICE_COUNT_THRESHOLD)
{
$this->_ignoreDevice($device, "Too many bogus errors");
}
else if(($count % self::BOGUS_REINSTALL_APP_EVERY_N) == 0)
{
$this->session->apk_installer->forgetInstall($device);
}
}
static function _parseExtMessage(string $message)
{
$msg_code = null;
$msg_text = $message;
//example: [DBG] this a debug message
if(preg_match('~^(\[[^\]]+\])(.*)$~', $message, $matches))
{
$msg_code = $matches[1];
$msg_text = $matches[2];
}
return array($msg_code, $msg_text);
}
static function _printToShellExtItem(Task $task, array $item)
{
$shell_msg = _atf_trim($item['message'], 200);
if(Task::isProblemCode($item['error']))
$shell_msg = "[PRB] Code:{$item['error']}, $shell_msg";
$shell_msg = "(".round($item['time'],1)."s) {$shell_msg} *{$task->device}*";
log($shell_msg);
}
function _postToMessengerExtStatusItem(Task $task, array $item)
{
$orig_msg = _atf_trim($item['message'], 3000);
$mm_msg = $orig_msg;
if($item['error'] == Task::CODE_EXCEPTION)
$mm_msg = "\n```\n$mm_msg\n```\n";
$mm_msg = '('.round($item['time'],1).'s) '.$mm_msg.' *'.$task->device.'*';
$resp = $this->post($mm_msg);
if(isset($resp['ok']) && $item['error'] == Task::CODE_EXCEPTION)
$this->session->tryShareToQAChannel($resp['id'], $orig_msg);
}
function post($msg, array $args = array())
{
$args['root_id'] = $this->im_thread_id;
return $this->session->conf->im->post($msg, $args);
}
function _analyzeExtStatusItemAsync(Task $task, array $item) : Amp\Promise
{
return Amp\call(function() use($task, $item) {
self::_printToShellExtItem($task, $item);
$task->addStatusCode($item['error'], $item['message']);
if($item['error'] == Task::CODE_EXCEPTION)
$this->session->trySendStatsEvent($task, 'exception');
list($msg_type, $msg_text) = self::_parseExtMessage($item['message']);
//NOTE: alive system messages are sent periodically by ExtBot itself,
// we'd like to ignore them for stuck detection
if($msg_type !== '[ALIVE]')
$task->last_stuck_check_time = microtime(true);
//NOTE: in case of any message from the device we update the alive check timestamp
$task->last_alive_check_time = microtime(true);
yield $this->_processExtStatusMessageAsync($task, $item, $msg_type, $msg_text);
});
}
function _processExtStatusMessageAsync($task, $item, $msg_type, $msg_text) : Amp\Promise
{
return Amp\call(function() use($task, $item, $msg_type, $msg_text) {
if($item['error'] == Task::CODE_EXCEPTION)
{
$this->_postToMessengerExtStatusItem($task, $item);
}
else if($msg_type === '[WRN]')
{
$task->addStatusCode(Task::CODE_WARN);
$this->_postToMessengerExtStatusItem($task, $item);
}
else if($msg_type === '[PRG]')
{
//let's reset hung stats since progress is going
$this->session->resetBogusDeviceCount($task->device);
$task->onProgress($msg_text);
}
else if($msg_type === '[STAT]')
yield $this->session->trySendStatsFromJzonAsync($task, $msg_text);
else if($msg_type === '[RPL]')
$this->_postReplayToMessenger($task, $msg_text);
});
}
function _postLastReplayToMessengerAsync(Task $task) : Amp\Promise
{
return Amp\call(function() use($task) {
if($this->session->conf->device_replay_path)
{
list($repl_err, $repl_txt) =
yield get_last_replay_async($this->session->conf->atf_host, $task->device, $this->session->conf->device_replay_path);
if($repl_err === 0)
$this->_postReplayToMessenger($task, $repl_txt);
}
});
}
function _postReplayToMessenger(Task $task, $repl_txt)
{
$repl_txt = _atf_try_lz4_replay($repl_txt);
$this->post("Last Replay: \n```\n$repl_txt\n```\n *{$task->device}*");
}
function _reportErrorFromLogcatToMessenger(Task $task, $limit)
{
$errors_log = _atf_trim(get_logcat_errors($task->device, $limit), 3950);
if($errors_log)
$this->post("\n```\n$errors_log\n```\n *{$task->device}*");
}
function _postScreenToMessenger(Task $task)
{
global $GAME_ROOT;
$png_data = device_screen($this->session->conf->atf_host, $task->device);
if($png_data)
{
$tmp_png = tempnam(sys_get_temp_dir(), 'device_screen') . '.png';
taskman\ensure_write($tmp_png, $png_data);
try
{
$this->session->conf->im->postPNG("screen: ".$task->device, $tmp_png, array('root_id' => $this->im_thread_id));
}
catch(Exception $e)
{}
finally
{
taskman\ensure_rm($tmp_png);
}
}
}
}
class Stats
{
public int $gones = 0;
public int $hungs = 0;
public int $stucks = 0;
public int $nostarts = 0;
public int $excepts = 0;
public int $warns = 0;
function add(Stats $other)
{
$this->gones += $other->gones;
$this->hungs += $other->hungs;
$this->stucks += $other->stucks;
$this->nostarts += $other->nostarts;
$this->excepts += $other->excepts;
$this->warns += $other->warns;
}
function getColor(bool $is_over) : string
{
if($this->gones > 0 ||
$this->hungs > 0 ||
$this->warns > 0 ||
$this->excepts > 0 ||
$this->stucks > 0 ||
$this->nostarts > 0
)
{
if($this->excepts)
return $is_over ? "danger" : "#FFCCCB";
else
return $is_over ? "warning" : "#FFCF9E";
}
else if($is_over)
return "good";
else
return "#D3D3D3";
}
}

492
session.inc.php Normal file
View File

@ -0,0 +1,492 @@
<?php
namespace ATF;
use Exception;
use taskman;
use Amp;
class SessionConfig
{
public string $atf_host;
public ?string $apk_path = '';
public string $proj_branch;
public string $proj_version;
public string $proj_rev_hash;
public bool $override_external_files = false;
public array $external_files = array();
public int $max_adb_installs_in_progress = 5;
public bool $adb_reboot = true;
public bool $reboot_devices = false;
public bool $error_pause = false;
public int $sleep_time = 5;
public int $dead_threshold = 80;
public int $stuck_threshold = 60;
public bool $share_with_qa_chan = true;
public IMChan $im;
public IMChan $im_qa;
public IStatsSender $stats;
public string $device_replay_path = 'replay.txt';
function __construct()
{
$this->im = new NullMessenger();
$this->im_qa = new NullMessenger();
$this->stats = new NullStatsSender();
}
function initHost(array $settings)
{
$this->atf_host = 'atf';
taskman\deploy_declare_node($this->atf_host, $settings);
}
}
class Session
{
const RUN_SLEEP = 3;
public string $name;
public SessionConfig $conf;
public int $start_time;
public IDevicePool $device_pool;
public ApkInstaller $apk_installer;
public string $id;
public string $guid;
public string $im_thread_id = '';
public array $plans = array();
public array $ignored_devices = array();
public array $bogus_device2counter = array();
public array $shared_qa_errors = array();
static function getId() : string
{
$id = getenv('ATF_SESSION_ID');
if($id)
return $id;
//sort of graceful fallback
return ''.getmypid();
}
function __construct(
string $name,
SessionConfig $conf,
IDevicePool $device_pool
)
{
$this->id = self::getId();
$this->name = $name;
$this->conf = $conf;
$this->guid = uniqid();
$this->device_pool = $device_pool;
}
function getTitle() : string
{
return "Session ({$this->id}) '{$this->name}' (devices:".sizeof($this->getDevices())." ?:".sizeof($this->ignored_devices) .
') (' . round($this->getDuration()/60, 1) . ' min)';
}
function getDuration() : int
{
return time() - $this->start_time;
}
function getDevices() : array
{
$devices = array_diff(
$this->device_pool->get(),
$this->ignored_devices
);
return $devices;
}
function ignoreDevice(string $device, string $reason)
{
err("Ignoring device *$device*: $reason");
$this->ignored_devices[] = $device;
}
function incBogusDevice(string $device)
{
if(!isset($this->bogus_device2counter[$device]))
$this->bogus_device2counter[$device] = 0;
return ++$this->bogus_device2counter[$device];
}
function resetBogusDeviceCount(string $device)
{
$this->bogus_device2counter[$device] = 0;
}
function resetBogusDevices()
{
$this->bogus_device2counter = array();
}
function addPlan(Plan $plan)
{
$plan->session = $this;
$this->plans[] = $plan;
}
function resetPlans()
{
$this->plans = array();
}
function runAsync() : Amp\Promise
{
return Amp\call(function() {
$this->start_time = time();
$apk_name = !$this->conf->apk_path ? null : basename($this->conf->apk_path);
$this->apk_installer = new ApkInstaller(
$this->conf->atf_host,
$this->conf->max_adb_installs_in_progress,
$apk_name,
$this->conf->override_external_files,
$this->conf->external_files
);
$this->_createMessengerThread();
//if it's not set the last one will be re-used
if($this->conf->apk_path)
$this->_deployApkToHost($apk_name);
if($this->conf->reboot_devices)
reboot_devices($this->conf->atf_host, $this->getDevices());
if($this->conf->adb_reboot)
adb_reboot($this->conf->atf_host);
$this->trySendApkStatsEvent($this->apk_installer->getSize());
$all_tasks = $this->_getAllTasks();
return yield $this->_runTasksAsync($all_tasks);
});
}
function _createMessengerThread()
{
for($i=0;$i<5;++$i)
{
$resp = $this->conf->im->post("", $this->_getMessengerThreadProps());
if(isset($resp['ok']))
{
$this->im_thread_id = $resp['id'];
return;
}
sleep(1);
}
throw new Exception("Could not create IM thread");
}
function _updateMessengerThread()
{
$this->conf->im->updatePost(
$this->im_thread_id,
$this->_getMessengerThreadProps()
);
}
function _getMessengerThreadProps() : array
{
return ['props' => ['attachments' => [
[
'text' => $this->getTitle(),
'color' => $this->_getThreadColor()
]
]]];
}
function _getThreadColor() : string
{
if(!$this->getDevices())
return "#000000";
$stats = new Stats();
foreach($this->plans as $plan)
$stats->add($plan->getStats());
return $stats->getColor($this->isOver());
}
function _deployApkToHost($apk_name)
{
if(!is_file($this->conf->apk_path))
throw new Exception("No such file '{$this->conf->apk_path}'");
host_put_file($this->conf->atf_host, $this->conf->apk_path, "%{dir}%/$apk_name");
}
function _getAllTasks()
{
$all_tasks = array();
foreach($this->plans as $plan)
{
foreach($plan->tasks as $task)
$all_tasks[] = $task;
}
return $all_tasks;
}
function _runTasksAsync(array $all_tasks) : Amp\Promise
{
return Amp\call(function() use($all_tasks) {
$active_tasks = array();
while(!$this->isOver() && (sizeof($all_tasks) > 0 || sizeof($active_tasks) > 0))
{
$free_devices = $this->_findFreeDevices($active_tasks);
$this->_assignTasksToDevices($all_tasks, $free_devices, $active_tasks);
yield Amp\delay(self::RUN_SLEEP*1000);
$this->_processDoneTasks($all_tasks, $active_tasks);
$this->_updateMessengerThread();
}
return sizeof($all_tasks) == 0 && sizeof($active_tasks) == 0;
});
}
function _assignTasksToDevices(array &$tasks, array $devices, array &$active_tasks)
{
static $next_run_id = 0;
//create new coroutines and assign them to free devices
foreach($devices as $device)
{
if(sizeof($tasks) == 0)
break;
$task = array_shift($tasks);
//explicitly assigning a device
$task->device = $device;
$task->run_id = ++$next_run_id;
$task->run_status = false;
$active_tasks[$task->run_id] = $task;
$task->plan->runTaskAsync($task)->onResolve(
function($error, $value) use($task)
{
$task->run_status = true;
$task->run_error = $error;
}
);
}
}
function _processDoneTasks(array &$tasks, array &$active_tasks)
{
$tmp_tasks = array();
foreach($active_tasks as $run_id => $task)
{
//task not in done ids, let's keep it
if(!$task->run_status)
{
$tmp_tasks[$run_id] = $task;
}
else if($task->run_error)
{
throw $task->run_error;
}
//task is done, let's check if it had a problem and if yes
//let's put it back into unscheduled tasks
else if($task->need_reschedule)
{
array_unshift($tasks, $task);
$task->need_reschedule = false;
}
}
$active_tasks = $tmp_tasks;
}
function _findFreeDevices(array $active_tasks)
{
$free = array();
foreach($this->getDevices() as $device)
{
//let's skip invalid devices
if(!$device)
continue;
$busy = false;
foreach($active_tasks as $task)
{
if($task->device == $device)
{
$busy = true;
break;
}
}
if(!$busy)
$free[] = $device;
}
return $free;
}
function isOver() : bool
{
if(!$this->getDevices())
return true;
foreach($this->plans as $plan)
if(!$plan->isOver())
return false;
return true;
}
function trySendStatsFromJzonAsync(Task $task, string $jzon) : Amp\Promise
{
return Amp\call(function() use($task, $jzon) {
try
{
$data = jzon_parse(trim(str_replace('\"', '"', $jzon)));
$table = $data['table'];
unset($data['table']);
$data = yield $this->inflateMissingStats($task, $table, $data);
$this->trySendStats($task, $table, $data);
}
catch(Exception $e)
{
echo $e->getMessage() . "\n";
}
});
}
//TODO: make it more flexible via plugin alike system?
function inflateMissingStats(Task $task, string $table, array $data) : Amp\Promise
{
return Amp\call(function() use($task, $table, $data) {
//NOTE: filling memory usage if it's not set
if(isset($data['deviceMemoryUsage']) && $data['deviceMemoryUsage'] === '')
{
$mem = yield device_mem_async($this->conf->atf_host, $task->device);
$data['deviceMemoryUsage'] = $mem['total'];
if($table === 'device_memory')
{
$data['deviceMemoryUsageNative'] = $mem['native'];
$data['deviceMemoryUsageSystem'] = $mem['system'];
$data['deviceMemoryUsageJava'] = $mem['java'];
$data['deviceMemoryUsageGraphics'] = $mem['graphics'];
}
}
//NOTE: filling CPU temperature
if(isset($data['event']) && $data['event'] == 'cpuTemp')
$data['value'] = yield device_temperature_async($this->conf->atf_host, $task->device);
return $data;
});
}
function trySendApkStatsEvent(int $apk_size)
{
try
{
$data['guid'] = $this->guid;
$data['time'] = time();
$data['version'] = $this->conf->proj_version;
$data['revHash'] = $this->conf->proj_rev_hash;
$data['branch'] = $this->conf->proj_branch;
$data['event'] = 'apk_size';
$data['value'] = $apk_size;
$this->conf->stats->send('event_value', $data);
}
catch(Exception $e)
{
echo $e->getMessage() . "\n";
}
}
function trySendStats(Task $task, string $table, array $data)
{
try
{
$data['guid'] = $this->guid;
$data['time'] = time();
$data['deviceId'] = $task->device;
$this->conf->stats->send($table, $data);
}
catch(Exception $e)
{
echo $e->getMessage() . "\n";
}
}
function trySendStatsEvent(Task $task, string $event, int $value = 1)
{
$this->trySendStats(
$task,
'event_value',
array(
'event' => $event,
'value' => $value,
'branch' => $this->conf->proj_branch,
'version' => $this->conf->proj_version,
'revHash' => $this->conf->proj_rev_hash
)
);
}
function tryShareToQAChannel($msg_id, $error)
{
if(!$this->conf->share_with_qa_chan)
return;
//let's skip similar already shared errors
if($this->_calcSharedErrorSimilarity($error) > 80)
return;
$this->shared_qa_errors[] = $error;
//let's share the exception message to the regency-qa channel
$permalink = $this->conf->im->getPermalink($msg_id);
$this->conf->im_qa->post($permalink);
}
function _calcSharedErrorSimilarity($error)
{
$max_perc = 0;
foreach($this->shared_qa_errors as $shared)
{
$perc = 0;
similar_text($shared, $error, $perc);
if($perc > $max_perc)
$max_perc = $perc;
}
return $max_perc;
}
}

180
slack.inc.php
View File

@ -1,180 +0,0 @@
<?php
namespace taskman;
use Amp;
use Exception;
use CURLFile;
function slack_msg($token, $channel, $msg, $fields = [])
{
$ch = curl_init("https://slack.com/api/chat.postMessage");
$fields["token"] = $token;
if(!isset($fields['channel']))
$fields["channel"] = $channel;
$fields["text"] = $msg;
$fields["as_user"] = "bitgames bot";
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
//curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
//curl_setopt($ch, CURLOPT_VERBOSE, true);
$result = curl_exec($ch);
curl_close($ch);
return json_decode($result, true);
}
function slack_upload_image($token, array $channels, $filepath)
{
$url = "https://slack.com/api/files.upload";
$cfile = new CURLFile(realpath($filepath), 'image/jpg', 'test.jpg');
$fields = array(
'token' => $token,
'channels' => implode(",", $channels),
'file' => $cfile,
'filename' => "test.jpg",
);
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_HEADER, false);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_VERBOSE, true);
$result = curl_exec($ch);
curl_close($ch);
$arr = json_decode($result, true);
if(isset($arr['ok']) && $arr['ok'] == 1)
{
if(isset($arr['file']['permalink_public']))
return $arr['file']['permalink_public'];
}
return "";
}
function _atf_slack_chan()
{
return get("ATF_SLACK_CHANNEL");
}
function _atf_slack_chan_qa()
{
return get("ATF_SLACK_CHANNEL");
}
function _atf_slack_token()
{
$token = "xoxb-141599046048-1567053090644-2gDAoWGiZxFTtXGLMOOeLG2u";
return $token;
}
task('slack_test', function()
{
// createSlackThread();
// atf_slack_post('hi', array('channel' => _atf_slack_chan_qa()));
});
function atf_slack_post($message, $fields = array())
{
if(!get("ATF_SLACK"))
return array('ok' => true, 'ts' => 0);
$ch = curl_init("https://slack.com/api/chat.postMessage");
$fields["token"] = _atf_slack_token();
if(!isset($fields['channel']))
$fields["channel"] = _atf_slack_chan();
$fields["text"] = $message;
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$result = curl_exec($ch);
curl_close($ch);
return json_decode($result, true);
}
function atf_slack_get_permalink($message_ts, $channel = null)
{
if(!get("ATF_SLACK"))
return array('permalink' => '');
$ch = curl_init("https://slack.com/api/chat.getPermalink");
$fields["token"] = _atf_slack_token();
if($channel === null)
$channel = _atf_slack_chan();
$fields["channel"] = $channel;
$fields["message_ts"] = $message_ts;
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$result = curl_exec($ch);
curl_close($ch);
return json_decode($result, true);
}
function atf_slack_update($ts, array $attachments)
{
if(!get("ATF_SLACK"))
return array('ok' => true);
$ch = curl_init("https://slack.com/api/chat.update");
$fields["token"] = _atf_slack_token();
$fields["channel"] = _atf_slack_chan();
$fields["ts"] = $ts;
$fields["attachments"] = json_encode($attachments);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields));
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
$result = curl_exec($ch);
curl_close($ch);
return json_decode($result, true);
}
function atf_slack_post_png($title, $png_file, $fields = array())
{
if(!get("ATF_SLACK"))
return array('ok' => true);
$file = new CurlFile($png_file, 'image/png');
$header = array();
$header[] = 'Content-Type: multipart/form-data';
$fields["token"] = _atf_slack_token();
$fields["channels"] = _atf_slack_chan();
$fields['file'] = $file;
$fields['title'] = $title;
$ch = curl_init("https://slack.com/api/files.upload");
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, $fields);
$result = curl_exec($ch);
curl_close($ch);
return json_decode($result, true);
}

56
stats.inc.php Normal file
View File

@ -0,0 +1,56 @@
<?php
namespace ATF;
use Exception;
use ClickHouseDB;
interface IStatsSender
{
function send(string $table, array $data);
}
class NullStatsSender implements IStatsSender
{
function send(string $table, array $data) {}
}
class ClickHouseStatsSender implements IStatsSender
{
private string $host;
private string $port;
private string $username;
private string $password;
private string $dbname;
function __construct(string $host, string $port, string $username, string $password, string $dbname)
{
$this->host = $host;
$this->port = $port;
$this->username = $username;
$this->password = $password;
$this->dbname = $dbname;
}
function getClient() : ClickHouseDB\Client
{
$config = [
'host' => $this->host,
'port' => $this->port,
'username' => $this->username,
'password' => $this->password,
];
$db = new ClickHouseDB\Client($config);
if($this->dbname)
$db->database($this->dbname);
$db->setTimeout(15);
$db->setConnectTimeOut(10);
return $db;
}
function send(string $table, array $data)
{
$this->getClient()->insert($table, array(array_values($data)), array_keys($data));
}
}

308
task.inc.php Normal file
View File

@ -0,0 +1,308 @@
<?php
namespace ATF;
use Exception;
use Amp;
class Cmd
{
public string $module;
public string $func;
function __construct(string $module, string $func)
{
$this->module = $module;
$this->func = $func;
}
function __toString()
{
return $this->module.':'.$this->func;
}
}
class Task
{
const CODE_MESSAGE = 0;
const CODE_EXCEPTION = 1;
const CODE_WARN = 2;
const CODE_NSTART = 124;
const CODE_STUCK = 125;
const CODE_GONE = 126;
const CODE_HUNG = 127;
const CODE_DONE = 128;
static function code2string(int $code) : string
{
switch($code)
{
case self::CODE_MESSAGE:
return "message";
case self::CODE_EXCEPTION:
return "exception";
case self::CODE_WARN:
return "warning";
case self::CODE_STUCK:
return "stuck";
case self::CODE_NSTART:
return "nstart";
case self::CODE_GONE:
return "gone";
case self::CODE_HUNG:
return "hung";
case self::CODE_DONE:
return "done";
}
return "???";
}
public ?Plan $plan = null;
//async execution id
public int $run_id = 0;
//'true' if async coroutine is resolved
public bool $run_status = false;
//async execution exception if any
public ?object $run_error = null;
//attached device
public ?string $device = null;
public int $attempts = 1;
//signal to dispatcher to re-start execution of the task
public bool $need_reschedule = false;
public Cmd $cmd;
public array $args = array();
public string $title = '';
public float $start_time = 0;
public float $reset_time = 0;
public float $last_progress = 0;
public int $last_done_arg_idx = -1;
public float $last_alive_check_time = 0;
public float $last_stuck_check_time = 1;
public float $last_ext_status_item_time = 0;
//array of tuples [code, msg]
public array $status_codes = array();
function __construct(Cmd $cmd, array $args, $title = '')
{
$this->cmd = $cmd;
$this->args = $args;
$this->title = $title;
}
function getCmd() : Cmd
{
return $this->cmd;
}
function getCmdArgs() : array
{
return $this->args;
}
function getProgress() : float
{
if($this->last_progress == 0 && $this->isDone())
return 1;
return $this->last_progress;
}
static function isProblemCode($code) : bool
{
return $code > self::CODE_MESSAGE && $code < self::CODE_DONE;
}
static function isFatalCode($code) : bool
{
return
$code == Task::CODE_STUCK ||
$code == Task::CODE_NSTART ||
$code == Task::CODE_GONE ||
$code == Task::CODE_HUNG ||
$code == Task::CODE_EXCEPTION;
}
function hasFatalProblem() : bool
{
return $this->getLastFatalProblem() != 0;
}
function getLastFatalProblem() : int
{
for($i=sizeof($this->status_codes);$i-- > 0;)
{
$item = $this->status_codes[$i];
if(self::isFatalCode($item[0]))
return $item[0];
}
return 0;
}
function hasStatusCode($code)
{
foreach($this->status_codes as $item)
{
if($item[0] === $code)
return true;
}
return false;
}
function isDone()
{
return $this->hasStatusCode(Task::CODE_DONE);
}
function addStatusCode($code, $msg = '')
{
//plan stores history of all status codes
$this->plan->addStatusCode($code, $msg);
$this->status_codes[] = array($code, $msg);
}
function onFatalProblem()
{
if($this->attempts < 3)
$this->reschedule();
}
function onProgress($jzon)
{
try
{
$data = jzon_parse(trim(str_replace('\"', '"', $jzon)));
if(isset($data['p']))
$this->last_progress = floatval(str_replace(',', '.', $data['p']));
if(isset($data['arg_idx']))
$this->last_done_arg_idx = (int)$data['arg_idx'];
}
catch(Exception $e)
{
echo $e->getMessage() . "\n";
}
}
function _resetCheckTimes()
{
$this->reset_time = microtime(true);
$this->last_alive_check_time = $this->reset_time;
$this->last_stuck_check_time = $this->reset_time;
$this->last_ext_status_item_time = 0;
}
function reschedule()
{
++$this->attempts;
$this->device = null;
$this->need_reschedule = true;
}
function start()
{
$this->_resetCheckTimes();
$this->status_codes = array();
$this->start_time = time();
}
function getDuration()
{
return time() - $this->start_time;
}
}
class TestLevelsTask extends Task
{
public $arg_offset = 0;
private $rseeds = array();
private $seed = null;
private $timescale = null;
function __construct(Cmd $cmd, array $args, $with_rseeds = false, $seed = null, $timescale = null, $title = '')
{
$level_ids = array();
//TODO: should not really be here
if($with_rseeds)
{
foreach($args as $item)
{
$level_ids[] = $item[0];
$this->rseeds[] = $item[1];
}
}
else
$level_ids = $args;
$this->seed = $seed;
$this->timescale = $timescale;
parent::__construct($cmd, $level_ids, $title);
}
function getCmdArgs() : array
{
$args = array();
if($this->rseeds)
{
foreach($this->args as $idx => $arg)
{
$args[] = $arg;
$args[] = $this->rseeds[$idx];
}
}
else
$args = $this->args;
//appending arg. offset as a first argument
array_unshift($args, ''.$this->arg_offset);
array_unshift($args, '--offset');
if($this->rseeds)
{
array_unshift($args, 'true');
array_unshift($args, '--levels-with-rseed');
}
if($this->seed !== null)
{
array_unshift($args, ''.$this->seed);
array_unshift($args, '--seed');
}
if($this->timescale !== null)
{
array_unshift($args, ''.$this->timescale);
array_unshift($args, '--timescale');
}
return $args;
}
function onFatalProblem()
{
$start_arg = isset($this->args[$this->arg_offset]) ? $this->args[$this->arg_offset] : '?';
$done_arg = isset($this->args[$this->last_done_arg_idx]) ? $this->args[$this->last_done_arg_idx] : '?';
$this->plan->post("Handling error, progress:".round($this->getProgress(),2).", last done arg:{$done_arg}, start arg:{$start_arg}, arg offset:{$this->arg_offset}, args:".sizeof($this->args)." *$this->device*");
//we're allowed to skip only in case of exceptions
if($this->getLastFatalProblem() == Task::CODE_EXCEPTION)
{
if($this->last_done_arg_idx > $this->arg_offset)
$this->arg_offset = $this->last_done_arg_idx;
++$this->arg_offset;
}
//let's reschedule the task if it still makes sense
if($this->arg_offset < sizeof($this->args))
$this->reschedule();
}
}