From c4cf2ea1959b0354be67f9267791c4c3142d7d3f Mon Sep 17 00:00:00 2001 From: Pavel Shevaev Date: Tue, 7 Nov 2023 15:42:40 +0300 Subject: [PATCH] Preparing new major release --- .gitignore | 1 + atf.inc.php | 1774 ++++++++++------------------------------------- composer.json | 12 +- device.inc.php | 161 +++++ im.inc.php | 183 +++++ plan.inc.php | 706 +++++++++++++++++++ session.inc.php | 492 +++++++++++++ slack.inc.php | 180 ----- stats.inc.php | 56 ++ task.inc.php | 308 ++++++++ 10 files changed, 2282 insertions(+), 1591 deletions(-) create mode 100644 .gitignore create mode 100644 device.inc.php create mode 100644 im.inc.php create mode 100644 plan.inc.php create mode 100644 session.inc.php delete mode 100644 slack.inc.php create mode 100644 stats.inc.php create mode 100644 task.inc.php diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6e92f57 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tags diff --git a/atf.inc.php b/atf.inc.php index 3dc095f..716ae74 100644 --- a/atf.inc.php +++ b/atf.inc.php @@ -1,1116 +1,17 @@ apk_name = $apk_name; - $this->override_conf = $override_conf; - } - - function countInstallsInProgress() - { - $c = 0; - foreach($this->installs as $status) - if($status == self::ST_IN_PROGRESS) - ++$c; - return $c; - } - - function installAsync($device) - { - return Amp\call(function() use($device) { - - while($this->countInstallsInProgress() >= self::MAX_INSTALLS_IN_PROGRESS) - yield Amp\delay(1000); - - if(isset($this->installs[$device])) - return; - $this->installs[$device] = self::ST_IN_PROGRESS; - try - { - yield atf_host_exec_async("%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK); - if($this->apk_name !== null) - { - yield atf_host_exec_async("%{adb}% -s $device uninstall %{package_id}%", DEPLOY_OPT_ERR_OK, 30); - yield atf_host_exec_async("%{adb}% -s $device install -r ./{$this->apk_name}", 0, 400); - } - - $this->installs[$device] = self::ST_INSTALLED; - } - catch(Exception $e) - { - atf_log("Error during install *$device*: " . $e->getMessage()); - unset($this->installs[$device]); - throw $e; - } - }); - } - - function getSize() - { - list($status, $lines) = atf_host_exec("ls -sd -- ./{$this->apk_name}", DEPLOY_OPT_ERR_OK|DEPLOY_OPT_SILENT); - if($status != 0) - return 0; - $items = explode(' ', $lines[0]); - //ls -s returns size in blocks where each block is 512 bytes long - return $items[0]*512; - } -} - - -class ATFAdbDevicePool implements IATFDevicePool -{ - function get() - { - return atf_get_devices(); - } -} - - -class ATFFixedDevicePool implements IATFDevicePool -{ - private $devices; - - function __construct(array $devices) - { - $this->devices = $devices; - } - - function get() - { - return $this->devices; - } -} - - -class ATFCachedDevices implements IATFDevicePool -{ - private $provider; - private $cached = null; - private $last_cache_time; - private $keep_cache_time; - - function __construct(IATFDevicePool $provider, $keep_cache_time) - { - $this->provider = $provider; - $this->keep_cache_time = $keep_cache_time; - } - - function get() - { - if($this->cached === null || (time() - $this->last_cache_time) > $this->keep_cache_time) - { - $this->cached = $this->provider->get(); - $this->last_cache_time = time(); - } - return $this->cached; - } -} - - - -class ATFSession -{ - public $guid; - public $version; - public $branch; - public $rev_hash; - - public $plans = array(); - - public $ignored_devices = array(); - public $bogus_device_count = array(); - - public $share_with_qa_chan = true; - public $shared_qa_errors = array(); - - function __construct() - { - $this->guid = atf_guid(); - $this->branch = get('GAME_VERSION_BRANCH'); - $this->version = get('GAME_VERSION'); - $this->rev_hash = get('GAME_REVISION_HASH'); - } - - function incBogusDevice($device) - { - if(!isset($this->bogus_device_count[$device])) - $this->bogus_device_count[$device] = 0; - return ++$this->bogus_device_count[$device]; - } - - function resetBogusDeviceCount($device) - { - $this->bogus_device_count[$device] = 0; - } - - function resetBogusDevices() - { - $this->bogus_device_count = array(); - } - - function addPlan(ATFPlan $plan) - { - $plan->session = $this; - $this->plans[] = $plan; - } - - function resetPlans() - { - $this->plans = array(); - } - - function run($apk_path = null, $apk_reuse = false, $override_conf = false, $adb_reboot = true) - { - $apk_name = $apk_path === null ? null : basename($apk_path); - - _atf_start_watchdog($this->guid); - - //1. deploy an apk file to the atf host - if(!$apk_reuse && $apk_path !== null) - { - if(!is_file($apk_path)) - throw new Exception("No such file '$apk_path'"); - - $local_crc = _atf_get_local_file_crc($apk_path); - $remote_crc = _atf_get_remote_file_crc($apk_name); - - if($remote_crc === '' || $local_crc !== $remote_crc) - atf_host_put_file($apk_path, $apk_name); - else - atf_log("Skipping same .apk upload"); - } - - //2. reboot just in case - if($adb_reboot) - atf_adb_reboot(); - - - $install = new ATFApkInstaller($apk_name, $override_conf); - $this->trySendApkStatsEvent($install->getSize()); - - foreach($this->plans as $plan) - $this->_runPlan($install, $plan); - - foreach($this->plans as $plan) - { - if($plan->hasFatalProblems() || - ($plan->isOver() && !$plan->getDevices())) - return false; - } - - return true; - } - - - function _runPlan(ATFApkInstaller $install, ATFPlan $plan) - { - atf_log("Starting " . $plan->getTitle() . "..."); - - //NOTE: hung_threshold must be larger than gone_threshold - Amp\Promise\wait($plan->runAsync($install, $sleep_time = 4, $hung_threshold = 80, $gone_threshold = 30, $stuck_threshold = 200)); - } - - function trySendStatsFromJzonAsync(ATFTask $task, $jzon) - { - return Amp\call(function() use($task, $jzon) { - - try - { - $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); - $table = $data['table']; - unset($data['table']); - - if(isset($data['deviceMemoryUsage']) && $data['deviceMemoryUsage'] === '') - { - $mem = yield atf_device_mem_async($task->device); - $data['deviceMemoryUsage'] = $mem['total']; - if($table === 'device_memory') - { - $data['deviceMemoryUsageNative'] = $mem['native']; - $data['deviceMemoryUsageSystem'] = $mem['system']; - $data['deviceMemoryUsageJava'] = $mem['java']; - $data['deviceMemoryUsageGraphics'] = $mem['graphics']; - } - } - $this->trySendStats($task, $table, $data); - } - catch(Exception $e) - { - echo $e->getMessage() . "\n"; - } - - }); - } - - function trySendApkStatsEvent($apk_size) - { - try - { - $msg = "version $this->version($this->rev_hash) size: $apk_size"; - atf_slack_post($msg, array('channel' => _atf_slack_chan_qa())); - - $data['time'] = time(); - $data['version'] = $this->version; - $data['revHash'] = $this->rev_hash; - $data['guid'] = $this->guid; - $data['branch'] = $this->branch; - $data['event'] = 'apk_size'; - $data['value'] = $apk_size; - atf_stats_send('event_value', $data); - } - catch(Exception $e) - { - echo $e->getMessage() . "\n"; - } - } - - - function trySendStats(ATFTask $task, $table, array $data) - { - try - { - $data['guid'] = $this->guid; - $data['time'] = time(); - $data['deviceId'] = $task->device; - atf_stats_send($table, $data); - } - catch(Exception $e) - { - echo $e->getMessage() . "\n"; - } - } - - function trySendStatsEvent(ATFTask $task, $event, $value = 1) - { - $this->trySendStats( - $task, - 'event_value', - array( - 'event' => $event, - 'value' => $value, - 'branch' => $this->branch, - 'version' => $this->version, - 'revHash' => $this->rev_hash - ) - ); - } - - function tryShareToQAChannel($msg_slack_id, $error) - { - if(!$this->share_with_qa_chan) - return; - - //let's skip similar already shared errors - if($this->_calcSharedErrorSimilarity($error) > 80) - return; - - $this->shared_qa_errors[] = $error; - - //let's share an exception message to goh-qa channel - $resp = atf_slack_get_permalink($msg_slack_id); - if(isset($resp['permalink'])) - atf_slack_post($resp['permalink'], array('channel' => _atf_slack_chan_qa())); - } - - function _calcSharedErrorSimilarity($error) - { - $max_perc = 0; - foreach($this->shared_qa_errors as $shared) - { - $perc = 0; - similar_text($shared, $error, $perc); - if($perc > $max_perc) - $max_perc = $perc; - } - return $max_perc; - } -} - - -class ATFPlan -{ - const EXCEPTIONS_THRESHOLD = 5; - const BOGUS_DEVICE_COUNT_THRESHOLD = 5; - - public $session; - public $name; - public $device_pool; - public $tasks = array(); - public $slack_thread_ts; - - function __construct($name, IATFDevicePool $device_pool) - { - $this->name = $name; - $this->device_pool = $device_pool; - } - - function getTitle() - { - return "Testing plan '{$this->name}' devices:".sizeof($this->getDevices())."(n/a:".sizeof($this->session->ignored_devices).")"; - } - - function createSlackThread() - { - for($i=0;$i<5;++$i) - { - $resp = atf_slack_post($this->getTitle()); - if(isset($resp['ok'])) - { - $this->slack_thread_ts = $resp['ts']; - return; - } - sleep(1); - } - throw new Exception("Could not create Slack thread"); - } - - function updateSlackThread() - { - $issues_summary = ''; - foreach($this->getProblemsHistogram() as $code => $count) - { - if($code == ATFTask::CODE_GONE) - $issues_summary .= " gones:$count"; - else if($code == ATFTask::CODE_NSTART) - $issues_summary .= " nstrts:$count"; - else if($code == ATFTask::CODE_STUCK) - $issues_summary .= " stucks:$count"; - else if($code == ATFTask::CODE_HUNG) - $issues_summary .= " hungs:$count"; - else if($code == ATFTask::CODE_EXCEPTION) - $issues_summary .= " excepts:$count"; - else if($code == ATFTask::CODE_WARN) - $issues_summary .= " warns:$count"; - } - - $running_count = 0; - $over_count = 0; - $total_progress = 0; - - foreach($this->tasks as $task) - { - $total_progress += $task->getProgress(); - if($task->isOver()) - $over_count++; - if($task->device) - $running_count++; - } - $progress_txt = ' tasks:' . ($running_count > 0 ? $running_count . '/' : '') . $over_count . '/' . sizeof($this->tasks) . ' (' . round($total_progress/sizeof($this->tasks)*100, 2) . '%)'; - - atf_slack_update($this->slack_thread_ts, - array(array( - "text" => $this->getTitle() . $issues_summary . $progress_txt, - "color" => $this->_getThreadColor(), - "mrkdwn_in" => array("text") - )) - ); - } - - function getProblemsHistogram() - { - $histogram = array(); - foreach($this->tasks as $task) - { - foreach($task->status_codes as $item) - { - list($code, $_) = $item; - if(!ATFTask::isProblemCode($code)) - continue; - - if(!isset($histogram[$code])) - $histogram[$code] = 0; - ++$histogram[$code]; - } - } - return $histogram; - } - - function hasStatusCode($code) - { - return $this->countStatusCode($code) > 0; - } - - function countStatusCode($code) - { - $c = 0; - foreach($this->tasks as $task) - $c += $task->countStatusCode($code); - return $c; - } - - function _getThreadColor() - { - if(!$this->getDevices()) - return "#000000"; - - $has_gones = $this->hasStatusCode(ATFTask::CODE_GONE); - $has_hungs = $this->hasStatusCode(ATFTask::CODE_HUNG); - $has_stucks = $this->hasStatusCode(ATFTask::CODE_STUCK); - $has_nostarts = $this->hasStatusCode(ATFTask::CODE_NSTART); - $has_excepts = $this->hasStatusCode(ATFTask::CODE_EXCEPTION); - $has_warns = $this->hasStatusCode(ATFTask::CODE_WARN); - - if($has_gones || $has_hungs || $has_warns || $has_excepts || $has_stucks || $has_nostarts) - { - if(!$has_excepts) - return $this->isOver() ? "warning" : "#FFCF9E"; - else - return $this->isOver() ? "danger" : "#FFCCCB"; - } - else if($this->isOver()) - return "good"; - else - return "#D3D3D3"; - } - - function _ignoreDevice($device, $reason) - { - $this->session->ignored_devices[] = $device; - atf_log("Ignoring device *$device*: $reason"); - atf_slack_post("Ignoring device *$device*: $reason", array('thread_ts' => $this->slack_thread_ts)); - } - - function getDevices() - { - $devices = array_diff($this->device_pool->get(), $this->session->ignored_devices); - return $devices; - } - - function addTask(ATFTask $task) - { - $this->tasks[] = $task; - } - - function isOver() - { - if(!$this->getDevices()) - { - return true; - } - if($this->countStatusCode(ATFTask::CODE_EXCEPTION) >= self::EXCEPTIONS_THRESHOLD) - { - return true; - } - foreach($this->tasks as $task) - { - if(!$task->isOver()) - return false; - } - return true; - } - - function hasFatalProblems() - { - foreach($this->tasks as $task) - { - foreach($task->status_codes as $item) - { - if(ATFTask::isFatalCode($item[0])) - return true; - } - } - return false; - } - - - function _findUnassignedTask() - { - foreach($this->tasks as $task) - { - if(!$task->device && !$task->isOver()) - return $task; - } - return null; - } - - function runAsync(ATFApkInstaller $install, $sleep_time, $hung_threshold, $gone_threshold, $stuck_threshold) - { - $this->createSlackThread(); - - $this->session->resetBogusDevices(); - - $cs = array(); - - foreach($this->tasks as $task) - $cs[] = $this->_runTaskAsync($task, $install, $sleep_time, $hung_threshold, $gone_threshold, $stuck_threshold); - - return Amp\Promise\all($cs); - } - - - function _runTaskAsync(ATFTask $task, ATFApkInstaller $install, $sleep_time, $hung_threshold, $gone_threshold, $stuck_threshold) - { - return Amp\call(function() use($task, $install, $sleep_time, $hung_threshold, $gone_threshold, $stuck_threshold) { - while(!$this->isOver() && !$task->isOver()) - { - $this->updateSlackThread(); - - yield Amp\delay((int)$sleep_time*1000); - - if(!$task->device) - { - if(!yield $this->_findDeviceAndStartAsync($task, $install)) - continue; - } - - $check_error = yield $this->_tryCheckExtStatusAsync($task, /*attempts*/3, /*timeout*/20); - if($check_error !== null) - { - $this->_ignoreDevice($device, $check_error); - if (!get('EXT_BOT_ERROR_PAUSE')) - $task->reschedule(); - } - - if(!$task->isOver()) - yield $this->_checkHealthAsync($task, $hung_threshold, $gone_threshold, $stuck_threshold); - - if($task->hasFatalProblem()) - { - yield $this->_processFatalProblemAsync($task); - } - else if($task->isDone()) - { - //let's reset hung stats for this device - $this->session->resetBogusDeviceCount($task->device); - //let's free the device - $task->device = null; - } - } - - $this->updateSlackThread(); - }); - } - - function _findDeviceAndStartAsync(ATFTask $task, ATFApkInstaller $install) - { - return Amp\call(function() use($task, $install) { - - $device = $this->_findFreeDevice(); - if(!$device) - return false; - - //let's mark the device as occupied ASAP - $task->device = $device; - try - { - atf_slack_post("Preparing *{$task->device}*", array('thread_ts' => $this->slack_thread_ts)); - - yield $install->installAsync($device); - yield atf_start_ext_cmd_on_device_async($device, $task->getCmd(), $task->getCmdArgs()); - } - catch(Exception $e) - { - $this->_ignoreDevice($device, $e->getMessage()); - $task->reschedule(); - return false; - } - - $task->start($this->slack_thread_ts); - return true; - - }); - } - - function _processFatalProblemAsync(ATFTask $task) - { - return Amp\call(function() use($task) { - - $this->_postScreenToSlack($task); - - $fatal_msg = "Fatal problem ({$task->last_fatal_problem} - ".ATFTask::code2string($task->last_fatal_problem)."), attempt:{$task->attempts} *{$task->device}*"; - - atf_log("[FTL] $fatal_msg"); - - atf_slack_post($fatal_msg, array('thread_ts' => $task->slack_thread_ts)); - - if($task->last_fatal_problem == ATFTask::CODE_GONE || - $task->last_fatal_problem == ATFTask::CODE_HUNG) - { - $app_log = atf_get_logcat_unity($task->device, 300); - $app_log = _atf_trim_start($app_log, 2000); - atf_slack_post("Last logs: ```$app_log``` *{$task->device}*", array('thread_ts' => $task->slack_thread_ts)); - } - - if (!get('EXT_BOT_ERROR_PAUSE')) - $task->reschedule(); - - }); - } - - - function _findFreeDevice() - { - $available = array(); - - foreach($this->getDevices() as $device) - { - $busy = false; - foreach($this->tasks as $task) - { - if($task->device == $device) - { - $busy = true; - break; - } - } - if(!$busy) - $available[] = $device; - } - - if(!$available) - return null; - return $available[mt_rand(0, sizeof($available) - 1)]; - } - - function _tryCheckExtStatusAsync(ATFTask $task, $attempts, $timeout) - { - return Amp\call(function() use($task, $attempts, $timeout) { - - $last_error = null; - - for($i=0;$i<$attempts;++$i) - { - try - { - yield $this->_checkExtStatusAsync($task, $timeout); - break; - } - catch(Exception $e) - { - $last_error = $e->getMessage(); - continue; - } - } - - return $last_error; - - }); - } - - function _checkExtStatusAsync(ATFTask $task, $timeout) - { - return Amp\call(function() use($task, $timeout) { - - $ext_status = yield atf_get_ext_status_async($task->device, $timeout); - if(!is_array($ext_status)) - return; - - $new_items = $this->_getExtStatusItemsSince($task->last_ext_status_item_time, $ext_status); - if(!$new_items) - return; - - $task->last_ext_status_item_time = end($new_items)['time']; - - foreach($new_items as $item) - yield $this->_analyzeExtStatusItemAsync($task, $item); - - }); - } - - - function _getExtStatusItemsSince($time, array $ext_status) - { - $new_items = array(); - foreach($ext_status['entries'] as $item) - { - if($item['time'] > $time) - $new_items[] = $item; - } - return $new_items; - } - - - function _checkHealthAsync(ATFTask $task, $hung_threshold, $gone_threshold, $stuck_threshold) - { - return Amp\call(function() use($task, $hung_threshold, $gone_threshold, $stuck_threshold) { - - $device_is_bogus = false; - - $not_alive_time = microtime(true) - $task->last_alive_check_time; - $stuck_time = microtime(true) - $task->last_stuck_check_time; - - if($not_alive_time > $hung_threshold) - { - $task->addStatusCode(ATFTask::CODE_HUNG); - $this->session->trySendStatsEvent($task, 'hung'); - atf_log("[HNG] No activity for $hung_threshold seconds *{$task->device}*"); - $device_is_bogus = true; - } - else if($not_alive_time > $gone_threshold) - { - list($status, $_) = yield atf_host_exec_async("%{adb}% -s {$task->device} shell pidof %{package_id}%", DEPLOY_OPT_ERR_OK); - if($status != 0) - { - if($task->last_alive_check_time === $task->reset_time) - { - $task->addStatusCode(ATFTask::CODE_NSTART); - $this->session->trySendStatsEvent($task, 'nstart'); - atf_log("[NFD] No app started after $hung_threshold seconds *{$task->device}*"); - } - else - { - $task->addStatusCode(ATFTask::CODE_GONE); - $this->session->trySendStatsEvent($task, 'gone'); - atf_log("[GNE] App is gone after $hung_threshold seconds *{$task->device}*"); - } - $device_is_bogus = true; - } - else - //let's tap the screen just in case app Activity is not in foreground - yield atf_host_exec_async("%{adb}% -s {$task->device} shell input tap 360 930", DEPLOY_OPT_ERR_OK); - } - else if($stuck_time > $stuck_threshold) - { - $task->addStatusCode(ATFTask::CODE_STUCK); - $this->session->trySendStatsEvent($task, 'stuck'); - atf_log("[STK] Stuck for $stuck_threshold seconds *{$task->device}*"); - } - - if($device_is_bogus) - { - $this->_reportErrorFromLogcatToSlack($task, 1000); - $this->_incAndCheckBogusDevice($task->device); - } - - }); - } - - function _incAndCheckBogusDevice($device) - { - if($this->session->incBogusDevice($device) >= self::BOGUS_DEVICE_COUNT_THRESHOLD) - { - $this->_ignoreDevice($device, "Too many N/As"); - return true; - } - return false; - } - - static function _parseExtMessage($message) - { - $msg_code = null; - $msg_text = $message; - //example: [DBG] this a debug message - if(preg_match('~^(\[[^\]]+\])(.*)$~', $message, $matches)) - { - $msg_code = $matches[1]; - $msg_text = $matches[2]; - } - return array($msg_code, $msg_text); - } - - static function _printToShellExtItem(ATFTask $task, array $item) - { - $shell_msg = _atf_trim($item['message'], 200); - if(ATFTask::isProblemCode($item['error'])) - $shell_msg = "[PROBLEM] Code:{$item['error']}, $shell_msg"; - $shell_msg = "(".round($item['time'],1)."s) {$shell_msg} *{$task->device}*"; - - atf_log($shell_msg); - } - - function _postToSlackExtStatusItem(ATFTask $task, array $item) - { - $orig_msg = _atf_trim($item['message'], 3000); - $slack_msg = $orig_msg; - - if($item['error'] == ATFTask::CODE_EXCEPTION) - $slack_msg = "```$slack_msg```"; - - $problem = $item['error'] == ATFTask::CODE_EXCEPTION || $item['error'] == ATFTask::CODE_STUCK; - - $slack_msg = '('.round($item['time'],1).'s) '.$slack_msg.' *'.$task->device.'*'; - - $mentions = getor("ATF_PROBLEM_MENTION_GROUPS", null); - if($mentions !== null && $problem && strlen($mentions) > 0) - { - $mentions = explode(",", $mentions); - $groups = ""; - foreach($mentions as $mention) - $groups .= ' '; // - - if(strlen($groups) > 0) - $slack_msg = $groups."\n".$slack_msg; - } - - $resp = atf_slack_post($slack_msg, array('thread_ts' => $task->slack_thread_ts)); - if(isset($resp['ok']) && $problem) - $this->session->tryShareToQAChannel($resp['ts'], $orig_msg); - } - - function _analyzeExtStatusItemAsync(ATFTask $task, array $item) - { - return Amp\call(function() use($task, $item) { - - self::_printToShellExtItem($task, $item); - - $task->addStatusCode($item['error'], $item['message']); - - if($item['error'] == ATFTask::CODE_EXCEPTION) - $this->session->trySendStatsEvent($task, 'exception'); - - list($msg_type, $msg_text) = self::_parseExtMessage($item['message']); - - //NOTE: alive system messages are sent periodically by ExtBot itself, - // we'd like to ignore them for stuck detection - if($msg_type !== '[ALIVE]') - $task->last_stuck_check_time = microtime(true); - - if($msg_type === '[DONE]') - { - $task->is_done = true; - $this->_postToSlackExtStatusItem($task, $item); - } - - //NOTE: in case of any message from the device we update the alive check timestamp - $task->last_alive_check_time = microtime(true); - - yield $this->_processExtStatusMessageAsync($task, $item, $msg_type, $msg_text); - - }); - } - - function _processExtStatusMessageAsync($task, $item, $msg_type, $msg_text) - { - return Amp\call(function() use($task, $item, $msg_type, $msg_text) { - - if($msg_type === '[PRG]') - { - //let's reset hung stats since progress is going - $this->session->resetBogusDeviceCount($task->device); - $task->onProgress($msg_text); - } - else if($msg_type === '[STAT]') - yield $this->session->trySendStatsFromJzonAsync($task, $msg_text); - else if($msg_type === '[WRN]') - { - $task->addStatusCode(ATFTask::CODE_WARN); - $this->_postToSlackExtStatusItem($task, $item); - } - else if($msg_type === null) - { - $this->_postToSlackExtStatusItem($task, $item); - } - - }); - } - - function _reportErrorFromLogcatToSlack(ATFTask $task, $limit) - { - $errors_log = _atf_trim(atf_get_logcat_errors($task->device, $limit), 3000); - if($errors_log) - atf_slack_post("```$errors_log``` *{$task->device}*", array('thread_ts' => $task->slack_thread_ts)); - } - - function _postScreenToSlack(ATFTask $task) - { - global $GAME_ROOT; - - $png_data = atf_screen($task->device); - if($png_data) - { - ensure_write("$GAME_ROOT/build/atf/screen.png", $png_data); - atf_slack_post_png("screen", "$GAME_ROOT/build/atf/screen.png", array('thread_ts' => $task->slack_thread_ts)); - } - } -} - - -class ATFTask -{ - const CODE_NONE = 0; - const CODE_EXCEPTION = 1; - const CODE_WARN = 2; - const CODE_NSTART = 124; - const CODE_STUCK = 125; - const CODE_GONE = 126; - const CODE_HUNG = 127; - const CODE_DONE = 128; - - - static function code2string($code) - { - switch($code) - { - case self::CODE_NONE: - return "none"; - case self::CODE_EXCEPTION: - return "exception"; - case self::CODE_WARN: - return "warning"; - case self::CODE_STUCK: - return "stuck"; - case self::CODE_NSTART: - return "nstart"; - case self::CODE_GONE: - return "gone"; - case self::CODE_HUNG: - return "hung"; - case self::CODE_DONE: - return "done"; - } - return "???"; - } - - public $cmd; - public $args; - public $title; - public $last_progress; - public $last_done_arg_idx; - public $slack_thread_ts; - public $reset_time = 0; - public $last_alive_check_time = 0; - public $last_stuck_check_time = 0; - public $last_ext_status_item_time = 0; - public $last_ext_input_req_id = 0; - public $status_codes = array(); - public $is_done; - public $last_fatal_problem = 0; - public $attempts = 1; - public $device; - - function __construct($cmd, array $args, $title = '') - { - $this->cmd = $cmd; - $this->args = $args; - $this->title = $title; - } - - function getCmd() - { - return $this->cmd; - } - - function getCmdArgs() - { - return $this->args; - } - - function getProgress() - { - return $this->last_progress; - } - - static function isProblemCode($code) - { - return $code > self::CODE_NONE && $code < self::CODE_DONE; - } - - static function isFatalCode($code) - { - return - $code == ATFTask::CODE_STUCK || - $code == ATFTask::CODE_NSTART || - $code == ATFTask::CODE_GONE || - $code == ATFTask::CODE_HUNG || - $code == ATFTask::CODE_EXCEPTION; - } - - function hasFatalProblem() - { - return $this->last_fatal_problem != 0; - } - - function hasStatusCode($code) - { - return $this->countStatusCode($code) > 0; - } - - function countStatusCode($code) - { - $c = 0; - foreach($this->status_codes as $item) - { - if($item[0] === $code) - $c++; - } - return $c; - } - - function isDone() - { - return $this->is_done; - } - - function isOver() - { - return $this->isDone() || $this->hasFatalProblem(); - } - - function addStatusCode($code, $msg = '') - { - $this->status_codes[] = array($code, $msg); - - if($code == ATFTask::CODE_DONE) - $this->is_done = true; - else if(self::isFatalCode($code)) - $this->last_fatal_problem = $code; - } - - - function onProgress($jzon) - { - try - { - $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); - if(isset($data['p'])) - $this->last_progress = floatval(str_replace(',', '.', str_replace('.', '', $data['p']))); - if(isset($data['arg_idx'])) - $this->last_done_arg_idx = (int)$data['arg_idx']; - } - catch(Exception $e) - { - echo $e->getMessage() . "\n"; - } - } - - function _resetCheckTimes() - { - $this->reset_time = microtime(true); - $this->last_alive_check_time = $this->reset_time; - $this->last_stuck_check_time = $this->reset_time; - $this->last_ext_status_item_time = 0; - } - - function reschedule() - { - ++$this->attempts; - $this->device = null; - } - - function start($slack_thread_ts) - { - $this->_resetCheckTimes(); - $this->slack_thread_ts = $slack_thread_ts; - } -} - -function atf_host_exec_async($cmd, $opts = 0, $timeout = 10) : Amp\Promise -{ - return Amp\call(function() use($cmd, $opts, $timeout) { - - $timeout_cmd = "/usr/local/bin/timeout -k 5s $timeout $cmd"; - - $res = yield deploy_ssh_exec_async(atf_host(), $timeout_cmd, $opts); + $res = yield taskman\deploy_ssh_exec_async($atf_host, $timeout_cmd, $opts); list($status, $lines) = current($res); //in case of timeout we set status to false explicitely @@ -1122,11 +23,11 @@ function atf_host_exec_async($cmd, $opts = 0, $timeout = 10) : Amp\Promise }); } -function atf_host_exec($cmd, $opts = 0, $timeout = 10) +function host_exec(string $atf_host, string $cmd, $opts = 0, $timeout = 10) { - $timeout_cmd = "/usr/local/bin/timeout -k 5s $timeout $cmd"; - - $res = deploy_ssh_exec(atf_host(), $timeout_cmd, $opts); + $timeout_cmd = "timeout -k 5s $timeout $cmd"; + + $res = taskman\deploy_ssh_exec($atf_host, $timeout_cmd, $opts); list($status, $lines) = current($res); //in case of timeout we set status to false explicitely @@ -1136,23 +37,36 @@ function atf_host_exec($cmd, $opts = 0, $timeout = 10) return array($status, $lines); } -function atf_host_get_file($file_name) +function host_get_file(string $atf_host, string $file_name) { - $file_data = current(deploy_get_file(atf_host(), $file_name)); + $file_data = current(taskman\deploy_get_file($atf_host, $file_name)); return $file_data; } -function atf_host_put_file($local_path, $remote_path) +function host_put_file(string $atf_host, string $local_path, string $remote_path) { - deploy_scp_put_file(atf_host(), $local_path, $remote_path); + taskman\deploy_rsync($atf_host, $local_path, $remote_path); } -function atf_host_put_file_async($local_path, $remote_path) +function host_put_file_async(string $atf_host, string $local_path, string $remote_path) { - return deploy_scp_put_file_async(atf_host(), $local_path, $remote_path); + return taskman\deploy_rsync_async($atf_host, $local_path, $remote_path); } -function _atf_slice(array $items, $max) +function _spread(array $items, $num) +{ + $k = sizeof($items) / $num; + + $res = array(); + for($i=0;$i<$num;++$i) + { + $tmp = array_slice($items, (int)floor($k*$i), (int)ceil($k)); + $res[$i] = $tmp; + } + return $res; +} + +function _slice(array $items, $max) { $sliced = array(); @@ -1169,256 +83,119 @@ function _atf_slice(array $items, $max) return $sliced; } -function _atf_get_remote_file_crc($path) -{ - list($status, $lines) = atf_host_exec("crc32 $path", DEPLOY_OPT_ERR_OK); - if($status != 0) - return ''; - return trim(implode("\n", $lines)); -} - -function _atf_get_local_file_crc($path) -{ - if(!is_file($path)) - return ''; - - shell("crc32 $path", $out); - return trim($out[0]); -} - -function atf_device_put_file_async($device, $data_or_file, $device_path, $is_file = false) -{ - return Amp\call(function() use($device, $data_or_file, $device_path, $is_file) { - - if($device_path[0] !== "/") - $device_path = atf_package_dir() . $device_path; - - $tmp_remote_file = uniqid(basename($device_path)."_"); - - if(!$is_file) - { - $local_path = tempnam("/tmp", "atf_"); - ensure_write($local_path, $data_or_file); - } - else - $local_path = $data_or_file; - - try - { - //1. let's copy local file to the ATF host as temp one - yield atf_host_put_file_async($local_path, $tmp_remote_file); - } - finally - { - if(!$is_file) - ensure_rm($local_path); - } - - //2. let's push the temp file to the device - $device_dir = dirname($device_path); - yield atf_host_exec_async("%{adb}% -s $device shell mkdir -p $device_dir"); - yield atf_host_exec_async("%{adb}% -s $device push $tmp_remote_file $device_path", 0, 30); - yield atf_host_exec_async("rm -rf $tmp_remote_file"); - -}); -} - -function atf_device_put_folder_async($device, $data, $device_path) -{ - return Amp\call(function() use($device, $data, $device_path) { - - if($device_path[0] !== "/") - $device_path = atf_package_dir() . $device_path; - - $device_dir = dirname($device_path); - - yield atf_host_exec_async("%{adb}% -s $device shell rm -rf $device_path"); - yield atf_host_exec_async("%{adb}% -s $device shell mkdir -p $device_dir"); - yield atf_host_exec_async("%{adb}% -s $device push $data $device_path", 0, 400); - }); -} - -function atf_device_del_file_async($device, $device_path, $opts = 0) -{ - if($device_path[0] !== "/") - $device_path = atf_package_dir() . $device_path; - - return atf_host_exec_async("%{adb}% -s $device shell rm -rf $device_path", $opts); -} - -function atf_device_blink($device, $times = 5) -{ - //disable auto brightness - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness_mode 0"); - - for($i=0;$i<$times;++$i) - { - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness 255"); - sleep(1); - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness 5"); - } - - //enable auto brightness - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness 100"); - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness_mode 1"); -} - -function atf_device_pull_file_async($device, $device_path, $timeout = 10, $throw_error_on_timeout = false) -{ - return Amp\call(function() use($device, $device_path, $throw_error_on_timeout, $timeout) { - - if($device_path[0] !== "/") - $device_path = atf_package_dir() . $device_path; - - $tmp_remote_file = uniqid(basename($device_path)."_"); - - list($status, $_) = yield atf_host_exec_async("%{adb}% -s $device pull $device_path $tmp_remote_file", DEPLOY_OPT_ERR_OK, $timeout); - - if($status !== 0) - { - if($status === false && $throw_error_on_timeout) - throw new Exception("Could not pull from device '$device' file '$device_path' due to timeout"); - return null; - } - - $file_data = atf_host_get_file($tmp_remote_file); - yield atf_host_exec_async("rm -rf $tmp_remote_file"); - return $file_data; - - }); -} - -function atf_screen($device) -{ - $screen_file_name = uniqid("screen_"); - try - { - atf_host_exec("%{adb}% -s $device exec-out screencap -p > $screen_file_name"); - } - catch(Exception $e) - { - return false; - } - $data = atf_host_get_file($screen_file_name); - atf_host_exec("rm -rf $screen_file_name"); - return $data; -} - -function atf_guid() -{ - return uniqid(); -} - -function _atf_start_watchdog($guid) -{ - global $GAME_ROOT; - - $script = "$GAME_ROOT/gamectl atf_watchdog " . getmypid() . ' ' . $guid; - exec("nohup $script > /dev/null &", $output, $ret); -} - -function _atf_trim($txt, $max_len) +function _trim($txt, $max_len) { return strlen($txt) > $max_len ? substr($txt, 0, $max_len) . "..." : $txt; } -function _atf_trim_start($txt, $max_len) +function _trim_start($txt, $max_len) { return strlen($txt) > $max_len ? "..." . substr($txt, strlen($txt) - $max_len) : $txt; } -function _atf_db() +function _trim_lines($txt, $max_lines) { - $config = array( - 'host' => '172.19.179.148', - 'port' => '8123', - 'username' => get('ATF_CLICKHOUSE_USERNAME'), - 'password' => get('ATF_CLICKHOUSE_PASSWORD') - ); - $db = new ClickHouseDB\Client($config); - $db->database(get('ATF_CLICKHOUSE_DB')); - $db->setTimeout(10); - $db->setConnectTimeOut(5); - return $db; + $lines = explode("\n", $txt, $max_lines+1); + $res = ''; + for($i=0;$iinsert($table, array(array_values($data)), array_keys($data)); + if(!$txt) + return $txt; + //let's check if it's already archieved + if(strlen($txt) > 2 && $txt[1] === ':') + return $txt; + if(!function_exists('lz4_compress')) + return $txt; + return "4:" . base64_encode(lz4_compress(base64_decode($txt))); } -function atf_device_mem_async($device) -{ - return Amp\call(function() use($device) { - list($code, $lines) = yield atf_host_exec_async("%{adb}% -s $device shell dumpsys meminfo -s %{package_id}%", DEPLOY_OPT_ERR_OK, 20); - - $res = array( - 'total' => 0, - 'native' => 0, - 'java' => 0, - 'system' => 0, - 'graphics' => 0, - ); - - if($code !== 0) - return $res; - - foreach($lines as $idx => $line) - { - $items = preg_split('/\s+/', trim($line)); - if($items && $items[0] === "TOTAL:") - $res['total'] = 1*$items[1]; - else if($items && $items[0] === "TOTAL" && $items[1] === "PSS:") - $res['total'] = 1*$items[2]; - else if($items && $items[0] === "Native" && $items[1] === "Heap:") - $res['native'] = 1*$items[2]; - else if($items && $items[0] === "Java" && $items[1] === "Heap:") - $res['java'] = 1*$items[2]; - else if($items && $items[0] === "System:") - $res['system'] = 1*$items[1]; - else if($items && $items[0] === "Graphics:") - $res['graphics'] = 1*$items[1]; - } - - return $res; - }); -} - -function atf_log($msg) +function log($msg) { echo date("Y-m-d H:i:s") . " " . $msg . "\n"; } -function atf_adb_reboot() +function err($msg) { - atf_host_exec("killall adb"); - atf_host_exec("%{adb}% kill-server"); - atf_host_exec("%{adb}% start-server"); + taskman\stderr(date("Y-m-d H:i:s") . " " . $msg . "\n"); } -function atf_reboot_devices(array $devices) +function _retry($max_tries, $func) { - atf_log("Rebooting devices..."); + for($i=0;$i<$max_tries;++$i) + { + try + { + $func(); + return; + } + catch(Exception $e) + { + if(($i+1) == $max_tries) + throw $e; + } + sleep(1); + } +} + +function adb_reboot(string $atf_host) +{ + host_exec($atf_host, "killall adb", DEPLOY_OPT_ERR_OK); + _retry(3, function() use($atf_host) { + host_exec($atf_host, "%{adb}% kill-server"); + }); + host_exec($atf_host, "%{adb}% start-server"); +} + +function reboot_devices(string $atf_host, array $devices, int $wait_after = 40) +{ + log("Rebooting devices..."); //let's reboot devices $ces = array(); foreach($devices as $device) { - $ces[] = Amp\call(function() use($device) { - yield atf_host_exec_async("%{adb}% -s $device reboot", DEPLOY_OPT_ERR_OK); - yield Amp\delay(40*1000); + $ces[] = Amp\call(function() use($atf_host, $device, $wait_after) { + yield host_exec_async($atf_host, "%{adb}% -s $device reboot", DEPLOY_OPT_ERR_OK); + yield Amp\delay($wait_after*1000); }); } Amp\Promise\wait(Amp\Promise\all($ces)); } -function atf_get_devices($extended = false) +function get_ext_status_async(string $atf_host, string $device, int $timeout = 20) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $timeout) { + + $file_data = yield device_pull_file_async($atf_host, $device, "atf_status.js", $timeout, /*throw on timeout*/true); + if($file_data === null) + return null; + $file_data = preg_replace('/[[:cntrl:]]/', '', $file_data); + return json_decode($file_data, true); + + }); +} + +function start_ext_cmd_on_device_async(string $atf_host, string $device, Cmd $cmd, array $cmd_args) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $cmd, $cmd_args) { + + yield device_del_file_async($atf_host, $device, 'atf_status.js', DEPLOY_OPT_ERR_OK); + yield host_exec_async($atf_host, "%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK, 30); + yield update_ext_cmd_async($atf_host, $device, $cmd, $cmd_args); + yield host_exec_async($atf_host, "%{adb}% -s $device shell am start -n %{package_id}%/%{activity}%", 0, 30); + + }); +} + +function get_devices(string $atf_host, bool $extended = false) { $devices = array(); - list($_, $lines) = atf_host_exec("%{adb}% devices -l", DEPLOY_OPT_SILENT); - + list($_, $lines) = host_exec($atf_host, "%{adb}% devices -l", DEPLOY_OPT_SILENT); foreach($lines as $line) { @@ -1438,63 +215,240 @@ function atf_get_devices($extended = false) $devices[] = $m[1]; } } - return $devices; } -function atf_get_ext_status_async($device, $timeout = 20) +function update_ext_cmd_async(string $atf_host, string $device, Cmd $cmd, array $args) { - return Amp\call(function() use($device, $timeout) { - - $file_data = yield atf_device_pull_file_async($device, "ext_status.js", $timeout, /*throw on timeout*/true); - if($file_data === null) - return null; - $file_data = preg_replace('/[[:cntrl:]]/', '', $file_data); - return json_decode($file_data, true); + $ext_cmd = array( + "device_id" => $device, + "module" => $cmd->module, + "func" => $cmd->func, + "args" => $args, + ); + + return device_put_file_async($atf_host, $device, json_encode($ext_cmd), 'atf_cmd.js'); +} + +function package_dir(string $atf_host) +{ + return "/storage/emulated/0/Android/data/".taskman\deploy_get($atf_host, 'package_id')."/files/"; +} + +function device_put_file_async(string $atf_host, string $device, string $data_or_file, string $device_path, bool $is_file = false, $timeout = 30) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $data_or_file, $device_path, $is_file, $timeout) { + + if($device_path[0] !== "/") + $device_path = package_dir($atf_host) . $device_path; + + $tmp_remote_file = '%{dir}%'.uniqid(basename($device_path)."_"); + + if(!$is_file) + { + $local_path = tempnam("/tmp", "atf_"); + taskman\ensure_write($local_path, $data_or_file); + } + else + { + $local_path = $data_or_file; + if(!is_file($local_path)) + throw new Exception("No such local file: $local_path"); + } + + try + { + //1. let's copy local file to the ATF host as temp one + yield host_put_file_async($atf_host, $local_path, $tmp_remote_file); + } + finally + { + if(!$is_file) + taskman\ensure_rm($local_path); + } + + //2. let's push the temp file to the device + $device_dir = dirname($device_path); + yield host_exec_async($atf_host, "%{adb}% -s $device shell mkdir -p $device_dir"); + yield host_exec_async($atf_host, "%{adb}% -s $device push $tmp_remote_file $device_path", 0, $timeout); + yield host_exec_async($atf_host, "%{adb}% -s $device shell chmod -R 777 $device_dir", DEPLOY_OPT_ERR_OK); + yield host_exec_async($atf_host, "rm -rf $tmp_remote_file"); }); } -function atf_update_ext_cmd_async($device, $main, array $args) +function device_del_file(string $atf_host, string $device, string $device_path, $opts = 0) { - $ext_cmd = array( - "device_id" => $device, - "main" => $main, - "args" => $args, - ); + if($device_path[0] !== "/") + $device_path = package_dir($atf_host) . $device_path; - return atf_device_put_file_async($device, json_encode($ext_cmd), 'ext_cmd.js'); + host_exec($atf_host, "%{adb}% -s $device shell rm -rf $device_path", $opts); } -function atf_package_dir() +function device_del_file_async(string $atf_host, string $device, string $device_path, $opts = 0) { - return "/storage/emulated/0/Android/data/".deploy_get(atf_host(), 'package_id')."/files/"; + if($device_path[0] !== "/") + $device_path = package_dir($atf_host) . $device_path; + + return host_exec_async($atf_host, "%{adb}% -s $device shell rm -rf $device_path", $opts); } -function atf_get_locat_errors_since($device, $since_stamp)//not using +function device_blink($atf_host, string $device, int $times = 5) +{ + //disable auto brightness + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness_mode 0"); + + for($i=0;$i<$times;++$i) + { + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness 255"); + sleep(1); + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness 5"); + } + + //enable auto brightness + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness 100"); + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness_mode 1"); +} + +function device_put_dir_async(string $atf_host, string $device, string $folder_path, string $device_parent_path, int $timeout = 60) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $folder_path, $device_parent_path, $timeout) { + + if($device_parent_path[0] !== "/") + $device_parent_path = package_dir($atf_host) . $device_parent_path; + + $remote_path = '/tmp/' . basename($folder_path); + + yield taskman\deploy_rsync_async($atf_host, "$folder_path/", "$remote_path/", '--delete'); + + yield host_exec_async($atf_host, "%{adb}% -s $device push --sync $remote_path $device_parent_path", 0, $timeout); + + }); +} + +function device_pull_file_async(string $atf_host, string $device, string $device_path, int $timeout = 10, bool $throw_error_on_timeout = false) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $device_path, $throw_error_on_timeout, $timeout) { + + if($device_path[0] !== "/") + $device_path = package_dir($atf_host) . $device_path; + + $tmp_remote_file = '%{dir}%/'.uniqid(basename($device_path)."_"); + + list($status, $_) = yield host_exec_async($atf_host, "%{adb}% -s $device pull $device_path $tmp_remote_file", DEPLOY_OPT_ERR_OK, $timeout); + + if($status !== 0) + { + if($status === false && $throw_error_on_timeout) + throw new Exception("Could not pull from device '$device' file '$device_path' due to timeout"); + return null; + } + + $file_data = host_get_file($atf_host, $tmp_remote_file); + yield host_exec_async($atf_host, "rm -rf $tmp_remote_file"); + return $file_data; + + }); +} + +function get_last_replay_async(string $atf_host, string $device, string $device_replay_path) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $device_replay_path) { + $file = package_dir($atf_host) . $device_replay_path; + list($status, $contents) = yield host_exec_async($atf_host, "%{adb}% -s $device shell cat $file", DEPLOY_OPT_ERR_OK); + return array($status, implode("\n", $contents)); + }); +} + +function _multi_curl_async($ch, $get_content) : Amp\Promise +{ + return Amp\call(function() use($ch, $get_content) { + $mh = curl_multi_init(); + curl_multi_add_handle($mh, $ch); + + do { + $status = curl_multi_exec($mh, $active); + yield Amp\delay(1); + } while ($active && $status == CURLM_OK); + + $result = null; + if($get_content) + $result = curl_multi_getcontent($ch); + + curl_multi_remove_handle($mh, $ch); + curl_multi_close($mh); + + return $result; + }); +} + +function device_screen(string $atf_host, string $device) +{ + $screen_file_name = '%{dir}%/'.uniqid("screen_").'.png'; + try + { + host_exec($atf_host, "%{adb}% -s $device exec-out screencap -p > $screen_file_name"); + } + catch(Exception $e) + { + return false; + } + $data = host_get_file($atf_host, $screen_file_name); + host_exec($atf_host, "rm -rf $screen_file_name"); + return $data; +} + +function del_external_files_async(string $atf_host, string $device, array $external_files) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $external_files) { + + foreach($external_files as $real_path => $remote_path) + yield device_del_file_async($atf_host, $device, $remote_path); + + }); +} + +function put_external_files_async(string $atf_host, string $device, array $external_files) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $external_files) { + + foreach($external_files as $real_path => $remote_path) + yield device_put_file_async($atf_host, $device, $real_path, $remote_path, true); + + }); +} + +function get_locat_errors_since(string $atf_host, string $device, int $since_stamp) { $time_spec = gmdate('m-d h:m:s.0', $since_stamp); - list($_, $lines) = atf_host_exec("%{adb}% -s $device logcat -s -v UTC -d -t '$time_spec' AndroidRuntime:E Unity:E", DEPLOY_OPT_SILENT | DEPLOY_OPT_ERR_OK); - _atf_filter_error_logs($lines); + list($_, $lines) = host_exec($atf_host, "%{adb}% -s $device logcat -s -v UTC -d -t '$time_spec' AndroidRuntime:E Unity:E", DEPLOY_OPT_ERR_OK); + _filter_error_logs($lines); return implode("\n", $lines); } -function atf_get_logcat_errors($device, $limit = 0) +function get_logcat_errors(string $atf_host, string $device, int $limit = 0) { - list($_, $lines) = atf_host_exec("%{adb}% -s $device logcat -s -d ".($limit > 0 ? "-t $limit" : "")." AndroidRuntime:E Unity:E", DEPLOY_OPT_SILENT | DEPLOY_OPT_ERR_OK); - _atf_filter_error_logs($lines); + list($_, $lines) = host_exec($atf_host, "%{adb}% -s $device logcat -s -d ".($limit > 0 ? "-t $limit" : "")." AndroidRuntime:E Unity:E", DEPLOY_OPT_ERR_OK); + _filter_error_logs($lines); return implode("\n", $lines); } -function atf_get_logcat_unity($device, $limit = 0) +function get_locat_unity_since(string $atf_host, string $device, int $since_stamp) { - list($_, $lines) = atf_host_exec("%{adb}% -s $device logcat -s -d Unity:'*'", DEPLOY_OPT_ERR_OK); + $time_spec = gmdate('m-d h:m:s.0', $since_stamp); + list($_, $lines) = host_exec($atf_host, "%{adb}% -s $device logcat -s -v UTC -d -t '$time_spec' Unity:'*'", DEPLOY_OPT_ERR_OK); + return implode("\n", $lines); +} + +function get_logcat_unity(string $atf_host, string $device, int $limit = 0) +{ + list($_, $lines) = host_exec($atf_host, "%{adb}% -s $device logcat -s -d Unity:'*'", DEPLOY_OPT_ERR_OK); if($limit > 0) array_splice($lines, 0, sizeof($lines) - $limit); return implode("\n", $lines); } -function _atf_filter_error_logs(array &$lines) +function _filter_error_logs(array &$lines) { $exception_lines = 0; $filtered = array(); @@ -1515,62 +469,68 @@ function _atf_filter_error_logs(array &$lines) $lines = $filtered; } -function atf_start_ext_cmd_on_device_async($device, $cmd_main, array $cmd_args) +function device_pss_async(string $atf_host, string $device) : Amp\Promise { - return Amp\call(function() use($device, $cmd_main, $cmd_args) { - - yield atf_device_del_file_async($device, 'ext_status.js', DEPLOY_OPT_ERR_OK); - yield atf_host_exec_async("%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK, 30); - yield atf_update_ext_cmd_async($device, $cmd_main, $cmd_args); - yield atf_host_exec_async("%{adb}% -s $device shell am start -n %{package_id}%/%{activity}%", 0, 30); - + return Amp\call(function() use($atf_host, $device) { + $res = yield device_mem_async($atf_host, $device); + return $res['total']; }); } -function atf_host() +function device_mem_async(string $atf_host, string $device) : Amp\Promise { - static $inited = false; + return Amp\call(function() use($atf_host, $device) { + list($code, $lines) = yield host_exec_async($atf_host, "%{adb}% -s $device shell dumpsys meminfo -s %{package_id}%", DEPLOY_OPT_ERR_OK, 20); - $node_name = 'atf'; - if($inited) - return $node_name; + $res = array( + 'total' => 0, + 'native' => 0, + 'java' => 0, + 'system' => 0, + 'graphics' => 0, + ); - deploy_declare_node($node_name, - array( - 'user' => get("ATF_USER"), - 'dir' => '', - 'ssh_key_str' => get("ATF_KEY"), - 'hosts' => array(get("ATF_HOST")), - 'adb' => get("ATF_HOST_ADB"), - 'package_id' => get("PACKAGE_ID"), - 'activity' => get("LAUNCHER_ACTIVITY") - ) - ); + if($code !== 0) + return $res; - $inited = true; - - return $node_name; -} - -function _atf_opt(array &$args, $opt, $default, $conv_fn = null) -{ - foreach($args as $idx => $arg) - { - if(strpos($arg, $opt) === 0) + foreach($lines as $idx => $line) { - $arg = substr($arg, strlen($opt)); - unset($args[$idx]); - return $conv_fn === null ? $arg : $conv_fn($arg); + $items = preg_split('/\s+/', trim($line)); + if($items && $items[0] === "TOTAL:") + $res['total'] = intval($items[1]); + else if($items && $items[0] === "TOTAL" && $items[1] === "PSS:") + $res['total'] = intval($items[2]); + else if($items && $items[0] === "Native" && $items[1] === "Heap:") + $res['native'] = intval($items[2]); + else if($items && $items[0] === "Java" && $items[1] === "Heap:") + $res['java'] = intval($items[2]); + else if($items && $items[0] === "System:") + $res['system'] = intval($items[1]); + else if($items && $items[0] === "Graphics:") + $res['graphics'] = intval($items[1]); } - } - return $default; + + return $res; + }); } -function _atf_opt_check_no_trailing(array $args) +function device_temperature_async(string $atf_host, string $device) : Amp\Promise { - foreach($args as $arg) - { - if(preg_match('~(--\w+)=.*~', $arg, $m)) - throw new Exception("Unknown option '{$m[1]}'"); - } -} \ No newline at end of file + return Amp\call(function() use($atf_host, $device) { + //NOTE: on current devices 16 thermal zone is responsible for GPU temperature probing + list($code, $lines) = yield host_exec_async($atf_host, "%{adb}% -s $device shell cat /sys/class/thermal/thermal_zone16/temp", DEPLOY_OPT_ERR_OK, 1); + + if($code !== 0) + return 0; + + foreach($lines as $idx => $line) + { + $line = trim($line); + if(!$line) + continue; + return intval($line); + } + + return 0; + }); +} diff --git a/composer.json b/composer.json index c3b7019..5372adc 100644 --- a/composer.json +++ b/composer.json @@ -1,12 +1,16 @@ { "name": "bit/taskman_atf", - "description": "android test farm", + "description": "Automation Test Farm routines", "homepage": "https://git.bit5.ru/composer/taskman_atf", "require": { "php": ">=7.4" }, "autoload": { - "classmap": ["atf.inc.php"], - "classmap": ["slack.inc.php"] + "files": ["atf.inc.php"], + "classmap" : [ + "session.inc.php", "plan.inc.php", + "tasks.inc.php", "device.inc.php", + "im.inc.php", "stats.inc.php" + ] } - } \ No newline at end of file + } diff --git a/device.inc.php b/device.inc.php new file mode 100644 index 0000000..5ea3247 --- /dev/null +++ b/device.inc.php @@ -0,0 +1,161 @@ +atf_host = $atf_host; + $this->max_installs_in_progress = $max_installs_in_progress; + $this->apk_name = $apk_name; + $this->override_external_files = $override_external_files; + $this->external_files = $external_files; + } + + function isInstalled(string $device) + { + return isset($this->installs[$device]) && $this->installs[$device] == self::ST_INSTALLED; + } + + function forgetInstall(string $device) + { + unset($this->installs[$device]); + } + + function countInstallsInProgress() + { + $c = 0; + foreach($this->installs as $status) + if($status == self::ST_IN_PROGRESS) + ++$c; + return $c; + } + + function installAsync(string $device, bool $force = false) : Amp\Promise + { + return Amp\call(function() use($device, $force) { + + while($this->countInstallsInProgress() >= $this->max_installs_in_progress) + yield Amp\delay(1000); + + if($force) + unset($this->installs[$device]); + + if(isset($this->installs[$device])) + return; + $this->installs[$device] = self::ST_IN_PROGRESS; + + try + { + yield host_exec_async($this->atf_host, "%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK); + if($this->apk_name !== null) + { + yield host_exec_async($this->atf_host, "%{adb}% -s $device uninstall %{package_id}%", DEPLOY_OPT_ERR_OK, 30); + yield host_exec_async($this->atf_host, "%{adb}% -s $device install -r %{dir}%/{$this->apk_name}", 0, 300); + + //delete overrides only if there's a new apk file uploaded + yield del_external_files_async($this->atf_host, $device, $this->external_files); + } + + if($this->override_external_files) + yield put_external_files_async($this->atf_host, $device, $this->external_files); + + $this->installs[$device] = self::ST_INSTALLED; + } + catch(Exception $e) + { + err("Error during install *$device*: " . $e->getMessage()); + unset($this->installs[$device]); + throw $e; + } + }); + } + + function getSize() + { + list($status, $lines) = host_exec($this->atf_host, "ls -sd -- %{dir}%/{$this->apk_name}", DEPLOY_OPT_ERR_OK|DEPLOY_OPT_SILENT); + if($status != 0) + return 0; + $items = explode(' ', $lines[0]); + //ls -s returns size in blocks where each block is 512 bytes long + return intval($items[0])*512; + } +} + +class AdbDevicePool implements IDevicePool +{ + private string $atf_host; + + function __construct(string $atf_host) + { + $this->atf_host = $atf_host; + } + + function get() : array + { + return get_devices($this->atf_host); + } +} + +class FixedDevicePool implements IDevicePool +{ + private array $devices; + + function __construct(array $devices) + { + $this->devices = $devices; + } + + function get() : array + { + return $this->devices; + } +} + +class CachedDevices implements IDevicePool +{ + private IDevicePool $provider; + private ?array $cached = null; + private int $last_cache_time; + private int $keep_cache_time; + + function __construct(IDevicePool $provider, int $keep_cache_time) + { + $this->provider = $provider; + $this->keep_cache_time = $keep_cache_time; + } + + function get() : array + { + if($this->cached === null || (time() - $this->last_cache_time) > $this->keep_cache_time) + { + $this->cached = $this->provider->get(); + $this->last_cache_time = time(); + } + return $this->cached; + } +} + diff --git a/im.inc.php b/im.inc.php new file mode 100644 index 0000000..f68b6c3 --- /dev/null +++ b/im.inc.php @@ -0,0 +1,183 @@ + true, 'id' => 1); + } + + function updatePost(string $id, array $fields = array()) : array + { + return array('ok' => true); + } + + function postPNG(string $title, string $png_file, array $fields = array()) : array + { + return array('ok' => true); + } + + function getPermalink(string $id) : string + { + return "https://fake/$id"; + } +} + +class MattermostChan implements IMChan +{ + private string $server; + private string $token; + private string $team; + private string $chan; + + function __construct(string $server, string $token, string $team, string $chan) + { + $this->server = $server; + $this->token = $token; + $this->team = $team; + $this->chan = $chan; + } + + function post(string $message, array $fields = array()) : array + { + return mm_post($this->server, $this->token, $this->chan, $message, $fields); + } + + function updatePost(string $id, array $fields = array()) : array + { + return mm_update($this->server, $this->token, $id, $fields); + } + + function postPNG(string $title, string $png_file, array $fields = array()) : array + { + return mm_post_png($this->server, $this->token, $this->chan, $title, $png_file, $fields); + } + + function getPermalink(string $id) : string + { + return mm_get_permalink($this->server, $this->team, $id); + } +} + +function mm_post(string $server, string $token, string $chan, string $message, $fields = array()) +{ + $ch = _atf_mm_post_curl($server, $token, $chan, $message, $fields); + $result = curl_exec($ch); + curl_close($ch); + + $json = json_decode($result, true); + if(!$json) + return array('ok' => false, 'id' => 0); + return array('ok' => true, 'id' => $json['id']); +} + +function mm_post_async(string $server, string $token, string $chan, string $message, $fields = array()) : Amp\Promise +{ + return Amp\call(function() use($server, $token, $chan, $message, $fields) { + + $ch = _atf_mm_post_curl($server, $token, $chan, $message, $fields); + $result = yield _atf_multi_curl_async($ch, true); + curl_close($ch); + + $json = json_decode($result, true); + if(!$json) + return array('ok' => false, 'id' => 0); + return array('ok' => true, 'id' => $json['id']); + }); +} + +function mm_update(string $server, string $token, string $id, array $fields) +{ + $ch = curl_init("$server/api/v4/posts/$id"); + + $headers = array( + 'Content-Type: application/json', + 'Authorization: Bearer ' . $token + ); + + $fields["post_id"] = $id; + $fields["id"] = $id; + + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT'); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($fields)); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + + $result = curl_exec($ch); + curl_close($ch); + + return json_decode($result, true); +} + +function mm_post_png(string $server, string $token, string $chan, string $title, string $png_file, $msg_fields = array()) +{ + $headers = array( + 'Content-Type: multipart/form-data', + 'Authorization: Bearer ' . $token + ); + + $file = new \CurlFile(realpath($png_file), 'image/png', basename($png_file)); + + $fields = array(); + $fields['channel_id'] = $chan; + $fields['files'] = $file; + + $ch = curl_init("$server/api/v4/files"); + curl_setopt($ch, CURLOPT_POST, 1); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_POSTFIELDS, $fields); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + + $result = curl_exec($ch); + $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if($status != 201) + throw new Exception("File failed with status: " . $status .", result: " . $result); + + $result = json_decode($result, true); + $file_id = $result['file_infos'][0]['id']; + + $msg_fields['file_ids'] = array($file_id); + + return mm_post($server, $token, $chan, $title, $msg_fields); +} + +function mm_get_permalink(string $server, string $team, string $msg_id) +{ + return "$server/$team/pl/$msg_id"; +} + +function _atf_mm_post_curl(string $server, string $token, string $chan, string $message, $fields = array()) +{ + $ch = curl_init("$server/api/v4/posts"); + + $headers = array( + 'Content-Type: application/json', + 'Authorization: Bearer ' . $token + ); + + if(!isset($fields['channel_id'])) + $fields['channel_id'] = $chan; + $fields['message'] = $message; + + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($fields)); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + + return $ch; +} diff --git a/plan.inc.php b/plan.inc.php new file mode 100644 index 0000000..07ec4f6 --- /dev/null +++ b/plan.inc.php @@ -0,0 +1,706 @@ +name = $name; + } + + function getTitle() : string + { + $issues_summary = ''; + foreach($this->getProblemsHistogram() as $code => $count) + { + if($code == Task::CODE_GONE) + $issues_summary .= " gones:$count"; + else if($code == Task::CODE_NSTART) + $issues_summary .= " nstrts:$count"; + else if($code == Task::CODE_STUCK) + $issues_summary .= " stucks:$count"; + else if($code == Task::CODE_HUNG) + $issues_summary .= " hungs:$count"; + else if($code == Task::CODE_EXCEPTION) + $issues_summary .= " excepts:$count"; + else if($code == Task::CODE_WARN) + $issues_summary .= " warns:$count"; + } + + $running_count = $this->countRunningTasks(); + $over_count = $this->countFinishedTasks(); + + $predicted_total_duration = $this->getPredictedTotalDuration(); + + $progress_txt = ' tasks:' . ($running_count > 0 ? $running_count . '|' : '') . $over_count . '/' . sizeof($this->tasks) . + ' (' . round($this->getProgress()*100, 2) . '%) (' . + round($this->getDuration()/60, 1) . '/' . ($predicted_total_duration > 0 ? round($predicted_total_duration/60, 1) : '?') . + ' min)'; + + return "({$this->session->id}) Plan '{$this->name}'" . $issues_summary . $progress_txt; + } + + function getDuration() : int + { + return time() - $this->start_time; + } + + function getPredictedTotalDuration() + { + if($this->getDuration() <= 0) + return 0; + + if($this->getProgress() <= 0) + return 0; + + return $this->getDuration() / $this->getProgress(); + } + + function _createMessengerThread() + { + if($this->im_thread_id) + return; + + for($i=0;$i<5;++$i) + { + $resp = $this->session->conf->im->post($this->getTitle(), $this->_getMessengerThreadProps()); + if(isset($resp['ok'])) + { + $this->im_thread_id = $resp['id']; + return; + } + sleep(1); + } + throw new Exception("Could not create IM thread"); + } + + function _updateMessengerThread(bool $force) + { + if(!$force && (time() - $this->im_thread_last_update) < self::IM_THREAD_UPDATE_INTERVAL) + return; + $this->im_thread_last_update = time(); + + if(!$this->im_thread_id) + return; + + $this->session->conf->im->updatePost($this->im_thread_id, $this->_getMessengerThreadProps()); + } + + function _getMessengerThreadProps() : array + { + return ['props' => ['attachments' => [ + [ + 'text' => '* ' . $this->getTitle(), + 'color' => $this->_getThreadColor() + ] + ]]]; + } + + function addStatusCode($code, $msg = '') + { + $this->status_codes[] = array($code, $msg); + } + + function getProblemsHistogram() + { + $histogram = array(); + foreach($this->status_codes as $item) + { + list($code, $_) = $item; + if(!Task::isProblemCode($code)) + continue; + + if(!isset($histogram[$code])) + $histogram[$code] = 0; + ++$histogram[$code]; + } + return $histogram; + } + + function hasStatusCode($code) : bool + { + return $this->countStatusCode($code) > 0; + } + + function countStatusCode($code) : int + { + $c = 0; + foreach($this->status_codes as $item) + { + if($item[0] === $code) + $c++; + } + return $c; + } + + function getStats() : Stats + { + $stats = new Stats(); + + $stats->gones = $this->countStatusCode(Task::CODE_GONE); + $stats->hungs = $this->countStatusCode(Task::CODE_HUNG); + $stats->stucks = $this->countStatusCode(Task::CODE_STUCK); + $stats->nostarts = $this->countStatusCode(Task::CODE_NSTART); + $stats->excepts = $this->countStatusCode(Task::CODE_EXCEPTION); + $stats->warns = $this->countStatusCode(Task::CODE_WARN); + + return $stats; + } + + function _getThreadColor() : string + { + if(!$this->session->getDevices()) + return "#000000"; + + $stats = $this->getStats(); + return $stats->getColor($this->isOver()); + } + + function addTask(Task $task) + { + $task->plan = $this; + $this->tasks[] = $task; + } + + function isOver() : bool + { + if($this->hasTooManyFatalProblems()) + return true; + + foreach($this->tasks as $task) + { + if(!$task->isDone()) + return false; + } + return true; + } + + function hasTooManyFatalProblems() : bool + { + return + ($this->countStatusCode(Task::CODE_EXCEPTION) >= self::EXCEPTIONS_THRESHOLD) || + ($this->countStatusCode(Task::CODE_STUCK) >= self::STUCK_THRESHOLD) || + ($this->countStatusCode(Task::CODE_GONE) >= self::GONE_THRESHOLD) || + ($this->countStatusCode(Task::CODE_HUNG) >= self::HUNG_THRESHOLD) + ; + } + + function getProgress() : float + { + $total_progress = 0; + foreach($this->tasks as $task) + $total_progress += $task->getProgress(); + return $total_progress / sizeof($this->tasks); + } + + function countRunningTasks() : int + { + $running_count = 0; + foreach($this->tasks as $task) + { + if($task->device) + $running_count++; + } + return $running_count; + } + + function countFinishedTasks() : int + { + $done_count = 0; + foreach($this->tasks as $task) + { + if($task->isDone()) + $done_count++; + } + return $done_count; + } + + function runTaskAsync(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + + yield $this->_startAsync($task); + + while(!$this->isOver()) + { + $this->_updateMessengerThread(false); + + yield Amp\delay((int)($this->session->conf->sleep_time*1000)); + + log("Task #{$task->run_id} {$task->cmd}(".sizeof($task->args).") progress {$task->getProgress()}, last ext time {$task->last_ext_status_item_time} *{$task->device}*..."); + + $check_error = yield $this->_tryCheckExtStatusAsync($task, /*attempts*/3, /*timeout*/20); + if($check_error !== null) + { + //something wrong with device, let's ignore the device and reschedule the task + $this->_ignoreDevice($task->device, $check_error); + $task->reschedule(); + break; + } + + if(!$task->isDone()) + yield $this->_checkHealthAsync($task); + + if($task->hasFatalProblem()) + { + yield $this->_processFatalProblemAsync($task); + break; + } + else if($task->isDone()) + { + //let's reset bogus stats for this device + $this->session->resetBogusDeviceCount($task->device); + //let's free the device + $task->device = null; + break; + } + } + + $this->_updateMessengerThread(true); + }); + } + + function _onStart() + { + $this->start_time = time(); + + log("Starting " . $this->getTitle() . "..."); + + $this->_createMessengerThread(); + } + + function _startAsync(Task $task) : Amp\Promise + { + //let's create a Slack thread synchronously if it's not present yet + if(!$this->im_thread_id) + $this->_onStart(); + + return Amp\call(function() use($task) { + + try + { + log("Preparing *{$task->device}*"); + + yield $this->session->apk_installer->installAsync($task->device); + yield start_ext_cmd_on_device_async( + $this->session->conf->atf_host, + $task->device, + $task->getCmd(), + $task->getCmdArgs() + ); + } + catch(Exception $e) + { + $this->_ignoreDevice($task->device, $e->getMessage()); + $task->reschedule(); + return false; + } + + $task->start(); + return true; + + }); + } + + function _ignoreDevice($device, $reason) + { + $this->session->ignoreDevice($device, $reason); + + $this->post("Ignoring device *$device*: $reason"); + } + + function _processFatalProblemAsync(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + + $fatal_msg = "Fatal problem ({$task->getLastFatalProblem()} - ".Task::code2string($task->getLastFatalProblem())."), attempt:{$task->attempts} *{$task->device}*"; + + err("[FTL] $fatal_msg"); + + $this->post($fatal_msg); + + $this->_postScreenToMessenger($task); + + if($task->getLastFatalProblem() == Task::CODE_EXCEPTION) + yield $this->_postLastReplayToMessengerAsync($task); + else if($task->getLastFatalProblem() == Task::CODE_GONE || + $task->getLastFatalProblem() == Task::CODE_HUNG) + { + $app_log = get_logcat_unity($this->session->conf->atf_host, $task->device, 300); + $app_log = _atf_trim_start($app_log, 2000); + $this->post("Last logs: \n```\n$app_log\n```\n *{$task->device}*"); + } + + if($this->session->conf->error_pause) + $this->_ignoreDevice($task->device, "Ignored for error investigation"); + //Let's reinstall application on this device if it's failing too often + else + $this->_incAndCheckBogusDevice($task->device); + + $task->onFatalProblem(); + + //let's free the device anyway, it's up to Task::onFatalProblem() to reschedule it or not + $task->device = null; + }); + } + + function _tryCheckExtStatusAsync(Task $task, int $attempts, int $timeout) : Amp\Promise + { + return Amp\call(function() use($task, $attempts, $timeout) { + + $last_error = null; + + for($i=0;$i<$attempts;++$i) + { + try + { + yield $this->_checkExtStatusAsync($task, $timeout); + break; + } + catch(Exception $e) + { + $last_error = $e->getMessage(); + continue; + } + } + + return $last_error; + + }); + } + + function _checkExtStatusAsync(Task $task, int $timeout) : Amp\Promise + { + return Amp\call(function() use($task, $timeout) { + + $ext_status = yield get_ext_status_async($this->session->conf->atf_host, $task->device, $timeout); + if(!is_array($ext_status)) + return; + + $new_items = $this->_getExtStatusItemsSince($task->last_ext_status_item_time, $ext_status); + + if(!$new_items) + return; + + $task->last_ext_status_item_time = end($new_items)['time']; + + foreach($new_items as $item) + yield $this->_analyzeExtStatusItemAsync($task, $item); + + }); + } + + function _getExtInputRequestsSince($id, array $all_reqs) : array + { + $new_reqs = array(); + foreach($all_reqs as $req) + { + if($req['id'] > $id) + $new_reqs[] = $req; + } + return $new_reqs; + } + + function _getExtStatusItemsSince($time, array $ext_status) : array + { + $new_items = array(); + foreach($ext_status['entries'] as $item) + { + if($item['time'] > $time) + $new_items[] = $item; + } + return $new_items; + } + + function _checkIfAppRunning(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + list($status, $_) = + yield host_exec_async($this->session->conf->atf_host, "%{adb}% -s {$task->device} shell pidof %{package_id}%", DEPLOY_OPT_ERR_OK); + return $status == 0; + }); + } + + function _reportGone(Task $task) + { + //check it application hasn't even started + if($task->last_alive_check_time === $task->reset_time) + { + $task->addStatusCode(Task::CODE_NSTART); + $this->session->trySendStatsEvent($task, 'nstart'); + err("[NFD] No app started after {$this->session->conf->dead_threshold} seconds *{$task->device}*"); + } + else + { + $task->addStatusCode(Task::CODE_GONE); + $this->session->trySendStatsEvent($task, 'gone'); + err("[GNE] App is gone after {$this->session->conf->dead_threshold} seconds *{$task->device}*"); + } + } + + function _checkHealthAsync(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + + $not_alive_time = microtime(true) - $task->last_alive_check_time; + $stuck_time = microtime(true) - $task->last_stuck_check_time; + + if($not_alive_time > $this->session->conf->dead_threshold) + { + if(yield $this->_checkIfAppRunning($task)) + { + $task->addStatusCode(Task::CODE_HUNG); + $this->session->trySendStatsEvent($task, 'hung'); + err("[HNG] No activity for {$this->session->conf->dead_threshold} seconds *{$task->device}*"); + } + else + $this->_reportGone($task); + + $this->_reportErrorFromLogcatToMessenger($task, 1000); + } + else if($stuck_time > $this->session->conf->stuck_threshold) + { + if(yield $this->_checkIfAppRunning($task)) + { + $task->addStatusCode(Task::CODE_STUCK); + $this->session->trySendStatsEvent($task, 'stuck'); + err("[STK] Stuck for {$this->session->conf->stuck_threshold} seconds *{$task->device}*"); + } + else + $this->_reportGone($task); + } + + }); + } + + function _incAndCheckBogusDevice(string $device) + { + $count = $this->session->incBogusDevice($device); + + if($count > self::BOGUS_DEVICE_COUNT_THRESHOLD) + { + $this->_ignoreDevice($device, "Too many bogus errors"); + } + else if(($count % self::BOGUS_REINSTALL_APP_EVERY_N) == 0) + { + $this->session->apk_installer->forgetInstall($device); + } + } + + static function _parseExtMessage(string $message) + { + $msg_code = null; + $msg_text = $message; + //example: [DBG] this a debug message + if(preg_match('~^(\[[^\]]+\])(.*)$~', $message, $matches)) + { + $msg_code = $matches[1]; + $msg_text = $matches[2]; + } + return array($msg_code, $msg_text); + } + + static function _printToShellExtItem(Task $task, array $item) + { + $shell_msg = _atf_trim($item['message'], 200); + if(Task::isProblemCode($item['error'])) + $shell_msg = "[PRB] Code:{$item['error']}, $shell_msg"; + $shell_msg = "(".round($item['time'],1)."s) {$shell_msg} *{$task->device}*"; + + log($shell_msg); + } + + function _postToMessengerExtStatusItem(Task $task, array $item) + { + $orig_msg = _atf_trim($item['message'], 3000); + $mm_msg = $orig_msg; + + if($item['error'] == Task::CODE_EXCEPTION) + $mm_msg = "\n```\n$mm_msg\n```\n"; + + $mm_msg = '('.round($item['time'],1).'s) '.$mm_msg.' *'.$task->device.'*'; + + $resp = $this->post($mm_msg); + + if(isset($resp['ok']) && $item['error'] == Task::CODE_EXCEPTION) + $this->session->tryShareToQAChannel($resp['id'], $orig_msg); + } + + function post($msg, array $args = array()) + { + $args['root_id'] = $this->im_thread_id; + return $this->session->conf->im->post($msg, $args); + } + + function _analyzeExtStatusItemAsync(Task $task, array $item) : Amp\Promise + { + return Amp\call(function() use($task, $item) { + + self::_printToShellExtItem($task, $item); + + $task->addStatusCode($item['error'], $item['message']); + + if($item['error'] == Task::CODE_EXCEPTION) + $this->session->trySendStatsEvent($task, 'exception'); + + list($msg_type, $msg_text) = self::_parseExtMessage($item['message']); + + //NOTE: alive system messages are sent periodically by ExtBot itself, + // we'd like to ignore them for stuck detection + if($msg_type !== '[ALIVE]') + $task->last_stuck_check_time = microtime(true); + + //NOTE: in case of any message from the device we update the alive check timestamp + $task->last_alive_check_time = microtime(true); + + yield $this->_processExtStatusMessageAsync($task, $item, $msg_type, $msg_text); + + }); + } + + function _processExtStatusMessageAsync($task, $item, $msg_type, $msg_text) : Amp\Promise + { + return Amp\call(function() use($task, $item, $msg_type, $msg_text) { + + if($item['error'] == Task::CODE_EXCEPTION) + { + $this->_postToMessengerExtStatusItem($task, $item); + } + else if($msg_type === '[WRN]') + { + $task->addStatusCode(Task::CODE_WARN); + $this->_postToMessengerExtStatusItem($task, $item); + } + else if($msg_type === '[PRG]') + { + //let's reset hung stats since progress is going + $this->session->resetBogusDeviceCount($task->device); + $task->onProgress($msg_text); + } + else if($msg_type === '[STAT]') + yield $this->session->trySendStatsFromJzonAsync($task, $msg_text); + else if($msg_type === '[RPL]') + $this->_postReplayToMessenger($task, $msg_text); + + }); + } + + function _postLastReplayToMessengerAsync(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + + if($this->session->conf->device_replay_path) + { + list($repl_err, $repl_txt) = + yield get_last_replay_async($this->session->conf->atf_host, $task->device, $this->session->conf->device_replay_path); + if($repl_err === 0) + $this->_postReplayToMessenger($task, $repl_txt); + } + + }); + } + + function _postReplayToMessenger(Task $task, $repl_txt) + { + $repl_txt = _atf_try_lz4_replay($repl_txt); + $this->post("Last Replay: \n```\n$repl_txt\n```\n *{$task->device}*"); + } + + function _reportErrorFromLogcatToMessenger(Task $task, $limit) + { + $errors_log = _atf_trim(get_logcat_errors($task->device, $limit), 3950); + if($errors_log) + $this->post("\n```\n$errors_log\n```\n *{$task->device}*"); + } + + function _postScreenToMessenger(Task $task) + { + global $GAME_ROOT; + + $png_data = device_screen($this->session->conf->atf_host, $task->device); + if($png_data) + { + $tmp_png = tempnam(sys_get_temp_dir(), 'device_screen') . '.png'; + taskman\ensure_write($tmp_png, $png_data); + try + { + $this->session->conf->im->postPNG("screen: ".$task->device, $tmp_png, array('root_id' => $this->im_thread_id)); + } + catch(Exception $e) + {} + finally + { + taskman\ensure_rm($tmp_png); + } + } + } +} + +class Stats +{ + public int $gones = 0; + public int $hungs = 0; + public int $stucks = 0; + public int $nostarts = 0; + public int $excepts = 0; + public int $warns = 0; + + function add(Stats $other) + { + $this->gones += $other->gones; + $this->hungs += $other->hungs; + $this->stucks += $other->stucks; + $this->nostarts += $other->nostarts; + $this->excepts += $other->excepts; + $this->warns += $other->warns; + } + + function getColor(bool $is_over) : string + { + if($this->gones > 0 || + $this->hungs > 0 || + $this->warns > 0 || + $this->excepts > 0 || + $this->stucks > 0 || + $this->nostarts > 0 + ) + { + if($this->excepts) + return $is_over ? "danger" : "#FFCCCB"; + else + return $is_over ? "warning" : "#FFCF9E"; + } + else if($is_over) + return "good"; + else + return "#D3D3D3"; + } +} + + diff --git a/session.inc.php b/session.inc.php new file mode 100644 index 0000000..f110347 --- /dev/null +++ b/session.inc.php @@ -0,0 +1,492 @@ +im = new NullMessenger(); + $this->im_qa = new NullMessenger(); + + $this->stats = new NullStatsSender(); + } + + function initHost(array $settings) + { + $this->atf_host = 'atf'; + taskman\deploy_declare_node($this->atf_host, $settings); + } +} + +class Session +{ + const RUN_SLEEP = 3; + + public string $name; + public SessionConfig $conf; + + public int $start_time; + + public IDevicePool $device_pool; + public ApkInstaller $apk_installer; + + public string $id; + public string $guid; + + public string $im_thread_id = ''; + + public array $plans = array(); + + public array $ignored_devices = array(); + public array $bogus_device2counter = array(); + + public array $shared_qa_errors = array(); + + static function getId() : string + { + $id = getenv('ATF_SESSION_ID'); + if($id) + return $id; + //sort of graceful fallback + return ''.getmypid(); + } + + function __construct( + string $name, + SessionConfig $conf, + IDevicePool $device_pool + ) + { + $this->id = self::getId(); + + $this->name = $name; + $this->conf = $conf; + + $this->guid = uniqid(); + + $this->device_pool = $device_pool; + } + + function getTitle() : string + { + return "Session ({$this->id}) '{$this->name}' (devices:".sizeof($this->getDevices())." ?:".sizeof($this->ignored_devices) . + ') (' . round($this->getDuration()/60, 1) . ' min)'; + } + + function getDuration() : int + { + return time() - $this->start_time; + } + + function getDevices() : array + { + $devices = array_diff( + $this->device_pool->get(), + $this->ignored_devices + ); + return $devices; + } + + function ignoreDevice(string $device, string $reason) + { + err("Ignoring device *$device*: $reason"); + + $this->ignored_devices[] = $device; + } + + function incBogusDevice(string $device) + { + if(!isset($this->bogus_device2counter[$device])) + $this->bogus_device2counter[$device] = 0; + return ++$this->bogus_device2counter[$device]; + } + + function resetBogusDeviceCount(string $device) + { + $this->bogus_device2counter[$device] = 0; + } + + function resetBogusDevices() + { + $this->bogus_device2counter = array(); + } + + function addPlan(Plan $plan) + { + $plan->session = $this; + $this->plans[] = $plan; + } + + function resetPlans() + { + $this->plans = array(); + } + + function runAsync() : Amp\Promise + { + return Amp\call(function() { + + $this->start_time = time(); + + $apk_name = !$this->conf->apk_path ? null : basename($this->conf->apk_path); + $this->apk_installer = new ApkInstaller( + $this->conf->atf_host, + $this->conf->max_adb_installs_in_progress, + $apk_name, + $this->conf->override_external_files, + $this->conf->external_files + ); + + $this->_createMessengerThread(); + + //if it's not set the last one will be re-used + if($this->conf->apk_path) + $this->_deployApkToHost($apk_name); + + if($this->conf->reboot_devices) + reboot_devices($this->conf->atf_host, $this->getDevices()); + + if($this->conf->adb_reboot) + adb_reboot($this->conf->atf_host); + + $this->trySendApkStatsEvent($this->apk_installer->getSize()); + + $all_tasks = $this->_getAllTasks(); + + return yield $this->_runTasksAsync($all_tasks); + + }); + } + + function _createMessengerThread() + { + for($i=0;$i<5;++$i) + { + $resp = $this->conf->im->post("", $this->_getMessengerThreadProps()); + if(isset($resp['ok'])) + { + $this->im_thread_id = $resp['id']; + return; + } + sleep(1); + } + throw new Exception("Could not create IM thread"); + } + + function _updateMessengerThread() + { + $this->conf->im->updatePost( + $this->im_thread_id, + $this->_getMessengerThreadProps() + ); + } + + function _getMessengerThreadProps() : array + { + return ['props' => ['attachments' => [ + [ + 'text' => $this->getTitle(), + 'color' => $this->_getThreadColor() + ] + ]]]; + } + + function _getThreadColor() : string + { + if(!$this->getDevices()) + return "#000000"; + + $stats = new Stats(); + foreach($this->plans as $plan) + $stats->add($plan->getStats()); + + return $stats->getColor($this->isOver()); + } + + function _deployApkToHost($apk_name) + { + if(!is_file($this->conf->apk_path)) + throw new Exception("No such file '{$this->conf->apk_path}'"); + + host_put_file($this->conf->atf_host, $this->conf->apk_path, "%{dir}%/$apk_name"); + } + + function _getAllTasks() + { + $all_tasks = array(); + foreach($this->plans as $plan) + { + foreach($plan->tasks as $task) + $all_tasks[] = $task; + } + return $all_tasks; + } + + function _runTasksAsync(array $all_tasks) : Amp\Promise + { + return Amp\call(function() use($all_tasks) { + + $active_tasks = array(); + + while(!$this->isOver() && (sizeof($all_tasks) > 0 || sizeof($active_tasks) > 0)) + { + $free_devices = $this->_findFreeDevices($active_tasks); + + $this->_assignTasksToDevices($all_tasks, $free_devices, $active_tasks); + + yield Amp\delay(self::RUN_SLEEP*1000); + + $this->_processDoneTasks($all_tasks, $active_tasks); + + $this->_updateMessengerThread(); + } + + return sizeof($all_tasks) == 0 && sizeof($active_tasks) == 0; + + }); + } + + function _assignTasksToDevices(array &$tasks, array $devices, array &$active_tasks) + { + static $next_run_id = 0; + + //create new coroutines and assign them to free devices + foreach($devices as $device) + { + if(sizeof($tasks) == 0) + break; + + $task = array_shift($tasks); + + //explicitly assigning a device + $task->device = $device; + + $task->run_id = ++$next_run_id; + $task->run_status = false; + + $active_tasks[$task->run_id] = $task; + + $task->plan->runTaskAsync($task)->onResolve( + function($error, $value) use($task) + { + $task->run_status = true; + $task->run_error = $error; + } + ); + } + } + + function _processDoneTasks(array &$tasks, array &$active_tasks) + { + $tmp_tasks = array(); + foreach($active_tasks as $run_id => $task) + { + //task not in done ids, let's keep it + if(!$task->run_status) + { + $tmp_tasks[$run_id] = $task; + } + else if($task->run_error) + { + throw $task->run_error; + } + //task is done, let's check if it had a problem and if yes + //let's put it back into unscheduled tasks + else if($task->need_reschedule) + { + array_unshift($tasks, $task); + $task->need_reschedule = false; + } + } + + $active_tasks = $tmp_tasks; + } + + function _findFreeDevices(array $active_tasks) + { + $free = array(); + + foreach($this->getDevices() as $device) + { + //let's skip invalid devices + if(!$device) + continue; + + $busy = false; + foreach($active_tasks as $task) + { + if($task->device == $device) + { + $busy = true; + break; + } + } + if(!$busy) + $free[] = $device; + } + + return $free; + } + + function isOver() : bool + { + if(!$this->getDevices()) + return true; + + foreach($this->plans as $plan) + if(!$plan->isOver()) + return false; + + return true; + } + + function trySendStatsFromJzonAsync(Task $task, string $jzon) : Amp\Promise + { + return Amp\call(function() use($task, $jzon) { + + try + { + $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); + $table = $data['table']; + unset($data['table']); + + $data = yield $this->inflateMissingStats($task, $table, $data); + + $this->trySendStats($task, $table, $data); + } + catch(Exception $e) + { + echo $e->getMessage() . "\n"; + } + + }); + } + + //TODO: make it more flexible via plugin alike system? + function inflateMissingStats(Task $task, string $table, array $data) : Amp\Promise + { + return Amp\call(function() use($task, $table, $data) { + //NOTE: filling memory usage if it's not set + if(isset($data['deviceMemoryUsage']) && $data['deviceMemoryUsage'] === '') + { + $mem = yield device_mem_async($this->conf->atf_host, $task->device); + $data['deviceMemoryUsage'] = $mem['total']; + if($table === 'device_memory') + { + $data['deviceMemoryUsageNative'] = $mem['native']; + $data['deviceMemoryUsageSystem'] = $mem['system']; + $data['deviceMemoryUsageJava'] = $mem['java']; + $data['deviceMemoryUsageGraphics'] = $mem['graphics']; + } + } + + //NOTE: filling CPU temperature + if(isset($data['event']) && $data['event'] == 'cpuTemp') + $data['value'] = yield device_temperature_async($this->conf->atf_host, $task->device); + + return $data; + }); + } + + function trySendApkStatsEvent(int $apk_size) + { + try + { + $data['guid'] = $this->guid; + $data['time'] = time(); + $data['version'] = $this->conf->proj_version; + $data['revHash'] = $this->conf->proj_rev_hash; + $data['branch'] = $this->conf->proj_branch; + $data['event'] = 'apk_size'; + $data['value'] = $apk_size; + $this->conf->stats->send('event_value', $data); + } + catch(Exception $e) + { + echo $e->getMessage() . "\n"; + } + } + + function trySendStats(Task $task, string $table, array $data) + { + try + { + $data['guid'] = $this->guid; + $data['time'] = time(); + $data['deviceId'] = $task->device; + $this->conf->stats->send($table, $data); + } + catch(Exception $e) + { + echo $e->getMessage() . "\n"; + } + } + + function trySendStatsEvent(Task $task, string $event, int $value = 1) + { + $this->trySendStats( + $task, + 'event_value', + array( + 'event' => $event, + 'value' => $value, + 'branch' => $this->conf->proj_branch, + 'version' => $this->conf->proj_version, + 'revHash' => $this->conf->proj_rev_hash + ) + ); + } + + function tryShareToQAChannel($msg_id, $error) + { + if(!$this->conf->share_with_qa_chan) + return; + + //let's skip similar already shared errors + if($this->_calcSharedErrorSimilarity($error) > 80) + return; + + $this->shared_qa_errors[] = $error; + + //let's share the exception message to the regency-qa channel + $permalink = $this->conf->im->getPermalink($msg_id); + $this->conf->im_qa->post($permalink); + } + + function _calcSharedErrorSimilarity($error) + { + $max_perc = 0; + foreach($this->shared_qa_errors as $shared) + { + $perc = 0; + similar_text($shared, $error, $perc); + if($perc > $max_perc) + $max_perc = $perc; + } + return $max_perc; + } +} diff --git a/slack.inc.php b/slack.inc.php deleted file mode 100644 index e027d42..0000000 --- a/slack.inc.php +++ /dev/null @@ -1,180 +0,0 @@ - $token, - 'channels' => implode(",", $channels), - 'file' => $cfile, - 'filename' => "test.jpg", - ); - - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HEADER, false); - curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, $fields); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_VERBOSE, true); - $result = curl_exec($ch); - curl_close($ch); - - $arr = json_decode($result, true); - if(isset($arr['ok']) && $arr['ok'] == 1) - { - if(isset($arr['file']['permalink_public'])) - return $arr['file']['permalink_public']; - } - - return ""; -} - -function _atf_slack_chan() -{ - return get("ATF_SLACK_CHANNEL"); -} - -function _atf_slack_chan_qa() -{ - return get("ATF_SLACK_CHANNEL"); -} - -function _atf_slack_token() -{ - $token = "xoxb-141599046048-1567053090644-2gDAoWGiZxFTtXGLMOOeLG2u"; - return $token; -} - -task('slack_test', function() -{ - // createSlackThread(); - // atf_slack_post('hi', array('channel' => _atf_slack_chan_qa())); -}); - -function atf_slack_post($message, $fields = array()) -{ - if(!get("ATF_SLACK")) - return array('ok' => true, 'ts' => 0); - - $ch = curl_init("https://slack.com/api/chat.postMessage"); - $fields["token"] = _atf_slack_token(); - if(!isset($fields['channel'])) - $fields["channel"] = _atf_slack_chan(); - $fields["text"] = $message; - - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); - curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields)); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - - $result = curl_exec($ch); - curl_close($ch); - - return json_decode($result, true); -} - -function atf_slack_get_permalink($message_ts, $channel = null) -{ - if(!get("ATF_SLACK")) - return array('permalink' => ''); - - $ch = curl_init("https://slack.com/api/chat.getPermalink"); - $fields["token"] = _atf_slack_token(); - if($channel === null) - $channel = _atf_slack_chan(); - $fields["channel"] = $channel; - $fields["message_ts"] = $message_ts; - - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); - curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields)); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - - $result = curl_exec($ch); - curl_close($ch); - - return json_decode($result, true); -} - -function atf_slack_update($ts, array $attachments) -{ - if(!get("ATF_SLACK")) - return array('ok' => true); - - $ch = curl_init("https://slack.com/api/chat.update"); - $fields["token"] = _atf_slack_token(); - $fields["channel"] = _atf_slack_chan(); - $fields["ts"] = $ts; - $fields["attachments"] = json_encode($attachments); - - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); - curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields)); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - - $result = curl_exec($ch); - curl_close($ch); - - return json_decode($result, true); -} - - -function atf_slack_post_png($title, $png_file, $fields = array()) -{ - if(!get("ATF_SLACK")) - return array('ok' => true); - - $file = new CurlFile($png_file, 'image/png'); - - $header = array(); - $header[] = 'Content-Type: multipart/form-data'; - - $fields["token"] = _atf_slack_token(); - $fields["channels"] = _atf_slack_chan(); - $fields['file'] = $file; - $fields['title'] = $title; - - $ch = curl_init("https://slack.com/api/files.upload"); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_HTTPHEADER, $header); - curl_setopt($ch, CURLOPT_POST, 1); - curl_setopt($ch, CURLOPT_POSTFIELDS, $fields); - - $result = curl_exec($ch); - curl_close($ch); - - return json_decode($result, true); -} \ No newline at end of file diff --git a/stats.inc.php b/stats.inc.php new file mode 100644 index 0000000..e8751ea --- /dev/null +++ b/stats.inc.php @@ -0,0 +1,56 @@ +host = $host; + $this->port = $port; + $this->username = $username; + $this->password = $password; + $this->dbname = $dbname; + } + + function getClient() : ClickHouseDB\Client + { + $config = [ + 'host' => $this->host, + 'port' => $this->port, + 'username' => $this->username, + 'password' => $this->password, + ]; + + $db = new ClickHouseDB\Client($config); + if($this->dbname) + $db->database($this->dbname); + $db->setTimeout(15); + $db->setConnectTimeOut(10); + return $db; + } + + function send(string $table, array $data) + { + $this->getClient()->insert($table, array(array_values($data)), array_keys($data)); + } +} + diff --git a/task.inc.php b/task.inc.php new file mode 100644 index 0000000..28896ce --- /dev/null +++ b/task.inc.php @@ -0,0 +1,308 @@ +module = $module; + $this->func = $func; + } + + function __toString() + { + return $this->module.':'.$this->func; + } +} + +class Task +{ + const CODE_MESSAGE = 0; + const CODE_EXCEPTION = 1; + const CODE_WARN = 2; + const CODE_NSTART = 124; + const CODE_STUCK = 125; + const CODE_GONE = 126; + const CODE_HUNG = 127; + const CODE_DONE = 128; + + static function code2string(int $code) : string + { + switch($code) + { + case self::CODE_MESSAGE: + return "message"; + case self::CODE_EXCEPTION: + return "exception"; + case self::CODE_WARN: + return "warning"; + case self::CODE_STUCK: + return "stuck"; + case self::CODE_NSTART: + return "nstart"; + case self::CODE_GONE: + return "gone"; + case self::CODE_HUNG: + return "hung"; + case self::CODE_DONE: + return "done"; + } + return "???"; + } + + public ?Plan $plan = null; + + //async execution id + public int $run_id = 0; + //'true' if async coroutine is resolved + public bool $run_status = false; + //async execution exception if any + public ?object $run_error = null; + + //attached device + public ?string $device = null; + + public int $attempts = 1; + //signal to dispatcher to re-start execution of the task + public bool $need_reschedule = false; + + public Cmd $cmd; + public array $args = array(); + + public string $title = ''; + + public float $start_time = 0; + public float $reset_time = 0; + + public float $last_progress = 0; + public int $last_done_arg_idx = -1; + public float $last_alive_check_time = 0; + public float $last_stuck_check_time = 1; + public float $last_ext_status_item_time = 0; + //array of tuples [code, msg] + public array $status_codes = array(); + + function __construct(Cmd $cmd, array $args, $title = '') + { + $this->cmd = $cmd; + $this->args = $args; + $this->title = $title; + } + + function getCmd() : Cmd + { + return $this->cmd; + } + + function getCmdArgs() : array + { + return $this->args; + } + + function getProgress() : float + { + if($this->last_progress == 0 && $this->isDone()) + return 1; + return $this->last_progress; + } + + static function isProblemCode($code) : bool + { + return $code > self::CODE_MESSAGE && $code < self::CODE_DONE; + } + + static function isFatalCode($code) : bool + { + return + $code == Task::CODE_STUCK || + $code == Task::CODE_NSTART || + $code == Task::CODE_GONE || + $code == Task::CODE_HUNG || + $code == Task::CODE_EXCEPTION; + } + + function hasFatalProblem() : bool + { + return $this->getLastFatalProblem() != 0; + } + + function getLastFatalProblem() : int + { + for($i=sizeof($this->status_codes);$i-- > 0;) + { + $item = $this->status_codes[$i]; + if(self::isFatalCode($item[0])) + return $item[0]; + } + return 0; + } + + function hasStatusCode($code) + { + foreach($this->status_codes as $item) + { + if($item[0] === $code) + return true; + } + return false; + } + + function isDone() + { + return $this->hasStatusCode(Task::CODE_DONE); + } + + function addStatusCode($code, $msg = '') + { + //plan stores history of all status codes + $this->plan->addStatusCode($code, $msg); + + $this->status_codes[] = array($code, $msg); + } + + function onFatalProblem() + { + if($this->attempts < 3) + $this->reschedule(); + } + + function onProgress($jzon) + { + try + { + $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); + if(isset($data['p'])) + $this->last_progress = floatval(str_replace(',', '.', $data['p'])); + if(isset($data['arg_idx'])) + $this->last_done_arg_idx = (int)$data['arg_idx']; + } + catch(Exception $e) + { + echo $e->getMessage() . "\n"; + } + } + + function _resetCheckTimes() + { + $this->reset_time = microtime(true); + $this->last_alive_check_time = $this->reset_time; + $this->last_stuck_check_time = $this->reset_time; + $this->last_ext_status_item_time = 0; + } + + function reschedule() + { + ++$this->attempts; + $this->device = null; + $this->need_reschedule = true; + } + + function start() + { + $this->_resetCheckTimes(); + $this->status_codes = array(); + $this->start_time = time(); + } + + function getDuration() + { + return time() - $this->start_time; + } +} + +class TestLevelsTask extends Task +{ + public $arg_offset = 0; + + private $rseeds = array(); + private $seed = null; + private $timescale = null; + + function __construct(Cmd $cmd, array $args, $with_rseeds = false, $seed = null, $timescale = null, $title = '') + { + $level_ids = array(); + //TODO: should not really be here + if($with_rseeds) + { + foreach($args as $item) + { + $level_ids[] = $item[0]; + $this->rseeds[] = $item[1]; + } + } + else + $level_ids = $args; + + $this->seed = $seed; + $this->timescale = $timescale; + + parent::__construct($cmd, $level_ids, $title); + } + + function getCmdArgs() : array + { + $args = array(); + + if($this->rseeds) + { + foreach($this->args as $idx => $arg) + { + $args[] = $arg; + $args[] = $this->rseeds[$idx]; + } + } + else + $args = $this->args; + + //appending arg. offset as a first argument + array_unshift($args, ''.$this->arg_offset); + array_unshift($args, '--offset'); + + if($this->rseeds) + { + array_unshift($args, 'true'); + array_unshift($args, '--levels-with-rseed'); + } + + if($this->seed !== null) + { + array_unshift($args, ''.$this->seed); + array_unshift($args, '--seed'); + } + + if($this->timescale !== null) + { + array_unshift($args, ''.$this->timescale); + array_unshift($args, '--timescale'); + } + + return $args; + } + + function onFatalProblem() + { + $start_arg = isset($this->args[$this->arg_offset]) ? $this->args[$this->arg_offset] : '?'; + $done_arg = isset($this->args[$this->last_done_arg_idx]) ? $this->args[$this->last_done_arg_idx] : '?'; + + $this->plan->post("Handling error, progress:".round($this->getProgress(),2).", last done arg:{$done_arg}, start arg:{$start_arg}, arg offset:{$this->arg_offset}, args:".sizeof($this->args)." *$this->device*"); + + //we're allowed to skip only in case of exceptions + if($this->getLastFatalProblem() == Task::CODE_EXCEPTION) + { + if($this->last_done_arg_idx > $this->arg_offset) + $this->arg_offset = $this->last_done_arg_idx; + ++$this->arg_offset; + } + + //let's reschedule the task if it still makes sense + if($this->arg_offset < sizeof($this->args)) + $this->reschedule(); + } +} +