diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6e92f57 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tags diff --git a/atf.inc.php b/atf.inc.php index 3dc095f..716ae74 100644 --- a/atf.inc.php +++ b/atf.inc.php @@ -1,1116 +1,17 @@ apk_name = $apk_name; - $this->override_conf = $override_conf; - } - - function countInstallsInProgress() - { - $c = 0; - foreach($this->installs as $status) - if($status == self::ST_IN_PROGRESS) - ++$c; - return $c; - } - - function installAsync($device) - { - return Amp\call(function() use($device) { - - while($this->countInstallsInProgress() >= self::MAX_INSTALLS_IN_PROGRESS) - yield Amp\delay(1000); - - if(isset($this->installs[$device])) - return; - $this->installs[$device] = self::ST_IN_PROGRESS; - try - { - yield atf_host_exec_async("%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK); - if($this->apk_name !== null) - { - yield atf_host_exec_async("%{adb}% -s $device uninstall %{package_id}%", DEPLOY_OPT_ERR_OK, 30); - yield atf_host_exec_async("%{adb}% -s $device install -r ./{$this->apk_name}", 0, 400); - } - - $this->installs[$device] = self::ST_INSTALLED; - } - catch(Exception $e) - { - atf_log("Error during install *$device*: " . $e->getMessage()); - unset($this->installs[$device]); - throw $e; - } - }); - } - - function getSize() - { - list($status, $lines) = atf_host_exec("ls -sd -- ./{$this->apk_name}", DEPLOY_OPT_ERR_OK|DEPLOY_OPT_SILENT); - if($status != 0) - return 0; - $items = explode(' ', $lines[0]); - //ls -s returns size in blocks where each block is 512 bytes long - return $items[0]*512; - } -} - - -class ATFAdbDevicePool implements IATFDevicePool -{ - function get() - { - return atf_get_devices(); - } -} - - -class ATFFixedDevicePool implements IATFDevicePool -{ - private $devices; - - function __construct(array $devices) - { - $this->devices = $devices; - } - - function get() - { - return $this->devices; - } -} - - -class ATFCachedDevices implements IATFDevicePool -{ - private $provider; - private $cached = null; - private $last_cache_time; - private $keep_cache_time; - - function __construct(IATFDevicePool $provider, $keep_cache_time) - { - $this->provider = $provider; - $this->keep_cache_time = $keep_cache_time; - } - - function get() - { - if($this->cached === null || (time() - $this->last_cache_time) > $this->keep_cache_time) - { - $this->cached = $this->provider->get(); - $this->last_cache_time = time(); - } - return $this->cached; - } -} - - - -class ATFSession -{ - public $guid; - public $version; - public $branch; - public $rev_hash; - - public $plans = array(); - - public $ignored_devices = array(); - public $bogus_device_count = array(); - - public $share_with_qa_chan = true; - public $shared_qa_errors = array(); - - function __construct() - { - $this->guid = atf_guid(); - $this->branch = get('GAME_VERSION_BRANCH'); - $this->version = get('GAME_VERSION'); - $this->rev_hash = get('GAME_REVISION_HASH'); - } - - function incBogusDevice($device) - { - if(!isset($this->bogus_device_count[$device])) - $this->bogus_device_count[$device] = 0; - return ++$this->bogus_device_count[$device]; - } - - function resetBogusDeviceCount($device) - { - $this->bogus_device_count[$device] = 0; - } - - function resetBogusDevices() - { - $this->bogus_device_count = array(); - } - - function addPlan(ATFPlan $plan) - { - $plan->session = $this; - $this->plans[] = $plan; - } - - function resetPlans() - { - $this->plans = array(); - } - - function run($apk_path = null, $apk_reuse = false, $override_conf = false, $adb_reboot = true) - { - $apk_name = $apk_path === null ? null : basename($apk_path); - - _atf_start_watchdog($this->guid); - - //1. deploy an apk file to the atf host - if(!$apk_reuse && $apk_path !== null) - { - if(!is_file($apk_path)) - throw new Exception("No such file '$apk_path'"); - - $local_crc = _atf_get_local_file_crc($apk_path); - $remote_crc = _atf_get_remote_file_crc($apk_name); - - if($remote_crc === '' || $local_crc !== $remote_crc) - atf_host_put_file($apk_path, $apk_name); - else - atf_log("Skipping same .apk upload"); - } - - //2. reboot just in case - if($adb_reboot) - atf_adb_reboot(); - - - $install = new ATFApkInstaller($apk_name, $override_conf); - $this->trySendApkStatsEvent($install->getSize()); - - foreach($this->plans as $plan) - $this->_runPlan($install, $plan); - - foreach($this->plans as $plan) - { - if($plan->hasFatalProblems() || - ($plan->isOver() && !$plan->getDevices())) - return false; - } - - return true; - } - - - function _runPlan(ATFApkInstaller $install, ATFPlan $plan) - { - atf_log("Starting " . $plan->getTitle() . "..."); - - //NOTE: hung_threshold must be larger than gone_threshold - Amp\Promise\wait($plan->runAsync($install, $sleep_time = 4, $hung_threshold = 80, $gone_threshold = 30, $stuck_threshold = 200)); - } - - function trySendStatsFromJzonAsync(ATFTask $task, $jzon) - { - return Amp\call(function() use($task, $jzon) { - - try - { - $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); - $table = $data['table']; - unset($data['table']); - - if(isset($data['deviceMemoryUsage']) && $data['deviceMemoryUsage'] === '') - { - $mem = yield atf_device_mem_async($task->device); - $data['deviceMemoryUsage'] = $mem['total']; - if($table === 'device_memory') - { - $data['deviceMemoryUsageNative'] = $mem['native']; - $data['deviceMemoryUsageSystem'] = $mem['system']; - $data['deviceMemoryUsageJava'] = $mem['java']; - $data['deviceMemoryUsageGraphics'] = $mem['graphics']; - } - } - $this->trySendStats($task, $table, $data); - } - catch(Exception $e) - { - echo $e->getMessage() . "\n"; - } - - }); - } - - function trySendApkStatsEvent($apk_size) - { - try - { - $msg = "version $this->version($this->rev_hash) size: $apk_size"; - atf_slack_post($msg, array('channel' => _atf_slack_chan_qa())); - - $data['time'] = time(); - $data['version'] = $this->version; - $data['revHash'] = $this->rev_hash; - $data['guid'] = $this->guid; - $data['branch'] = $this->branch; - $data['event'] = 'apk_size'; - $data['value'] = $apk_size; - atf_stats_send('event_value', $data); - } - catch(Exception $e) - { - echo $e->getMessage() . "\n"; - } - } - - - function trySendStats(ATFTask $task, $table, array $data) - { - try - { - $data['guid'] = $this->guid; - $data['time'] = time(); - $data['deviceId'] = $task->device; - atf_stats_send($table, $data); - } - catch(Exception $e) - { - echo $e->getMessage() . "\n"; - } - } - - function trySendStatsEvent(ATFTask $task, $event, $value = 1) - { - $this->trySendStats( - $task, - 'event_value', - array( - 'event' => $event, - 'value' => $value, - 'branch' => $this->branch, - 'version' => $this->version, - 'revHash' => $this->rev_hash - ) - ); - } - - function tryShareToQAChannel($msg_slack_id, $error) - { - if(!$this->share_with_qa_chan) - return; - - //let's skip similar already shared errors - if($this->_calcSharedErrorSimilarity($error) > 80) - return; - - $this->shared_qa_errors[] = $error; - - //let's share an exception message to goh-qa channel - $resp = atf_slack_get_permalink($msg_slack_id); - if(isset($resp['permalink'])) - atf_slack_post($resp['permalink'], array('channel' => _atf_slack_chan_qa())); - } - - function _calcSharedErrorSimilarity($error) - { - $max_perc = 0; - foreach($this->shared_qa_errors as $shared) - { - $perc = 0; - similar_text($shared, $error, $perc); - if($perc > $max_perc) - $max_perc = $perc; - } - return $max_perc; - } -} - - -class ATFPlan -{ - const EXCEPTIONS_THRESHOLD = 5; - const BOGUS_DEVICE_COUNT_THRESHOLD = 5; - - public $session; - public $name; - public $device_pool; - public $tasks = array(); - public $slack_thread_ts; - - function __construct($name, IATFDevicePool $device_pool) - { - $this->name = $name; - $this->device_pool = $device_pool; - } - - function getTitle() - { - return "Testing plan '{$this->name}' devices:".sizeof($this->getDevices())."(n/a:".sizeof($this->session->ignored_devices).")"; - } - - function createSlackThread() - { - for($i=0;$i<5;++$i) - { - $resp = atf_slack_post($this->getTitle()); - if(isset($resp['ok'])) - { - $this->slack_thread_ts = $resp['ts']; - return; - } - sleep(1); - } - throw new Exception("Could not create Slack thread"); - } - - function updateSlackThread() - { - $issues_summary = ''; - foreach($this->getProblemsHistogram() as $code => $count) - { - if($code == ATFTask::CODE_GONE) - $issues_summary .= " gones:$count"; - else if($code == ATFTask::CODE_NSTART) - $issues_summary .= " nstrts:$count"; - else if($code == ATFTask::CODE_STUCK) - $issues_summary .= " stucks:$count"; - else if($code == ATFTask::CODE_HUNG) - $issues_summary .= " hungs:$count"; - else if($code == ATFTask::CODE_EXCEPTION) - $issues_summary .= " excepts:$count"; - else if($code == ATFTask::CODE_WARN) - $issues_summary .= " warns:$count"; - } - - $running_count = 0; - $over_count = 0; - $total_progress = 0; - - foreach($this->tasks as $task) - { - $total_progress += $task->getProgress(); - if($task->isOver()) - $over_count++; - if($task->device) - $running_count++; - } - $progress_txt = ' tasks:' . ($running_count > 0 ? $running_count . '/' : '') . $over_count . '/' . sizeof($this->tasks) . ' (' . round($total_progress/sizeof($this->tasks)*100, 2) . '%)'; - - atf_slack_update($this->slack_thread_ts, - array(array( - "text" => $this->getTitle() . $issues_summary . $progress_txt, - "color" => $this->_getThreadColor(), - "mrkdwn_in" => array("text") - )) - ); - } - - function getProblemsHistogram() - { - $histogram = array(); - foreach($this->tasks as $task) - { - foreach($task->status_codes as $item) - { - list($code, $_) = $item; - if(!ATFTask::isProblemCode($code)) - continue; - - if(!isset($histogram[$code])) - $histogram[$code] = 0; - ++$histogram[$code]; - } - } - return $histogram; - } - - function hasStatusCode($code) - { - return $this->countStatusCode($code) > 0; - } - - function countStatusCode($code) - { - $c = 0; - foreach($this->tasks as $task) - $c += $task->countStatusCode($code); - return $c; - } - - function _getThreadColor() - { - if(!$this->getDevices()) - return "#000000"; - - $has_gones = $this->hasStatusCode(ATFTask::CODE_GONE); - $has_hungs = $this->hasStatusCode(ATFTask::CODE_HUNG); - $has_stucks = $this->hasStatusCode(ATFTask::CODE_STUCK); - $has_nostarts = $this->hasStatusCode(ATFTask::CODE_NSTART); - $has_excepts = $this->hasStatusCode(ATFTask::CODE_EXCEPTION); - $has_warns = $this->hasStatusCode(ATFTask::CODE_WARN); - - if($has_gones || $has_hungs || $has_warns || $has_excepts || $has_stucks || $has_nostarts) - { - if(!$has_excepts) - return $this->isOver() ? "warning" : "#FFCF9E"; - else - return $this->isOver() ? "danger" : "#FFCCCB"; - } - else if($this->isOver()) - return "good"; - else - return "#D3D3D3"; - } - - function _ignoreDevice($device, $reason) - { - $this->session->ignored_devices[] = $device; - atf_log("Ignoring device *$device*: $reason"); - atf_slack_post("Ignoring device *$device*: $reason", array('thread_ts' => $this->slack_thread_ts)); - } - - function getDevices() - { - $devices = array_diff($this->device_pool->get(), $this->session->ignored_devices); - return $devices; - } - - function addTask(ATFTask $task) - { - $this->tasks[] = $task; - } - - function isOver() - { - if(!$this->getDevices()) - { - return true; - } - if($this->countStatusCode(ATFTask::CODE_EXCEPTION) >= self::EXCEPTIONS_THRESHOLD) - { - return true; - } - foreach($this->tasks as $task) - { - if(!$task->isOver()) - return false; - } - return true; - } - - function hasFatalProblems() - { - foreach($this->tasks as $task) - { - foreach($task->status_codes as $item) - { - if(ATFTask::isFatalCode($item[0])) - return true; - } - } - return false; - } - - - function _findUnassignedTask() - { - foreach($this->tasks as $task) - { - if(!$task->device && !$task->isOver()) - return $task; - } - return null; - } - - function runAsync(ATFApkInstaller $install, $sleep_time, $hung_threshold, $gone_threshold, $stuck_threshold) - { - $this->createSlackThread(); - - $this->session->resetBogusDevices(); - - $cs = array(); - - foreach($this->tasks as $task) - $cs[] = $this->_runTaskAsync($task, $install, $sleep_time, $hung_threshold, $gone_threshold, $stuck_threshold); - - return Amp\Promise\all($cs); - } - - - function _runTaskAsync(ATFTask $task, ATFApkInstaller $install, $sleep_time, $hung_threshold, $gone_threshold, $stuck_threshold) - { - return Amp\call(function() use($task, $install, $sleep_time, $hung_threshold, $gone_threshold, $stuck_threshold) { - while(!$this->isOver() && !$task->isOver()) - { - $this->updateSlackThread(); - - yield Amp\delay((int)$sleep_time*1000); - - if(!$task->device) - { - if(!yield $this->_findDeviceAndStartAsync($task, $install)) - continue; - } - - $check_error = yield $this->_tryCheckExtStatusAsync($task, /*attempts*/3, /*timeout*/20); - if($check_error !== null) - { - $this->_ignoreDevice($device, $check_error); - if (!get('EXT_BOT_ERROR_PAUSE')) - $task->reschedule(); - } - - if(!$task->isOver()) - yield $this->_checkHealthAsync($task, $hung_threshold, $gone_threshold, $stuck_threshold); - - if($task->hasFatalProblem()) - { - yield $this->_processFatalProblemAsync($task); - } - else if($task->isDone()) - { - //let's reset hung stats for this device - $this->session->resetBogusDeviceCount($task->device); - //let's free the device - $task->device = null; - } - } - - $this->updateSlackThread(); - }); - } - - function _findDeviceAndStartAsync(ATFTask $task, ATFApkInstaller $install) - { - return Amp\call(function() use($task, $install) { - - $device = $this->_findFreeDevice(); - if(!$device) - return false; - - //let's mark the device as occupied ASAP - $task->device = $device; - try - { - atf_slack_post("Preparing *{$task->device}*", array('thread_ts' => $this->slack_thread_ts)); - - yield $install->installAsync($device); - yield atf_start_ext_cmd_on_device_async($device, $task->getCmd(), $task->getCmdArgs()); - } - catch(Exception $e) - { - $this->_ignoreDevice($device, $e->getMessage()); - $task->reschedule(); - return false; - } - - $task->start($this->slack_thread_ts); - return true; - - }); - } - - function _processFatalProblemAsync(ATFTask $task) - { - return Amp\call(function() use($task) { - - $this->_postScreenToSlack($task); - - $fatal_msg = "Fatal problem ({$task->last_fatal_problem} - ".ATFTask::code2string($task->last_fatal_problem)."), attempt:{$task->attempts} *{$task->device}*"; - - atf_log("[FTL] $fatal_msg"); - - atf_slack_post($fatal_msg, array('thread_ts' => $task->slack_thread_ts)); - - if($task->last_fatal_problem == ATFTask::CODE_GONE || - $task->last_fatal_problem == ATFTask::CODE_HUNG) - { - $app_log = atf_get_logcat_unity($task->device, 300); - $app_log = _atf_trim_start($app_log, 2000); - atf_slack_post("Last logs: ```$app_log``` *{$task->device}*", array('thread_ts' => $task->slack_thread_ts)); - } - - if (!get('EXT_BOT_ERROR_PAUSE')) - $task->reschedule(); - - }); - } - - - function _findFreeDevice() - { - $available = array(); - - foreach($this->getDevices() as $device) - { - $busy = false; - foreach($this->tasks as $task) - { - if($task->device == $device) - { - $busy = true; - break; - } - } - if(!$busy) - $available[] = $device; - } - - if(!$available) - return null; - return $available[mt_rand(0, sizeof($available) - 1)]; - } - - function _tryCheckExtStatusAsync(ATFTask $task, $attempts, $timeout) - { - return Amp\call(function() use($task, $attempts, $timeout) { - - $last_error = null; - - for($i=0;$i<$attempts;++$i) - { - try - { - yield $this->_checkExtStatusAsync($task, $timeout); - break; - } - catch(Exception $e) - { - $last_error = $e->getMessage(); - continue; - } - } - - return $last_error; - - }); - } - - function _checkExtStatusAsync(ATFTask $task, $timeout) - { - return Amp\call(function() use($task, $timeout) { - - $ext_status = yield atf_get_ext_status_async($task->device, $timeout); - if(!is_array($ext_status)) - return; - - $new_items = $this->_getExtStatusItemsSince($task->last_ext_status_item_time, $ext_status); - if(!$new_items) - return; - - $task->last_ext_status_item_time = end($new_items)['time']; - - foreach($new_items as $item) - yield $this->_analyzeExtStatusItemAsync($task, $item); - - }); - } - - - function _getExtStatusItemsSince($time, array $ext_status) - { - $new_items = array(); - foreach($ext_status['entries'] as $item) - { - if($item['time'] > $time) - $new_items[] = $item; - } - return $new_items; - } - - - function _checkHealthAsync(ATFTask $task, $hung_threshold, $gone_threshold, $stuck_threshold) - { - return Amp\call(function() use($task, $hung_threshold, $gone_threshold, $stuck_threshold) { - - $device_is_bogus = false; - - $not_alive_time = microtime(true) - $task->last_alive_check_time; - $stuck_time = microtime(true) - $task->last_stuck_check_time; - - if($not_alive_time > $hung_threshold) - { - $task->addStatusCode(ATFTask::CODE_HUNG); - $this->session->trySendStatsEvent($task, 'hung'); - atf_log("[HNG] No activity for $hung_threshold seconds *{$task->device}*"); - $device_is_bogus = true; - } - else if($not_alive_time > $gone_threshold) - { - list($status, $_) = yield atf_host_exec_async("%{adb}% -s {$task->device} shell pidof %{package_id}%", DEPLOY_OPT_ERR_OK); - if($status != 0) - { - if($task->last_alive_check_time === $task->reset_time) - { - $task->addStatusCode(ATFTask::CODE_NSTART); - $this->session->trySendStatsEvent($task, 'nstart'); - atf_log("[NFD] No app started after $hung_threshold seconds *{$task->device}*"); - } - else - { - $task->addStatusCode(ATFTask::CODE_GONE); - $this->session->trySendStatsEvent($task, 'gone'); - atf_log("[GNE] App is gone after $hung_threshold seconds *{$task->device}*"); - } - $device_is_bogus = true; - } - else - //let's tap the screen just in case app Activity is not in foreground - yield atf_host_exec_async("%{adb}% -s {$task->device} shell input tap 360 930", DEPLOY_OPT_ERR_OK); - } - else if($stuck_time > $stuck_threshold) - { - $task->addStatusCode(ATFTask::CODE_STUCK); - $this->session->trySendStatsEvent($task, 'stuck'); - atf_log("[STK] Stuck for $stuck_threshold seconds *{$task->device}*"); - } - - if($device_is_bogus) - { - $this->_reportErrorFromLogcatToSlack($task, 1000); - $this->_incAndCheckBogusDevice($task->device); - } - - }); - } - - function _incAndCheckBogusDevice($device) - { - if($this->session->incBogusDevice($device) >= self::BOGUS_DEVICE_COUNT_THRESHOLD) - { - $this->_ignoreDevice($device, "Too many N/As"); - return true; - } - return false; - } - - static function _parseExtMessage($message) - { - $msg_code = null; - $msg_text = $message; - //example: [DBG] this a debug message - if(preg_match('~^(\[[^\]]+\])(.*)$~', $message, $matches)) - { - $msg_code = $matches[1]; - $msg_text = $matches[2]; - } - return array($msg_code, $msg_text); - } - - static function _printToShellExtItem(ATFTask $task, array $item) - { - $shell_msg = _atf_trim($item['message'], 200); - if(ATFTask::isProblemCode($item['error'])) - $shell_msg = "[PROBLEM] Code:{$item['error']}, $shell_msg"; - $shell_msg = "(".round($item['time'],1)."s) {$shell_msg} *{$task->device}*"; - - atf_log($shell_msg); - } - - function _postToSlackExtStatusItem(ATFTask $task, array $item) - { - $orig_msg = _atf_trim($item['message'], 3000); - $slack_msg = $orig_msg; - - if($item['error'] == ATFTask::CODE_EXCEPTION) - $slack_msg = "```$slack_msg```"; - - $problem = $item['error'] == ATFTask::CODE_EXCEPTION || $item['error'] == ATFTask::CODE_STUCK; - - $slack_msg = '('.round($item['time'],1).'s) '.$slack_msg.' *'.$task->device.'*'; - - $mentions = getor("ATF_PROBLEM_MENTION_GROUPS", null); - if($mentions !== null && $problem && strlen($mentions) > 0) - { - $mentions = explode(",", $mentions); - $groups = ""; - foreach($mentions as $mention) - $groups .= ' '; // - - if(strlen($groups) > 0) - $slack_msg = $groups."\n".$slack_msg; - } - - $resp = atf_slack_post($slack_msg, array('thread_ts' => $task->slack_thread_ts)); - if(isset($resp['ok']) && $problem) - $this->session->tryShareToQAChannel($resp['ts'], $orig_msg); - } - - function _analyzeExtStatusItemAsync(ATFTask $task, array $item) - { - return Amp\call(function() use($task, $item) { - - self::_printToShellExtItem($task, $item); - - $task->addStatusCode($item['error'], $item['message']); - - if($item['error'] == ATFTask::CODE_EXCEPTION) - $this->session->trySendStatsEvent($task, 'exception'); - - list($msg_type, $msg_text) = self::_parseExtMessage($item['message']); - - //NOTE: alive system messages are sent periodically by ExtBot itself, - // we'd like to ignore them for stuck detection - if($msg_type !== '[ALIVE]') - $task->last_stuck_check_time = microtime(true); - - if($msg_type === '[DONE]') - { - $task->is_done = true; - $this->_postToSlackExtStatusItem($task, $item); - } - - //NOTE: in case of any message from the device we update the alive check timestamp - $task->last_alive_check_time = microtime(true); - - yield $this->_processExtStatusMessageAsync($task, $item, $msg_type, $msg_text); - - }); - } - - function _processExtStatusMessageAsync($task, $item, $msg_type, $msg_text) - { - return Amp\call(function() use($task, $item, $msg_type, $msg_text) { - - if($msg_type === '[PRG]') - { - //let's reset hung stats since progress is going - $this->session->resetBogusDeviceCount($task->device); - $task->onProgress($msg_text); - } - else if($msg_type === '[STAT]') - yield $this->session->trySendStatsFromJzonAsync($task, $msg_text); - else if($msg_type === '[WRN]') - { - $task->addStatusCode(ATFTask::CODE_WARN); - $this->_postToSlackExtStatusItem($task, $item); - } - else if($msg_type === null) - { - $this->_postToSlackExtStatusItem($task, $item); - } - - }); - } - - function _reportErrorFromLogcatToSlack(ATFTask $task, $limit) - { - $errors_log = _atf_trim(atf_get_logcat_errors($task->device, $limit), 3000); - if($errors_log) - atf_slack_post("```$errors_log``` *{$task->device}*", array('thread_ts' => $task->slack_thread_ts)); - } - - function _postScreenToSlack(ATFTask $task) - { - global $GAME_ROOT; - - $png_data = atf_screen($task->device); - if($png_data) - { - ensure_write("$GAME_ROOT/build/atf/screen.png", $png_data); - atf_slack_post_png("screen", "$GAME_ROOT/build/atf/screen.png", array('thread_ts' => $task->slack_thread_ts)); - } - } -} - - -class ATFTask -{ - const CODE_NONE = 0; - const CODE_EXCEPTION = 1; - const CODE_WARN = 2; - const CODE_NSTART = 124; - const CODE_STUCK = 125; - const CODE_GONE = 126; - const CODE_HUNG = 127; - const CODE_DONE = 128; - - - static function code2string($code) - { - switch($code) - { - case self::CODE_NONE: - return "none"; - case self::CODE_EXCEPTION: - return "exception"; - case self::CODE_WARN: - return "warning"; - case self::CODE_STUCK: - return "stuck"; - case self::CODE_NSTART: - return "nstart"; - case self::CODE_GONE: - return "gone"; - case self::CODE_HUNG: - return "hung"; - case self::CODE_DONE: - return "done"; - } - return "???"; - } - - public $cmd; - public $args; - public $title; - public $last_progress; - public $last_done_arg_idx; - public $slack_thread_ts; - public $reset_time = 0; - public $last_alive_check_time = 0; - public $last_stuck_check_time = 0; - public $last_ext_status_item_time = 0; - public $last_ext_input_req_id = 0; - public $status_codes = array(); - public $is_done; - public $last_fatal_problem = 0; - public $attempts = 1; - public $device; - - function __construct($cmd, array $args, $title = '') - { - $this->cmd = $cmd; - $this->args = $args; - $this->title = $title; - } - - function getCmd() - { - return $this->cmd; - } - - function getCmdArgs() - { - return $this->args; - } - - function getProgress() - { - return $this->last_progress; - } - - static function isProblemCode($code) - { - return $code > self::CODE_NONE && $code < self::CODE_DONE; - } - - static function isFatalCode($code) - { - return - $code == ATFTask::CODE_STUCK || - $code == ATFTask::CODE_NSTART || - $code == ATFTask::CODE_GONE || - $code == ATFTask::CODE_HUNG || - $code == ATFTask::CODE_EXCEPTION; - } - - function hasFatalProblem() - { - return $this->last_fatal_problem != 0; - } - - function hasStatusCode($code) - { - return $this->countStatusCode($code) > 0; - } - - function countStatusCode($code) - { - $c = 0; - foreach($this->status_codes as $item) - { - if($item[0] === $code) - $c++; - } - return $c; - } - - function isDone() - { - return $this->is_done; - } - - function isOver() - { - return $this->isDone() || $this->hasFatalProblem(); - } - - function addStatusCode($code, $msg = '') - { - $this->status_codes[] = array($code, $msg); - - if($code == ATFTask::CODE_DONE) - $this->is_done = true; - else if(self::isFatalCode($code)) - $this->last_fatal_problem = $code; - } - - - function onProgress($jzon) - { - try - { - $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); - if(isset($data['p'])) - $this->last_progress = floatval(str_replace(',', '.', str_replace('.', '', $data['p']))); - if(isset($data['arg_idx'])) - $this->last_done_arg_idx = (int)$data['arg_idx']; - } - catch(Exception $e) - { - echo $e->getMessage() . "\n"; - } - } - - function _resetCheckTimes() - { - $this->reset_time = microtime(true); - $this->last_alive_check_time = $this->reset_time; - $this->last_stuck_check_time = $this->reset_time; - $this->last_ext_status_item_time = 0; - } - - function reschedule() - { - ++$this->attempts; - $this->device = null; - } - - function start($slack_thread_ts) - { - $this->_resetCheckTimes(); - $this->slack_thread_ts = $slack_thread_ts; - } -} - -function atf_host_exec_async($cmd, $opts = 0, $timeout = 10) : Amp\Promise -{ - return Amp\call(function() use($cmd, $opts, $timeout) { - - $timeout_cmd = "/usr/local/bin/timeout -k 5s $timeout $cmd"; - - $res = yield deploy_ssh_exec_async(atf_host(), $timeout_cmd, $opts); + $res = yield taskman\deploy_ssh_exec_async($atf_host, $timeout_cmd, $opts); list($status, $lines) = current($res); //in case of timeout we set status to false explicitely @@ -1122,11 +23,11 @@ function atf_host_exec_async($cmd, $opts = 0, $timeout = 10) : Amp\Promise }); } -function atf_host_exec($cmd, $opts = 0, $timeout = 10) +function host_exec(string $atf_host, string $cmd, $opts = 0, $timeout = 10) { - $timeout_cmd = "/usr/local/bin/timeout -k 5s $timeout $cmd"; - - $res = deploy_ssh_exec(atf_host(), $timeout_cmd, $opts); + $timeout_cmd = "timeout -k 5s $timeout $cmd"; + + $res = taskman\deploy_ssh_exec($atf_host, $timeout_cmd, $opts); list($status, $lines) = current($res); //in case of timeout we set status to false explicitely @@ -1136,23 +37,36 @@ function atf_host_exec($cmd, $opts = 0, $timeout = 10) return array($status, $lines); } -function atf_host_get_file($file_name) +function host_get_file(string $atf_host, string $file_name) { - $file_data = current(deploy_get_file(atf_host(), $file_name)); + $file_data = current(taskman\deploy_get_file($atf_host, $file_name)); return $file_data; } -function atf_host_put_file($local_path, $remote_path) +function host_put_file(string $atf_host, string $local_path, string $remote_path) { - deploy_scp_put_file(atf_host(), $local_path, $remote_path); + taskman\deploy_rsync($atf_host, $local_path, $remote_path); } -function atf_host_put_file_async($local_path, $remote_path) +function host_put_file_async(string $atf_host, string $local_path, string $remote_path) { - return deploy_scp_put_file_async(atf_host(), $local_path, $remote_path); + return taskman\deploy_rsync_async($atf_host, $local_path, $remote_path); } -function _atf_slice(array $items, $max) +function _spread(array $items, $num) +{ + $k = sizeof($items) / $num; + + $res = array(); + for($i=0;$i<$num;++$i) + { + $tmp = array_slice($items, (int)floor($k*$i), (int)ceil($k)); + $res[$i] = $tmp; + } + return $res; +} + +function _slice(array $items, $max) { $sliced = array(); @@ -1169,256 +83,119 @@ function _atf_slice(array $items, $max) return $sliced; } -function _atf_get_remote_file_crc($path) -{ - list($status, $lines) = atf_host_exec("crc32 $path", DEPLOY_OPT_ERR_OK); - if($status != 0) - return ''; - return trim(implode("\n", $lines)); -} - -function _atf_get_local_file_crc($path) -{ - if(!is_file($path)) - return ''; - - shell("crc32 $path", $out); - return trim($out[0]); -} - -function atf_device_put_file_async($device, $data_or_file, $device_path, $is_file = false) -{ - return Amp\call(function() use($device, $data_or_file, $device_path, $is_file) { - - if($device_path[0] !== "/") - $device_path = atf_package_dir() . $device_path; - - $tmp_remote_file = uniqid(basename($device_path)."_"); - - if(!$is_file) - { - $local_path = tempnam("/tmp", "atf_"); - ensure_write($local_path, $data_or_file); - } - else - $local_path = $data_or_file; - - try - { - //1. let's copy local file to the ATF host as temp one - yield atf_host_put_file_async($local_path, $tmp_remote_file); - } - finally - { - if(!$is_file) - ensure_rm($local_path); - } - - //2. let's push the temp file to the device - $device_dir = dirname($device_path); - yield atf_host_exec_async("%{adb}% -s $device shell mkdir -p $device_dir"); - yield atf_host_exec_async("%{adb}% -s $device push $tmp_remote_file $device_path", 0, 30); - yield atf_host_exec_async("rm -rf $tmp_remote_file"); - -}); -} - -function atf_device_put_folder_async($device, $data, $device_path) -{ - return Amp\call(function() use($device, $data, $device_path) { - - if($device_path[0] !== "/") - $device_path = atf_package_dir() . $device_path; - - $device_dir = dirname($device_path); - - yield atf_host_exec_async("%{adb}% -s $device shell rm -rf $device_path"); - yield atf_host_exec_async("%{adb}% -s $device shell mkdir -p $device_dir"); - yield atf_host_exec_async("%{adb}% -s $device push $data $device_path", 0, 400); - }); -} - -function atf_device_del_file_async($device, $device_path, $opts = 0) -{ - if($device_path[0] !== "/") - $device_path = atf_package_dir() . $device_path; - - return atf_host_exec_async("%{adb}% -s $device shell rm -rf $device_path", $opts); -} - -function atf_device_blink($device, $times = 5) -{ - //disable auto brightness - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness_mode 0"); - - for($i=0;$i<$times;++$i) - { - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness 255"); - sleep(1); - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness 5"); - } - - //enable auto brightness - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness 100"); - atf_host_exec("%{adb}% -s $device shell settings put system screen_brightness_mode 1"); -} - -function atf_device_pull_file_async($device, $device_path, $timeout = 10, $throw_error_on_timeout = false) -{ - return Amp\call(function() use($device, $device_path, $throw_error_on_timeout, $timeout) { - - if($device_path[0] !== "/") - $device_path = atf_package_dir() . $device_path; - - $tmp_remote_file = uniqid(basename($device_path)."_"); - - list($status, $_) = yield atf_host_exec_async("%{adb}% -s $device pull $device_path $tmp_remote_file", DEPLOY_OPT_ERR_OK, $timeout); - - if($status !== 0) - { - if($status === false && $throw_error_on_timeout) - throw new Exception("Could not pull from device '$device' file '$device_path' due to timeout"); - return null; - } - - $file_data = atf_host_get_file($tmp_remote_file); - yield atf_host_exec_async("rm -rf $tmp_remote_file"); - return $file_data; - - }); -} - -function atf_screen($device) -{ - $screen_file_name = uniqid("screen_"); - try - { - atf_host_exec("%{adb}% -s $device exec-out screencap -p > $screen_file_name"); - } - catch(Exception $e) - { - return false; - } - $data = atf_host_get_file($screen_file_name); - atf_host_exec("rm -rf $screen_file_name"); - return $data; -} - -function atf_guid() -{ - return uniqid(); -} - -function _atf_start_watchdog($guid) -{ - global $GAME_ROOT; - - $script = "$GAME_ROOT/gamectl atf_watchdog " . getmypid() . ' ' . $guid; - exec("nohup $script > /dev/null &", $output, $ret); -} - -function _atf_trim($txt, $max_len) +function _trim($txt, $max_len) { return strlen($txt) > $max_len ? substr($txt, 0, $max_len) . "..." : $txt; } -function _atf_trim_start($txt, $max_len) +function _trim_start($txt, $max_len) { return strlen($txt) > $max_len ? "..." . substr($txt, strlen($txt) - $max_len) : $txt; } -function _atf_db() +function _trim_lines($txt, $max_lines) { - $config = array( - 'host' => '172.19.179.148', - 'port' => '8123', - 'username' => get('ATF_CLICKHOUSE_USERNAME'), - 'password' => get('ATF_CLICKHOUSE_PASSWORD') - ); - $db = new ClickHouseDB\Client($config); - $db->database(get('ATF_CLICKHOUSE_DB')); - $db->setTimeout(10); - $db->setConnectTimeOut(5); - return $db; + $lines = explode("\n", $txt, $max_lines+1); + $res = ''; + for($i=0;$iinsert($table, array(array_values($data)), array_keys($data)); + if(!$txt) + return $txt; + //let's check if it's already archieved + if(strlen($txt) > 2 && $txt[1] === ':') + return $txt; + if(!function_exists('lz4_compress')) + return $txt; + return "4:" . base64_encode(lz4_compress(base64_decode($txt))); } -function atf_device_mem_async($device) -{ - return Amp\call(function() use($device) { - list($code, $lines) = yield atf_host_exec_async("%{adb}% -s $device shell dumpsys meminfo -s %{package_id}%", DEPLOY_OPT_ERR_OK, 20); - - $res = array( - 'total' => 0, - 'native' => 0, - 'java' => 0, - 'system' => 0, - 'graphics' => 0, - ); - - if($code !== 0) - return $res; - - foreach($lines as $idx => $line) - { - $items = preg_split('/\s+/', trim($line)); - if($items && $items[0] === "TOTAL:") - $res['total'] = 1*$items[1]; - else if($items && $items[0] === "TOTAL" && $items[1] === "PSS:") - $res['total'] = 1*$items[2]; - else if($items && $items[0] === "Native" && $items[1] === "Heap:") - $res['native'] = 1*$items[2]; - else if($items && $items[0] === "Java" && $items[1] === "Heap:") - $res['java'] = 1*$items[2]; - else if($items && $items[0] === "System:") - $res['system'] = 1*$items[1]; - else if($items && $items[0] === "Graphics:") - $res['graphics'] = 1*$items[1]; - } - - return $res; - }); -} - -function atf_log($msg) +function log($msg) { echo date("Y-m-d H:i:s") . " " . $msg . "\n"; } -function atf_adb_reboot() +function err($msg) { - atf_host_exec("killall adb"); - atf_host_exec("%{adb}% kill-server"); - atf_host_exec("%{adb}% start-server"); + taskman\stderr(date("Y-m-d H:i:s") . " " . $msg . "\n"); } -function atf_reboot_devices(array $devices) +function _retry($max_tries, $func) { - atf_log("Rebooting devices..."); + for($i=0;$i<$max_tries;++$i) + { + try + { + $func(); + return; + } + catch(Exception $e) + { + if(($i+1) == $max_tries) + throw $e; + } + sleep(1); + } +} + +function adb_reboot(string $atf_host) +{ + host_exec($atf_host, "killall adb", DEPLOY_OPT_ERR_OK); + _retry(3, function() use($atf_host) { + host_exec($atf_host, "%{adb}% kill-server"); + }); + host_exec($atf_host, "%{adb}% start-server"); +} + +function reboot_devices(string $atf_host, array $devices, int $wait_after = 40) +{ + log("Rebooting devices..."); //let's reboot devices $ces = array(); foreach($devices as $device) { - $ces[] = Amp\call(function() use($device) { - yield atf_host_exec_async("%{adb}% -s $device reboot", DEPLOY_OPT_ERR_OK); - yield Amp\delay(40*1000); + $ces[] = Amp\call(function() use($atf_host, $device, $wait_after) { + yield host_exec_async($atf_host, "%{adb}% -s $device reboot", DEPLOY_OPT_ERR_OK); + yield Amp\delay($wait_after*1000); }); } Amp\Promise\wait(Amp\Promise\all($ces)); } -function atf_get_devices($extended = false) +function get_ext_status_async(string $atf_host, string $device, int $timeout = 20) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $timeout) { + + $file_data = yield device_pull_file_async($atf_host, $device, "atf_status.js", $timeout, /*throw on timeout*/true); + if($file_data === null) + return null; + $file_data = preg_replace('/[[:cntrl:]]/', '', $file_data); + return json_decode($file_data, true); + + }); +} + +function start_ext_cmd_on_device_async(string $atf_host, string $device, Cmd $cmd, array $cmd_args) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $cmd, $cmd_args) { + + yield device_del_file_async($atf_host, $device, 'atf_status.js', DEPLOY_OPT_ERR_OK); + yield host_exec_async($atf_host, "%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK, 30); + yield update_ext_cmd_async($atf_host, $device, $cmd, $cmd_args); + yield host_exec_async($atf_host, "%{adb}% -s $device shell am start -n %{package_id}%/%{activity}%", 0, 30); + + }); +} + +function get_devices(string $atf_host, bool $extended = false) { $devices = array(); - list($_, $lines) = atf_host_exec("%{adb}% devices -l", DEPLOY_OPT_SILENT); - + list($_, $lines) = host_exec($atf_host, "%{adb}% devices -l", DEPLOY_OPT_SILENT); foreach($lines as $line) { @@ -1438,63 +215,240 @@ function atf_get_devices($extended = false) $devices[] = $m[1]; } } - return $devices; } -function atf_get_ext_status_async($device, $timeout = 20) +function update_ext_cmd_async(string $atf_host, string $device, Cmd $cmd, array $args) { - return Amp\call(function() use($device, $timeout) { - - $file_data = yield atf_device_pull_file_async($device, "ext_status.js", $timeout, /*throw on timeout*/true); - if($file_data === null) - return null; - $file_data = preg_replace('/[[:cntrl:]]/', '', $file_data); - return json_decode($file_data, true); + $ext_cmd = array( + "device_id" => $device, + "module" => $cmd->module, + "func" => $cmd->func, + "args" => $args, + ); + + return device_put_file_async($atf_host, $device, json_encode($ext_cmd), 'atf_cmd.js'); +} + +function package_dir(string $atf_host) +{ + return "/storage/emulated/0/Android/data/".taskman\deploy_get($atf_host, 'package_id')."/files/"; +} + +function device_put_file_async(string $atf_host, string $device, string $data_or_file, string $device_path, bool $is_file = false, $timeout = 30) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $data_or_file, $device_path, $is_file, $timeout) { + + if($device_path[0] !== "/") + $device_path = package_dir($atf_host) . $device_path; + + $tmp_remote_file = '%{dir}%'.uniqid(basename($device_path)."_"); + + if(!$is_file) + { + $local_path = tempnam("/tmp", "atf_"); + taskman\ensure_write($local_path, $data_or_file); + } + else + { + $local_path = $data_or_file; + if(!is_file($local_path)) + throw new Exception("No such local file: $local_path"); + } + + try + { + //1. let's copy local file to the ATF host as temp one + yield host_put_file_async($atf_host, $local_path, $tmp_remote_file); + } + finally + { + if(!$is_file) + taskman\ensure_rm($local_path); + } + + //2. let's push the temp file to the device + $device_dir = dirname($device_path); + yield host_exec_async($atf_host, "%{adb}% -s $device shell mkdir -p $device_dir"); + yield host_exec_async($atf_host, "%{adb}% -s $device push $tmp_remote_file $device_path", 0, $timeout); + yield host_exec_async($atf_host, "%{adb}% -s $device shell chmod -R 777 $device_dir", DEPLOY_OPT_ERR_OK); + yield host_exec_async($atf_host, "rm -rf $tmp_remote_file"); }); } -function atf_update_ext_cmd_async($device, $main, array $args) +function device_del_file(string $atf_host, string $device, string $device_path, $opts = 0) { - $ext_cmd = array( - "device_id" => $device, - "main" => $main, - "args" => $args, - ); + if($device_path[0] !== "/") + $device_path = package_dir($atf_host) . $device_path; - return atf_device_put_file_async($device, json_encode($ext_cmd), 'ext_cmd.js'); + host_exec($atf_host, "%{adb}% -s $device shell rm -rf $device_path", $opts); } -function atf_package_dir() +function device_del_file_async(string $atf_host, string $device, string $device_path, $opts = 0) { - return "/storage/emulated/0/Android/data/".deploy_get(atf_host(), 'package_id')."/files/"; + if($device_path[0] !== "/") + $device_path = package_dir($atf_host) . $device_path; + + return host_exec_async($atf_host, "%{adb}% -s $device shell rm -rf $device_path", $opts); } -function atf_get_locat_errors_since($device, $since_stamp)//not using +function device_blink($atf_host, string $device, int $times = 5) +{ + //disable auto brightness + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness_mode 0"); + + for($i=0;$i<$times;++$i) + { + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness 255"); + sleep(1); + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness 5"); + } + + //enable auto brightness + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness 100"); + host_exec($atf_host, "%{adb}% -s $device shell settings put system screen_brightness_mode 1"); +} + +function device_put_dir_async(string $atf_host, string $device, string $folder_path, string $device_parent_path, int $timeout = 60) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $folder_path, $device_parent_path, $timeout) { + + if($device_parent_path[0] !== "/") + $device_parent_path = package_dir($atf_host) . $device_parent_path; + + $remote_path = '/tmp/' . basename($folder_path); + + yield taskman\deploy_rsync_async($atf_host, "$folder_path/", "$remote_path/", '--delete'); + + yield host_exec_async($atf_host, "%{adb}% -s $device push --sync $remote_path $device_parent_path", 0, $timeout); + + }); +} + +function device_pull_file_async(string $atf_host, string $device, string $device_path, int $timeout = 10, bool $throw_error_on_timeout = false) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $device_path, $throw_error_on_timeout, $timeout) { + + if($device_path[0] !== "/") + $device_path = package_dir($atf_host) . $device_path; + + $tmp_remote_file = '%{dir}%/'.uniqid(basename($device_path)."_"); + + list($status, $_) = yield host_exec_async($atf_host, "%{adb}% -s $device pull $device_path $tmp_remote_file", DEPLOY_OPT_ERR_OK, $timeout); + + if($status !== 0) + { + if($status === false && $throw_error_on_timeout) + throw new Exception("Could not pull from device '$device' file '$device_path' due to timeout"); + return null; + } + + $file_data = host_get_file($atf_host, $tmp_remote_file); + yield host_exec_async($atf_host, "rm -rf $tmp_remote_file"); + return $file_data; + + }); +} + +function get_last_replay_async(string $atf_host, string $device, string $device_replay_path) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $device_replay_path) { + $file = package_dir($atf_host) . $device_replay_path; + list($status, $contents) = yield host_exec_async($atf_host, "%{adb}% -s $device shell cat $file", DEPLOY_OPT_ERR_OK); + return array($status, implode("\n", $contents)); + }); +} + +function _multi_curl_async($ch, $get_content) : Amp\Promise +{ + return Amp\call(function() use($ch, $get_content) { + $mh = curl_multi_init(); + curl_multi_add_handle($mh, $ch); + + do { + $status = curl_multi_exec($mh, $active); + yield Amp\delay(1); + } while ($active && $status == CURLM_OK); + + $result = null; + if($get_content) + $result = curl_multi_getcontent($ch); + + curl_multi_remove_handle($mh, $ch); + curl_multi_close($mh); + + return $result; + }); +} + +function device_screen(string $atf_host, string $device) +{ + $screen_file_name = '%{dir}%/'.uniqid("screen_").'.png'; + try + { + host_exec($atf_host, "%{adb}% -s $device exec-out screencap -p > $screen_file_name"); + } + catch(Exception $e) + { + return false; + } + $data = host_get_file($atf_host, $screen_file_name); + host_exec($atf_host, "rm -rf $screen_file_name"); + return $data; +} + +function del_external_files_async(string $atf_host, string $device, array $external_files) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $external_files) { + + foreach($external_files as $real_path => $remote_path) + yield device_del_file_async($atf_host, $device, $remote_path); + + }); +} + +function put_external_files_async(string $atf_host, string $device, array $external_files) : Amp\Promise +{ + return Amp\call(function() use($atf_host, $device, $external_files) { + + foreach($external_files as $real_path => $remote_path) + yield device_put_file_async($atf_host, $device, $real_path, $remote_path, true); + + }); +} + +function get_locat_errors_since(string $atf_host, string $device, int $since_stamp) { $time_spec = gmdate('m-d h:m:s.0', $since_stamp); - list($_, $lines) = atf_host_exec("%{adb}% -s $device logcat -s -v UTC -d -t '$time_spec' AndroidRuntime:E Unity:E", DEPLOY_OPT_SILENT | DEPLOY_OPT_ERR_OK); - _atf_filter_error_logs($lines); + list($_, $lines) = host_exec($atf_host, "%{adb}% -s $device logcat -s -v UTC -d -t '$time_spec' AndroidRuntime:E Unity:E", DEPLOY_OPT_ERR_OK); + _filter_error_logs($lines); return implode("\n", $lines); } -function atf_get_logcat_errors($device, $limit = 0) +function get_logcat_errors(string $atf_host, string $device, int $limit = 0) { - list($_, $lines) = atf_host_exec("%{adb}% -s $device logcat -s -d ".($limit > 0 ? "-t $limit" : "")." AndroidRuntime:E Unity:E", DEPLOY_OPT_SILENT | DEPLOY_OPT_ERR_OK); - _atf_filter_error_logs($lines); + list($_, $lines) = host_exec($atf_host, "%{adb}% -s $device logcat -s -d ".($limit > 0 ? "-t $limit" : "")." AndroidRuntime:E Unity:E", DEPLOY_OPT_ERR_OK); + _filter_error_logs($lines); return implode("\n", $lines); } -function atf_get_logcat_unity($device, $limit = 0) +function get_locat_unity_since(string $atf_host, string $device, int $since_stamp) { - list($_, $lines) = atf_host_exec("%{adb}% -s $device logcat -s -d Unity:'*'", DEPLOY_OPT_ERR_OK); + $time_spec = gmdate('m-d h:m:s.0', $since_stamp); + list($_, $lines) = host_exec($atf_host, "%{adb}% -s $device logcat -s -v UTC -d -t '$time_spec' Unity:'*'", DEPLOY_OPT_ERR_OK); + return implode("\n", $lines); +} + +function get_logcat_unity(string $atf_host, string $device, int $limit = 0) +{ + list($_, $lines) = host_exec($atf_host, "%{adb}% -s $device logcat -s -d Unity:'*'", DEPLOY_OPT_ERR_OK); if($limit > 0) array_splice($lines, 0, sizeof($lines) - $limit); return implode("\n", $lines); } -function _atf_filter_error_logs(array &$lines) +function _filter_error_logs(array &$lines) { $exception_lines = 0; $filtered = array(); @@ -1515,62 +469,68 @@ function _atf_filter_error_logs(array &$lines) $lines = $filtered; } -function atf_start_ext_cmd_on_device_async($device, $cmd_main, array $cmd_args) +function device_pss_async(string $atf_host, string $device) : Amp\Promise { - return Amp\call(function() use($device, $cmd_main, $cmd_args) { - - yield atf_device_del_file_async($device, 'ext_status.js', DEPLOY_OPT_ERR_OK); - yield atf_host_exec_async("%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK, 30); - yield atf_update_ext_cmd_async($device, $cmd_main, $cmd_args); - yield atf_host_exec_async("%{adb}% -s $device shell am start -n %{package_id}%/%{activity}%", 0, 30); - + return Amp\call(function() use($atf_host, $device) { + $res = yield device_mem_async($atf_host, $device); + return $res['total']; }); } -function atf_host() +function device_mem_async(string $atf_host, string $device) : Amp\Promise { - static $inited = false; + return Amp\call(function() use($atf_host, $device) { + list($code, $lines) = yield host_exec_async($atf_host, "%{adb}% -s $device shell dumpsys meminfo -s %{package_id}%", DEPLOY_OPT_ERR_OK, 20); - $node_name = 'atf'; - if($inited) - return $node_name; + $res = array( + 'total' => 0, + 'native' => 0, + 'java' => 0, + 'system' => 0, + 'graphics' => 0, + ); - deploy_declare_node($node_name, - array( - 'user' => get("ATF_USER"), - 'dir' => '', - 'ssh_key_str' => get("ATF_KEY"), - 'hosts' => array(get("ATF_HOST")), - 'adb' => get("ATF_HOST_ADB"), - 'package_id' => get("PACKAGE_ID"), - 'activity' => get("LAUNCHER_ACTIVITY") - ) - ); + if($code !== 0) + return $res; - $inited = true; - - return $node_name; -} - -function _atf_opt(array &$args, $opt, $default, $conv_fn = null) -{ - foreach($args as $idx => $arg) - { - if(strpos($arg, $opt) === 0) + foreach($lines as $idx => $line) { - $arg = substr($arg, strlen($opt)); - unset($args[$idx]); - return $conv_fn === null ? $arg : $conv_fn($arg); + $items = preg_split('/\s+/', trim($line)); + if($items && $items[0] === "TOTAL:") + $res['total'] = intval($items[1]); + else if($items && $items[0] === "TOTAL" && $items[1] === "PSS:") + $res['total'] = intval($items[2]); + else if($items && $items[0] === "Native" && $items[1] === "Heap:") + $res['native'] = intval($items[2]); + else if($items && $items[0] === "Java" && $items[1] === "Heap:") + $res['java'] = intval($items[2]); + else if($items && $items[0] === "System:") + $res['system'] = intval($items[1]); + else if($items && $items[0] === "Graphics:") + $res['graphics'] = intval($items[1]); } - } - return $default; + + return $res; + }); } -function _atf_opt_check_no_trailing(array $args) +function device_temperature_async(string $atf_host, string $device) : Amp\Promise { - foreach($args as $arg) - { - if(preg_match('~(--\w+)=.*~', $arg, $m)) - throw new Exception("Unknown option '{$m[1]}'"); - } -} \ No newline at end of file + return Amp\call(function() use($atf_host, $device) { + //NOTE: on current devices 16 thermal zone is responsible for GPU temperature probing + list($code, $lines) = yield host_exec_async($atf_host, "%{adb}% -s $device shell cat /sys/class/thermal/thermal_zone16/temp", DEPLOY_OPT_ERR_OK, 1); + + if($code !== 0) + return 0; + + foreach($lines as $idx => $line) + { + $line = trim($line); + if(!$line) + continue; + return intval($line); + } + + return 0; + }); +} diff --git a/composer.json b/composer.json index c3b7019..5372adc 100644 --- a/composer.json +++ b/composer.json @@ -1,12 +1,16 @@ { "name": "bit/taskman_atf", - "description": "android test farm", + "description": "Automation Test Farm routines", "homepage": "https://git.bit5.ru/composer/taskman_atf", "require": { "php": ">=7.4" }, "autoload": { - "classmap": ["atf.inc.php"], - "classmap": ["slack.inc.php"] + "files": ["atf.inc.php"], + "classmap" : [ + "session.inc.php", "plan.inc.php", + "tasks.inc.php", "device.inc.php", + "im.inc.php", "stats.inc.php" + ] } - } \ No newline at end of file + } diff --git a/device.inc.php b/device.inc.php new file mode 100644 index 0000000..5ea3247 --- /dev/null +++ b/device.inc.php @@ -0,0 +1,161 @@ +atf_host = $atf_host; + $this->max_installs_in_progress = $max_installs_in_progress; + $this->apk_name = $apk_name; + $this->override_external_files = $override_external_files; + $this->external_files = $external_files; + } + + function isInstalled(string $device) + { + return isset($this->installs[$device]) && $this->installs[$device] == self::ST_INSTALLED; + } + + function forgetInstall(string $device) + { + unset($this->installs[$device]); + } + + function countInstallsInProgress() + { + $c = 0; + foreach($this->installs as $status) + if($status == self::ST_IN_PROGRESS) + ++$c; + return $c; + } + + function installAsync(string $device, bool $force = false) : Amp\Promise + { + return Amp\call(function() use($device, $force) { + + while($this->countInstallsInProgress() >= $this->max_installs_in_progress) + yield Amp\delay(1000); + + if($force) + unset($this->installs[$device]); + + if(isset($this->installs[$device])) + return; + $this->installs[$device] = self::ST_IN_PROGRESS; + + try + { + yield host_exec_async($this->atf_host, "%{adb}% -s $device shell am force-stop %{package_id}%", DEPLOY_OPT_ERR_OK); + if($this->apk_name !== null) + { + yield host_exec_async($this->atf_host, "%{adb}% -s $device uninstall %{package_id}%", DEPLOY_OPT_ERR_OK, 30); + yield host_exec_async($this->atf_host, "%{adb}% -s $device install -r %{dir}%/{$this->apk_name}", 0, 300); + + //delete overrides only if there's a new apk file uploaded + yield del_external_files_async($this->atf_host, $device, $this->external_files); + } + + if($this->override_external_files) + yield put_external_files_async($this->atf_host, $device, $this->external_files); + + $this->installs[$device] = self::ST_INSTALLED; + } + catch(Exception $e) + { + err("Error during install *$device*: " . $e->getMessage()); + unset($this->installs[$device]); + throw $e; + } + }); + } + + function getSize() + { + list($status, $lines) = host_exec($this->atf_host, "ls -sd -- %{dir}%/{$this->apk_name}", DEPLOY_OPT_ERR_OK|DEPLOY_OPT_SILENT); + if($status != 0) + return 0; + $items = explode(' ', $lines[0]); + //ls -s returns size in blocks where each block is 512 bytes long + return intval($items[0])*512; + } +} + +class AdbDevicePool implements IDevicePool +{ + private string $atf_host; + + function __construct(string $atf_host) + { + $this->atf_host = $atf_host; + } + + function get() : array + { + return get_devices($this->atf_host); + } +} + +class FixedDevicePool implements IDevicePool +{ + private array $devices; + + function __construct(array $devices) + { + $this->devices = $devices; + } + + function get() : array + { + return $this->devices; + } +} + +class CachedDevices implements IDevicePool +{ + private IDevicePool $provider; + private ?array $cached = null; + private int $last_cache_time; + private int $keep_cache_time; + + function __construct(IDevicePool $provider, int $keep_cache_time) + { + $this->provider = $provider; + $this->keep_cache_time = $keep_cache_time; + } + + function get() : array + { + if($this->cached === null || (time() - $this->last_cache_time) > $this->keep_cache_time) + { + $this->cached = $this->provider->get(); + $this->last_cache_time = time(); + } + return $this->cached; + } +} + diff --git a/im.inc.php b/im.inc.php new file mode 100644 index 0000000..f68b6c3 --- /dev/null +++ b/im.inc.php @@ -0,0 +1,183 @@ + true, 'id' => 1); + } + + function updatePost(string $id, array $fields = array()) : array + { + return array('ok' => true); + } + + function postPNG(string $title, string $png_file, array $fields = array()) : array + { + return array('ok' => true); + } + + function getPermalink(string $id) : string + { + return "https://fake/$id"; + } +} + +class MattermostChan implements IMChan +{ + private string $server; + private string $token; + private string $team; + private string $chan; + + function __construct(string $server, string $token, string $team, string $chan) + { + $this->server = $server; + $this->token = $token; + $this->team = $team; + $this->chan = $chan; + } + + function post(string $message, array $fields = array()) : array + { + return mm_post($this->server, $this->token, $this->chan, $message, $fields); + } + + function updatePost(string $id, array $fields = array()) : array + { + return mm_update($this->server, $this->token, $id, $fields); + } + + function postPNG(string $title, string $png_file, array $fields = array()) : array + { + return mm_post_png($this->server, $this->token, $this->chan, $title, $png_file, $fields); + } + + function getPermalink(string $id) : string + { + return mm_get_permalink($this->server, $this->team, $id); + } +} + +function mm_post(string $server, string $token, string $chan, string $message, $fields = array()) +{ + $ch = _atf_mm_post_curl($server, $token, $chan, $message, $fields); + $result = curl_exec($ch); + curl_close($ch); + + $json = json_decode($result, true); + if(!$json) + return array('ok' => false, 'id' => 0); + return array('ok' => true, 'id' => $json['id']); +} + +function mm_post_async(string $server, string $token, string $chan, string $message, $fields = array()) : Amp\Promise +{ + return Amp\call(function() use($server, $token, $chan, $message, $fields) { + + $ch = _atf_mm_post_curl($server, $token, $chan, $message, $fields); + $result = yield _atf_multi_curl_async($ch, true); + curl_close($ch); + + $json = json_decode($result, true); + if(!$json) + return array('ok' => false, 'id' => 0); + return array('ok' => true, 'id' => $json['id']); + }); +} + +function mm_update(string $server, string $token, string $id, array $fields) +{ + $ch = curl_init("$server/api/v4/posts/$id"); + + $headers = array( + 'Content-Type: application/json', + 'Authorization: Bearer ' . $token + ); + + $fields["post_id"] = $id; + $fields["id"] = $id; + + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'PUT'); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($fields)); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + + $result = curl_exec($ch); + curl_close($ch); + + return json_decode($result, true); +} + +function mm_post_png(string $server, string $token, string $chan, string $title, string $png_file, $msg_fields = array()) +{ + $headers = array( + 'Content-Type: multipart/form-data', + 'Authorization: Bearer ' . $token + ); + + $file = new \CurlFile(realpath($png_file), 'image/png', basename($png_file)); + + $fields = array(); + $fields['channel_id'] = $chan; + $fields['files'] = $file; + + $ch = curl_init("$server/api/v4/files"); + curl_setopt($ch, CURLOPT_POST, 1); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_POSTFIELDS, $fields); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + + $result = curl_exec($ch); + $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); + + if($status != 201) + throw new Exception("File failed with status: " . $status .", result: " . $result); + + $result = json_decode($result, true); + $file_id = $result['file_infos'][0]['id']; + + $msg_fields['file_ids'] = array($file_id); + + return mm_post($server, $token, $chan, $title, $msg_fields); +} + +function mm_get_permalink(string $server, string $team, string $msg_id) +{ + return "$server/$team/pl/$msg_id"; +} + +function _atf_mm_post_curl(string $server, string $token, string $chan, string $message, $fields = array()) +{ + $ch = curl_init("$server/api/v4/posts"); + + $headers = array( + 'Content-Type: application/json', + 'Authorization: Bearer ' . $token + ); + + if(!isset($fields['channel_id'])) + $fields['channel_id'] = $chan; + $fields['message'] = $message; + + curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($fields)); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); + + return $ch; +} diff --git a/plan.inc.php b/plan.inc.php new file mode 100644 index 0000000..07ec4f6 --- /dev/null +++ b/plan.inc.php @@ -0,0 +1,706 @@ +name = $name; + } + + function getTitle() : string + { + $issues_summary = ''; + foreach($this->getProblemsHistogram() as $code => $count) + { + if($code == Task::CODE_GONE) + $issues_summary .= " gones:$count"; + else if($code == Task::CODE_NSTART) + $issues_summary .= " nstrts:$count"; + else if($code == Task::CODE_STUCK) + $issues_summary .= " stucks:$count"; + else if($code == Task::CODE_HUNG) + $issues_summary .= " hungs:$count"; + else if($code == Task::CODE_EXCEPTION) + $issues_summary .= " excepts:$count"; + else if($code == Task::CODE_WARN) + $issues_summary .= " warns:$count"; + } + + $running_count = $this->countRunningTasks(); + $over_count = $this->countFinishedTasks(); + + $predicted_total_duration = $this->getPredictedTotalDuration(); + + $progress_txt = ' tasks:' . ($running_count > 0 ? $running_count . '|' : '') . $over_count . '/' . sizeof($this->tasks) . + ' (' . round($this->getProgress()*100, 2) . '%) (' . + round($this->getDuration()/60, 1) . '/' . ($predicted_total_duration > 0 ? round($predicted_total_duration/60, 1) : '?') . + ' min)'; + + return "({$this->session->id}) Plan '{$this->name}'" . $issues_summary . $progress_txt; + } + + function getDuration() : int + { + return time() - $this->start_time; + } + + function getPredictedTotalDuration() + { + if($this->getDuration() <= 0) + return 0; + + if($this->getProgress() <= 0) + return 0; + + return $this->getDuration() / $this->getProgress(); + } + + function _createMessengerThread() + { + if($this->im_thread_id) + return; + + for($i=0;$i<5;++$i) + { + $resp = $this->session->conf->im->post($this->getTitle(), $this->_getMessengerThreadProps()); + if(isset($resp['ok'])) + { + $this->im_thread_id = $resp['id']; + return; + } + sleep(1); + } + throw new Exception("Could not create IM thread"); + } + + function _updateMessengerThread(bool $force) + { + if(!$force && (time() - $this->im_thread_last_update) < self::IM_THREAD_UPDATE_INTERVAL) + return; + $this->im_thread_last_update = time(); + + if(!$this->im_thread_id) + return; + + $this->session->conf->im->updatePost($this->im_thread_id, $this->_getMessengerThreadProps()); + } + + function _getMessengerThreadProps() : array + { + return ['props' => ['attachments' => [ + [ + 'text' => '* ' . $this->getTitle(), + 'color' => $this->_getThreadColor() + ] + ]]]; + } + + function addStatusCode($code, $msg = '') + { + $this->status_codes[] = array($code, $msg); + } + + function getProblemsHistogram() + { + $histogram = array(); + foreach($this->status_codes as $item) + { + list($code, $_) = $item; + if(!Task::isProblemCode($code)) + continue; + + if(!isset($histogram[$code])) + $histogram[$code] = 0; + ++$histogram[$code]; + } + return $histogram; + } + + function hasStatusCode($code) : bool + { + return $this->countStatusCode($code) > 0; + } + + function countStatusCode($code) : int + { + $c = 0; + foreach($this->status_codes as $item) + { + if($item[0] === $code) + $c++; + } + return $c; + } + + function getStats() : Stats + { + $stats = new Stats(); + + $stats->gones = $this->countStatusCode(Task::CODE_GONE); + $stats->hungs = $this->countStatusCode(Task::CODE_HUNG); + $stats->stucks = $this->countStatusCode(Task::CODE_STUCK); + $stats->nostarts = $this->countStatusCode(Task::CODE_NSTART); + $stats->excepts = $this->countStatusCode(Task::CODE_EXCEPTION); + $stats->warns = $this->countStatusCode(Task::CODE_WARN); + + return $stats; + } + + function _getThreadColor() : string + { + if(!$this->session->getDevices()) + return "#000000"; + + $stats = $this->getStats(); + return $stats->getColor($this->isOver()); + } + + function addTask(Task $task) + { + $task->plan = $this; + $this->tasks[] = $task; + } + + function isOver() : bool + { + if($this->hasTooManyFatalProblems()) + return true; + + foreach($this->tasks as $task) + { + if(!$task->isDone()) + return false; + } + return true; + } + + function hasTooManyFatalProblems() : bool + { + return + ($this->countStatusCode(Task::CODE_EXCEPTION) >= self::EXCEPTIONS_THRESHOLD) || + ($this->countStatusCode(Task::CODE_STUCK) >= self::STUCK_THRESHOLD) || + ($this->countStatusCode(Task::CODE_GONE) >= self::GONE_THRESHOLD) || + ($this->countStatusCode(Task::CODE_HUNG) >= self::HUNG_THRESHOLD) + ; + } + + function getProgress() : float + { + $total_progress = 0; + foreach($this->tasks as $task) + $total_progress += $task->getProgress(); + return $total_progress / sizeof($this->tasks); + } + + function countRunningTasks() : int + { + $running_count = 0; + foreach($this->tasks as $task) + { + if($task->device) + $running_count++; + } + return $running_count; + } + + function countFinishedTasks() : int + { + $done_count = 0; + foreach($this->tasks as $task) + { + if($task->isDone()) + $done_count++; + } + return $done_count; + } + + function runTaskAsync(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + + yield $this->_startAsync($task); + + while(!$this->isOver()) + { + $this->_updateMessengerThread(false); + + yield Amp\delay((int)($this->session->conf->sleep_time*1000)); + + log("Task #{$task->run_id} {$task->cmd}(".sizeof($task->args).") progress {$task->getProgress()}, last ext time {$task->last_ext_status_item_time} *{$task->device}*..."); + + $check_error = yield $this->_tryCheckExtStatusAsync($task, /*attempts*/3, /*timeout*/20); + if($check_error !== null) + { + //something wrong with device, let's ignore the device and reschedule the task + $this->_ignoreDevice($task->device, $check_error); + $task->reschedule(); + break; + } + + if(!$task->isDone()) + yield $this->_checkHealthAsync($task); + + if($task->hasFatalProblem()) + { + yield $this->_processFatalProblemAsync($task); + break; + } + else if($task->isDone()) + { + //let's reset bogus stats for this device + $this->session->resetBogusDeviceCount($task->device); + //let's free the device + $task->device = null; + break; + } + } + + $this->_updateMessengerThread(true); + }); + } + + function _onStart() + { + $this->start_time = time(); + + log("Starting " . $this->getTitle() . "..."); + + $this->_createMessengerThread(); + } + + function _startAsync(Task $task) : Amp\Promise + { + //let's create a Slack thread synchronously if it's not present yet + if(!$this->im_thread_id) + $this->_onStart(); + + return Amp\call(function() use($task) { + + try + { + log("Preparing *{$task->device}*"); + + yield $this->session->apk_installer->installAsync($task->device); + yield start_ext_cmd_on_device_async( + $this->session->conf->atf_host, + $task->device, + $task->getCmd(), + $task->getCmdArgs() + ); + } + catch(Exception $e) + { + $this->_ignoreDevice($task->device, $e->getMessage()); + $task->reschedule(); + return false; + } + + $task->start(); + return true; + + }); + } + + function _ignoreDevice($device, $reason) + { + $this->session->ignoreDevice($device, $reason); + + $this->post("Ignoring device *$device*: $reason"); + } + + function _processFatalProblemAsync(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + + $fatal_msg = "Fatal problem ({$task->getLastFatalProblem()} - ".Task::code2string($task->getLastFatalProblem())."), attempt:{$task->attempts} *{$task->device}*"; + + err("[FTL] $fatal_msg"); + + $this->post($fatal_msg); + + $this->_postScreenToMessenger($task); + + if($task->getLastFatalProblem() == Task::CODE_EXCEPTION) + yield $this->_postLastReplayToMessengerAsync($task); + else if($task->getLastFatalProblem() == Task::CODE_GONE || + $task->getLastFatalProblem() == Task::CODE_HUNG) + { + $app_log = get_logcat_unity($this->session->conf->atf_host, $task->device, 300); + $app_log = _atf_trim_start($app_log, 2000); + $this->post("Last logs: \n```\n$app_log\n```\n *{$task->device}*"); + } + + if($this->session->conf->error_pause) + $this->_ignoreDevice($task->device, "Ignored for error investigation"); + //Let's reinstall application on this device if it's failing too often + else + $this->_incAndCheckBogusDevice($task->device); + + $task->onFatalProblem(); + + //let's free the device anyway, it's up to Task::onFatalProblem() to reschedule it or not + $task->device = null; + }); + } + + function _tryCheckExtStatusAsync(Task $task, int $attempts, int $timeout) : Amp\Promise + { + return Amp\call(function() use($task, $attempts, $timeout) { + + $last_error = null; + + for($i=0;$i<$attempts;++$i) + { + try + { + yield $this->_checkExtStatusAsync($task, $timeout); + break; + } + catch(Exception $e) + { + $last_error = $e->getMessage(); + continue; + } + } + + return $last_error; + + }); + } + + function _checkExtStatusAsync(Task $task, int $timeout) : Amp\Promise + { + return Amp\call(function() use($task, $timeout) { + + $ext_status = yield get_ext_status_async($this->session->conf->atf_host, $task->device, $timeout); + if(!is_array($ext_status)) + return; + + $new_items = $this->_getExtStatusItemsSince($task->last_ext_status_item_time, $ext_status); + + if(!$new_items) + return; + + $task->last_ext_status_item_time = end($new_items)['time']; + + foreach($new_items as $item) + yield $this->_analyzeExtStatusItemAsync($task, $item); + + }); + } + + function _getExtInputRequestsSince($id, array $all_reqs) : array + { + $new_reqs = array(); + foreach($all_reqs as $req) + { + if($req['id'] > $id) + $new_reqs[] = $req; + } + return $new_reqs; + } + + function _getExtStatusItemsSince($time, array $ext_status) : array + { + $new_items = array(); + foreach($ext_status['entries'] as $item) + { + if($item['time'] > $time) + $new_items[] = $item; + } + return $new_items; + } + + function _checkIfAppRunning(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + list($status, $_) = + yield host_exec_async($this->session->conf->atf_host, "%{adb}% -s {$task->device} shell pidof %{package_id}%", DEPLOY_OPT_ERR_OK); + return $status == 0; + }); + } + + function _reportGone(Task $task) + { + //check it application hasn't even started + if($task->last_alive_check_time === $task->reset_time) + { + $task->addStatusCode(Task::CODE_NSTART); + $this->session->trySendStatsEvent($task, 'nstart'); + err("[NFD] No app started after {$this->session->conf->dead_threshold} seconds *{$task->device}*"); + } + else + { + $task->addStatusCode(Task::CODE_GONE); + $this->session->trySendStatsEvent($task, 'gone'); + err("[GNE] App is gone after {$this->session->conf->dead_threshold} seconds *{$task->device}*"); + } + } + + function _checkHealthAsync(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + + $not_alive_time = microtime(true) - $task->last_alive_check_time; + $stuck_time = microtime(true) - $task->last_stuck_check_time; + + if($not_alive_time > $this->session->conf->dead_threshold) + { + if(yield $this->_checkIfAppRunning($task)) + { + $task->addStatusCode(Task::CODE_HUNG); + $this->session->trySendStatsEvent($task, 'hung'); + err("[HNG] No activity for {$this->session->conf->dead_threshold} seconds *{$task->device}*"); + } + else + $this->_reportGone($task); + + $this->_reportErrorFromLogcatToMessenger($task, 1000); + } + else if($stuck_time > $this->session->conf->stuck_threshold) + { + if(yield $this->_checkIfAppRunning($task)) + { + $task->addStatusCode(Task::CODE_STUCK); + $this->session->trySendStatsEvent($task, 'stuck'); + err("[STK] Stuck for {$this->session->conf->stuck_threshold} seconds *{$task->device}*"); + } + else + $this->_reportGone($task); + } + + }); + } + + function _incAndCheckBogusDevice(string $device) + { + $count = $this->session->incBogusDevice($device); + + if($count > self::BOGUS_DEVICE_COUNT_THRESHOLD) + { + $this->_ignoreDevice($device, "Too many bogus errors"); + } + else if(($count % self::BOGUS_REINSTALL_APP_EVERY_N) == 0) + { + $this->session->apk_installer->forgetInstall($device); + } + } + + static function _parseExtMessage(string $message) + { + $msg_code = null; + $msg_text = $message; + //example: [DBG] this a debug message + if(preg_match('~^(\[[^\]]+\])(.*)$~', $message, $matches)) + { + $msg_code = $matches[1]; + $msg_text = $matches[2]; + } + return array($msg_code, $msg_text); + } + + static function _printToShellExtItem(Task $task, array $item) + { + $shell_msg = _atf_trim($item['message'], 200); + if(Task::isProblemCode($item['error'])) + $shell_msg = "[PRB] Code:{$item['error']}, $shell_msg"; + $shell_msg = "(".round($item['time'],1)."s) {$shell_msg} *{$task->device}*"; + + log($shell_msg); + } + + function _postToMessengerExtStatusItem(Task $task, array $item) + { + $orig_msg = _atf_trim($item['message'], 3000); + $mm_msg = $orig_msg; + + if($item['error'] == Task::CODE_EXCEPTION) + $mm_msg = "\n```\n$mm_msg\n```\n"; + + $mm_msg = '('.round($item['time'],1).'s) '.$mm_msg.' *'.$task->device.'*'; + + $resp = $this->post($mm_msg); + + if(isset($resp['ok']) && $item['error'] == Task::CODE_EXCEPTION) + $this->session->tryShareToQAChannel($resp['id'], $orig_msg); + } + + function post($msg, array $args = array()) + { + $args['root_id'] = $this->im_thread_id; + return $this->session->conf->im->post($msg, $args); + } + + function _analyzeExtStatusItemAsync(Task $task, array $item) : Amp\Promise + { + return Amp\call(function() use($task, $item) { + + self::_printToShellExtItem($task, $item); + + $task->addStatusCode($item['error'], $item['message']); + + if($item['error'] == Task::CODE_EXCEPTION) + $this->session->trySendStatsEvent($task, 'exception'); + + list($msg_type, $msg_text) = self::_parseExtMessage($item['message']); + + //NOTE: alive system messages are sent periodically by ExtBot itself, + // we'd like to ignore them for stuck detection + if($msg_type !== '[ALIVE]') + $task->last_stuck_check_time = microtime(true); + + //NOTE: in case of any message from the device we update the alive check timestamp + $task->last_alive_check_time = microtime(true); + + yield $this->_processExtStatusMessageAsync($task, $item, $msg_type, $msg_text); + + }); + } + + function _processExtStatusMessageAsync($task, $item, $msg_type, $msg_text) : Amp\Promise + { + return Amp\call(function() use($task, $item, $msg_type, $msg_text) { + + if($item['error'] == Task::CODE_EXCEPTION) + { + $this->_postToMessengerExtStatusItem($task, $item); + } + else if($msg_type === '[WRN]') + { + $task->addStatusCode(Task::CODE_WARN); + $this->_postToMessengerExtStatusItem($task, $item); + } + else if($msg_type === '[PRG]') + { + //let's reset hung stats since progress is going + $this->session->resetBogusDeviceCount($task->device); + $task->onProgress($msg_text); + } + else if($msg_type === '[STAT]') + yield $this->session->trySendStatsFromJzonAsync($task, $msg_text); + else if($msg_type === '[RPL]') + $this->_postReplayToMessenger($task, $msg_text); + + }); + } + + function _postLastReplayToMessengerAsync(Task $task) : Amp\Promise + { + return Amp\call(function() use($task) { + + if($this->session->conf->device_replay_path) + { + list($repl_err, $repl_txt) = + yield get_last_replay_async($this->session->conf->atf_host, $task->device, $this->session->conf->device_replay_path); + if($repl_err === 0) + $this->_postReplayToMessenger($task, $repl_txt); + } + + }); + } + + function _postReplayToMessenger(Task $task, $repl_txt) + { + $repl_txt = _atf_try_lz4_replay($repl_txt); + $this->post("Last Replay: \n```\n$repl_txt\n```\n *{$task->device}*"); + } + + function _reportErrorFromLogcatToMessenger(Task $task, $limit) + { + $errors_log = _atf_trim(get_logcat_errors($task->device, $limit), 3950); + if($errors_log) + $this->post("\n```\n$errors_log\n```\n *{$task->device}*"); + } + + function _postScreenToMessenger(Task $task) + { + global $GAME_ROOT; + + $png_data = device_screen($this->session->conf->atf_host, $task->device); + if($png_data) + { + $tmp_png = tempnam(sys_get_temp_dir(), 'device_screen') . '.png'; + taskman\ensure_write($tmp_png, $png_data); + try + { + $this->session->conf->im->postPNG("screen: ".$task->device, $tmp_png, array('root_id' => $this->im_thread_id)); + } + catch(Exception $e) + {} + finally + { + taskman\ensure_rm($tmp_png); + } + } + } +} + +class Stats +{ + public int $gones = 0; + public int $hungs = 0; + public int $stucks = 0; + public int $nostarts = 0; + public int $excepts = 0; + public int $warns = 0; + + function add(Stats $other) + { + $this->gones += $other->gones; + $this->hungs += $other->hungs; + $this->stucks += $other->stucks; + $this->nostarts += $other->nostarts; + $this->excepts += $other->excepts; + $this->warns += $other->warns; + } + + function getColor(bool $is_over) : string + { + if($this->gones > 0 || + $this->hungs > 0 || + $this->warns > 0 || + $this->excepts > 0 || + $this->stucks > 0 || + $this->nostarts > 0 + ) + { + if($this->excepts) + return $is_over ? "danger" : "#FFCCCB"; + else + return $is_over ? "warning" : "#FFCF9E"; + } + else if($is_over) + return "good"; + else + return "#D3D3D3"; + } +} + + diff --git a/session.inc.php b/session.inc.php new file mode 100644 index 0000000..f110347 --- /dev/null +++ b/session.inc.php @@ -0,0 +1,492 @@ +im = new NullMessenger(); + $this->im_qa = new NullMessenger(); + + $this->stats = new NullStatsSender(); + } + + function initHost(array $settings) + { + $this->atf_host = 'atf'; + taskman\deploy_declare_node($this->atf_host, $settings); + } +} + +class Session +{ + const RUN_SLEEP = 3; + + public string $name; + public SessionConfig $conf; + + public int $start_time; + + public IDevicePool $device_pool; + public ApkInstaller $apk_installer; + + public string $id; + public string $guid; + + public string $im_thread_id = ''; + + public array $plans = array(); + + public array $ignored_devices = array(); + public array $bogus_device2counter = array(); + + public array $shared_qa_errors = array(); + + static function getId() : string + { + $id = getenv('ATF_SESSION_ID'); + if($id) + return $id; + //sort of graceful fallback + return ''.getmypid(); + } + + function __construct( + string $name, + SessionConfig $conf, + IDevicePool $device_pool + ) + { + $this->id = self::getId(); + + $this->name = $name; + $this->conf = $conf; + + $this->guid = uniqid(); + + $this->device_pool = $device_pool; + } + + function getTitle() : string + { + return "Session ({$this->id}) '{$this->name}' (devices:".sizeof($this->getDevices())." ?:".sizeof($this->ignored_devices) . + ') (' . round($this->getDuration()/60, 1) . ' min)'; + } + + function getDuration() : int + { + return time() - $this->start_time; + } + + function getDevices() : array + { + $devices = array_diff( + $this->device_pool->get(), + $this->ignored_devices + ); + return $devices; + } + + function ignoreDevice(string $device, string $reason) + { + err("Ignoring device *$device*: $reason"); + + $this->ignored_devices[] = $device; + } + + function incBogusDevice(string $device) + { + if(!isset($this->bogus_device2counter[$device])) + $this->bogus_device2counter[$device] = 0; + return ++$this->bogus_device2counter[$device]; + } + + function resetBogusDeviceCount(string $device) + { + $this->bogus_device2counter[$device] = 0; + } + + function resetBogusDevices() + { + $this->bogus_device2counter = array(); + } + + function addPlan(Plan $plan) + { + $plan->session = $this; + $this->plans[] = $plan; + } + + function resetPlans() + { + $this->plans = array(); + } + + function runAsync() : Amp\Promise + { + return Amp\call(function() { + + $this->start_time = time(); + + $apk_name = !$this->conf->apk_path ? null : basename($this->conf->apk_path); + $this->apk_installer = new ApkInstaller( + $this->conf->atf_host, + $this->conf->max_adb_installs_in_progress, + $apk_name, + $this->conf->override_external_files, + $this->conf->external_files + ); + + $this->_createMessengerThread(); + + //if it's not set the last one will be re-used + if($this->conf->apk_path) + $this->_deployApkToHost($apk_name); + + if($this->conf->reboot_devices) + reboot_devices($this->conf->atf_host, $this->getDevices()); + + if($this->conf->adb_reboot) + adb_reboot($this->conf->atf_host); + + $this->trySendApkStatsEvent($this->apk_installer->getSize()); + + $all_tasks = $this->_getAllTasks(); + + return yield $this->_runTasksAsync($all_tasks); + + }); + } + + function _createMessengerThread() + { + for($i=0;$i<5;++$i) + { + $resp = $this->conf->im->post("", $this->_getMessengerThreadProps()); + if(isset($resp['ok'])) + { + $this->im_thread_id = $resp['id']; + return; + } + sleep(1); + } + throw new Exception("Could not create IM thread"); + } + + function _updateMessengerThread() + { + $this->conf->im->updatePost( + $this->im_thread_id, + $this->_getMessengerThreadProps() + ); + } + + function _getMessengerThreadProps() : array + { + return ['props' => ['attachments' => [ + [ + 'text' => $this->getTitle(), + 'color' => $this->_getThreadColor() + ] + ]]]; + } + + function _getThreadColor() : string + { + if(!$this->getDevices()) + return "#000000"; + + $stats = new Stats(); + foreach($this->plans as $plan) + $stats->add($plan->getStats()); + + return $stats->getColor($this->isOver()); + } + + function _deployApkToHost($apk_name) + { + if(!is_file($this->conf->apk_path)) + throw new Exception("No such file '{$this->conf->apk_path}'"); + + host_put_file($this->conf->atf_host, $this->conf->apk_path, "%{dir}%/$apk_name"); + } + + function _getAllTasks() + { + $all_tasks = array(); + foreach($this->plans as $plan) + { + foreach($plan->tasks as $task) + $all_tasks[] = $task; + } + return $all_tasks; + } + + function _runTasksAsync(array $all_tasks) : Amp\Promise + { + return Amp\call(function() use($all_tasks) { + + $active_tasks = array(); + + while(!$this->isOver() && (sizeof($all_tasks) > 0 || sizeof($active_tasks) > 0)) + { + $free_devices = $this->_findFreeDevices($active_tasks); + + $this->_assignTasksToDevices($all_tasks, $free_devices, $active_tasks); + + yield Amp\delay(self::RUN_SLEEP*1000); + + $this->_processDoneTasks($all_tasks, $active_tasks); + + $this->_updateMessengerThread(); + } + + return sizeof($all_tasks) == 0 && sizeof($active_tasks) == 0; + + }); + } + + function _assignTasksToDevices(array &$tasks, array $devices, array &$active_tasks) + { + static $next_run_id = 0; + + //create new coroutines and assign them to free devices + foreach($devices as $device) + { + if(sizeof($tasks) == 0) + break; + + $task = array_shift($tasks); + + //explicitly assigning a device + $task->device = $device; + + $task->run_id = ++$next_run_id; + $task->run_status = false; + + $active_tasks[$task->run_id] = $task; + + $task->plan->runTaskAsync($task)->onResolve( + function($error, $value) use($task) + { + $task->run_status = true; + $task->run_error = $error; + } + ); + } + } + + function _processDoneTasks(array &$tasks, array &$active_tasks) + { + $tmp_tasks = array(); + foreach($active_tasks as $run_id => $task) + { + //task not in done ids, let's keep it + if(!$task->run_status) + { + $tmp_tasks[$run_id] = $task; + } + else if($task->run_error) + { + throw $task->run_error; + } + //task is done, let's check if it had a problem and if yes + //let's put it back into unscheduled tasks + else if($task->need_reschedule) + { + array_unshift($tasks, $task); + $task->need_reschedule = false; + } + } + + $active_tasks = $tmp_tasks; + } + + function _findFreeDevices(array $active_tasks) + { + $free = array(); + + foreach($this->getDevices() as $device) + { + //let's skip invalid devices + if(!$device) + continue; + + $busy = false; + foreach($active_tasks as $task) + { + if($task->device == $device) + { + $busy = true; + break; + } + } + if(!$busy) + $free[] = $device; + } + + return $free; + } + + function isOver() : bool + { + if(!$this->getDevices()) + return true; + + foreach($this->plans as $plan) + if(!$plan->isOver()) + return false; + + return true; + } + + function trySendStatsFromJzonAsync(Task $task, string $jzon) : Amp\Promise + { + return Amp\call(function() use($task, $jzon) { + + try + { + $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); + $table = $data['table']; + unset($data['table']); + + $data = yield $this->inflateMissingStats($task, $table, $data); + + $this->trySendStats($task, $table, $data); + } + catch(Exception $e) + { + echo $e->getMessage() . "\n"; + } + + }); + } + + //TODO: make it more flexible via plugin alike system? + function inflateMissingStats(Task $task, string $table, array $data) : Amp\Promise + { + return Amp\call(function() use($task, $table, $data) { + //NOTE: filling memory usage if it's not set + if(isset($data['deviceMemoryUsage']) && $data['deviceMemoryUsage'] === '') + { + $mem = yield device_mem_async($this->conf->atf_host, $task->device); + $data['deviceMemoryUsage'] = $mem['total']; + if($table === 'device_memory') + { + $data['deviceMemoryUsageNative'] = $mem['native']; + $data['deviceMemoryUsageSystem'] = $mem['system']; + $data['deviceMemoryUsageJava'] = $mem['java']; + $data['deviceMemoryUsageGraphics'] = $mem['graphics']; + } + } + + //NOTE: filling CPU temperature + if(isset($data['event']) && $data['event'] == 'cpuTemp') + $data['value'] = yield device_temperature_async($this->conf->atf_host, $task->device); + + return $data; + }); + } + + function trySendApkStatsEvent(int $apk_size) + { + try + { + $data['guid'] = $this->guid; + $data['time'] = time(); + $data['version'] = $this->conf->proj_version; + $data['revHash'] = $this->conf->proj_rev_hash; + $data['branch'] = $this->conf->proj_branch; + $data['event'] = 'apk_size'; + $data['value'] = $apk_size; + $this->conf->stats->send('event_value', $data); + } + catch(Exception $e) + { + echo $e->getMessage() . "\n"; + } + } + + function trySendStats(Task $task, string $table, array $data) + { + try + { + $data['guid'] = $this->guid; + $data['time'] = time(); + $data['deviceId'] = $task->device; + $this->conf->stats->send($table, $data); + } + catch(Exception $e) + { + echo $e->getMessage() . "\n"; + } + } + + function trySendStatsEvent(Task $task, string $event, int $value = 1) + { + $this->trySendStats( + $task, + 'event_value', + array( + 'event' => $event, + 'value' => $value, + 'branch' => $this->conf->proj_branch, + 'version' => $this->conf->proj_version, + 'revHash' => $this->conf->proj_rev_hash + ) + ); + } + + function tryShareToQAChannel($msg_id, $error) + { + if(!$this->conf->share_with_qa_chan) + return; + + //let's skip similar already shared errors + if($this->_calcSharedErrorSimilarity($error) > 80) + return; + + $this->shared_qa_errors[] = $error; + + //let's share the exception message to the regency-qa channel + $permalink = $this->conf->im->getPermalink($msg_id); + $this->conf->im_qa->post($permalink); + } + + function _calcSharedErrorSimilarity($error) + { + $max_perc = 0; + foreach($this->shared_qa_errors as $shared) + { + $perc = 0; + similar_text($shared, $error, $perc); + if($perc > $max_perc) + $max_perc = $perc; + } + return $max_perc; + } +} diff --git a/slack.inc.php b/slack.inc.php deleted file mode 100644 index e027d42..0000000 --- a/slack.inc.php +++ /dev/null @@ -1,180 +0,0 @@ - $token, - 'channels' => implode(",", $channels), - 'file' => $cfile, - 'filename' => "test.jpg", - ); - - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HEADER, false); - curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, $fields); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - curl_setopt($ch, CURLOPT_VERBOSE, true); - $result = curl_exec($ch); - curl_close($ch); - - $arr = json_decode($result, true); - if(isset($arr['ok']) && $arr['ok'] == 1) - { - if(isset($arr['file']['permalink_public'])) - return $arr['file']['permalink_public']; - } - - return ""; -} - -function _atf_slack_chan() -{ - return get("ATF_SLACK_CHANNEL"); -} - -function _atf_slack_chan_qa() -{ - return get("ATF_SLACK_CHANNEL"); -} - -function _atf_slack_token() -{ - $token = "xoxb-141599046048-1567053090644-2gDAoWGiZxFTtXGLMOOeLG2u"; - return $token; -} - -task('slack_test', function() -{ - // createSlackThread(); - // atf_slack_post('hi', array('channel' => _atf_slack_chan_qa())); -}); - -function atf_slack_post($message, $fields = array()) -{ - if(!get("ATF_SLACK")) - return array('ok' => true, 'ts' => 0); - - $ch = curl_init("https://slack.com/api/chat.postMessage"); - $fields["token"] = _atf_slack_token(); - if(!isset($fields['channel'])) - $fields["channel"] = _atf_slack_chan(); - $fields["text"] = $message; - - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); - curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields)); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - - $result = curl_exec($ch); - curl_close($ch); - - return json_decode($result, true); -} - -function atf_slack_get_permalink($message_ts, $channel = null) -{ - if(!get("ATF_SLACK")) - return array('permalink' => ''); - - $ch = curl_init("https://slack.com/api/chat.getPermalink"); - $fields["token"] = _atf_slack_token(); - if($channel === null) - $channel = _atf_slack_chan(); - $fields["channel"] = $channel; - $fields["message_ts"] = $message_ts; - - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); - curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields)); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - - $result = curl_exec($ch); - curl_close($ch); - - return json_decode($result, true); -} - -function atf_slack_update($ts, array $attachments) -{ - if(!get("ATF_SLACK")) - return array('ok' => true); - - $ch = curl_init("https://slack.com/api/chat.update"); - $fields["token"] = _atf_slack_token(); - $fields["channel"] = _atf_slack_chan(); - $fields["ts"] = $ts; - $fields["attachments"] = json_encode($attachments); - - curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST'); - curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query($fields)); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); - - $result = curl_exec($ch); - curl_close($ch); - - return json_decode($result, true); -} - - -function atf_slack_post_png($title, $png_file, $fields = array()) -{ - if(!get("ATF_SLACK")) - return array('ok' => true); - - $file = new CurlFile($png_file, 'image/png'); - - $header = array(); - $header[] = 'Content-Type: multipart/form-data'; - - $fields["token"] = _atf_slack_token(); - $fields["channels"] = _atf_slack_chan(); - $fields['file'] = $file; - $fields['title'] = $title; - - $ch = curl_init("https://slack.com/api/files.upload"); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_HTTPHEADER, $header); - curl_setopt($ch, CURLOPT_POST, 1); - curl_setopt($ch, CURLOPT_POSTFIELDS, $fields); - - $result = curl_exec($ch); - curl_close($ch); - - return json_decode($result, true); -} \ No newline at end of file diff --git a/stats.inc.php b/stats.inc.php new file mode 100644 index 0000000..e8751ea --- /dev/null +++ b/stats.inc.php @@ -0,0 +1,56 @@ +host = $host; + $this->port = $port; + $this->username = $username; + $this->password = $password; + $this->dbname = $dbname; + } + + function getClient() : ClickHouseDB\Client + { + $config = [ + 'host' => $this->host, + 'port' => $this->port, + 'username' => $this->username, + 'password' => $this->password, + ]; + + $db = new ClickHouseDB\Client($config); + if($this->dbname) + $db->database($this->dbname); + $db->setTimeout(15); + $db->setConnectTimeOut(10); + return $db; + } + + function send(string $table, array $data) + { + $this->getClient()->insert($table, array(array_values($data)), array_keys($data)); + } +} + diff --git a/task.inc.php b/task.inc.php new file mode 100644 index 0000000..28896ce --- /dev/null +++ b/task.inc.php @@ -0,0 +1,308 @@ +module = $module; + $this->func = $func; + } + + function __toString() + { + return $this->module.':'.$this->func; + } +} + +class Task +{ + const CODE_MESSAGE = 0; + const CODE_EXCEPTION = 1; + const CODE_WARN = 2; + const CODE_NSTART = 124; + const CODE_STUCK = 125; + const CODE_GONE = 126; + const CODE_HUNG = 127; + const CODE_DONE = 128; + + static function code2string(int $code) : string + { + switch($code) + { + case self::CODE_MESSAGE: + return "message"; + case self::CODE_EXCEPTION: + return "exception"; + case self::CODE_WARN: + return "warning"; + case self::CODE_STUCK: + return "stuck"; + case self::CODE_NSTART: + return "nstart"; + case self::CODE_GONE: + return "gone"; + case self::CODE_HUNG: + return "hung"; + case self::CODE_DONE: + return "done"; + } + return "???"; + } + + public ?Plan $plan = null; + + //async execution id + public int $run_id = 0; + //'true' if async coroutine is resolved + public bool $run_status = false; + //async execution exception if any + public ?object $run_error = null; + + //attached device + public ?string $device = null; + + public int $attempts = 1; + //signal to dispatcher to re-start execution of the task + public bool $need_reschedule = false; + + public Cmd $cmd; + public array $args = array(); + + public string $title = ''; + + public float $start_time = 0; + public float $reset_time = 0; + + public float $last_progress = 0; + public int $last_done_arg_idx = -1; + public float $last_alive_check_time = 0; + public float $last_stuck_check_time = 1; + public float $last_ext_status_item_time = 0; + //array of tuples [code, msg] + public array $status_codes = array(); + + function __construct(Cmd $cmd, array $args, $title = '') + { + $this->cmd = $cmd; + $this->args = $args; + $this->title = $title; + } + + function getCmd() : Cmd + { + return $this->cmd; + } + + function getCmdArgs() : array + { + return $this->args; + } + + function getProgress() : float + { + if($this->last_progress == 0 && $this->isDone()) + return 1; + return $this->last_progress; + } + + static function isProblemCode($code) : bool + { + return $code > self::CODE_MESSAGE && $code < self::CODE_DONE; + } + + static function isFatalCode($code) : bool + { + return + $code == Task::CODE_STUCK || + $code == Task::CODE_NSTART || + $code == Task::CODE_GONE || + $code == Task::CODE_HUNG || + $code == Task::CODE_EXCEPTION; + } + + function hasFatalProblem() : bool + { + return $this->getLastFatalProblem() != 0; + } + + function getLastFatalProblem() : int + { + for($i=sizeof($this->status_codes);$i-- > 0;) + { + $item = $this->status_codes[$i]; + if(self::isFatalCode($item[0])) + return $item[0]; + } + return 0; + } + + function hasStatusCode($code) + { + foreach($this->status_codes as $item) + { + if($item[0] === $code) + return true; + } + return false; + } + + function isDone() + { + return $this->hasStatusCode(Task::CODE_DONE); + } + + function addStatusCode($code, $msg = '') + { + //plan stores history of all status codes + $this->plan->addStatusCode($code, $msg); + + $this->status_codes[] = array($code, $msg); + } + + function onFatalProblem() + { + if($this->attempts < 3) + $this->reschedule(); + } + + function onProgress($jzon) + { + try + { + $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); + if(isset($data['p'])) + $this->last_progress = floatval(str_replace(',', '.', $data['p'])); + if(isset($data['arg_idx'])) + $this->last_done_arg_idx = (int)$data['arg_idx']; + } + catch(Exception $e) + { + echo $e->getMessage() . "\n"; + } + } + + function _resetCheckTimes() + { + $this->reset_time = microtime(true); + $this->last_alive_check_time = $this->reset_time; + $this->last_stuck_check_time = $this->reset_time; + $this->last_ext_status_item_time = 0; + } + + function reschedule() + { + ++$this->attempts; + $this->device = null; + $this->need_reschedule = true; + } + + function start() + { + $this->_resetCheckTimes(); + $this->status_codes = array(); + $this->start_time = time(); + } + + function getDuration() + { + return time() - $this->start_time; + } +} + +class TestLevelsTask extends Task +{ + public $arg_offset = 0; + + private $rseeds = array(); + private $seed = null; + private $timescale = null; + + function __construct(Cmd $cmd, array $args, $with_rseeds = false, $seed = null, $timescale = null, $title = '') + { + $level_ids = array(); + //TODO: should not really be here + if($with_rseeds) + { + foreach($args as $item) + { + $level_ids[] = $item[0]; + $this->rseeds[] = $item[1]; + } + } + else + $level_ids = $args; + + $this->seed = $seed; + $this->timescale = $timescale; + + parent::__construct($cmd, $level_ids, $title); + } + + function getCmdArgs() : array + { + $args = array(); + + if($this->rseeds) + { + foreach($this->args as $idx => $arg) + { + $args[] = $arg; + $args[] = $this->rseeds[$idx]; + } + } + else + $args = $this->args; + + //appending arg. offset as a first argument + array_unshift($args, ''.$this->arg_offset); + array_unshift($args, '--offset'); + + if($this->rseeds) + { + array_unshift($args, 'true'); + array_unshift($args, '--levels-with-rseed'); + } + + if($this->seed !== null) + { + array_unshift($args, ''.$this->seed); + array_unshift($args, '--seed'); + } + + if($this->timescale !== null) + { + array_unshift($args, ''.$this->timescale); + array_unshift($args, '--timescale'); + } + + return $args; + } + + function onFatalProblem() + { + $start_arg = isset($this->args[$this->arg_offset]) ? $this->args[$this->arg_offset] : '?'; + $done_arg = isset($this->args[$this->last_done_arg_idx]) ? $this->args[$this->last_done_arg_idx] : '?'; + + $this->plan->post("Handling error, progress:".round($this->getProgress(),2).", last done arg:{$done_arg}, start arg:{$start_arg}, arg offset:{$this->arg_offset}, args:".sizeof($this->args)." *$this->device*"); + + //we're allowed to skip only in case of exceptions + if($this->getLastFatalProblem() == Task::CODE_EXCEPTION) + { + if($this->last_done_arg_idx > $this->arg_offset) + $this->arg_offset = $this->last_done_arg_idx; + ++$this->arg_offset; + } + + //let's reschedule the task if it still makes sense + if($this->arg_offset < sizeof($this->args)) + $this->reschedule(); + } +} +