bit
/
taskman_atf
26
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

clickhouse + logs analytics

This commit is contained in:
Bimmy 2022-08-03 17:40:44 +03:00
parent 933362cc6a
commit a6b8fd6228
1 changed files with 118 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
atf.inc.php
View File

@ -1,6 +1,8 @@
<?php <?php
namespace taskman; namespace taskman;
use Amp; use Amp;
use ClickHouseDB;
use Exception;
interface IATFDevicePool interface IATFDevicePool
{ {
@ -216,101 +218,101 @@ class ATFSession
Amp\Promise\wait($plan->runAsync($install, $sleep_time = 4, $hung_threshold = 80, $gone_threshold = 30, $stuck_threshold = 200)); Amp\Promise\wait($plan->runAsync($install, $sleep_time = 4, $hung_threshold = 80, $gone_threshold = 30, $stuck_threshold = 200));
} }
// function trySendStatsFromJzonAsync(ATFTask $task, $jzon) function trySendStatsFromJzonAsync(ATFTask $task, $jzon)
// { {
// return Amp\call(function() use($task, $jzon) { return Amp\call(function() use($task, $jzon) {
// try try
// { {
// $data = jzon_parse(trim(str_replace('\"', '"', $jzon))); $data = jzon_parse(trim(str_replace('\"', '"', $jzon)));
// $table = $data['table']; $table = $data['table'];
// unset($data['table']); unset($data['table']);
// if(isset($data['deviceMemoryUsage']) && $data['deviceMemoryUsage'] === '') if(isset($data['deviceMemoryUsage']) && $data['deviceMemoryUsage'] === '')
// { {
// $mem = yield atf_device_mem_async($task->device); $mem = yield atf_device_mem_async($task->device);
// $data['deviceMemoryUsage'] = $mem['total']; $data['deviceMemoryUsage'] = $mem['total'];
// if($table === 'device_memory') if($table === 'device_memory')
// { {
// $data['deviceMemoryUsageNative'] = $mem['native']; $data['deviceMemoryUsageNative'] = $mem['native'];
// $data['deviceMemoryUsageSystem'] = $mem['system']; $data['deviceMemoryUsageSystem'] = $mem['system'];
// $data['deviceMemoryUsageJava'] = $mem['java']; $data['deviceMemoryUsageJava'] = $mem['java'];
// $data['deviceMemoryUsageGraphics'] = $mem['graphics']; $data['deviceMemoryUsageGraphics'] = $mem['graphics'];
// } }
// } }
// $this->trySendStats($task, $table, $data); $this->trySendStats($task, $table, $data);
// } }
// catch(Exception $e) catch(Exception $e)
// { {
// echo $e->getMessage() . "\n"; echo $e->getMessage() . "\n";
// } }
// }); });
// } }
// function trySendApkStatsEvent() function trySendApkStatsEvent()
// { {
// try try
// { {
// $apk_sizeB = get_apk_size(); $apk_sizeB = get_apk_size();
// $apk_sizeMb = round($apk_sizeB / 1024 / 1024,4); $apk_sizeMb = round($apk_sizeB / 1024 / 1024,4);
// $version = taskman_prop('GAME_VERSION'); $version = taskman_prop('GAME_VERSION');
// $rev_hash = taskman_prop('GAME_REVISION_HASH'); $rev_hash = taskman_prop('GAME_REVISION_HASH');
// $msg = "version $version($rev_hash) size: $apk_sizeMb Mb ($apk_sizeB B)"; $msg = "version $version($rev_hash) size: $apk_sizeMb Mb ($apk_sizeB B)";
// // atf_slack_post($msg, array('channel' => _atf_slack_chan_qa())); // atf_slack_post($msg, array('channel' => _atf_slack_chan_qa()));
// $data['guid'] = $this->guid; $data['guid'] = $this->guid;
// $data['time'] = time(); $data['time'] = time();
// $data['version'] = $version; $data['version'] = $version;
// $data['revHash'] = $rev_hash; $data['revHash'] = $rev_hash;
// $data['apk_size'] = $apk_sizeB; $data['apk_size'] = $apk_sizeB;
// atf_stats_send('apk_size', $data); atf_stats_send('apk_size', $data);
// $data['guid'] = $this->guid; $data['guid'] = $this->guid;
// $data['time'] = time(); $data['time'] = time();
// $data['version'] = $this->version; $data['version'] = $this->version;
// $data['revHash'] = $this->rev_hash; $data['revHash'] = $this->rev_hash;
// $data['branch'] = $this->branch; $data['branch'] = $this->branch;
// $data['event'] = 'apk_size'; $data['event'] = 'apk_size';
// $data['value'] = $apk_size; $data['value'] = $apk_size;
// atf_stats_send('event_value', $data); atf_stats_send('event_value', $data);
// } }
// catch(Exception $e) catch(Exception $e)
// { {
// echo $e->getMessage() . "\n"; echo $e->getMessage() . "\n";
// } }
// } }
// function trySendStats(ATFTask $task, $table, array $data) function trySendStats(ATFTask $task, $table, array $data)
// { {
// try try
// { {
// $data['guid'] = $this->guid; $data['guid'] = $this->guid;
// $data['time'] = time(); $data['time'] = time();
// $data['deviceId'] = $task->device; $data['deviceId'] = $task->device;
// atf_stats_send($table, $data); atf_stats_send($table, $data);
// } }
// catch(Exception $e) catch(Exception $e)
// { {
// echo $e->getMessage() . "\n"; echo $e->getMessage() . "\n";
// } }
// } }
// function trySendStatsEvent(ATFTask $task, $event, $value = 1) function trySendStatsEvent(ATFTask $task, $event, $value = 1)
// { {
// $this->trySendStats( $this->trySendStats(
// $task, $task,
// 'event_value', 'event_value',
// array( array(
// 'event' => $event, 'event' => $event,
// 'value' => $value, 'value' => $value,
// 'branch' => $this->branch, 'branch' => $this->branch,
// 'version' => $this->version, 'version' => $this->version,
// 'revHash' => $this->rev_hash 'revHash' => $this->rev_hash
// ) )
// ); );
// } }
// function tryShareToQAChannel($msg_slack_id, $error) // function tryShareToQAChannel($msg_slack_id, $error)
// { // {
@ -749,7 +751,7 @@ class ATFPlan
if($not_alive_time > $hung_threshold) if($not_alive_time > $hung_threshold)
{ {
$task->addStatusCode(ATFTask::CODE_HUNG); $task->addStatusCode(ATFTask::CODE_HUNG);
// $this->session->trySendStatsEvent($task, 'hung'); $this->session->trySendStatsEvent($task, 'hung');
atf_log("[HNG] No activity for $hung_threshold seconds *{$task->device}*"); atf_log("[HNG] No activity for $hung_threshold seconds *{$task->device}*");
$device_is_bogus = true; $device_is_bogus = true;
} }
@ -761,13 +763,13 @@ class ATFPlan
if($task->last_alive_check_time === $task->reset_time) if($task->last_alive_check_time === $task->reset_time)
{ {
$task->addStatusCode(ATFTask::CODE_NSTART); $task->addStatusCode(ATFTask::CODE_NSTART);
// $this->session->trySendStatsEvent($task, 'nstart'); $this->session->trySendStatsEvent($task, 'nstart');
atf_log("[NFD] No app started after $hung_threshold seconds *{$task->device}*"); atf_log("[NFD] No app started after $hung_threshold seconds *{$task->device}*");
} }
else else
{ {
$task->addStatusCode(ATFTask::CODE_GONE); $task->addStatusCode(ATFTask::CODE_GONE);
// $this->session->trySendStatsEvent($task, 'gone'); $this->session->trySendStatsEvent($task, 'gone');
atf_log("[GNE] App is gone after $hung_threshold seconds *{$task->device}*"); atf_log("[GNE] App is gone after $hung_threshold seconds *{$task->device}*");
} }
$device_is_bogus = true; $device_is_bogus = true;
@ -779,7 +781,7 @@ class ATFPlan
else if($stuck_time > $stuck_threshold) else if($stuck_time > $stuck_threshold)
{ {
$task->addStatusCode(ATFTask::CODE_STUCK); $task->addStatusCode(ATFTask::CODE_STUCK);
// $this->session->trySendStatsEvent($task, 'stuck'); $this->session->trySendStatsEvent($task, 'stuck');
atf_log("[STK] Stuck for $stuck_threshold seconds *{$task->device}*"); atf_log("[STK] Stuck for $stuck_threshold seconds *{$task->device}*");
} }
@ -849,8 +851,8 @@ class ATFPlan
$task->addStatusCode($item['error'], $item['message']); $task->addStatusCode($item['error'], $item['message']);
// if($item['error'] == ATFTask::CODE_EXCEPTION) if($item['error'] == ATFTask::CODE_EXCEPTION)
// $this->session->trySendStatsEvent($task, 'exception'); $this->session->trySendStatsEvent($task, 'exception');
list($msg_type, $msg_text) = self::_parseExtMessage($item['message']); list($msg_type, $msg_text) = self::_parseExtMessage($item['message']);
@ -883,8 +885,8 @@ class ATFPlan
$this->session->resetBogusDeviceCount($task->device); $this->session->resetBogusDeviceCount($task->device);
$task->onProgress($msg_text); $task->onProgress($msg_text);
} }
// else if($msg_type === '[STAT]') else if($msg_type === '[STAT]')
// yield $this->session->trySendStatsFromJzonAsync($task, $msg_text); yield $this->session->trySendStatsFromJzonAsync($task, $msg_text);
else if($msg_type === '[WRN]') else if($msg_type === '[WRN]')
{ {
$task->addStatusCode(ATFTask::CODE_WARN); $task->addStatusCode(ATFTask::CODE_WARN);
@ -1292,6 +1294,27 @@ function _atf_trim_start($txt, $max_len)
return strlen($txt) > $max_len ? "..." . substr($txt, strlen($txt) - $max_len) : $txt; return strlen($txt) > $max_len ? "..." . substr($txt, strlen($txt) - $max_len) : $txt;
} }
function _atf_db()
{
$config = array(
'host' => '172.19.179.148',
'port' => '8123',
'username' => get('ATF_CLICKHOUSE_USERNAME'),
'password' => get('ATF_CLICKHOUSE_PASSWORD')
);
$db = new ClickHouseDB\Client($config);
$db->database(get('ATF_CLICKHOUSE_DB'));
$db->setTimeout(10);
$db->setConnectTimeOut(5);
return $db;
}
function atf_stats_send($table, array $data)
{
$db = _atf_db();
return;
$db->insert($table, array(array_values($data)), array_keys($data));
}
function atf_log($msg) function atf_log($msg)
{ {
@ -1459,7 +1482,6 @@ function atf_host()
return $node_name; return $node_name;
} }
function _atf_opt(array &$args, $opt, $default, $conv_fn = null) function _atf_opt(array &$args, $opt, $default, $conv_fn = null)
{ {
foreach($args as $idx => $arg) foreach($args as $idx => $arg)