micrOSDevToolKit 2.1.5__py3-none-any.whl → 2.26.1__py3-none-any.whl

toolkit/dashboard_apps/SystemTest.py

@@ -5,44 +5,111 @@ import sys
 import time
 import socket
 import ast
+from pprint import pprint
+try:
+    from ._app_base import AppBase
+except:
+    from _app_base import AppBase
+
 MYPATH = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.dirname(MYPATH))
-import socketClient
+REPORT_OUTPUT_PATH = os.path.join(MYPATH, '../../micrOS/release_info/micrOS_ReleaseInfo/devices_system_metrics.json')
 sys.path.append(os.path.join(MYPATH, '../lib/'))
 from TerminalColors import Colors
 import requests
 
-# FILL OUT
-DEVICE = '__simulator__'
-PASSWD = None
-
-
-def base_cmd():
-    if PASSWD is None:
-        return ['--dev', DEVICE]
-    return ['--dev', DEVICE, '--password', PASSWD]
+CLIENT = None
+TIMEOUT_SEC = 5
 
+# COLLECT AND SAVE SYSTEM TEST METRICS
+ENABLE_DATA_COLLECTION = os.environ.get("ENABLE_DATA_COLLECTION", False)
+REPORT_DICT = {}
+METRICS = {}
 
-def get_device():
-    return DEVICE
-
+#####################################
+#               METRICS             #
+#####################################
+def load_reports():
+    global REPORT_DICT
+    previous_reports = {}
+    try:
+        with open(REPORT_OUTPUT_PATH, 'r') as f:
+            previous_reports = json.loads(f.read())
+    except FileNotFoundError:
+        pass
+    finally:
+        REPORT_DICT = previous_reports
+
+def create_report(device_name:str, metrics:dict) -> None:
+    global REPORT_DICT
+
+    if not ENABLE_DATA_COLLECTION:
+        print(f"[SKIP] system test metrics export, ENABLE_DATA_COLLECTION={ENABLE_DATA_COLLECTION}")
+        pprint(metrics)
+        return
+    load_reports()
+    version = metrics.get('version')
+    device_type = metrics.get('board_type')
+    device_type = '???' if device_type is None else device_type.split()[-1]
+    if REPORT_DICT.get(version) is None:
+        REPORT_DICT[version] = {}
+    if REPORT_DICT.get(version).get(device_type) is None:
+        REPORT_DICT[version][device_type] = {}
+    REPORT_DICT[version][device_type][device_name] = metrics
+    # Cleanup - remove redundant elements
+    REPORT_DICT[version][device_type][device_name].pop("board_type", None)
+    REPORT_DICT[version][device_type][device_name].pop("version", None)
+    # Dump report
+    print(f"Save system test report: {REPORT_OUTPUT_PATH}")
+    pprint(metrics)
+    with open(REPORT_OUTPUT_PATH, 'w') as f:
+        f.write(json.dumps(REPORT_DICT, indent=4))
+
+
+def _add_metrics(key:str, value):
+    global METRICS
+    METRICS[key] = value
+
+#####################################
+#               TESTS               #
+#####################################
 
 def single_cmd_exec_check():
     info = "[ST] Run single command execution check [hello]"
     print(info)
     cmd_list = ['hello']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if output[1].startswith("hello:"):
             return True, info
     return False, info
 
+def shell_cmds_check():
+    info = "[ST] Run built-in shell commands [modules|version|help]"
+    print(info)
+    cmd_list = ['modules']
+    output = CLIENT.execute(cmd_list)
+    if output[0]:
+        _add_metrics("modules", ast.literal_eval(output[1]))
+        if not (output[1].startswith("[") and output[1].endswith("]")):
+            return False, f"{info}modules: {output[1]}"
+    cmd_list = ['version']
+    output = CLIENT.execute(cmd_list)
+    if output[0]:
+        if not ('.' in output[1] and '-' in output[1]):
+            return False, f"{info}version: {output[1]}"
+    cmd_list = ['help']
+    output = CLIENT.execute(cmd_list)
+    if output[0]:
+        if not ('[MICROS]' in output[1] and '[CONF]' in output[1] and '[TASK]' in output[1] and '[EXEC]' in output[1]):
+            return False, f"{info}help: {output[1]}"
+    return True, info
+
 
 def lm_cmd_exec_check():
     info = "[ST] Run Load Module command execution check [system heartbeat]"
     print(info)
     cmd_list = ['system heartbeat']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if output[1].strip() == '<3 heartbeat <3':
             return True, info
@@ -53,7 +120,7 @@ def micrOS_config_get():
     info = "[ST] Run micrOS config get [conf -> socport]"
     print(info)
     cmd_list = ['config <a> socport']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if output[1].strip() == '9008':
             return True, info
@@ -67,7 +134,7 @@ def micrOS_config_set():
 
     # [1] Get actual utc value
     cmd_list = ['config <a> utc']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         try:
             utc_bak = int(output[1].strip())
@@ -77,21 +144,21 @@ def micrOS_config_set():
     # [2] Set x+1 value as expected
     utc_expected = utc_bak+1
     cmd_list = ['config <a> utc {}'.format(utc_expected)]
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if output[1].strip() != 'Saved':
             return False, f"{info} + utc overwrite issue: {output[1]}"
 
     # [3] Get modified utc value - veridy [2] step
     cmd_list = ['config <a> utc']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if int(output[1].strip()) != utc_expected:
             return False, f"{info} + utc modified value error: {output[1]} != {utc_expected}"
 
     # Restore original value
     cmd_list = ['config <a> utc {}'.format(utc_bak)]
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if output[1].strip() != 'Saved':
             return False, f"{info} + utc overwrite issue: {output[1]}"
@@ -101,67 +168,51 @@ def micrOS_config_set():
 
 
 def micrOS_bgjob_one_shot_check():
-    info = "[ST] Run micrOS BgJob check [system clock &]"
+    info = "[ST] Run async microTask check [system clock &]"
     print(info)
-
     # Initial task cleanup...
-    execute(['task kill system.clock'])
-
-    async_available_cmd_list = ['help']
-    output = execute(async_available_cmd_list)
+    CLIENT.execute(['task kill system.clock'])
+    time.sleep(1)
+    cmd_list = ['system clock &']
+    output = CLIENT.execute(cmd_list)
     if output[0]:
-        if "[TASK]" not in output[1]:
-            return False, f'[ASYNC] task function not available: {output[1]}'
-    else:
-        return False, f'[ASYNC] check: help cmd return error.'
-
-    for _ in range(0, 2):
-        cmd_list = ['system clock &']
-        output = execute(cmd_list)
-        time.sleep(1)
-        if output[0]:
-            if 'Start system.clock' not in output[1].strip():
-                return False, f'{info} + not expected return: {output[1]}'
-    return True, info
+        # Legacy return OR New dict return
+        if 'Start system.clock' in output[1].strip() or "{'system.clock': 'Starting'}" in output[1].strip():
+            return True, info
+    return False, f'{info} + not expected return: {output[1]}'
 
 
 def micrOS_bgjob_loop_check():
-    info = "[ST] Run micrOS Async Task check [system clock &&] + task kill"
+    info = "[ST] Run async microTask check [system clock &&] + task kill"
     print(info)
 
-    async_available_cmd_list = ['help']
-    output = execute(async_available_cmd_list)
-    if output[0]:
-        if "[TASK]" not in output[1]:
-            return False, f'[ASYNC] task function not available: {output[1]}'
-    else:
-        return False, f'[ASYNC] check: help cmd return error.'
-
     # Start background task loop
     cmd_list = ['system clock &&']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
-        if 'Start system.clock' not in output[1].strip():
+        # Legacy return OR New dict return
+        if not ('Start system.clock' in output[1].strip() or "{'system.clock': 'Starting'}" in output[1]):
             return False, f'[Start Task error] {info} + not expected return: {output[1]}'
 
     # Attempt to overload background thread
     cmd_list = ['system clock &&']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
-        if 'system.clock is Busy' not in output[1].strip():
+        # Legacy return OR New dict return
+        if not ('system.clock is Busy' in output[1].strip() or "{'system.clock': 'Already running'}" in output[1]):
             return False, f'[Overload task - run same] {info} + not expected return: {output[1]}'
 
     # Show task output by task tag
     time.sleep(0.1)                           # Give time for the task to start and update buffer... (sim)
     cmd_list = ['task show system.clock']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if "No task found" in output[1].strip() or len(output[1].strip()) == 0:
             return False, f'[No task output] {info} + not expected return: {output[1]}'
 
     # Stop BgJob
     cmd_list = ['task kill system.clock']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if 'Kill:' in output[1].strip() or 'system.clock' in output[1].strip():
             return True, f'[Stop task] {info}'
@@ -169,12 +220,13 @@ def micrOS_bgjob_loop_check():
     # Failed verdict
     return False, f'[Thread not stopped]{info} + not expected return: {output[1]}'
 
+
 def micrOS_task_list_check():
     info = "[ST] Run micrOS Task list feature check [task list][task list >json]"
     print(info)
 
     async_available_cmd_list = ['task list']
-    output = execute(async_available_cmd_list)
+    output = CLIENT.execute(async_available_cmd_list)
     if output[0]:
         if "---- micrOS  top ----" not in output[1]:
             return False, f"{info} - ERROR: missing response prefix {output[1]}"
@@ -182,7 +234,7 @@ def micrOS_task_list_check():
         return False, f"{info} - task list error: {output[1]}"
 
     async_available_cmd_list = ['task list >json']
-    output = execute(async_available_cmd_list)
+    output = CLIENT.execute(async_available_cmd_list)
     if output[0]:
         starts = '{'
         ends = '}'
@@ -192,14 +244,16 @@ def micrOS_task_list_check():
         print(f"{info} - task list >json error: {output[1]}")
     return True, info
 
+
 def micrOS_get_version():
     info = "[ST] Run micrOS get version [version]"
     print(info)
     cmd_list = ['version']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if '.' in output[1].strip() and '-' in output[1].strip():
             return True, f"{info} v:{output[1].strip()}"
+    _add_metrics('version', output[1])
     return False, f"{info} out: {output[1]}"
 
 
@@ -207,7 +261,7 @@ def json_format_check():
     info = "[ST] Run micrOS raw output check aka >json [system rssi >json]"
     print(info)
     cmd_list = ['system rssi >json']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0] and output[1].startswith("{") and output[1].endswith("}"):
         return True, info + f" out: {output[1]}"
     return False, info + f" out: {output[1]}"
@@ -218,25 +272,25 @@ def negative_interface_check():
     print(info)
 
     cmd_list = ['Apple']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if 'Shell: for hints type help.' not in output[1].strip():
             return False, f'[Wrong single command] {info} + not expected return: {output[1]}'
 
     cmd_list = ['Apple test']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if 'no module named' not in output[1].strip().lower():
             return False, f'[Missing module] {info} + not expected return: {output[1]}'
 
     cmd_list = ['conf', 'gmttimaaaa']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if output[1].strip() != 'None':
             return False, f'[Config invalid key] {info} + not expected return: {output[1]}'
 
     cmd_list = ['conf', 'utc "type"']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0]:
         if output[1].strip() != 'Failed to save':
             return False, f'[Config invalid key type] {info} + not expected return: {output[1]}'
@@ -260,11 +314,12 @@ def measure_package_response_time():
     # Start time
     start = time.time()
     # Command exec
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     # Stop time
     end = time.time() - start
     # Get average response time
     delta_cmd_rep_time = round(end/10, 4)
+    _add_metrics('shell_heartbeatX10_response_ms', int(delta_cmd_rep_time*1000))
     # Create verdict
     print(output)
     if output[0] and "<3 heartbeat <3" in output[1]:
@@ -276,7 +331,7 @@ def micros_alarm_check():
     info = "[ST] Test alarm state - system alarms should be null"
     print(info)
     cmd_list = ['system alarms']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     alarm_cnt = 0
     if output[0]:
         try:
@@ -287,7 +342,7 @@ def micros_alarm_check():
             print(e)
         # Clean alarms
         cmd_list = ['system alarms True']
-        execute(cmd_list)
+        CLIENT.execute(cmd_list)
         # Evaluation
         if alarm_cnt > 0:
             return True, info + f" -1 !!!WARN!!! [{alarm_cnt}] out: {output[1]}"
@@ -295,11 +350,11 @@ def micros_alarm_check():
 
 
 def oled_msg_end_result(result):
-    cmd_list = ['lmpacman module >json']
-    output = execute(cmd_list)
+    cmd_list = ['pacman moduls >json']
+    output = CLIENT.execute(cmd_list)
     if output[0] and 'LM_oled_ui' in output[1]:
         cmd_list = [f'oled_ui msgbox "{result} %"']
-        print(execute(cmd_list))
+        print(CLIENT.execute(cmd_list))
 
 
 def check_device_by_hostname(dev):
@@ -320,7 +375,7 @@ def check_robustness_exception():
     info_msg = '[ST] Check robustness - exception [robustness raise_error]'
     print(info_msg)
     cmd_list = ['robustness raise_error']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0] and "Core error: LM_robustness->raise_error: Test exception" in output[1]:
         return True, f'{info_msg}: Valid error msg: exec_lm_core *->raise_error: *'
     else:
@@ -331,7 +386,7 @@ def check_robustness_memory():
     info_msg = '[ST] Check robustness - memory_leak [robustness memory_leak 12]'
     print(info_msg)
     cmd_list = ['robustness memory_leak 12']
-    output = execute(cmd_list)
+    output = CLIENT.execute(cmd_list)
     if output[0] and "[12] RAM Alloc" in output[1]:
         end_result = output[1].split("\n")[-1]
         return True, f'{info_msg}: Mem alloc: {end_result}'
@@ -343,7 +398,7 @@ def check_robustness_recursion():
     info_msg = '[ST] Check robustness - recursion [robustness recursion_limit 8]'
     print(info_msg)
     cmd_list = ['robustness recursion_limit 8']
-    output = execute(cmd_list, tout=10)
+    output = CLIENT.execute(cmd_list, tout=10)
     if output[0]:
         last_line = output[1].split("\n")[-1]
         if "Recursion limit:" in last_line:
@@ -355,9 +410,14 @@ def check_robustness_recursion():
 def check_intercon(host=None):
     def _convert_return_to_dict(data):
         try:
-            data_dict = ast.literal_eval(data[1])
+            data_dict = ast.literal_eval(data[1].split("\n")[0])
         except Exception as e:
+            print(f"WARNING: cannot parse output as dir: {e}")
             data_dict = {'tag': None, 'verdict': f'{data}: {str(e)}'}
+        # Handle new syntax: {"taskID": "verdict"}
+        if data[0] and data_dict.get("tag") is None:
+            _ttag = list(data_dict.keys())[0]
+            data_dict = {'tag': _ttag, 'verdict': f'{list(data_dict.values())[0]}: task show {_ttag}'}
         return data[0], data_dict
 
     def _get_intercon_output(tag):
@@ -366,23 +426,29 @@ def check_intercon(host=None):
         _output = None
         for _ in range(0, 2):
             time.sleep(1)
-            _output = execute(_cmd_list, tout=8)
+            _output = CLIENT.execute(_cmd_list, tout=8)
             if _output[0] and 'No task found:' not in _output[1]:
                 _state = True
                 break
         return _state, _output
 
+    def run_intercon_hello(tout=8):
+        nonlocal host
+        cmd_list = [f'hello >>{host}']
+        _output = CLIENT.execute(cmd_list, tout=tout)
+        return _output
+
     info_msg = '[ST] Check device-device connectivity'
     print(info_msg)
     host = 'test.local' if host is None else host
-    cmd_list = ['intercon sendcmd "{}" "hello" >json'.format(host)]
-    output = execute(cmd_list, tout=8)
+    output = run_intercon_hello(tout=8)
     output = _convert_return_to_dict(output)
+
     device_was_found = False
     if output[0] is False or output[1] is None:
         output = 'Device was not found: {}:{}'.format(host, output)
         return False, output
-    elif output[1] == '[]':
+    if output[1] == '[]':
         # Valid input, device was not found
         output = 'Device was not found: {}:{}'.format(host, output)
         state = True, f'{info_msg}:\n\t\t{output}'
@@ -390,30 +456,37 @@ def check_intercon(host=None):
         response_state, response = _get_intercon_output(output[1]['tag'])
         # Valid input on online device
         output = "Device was found: {}:{}".format(host, f"{output}: {response}")
-        state = True & response_state, f'{info_msg}:\n\t\t{output}'
+        state = response_state, f'{info_msg}:\n\t\t{output}'
         device_was_found = True
     else:
-        state = False, output
+        state = False, f" InterCon Failed: {output}"
 
     if device_was_found:
         # DO Negative testing as well
-        cmd_list = ['intercon sendcmd "notavailable.local" "hello" >json']
-        output_neg = execute(cmd_list, tout=15)
+        host = "notavailable.local"
+        output_neg = run_intercon_hello(tout=20)
+        if output_neg[0] and output_neg[1] == '':
+            # NO HOST TIMEOUT ...
+            output_neg = f'Device was not found (dhcp timeout): {host}:{output_neg}: {output_neg[1]}'
+            return True & state[0], f"{state[1]}\n\t\tNegative test: {output_neg}"
+        # Valid return on negative testing
         output_neg = _convert_return_to_dict(output_neg)
-        state_neg = False, output_neg
+        state_neg = False, f"Negative test failed: {output_neg}"
         if len(output_neg[1]) > 1 and "hello" in output_neg[1]['verdict']:
             response_state, response = _get_intercon_output(output_neg[1]['tag'])
-            output_neg = f'Device was not found: "notavailable.local":{output_neg}: {response}'
-            state_neg = True & response_state, output_neg
-        return state[0] & state_neg[0], "{}\n\t\tNegative test: {}".format(state[1], state_neg[1])
+            output_neg = f'Device was not found: {host}":{output_neg}: {response}'
+            state_neg = response_state, output_neg
+        return state[0] & state_neg[0], f"{state[1]}\n\t\tNegative test: {state_neg[1]}"
     return state
 
 
 def measure_conn_metrics():
     try:
-        verdict = socketClient.connection_metrics(f"{get_device().strip()}.local")
+        verdict, delta_t_single, delta_t_multi = CLIENT.connection_metrics()
+        _add_metrics("shell_single_session_dt_ms", int(delta_t_single*1000))
+        _add_metrics("shell_multi_session_dt_ms", int(delta_t_multi*1000))
         for k in verdict:
-            print("\t\t{}".format(k))
+            print(f"\t\t{k}")
         state = True if len(verdict) > 0 else False
     except Exception as e:
         state = False
@@ -423,7 +496,7 @@ def measure_conn_metrics():
 
 def memory_usage():
     cmd = ['system memory_usage >json']
-    out = execute(cmd, tout=3)
+    out = CLIENT.execute(cmd, tout=TIMEOUT_SEC)
     state, raw_output = out[0], out[1]
     try:
         json_out = json.loads(raw_output)
@@ -431,7 +504,9 @@ def memory_usage():
         return False, '[ST] {}ERR{}: {}: {}'.format(Colors.ERR, Colors.NC, raw_output, e)
 
     # {"percent": 93.11, "mem_used": 103504}
-    if json_out.get('percent') > 80:        # MEM USAGE WARNING INDICATOR: 85%
+    _add_metrics('mem_percent', json_out.get('percent'))
+    _add_metrics('mem_used_byte', json_out.get('mem_used'))
+    if json_out.get('percent') > 70:        # MEM USAGE WARNING INDICATOR: 80%
         return state, '[ST] {}WARNING{}: memory usage {}% ({} bytes)'.format(Colors.WARN, Colors.NC,
                                                                              json_out.get('percent'),
                                                                              json_out.get('mem_used'))
@@ -444,7 +519,7 @@ def disk_usage():
     Check disk usage - manually defined 16% (336_000 byte) - check degradations...
     """
     cmd = ['system disk_usage >json']
-    out = execute(cmd, tout=3)
+    out = CLIENT.execute(cmd, tout=TIMEOUT_SEC)
     state, raw_output = out[0], out[1]
     try:
         json_out = json.loads(raw_output)
@@ -452,7 +527,9 @@ def disk_usage():
         return False, '[ST] {}ERR{}: {}: {}'.format(Colors.ERR, Colors.NC, raw_output, e)
 
     # {"percent": 15.4, "fs_used": 323_584}
-    if json_out.get('fs_used') > 500_000:        # MEM USAGE WARNING INDICATOR: 500_000 bytes (500kb)
+    _add_metrics('fs_percent', json_out.get('percent'))
+    _add_metrics('fs_used_byte', json_out.get('fs_used'))
+    if json_out.get('fs_used') > 800_000:        # MEM USAGE WARNING INDICATOR: (800kb)
         return state, '[ST] {}WARNING{}: disk usage {}% ({} bytes)'.format(Colors.WARN, Colors.NC,
                                                                              json_out.get('percent'),
                                                                              json_out.get('fs_used'))
@@ -462,7 +539,7 @@ def disk_usage():
 
 def task_list():
     cmd = ['task list']
-    out = execute(cmd, tout=3)
+    out = CLIENT.execute(cmd, tout=TIMEOUT_SEC)
     state, output = out[0], out[1]
     if state:
         return state, output.replace('\n', f'\n{" "*51}')        # TODO format output
@@ -472,40 +549,51 @@ def task_list():
 def webcli_test():
     endpoints = []
     cmd = ['conf', 'webui']
-    out = execute(cmd, tout=3)
+    out = CLIENT.execute(cmd, tout=TIMEOUT_SEC)
     state, output = out[0], out[1]
     if state:
         verdict = f"[ST] WEBUI IS ENABLED ({output})" if output.strip() == 'True' else f"[ST] WEBUI IS DISABLED ({output})"
         if output.strip() == 'True':
-            out = execute(['conf', 'devip'], tout=3)
+            out = CLIENT.execute(['conf', 'devip'], tout=TIMEOUT_SEC)
             if out[0] and out[1] is not None:
                 devip = out[1]
-                endpoints.append(f'http://{devip}')
                 endpoints.append(f'http://{devip}/rest')
+                endpoints.append(f'http://{devip}')
 
             for endpoint in endpoints:
                 _start_t = time.time()
+                delta_t = -1
                 try:
                     response = requests.get(endpoint, timeout=5)
+                    delta_t = round(time.time() - _start_t, 2)
                     # Check if the request was successful
                     if not (response.status_code == 200 and ('<!DOCTYPE html>' in str(response.content) or '"micrOS"' in str(response.content))):
-                        verdict += f" Endpoint: {endpoint} [{Colors.ERR}NOK{Colors.NC}]({round(time.time()-_start_t, 2)}s)"
-                        print(response.content)
-                        state = False
+                        if response.status_code == 400 and "Low memory" in str(response.content):
+                            verdict += f" Endpoint: {endpoint} - Low memory mode [{Colors.WARN}OK{Colors.NC}]({delta_t}s)"
+                            print(response.content)
+                        else:
+                            verdict += f" Endpoint: {endpoint} [{Colors.ERR}NOK{Colors.NC}]({delta_t}s)"
+                            print(response.content)
+                            state = False
                     else:
-                        verdict += f" Endpoint: {endpoint} [{Colors.OK}OK{Colors.NC}]({round(time.time()-_start_t, 2)}s)"
+                        verdict += f" Endpoint: {endpoint} [{Colors.OK}OK{Colors.NC}]({delta_t}s)"
+                        metrics_name = 'landingpage' if "." in endpoint.split('/')[-1] else endpoint.split('/')[-1]
+                        _add_metrics(f"web_{metrics_name}_response_ms", int(delta_t*1000))
                 except Exception as e:
-                    verdict += f" Endpoint: {endpoint} [{Colors.ERR}NOK{Colors.NC}]({round(time.time()-_start_t, 2)}s)"
+                    verdict += f" Endpoint: {endpoint} [{Colors.ERR}NOK{Colors.NC}]({delta_t}s)"
                     print(f"webcli_test error: {e}")
                     state  = False
         return state, verdict
     return state, output
 
+#####################################
+#               HELPERS             #
+#####################################
 
 def after_st_reboot():
     verdict = False, 'reboot -h failed'
     cmd = ['reboot -h']
-    out = execute(cmd, tout=3)
+    out = CLIENT.execute(cmd, tout=TIMEOUT_SEC)
     state, output = out[0], out[1]
     if state:
         verdict = state, f'[reboot-h] commad out: {output}'
@@ -513,7 +601,7 @@ def after_st_reboot():
             print(f"[reboot-h] Wait for node up again ({retry}/{retry*2}sec)")
             time.sleep(2)
             try:
-                up, o = execute(['hello'], tout=3)
+                up, o = CLIENT.execute(['hello'], tout=TIMEOUT_SEC)
                 if up:
                     verdict = True, f"[reboot-h][OK] successfully rebooted: {o} (boot time: ~{retry * 2}sec)"
                     print(verdict[1])
@@ -526,21 +614,40 @@ def after_st_reboot():
 
 def get_dev_version():
     cmd = ['version']
-    out = execute(cmd, tout=3)
+    out = CLIENT.execute(cmd, tout=TIMEOUT_SEC)
     state, output = out[0], out[1]
+    _add_metrics("version", output)
     if state:
         return output
     return '0.0.0-0'
 
+def get_dev_board_type():
+    cmd = ['system info >json']
+    out = CLIENT.execute(cmd, tout=TIMEOUT_SEC)
+    state, output = out[0], out[1]
+    output = ast.literal_eval(output)
+    board = output.get("board")
+    upython = output.get("upython")
+    _add_metrics("board_type", board)
+    _add_metrics("micropython_version", str(upython))
+    if state:
+        return board, upython
+    return None, None
+
+#####################################
+#              MAIN APP             #
+#####################################
+
 def app(devfid=None, pwd=None):
-    global DEVICE, PASSWD
-    if devfid is not None:
-        DEVICE = devfid
-    if pwd is not None:
-        PASSWD = pwd
+    global CLIENT
+    CLIENT = AppBase(device=devfid, password=pwd)
 
+    # Get device info
+    version = get_dev_version()
+    get_dev_board_type()        # update metrics
     # Get test verdict
     verdict = {'single_cmds': single_cmd_exec_check(),
+               'shell_cmds': shell_cmds_check(),
                'lm_cmd_exec': lm_cmd_exec_check(),
                'config_get': micrOS_config_get(),
                'config_set': micrOS_config_set(),
@@ -551,7 +658,7 @@ def app(devfid=None, pwd=None):
                'json_check': json_format_check(),
                'response_time': measure_package_response_time(),
                'negative_api': negative_interface_check(),
-               'dhcp_hostname': check_device_by_hostname(DEVICE),
+               'dhcp_hostname': check_device_by_hostname(CLIENT.get_device()),
                'lm_exception': check_robustness_exception(),
                'mem_usage': memory_usage(),
                'disk_usage': disk_usage(),
@@ -565,11 +672,12 @@ def app(devfid=None, pwd=None):
                'clean-reboot': after_st_reboot()
                }
 
+
     # Test Evaluation
     final_state = True
     ok_cnt = 0
-    version = get_dev_version()
-    print(f"\n----------------------------------- micrOS System Test results on {DEVICE}:{version} device -----------------------------------")
+    create_report(CLIENT.get_device(), METRICS)
+    print(f"\n----------------------------------- micrOS System Test results on {CLIENT.get_device()}:{version} device -----------------------------------")
     print("\tTEST NAME\t\tSTATE\t\tDescription\n")
     for test, state_data in verdict.items():
         state = Colors.ERR + 'NOK' + Colors.NC
@@ -589,11 +697,5 @@ def app(devfid=None, pwd=None):
     oled_msg_end_result(f"System[{state}] {pass_rate}")
 
 
-def execute(cmd_list, tout=5):
-    cmd_args = base_cmd() + cmd_list
-    print("[ST] test cmd: {}".format(cmd_args))
-    return socketClient.run(cmd_args, timeout=tout)
-
-
 if __name__ == "__main__":
     app()