wiliot-certificate 4.5.0__py3-none-any.whl → 4.5.0a1__py3-none-any.whl

certificate/cert_results.py

@@ -31,13 +31,11 @@ CERT_VER =                         "Certificate Version"
 TESTING_DEVICE_INFO =              "Testing Device Information"
 TESTED_DEVICE_INFO =               "Tested Device Information"
 SIM =                              "Simulator"
-GATEWAY =                          "Gateway"
 BRG =                              "Bridge"
 BLE_MAC_ADDRESS =                  "BLE MAC Address"
 BOARD_TYPE =                       "Board Type"
 BLE_VER =                          "BLE Version"
 SUP_API_VER =                      "Supported API Version"
-ADD_INFO =                         "Additional information"
 # TODO: This is a temporary list of all schema module names - remove this when auto generated
 MODULE_SCHEMA_NAMES_LIST = ["Cloud Connectivity", "Edge Management", "Calibration", "Datapath", "Energizer 2.4GHz",
                             "Energizer Sub-1GHz", "Power Management", "BLE Sensor", "Custom"]
@@ -114,24 +112,32 @@ def get_important_tests_info():
     patterns = ["DISCONNECTED", "WLT_ERROR", "free heap size", "python_mqtt_disconnect"]
     return "".join([l.strip(' "') for l in open(os.path.join(ARTIFACTS_DIR, CERT_MQTT_LOG_FILE)).readlines() if any([p in l for p in patterns])])
 
-def generate_results_files(html=True, pdf=True, failures=0, skipped=0, start_time=datetime.datetime.now(), tests=[], error=None, non_cert_run=False):
+def generate_results_files(dut=None, tester=None, html=True, pdf=True, failures=0, skipped=0, start_time=datetime.datetime.now(), tests=[], error=None, non_cert_run=False):
     print(cert_prints.SEP)
     duration = (datetime.datetime.now()-start_time)
     print("Tests duration: {}".format(str(duration).split(".")[0]))
     pipeline = cert_prints.pipeline_running()
-    test = tests[-1] if tests else None
-    if test and test.dut_is_bridge():
-        print("Bridge version: {}".format(test.dut.version))
+    brg = None
+    if dut:
+        brg = dut.internal_brg if isinstance(dut, cert_utils.Gateway) and dut.has_internal_brg() else dut
+        if isinstance(brg, cert_utils.Bridge):
+            print("Bridge version: {}".format(brg.version))
     if not error:
         print(generate_tests_table(tests))
         print(tabulate.tabulate([[len(tests), len(tests)-(failures+skipped), skipped, failures, len(tests)]],
                                 headers=["FINISHED", "PASSED", "SKIPPED", "FAILED", "TOTAL"], tablefmt="pretty"))
         print(cert_prints.WIL_CERT_TEXT)
         cert_prints.print_warn(get_important_tests_info())
-    print(f"Run artifacts saved in \"{ARTIFACTS_DIR}\" directory")
     run_type = headline_get(non_cert_run, "").strip()
     cert_prints.print_pass_or_fail(not failures and not error, run_type)
 
+    # Get internal_brg from dut or tester
+    internal_brg = None
+    if dut and isinstance(dut, cert_utils.Gateway) and dut.has_internal_brg():
+        internal_brg = dut.internal_brg
+    elif tester:
+        internal_brg = tester.internal_brg
+    
     # Generate HTML file
     if html:
         file_path = os.path.join(ARTIFACTS_DIR, UT_RESULT_FILE_HTML)
@@ -153,8 +159,8 @@ def generate_results_files(html=True, pdf=True, failures=0, skipped=0, start_tim
             f.write(update_status + "<br><br>")
             f.write(error + "<br><br>")
             f.write(f"{RUN_DUR}: {str(duration).split('.')[0]} <br><br>")
-            if test and test.dut_is_bridge():
-                f.write(f"{BRG} {BLE_VER}: {test.dut.version} <br><br>")
+            if brg:
+                f.write(f"{BRG} {BLE_VER}: {brg.version} <br><br>")
         elif tests:
             if not failures and ("successfully!" in update_status or "skipped!" in update_status or not pipeline):
                 headline = headline_get(non_cert_run, "Passed!")
@@ -169,20 +175,12 @@ def generate_results_files(html=True, pdf=True, failures=0, skipped=0, start_tim
             f.write(f"{RUN_DATETIME}: {start_time.strftime('%d/%m/%Y, %H:%M:%S')} <br><br>")
             f.write(f"{RUN_DUR}: {str(duration).split('.')[0]} <br><br>")
             f.write(f"{CERT_VER}: {CERT_VERSION} <br><br>")
-            if test and test.dut_is_gateway():
-                f.write(f"{GATEWAY} ID: {test.dut.id_str} <br><br>")
-                f.write(f"{GATEWAY} {SUP_API_VER}: {test.dut.gw_api_version} <br><br>")
-                f.write(f"{GATEWAY} {SERIALIZATION_FORMAT}: {PROTOBUF if test.dut.protobuf else JSON} <br><br>")
-                if test.dut_is_combo():
-                    f.write(f"{BRG} {BLE_MAC_ADDRESS}: {test.dut.internal_brg.id_str} <br><br>")
-                    f.write(f"{BRG} {BLE_VER}: {test.dut.internal_brg.version} <br><br>")
-            if test.dut_is_bridge():
-                f.write(f"{BRG} {BLE_MAC_ADDRESS}: {test.dut.id_str} <br><br>")
-                f.write(f"{BRG} {BLE_VER}: {test.dut.version} <br><br>")
-            if test.tester:
-                f.write(f"{SIM} {BLE_MAC_ADDRESS}: {test.tester.internal_brg.id_str} <br><br>")
-                f.write(f"{SIM} {BLE_VER}: {test.tester.internal_brg.version} <br><br>")
-
+            if internal_brg:
+                f.write(f"{SIM} {BLE_MAC_ADDRESS}: {internal_brg.id_str} <br><br>")
+                f.write(f"{SIM} {BLE_VER}: {internal_brg.version} <br><br>")
+            if isinstance(brg, cert_utils.Bridge):
+                f.write(f"{BRG} {BLE_MAC_ADDRESS}: {brg.id_str} <br><br>")
+                f.write(f"{BRG} {BLE_VER}: {brg.version} <br><br>")
             f.write(tabulate.tabulate([[len(tests)-(failures+skipped), skipped, failures, len(tests)]], headers=["PASSED", "SKIPPED", "FAILED", "TOTAL"], tablefmt="html"))
             f.write(generate_tests_table(tests, html=True))
             f.write("<br><br>")
@@ -238,48 +236,34 @@ def generate_results_files(html=True, pdf=True, failures=0, skipped=0, start_tim
         
         hdr_page.append(Spacer(1, 20))
         hdr_page.append(Paragraph(f"<a name='{SUMMARY_HEADLINE}'/><b>{SUMMARY_HEADLINE}</b>", module_header))
-        hdr_page.append(Spacer(1, 10))
+        hdr_page.append(Spacer(1, 20))
         hdr_page.append(Paragraph(f"<u>{RUN_INFO}:</u>", bold_text_style))
         hdr_page.append(Paragraph(f"{RUN_DATETIME}: {start_time.strftime('%d/%m/%Y, %H:%M:%S')}", bold_text_style))
         hdr_page.append(Paragraph(f"{RUN_DUR}: {str(duration).split('.')[0]}", bold_text_style))
         hdr_page.append(Paragraph(f"{CERT_VER}: {CERT_VERSION}", bold_text_style))
         hdr_page.append(Spacer(1, 10))
-        if test:
+        if brg:
             hdr_page.append(Paragraph(f"<u>{TESTED_DEVICE_INFO}:</u>", bold_text_style))
-            if test.dut_is_gateway():
-                hdr_page.append(Paragraph(f"{GATEWAY} ID: {test.dut.id_str}", bold_text_style))
-                hdr_page.append(Paragraph(f"{GATEWAY} {SUP_API_VER}: {test.dut.gw_api_version}", bold_text_style))
-                hdr_page.append(Paragraph(f"{GATEWAY} {SERIALIZATION_FORMAT}: {PROTOBUF if test.dut.protobuf else JSON}", bold_text_style))
-            if test.dut_is_combo() or test.dut_is_bridge():
-                brg = test.dut.internal_brg if test.dut_is_combo() else test.dut
-                hdr_page.append(Paragraph(f"{BRG} {BLE_MAC_ADDRESS}: {brg.id_str}", bold_text_style))
+            hdr_page.append(Paragraph(f"{BRG} {BLE_MAC_ADDRESS}: {brg.id_str}", bold_text_style))
+            if isinstance(brg, cert_utils.Bridge):
                 hdr_page.append(Paragraph(f"{BRG} {BOARD_TYPE}: {ag.BOARD_TYPES_LIST[brg.board_type]}", bold_text_style))
                 hdr_page.append(Paragraph(f"{BRG} {BLE_VER}: {brg.version}", bold_text_style))
                 hdr_page.append(Paragraph(f"{BRG} {SUP_API_VER}: {brg.api_version}", bold_text_style))
-            hdr_page.append(Spacer(1, 10))
-
-            # In wlt-cert-reg there is no tester
-            if test.tester:
-                hdr_page.append(Paragraph(f"<u>{TESTING_DEVICE_INFO}:</u>", bold_text_style))
-                hdr_page.append(Paragraph(f"{SIM} {BLE_MAC_ADDRESS}: {test.tester.internal_brg.id_str}", bold_text_style))
-                hdr_page.append(Paragraph(f"{SIM} {BOARD_TYPE}: {ag.BOARD_TYPES_LIST[test.tester.internal_brg.board_type]}", bold_text_style))
-                hdr_page.append(Paragraph(f"{SIM} {BLE_VER}: {test.tester.internal_brg.version}", bold_text_style))
                 hdr_page.append(Spacer(1, 10))
-        notes_file_path = os.path.join(ARTIFACTS_DIR, RESULT_NOTES_FILE)
-        if os.path.exists(notes_file_path):
-            hdr_page.append(Paragraph(f"<u>{ADD_INFO}:</u>", bold_text_style))
-            with open(notes_file_path, 'r') as f:
-                for line in f.readlines():
-                    hdr_page.append(Paragraph(f"{line}", bold_text_style))
-
-        hdr_page.append(Spacer(1, 15))
+        if internal_brg:
+            hdr_page.append(Paragraph(f"<u>{TESTING_DEVICE_INFO}:</u>", bold_text_style))
+            hdr_page.append(Paragraph(f"{SIM} {BLE_MAC_ADDRESS}: {internal_brg.id_str}", bold_text_style))
+            hdr_page.append(Paragraph(f"{SIM} {BOARD_TYPE}: {ag.BOARD_TYPES_LIST[internal_brg.board_type]}", bold_text_style))
+            hdr_page.append(Paragraph(f"{SIM} {BLE_VER}: {internal_brg.version}", bold_text_style))
+            hdr_page.append(Paragraph(f"{SIM} {SUP_API_VER}: {internal_brg.api_version}", bold_text_style))
+        hdr_page.append(Spacer(1, 20))
 
         # Analyze results per module and generate results per module for PDF
         results_per_module = generate_results_per_module_for_pdf(tests=tests)
 
         # Modules Table
         hdr_page.append(Paragraph(f"<b>Modules Overview</b>", module_header))
-        hdr_page.append(Spacer(1, 15))
+        hdr_page.append(Spacer(1, 20))
         modules_overview = get_modules_overview(results_per_module)
         module_stats_table_data = []
         for module_name, status in modules_overview.items():
@@ -287,6 +271,7 @@ def generate_results_files(html=True, pdf=True, failures=0, skipped=0, start_tim
         module_stats_table = Table([["Module", "Status"]] + module_stats_table_data, colWidths=[100, 100])
         module_stats_table.setStyle(inner_table_style('CENTER'))
         hdr_page.append(module_stats_table)
+        hdr_page.append(Spacer(1, 20))
         hdr_page.append(PageBreak())
 
         # Tests Tables
@@ -301,7 +286,7 @@ def generate_results_files(html=True, pdf=True, failures=0, skipped=0, start_tim
         count_table = Table(count_data)
         count_table.setStyle(inner_table_style('CENTER'))
         hdr_page.append(count_table)
-        hdr_page.append(Spacer(1, 10))
+        hdr_page.append(Spacer(1, 20))
         
         # Tests Results Table
         summary_data = []
@@ -347,12 +332,8 @@ def generate_results_files(html=True, pdf=True, failures=0, skipped=0, start_tim
         import io
         from botocore.exceptions import ClientError
 
-        if test:
-            device_id = f"{test.dut.id_str}:{test.dut.internal_brg.id_str}" if test.dut_is_combo() else test.dut.id_str
-            version = test.dut.internal_brg.version if test.dut_is_combo() else test.dut.version
-        else:
-            device_id = "NO_ID"
-            version = "0.0.0"
+        device_id = brg.id_str if brg else internal_brg.id_str if internal_brg else "NO_ID"
+        version = brg.version if brg else internal_brg.version if internal_brg else "0.0.0"
         json_data = {
             "setup_name": os.environ['PIPELINE_NAME'],
             "device_id": device_id,

certificate/cert_utils.py

@@ -26,6 +26,10 @@ ORIGINAL_AG_FILE =  "wlt_types_ag.py"
 TEST_MODULES_MAP = {"calibration": ag.MODULE_CALIBRATION, "datapath": ag.MODULE_DATAPATH, "energy2400": ag.MODULE_ENERGY_2400, "energy_sub1g": ag.MODULE_ENERGY_SUB1G,
                     "pwr_mgmt": ag.MODULE_PWR_MGMT, "sensors": ag.MODULE_EXT_SENSORS, "custom": ag.MODULE_CUSTOM}
 
+# Used when mac is not really important for the test (e.g: gw_sim) or to randomize pkt data
+def get_random_hex_str(len=12):
+    return ''.join([random.choice('0123456789ABCDEF') for i in range(len)])
+
 def module2name(module_id):
     for k, v in TEST_MODULES_MAP.items():
         if module_id == v:
@@ -159,22 +163,8 @@ class WltTest:
 
     def create_test_phases_and_params(self, line_params):
         self.params = []
-        phases_source = []
-        dynamic_parameters = "dynamic_parameters" in self.test_json[ALL_SUPPORTED_VALUES]
-        if dynamic_parameters:
-            self.test_json[ALL_SUPPORTED_VALUES].remove("dynamic_parameters")
         if len(self.test_json[ALL_SUPPORTED_VALUES]) > 0:
-            if dynamic_parameters:
-                if line_params:
-                    phases_source = line_params
-                elif len(self.test_json[ALL_SUPPORTED_VALUES]) > 0:
-                    phases_source = self.test_json[ALL_SUPPORTED_VALUES]
-                else:
-                    error = f"ERROR: No dynamic parameters provided for test {self.name}! Check test list file and update the supported values!\n{[f.__dict__ for f in self.phases]}"
-                    handle_error(error, datetime.datetime.now())
-            else:
-                phases_source = self.test_json[ALL_SUPPORTED_VALUES]
-            self.phases = [Phase(PRE_CONFIG)] + [Phase(phase) for phase in phases_source] + [Phase(RESTORE_CONFIG)]
+            self.phases = [Phase(PRE_CONFIG)] + [Phase(param) for param in self.test_json[ALL_SUPPORTED_VALUES]] + [Phase(RESTORE_CONFIG)]
             for param_phase in self.phases:
                 param = Param(param_phase.name)
                 if (param.name in line_params or param.value in [eval_param(p) for p in line_params]):
@@ -230,9 +220,6 @@ class WltTest:
                 return phase
         return None
 
-    def add_phase(self, phase):
-        self.phases[-1:-1] = [phase]
-
     def update_overall_rc(self):
         if any([phase.rc == TEST_FAILED for phase in self.phases]):
             self.rc = TEST_FAILED
@@ -318,8 +305,6 @@ class Bridge:
         self.cfg_hash = interface_pkt.cfg_hash if interface_pkt else cfg_hash
         self.api_version = interface_pkt.api_version if interface_pkt else api_version
         if import_defs:
-            if self.board_type >= len(ag.BOARD_TYPES_LIST):
-                handle_error(f"ERROR: Unknown board type {self.board_type} for bridge {self.id_str}!", datetime.datetime.now())
             self.defines_file_name = f'{ag.BOARD_TYPES_LIST[self.board_type]}_defines.cfg'
             overwrite_dict = {}
             # Overwrite auto-generated defines for the specific bridge
@@ -400,7 +385,7 @@ def ut_prep_brg(args, start_time, tester, brg_id, tester_flag=False, validation_
     if args.latest or args.rc:
         return handle_prep_brg_for_latest(test, interface, brg_id, start_time)
     elif test.rc == TEST_FAILED:
-        error = f"ERROR: Didn't get ModuleIfV{test.active_brg.api_version} from BRG:{brg.id_str}!\nCheck that the brg responded with the correct module"
+        error = f"ERROR: Didn't get ModuleIfV{test.active_brg.api_version} from BRG:{brg.id_str}!"
         handle_error(error, start_time)
     version = f"{interface.major_ver}.{interface.minor_ver}.{interface.patch_ver}"
     board_type = interface.board_type
@@ -477,7 +462,7 @@ class Gateway:
 
 def get_tester_id(tester):
     if not tester or tester == GW_SIM_PREFIX:
-        return f"GW_SIM_{get_random_hex_str(12)}"
+        return f"GW_SIM_{get_random_hex_str()}"
     else:
         # Allow tester to be specified as tester_id:ble_addr
         if ':' in tester:
@@ -527,16 +512,16 @@ def prep_dut(args, tester, validation_schema, mqttc, start_time, upload_wait_tim
         # Prepare gateway's internal BRG
         if brg_flag(validation_schema):
             dut.internal_brg = ut_prep_brg(args, start_time, tester=test.dut, brg_id=args.combo_ble_addr, validation_schema=validation_schema['modules'])
-            if dut.internal_brg.api_version < API_OLDEST_SUPPORTED_VERSION:
-                 handle_error(f"ERROR: DUT internal brg FW api_version={dut.internal_brg.api_version} is lower then the oldest supported = {API_OLDEST_SUPPORTED_VERSION}! Please upgrade the internal brg FW!", start_time)
+            if dut.internal_brg.api_version != ag.API_VERSION_LATEST:
+                handle_error(f"ERROR: Certificate FW api_version={dut.internal_brg.api_version} instead of api_version={ag.API_VERSION_LATEST}! Please upgrade the FW!", start_time)
         # Return Gateway object
         return dut
 
     elif brg_flag(validation_schema):
         # Prepare standalone bridge using prepared tester
         brg = ut_prep_brg(args, start_time, tester=tester, brg_id=args.dut, validation_schema=validation_schema['modules'])
-        if brg.api_version < API_OLDEST_SUPPORTED_VERSION:
-            handle_error(f"ERROR: DUT brg FW api_version={brg.api_version} is lower then the oldest supported = {API_OLDEST_SUPPORTED_VERSION}! Please upgrade the brg FW!", start_time)
+        if brg.api_version != ag.API_VERSION_LATEST:
+            handle_error(f"ERROR: DUT FW api_version={brg.api_version} instead of api_version={ag.API_VERSION_LATEST}!", start_time)
         return brg
 
 
@@ -591,6 +576,10 @@ def prep_tester(args, mqttc, start_time, gw_sim_thread=None):
 
     # Prepare tester's internal BRG
     tester.internal_brg = ut_prep_brg(args, start_time, tester, internal_brg_mac_addr, tester_flag=True, validation_schema=None)
+    if gw_sim_thread and tester.internal_brg.version not in TESTER_FW_VERSIONS:
+        utPrint(f"ERROR: Certificate FW version={tester.internal_brg.version} instead of versions={TESTER_FW_VERSIONS}!\n"
+                f"Please run the command wlt-cert-tester-upgrade to upgrade the tester firmware!", "RED")
+        handle_error("ERROR: Tester FW version mismatch!", start_time)
     # Return Gateway object
     return tester
 
@@ -621,7 +610,6 @@ def prep_gw_info_action(test, start_time, brg_flag, target):
             gw.gw_version = {BLE_VERSION : info[BLE_VERSION], WIFI_VERSION : info[WIFI_VERSION]}
         if brg_flag and BLE_MAC_ADDR in info:
             internal_brg_mac_addr = info[BLE_MAC_ADDR]
-        # For internal use only in versions update test 
         if GW_API_VERSION in info:
             gw.gw_api_version = info[GW_API_VERSION]
 

certificate/certificate.py

@@ -51,11 +51,6 @@ def skip_test_check(test, validation_schema):
         skip_string = f"Skipped {test.module_name} because it can run only with Gateway or Combo devices"
     elif BRIDGE_ONLY_TEST in test.test_json and test.test_json[BRIDGE_ONLY_TEST] and not test.dut_is_bridge():
         skip_string = f"Skipped {test.module_name} because it can run only with Bridge devices without Gateway functionality"
-    elif SUPPORTED_FROM_API_VERSION in test.test_json:
-        if test.test_json[MODULE] == "Cloud Connectivity" and test.dut_is_gateway() and int(test.dut.gw_api_version) < test.test_json[SUPPORTED_FROM_API_VERSION]:
-            skip_string = f"Skipped {test.module_name} because it is supported from api version {test.test_json[SUPPORTED_FROM_API_VERSION]} and dut api version is {test.dut.gw_api_version}"
-        elif test.test_json[MODULE] != "Cloud Connectivity" and test.active_brg.api_version < test.test_json[SUPPORTED_FROM_API_VERSION]:
-            skip_string = f"Skipped {test.module_name} because it is supported from api version {test.test_json[SUPPORTED_FROM_API_VERSION]} and dut api version is {test.active_brg.api_version}"
     if skip_string:
         utPrint(f"{SEP}{skip_string}{SEP}", "WARNING")
         test.reason = skip_string
@@ -106,12 +101,11 @@ def main(args):
 
     utPrint(f"wiliot_certificate version: {CERT_VERSION}")
     utPrint(str(args.__dict__))
+    os.makedirs(ARTIFACTS_DIR, exist_ok=True)
     start_time = datetime.datetime.now()
 
     # Clean
     clean(args)
-    # Create artifacts dir
-    os.makedirs(ARTIFACTS_DIR, exist_ok=True)
 
     # Filter tests
     test_list = TEST_LIST_FW_UPDATE_FILE if args.latest or args.rc else args.tl
@@ -136,9 +130,9 @@ def main(args):
         # Run Gateway Simulator in separate thread
         gw_sim_thread = threading.Thread(target=cert_gw_sim.gw_sim_run, daemon=True, kwargs={'port':args.port, 'gw_id': args.tester,
                                                                                              'custom_broker':args.custom_broker,
-                                                                                             'disable_interference_analyzer':args.disable_interference_analyzer})
+                                                                                             'analyze_interference':args.analyze_interference})
         gw_sim_thread.start()
-        sleep_time = (len(cert_gw_sim.CHANNELS_TO_ANALYZE) * 30) + 15 if not args.disable_interference_analyzer else 10
+        sleep_time = (len(cert_gw_sim.CHANNELS_TO_ANALYZE) * 30) + 15 if args.analyze_interference else 10
         time.sleep(sleep_time)
     tester = cert_utils.prep_tester(args, tester_mqttc, start_time, gw_sim_thread)
     
@@ -188,13 +182,12 @@ def main(args):
         print(f"Test Duration: {test.duration}")
         print(tabulate.tabulate([[i+1, i+1-(failures+skipped), skipped, failures, len(tests)]],
                             headers=["FINISHED", "PASSED", "SKIPPED", "FAILED", "TOTAL"], tablefmt="pretty"))
-        if test.rc != TEST_SKIPPED:
-            wait_time_n_print(2)
+        wait_time_n_print(2)
         if exit_on_test_failure and test.rc == TEST_FAILED:
             break
 
     # Print results
-    cert_results.generate_results_files(html=True, pdf=True, failures=failures, skipped=skipped, start_time=start_time, tests=tests, non_cert_run=args.non_cert_run)
+    cert_results.generate_results_files(test.dut, test.tester, html=True, pdf=True, failures=failures, skipped=skipped, start_time=start_time, tests=tests, non_cert_run=args.non_cert_run)
     if not pipeline_running():
         webbrowser.open('file://' + os.path.realpath(os.path.join(ARTIFACTS_DIR, UT_RESULT_FILE_PDF)))
 

certificate/certificate_cli.py

@@ -24,23 +24,25 @@ class CertificateCLI:
             description='Certificate - CLI Tool to test Wiliot Devices',
             epilog=
             "run examples:\n"
-            "  Run command example with COM PORT tester:\n"
-            "  wlt-cert-cli --tester SIM --dut <XXXXXXXXXXXX> --port <COM_PORT>\n"
-            "  Run command example with remote GW tester:\n"
-            "  wlt-cert-cli --tester <YYYYYYYYYYYY> --dut <XXXXXXXXXXXX>\n"
+            "  Run command example with COM PORT connection:\n"
+            "  wlt-cert-cli --gw SIM --brg <XXXXXXXXXXXX> --port <COM_PORT>\n"
+            "  Run command example with remote GW connection:\n"
+            "  wlt-cert-cli --gw <YYYYYYYYYYYY> --brg <XXXXXXXXXXXX>\n"
             "  Run command example for running datapath module tests only:\n"
-            "  wlt-cert-cli --tester <YYYYYYYYYYYY> --dut <XXXXXXXXXXXX> --run datapath\n"
+            "  wlt-cert-cli --gw <YYYYYYYYYYYY> --brg <XXXXXXXXXXXX> --run datapath\n"
             "  Run command example with sanity test list:\n"
-            "  wlt-cert-cli --tester <YYYYYYYYYYYY> --dut <XXXXXXXXXXXX> --tl certificate_sanity_test_list.txt\n"
-            "  Run command example with COM PORT tester for combo device:\n"
-            "  wlt-cert-cli --tester SIM --dut <XXXXXXXXXXXX>:<YYYYYYYYYYYY> --port <COM_PORT>\n",
+            "  wlt-cert-cli --gw <YYYYYYYYYYYY> --brg <XXXXXXXXXXXX> --tl certificate_sanity_test_list.txt\n"
+            "  Run command example with COM PORT connection for bridge with cloud connectivity:\n"
+            "  wlt-cert-cli --gw SIM --brg_cloud_connectivity <XXXXXXXXXXXX> --tl certificate_bcc_test_list.txt --port <COM_PORT>\n"
+            "  Run command example with remote GW connection for bridge with cloud connectivity:\n"
+            "  wlt-cert-cli --gw <YYYYYYYYYYYY> --brg_cloud_connectivity <XXXXXXXXXXXX> --tl certificate_bcc_test_list.txt\n",
             formatter_class=argparse.RawDescriptionHelpFormatter
         )
         self.parser.add_argument('--validation_schema', '-vs', required=True, help='Validation schema json file path to use for the run')
         self.parser.add_argument('--dut', required=True, help="Tested device ID. When DUT is a combo device, use <Gateway_ID>:<BLE_MAC>")
         self.parser.add_argument('--combo_ble_addr', default="", help="Combo device BLE MAC address" + argparse.SUPPRESS)
         self.parser.add_argument('--tester', default=GW_SIM_PREFIX, help='Tester id to run on the test, SIM prefix is default and used for COM tester')
-        self.parser.add_argument('--port', '-p', default='', help='UART PORT connection to use for the run, if not provided tries to scan for tester port')
+        self.parser.add_argument('--port', '-p', default='', help='UART PORT connection to use for the run, if not provided last available port is used')
         self.parser.add_argument('--custom_broker', '-cb', required=True, help='Choose custom MQTT broker configuration json file to use for the run,' \
                                                                               ' for explanation of the format see https://community.wiliot.com/customers/s/article/Wiliot-Gateway-Certification')
         self.parser.add_argument('--clean', default=False, action='store_true', help='Clean old logs before running the tests')
@@ -50,7 +52,8 @@ class CertificateCLI:
         self.parser.add_argument('--exit_on_test_failure', default=False, action='store_true', help='Stop running the tests if a test failed ' + argparse.SUPPRESS)
         self.parser.add_argument('--exit_on_param_failure', default=False, action='store_true', help='Sets exit_on_param_failure mode to true in order to prevent \
                                  tests from continuing iteration over all possibilities in case of failure ' + argparse.SUPPRESS)
-        self.parser.add_argument('--disable_interference_analyzer', '-dia', default=False, action='store_true', help='Disable interference analysis before tests')
+        self.parser.add_argument('--analyze_interference', '-ai', default=False, action='store_true', help='Analyze interference before tests start \
+                                 (relevant only for Gateway Simulator)')
         self.parser.add_argument('--unsterile_run', '-ur', default=False, action='store_true',
                                  help="Set unsterile run mode for the run if there are pixels or energizing devices nearby")
         self.parser.add_argument("--overwrite_defaults", '-od', type=parse_dict_arg, help='Manually overwrite default defines for the bridge (key1=value1,key2=value2...)')

certificate/certificate_eth_test_list.txt

@@ -22,7 +22,6 @@ energy2400/output_power_test OUTPUT_POWER_2_4_MAX_MINUS_26 OUTPUT_POWER_2_4_MAX_
 energy2400/duty_cycle_test 1 25 50 75
 energy2400/signal_indicator_test rssi_threshold rx tx_eu_pattern disable_rx disable_tx rx_tx
 energy2400/signal_indicator_ble5_test rssi_threshold rx tx disable_rx disable_tx rx_tx
-energy2400/signal_indicator_ext_adv_test ext_adv_rx37 ext_adv_rx10
 
 # ------------- energy_sub1g -------------
 energy_sub1g/pattern_test SUB1G_ENERGY_PATTERN_NO_ENERGIZING SUB1G_ENERGY_PATTERN_SINGLE_TONE_915000 SUB1G_ENERGY_PATTERN_FCC_HOPPING SUB1G_ENERGY_PATTERN_SINGLE_TONE_917500 SUB1G_ENERGY_PATTERN_NZ_HOPPING
@@ -49,7 +48,7 @@ datapath/rx_rate_gen2_test mid_values diff_pacer min_value max_value diff_rate
 datapath/rx_rate_gen3_test mid_values diff_pacer min_value max_value diff_rate
 datapath/stress_test rep3 rep2 rep1
 datapath/stress_gen3_test rep3 rep2 rep1
-datapath/num_of_tags_test 2500
+datapath/num_of_tags_test 5000
 
 # ------------- pwr_mgmt -------------
 pwr_mgmt/pwr_mgmt_test
@@ -68,9 +67,8 @@ cloud_connectivity/uplink_test mgmt_pkt pixels_pkt sensor_pkt
 cloud_connectivity/downlink_test
 cloud_connectivity/uplink_ext_adv_test
 cloud_connectivity/reboot_test
-cloud_connectivity/stress_test
-cloud_connectivity/ext_adv_stress_test
+cloud_connectivity/stress_test 20 40 60 80 100 120 140 160 180 200 222 250 285 333
+cloud_connectivity/ext_adv_stress_test 20 40 60 80 100 120 140 160 180 200
 cloud_connectivity/channel_scan_behaviour_test
 cloud_connectivity/acl_test deny allow
-cloud_connectivity/acl_ext_adv_test deny allow
-cloud_connectivity/deduplication_test
+cloud_connectivity/acl_ext_adv_test deny allow

certificate/certificate_sanity_test_list.txt

@@ -22,7 +22,6 @@ energy2400/output_power_test OUTPUT_POWER_2_4_MAX_MINUS_14 OUTPUT_POWER_2_4_MAX_
 energy2400/duty_cycle_test 1 50 75
 energy2400/signal_indicator_test rx_tx
 energy2400/signal_indicator_ble5_test rx_tx
-energy2400/signal_indicator_ext_adv_test ext_adv_rx37
 
 # ------------- energy_sub1g -------------
 energy_sub1g/pattern_test SUB1G_ENERGY_PATTERN_NO_ENERGIZING SUB1G_ENERGY_PATTERN_FCC_HOPPING SUB1G_ENERGY_PATTERN_SINGLE_TONE_917500 SUB1G_ENERGY_PATTERN_NZ_HOPPING
@@ -60,7 +59,7 @@ cloud_connectivity/uplink_test mgmt_pkt pixels_pkt
 cloud_connectivity/downlink_test
 cloud_connectivity/uplink_ext_adv_test
 cloud_connectivity/reboot_test
-cloud_connectivity/stress_test
-cloud_connectivity/ext_adv_stress_test
+cloud_connectivity/stress_test 100 222
+cloud_connectivity/ext_adv_stress_test 20 100
 cloud_connectivity/acl_test deny
 cloud_connectivity/acl_ext_adv_test allow

certificate/certificate_test_list.txt

@@ -22,7 +22,7 @@ energy2400/output_power_test OUTPUT_POWER_2_4_MAX_MINUS_26 OUTPUT_POWER_2_4_MAX_
 energy2400/duty_cycle_test 1 25 50 75
 energy2400/signal_indicator_test rssi_threshold rx tx_eu_pattern disable_rx disable_tx rx_tx
 energy2400/signal_indicator_ble5_test rssi_threshold rx tx disable_rx disable_tx rx_tx
-energy2400/signal_indicator_ext_adv_test ext_adv_rx37 ext_adv_rx10
+#energy2400/signal_indicator_ext_adv_test rssi_threshold brg0_tx_brg1_rx brg0_rxtx_brg1_rxtx
 
 # ------------- energy_sub1g -------------
 energy_sub1g/pattern_test SUB1G_ENERGY_PATTERN_NO_ENERGIZING SUB1G_ENERGY_PATTERN_SINGLE_TONE_915000 SUB1G_ENERGY_PATTERN_FCC_HOPPING SUB1G_ENERGY_PATTERN_SINGLE_TONE_917500 SUB1G_ENERGY_PATTERN_NZ_HOPPING
@@ -49,7 +49,7 @@ datapath/rx_rate_gen2_test mid_values diff_pacer min_value max_value diff_rate
 datapath/rx_rate_gen3_test mid_values diff_pacer min_value max_value diff_rate
 datapath/stress_test rep3 rep2 rep1
 datapath/stress_gen3_test rep3 rep2 rep1
-datapath/num_of_tags_test 4500
+datapath/num_of_tags_test 5000
 
 # ------------- pwr_mgmt -------------
 pwr_mgmt/pwr_mgmt_test
@@ -67,10 +67,9 @@ cloud_connectivity/uplink_test mgmt_pkt pixels_pkt sensor_pkt geolocation
 cloud_connectivity/downlink_test
 cloud_connectivity/uplink_ext_adv_test
 cloud_connectivity/reboot_test
-cloud_connectivity/stress_test
-cloud_connectivity/ext_adv_stress_test
+cloud_connectivity/stress_test 20 40 60 80 100 120 140 160 180 200 222 250 285 333
+cloud_connectivity/ext_adv_stress_test 20 40 60 80 100 120 140 160 180 200
 cloud_connectivity/channel_scan_behaviour_test
 cloud_connectivity/acl_test deny allow
 cloud_connectivity/acl_ext_adv_test deny allow
 cloud_connectivity/brg_ota_test
-cloud_connectivity/deduplication_test

certificate/tests/calibration/interval_test/interval_test.json

@@ -10,6 +10,5 @@
     "multiBridgeTest": 0,
     "gwOnlyTest": 0,
     "dataSimOnlyTest": 0,
-    "SupportedFromApiVersion": 12,
     "allSupportedValues": [10, 25, 50, 100, 255]
 }

certificate/tests/calibration/output_power_test/output_power_test.json

@@ -12,7 +12,6 @@
     "multiBridgeTest": 0,
     "gwOnlyTest": 0,
     "dataSimOnlyTest": 0,
-    "SupportedFromApiVersion": 13,
     "allSupportedValues": ["OUTPUT_POWER_2_4_MAX", "OUTPUT_POWER_2_4_MAX_MINUS_2", "OUTPUT_POWER_2_4_MAX_MINUS_3",
                            "OUTPUT_POWER_2_4_MAX_MINUS_4", "OUTPUT_POWER_2_4_MAX_MINUS_6", "OUTPUT_POWER_2_4_MAX_MINUS_7",
                            "OUTPUT_POWER_2_4_MAX_MINUS_8", "OUTPUT_POWER_2_4_MAX_MINUS_10", "OUTPUT_POWER_2_4_MAX_MINUS_11",

certificate/tests/calibration/pattern_test/pattern_test.json

@@ -12,6 +12,5 @@
     "multiBridgeTest": 0,
     "gwOnlyTest": 0,
     "dataSimOnlyTest": 0,
-    "SupportedFromApiVersion": 12,
     "allSupportedValues": ["CALIBRATION_PATTERN_STANDARD", "CALIBRATION_PATTERN_38_38_39", "CALIBRATION_PATTERN_EU_PATTERN", "CALIBRATION_PATTERN_DISABLE_BEACON"]
 }

certificate/tests/cloud_connectivity/acl_ext_adv_test/acl_ext_adv_test.json

@@ -2,7 +2,7 @@
     "name": "Access Control List using Extended Advertising",
     "module": "Cloud Connectivity",
     "purpose": "Test the access control list (ACL) functionality, configure allow list and deny list and test the behavior, while the Bridge is transmitting in extended advertising",
-    "documentation": "https://community.wiliot.com/customers/s/article/Wiliot-Network---Gateway-Extended-Advertising-Implementation",
+    "documentation": "<TEST_DOCUMENTATION_LINK>",
     "initialCondition": "Gateway configured to defaults",
     "procedure": ["Test prolog", "configure the brg to be on the deny list", "configure the brg to be on the allow list", "Test epilog"],
     "expectedOutcome": "For the deny list - no unified packets are found, for the allow list - unified packets are found",
@@ -10,6 +10,5 @@
     "multiBridgeTest": 0,
     "gwOnlyTest": 1,
     "dataSimOnlyTest": 1,
-    "SupportedFromApiVersion": 206,
     "allSupportedValues": ["deny", "allow"]
 }

certificate/tests/cloud_connectivity/acl_test/acl_test.json

@@ -2,7 +2,7 @@
     "name": "Access Control List",
     "module": "Cloud Connectivity",
     "purpose": "Test the access control list (ACL) functionality, configure allow list and deny list and test the behavior",
-    "documentation": "https://community.wiliot.com/customers/s/article/Access-Control-List",
+    "documentation": "<TEST_DOCUMENTATION_LINK>",
     "initialCondition": "Gateway configured to defaults",
     "procedure": ["Test prolog", "configure the brg to be on the deny list", "configure the brg to be on the allow list", "Test epilog"],
     "expectedOutcome": "For the deny list - no unified packets are found, for the allow list - unified packets are found",
@@ -10,6 +10,5 @@
     "multiBridgeTest": 0,
     "gwOnlyTest": 1,
     "dataSimOnlyTest": 1,
-    "SupportedFromApiVersion": 205,
     "allSupportedValues": ["deny", "allow"]
 }

certificate/tests/cloud_connectivity/acl_test/acl_test.py

@@ -8,7 +8,7 @@ import certificate.cert_data_sim as cert_data_sim
 from certificate.ag.wlt_types_ag import *
 
 SCAN_TIMEOUT = 30
-NUM_OF_BRGS = 4
+
 # DEFINES
 test_indicator = cert_data_sim.PIXEL_SIM_INDICATOR
 
@@ -20,11 +20,11 @@ def run(test):
         return cert_common.test_epilog(test)
 
     # Generate management and data packets
-    bridge_ids = [cert_common.hex2alias_id_get(get_random_hex_str(12)) for _ in range(NUM_OF_BRGS)]
-    pixels_pkts, _ = cert_data_sim.brg_pkt_gen(num_of_pkts_per_brg=20, num_of_brgs=NUM_OF_BRGS, brgs_list=bridge_ids,
-                                               pkt_type=PIXELS_PKT, indicator=test_indicator)
+    bridge_ids = []
+    pixels_pkts, bridge_ids = cert_data_sim.brg_pkt_gen(num_of_pkts_per_brg=20, num_of_brgs=4,
+                                                        pkt_type=PIXELS_PKT, indicator=test_indicator)
 
-    mgmt_pkts, _ = cert_data_sim.brg_pkt_gen(num_of_pkts_per_brg=10, num_of_brgs=NUM_OF_BRGS, brgs_list=bridge_ids,
+    mgmt_pkts, _ = cert_data_sim.brg_pkt_gen(num_of_pkts_per_brg=10, num_of_brgs=4, brgs_list=bridge_ids,
                                              pkt_type=MGMT_PKT, indicator=test_indicator)
     # Use first 3 bridges for ACL, 4th bridge is not in ACL
     acl_bridge_ids = bridge_ids[:3]
@@ -41,7 +41,7 @@ def run(test):
         cfg = cert_config.get_default_gw_dict(test)
         cfg[ACL][ACL_BRIDGE_IDS] = acl_bridge_ids
         cfg[ACL][ACL_MODE] = param.value
-        test, _ = cert_config.gw_configure(test, cfg=cfg, wait=True)
+        test, ret = cert_config.gw_configure(test, cfg=cfg, wait=True)
         print_update_wait(1)
 
         test.get_mqttc_by_target(DUT).flush_pkts()
@@ -51,8 +51,8 @@ def run(test):
         # Analyze pass/fail
         # Get all received packets with the test indicator and filter packets by bridge IDs in ACL list
         received_pkts = cert_mqtt.get_unified_data_pkts(test, only_active_brg=False, indicator=test_indicator)
-        acl_bridge_pkts = [pkt for pkt in received_pkts if any([id in pkt[PAYLOAD] for id in acl_bridge_ids])]
-        non_acl_bridge_pkts = [pkt for pkt in received_pkts if non_acl_bridge_id in pkt[PAYLOAD]]
+        acl_bridge_pkts = [pkt for pkt in received_pkts if pkt[ALIAS_BRIDGE_ID] in acl_bridge_ids]
+        non_acl_bridge_pkts = [pkt for pkt in received_pkts if pkt[ALIAS_BRIDGE_ID] in non_acl_bridge_id]
 
         # In deny mode - we want to make sure bridges in ACL list are filtered
         if param.value == ACL_DENY:
@@ -71,18 +71,20 @@ def run(test):
                 test.add_reason(f"Phase failed! Bridge {non_acl_bridge_id} was not on the allow list "
                                 f"and {len(non_acl_bridge_pkts)} packets were found from it")
             # Check that we received packets from all expected bridges
-            received_alias_bridge_ids = set([pkt[ALIAS_BRIDGE_ID] for pkt in acl_bridge_pkts])
-            if len(received_alias_bridge_ids) != len(acl_bridge_ids):
+            received_bridge_ids = set([pkt[ALIAS_BRIDGE_ID] for pkt in acl_bridge_pkts])
+            if received_bridge_ids != set(acl_bridge_ids):
                 test.rc = TEST_FAILED
                 test.add_reason("Phase failed! Didn't receive packets from all bridges that were on the allow list")
 
         # Check that management packets are received from all bridges (ACL doesn't filter mgmt packets)
         all_mgmt_pkts = cert_mqtt.get_all_mgmt_pkts(test.get_mqttc_by_target(DUT), indicator=test_indicator)
-        mgmt_alias_bridge_ids = set([pkt[ALIAS_BRIDGE_ID] for pkt in all_mgmt_pkts])
+        mgmt_bridge_ids = set([pkt[ALIAS_BRIDGE_ID] for pkt in all_mgmt_pkts])
 
-        if len(mgmt_alias_bridge_ids) != NUM_OF_BRGS:
+        if not set(bridge_ids).issubset(mgmt_bridge_ids):
             test.rc = TEST_FAILED
-            test.add_reason("Phase failed! Didn't receive management packets from all bridges")
+            missing_bridges = set(bridge_ids) - mgmt_bridge_ids
+            test.add_reason(f"Phase failed! Expected management packets from all bridges, "
+                            f"whether they are on the ACL or not, but missing from BRG {missing_bridges}")
 
         field_functionality_pass_fail_print(test, "ACL", value=param.value)
         test.set_phase_rc(param.name, test.rc)

certificate/tests/cloud_connectivity/brg_ota_test/brg_ota_test.json

@@ -9,11 +9,10 @@
                   "Analyze if bridge was upgraded and action status message was uploaded",
                   "Test epilog"],
     "expectedOutcome": "Tester brg advertises ModuleIF packet with the new version and DUT Gateway publish an actionStatus message indicating success and  within timeout",
-    "mandatory": 0,
+    "mandatory": 1,
     "multiBridgeTest": 0,
     "gwOnlyTest": 1,
     "internalBridge": 1,
-    "SupportedFromApiVersion": 203,
     "allSupportedValues": []
 
 }