wiliot-certificate 4.4.3__py3-none-any.whl → 4.5.0a1__py3-none-any.whl

certificate/tests/cloud_connectivity/connection_test/connection_test.py

@@ -0,0 +1,74 @@
+import datetime
+
+from certificate import cert_config
+from certificate.cert_prints import *
+from certificate.cert_defines import *
+from certificate.wlt_types import *
+import certificate.cert_common as cert_common
+import certificate.cert_mqtt as cert_mqtt
+from common.api_if.api_validation import api_validation
+
+# DEFINES
+TIMEOUT_IN_MINUTES = 3
+
+
+def run(test):
+    test = cert_common.test_prolog(test)
+    if test.rc == TEST_FAILED:
+        return cert_common.test_epilog(test)
+
+    # Stage setup
+    phase_run_print("Connection test started")
+    dut_mqttc = test.get_mqttc_by_target(DUT)
+
+    # Initiate action
+    if not cert_common.pipeline_running():
+        warning_prefix = COLORS["WARNING"]
+        warning_suffix = COLORS["ENDC"]
+        input(
+            (
+                f"{warning_prefix}The GW is expected to publish a configuration "
+                "JSON/Protobuf message through the status topic upon connecting "
+                "to mqtt:\nPlease unplug GW from power. Press enter when unplugged"
+                f"{warning_suffix}"
+            )
+        )
+        dut_mqttc.flush_pkts()
+        input(
+            f"{warning_prefix}Please plug GW back to power. "
+            f"Press enter when plugged{warning_suffix}"
+        )
+    else:
+        # If this test runs as part of a pipeline, act like the reboot test and don't demand a manual disconnect & connect
+        utPrint(f"Publishing reboot action to {dut_mqttc.update_topic}. Awaiting reconnect.. (timeout is {TIMEOUT_IN_MINUTES} minutes)")
+        dut_mqttc.flush_pkts()
+        cert_config.gw_action(test, f"{REBOOT_GW_ACTION}", target=DUT)
+
+    # Wait for response
+    gw_type = None
+    utPrint(f'Waiting for GW to connect... (Timeout {TIMEOUT_IN_MINUTES} minutes)')
+    timeout = datetime.datetime.now() + datetime.timedelta(minutes=TIMEOUT_IN_MINUTES)
+    while datetime.datetime.now() < timeout:
+        gw_type, msg = cert_common.get_gw_type(dut_mqttc)
+        if gw_type is not None:
+            break
+        print_update_wait(5)
+
+    # generate logs
+    cert_mqtt.generate_log_file(test, "reboot")
+
+    # Analyze results
+    if gw_type is None:
+        test.rc = TEST_FAILED
+        test.reason = "The gateway did not reboot properly, status message was not received"
+    elif gw_type == "other":
+        test.rc = TEST_FAILED
+        test.reason = f"gatewayType must be defined in the status message {msg}"
+    else:
+        utPrint("Gateway rebooted and uploaded a configuration message as expected.", "GREEN")
+        utPrint(f"The configuration message received:\n {msg}")
+
+    utPrint("Checking the status message matches API format...")
+    test = api_validation(test)
+
+    return cert_common.test_epilog(test)

certificate/tests/cloud_connectivity/downlink_test/downlink_test.json

@@ -0,0 +1,20 @@
+{
+    "name": "Gateway Downlink",
+    "module": "Cloud Connectivity",
+    "purpose": "Validate device advertises requested txPackets uniformly across primary channels",
+    "documentation": ["https://community.wiliot.com/customers/s/article/Wiliot-Gateway-Actions"],
+    "initialCondition": "Device is online, connected to MQTT and supports downlink",
+    "procedure": [
+        "Test prolog",
+        "Correlation - Trigger downlink packet; validate advertised packets increase in time and count per txMaxDuration",
+        "Uniformity - Confirm equal distribution across the 3 primary channels",
+        "Test epilog"
+    ],
+    "expectedOutcome": "Equal advertising across 3 primary channels with constant intervals over the specified txMaxDuration",
+    "mandatory": 1,
+    "multiBridgeTest": 0,
+    "gwOnlyTest": 1,
+    "dataSimOnlyTest": 0,
+    "allSupportedValues": []
+
+}

certificate/tests/cloud_connectivity/downlink_test/downlink_test.py

@@ -0,0 +1,204 @@
+import time
+import pandas as pd
+import plotly.express as px
+
+from certificate.cert_prints import *
+from certificate.cert_defines import *
+from certificate.wlt_types import *
+import certificate.cert_common as cert_common
+import certificate.cert_config as cert_config
+import certificate.cert_mqtt as cert_mqtt
+from certificate.cert_gw_sim import DEDUPLICATION_PKTS
+
+
+# DEFINES
+RAW_TX_DATA = "1E16C6FC123456789123456789123456789123456789123456789100000000"
+STAGE_DURATIONS = [100, 1000, 16000]
+DOWNLINK_PUBLISH_COUNT = range(3)
+CHANNELS = [(37, ag.RX_CHANNEL_37), (38, ag.RX_CHANNEL_38), (39, ag.RX_CHANNEL_39)]
+
+# HELPER FUNCTIONS
+
+
+def create_unique_raw_packet(base_raw_packet: str, duration: int, retry: int, channel: int) -> str:
+    """
+    Modifies the base_raw_packet string by replacing its tail with a padded duration (5 digits) and retry digit.
+    Examples:
+        create_unique_raw_packet(RAW_TX_DATA, 700, 2, 37)   -> '...0000000000000000000000000 00700237'
+        create_unique_raw_packet(RAW_TX_DATA, 1500, 1, 38)  -> '...0000000000000000000000000 01500138'
+        create_unique_raw_packet(RAW_TX_DATA, 16000, 9, 39) -> '...0000000000000000000000000 16000939'
+    """
+    return f"{base_raw_packet[:-8]}{duration:05d}{retry:01d}{channel:02d}"
+
+
+def send_tested_payloads(test, rx_channel, tx_max_durations, downlink_publish_count, sniffed_pkts):
+    for duration in tx_max_durations:
+        for count_num in downlink_publish_count:
+            retries = duration // 20
+            raw_tx_data = create_unique_raw_packet(RAW_TX_DATA, duration, count_num, rx_channel)
+            dut_mqttc = test.get_mqttc_by_target(DUT)
+            utPrint(
+                f'Publishing to topic: {dut_mqttc.update_topic} packets with Duration of: {duration} for channel {rx_channel}',
+                "BLUE"
+            )
+            cert_config.gw_downlink(test, raw_tx_data=raw_tx_data, max_duration=duration, max_retries=retries)
+            sniffed_pkts[rx_channel][raw_tx_data] = {
+                'tx_max_duration': duration,
+                'tx_max_retries': retries,
+                'retry': count_num,
+                'num_pkts_received': 0
+            }
+            scan_duration = max(0.5, (duration / 1000) * 1.2)
+            mqtt_scan_wait(test, duration=scan_duration, target=TESTER)
+
+
+def process_sniffed_packets(test, channel, sniffed_pkts):
+    data_pkts = cert_mqtt.get_all_data_pkts(test.get_mqttc_by_target(TESTER))
+    for pkt in data_pkts:
+        for sent_pkt in sniffed_pkts[channel]:
+            if sent_pkt in pkt[PAYLOAD]:
+                sniffed_pkts[channel][sent_pkt]['num_pkts_received'] += 1
+
+
+def calc_for_stage_downlink(rsquared, slope):
+    error_msg = ''
+    if slope == 0:  # Meaning no packets received
+        error_msg = (
+            "Actions were sent to the GW to advertise packets, but no packets "
+            "were received by the tester"
+        )
+        return "failed", error_msg
+    elif rsquared > 0.6 and slope > 0:
+        return "passed", error_msg
+    else:
+        error_msg = (
+            "The correlation between increasing advertising duration and the "
+            "number of packets received by the tester is weak"
+        )
+        return "failed", error_msg
+
+
+def report_and_results(test, sniffed_pkts):
+    utPrint('-' * 50)
+    for channel, pkt_map in sniffed_pkts.items():
+        if not pkt_map:
+            continue
+        received_width = max(len(str(meta.get('num_pkts_received', 0))) for meta in pkt_map.values())
+        sent_width = max(len(str(meta.get('tx_max_retries', 0))) for meta in pkt_map.values())
+        duration_width = max(len(str(meta.get('tx_max_duration', ''))) for meta in pkt_map.values())
+        retry_width = max(len(str(meta.get('retry', ''))) for meta in pkt_map.values())
+        for _, meta in pkt_map.items():
+            received_raw = meta.get('num_pkts_received', 0)
+            # Guard against fractional artifacts; display only whole packets
+            received = int(received_raw)
+            sent = int(meta.get('tx_max_retries', 0))
+            percentage = (received / sent * 100) if sent else 0.0
+            duration = meta.get('tx_max_duration', '')
+            retry = meta.get('retry', '')
+            print(
+                f"Number of pkts received on channel {channel} for duration {duration:>{duration_width}} "
+                f"and retry {retry:>{retry_width}} got {received:>{received_width}} packets | "
+                f"Sent: {sent:>{sent_width}} [{percentage:6.2f}%]"
+            )
+    utPrint('-' * 50)
+    # Flatten nested dict into list of row dicts
+    rows = []
+    for channel, pkt_map in sniffed_pkts.items():
+        for raw_payload, meta in pkt_map.items():
+            row = {'channel': str(channel),
+                   'raw_tx_data': raw_payload,
+                   'tx_max_duration': meta.get('tx_max_duration'),
+                   'tx_max_retries': meta.get('tx_max_retries'),
+                   'retry': meta.get('retry'),
+                   'num_pkts_received': meta.get('num_pkts_received')
+                   }
+            rows.append(row)
+    df_received_pkts = pd.DataFrame(rows)
+
+    if df_received_pkts.empty:
+        utPrint('No packets collected; skipping graph generation.', 'WARNING')
+        test.rc = TEST_FAILED
+        test.reason = 'No packets collected.'
+        return
+
+    if test.active_brg.board_type == ag.BOARD_TYPE_FANSTEL_WIFI_V0:
+        x_value = ('tx_max_retries', 'TX Max Retries')
+    else:
+        x_value = ('tx_max_duration', 'TX Max Duration')
+
+    # Create scatter with OLS trendline per channel
+    fig = px.scatter(
+        df_received_pkts,
+        x=x_value[0], y='num_pkts_received', color='channel', title=f'Packets Received by Sniffer / {x_value[1]}', trendline='ols',
+        labels={x_value[0]: x_value[1], 'num_pkts_received': 'Number of packets received', 'channel': 'BLE Adv. Channel'}
+    )
+    fig.update_layout(scattermode='group', scattergap=0.95)
+
+    # Extract regression results
+    trendline_info = px.get_trendline_results(fig)
+
+    # Calculate whether stage pass/failed
+    for channel, channel_df in trendline_info.groupby('BLE Adv. Channel'):
+        channel_pkts = df_received_pkts[df_received_pkts['channel'] == channel]
+        channel_trendline = channel_df['px_fit_results'].iloc[0]
+        slope = channel_trendline.params[1]
+        rsquared = channel_trendline.rsquared
+        total_received = int(channel_pkts['num_pkts_received'].sum())
+        channel_rc, channel_err_summary = calc_for_stage_downlink(rsquared, slope)
+
+        utPrint(f'Channel {channel}: {channel_rc}')
+        utPrint(f'- Total {len(channel_pkts)} MQTT payloads sent')
+        utPrint(f'- Total {total_received} BLE packets received by sniffer (including duplicates)')
+        utPrint(f'- R Value: {rsquared} | Slope: {slope}')
+
+        if channel_rc == 'failed':
+            test.rc = TEST_FAILED
+            test.reason = channel_err_summary
+
+    # Export graph
+    html_file_path = os.path.join(ARTIFACTS_DIR, test.dir, 'downlink_graph.html')
+    fig.write_html(html_file_path)
+    utPrint('-' * 50)
+
+
+def run(test):
+    test = cert_common.test_prolog(test)
+    if test.rc == TEST_FAILED:
+        return cert_common.test_epilog(test)
+
+    # Stage setup
+    utPrint("Setting TESTER GW to allow duplicate packets", "GREEN")
+    cert_config.gw_action(test, f"{DEDUPLICATION_PKTS} 0", target=TESTER)
+
+    # Clean
+    sniffed_pkts = {}
+
+    for channel, channel_idx in CHANNELS:
+        # Setup
+        test.flush_all_mqtt_packets()
+        sniffed_pkts[channel] = {}
+
+        # Configuring
+        utPrint(f'Configuring TESTER to RX channel {channel}', "GREEN")
+        test = cert_config.brg_configure(test, fields=[BRG_RX_CHANNEL],
+                                         values=[channel_idx],
+                                         module=test.tester.internal_brg.datapath,
+                                         target=TESTER)[0]
+        time.sleep(2)  # Wait for sniffer to start
+
+        # Send
+        send_tested_payloads(test, channel, STAGE_DURATIONS, DOWNLINK_PUBLISH_COUNT, sniffed_pkts)
+
+        # Receive packets
+        process_sniffed_packets(test, channel, sniffed_pkts)
+
+        # Channel switch setup
+        cert_mqtt.generate_log_file(test, f'{channel}', TESTER)
+
+    # Analyze and report
+    report_and_results(test, sniffed_pkts)
+    utPrint("Resetting TESTER GW to re-enable packet deduplication", "GREEN")
+    cert_config.gw_action(test, f"{DEDUPLICATION_PKTS} 1", target=TESTER)
+    test = cert_config.config_brg_defaults(test, modules=[test.tester.internal_brg.datapath], target=TESTER)[0]
+
+    return cert_common.test_epilog(test)

certificate/tests/cloud_connectivity/ext_adv_stress_test/ext_adv_stress_test.json

@@ -0,0 +1,16 @@
+{
+    "name": "Gateway Uplink Extended Advertising Stress",
+    "module": "Cloud Connectivity",
+    "purpose": "Test gateway's stress handling capabilities for different rates and durations when scanning in extended advertising",
+    "documentation": "https://community.wiliot.com/customers/s/article/Wiliot-Network---Gateway-Extended-Advertising-Implementation",
+    "initialCondition": "Gateway configured to defaults, in a BLE-clean environment",
+    "procedure": ["Test prolog",
+                  "For each pps value, advertise extended advertising packets at that rate, with pointers sent on each of the three BLE advertising channels, and compare the advertised vs uploaded rate",
+                  "Test epilog"],
+    "expectedOutcome": "Find the maximum PPS supported by the gateway when scanning in extended advertising",
+    "mandatory": 0,
+    "multiBridgeTest": 0,
+    "gwOnlyTest": 1,
+    "dataSimOnlyTest": 1,
+    "allSupportedValues": [20, 40, 60, 80, 100, 120, 140, 160, 180, 200]
+}

certificate/tests/cloud_connectivity/ext_adv_stress_test/ext_adv_stress_test.py

@@ -0,0 +1,98 @@
+
+from certificate.cert_prints import *
+from certificate.cert_defines import *
+from certificate.wlt_types import *
+from certificate.ag.wlt_types_ag import OUTPUT_POWER_2_4_MAX
+import certificate.cert_common as cert_common
+import certificate.cert_utils as cert_utils
+import certificate.cert_mqtt as cert_mqtt
+import certificate.cert_config as cert_config
+import certificate.cert_gw_sim as cert_gw_sim
+import math
+
+
+# DEFINES
+STRESS_TEST_INDICATOR = cert_utils.get_random_hex_str(len=6)
+
+DUPLICATES = 1
+ADVA_ASCII_LEN = 12
+
+LOWEST_PPS = 20
+ADV_DURATION_LOWEST_PPS = 70
+ADV_DURATION_DEFAULT = 30
+
+
+# HELPER FUNCTIONS
+
+def generate_extended_adv_payloads_list(pkt, duplicates):
+    """
+    Should always stay synced to how cmd_ble_sim generate additional packets with unique_pkts_count
+    """
+    data_payload = pkt[12:]
+    ext_adv_payload = data_payload * 7
+    last_hex = pkt[-8:]
+    last_int = int.from_bytes(bytes.fromhex(last_hex), byteorder='little', signed=False)
+
+    pkts_list = []
+    for i in range(duplicates):
+        value = last_int + i
+        # Convert back to 4 bytes in little-endian and then to hex
+        new_hex = value.to_bytes(4, byteorder='little', signed=False).hex().upper()
+
+        pkts_list.append(pkt[:12] + ext_adv_payload[:-8] + new_hex)
+
+    return pkts_list
+
+
+def run(test):
+    test = cert_common.test_prolog(test)
+    if test.rc == TEST_FAILED:
+        return cert_common.test_epilog(test)
+
+    cert_config.gw_action(test, f'{cert_gw_sim.BLE_SIM_INIT} 1', TESTER)
+
+    test_pkts_received = []
+    data_pkts_received = []
+    results = []
+    for param in test.params:
+        phase_run_print(f"PPS = {param}")
+        test.flush_all_mqtt_packets()
+        pps = param.value
+        # adv_duration is increased only for lowest pps to increase packets sample for reliable results
+        adv_duration = ADV_DURATION_LOWEST_PPS if pps == LOWEST_PPS else ADV_DURATION_DEFAULT
+        delay = math.floor(1000 / pps)  # This should be bigger than 2ms
+        upload_wait_time = test.dut.upload_wait_time + 15
+
+        # Generate pkts and get ready to advertise
+        payload = cert_common.generate_adv_payload(STRESS_TEST_INDICATOR)
+        generated_payloads = generate_extended_adv_payloads_list(payload, adv_duration * pps)
+
+        # We provide the tester with the first pkt only. It then advertises in a loop pkts identical
+        # to the ones we generated (by incrementing the last bytes).
+        gw_action_cmd = (f'{cert_gw_sim.BLE_SIM_EXT_ADV} {payload} {DUPLICATES} '
+                         f'{OUTPUT_POWER_2_4_MAX} {delay} {adv_duration * pps}')
+        cert_config.gw_action(test, gw_action_cmd, TESTER)
+        mqtt_scan_wait(test, adv_duration + upload_wait_time)
+
+        cert_mqtt.dump_pkts(test, log=param.name)
+        phase_pkts_received = cert_mqtt.get_all_aggregated_data_pkts(test.get_mqttc_by_target(DUT), indicator=STRESS_TEST_INDICATOR)
+
+        test, percentage_received = cert_common.stress_analysis(
+            test, pps, generated_payloads,
+            [(p[ALIAS_BRIDGE_ID] + p[AGGREGATED_PAYLOAD]) for p in phase_pkts_received])
+        results.extend([pps, percentage_received])
+        cert_mqtt.generate_log_file(test, param.name)
+
+        test.set_phase_rc(param.name, test.rc)
+        test.add_phase_reason(param.name, test.reason)
+        if test.rc == TEST_FAILED and test.exit_on_param_failure:
+            break  # break the whole for loop and keep the test as failed
+        test.reset_result()  # reset result and continue to next param
+        data_pkts_received.extend(cert_mqtt.get_all_data_pkts(test.get_mqttc_by_target(DUT)))
+        test_pkts_received.extend(test.get_mqttc_by_target(DUT)._userdata[PKTS].data)
+    cert_config.gw_action(test, f'{cert_gw_sim.BLE_SIM_INIT} 0', TESTER)
+
+    # Generate stress test graphs
+    cert_common.generate_graph_stress_test(test, results, data_pkts_received)
+
+    return cert_common.test_epilog(test)

certificate/tests/cloud_connectivity/reboot_test/reboot_test.json

@@ -0,0 +1,17 @@
+{
+    "name": "GW Reboot Action",
+    "module": "Cloud Connectivity",
+    "purpose": "Verify the device reboots upon receiving reboot action",
+    "documentation": ["https://community.wiliot.com/customers/s/article/Wiliot-Gateway-Actions"],
+    "initialCondition": "Gateway online and connected to MQTT",
+    "procedure": ["Test prolog",
+                  "Reboot action sent to tested GW",
+                  "Wait for device to connect to MQTT by verifying a status message or fail when timeout occurs",
+                  "Test epilog"],
+    "expectedOutcome": "Device reboot (disconnect from MQTT then reconnect and publish its configurations to the status topic) before timeout",
+    "mandatory": 0,
+    "multiBridgeTest": 0,
+    "gwOnlyTest": 1,
+    "allSupportedValues": []
+
+}

certificate/tests/cloud_connectivity/reboot_test/reboot_test.py

@@ -0,0 +1,57 @@
+import datetime
+
+from certificate.cert_prints import *
+from certificate.cert_defines import *
+from certificate.wlt_types import *
+import certificate.cert_common as cert_common
+import certificate.cert_config as cert_config
+import certificate.cert_mqtt as cert_mqtt
+from common.api_if.api_validation import api_validation
+
+
+# DEFINES
+TIMEOUT_IN_MINUTES = 3
+
+
+def run(test):
+    test = cert_common.test_prolog(test)
+    if test.rc == TEST_FAILED:
+        return cert_common.test_epilog(test)
+
+    # Stage setup
+    phase_run_print("Reboot started")
+
+    # Initiate action
+    dut_mqttc = test.get_mqttc_by_target(DUT)
+    utPrint(f"Publishing reboot action to {dut_mqttc.update_topic}. Awaiting reboot.. (timeout is {TIMEOUT_IN_MINUTES} minutes)")
+    cert_config.gw_action(test, f"{REBOOT_GW_ACTION}", target=DUT)
+    dut_mqttc.flush_pkts()
+
+    # Wait for response
+    gw_type = None
+    utPrint(f'Waiting for GW to connect... (Timeout {TIMEOUT_IN_MINUTES} minutes)')
+    timeout = datetime.datetime.now() + datetime.timedelta(minutes=TIMEOUT_IN_MINUTES)
+    while datetime.datetime.now() < timeout:
+        gw_type, msg = cert_common.get_gw_type(dut_mqttc)
+        if gw_type is not None:
+            break
+        print_update_wait(5)
+
+    # generate logs
+    cert_mqtt.generate_log_file(test, "reboot")
+
+    # Analyze results
+    if gw_type is None:
+        test.rc = TEST_FAILED
+        test.reason = "The gateway did not reboot properly, status message was not received"
+    elif gw_type == "other":
+        test.rc = TEST_FAILED
+        test.reason = f"gatewayType must be defined in the status message {msg}"
+    else:
+        utPrint("Gateway rebooted and uploaded a configuration message as expected.", "GREEN")
+        utPrint(f"The configuration message received:\n {msg}")
+
+    utPrint("Checking the status message matches API format...")
+    test = api_validation(test)
+
+    return cert_common.test_epilog(test)

certificate/tests/cloud_connectivity/registration_test/registration_test.json

@@ -0,0 +1,19 @@
+{
+    "name": "Registration",
+    "module": "Cloud Connectivity",
+    "purpose": "Verify the device register, connect to MQTT, upload a status message & manage/refresh tokens correctly",
+    "documentation": "https://community.wiliot.com/customers/s/article/Registering-Third-Party-Gateways",
+    "initialCondition": "Gateway unregistered anywhere, attempts to connect to Wiliot's AWS MQTT broker. Testing PC has API key as environment variable",
+    "procedure": ["Test prolog",
+                  "Registry - Pre-register the gw, then device-authorize & registry requests are expected",
+                  "Online - Token acquisition and MQTT connection",
+                  "Active - Status message upload",
+                  "Refresh - Token refreshed by GW",
+                  "Test epilog"],
+    "expectedOutcome": "Device goes through the Wiliot registration flow & token refresh",
+    "mandatory": 1,
+    "multiBridgeTest": 0,
+    "gwOnlyTest": 1,
+    "internalBridge": 0,
+    "allSupportedValues": ["registry", "online", "refresh"]
+}