hubble-gateway-sdk-python 0.1.0__tar.gz → 0.1.1__tar.gz

hubble_gateway_sdk_python-0.1.1/.github/workflows/ci.yml

@@ -0,0 +1,65 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install
+        run: python -m pip install -e ".[dev]"
+
+      # `ruff format --check` is not enforced yet: the existing sources are not
+      # format-clean, and reformatting them belongs in its own PR.
+      - name: Ruff
+        run: ruff check .
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: pyproject.toml
+
+      - name: Install
+        run: python -m pip install -e ".[dev]"
+
+      # pytest exits 5 when it collects no tests. main has no tests yet, so
+      # treat 5 as success until the first test file lands; every other
+      # non-zero code still fails the job.
+      - name: Pytest
+        run: |
+          set +e
+          pytest -q
+          code=$?
+          if [ "$code" -eq 5 ]; then
+            echo "::notice::pytest collected no tests (exit 5); treating as success."
+            exit 0
+          fi
+          exit "$code"

{hubble_gateway_sdk_python-0.1.0 → hubble_gateway_sdk_python-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: hubble-gateway-sdk-python
-Version: 0.1.0
+Version: 0.1.1
 Summary: Python SDK for building Hubble Network BLE gateways
 Project-URL: Homepage, https://github.com/HubbleNetwork/gateway-sdk-python
 Project-URL: Documentation, https://github.com/HubbleNetwork/gateway-sdk-python#readme
@@ -42,3 +42,39 @@ Description-Content-Type: text/markdown
 # hubble-gateway
 
 Python SDK for building Hubble Network BLE gateways. Import `hubble_gateway` in your application; this package is a library only (no CLI).
+
+## Server config
+
+gateway-api sends a `config` object (or `null`) in the registration, heartbeat
+and token-refresh responses. `GatewayAuth` stores it as `server_config`.
+`GatewaySender` reads it on every use, so each new response takes effect
+without a restart.
+
+| Key | Range | Overrides |
+| --- | --- | --- |
+| `uploadIntervalMs` | 60,000 – 3,600,000 | `upload_interval_s` |
+| `uploadBatchSize` | 1 – 500 | `batch_size` |
+| `dedupWindowMs` | 5,000 – 3,600,000 | `dedup_window_s` |
+
+- Each response replaces the stored config. It does not merge into it.
+- `config: null` clears it, and the local values apply again.
+- Out-of-range values are clamped. Values of the wrong type are ignored.
+- Uploaded packets carry the 16-bit form (`FCA6`) as `serviceUuid`.
+- Other keys (`activeUploadIntervalMs`, `activeDedupWindowMs`,
+  `uploadOnCellular`) are ignored: this SDK has no motion state or cellular gating.
+
+The local default upload interval is 60 s, the same as the server's floor.
+
+## Deduplication
+
+The dedup key is `(device address, 16-bit service UUID, service data hex)`.
+For a custom 128-bit UUID, the full UUID takes the place of the 16-bit one.
+
+- The first sighting of a key opens a window of `dedup_window_s`.
+- Repeats inside the window are dropped. If the first sighting is not uploaded
+  yet, a repeat adds to its sighting count and can raise its RSSI.
+- The window is fixed at the first sighting. Repeats do not extend it, so a
+  device that beacons all the time is reported once per window.
+- Windows survive uploads. An upload does not reset them.
+- Expired keys are removed. The map is capped at `MAX_DEDUP_KEYS` (100,000);
+  past the cap, the oldest key is removed first.

hubble_gateway_sdk_python-0.1.1/README.md

@@ -0,0 +1,39 @@
+# hubble-gateway
+
+Python SDK for building Hubble Network BLE gateways. Import `hubble_gateway` in your application; this package is a library only (no CLI).
+
+## Server config
+
+gateway-api sends a `config` object (or `null`) in the registration, heartbeat
+and token-refresh responses. `GatewayAuth` stores it as `server_config`.
+`GatewaySender` reads it on every use, so each new response takes effect
+without a restart.
+
+| Key | Range | Overrides |
+| --- | --- | --- |
+| `uploadIntervalMs` | 60,000 – 3,600,000 | `upload_interval_s` |
+| `uploadBatchSize` | 1 – 500 | `batch_size` |
+| `dedupWindowMs` | 5,000 – 3,600,000 | `dedup_window_s` |
+
+- Each response replaces the stored config. It does not merge into it.
+- `config: null` clears it, and the local values apply again.
+- Out-of-range values are clamped. Values of the wrong type are ignored.
+- Uploaded packets carry the 16-bit form (`FCA6`) as `serviceUuid`.
+- Other keys (`activeUploadIntervalMs`, `activeDedupWindowMs`,
+  `uploadOnCellular`) are ignored: this SDK has no motion state or cellular gating.
+
+The local default upload interval is 60 s, the same as the server's floor.
+
+## Deduplication
+
+The dedup key is `(device address, 16-bit service UUID, service data hex)`.
+For a custom 128-bit UUID, the full UUID takes the place of the 16-bit one.
+
+- The first sighting of a key opens a window of `dedup_window_s`.
+- Repeats inside the window are dropped. If the first sighting is not uploaded
+  yet, a repeat adds to its sighting count and can raise its RSSI.
+- The window is fixed at the first sighting. Repeats do not extend it, so a
+  device that beacons all the time is reported once per window.
+- Windows survive uploads. An upload does not reset them.
+- Expired keys are removed. The map is capped at `MAX_DEDUP_KEYS` (100,000);
+  past the cap, the oldest key is removed first.

{hubble_gateway_sdk_python-0.1.0 → hubble_gateway_sdk_python-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hubble-gateway-sdk-python"
-version = "0.1.0"
+version = "0.1.1"
 description = "Python SDK for building Hubble Network BLE gateways"
 readme = "README.md"
 license = "Apache-2.0"

{hubble_gateway_sdk_python-0.1.0 → hubble_gateway_sdk_python-0.1.1}/src/hubble_gateway/__init__.py

@@ -5,7 +5,7 @@ Import the components you need to build a gateway:
     from hubble_gateway import Scanner, GatewaySender, GatewayAuth, Location, BLEPacket
 """
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"
 
 from hubble_gateway.auth import GatewayAuth
 from hubble_gateway.config import (

{hubble_gateway_sdk_python-0.1.0 → hubble_gateway_sdk_python-0.1.1}/src/hubble_gateway/auth.py

@@ -2,6 +2,7 @@
 
 import json
 import locale
+import math
 import platform
 import time
 from pathlib import Path
@@ -11,11 +12,65 @@ import httpx
 import structlog
 
 from hubble_gateway import __version__
+from hubble_gateway.config import HUBBLE_SERVICE_UUID
+from hubble_gateway.models import is_bluetooth_base_uuid, normalize_service_uuid
 
 logger = structlog.get_logger(__name__)
 
 _TOKEN_EXPIRY_MARGIN_S = 60
 
+# Server config keys this SDK applies, with the inclusive range gateway-api
+# allows for them. gateway-api may also send activeUploadIntervalMs,
+# activeDedupWindowMs and uploadOnCellular; Python has no motion state or
+# cellular gating, so those are dropped.
+SERVER_CONFIG_RANGES: dict[str, tuple[int, int]] = {
+    "uploadIntervalMs": (60_000, 3_600_000),
+    "uploadBatchSize": (1, 500),
+    "dedupWindowMs": (5_000, 3_600_000),
+}
+
+MAX_SCAN_SERVICE_UUIDS = 16
+
+
+def _sanitize_scan_service_uuids(value: Any) -> list[str] | None:
+    """Normalize a UUID list to uppercase 128-bit UUIDs, FCA6 first.
+
+    Only 16-bit services are valid (short form or written out on the Bluetooth
+    Base UUID), because gateway-api ingest keys packets on 16-bit service IDs.
+    Other entries are dropped; FCA6 is always kept, as gateway-api does.
+    Returns None when the value is not a list.
+    """
+    if not isinstance(value, list):
+        return None
+    result = [HUBBLE_SERVICE_UUID]
+    for item in value:
+        u = normalize_service_uuid(item) if isinstance(item, str) else None
+        if u is not None and is_bluetooth_base_uuid(u) and u not in result:
+            result.append(u)
+    return result[:MAX_SCAN_SERVICE_UUIDS]
+
+
+def sanitize_server_config(config: Any) -> dict[str, Any]:
+    """Keep the keys this SDK uses, clamped to their ranges.
+
+    Values of the wrong type (null, strings, bools, NaN) are dropped, so the
+    local setting applies for that key.
+    """
+    if not isinstance(config, dict):
+        return {}
+    result: dict[str, Any] = {}
+    uuids = _sanitize_scan_service_uuids(config.get("scanServiceUuids"))
+    if uuids is not None:
+        result["scanServiceUuids"] = uuids
+    for key, (low, high) in SERVER_CONFIG_RANGES.items():
+        value = config.get(key)
+        if isinstance(value, bool) or not isinstance(value, (int, float)):
+            continue
+        if isinstance(value, float) and not math.isfinite(value):
+            continue
+        result[key] = min(max(int(value), low), high)
+    return result
+
 
 def _device_id() -> str:
     machine_id_path = Path("/etc/machine-id")
@@ -188,6 +243,7 @@ class GatewayAuth:
         self._device_token = dt
         self._refresh_token = rt
         self._expires_at_ms = int(time.time() * 1000) + exp * 1000
+        self._apply_server_config(data)
         self._persist_tokens()
         logger.info("Token refreshed", expires_in_s=exp)
         return True
@@ -201,6 +257,7 @@ class GatewayAuth:
         )
         if resp.status_code == 200:
             self._apply_server_config(resp.json())
+            self._persist_tokens()
 
     async def _request(
         self, method: str, path: str, json_body: dict[str, Any] | None = None,
@@ -216,11 +273,13 @@ class GatewayAuth:
             method, f"{self._base_url}{path}", json=json_body, headers=headers,
         )
 
-    def _apply_server_config(self, data: dict[str, Any]) -> None:
-        config = data.get("config") or {}
-        for key in ("uploadIntervalMs", "uploadBatchSize", "scanIntervalMs", "featureFlags"):
-            if key in config:
-                self.server_config[key] = config[key]
+    def _apply_server_config(self, data: Any) -> None:
+        # The server sends the full effective config each time, so replace
+        # rather than merge: a removed override must not stick. A response
+        # without a "config" key says nothing and keeps the current config.
+        if not isinstance(data, dict) or "config" not in data:
+            return
+        self.server_config = sanitize_server_config(data["config"])
 
     def _persist_tokens(self) -> None:
         self._token_path.parent.mkdir(parents=True, exist_ok=True)
@@ -242,7 +301,7 @@ class GatewayAuth:
             self._device_token = data.get("deviceToken")
             self._refresh_token = data.get("refreshToken")
             self._expires_at_ms = data.get("expiresAtMs", 0)
-            self.server_config = data.get("serverConfig", {})
+            self.server_config = sanitize_server_config(data.get("serverConfig"))
             logger.info("Loaded persisted auth", gateway_id=self._gateway_id)
         except (json.JSONDecodeError, OSError) as exc:
             logger.warning("Failed to load persisted auth", error=str(exc))

{hubble_gateway_sdk_python-0.1.0 → hubble_gateway_sdk_python-0.1.1}/src/hubble_gateway/config.py

@@ -53,15 +53,15 @@ class Settings(BaseSettings):
     # Batching and deduplication
     batch_size: int = Field(
         default=500,
-        description="Max packets per upload batch",
+        description="Max packets per upload batch (server uploadBatchSize overrides)",
     )
     upload_interval_seconds: float = Field(
-        default=5.0,
-        description="Seconds between upload flushes",
+        default=60.0,
+        description="Seconds between upload flushes (server uploadIntervalMs overrides)",
     )
     dedup_window_seconds: float = Field(
         default=300.0,
-        description="Deduplication window in seconds",
+        description="Deduplication window in seconds (server dedupWindowMs overrides)",
     )
 
     # Location

{hubble_gateway_sdk_python-0.1.0 → hubble_gateway_sdk_python-0.1.1}/src/hubble_gateway/models.py

@@ -1,18 +1,49 @@
 """Data models for BLE packets and locations."""
 
 import base64
+import re
 from datetime import datetime, timezone
 from typing import Any
 
 from pydantic import BaseModel, Field
 
+BLUETOOTH_BASE_UUID_SUFFIX = "-0000-1000-8000-00805F9B34FB"
+
+_UUID16_PATTERN = re.compile(r"^[0-9A-F]{4}$")
+_UUID128_PATTERN = re.compile(
+    r"^[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}$"
+)
+
+
+def normalize_service_uuid(value: str) -> str | None:
+    """Return the uppercase 128-bit form of a BLE service UUID, or None if invalid.
+
+    ``FCA6`` -> ``0000FCA6-0000-1000-8000-00805F9B34FB``. This is the form
+    bleak reports and filters on.
+    """
+    u = value.strip().upper()
+    if _UUID16_PATTERN.match(u):
+        return f"0000{u}{BLUETOOTH_BASE_UUID_SUFFIX}"
+    if _UUID128_PATTERN.match(u):
+        return u
+    return None
+
+
+def is_bluetooth_base_uuid(full_uuid: str) -> bool:
+    u = full_uuid.upper()
+    return len(u) == 36 and u.startswith("0000") and u.endswith(BLUETOOTH_BASE_UUID_SUFFIX)
+
 
 def uuid_to_16bit(full_uuid: str) -> str:
-    """Extract 16-bit short form from a full 128-bit BLE UUID.
+    """Short form of a BLE UUID, as the gateway-api packet schema expects.
 
-    ``0000FCA6-0000-1000-8000-00805F9B34FB`` -> ``FCA6``
+    ``0000FCA6-0000-1000-8000-00805F9B34FB`` -> ``FCA6``. A UUID that is not
+    on the Bluetooth Base UUID has no 16-bit form, so it is returned whole
+    (uppercase) rather than as a misleading slice.
     """
-    return full_uuid[4:8].upper()
+    if is_bluetooth_base_uuid(full_uuid):
+        return full_uuid[4:8].upper()
+    return full_uuid.upper()
 
 
 class Location(BaseModel):
@@ -89,7 +120,7 @@ class BLEPacket(BaseModel):
 
     @property
     def service_uuid_16bit(self) -> str | None:
-        """16-bit short form of matched UUID (e.g. 'FCA6')."""
+        """Short form of matched UUID (e.g. 'FCA6'); full form if not a Base UUID."""
         if self.service_uuid:
             return uuid_to_16bit(self.service_uuid)
         return None

{hubble_gateway_sdk_python-0.1.0 → hubble_gateway_sdk_python-0.1.1}/src/hubble_gateway/scanner.py

@@ -3,6 +3,7 @@
 import asyncio
 from collections.abc import Callable
 from datetime import datetime, timezone
+from typing import TYPE_CHECKING
 
 import structlog
 from bleak import BleakScanner
@@ -10,7 +11,10 @@ from bleak.backends.device import BLEDevice
 from bleak.backends.scanner import AdvertisementData
 
 from hubble_gateway.config import DEFAULT_SERVICE_UUIDS
-from hubble_gateway.models import BLEPacket, uuid_to_16bit
+from hubble_gateway.models import BLEPacket, is_bluetooth_base_uuid, uuid_to_16bit
+
+if TYPE_CHECKING:
+    from hubble_gateway.auth import GatewayAuth
 
 logger = structlog.get_logger(__name__)
 
@@ -29,8 +33,12 @@ class Scanner:
         scan_duration: float = 5.0,
         on_packet: Callable[[BLEPacket], None] | None = None,
         adapter: str | None = None,
+        auth: "GatewayAuth | None" = None,
     ) -> None:
-        self._service_uuids = [u.lower() for u in (service_uuids or DEFAULT_SERVICE_UUIDS)]
+        self._local_service_uuids = list(service_uuids or DEFAULT_SERVICE_UUIDS)
+        self._auth = auth
+        self._service_uuids: list[str] = []
+        self._uuid_variants: dict[str, str] = {}
         self._scan_duration = scan_duration
         self._on_packet = on_packet
         self._adapter = adapter or None
@@ -39,14 +47,35 @@ class Scanner:
         self._seen_devices: set[str] = set()
         self._devices_this_scan: set[str] = set()
         self._total_packets = 0
+        self._refresh_service_uuids()
 
-        self._uuid_variants: dict[str, str] = {}
-        for full_uuid in self._service_uuids:
-            short = full_uuid[4:8]
-            canonical = f"0000{short}-0000-1000-8000-00805f9b34fb"
-            self._uuid_variants[full_uuid] = full_uuid
-            self._uuid_variants[short] = full_uuid
-            self._uuid_variants[canonical] = full_uuid
+    @property
+    def service_uuids(self) -> list[str]:
+        """The UUIDs the current scan cycle filters on (lowercase 128-bit)."""
+        return list(self._service_uuids)
+
+    def _effective_service_uuids(self) -> list[str]:
+        if self._auth is not None:
+            server = self._auth.server_config.get("scanServiceUuids")
+            if server:
+                return server
+        return self._local_service_uuids
+
+    def _refresh_service_uuids(self) -> bool:
+        """Adopt the effective UUID list; return True if it changed."""
+        uuids = [u.lower() for u in self._effective_service_uuids()]
+        if uuids == self._service_uuids:
+            return False
+        variants: dict[str, str] = {}
+        for full_uuid in uuids:
+            variants[full_uuid] = full_uuid
+            # Only a Bluetooth Base UUID has a 16-bit alias; slicing any other
+            # UUID would match an unrelated 16-bit service.
+            if is_bluetooth_base_uuid(full_uuid):
+                variants[full_uuid[4:8]] = full_uuid
+        self._service_uuids = uuids
+        self._uuid_variants = variants
+        return True
 
     def _match_service_uuid(self, advertised_uuids: list[str]) -> str | None:
         for adv_uuid in advertised_uuids:
@@ -103,17 +132,21 @@ class Scanner:
             adapter=self._adapter or "default",
         )
 
-        scanner_kwargs: dict = {
-            "detection_callback": self._detection_callback,
-            "service_uuids": self._service_uuids,
-        }
-        if self._adapter:
-            scanner_kwargs["adapter"] = self._adapter
-
         while self._running:
             if self._paused:
                 await asyncio.sleep(0.5)
                 continue
+            if self._refresh_service_uuids():
+                logger.info(
+                    "Scan UUIDs changed",
+                    service_uuids=[uuid_to_16bit(u) for u in self._service_uuids],
+                )
+            scanner_kwargs: dict = {
+                "detection_callback": self._detection_callback,
+                "service_uuids": self._service_uuids,
+            }
+            if self._adapter:
+                scanner_kwargs["adapter"] = self._adapter
             try:
                 self._devices_this_scan.clear()
                 async with BleakScanner(**scanner_kwargs) as _scanner:

{hubble_gateway_sdk_python-0.1.0 → hubble_gateway_sdk_python-0.1.1}/src/hubble_gateway/sender.py

@@ -13,15 +13,21 @@ from hubble_gateway.models import BLEPacket, Location
 
 logger = structlog.get_logger(__name__)
 
+_DedupKey = tuple[str, str | None, str]
+
+# Hard cap on remembered dedup keys. Age-based expiry normally keeps the map
+# far smaller; the cap only bounds memory if a flood of distinct keys arrives
+# inside one window.
+MAX_DEDUP_KEYS = 100_000
+
 
 class _DedupEntry:
-    __slots__ = ("packet", "strongest_rssi", "sighting_count", "first_seen_ms")
+    __slots__ = ("packet", "strongest_rssi", "sighting_count")
 
     def __init__(self, packet: BLEPacket) -> None:
         self.packet = packet
         self.strongest_rssi = packet.rssi
         self.sighting_count = 1
-        self.first_seen_ms = int(packet.timestamp.timestamp() * 1000)
 
     def merge(self, packet: BLEPacket) -> None:
         self.sighting_count += 1
@@ -38,34 +44,41 @@ class GatewaySender:
         auth: GatewayAuth,
         get_location: Callable[[], Awaitable[Location]] | None = None,
         batch_size: int = 500,
-        upload_interval_s: float = 5.0,
+        upload_interval_s: float = 60.0,
         dedup_window_s: float = 300.0,
     ) -> None:
         self._auth = auth
         self._get_location = get_location
-        self._batch_size = batch_size
-        self._upload_interval = upload_interval_s
-        self._dedup_window = dedup_window_s
-        self._dedup_map: dict[tuple[str, str | None, str], _DedupEntry] = {}
+        self._local_batch_size = batch_size
+        self._local_upload_interval_s = upload_interval_s
+        self._local_dedup_window_s = dedup_window_s
+        # Sightings waiting for the next upload.
+        self._pending: list[_DedupEntry] = []
+        # Pending entry per key for the window it opened, so in-window repeats
+        # before the upload add to its sighting count.
+        self._open: dict[_DedupKey, _DedupEntry] = {}
+        # When each key last opened a window. Survives uploads; kept in
+        # ascending time order so expiry and the cap pop from the front.
+        self._window_start_ms: dict[_DedupKey, int] = {}
         self._dedup_lock = asyncio.Lock()
         self._running = False
         self._stats_sends = 0
         self._stats_total = 0
         self._stats_uploaded = 0
+        self._stats_dropped = 0
         self._stats_devices: set[str] = set()
 
     async def start(self) -> None:
         self._running = True
-        self._apply_server_config()
         asyncio.create_task(self._upload_loop())
         if self._get_location:
             asyncio.create_task(self._location_loop())
         logger.info(
             "Sender started",
             gateway_id=self._auth.gateway_id,
-            upload_interval=self._upload_interval,
-            dedup_window=self._dedup_window,
-            batch_size=self._batch_size,
+            upload_interval=self.upload_interval_s,
+            dedup_window=self.dedup_window_s,
+            batch_size=self.batch_size,
         )
 
     async def stop(self) -> None:
@@ -82,14 +95,55 @@ class GatewaySender:
     def add_packet(self, packet: BLEPacket) -> None:
         self._stats_total += 1
         self._stats_devices.add(packet.device_address)
-        key = (packet.device_address, packet.service_uuid_16bit, packet.service_data_hex)
-        existing = self._dedup_map.get(key)
-        if existing is not None:
-            age_ms = int(packet.timestamp.timestamp() * 1000) - existing.first_seen_ms
-            if age_ms < self._dedup_window * 1000:
-                existing.merge(packet)
-                return
-        self._dedup_map[key] = _DedupEntry(packet)
+        key: _DedupKey = (
+            packet.device_address, packet.service_uuid_16bit, packet.service_data_hex,
+        )
+        now_ms = int(packet.timestamp.timestamp() * 1000)
+        window_ms = self.dedup_window_s * 1000
+        self._expire_dedup_keys(now_ms, window_ms)
+
+        # The window is anchored at the sighting that opened it, not slid by
+        # repeats: a device that beacons nonstop is still reported once per
+        # window instead of never again.
+        start_ms = self._window_start_ms.get(key)
+        if start_ms is not None and now_ms - start_ms < window_ms:
+            entry = self._open.get(key)
+            if entry is not None:
+                entry.merge(packet)
+            else:
+                self._stats_dropped += 1
+            return
+
+        self._window_start_ms.pop(key, None)
+        self._window_start_ms[key] = now_ms
+        entry = _DedupEntry(packet)
+        self._pending.append(entry)
+        self._open[key] = entry
+
+    def _expire_dedup_keys(self, now_ms: int, window_ms: float) -> None:
+        starts = self._window_start_ms
+        while starts:
+            key = next(iter(starts))
+            if now_ms - starts[key] < window_ms and len(starts) < MAX_DEDUP_KEYS:
+                break
+            del starts[key]
+
+    # Read the auth's server config on every use rather than caching it, so a
+    # new config from any registration, heartbeat or token refresh applies at
+    # once, and a cleared config falls back to the local values.
+    @property
+    def batch_size(self) -> int:
+        return self._auth.server_config.get("uploadBatchSize", self._local_batch_size)
+
+    @property
+    def upload_interval_s(self) -> float:
+        ms = self._auth.server_config.get("uploadIntervalMs")
+        return self._local_upload_interval_s if ms is None else ms / 1000
+
+    @property
+    def dedup_window_s(self) -> float:
+        ms = self._auth.server_config.get("dedupWindowMs")
+        return self._local_dedup_window_s if ms is None else ms / 1000
 
     @property
     def stats(self) -> dict[str, int]:
@@ -98,13 +152,15 @@ class GatewaySender:
             "uploaded_packets": self._stats_uploaded,
             "sends": self._stats_sends,
             "unique_devices": len(self._stats_devices),
-            "buffer_size": len(self._dedup_map),
+            "dropped_duplicates": self._stats_dropped,
+            "buffer_size": len(self._pending),
+            "dedup_keys": len(self._window_start_ms),
         }
 
     async def _upload_loop(self) -> None:
         while self._running:
-            await asyncio.sleep(self._upload_interval)
-            if not self._dedup_map:
+            await asyncio.sleep(self.upload_interval_s)
+            if not self._pending:
                 continue
             await self._flush()
 
@@ -123,10 +179,11 @@ class GatewaySender:
 
     async def _flush(self) -> None:
         async with self._dedup_lock:
-            if not self._dedup_map:
+            if not self._pending:
                 return
-            entries = list(self._dedup_map.values())
-            self._dedup_map.clear()
+            entries = self._pending
+            self._pending = []
+            self._open.clear()
 
         location = Location()
         if self._get_location:
@@ -135,8 +192,9 @@ class GatewaySender:
             except Exception as e:
                 logger.warning("Failed to get location", error=str(e))
 
-        for i in range(0, len(entries), self._batch_size):
-            batch = entries[i : i + self._batch_size]
+        batch_size = self.batch_size
+        for i in range(0, len(entries), batch_size):
+            batch = entries[i : i + batch_size]
             await self._upload_batch(batch, location)
 
     async def _upload_batch(self, entries: list[_DedupEntry], location: Location) -> None:
@@ -193,10 +251,3 @@ class GatewaySender:
             )
         except Exception as e:
             logger.warning("Location upload failed", error=str(e))
-
-    def _apply_server_config(self) -> None:
-        cfg = self._auth.server_config
-        if "uploadBatchSize" in cfg:
-            self._batch_size = min(int(cfg["uploadBatchSize"]), 500)
-        if "uploadIntervalMs" in cfg:
-            self._upload_interval = max(int(cfg["uploadIntervalMs"]) / 1000, 1)

hubble_gateway_sdk_python-0.1.1/tests/test_scan_service_uuids.py

@@ -0,0 +1,156 @@
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+from hubble_gateway import scanner as scanner_mod
+from hubble_gateway.auth import GatewayAuth, sanitize_server_config
+from hubble_gateway.config import HUBBLE_SERVICE_UUID, TILE_SERVICE_UUID
+from hubble_gateway.models import uuid_to_16bit
+from hubble_gateway.scanner import Scanner
+
+FCA6 = HUBBLE_SERVICE_UUID
+FEED = TILE_SERVICE_UUID
+CUSTOM = "12345678-ABCD-1234-ABCD-1234567890AB"
+BASE_180F = "0000180F-0000-1000-8000-00805F9B34FB"
+
+
+def _scan(raw):
+    return sanitize_server_config({"scanServiceUuids": raw}).get("scanServiceUuids")
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        (["FCA6", "FEED"], [FCA6, FEED]),
+        (["feed"], [FCA6, FEED]),
+        ([], [FCA6]),
+        (["180F", FCA6.lower()], [FCA6, BASE_180F]),
+        ([BASE_180F.lower(), "180F"], [FCA6, BASE_180F]),
+        ([CUSTOM.lower()], [FCA6]),
+        (["FEED", CUSTOM], [FCA6, FEED]),
+        (["00010000-0000-1000-8000-00805F9B34FB"], [FCA6]),
+        (["FEED", "nope", 7, None, "FCA", "FEED"], [FCA6, FEED]),
+    ],
+)
+def test_scan_list_normalized(raw, expected):
+    assert _scan(raw) == expected
+
+
+def test_scan_list_capped_at_16():
+    raw = [f"{i:04X}" for i in range(1, 40)]
+    result = _scan(raw)
+    assert len(result) == 16
+    assert result[0] == FCA6
+
+
+@pytest.mark.parametrize("raw", [None, "FCA6,FEED", {"a": 1}])
+def test_scan_list_wrong_type_ignored(raw):
+    assert _scan(raw) is None
+
+
+def test_uuid_to_16bit():
+    assert uuid_to_16bit(FCA6.lower()) == "FCA6"
+    assert uuid_to_16bit(CUSTOM.lower()) == CUSTOM
+
+
+def _auth(config=None):
+    return SimpleNamespace(server_config=config or {})
+
+
+def _adv(uuid: str, data: bytes = b"\x01"):
+    device = SimpleNamespace(address="AA:BB:CC:DD:EE:FF", name=None)
+    adv = SimpleNamespace(
+        service_uuids=[uuid],
+        service_data={uuid: data},
+        manufacturer_data={},
+        rssi=-50,
+        tx_power=None,
+        local_name=None,
+    )
+    return device, adv
+
+
+def test_scanner_local_list_without_server_config():
+    s = Scanner(service_uuids=[FCA6, FEED], auth=_auth())
+    assert s.service_uuids == [FCA6.lower(), FEED.lower()]
+
+
+def test_scanner_server_list_replaces_local():
+    packets = []
+    s = Scanner(
+        service_uuids=[FCA6, FEED],
+        on_packet=packets.append,
+        auth=_auth({"scanServiceUuids": [FCA6, BASE_180F]}),
+    )
+    assert s.service_uuids == [FCA6.lower(), BASE_180F.lower()]
+
+    s._detection_callback(*_adv(FEED.lower()))
+    assert packets == []
+
+    s._detection_callback(*_adv(BASE_180F.lower()))
+    assert packets[0].to_gateway_packet()["serviceUuid"] == "180F"
+
+    s._detection_callback(*_adv("fca6"))
+    assert packets[1].to_gateway_packet()["serviceUuid"] == "FCA6"
+
+
+def test_local_custom_uuid_has_no_16bit_alias():
+    s = Scanner(service_uuids=[CUSTOM])
+    assert s._match_service_uuid(["5678"]) is None
+
+
+async def test_scanner_applies_new_list_next_cycle(monkeypatch):
+    auth = _auth()
+    calls: list[list[str]] = []
+    s = Scanner(service_uuids=[FCA6, FEED], scan_duration=0, auth=auth)
+
+    class FakeBleak:
+        def __init__(self, **kwargs):
+            calls.append(list(kwargs["service_uuids"]))
+
+        async def __aenter__(self):
+            if len(calls) == 1:
+                auth.server_config = {"scanServiceUuids": [FCA6, BASE_180F]}
+            elif len(calls) == 2:
+                auth.server_config = {}
+            else:
+                s._running = False
+            return self
+
+        async def __aexit__(self, *exc):
+            return False
+
+    monkeypatch.setattr(scanner_mod, "BleakScanner", FakeBleak)
+    await s.start()
+    assert calls == [
+        [FCA6.lower(), FEED.lower()],
+        [FCA6.lower(), BASE_180F.lower()],
+        [FCA6.lower(), FEED.lower()],
+    ]
+
+
+async def test_auth_scan_list_absent_keeps_null_clears(tmp_path):
+    bodies = {"heartbeat": {}}
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path == "/api/v1/gateways":
+            return httpx.Response(201, json={
+                "gatewayId": "gw-1", "deviceToken": "dt", "refreshToken": "rt",
+                "expiresInSeconds": 3600, "config": {"scanServiceUuids": ["FEED", "180F"]},
+            })
+        return httpx.Response(200, json=bodies["heartbeat"])
+
+    auth = GatewayAuth("k", "https://gw.test", token_path=str(tmp_path / "a.json"))
+    auth._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    assert await auth.ensure_authenticated()
+    s = Scanner(service_uuids=[FCA6, FEED], auth=auth)
+    assert auth.server_config["scanServiceUuids"] == [FCA6, FEED, BASE_180F]
+
+    await auth.heartbeat()
+    assert auth.server_config["scanServiceUuids"] == [FCA6, FEED, BASE_180F]
+
+    bodies["heartbeat"] = {"config": None}
+    await auth.heartbeat()
+    s._refresh_service_uuids()
+    assert s.service_uuids == [FCA6.lower(), FEED.lower()]

hubble_gateway_sdk_python-0.1.1/tests/test_server_config.py

@@ -0,0 +1,291 @@
+import json
+from datetime import datetime, timedelta, timezone
+
+import httpx
+import pytest
+
+from hubble_gateway.auth import GatewayAuth, sanitize_server_config
+from hubble_gateway.config import Settings
+from hubble_gateway.models import BLEPacket
+from hubble_gateway.sender import GatewaySender
+
+NORMAL = {"uploadIntervalMs": 300_000, "uploadBatchSize": 500, "dedupWindowMs": 300_000}
+
+
+def _register_body(config):
+    return {
+        "gatewayId": "gw-1",
+        "deviceToken": "dt",
+        "refreshToken": "rt",
+        "expiresInSeconds": 3600,
+        "config": config,
+    }
+
+
+class FakeServer:
+    """Stands in for gateway-api; tests set the body each endpoint returns."""
+
+    def __init__(self) -> None:
+        self.register_config: object = None
+        self.heartbeat_body: dict = {}
+        self.refresh_body: dict = {}
+
+    def handler(self, request: httpx.Request) -> httpx.Response:
+        path = request.url.path
+        if request.method == "PUT" and path == "/api/v1/gateways":
+            body = json.loads(request.content)
+            assert body["platform"]["sdk"] == "hubble-gateway-python"
+            return httpx.Response(201, json=_register_body(self.register_config))
+        if request.method == "PUT" and path == "/api/v1/gateways/gw-1":
+            return httpx.Response(200, json=self.heartbeat_body)
+        if request.method == "POST" and path == "/api/v1/auth/refresh":
+            return httpx.Response(200, json=self.refresh_body)
+        return httpx.Response(404)
+
+
+@pytest.fixture
+def server():
+    return FakeServer()
+
+
+@pytest.fixture
+def auth(tmp_path, server):
+    a = GatewayAuth("sdk-key", "https://gw.test", token_path=str(tmp_path / "auth.json"))
+    a._client = httpx.AsyncClient(transport=httpx.MockTransport(server.handler))
+    return a
+
+
+def _packet(addr: str, ts: datetime) -> BLEPacket:
+    return BLEPacket(
+        device_address=addr,
+        rssi=-60,
+        service_data={"0000fca6-0000-1000-8000-00805f9b34fb": b"\x01"},
+        service_uuid="0000fca6-0000-1000-8000-00805f9b34fb",
+        timestamp=ts,
+    )
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        (NORMAL, NORMAL),
+        ({"uploadIntervalMs": 1_000}, {"uploadIntervalMs": 60_000}),
+        ({"uploadIntervalMs": 99_000_000}, {"uploadIntervalMs": 3_600_000}),
+        ({"uploadBatchSize": 0}, {"uploadBatchSize": 1}),
+        ({"uploadBatchSize": 10_000}, {"uploadBatchSize": 500}),
+        ({"dedupWindowMs": 1}, {"dedupWindowMs": 5_000}),
+        ({"dedupWindowMs": 99_000_000}, {"dedupWindowMs": 3_600_000}),
+        ({"uploadIntervalMs": 120_000.0}, {"uploadIntervalMs": 120_000}),
+    ],
+)
+def test_sanitize_clamps(raw, expected):
+    assert sanitize_server_config(raw) == expected
+
+
+@pytest.mark.parametrize(
+    "raw",
+    [
+        {"uploadIntervalMs": None},
+        {"uploadIntervalMs": "300000"},
+        {"uploadBatchSize": True},
+        {"dedupWindowMs": float("nan")},
+        {"dedupWindowMs": [5_000]},
+    ],
+)
+def test_sanitize_ignores_bad_types(raw):
+    assert sanitize_server_config(raw) == {}
+
+
+@pytest.mark.parametrize("raw", [None, "eco", [], 42])
+def test_sanitize_non_object(raw):
+    assert sanitize_server_config(raw) == {}
+
+
+def test_sanitize_drops_unused_keys():
+    raw = {
+        **NORMAL,
+        "scanIntervalMs": 10_000,
+        "featureFlags": {"x": True},
+        "activeUploadIntervalMs": 60_000,
+        "activeDedupWindowMs": 60_000,
+        "uploadOnCellular": False,
+    }
+    assert sanitize_server_config(raw) == NORMAL
+
+
+async def test_heartbeat_replaces_not_merges(auth, server):
+    server.register_config = NORMAL
+    assert await auth.ensure_authenticated()
+    assert auth.server_config == NORMAL
+
+    server.heartbeat_body = {"config": {"uploadIntervalMs": 900_000}}
+    await auth.heartbeat()
+    assert auth.server_config == {"uploadIntervalMs": 900_000}
+
+
+async def test_null_config_clears(auth, server):
+    server.register_config = NORMAL
+    assert await auth.ensure_authenticated()
+
+    server.heartbeat_body = {"config": None}
+    await auth.heartbeat()
+    assert auth.server_config == {}
+
+
+async def test_missing_config_key_keeps_config(auth, server):
+    server.register_config = NORMAL
+    assert await auth.ensure_authenticated()
+
+    server.heartbeat_body = {}
+    await auth.heartbeat()
+    assert auth.server_config == NORMAL
+
+
+async def test_refresh_applies_config(auth, server):
+    server.register_config = NORMAL
+    assert await auth.ensure_authenticated()
+
+    server.refresh_body = {
+        "deviceToken": "dt2",
+        "expiresInSeconds": 3600,
+        "config": {"uploadIntervalMs": 60_000, "dedupWindowMs": 60_000},
+    }
+    assert await auth._refresh()
+    assert auth.server_config == {"uploadIntervalMs": 60_000, "dedupWindowMs": 60_000}
+
+
+async def test_heartbeat_persists_config(auth, server, tmp_path):
+    server.register_config = NORMAL
+    assert await auth.ensure_authenticated()
+    server.heartbeat_body = {"config": None}
+    await auth.heartbeat()
+
+    reloaded = GatewayAuth("sdk-key", "https://gw.test", token_path=str(tmp_path / "auth.json"))
+    assert reloaded.server_config == {}
+
+
+def test_persisted_stale_keys_dropped(tmp_path):
+    path = tmp_path / "auth.json"
+    path.write_text(json.dumps({
+        "gatewayId": "gw-1",
+        "serverConfig": {"uploadIntervalMs": 1_000, "scanIntervalMs": 5, "featureFlags": {}},
+    }))
+    a = GatewayAuth("sdk-key", "https://gw.test", token_path=str(path))
+    assert a.server_config == {"uploadIntervalMs": 60_000}
+
+
+async def test_sender_reapplies_after_heartbeat(auth, server):
+    sender = GatewaySender(auth, batch_size=200, upload_interval_s=5.0, dedup_window_s=30.0)
+    assert (sender.batch_size, sender.upload_interval_s, sender.dedup_window_s) == (200, 5.0, 30.0)
+
+    server.register_config = NORMAL
+    assert await auth.ensure_authenticated()
+    assert (sender.batch_size, sender.upload_interval_s, sender.dedup_window_s) == (500, 300, 300)
+
+    server.heartbeat_body = {"config": {"uploadIntervalMs": 900_000, "dedupWindowMs": 600_000}}
+    await auth.heartbeat()
+    assert (sender.batch_size, sender.upload_interval_s, sender.dedup_window_s) == (200, 900, 600)
+
+    server.heartbeat_body = {"config": None}
+    await auth.heartbeat()
+    assert (sender.batch_size, sender.upload_interval_s, sender.dedup_window_s) == (200, 5.0, 30.0)
+
+
+async def test_sender_applies_dedup_window(auth, server):
+    server.register_config = {"dedupWindowMs": 60_000}
+    assert await auth.ensure_authenticated()
+    sender = GatewaySender(auth, dedup_window_s=1.0)
+
+    t0 = datetime.now(timezone.utc)
+    sender.add_packet(_packet("AA", t0))
+    sender.add_packet(_packet("AA", t0 + timedelta(seconds=30)))
+    assert sender.stats["buffer_size"] == 1
+    assert sender._pending[0].sighting_count == 2
+
+    sender.add_packet(_packet("AA", t0 + timedelta(seconds=61)))
+    assert sender.stats["buffer_size"] == 2
+
+
+def _capture_uploads(auth, server) -> list[list[str]]:
+    uploads: list[list[str]] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path.endswith("/packets"):
+            packets = json.loads(request.content)["packets"]
+            uploads.append([p["deviceAddress"] for p in packets])
+            return httpx.Response(202)
+        return server.handler(request)
+
+    auth._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    return uploads
+
+
+async def test_duplicate_dropped_across_flush(auth, server):
+    server.register_config = {"dedupWindowMs": 60_000}
+    assert await auth.ensure_authenticated()
+    uploads = _capture_uploads(auth, server)
+    sender = GatewaySender(auth)
+
+    t0 = datetime.now(timezone.utc)
+    sender.add_packet(_packet("AA", t0))
+    await sender._flush()
+    assert uploads == [["AA"]]
+
+    sender.add_packet(_packet("AA", t0 + timedelta(seconds=30)))
+    assert sender.stats["buffer_size"] == 0
+    assert sender.stats["dropped_duplicates"] == 1
+    await sender._flush()
+    assert uploads == [["AA"]]
+
+    sender.add_packet(_packet("AA", t0 + timedelta(seconds=61)))
+    await sender._flush()
+    assert uploads == [["AA"], ["AA"]]
+
+
+async def test_dedup_keys_expire_by_age(auth):
+    sender = GatewaySender(auth, dedup_window_s=10.0)
+    t0 = datetime.now(timezone.utc)
+    for addr in ("A1", "A2", "A3"):
+        sender.add_packet(_packet(addr, t0))
+    assert sender.stats["dedup_keys"] == 3
+
+    sender.add_packet(_packet("B1", t0 + timedelta(seconds=11)))
+    assert sender.stats["dedup_keys"] == 1
+
+
+async def test_dedup_keys_capped(auth, monkeypatch):
+    monkeypatch.setattr("hubble_gateway.sender.MAX_DEDUP_KEYS", 3)
+    sender = GatewaySender(auth, dedup_window_s=600.0)
+    t0 = datetime.now(timezone.utc)
+    for addr in ("A1", "A2", "A3", "A4", "A5"):
+        sender.add_packet(_packet(addr, t0))
+    assert sender.stats["dedup_keys"] == 3
+    assert list(sender._window_start_ms) == [("A3", "FCA6", "01"), ("A4", "FCA6", "01"),
+                                             ("A5", "FCA6", "01")]
+
+
+def test_default_upload_interval_is_60s(auth, monkeypatch):
+    monkeypatch.delenv("HUBBLE_UPLOAD_INTERVAL_SECONDS", raising=False)
+    assert Settings(_env_file=None).upload_interval_seconds == 60.0
+    assert GatewaySender(auth).upload_interval_s == 60.0
+
+
+async def test_sender_flush_uses_server_batch_size(auth, server):
+    server.register_config = {"uploadBatchSize": 2}
+    assert await auth.ensure_authenticated()
+
+    batches: list[int] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        if request.url.path.endswith("/packets"):
+            batches.append(len(json.loads(request.content)["packets"]))
+            return httpx.Response(202)
+        return server.handler(request)
+
+    auth._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+    sender = GatewaySender(auth, batch_size=500)
+    t0 = datetime.now(timezone.utc)
+    for addr in ("A1", "A2", "A3", "A4", "A5"):
+        sender.add_packet(_packet(addr, t0))
+    await sender._flush()
+    assert batches == [2, 2, 1]

hubble_gateway_sdk_python-0.1.0/README.md

@@ -1,3 +0,0 @@
-# hubble-gateway
-
-Python SDK for building Hubble Network BLE gateways. Import `hubble_gateway` in your application; this package is a library only (no CLI).