rtlamr-python 1.0.0__py3-none-any.whl

rtlamr_python/__init__.py

@@ -0,0 +1,17 @@
+from rtlamr_python.listener import (
+    ALL_PROTOCOLS,
+    DEFAULT_PROTOCOLS,
+    ListenerHandle,
+    listen,
+    listen_once,
+    start_listening,
+)
+
+__all__ = [
+    "ALL_PROTOCOLS",
+    "DEFAULT_PROTOCOLS",
+    "ListenerHandle",
+    "listen",
+    "listen_once",
+    "start_listening",
+]

rtlamr_python/__main__.py

@@ -0,0 +1,4 @@
+from rtlamr_python.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

rtlamr_python/cli.py

@@ -0,0 +1,233 @@
+"""rtlamr-python — Multi-protocol ERT smart meter receiver.
+
+Reads IQ samples from an RTL-SDR dongle (or a raw sample file for testing),
+decodes ERT packets, and prints each message as a JSON line to stdout.
+
+Supported protocols (use --protocol to select; defaults to all Manchester ones):
+  scmplus — Standard Consumption Message Plus (16 bytes, most electric meters)
+  scm     — Standard Consumption Message (12 bytes, older electric meters)
+  idm     — Interval Data Message (92 bytes, hourly interval data)
+  netidm  — Net Meter Interval Data Message (92 bytes, net-metering variant)
+  r900    — Neptune R900 water meters (different center freq: 912.38 MHz)
+
+Usage:
+  # All Manchester protocols on live hardware
+  rtlamr
+
+  # Single protocol
+  rtlamr --protocol scmplus
+
+  # From a recorded capture file
+  rtlamr --sample-file /path/to/capture.bin
+
+  # Filter to specific meters
+  rtlamr --meter-id 12345678
+
+  # Alternate between Manchester and R900 (different center frequencies)
+  rtlamr --protocol scmplus r900
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import signal
+import sys
+import threading
+
+from rtlamr_python.listener import ALL_PROTOCOLS, DEFAULT_PROTOCOLS, listen
+from rtlamr_python.poster import ApiPoster
+
+LOG = logging.getLogger(__name__)
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    p = argparse.ArgumentParser(description="Multi-protocol ERT smart meter receiver")
+    p.add_argument(
+        "--config",
+        default=None,
+        metavar="PATH",
+        help="TOML config file (default: none)",
+    )
+    p.add_argument(
+        "--verbose",
+        action="store_true",
+        default=None,
+        help="print per-decoder stats to stderr every 500 blocks",
+    )
+    p.add_argument(
+        "--chip-length",
+        type=int,
+        default=None,
+        metavar="N",
+        help="chip length in samples (default: 72 → ~2.36 MHz sample rate)",
+    )
+    p.add_argument(
+        "--gain",
+        default=None,
+        metavar="GAIN",
+        help='tuner gain in dB or "auto" (default: auto)',
+    )
+    p.add_argument(
+        "--freq-correction",
+        type=int,
+        default=None,
+        metavar="PPM",
+        dest="freq_correction",
+        help="frequency correction for the RTL-SDR oscillator in parts per million "
+             "(negative if signals appear below their expected frequency)",
+    )
+    p.add_argument(
+        "--meter-id",
+        type=int,
+        nargs="+",
+        default=None,
+        metavar="ID",
+        dest="meter_id",
+        help="only forward readings from these endpoint IDs (space-separated); "
+             "overrides meter_ids in the config file",
+    )
+    p.add_argument(
+        "--sample-file",
+        default=None,
+        metavar="PATH",
+        help="read raw IQ bytes from a file instead of live hardware",
+    )
+    p.add_argument(
+        "--duration",
+        type=float,
+        default=None,
+        metavar="SECONDS",
+        help="stop after this many seconds (0 = run forever)",
+    )
+    p.add_argument(
+        "--protocol",
+        nargs="+",
+        choices=list(ALL_PROTOCOLS),
+        default=None,
+        metavar="PROTO",
+        help="protocols to decode (space-separated); default: all Manchester",
+    )
+    p.add_argument(
+        "--api-url",
+        default=None,
+        metavar="URL",
+        help='base URL of the REST API (e.g. "http://localhost:8000/api"); '
+             "if omitted, readings are only written to stdout",
+    )
+    p.add_argument(
+        "--api-key",
+        default=None,
+        metavar="KEY",
+        help="value for the X-API-Key header (only needed when the API requires auth)",
+    )
+    p.add_argument(
+        "--switch-timeout",
+        type=float,
+        default=None,
+        metavar="SECONDS",
+        help="alternating mode: switch frequency after this many seconds without "
+             "a message (default: 60)",
+    )
+    return p.parse_args(argv)
+
+
+def _load_config(path: str | None) -> dict:
+    if path is None:
+        return {}
+    import tomllib
+    try:
+        with open(path, "rb") as f:
+            return tomllib.load(f)
+    except FileNotFoundError:
+        return {}
+
+
+_DEFAULTS: dict = {
+    "chip_length": 72,
+    "gain": "auto",
+    "freq_correction": 0,
+    "duration": 0.0,
+    "verbose": False,
+    "switch_timeout": 60.0,
+}
+
+_CONFIG_KEYS = {
+    "api_url", "api_key", "meter_id", "meter_ids", "protocol", "switch_timeout",
+    "gain", "freq_correction", "chip_length", "duration", "verbose",
+}
+
+
+def _apply_config(args: argparse.Namespace, cfg: dict) -> None:
+    """Back-fill args still at None from cfg, then apply built-in defaults."""
+    # Normalize meter ID to list[int] or None.
+    if args.meter_id is None:
+        if "meter_ids" in cfg:
+            args.meter_id = list(cfg["meter_ids"])
+        elif "meter_id" in cfg:
+            args.meter_id = [cfg["meter_id"]]  # legacy single-int key
+
+    # Normalize protocol to list[str] or None.
+    if args.protocol is None and "protocol" in cfg:
+        raw = cfg["protocol"]
+        if isinstance(raw, list):
+            args.protocol = raw
+        else:
+            args.protocol = [p.strip() for p in str(raw).split(",")]
+
+    for key in _CONFIG_KEYS - {"meter_id", "meter_ids", "protocol"}:
+        if getattr(args, key) is None and key in cfg:
+            setattr(args, key, cfg[key])
+    for key, default in _DEFAULTS.items():
+        if getattr(args, key) is None:
+            setattr(args, key, default)
+
+
+def main(argv: list[str] | None = None) -> int:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(message)s",
+        stream=sys.stderr,
+    )
+
+    args = parse_args(argv)
+    _apply_config(args, _load_config(args.config))
+
+    poster = ApiPoster(args.api_url, args.api_key) if args.api_url else None
+
+    # Graceful shutdown on SIGINT.
+    stop_event = threading.Event()
+
+    def _stop(signum, frame):
+        LOG.info("Received signal %s, stopping.", signum)
+        stop_event.set()
+
+    signal.signal(signal.SIGINT, _stop)
+
+    try:
+        for record in listen(
+            protocols=args.protocol or list(DEFAULT_PROTOCOLS),
+            meter_id=args.meter_id,
+            chip_length=args.chip_length,
+            gain=args.gain,
+            freq_correction=args.freq_correction,
+            sample_file=args.sample_file,
+            switch_timeout=args.switch_timeout,
+            duration=args.duration,
+            verbose=args.verbose,
+            stop_event=stop_event,
+        ):
+            sys.stdout.write(json.dumps(record) + "\n")
+            sys.stdout.flush()
+            if poster:
+                poster.submit(record)
+    except ValueError as exc:
+        LOG.error(str(exc))
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

rtlamr_python/crc.py

@@ -0,0 +1,75 @@
+"""CCITT-16 CRC used by the SCM+ protocol (CRC-16/GENIBUS variant).
+
+Port of rtlamr-go/crc/crc.go.
+
+Polynomial : 0x1021
+Init value : 0xFFFF
+XOR-out    : 0xFFFF  (meters complement the CRC before embedding it as PacketCRC)
+Valid residue: 0x1D0F  (checksum over bytes[2:] of a valid SCM+ packet equals this)
+
+To embed a valid PacketCRC when constructing a packet:
+    pkt_crc = checksum(bytes[2:14]) ^ 0xFFFF
+
+To verify a received packet:
+    valid(packet)  — checks checksum(packet[2:]) == RESIDUE
+"""
+
+_POLY = 0x1021
+_INIT = 0xFFFF
+RESIDUE = 0x1D0F
+
+# Build 256-entry lookup table once at import time.
+_table: list[int] = []
+for _i in range(256):
+    _crc = _i << 8
+    for _ in range(8):
+        if _crc & 0x8000:
+            _crc = (_crc << 1) ^ _POLY
+        else:
+            _crc <<= 1
+        _crc &= 0xFFFF
+    _table.append(_crc)
+
+
+def checksum(data: bytes | bytearray, init: int = _INIT) -> int:
+    """Return the CCITT-16 CRC of *data* starting from *init*."""
+    crc = init
+    for byte in data:
+        crc = ((crc << 8) ^ _table[(crc >> 8) ^ byte]) & 0xFFFF
+    return crc
+
+
+def valid(packet: bytes | bytearray) -> bool:
+    """Return True when the CRC of packet[2:] equals the expected residue."""
+    return checksum(packet[2:]) == RESIDUE
+
+
+# ---------------------------------------------------------------------------
+# BCH-16 CRC used by the SCM protocol
+# ---------------------------------------------------------------------------
+
+_BCH_POLY = 0x6F63
+
+_bch_table: list[int] = []
+for _i in range(256):
+    _crc = _i << 8
+    for _ in range(8):
+        if _crc & 0x8000:
+            _crc = (_crc << 1) ^ _BCH_POLY
+        else:
+            _crc <<= 1
+        _crc &= 0xFFFF
+    _bch_table.append(_crc)
+
+
+def bch_checksum(data: bytes | bytearray, init: int = 0) -> int:
+    """Return the BCH-16 CRC of *data* (poly=0x6F63, init=0, xorout=0)."""
+    crc = init
+    for byte in data:
+        crc = ((crc << 8) ^ _bch_table[(crc >> 8) ^ byte]) & 0xFFFF
+    return crc
+
+
+def bch_valid(packet: bytes | bytearray) -> bool:
+    """Return True when bch_checksum(packet[2:12]) == 0 (SCM validation)."""
+    return bch_checksum(packet[2:12]) == 0

rtlamr_python/decoder.py

@@ -0,0 +1,275 @@
+"""Signal processing pipeline for ERT smart meter packets.
+
+Port of rtlamr-go/protocol/decode.go.
+
+Pipeline per block:
+  1. magnitude()      — uint8 IQ pairs → float64 signal via MagLUT
+  2. matched_filter() — Manchester-coded cumulative-sum filter → quantized bits
+  3. _search()        — find preamble positions in quantized signal
+  4. _slice_packets() — extract packet bytes at each preamble position
+  5. parser callable  — protocol-specific CRC check and field extraction
+
+The Decoder class maintains two rolling buffers (signal + quantized) so that
+packets spanning a block boundary are not lost.
+
+Each protocol supplies its own Config (via make_config()) and parse callable.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+import numpy as np
+
+# SCM+ protocol constants (from rtlamr-go/scmplus/scmplus.go)
+_CENTER_FREQ = 912_600_155  # Hz
+_DATA_RATE = 32_768         # bps
+
+# Default preamble for SCM+ — 0x16A3 = 0001011010100011
+_PREAMBLE_BITS = np.array(
+    [int(b) for b in "0001011010100011"], dtype=np.uint8
+)
+_PACKET_SYMBOLS = 128  # 16 bytes × 8 bits
+
+
+@dataclass
+class Config:
+    chip_length: int = 72
+    preamble_bits: np.ndarray = field(
+        default_factory=lambda: _PREAMBLE_BITS.copy()
+    )
+    packet_symbols: int = _PACKET_SYMBOLS
+    center_freq: int = _CENTER_FREQ
+
+    # All fields below are derived from the above in __post_init__.
+    symbol_length: int = field(init=False)
+    sample_rate: int = field(init=False)
+    preamble_symbols: int = field(init=False)
+    preamble_length: int = field(init=False)
+    packet_length: int = field(init=False)
+    block_size: int = field(init=False)
+    block_size2: int = field(init=False)
+    buffer_length: int = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.preamble_symbols = len(self.preamble_bits)
+        self.symbol_length = self.chip_length * 2
+        self.sample_rate = _DATA_RATE * self.chip_length
+        self.preamble_length = self.preamble_symbols * self.symbol_length
+        self.packet_length = self.packet_symbols * self.symbol_length
+        self.block_size = _next_power_of_2(self.preamble_length)
+        self.block_size2 = self.block_size * 2
+        self.buffer_length = self.packet_length + self.block_size
+
+
+def _next_power_of_2(v: int) -> int:
+    return 1 << math.ceil(math.log2(v))
+
+
+# ---------------------------------------------------------------------------
+# MagLUT — pre-computed normalised squared magnitudes for each uint8 value
+# ---------------------------------------------------------------------------
+
+_mag_lut: np.ndarray = ((127.5 - np.arange(256, dtype=np.float64)) / 127.5) ** 2
+
+
+def magnitude(block: np.ndarray) -> np.ndarray:
+    """Convert raw uint8 IQ block to float64 magnitude array.
+
+    Input shape: (2N,) — interleaved I, Q bytes.
+    Output shape: (N,) — sum of squared normalised I and Q.
+    """
+    samples = _mag_lut[block]
+    return samples[0::2] + samples[1::2]
+
+
+# ---------------------------------------------------------------------------
+# Matched filter for Manchester-coded signal
+# ---------------------------------------------------------------------------
+
+def matched_filter(signal: np.ndarray, chip_length: int) -> np.ndarray:
+    """Apply Manchester-coded matched filter; return quantized 0/1 uint8 array.
+
+    Uses a cumulative-sum approach identical to Decoder.Filter in the Go code.
+    """
+    symbol_length = chip_length * 2
+    csum = np.empty(len(signal) + 1, dtype=np.float64)
+    csum[0] = 0.0
+    np.cumsum(signal, out=csum[1:])
+
+    n_out = len(signal) - symbol_length + 1
+    lower = csum[chip_length: chip_length + n_out]
+    base = csum[:n_out]
+    upper = csum[symbol_length: symbol_length + n_out]
+
+    f = (lower - base) - (upper - lower)
+    return (f >= 0).astype(np.uint8)[:n_out]
+
+
+# ---------------------------------------------------------------------------
+# Preamble search
+# ---------------------------------------------------------------------------
+
+def _search(quantized: np.ndarray, preamble: np.ndarray, cfg: Config) -> list[int]:
+    """Return sample indices where *preamble* exists in *quantized*.
+
+    Two-pass approach ported from Decoder.Search:
+      1. Coarse byte-level pass — eliminate bytes that can't contain preamble start.
+      2. Fine bit-level pass — verify exact preamble match at symbol boundaries.
+    """
+    sym = cfg.symbol_length
+    block = cfg.block_size
+
+    # Pack quantized signal to bytes (8 bits each).
+    packed = np.packbits(quantized[:block + cfg.preamble_length])
+
+    sym_bytes = sym >> 3
+
+    # Coarse pass: find byte positions where the first preamble bit might start.
+    first_bit_mask = 0x00 if preamble[0] == 1 else 0xFF
+    candidates = [i for i, b in enumerate(packed[:block >> 3]) if b != first_bit_mask]
+
+    # Eliminate byte-level candidates for each subsequent preamble bit.
+    for p_idx in range(1, len(preamble)):
+        bit_mask = 0x00 if preamble[p_idx] == 1 else 0xFF
+        offset = p_idx * sym_bytes
+        candidates = [i for i in candidates if packed[i + offset] != bit_mask]
+        if not candidates:
+            return []
+
+    # Expand byte candidates to bit-level indices.
+    bit_candidates = [byte_idx * 8 + bit_off
+                      for byte_idx in candidates
+                      for bit_off in range(8)]
+
+    # Fine pass: verify the full preamble at each bit-level index.
+    valid: list[int] = []
+    q = quantized
+    for idx in bit_candidates:
+        for p_idx, p_bit in enumerate(preamble):
+            if q[idx + p_idx * sym] != p_bit:
+                break
+        else:
+            valid.append(idx)
+
+    return valid
+
+
+# ---------------------------------------------------------------------------
+# Packet slicing
+# ---------------------------------------------------------------------------
+
+def _slice_packets(quantized: np.ndarray, indices: list[int], cfg: Config) -> list[bytes]:
+    """Extract and pack packet bits at each preamble index."""
+    packets = []
+    for idx in indices:
+        if idx > cfg.block_size:
+            # Packet will be fully available in the next block.
+            continue
+        bits = quantized[idx: idx + cfg.packet_symbols * cfg.symbol_length: cfg.symbol_length]
+        if len(bits) < cfg.packet_symbols:
+            continue
+        packed = np.packbits(bits[:cfg.packet_symbols])
+        packets.append(packed.tobytes())
+    return packets
+
+
+# ---------------------------------------------------------------------------
+# Decoder
+# ---------------------------------------------------------------------------
+
+class Decoder:
+    """Stateful decoder that accumulates sample blocks and emits messages.
+
+    Matches Go's rolling-buffer design:
+      _signal    — length block_size + symbol_length  (short window for filter continuity)
+      _quantized — length buffer_length               (shifted left each block, tail filled)
+
+    Pass a protocol-specific parser callable to decode into a specific message type.
+    Defaults to SCM+ if no parser is provided.
+    """
+
+    def __init__(self, cfg: Config | None = None, parser: Callable | None = None) -> None:
+        self.cfg = cfg or Config()
+
+        if parser is None:
+            from rtlamr_python.protocols.scmplus import parse as _default_parse
+            self._parser: Callable = _default_parse
+        else:
+            self._parser = parser
+
+        # Signal window: [symbol_length carry-over | block_size new magnitudes]
+        self._signal = np.zeros(self.cfg.block_size + self.cfg.symbol_length, dtype=np.float64)
+
+        # Quantized history: buffer_length, shifted left each block.
+        self._quantized = np.zeros(self.cfg.buffer_length, dtype=np.uint8)
+
+        # De-duplicate messages seen in consecutive blocks (same packet at block boundary).
+        self._prev_digests: set[bytes] = set()
+        self._next_digests: set[bytes] = set()
+
+        # Diagnostic counters — reset never; read externally for logging.
+        self.stats: dict[str, int] = {
+            "candidates": 0,   # preamble positions found by _search
+            "raw_packets": 0,  # packets sliced from those positions
+            "parse_ok": 0,     # packets accepted by the parser
+            "dedup_drop": 0,   # valid packets suppressed as duplicates
+        }
+
+    @property
+    def block_size2(self) -> int:
+        return self.cfg.block_size2
+
+    def reset(self) -> None:
+        """Zero rolling buffers and clear dedup sets. Call after an SDR retune."""
+        self._signal[:] = 0
+        self._quantized[:] = 0
+        self._prev_digests.clear()
+        self._next_digests.clear()
+
+    def decode(self, raw_block: bytes) -> list[Any]:
+        """Process one raw IQ block; return any newly decoded messages."""
+        cfg = self.cfg
+        block = np.frombuffer(raw_block, dtype=np.uint8)
+
+        # Shift signal left by block_size, keeping symbol_length samples as carry-over.
+        self._signal[:cfg.symbol_length] = self._signal[cfg.block_size:]
+        mag = magnitude(block)  # block_size2 bytes → block_size magnitudes
+        self._signal[cfg.symbol_length:] = mag
+
+        # Filter signal window → trim to exactly block_size quantized outputs.
+        filt = matched_filter(self._signal, cfg.chip_length)[:cfg.block_size]
+
+        # Shift quantized left by block_size, fill tail with new filter output.
+        self._quantized[:cfg.packet_length] = self._quantized[cfg.block_size:]
+        self._quantized[cfg.packet_length:] = filt
+
+        indices = _search(self._quantized, cfg.preamble_bits, cfg)
+        raw_packets = _slice_packets(self._quantized, indices, cfg)
+
+        self.stats["candidates"] += len(indices)
+        self.stats["raw_packets"] += len(raw_packets)
+
+        self._next_digests.clear()
+        messages: list[Any] = []
+
+        for raw in raw_packets:
+            msg = self._parser(raw)
+            if msg is None:
+                continue
+
+            self.stats["parse_ok"] += 1
+
+            # Use raw bytes as dedup key — same transmission → same bytes.
+            if raw in self._prev_digests or raw in self._next_digests:
+                self.stats["dedup_drop"] += 1
+                continue
+
+            self._next_digests.add(raw)
+            messages.append(msg)
+
+        self._prev_digests, self._next_digests = self._next_digests, self._prev_digests
+
+        return messages