eigsep_observing 2.5.0__py3-none-any.whl

eigsep_observing/__init__.py

@@ -0,0 +1,25 @@
+__author__ = "Christian Hellum Bye"
+__version__ = "2.5.0"
+
+from .client import PandaClient
+from .motion_switch import MotionSwitchCoordinator
+from .observer import EigObserver
+from .fpga import EigsepFpga
+from .motor_client import MotorClient
+from .motor_zeroer import MotorZeroer
+from .status_log_handler import StatusStreamHandler
+from .tempctrl_client import TempCtrlClient
+
+try:
+    from . import testing
+except ImportError as e:
+    import logging
+
+    # Use a named logger (not the root convenience function) so the
+    # warning does NOT trigger logging.basicConfig() and silently
+    # install a stderr StreamHandler on the root logger — that would
+    # defeat configure_eig_logger(console=False) callers downstream.
+    logging.getLogger(__name__).warning(
+        f"Could not import testing module: {e}, use pip install .[dev] to "
+        "install the required dependencies for testing if needed."
+    )

eigsep_observing/_scripts_util.py

@@ -0,0 +1,115 @@
+"""Shared helpers for the standalone manual test scripts under ``scripts/``.
+
+Three responsibilities, all deliberately small:
+
+- :func:`build_transport` mirrors the ``--dummy`` bootstrap used by
+  the multi-pico manual scripts (``motor_manual.py`` / ``motor_control.py``
+  / ``potmon_manual.py`` / ...) so every per-app manual script can
+  share one well-tested transport path.
+- :func:`build_transport_bare` is the equivalent for bring-up scripts
+  that build their *own* minimal producer surface and explicitly do
+  not want a ``DummyPandaClient`` masquerading on the transport in
+  dummy mode (see ``scripts/CLAUDE.md`` for the contract).
+- :func:`require_pico` is a structural availability check the manual
+  scripts run before issuing any command, so an operator who forgot to
+  start ``pico-manager.service`` (or who flashed the wrong pico) gets a
+  one-line actionable error instead of a silent ``send_command`` no-op.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+
+from eigsep_redis import Transport
+
+
+logger = logging.getLogger(__name__)
+
+
+def build_transport(
+    dummy: bool,
+    *,
+    host: str = "localhost",
+    real_port: int = 6379,
+    dummy_port: int = 6380,
+) -> Transport:
+    """Return a :class:`Transport` matching the manual-script convention.
+
+    Real mode just constructs a ``Transport(host, real_port)``.
+
+    Dummy mode constructs a transport against a separate fakeredis port
+    (``dummy_port``, conventionally 6380), resets the fakeredis state,
+    and attaches a fully-emulated ``DummyPandaClient`` to the transport
+    so every dummy pico (motor, rfswitch, tempctrl, potmon, imu_el,
+    imu_az, lidar) is registered and emitting status before the caller
+    starts issuing proxy commands. The client is stashed on
+    ``transport._dummy_client`` to keep it alive — without that
+    reference Python would garbage-collect the embedded PicoManager
+    threads and the proxy would see ``is_available=False``.
+
+    Bring-up scripts that build their own minimal producer surface
+    (e.g. ``vna_manual.py`` / ``record_vna.py``) want
+    :func:`build_transport_bare` instead — the auto-attached
+    ``DummyPandaClient`` here violates the no-PandaClient-masquerade
+    rule documented in ``scripts/CLAUDE.md`` and competes with the
+    script's own dummy-pico bootstrap.
+    """
+    if dummy:
+        logger.warning("Running in DUMMY mode, no hardware will be used.")
+        from .testing import DummyPandaClient
+
+        transport = Transport(host=host, port=dummy_port)
+        transport.reset()
+        transport._dummy_client = DummyPandaClient(transport=transport)
+        return transport
+    return Transport(host=host, port=real_port)
+
+
+def build_transport_bare(
+    dummy: bool,
+    *,
+    host: str = "localhost",
+    real_port: int = 6379,
+    dummy_port: int = 6380,
+) -> Transport:
+    """Return a :class:`Transport` with no client attached.
+
+    The bring-up-script variant of :func:`build_transport`: same port
+    convention, same fakeredis reset on dummy, but explicitly does
+    **not** instantiate a ``DummyPandaClient`` on the transport.
+
+    Bring-up scripts (``vna_manual.py``, ``record_vna.py``, etc.) build
+    their own minimal producer surface (see ``scripts/CLAUDE.md``) and
+    spin up only the dummy picos they actually need. Attaching a
+    ``DummyPandaClient`` from this helper would (a) start a heartbeat
+    thread that collides with whatever the script is meant to test and
+    (b) double-register dummy picos if the script also calls
+    ``start_dummy_pico_manager`` directly.
+    """
+    if dummy:
+        logger.warning("Running in DUMMY mode, no hardware will be used.")
+        transport = Transport(host=host, port=dummy_port)
+        transport.reset()
+        return transport
+    return Transport(host=host, port=real_port)
+
+
+def require_pico(proxy, *, hint_script: str = "pico_preflight.py") -> None:
+    """Exit with a clear message if ``proxy``'s heartbeat is missing.
+
+    Manual scripts go through :class:`picohost.proxy.PicoProxy`, which
+    silently returns ``None`` from :meth:`send_command` when the
+    targeted pico's heartbeat is absent. That's the right behavior for
+    long-running observing loops, but a manual operator running a
+    bring-up script needs to know immediately that the pico-manager
+    service isn't reachable.
+    """
+    if proxy.is_available:
+        return
+    sys.stderr.write(
+        f"ERROR: pico {proxy.name!r} is not registered with PicoManager.\n"
+        f"  - Is pico-manager.service running on the panda?\n"
+        f"  - Was the pico flashed? Run scripts/{hint_script} to verify.\n"
+    )
+    raise SystemExit(2)

eigsep_observing/_test_fixtures.py

@@ -0,0 +1,293 @@
+"""Golden fixtures for the eigsep_observing test suite and for the
+producer-contract tests shipped under ``eigsep_observing.contract_tests``.
+
+These constants and helpers are deliberately constructed to mirror the
+shape and types of real production data. See the "Testing philosophy"
+section in CLAUDE.md: fixtures should look like what producers actually
+emit so tests catch contract drift, and any deviations should be called
+out explicitly.
+
+They live under ``src/`` (rather than in ``tests/conftest.py``) so the
+producer-contract suite can ship inside the installed wheel — the
+eigsep-field CLI runs it via ``pytest --pyargs
+eigsep_observing.contract_tests`` on nodes that only have the wheel
+(the Pi), not the test tree. The leading underscore marks this module
+as private: it is not part of the supported public API of
+eigsep_observing and its shape can change without a deprecation cycle.
+The in-repo ``tests/conftest.py`` re-exports from here so existing
+``from conftest import HEADER`` imports in ``test_io.py`` keep working
+unchanged.
+
+These are kept as plain module-level constants rather than
+``@pytest.fixture`` functions because they are referenced from inside
+nested data structures (e.g. ``CORR_METADATA``) and from helper module
+imports — both of which the parameter-injection style does not support
+without significant test rewrites.
+"""
+
+import numpy as np
+from picohost.base import PicoPeltier
+from picohost.testing import TempCtrlEmulator
+
+# One corr file accumulates NTIMES integrations, each of duration
+# INTEGRATION_TIME seconds. FILE_TIME = NTIMES * INTEGRATION_TIME is the
+# wall-clock duration of the file and is included in HEADER so the
+# relationship is explicit at the fixture level (rather than being two
+# independently-set numbers that can drift apart).
+NTIMES = 60
+INTEGRATION_TIME = 1.0  # seconds
+FILE_TIME = NTIMES * INTEGRATION_TIME  # seconds
+
+# HEADER mimics EigsepFpga.header: the static-configuration portion of a
+# corr file. Units match corr_config.yaml: sample_rate is in MHz (NOT Hz).
+#
+# ``pol_delay`` is a nested dict (one key per pol-pair) matching the
+# shape emitted by the real header property — not three flat keys.
+# ``wiring`` is the hardware manifest (split out of the old ``rf_chain``
+# key in corr_config.yaml); a single antenna with no ``pam:`` block is
+# included so consumers exercise the PAM-absent code path.
+HEADER = {
+    "dtype": ">i4",
+    "acc_bins": 2,
+    "avg_even_odd": True,
+    "nchan": 1024,
+    "fpg_file": "fpg_files/eigsep_fengine.fpg",
+    "fpg_version": [0, 0],
+    "corr_acc_len": 2**28,
+    "corr_scalar": 2**9,
+    "pol_delay": {"01": 0, "23": 0, "45": 0},
+    "fft_shift": 0x00FF,
+    "sample_rate": 500.0,  # MHz, matching corr_config.yaml convention
+    "adc_gain": 4,
+    "wiring": {
+        "snap_id": "C000069",
+        "ants": {
+            "viv1-N": {
+                "fem": {"id": 32, "pol": "N"},
+                "snap": {"input": 2, "label": "N4"},
+            },
+        },
+    },
+    "sync_time": 1748732903.4203713,
+    "integration_time": INTEGRATION_TIME,
+    "file_time": FILE_TIME,
+}
+
+# Schema-conformant raw IMU reading (as emitted by a pico and pushed into
+# stream:imu_el by picohost). Mirrors the BNO085 UART RVC payload
+# introduced in picohost 1.0.0: yaw/pitch/roll orientation in degrees and
+# accel_x/y/z in m/s². Used to build CORR_METADATA entries and by tests
+# that feed raw stream data into File.add_data.
+IMU_READING = {
+    "sensor_name": "imu_el",
+    "status": "update",
+    "app_id": 3,
+    "yaw": 0.0,
+    "pitch": 0.0,
+    "roll": 0.0,
+    "accel_x": 0.0,
+    "accel_y": 0.0,
+    "accel_z": 9.81,
+}
+
+
+def _imu_avg_entry(yaw):
+    """One per-sample IMU entry as avg_metadata would emit it.
+
+    All numeric fields are float and take the float→mean reduction in
+    ``_avg_sensor_values``. ``yaw`` is the per-sample varying axis used
+    by tests that need to assert on a non-constant float field.
+    """
+    return {
+        "sensor_name": "imu_el",
+        "status": "update",
+        "app_id": 3,
+        "yaw": yaw,
+        "pitch": 0.0,
+        "roll": 0.0,
+        "accel_x": 0.0,
+        "accel_y": 0.0,
+        "accel_z": 9.81,
+    }
+
+
+def _lidar_avg_entry(distance_m):
+    """One per-sample lidar entry as avg_metadata would emit it."""
+    return {
+        "sensor_name": "lidar",
+        "status": "update",
+        "app_id": 4,
+        "distance_m": distance_m,
+    }
+
+
+def tempctrl_post_handler_reading(stream_name):
+    """One per-channel tempctrl reading after ``_peltier_redis_handler``.
+
+    The firmware/emulator emits one combined ``sensor_name="tempctrl"``
+    dict per status tick; ``PicoPeltier._peltier_redis_handler`` fans
+    that into two ``writer.add(...)`` calls — one per channel — and is
+    the boundary that produces the actual Redis-stream shapes. Composing
+    the emulator with the handler here means every test fixture downstream
+    is anchored to the real producer output rather than drifting on
+    hand-typed steady-state values.
+    """
+    pel = PicoPeltier.__new__(PicoPeltier)
+    captured = []
+    pel._base_redis_handler = lambda d: captured.append(dict(d))
+    pel._peltier_redis_handler(TempCtrlEmulator().get_status())
+    for entry in captured:
+        if entry.get("sensor_name") == stream_name:
+            return entry
+    raise AssertionError(
+        f"_peltier_redis_handler did not emit a {stream_name!r} entry; "
+        f"got sensor_names={[e.get('sensor_name') for e in captured]}"
+    )
+
+
+def _tempctrl_channel_entry(sensor_name, t_now, timestamp):
+    """One per-sample tempctrl channel (``tempctrl_lna`` or ``tempctrl_load``).
+
+    Steady-state fields are derived from
+    ``tempctrl_post_handler_reading`` so the fixture stays anchored to
+    real ``TempCtrlEmulator`` + ``PicoPeltier._peltier_redis_handler``
+    output. Only ``T_now`` and ``timestamp`` are per-sample test inputs;
+    everything else (``T_target``, drive flags, watchdog) is the
+    emulator's steady-state value.
+    """
+    entry = tempctrl_post_handler_reading(sensor_name)
+    entry["T_now"] = t_now
+    entry["timestamp"] = timestamp
+    return entry
+
+
+def _potmon_avg_entry(pot_el_voltage):
+    """One per-sample potmon entry as avg_metadata would emit it.
+
+    Mirrors the post-``_pot_redis_handler`` shape that lands in Redis:
+    raw voltages plus the flattened cal slope/intercept and the derived
+    angle. All-scalar per the picohost scalar-only contract; the cal
+    fields are de-facto invariants for the lifetime of a stream.
+
+    A *calibrated* reading is used here so every cal/angle field is a
+    real float, exercising the float→mean reduction in
+    ``_avg_sensor_values``. The uncalibrated-stream case (cal/angle
+    fields all ``None``) is a first-class producer state — see the
+    ``potmon`` schema comment in ``io.py`` — but it is intentionally
+    not exercised by this golden fixture because it would force the
+    round-trip assertion to special-case ``None`` survivors and obscure
+    the steady-state contract this fixture is meant to pin. Tests that
+    need to cover the uncalibrated path should build their own samples.
+    Same rationale as ``_potmon_post_handler_reading`` in
+    ``test_producer_contracts.py``.
+    """
+    return {
+        "sensor_name": "potmon",
+        "status": "update",
+        "app_id": 2,
+        "pot_el_voltage": pot_el_voltage,
+        "pot_az_voltage": 1.5,
+        "pot_el_cal_slope": 100.0,
+        "pot_el_cal_intercept": -50.0,
+        "pot_az_cal_slope": 200.0,
+        "pot_az_cal_intercept": -100.0,
+        "pot_el_angle": 100.0 * pot_el_voltage - 50.0,
+        "pot_az_angle": 200.0 * 1.5 - 100.0,
+    }
+
+
+ERROR_INTEGRATION_INDEX = 30
+
+
+def _imu_errored_integration_entry(yaw):
+    return {**_imu_avg_entry(yaw), "status": "error"}
+
+
+CORR_METADATA = {
+    "imu_el": [
+        _imu_avg_entry(0.001 * i)
+        if i != ERROR_INTEGRATION_INDEX
+        else _imu_errored_integration_entry(0.001 * i)
+        for i in range(NTIMES)
+    ],
+    "imu_az": [
+        {**_imu_avg_entry(0.002 * i), "sensor_name": "imu_az", "app_id": 6}
+        for i in range(NTIMES)
+    ],
+    "lidar": [_lidar_avg_entry(1.5 + 0.001 * i) for i in range(NTIMES)],
+    "potmon": [_potmon_avg_entry(1.5 + 0.001 * i) for i in range(NTIMES)],
+    "tempctrl_lna": [
+        _tempctrl_channel_entry("tempctrl_lna", 30.0 + 0.01 * i, 1.0 + i)
+        for i in range(NTIMES)
+    ],
+    "tempctrl_load": [
+        _tempctrl_channel_entry("tempctrl_load", 25.0 + 0.01 * i, 1.0 + i)
+        for i in range(NTIMES)
+    ],
+    "rfswitch": (
+        ["RFANT"] * 20  # steady state
+        + ["UNKNOWN"] * 5  # transition window
+        + ["RFNOFF"] * 20  # new steady state
+        + [None] * 5  # sensor dropout (gap-fill pad in _insert_sample)
+        + ["RFNOFF"] * 10  # recovery
+    ),
+}
+
+# VNA_METADATA mirrors the flat ``{key: value}`` dict returned by
+# ``MetadataSnapshotReader.get()`` — the snapshot path used by the VNA
+# code in ``PandaClient.measure_s11``. Values are whatever the producer
+# last pushed via ``MetadataWriter.add``:
+#   - picohost pushes the raw sensor dict for each sensor key
+#   - ``MetadataWriter.add`` auto-appends a ``{key}_ts`` Unix-seconds float
+#   - misc. scalars (e.g. ``corr_sync_time``) go in as floats
+# There is NO averaging on this path; unlike CORR_METADATA, values are
+# scalars or nested dicts, never per-sample lists.
+_SNAPSHOT_TS = 1775997296.789012
+VNA_METADATA = {
+    "imu_el": IMU_READING,
+    "imu_el_ts": _SNAPSHOT_TS,
+    "imu_az": {**IMU_READING, "sensor_name": "imu_az", "app_id": 6},
+    "imu_az_ts": _SNAPSHOT_TS,
+    "lidar": {
+        "sensor_name": "lidar",
+        "status": "update",
+        "app_id": 4,
+        "distance_m": 1.52,
+    },
+    "lidar_ts": _SNAPSHOT_TS,
+    "potmon": {
+        "sensor_name": "potmon",
+        "status": "update",
+        "app_id": 2,
+        "pot_el_voltage": 1.5,
+        "pot_az_voltage": 1.5,
+        "pot_el_cal_slope": 100.0,
+        "pot_el_cal_intercept": -50.0,
+        "pot_az_cal_slope": 200.0,
+        "pot_az_cal_intercept": -100.0,
+        "pot_el_angle": 100.0,
+        "pot_az_angle": 200.0,
+    },
+    "potmon_ts": _SNAPSHOT_TS,
+    "rfswitch": {
+        "sensor_name": "rfswitch",
+        "status": "update",
+        "app_id": 5,
+        "sw_state": 3,
+        "sw_state_name": "VNAS",
+    },
+    "rfswitch_ts": _SNAPSHOT_TS,
+    "corr_sync_time": 1748732903.4203713,
+    "corr_sync_time_ts": _SNAPSHOT_TS,
+}
+
+S11_HEADER = {
+    "fstart": 1e6,
+    "fstop": 250e6,
+    "npoints": 1000,
+    "ifbw": 100,
+    "power_dBm": 0,
+    "freqs": np.linspace(1e6, 250e6, 1000),
+    "mode": "ant",
+    "metadata_snapshot_unix": 1748734379.905014,
+}

eigsep_observing/adc.py

@@ -0,0 +1,134 @@
+import json
+import logging
+
+import numpy as np
+from eigsep_redis import SingleStreamReader, SingleStreamWriter
+
+from .keys import ADC_SNAPSHOT_STREAM
+
+logger = logging.getLogger(__name__)
+
+
+class AdcSnapshotWriter(SingleStreamWriter):
+    """
+    Publish raw ADC snapshot arrays onto the diagnostic snapshot stream.
+
+    Each entry carries the bare int8 samples pulled from the SNAP's
+    ``input_snapshot_bram`` by :meth:`Input.get_adc_snapshot`, stacked
+    across all antennas, plus a JSON sidecar with timing context and
+    wiring so a downstream live-status app can label cores without
+    hardcoding the layout. Not routed through ``MetadataWriter`` —
+    snapshots are binary and bypass the metadata-averaging path; they
+    live in Redis only and are not folded into the HDF5 corr file.
+
+    ``maxlen`` is a dead-reader failsafe: at the 1 Hz default publish
+    cadence, 60 entries is one minute of headroom, which covers any
+    realistic live-status reconnect window. Snapshots are diagnostic
+    and not recoverable from a different source, so we don't need
+    hours of buffering — if the reader fell behind that far, the data
+    it would pull is already stale for debugging purposes.
+    """
+
+    stream = ADC_SNAPSHOT_STREAM
+    maxlen = 60
+
+    def _encode(
+        self,
+        data,
+        unix_ts,
+        sync_time=None,
+        corr_acc_cnt=None,
+        wiring=None,
+    ):
+        if not isinstance(data, np.ndarray):
+            raise ValueError("data must be a numpy array")
+        # Force C-contiguous so tobytes() and the reader's reshape
+        # always agree on memory layout regardless of what the caller
+        # passed (views, F-contig, strided, etc.).
+        data = np.ascontiguousarray(data)
+        arr_meta = {
+            "dtype": data.dtype.str,
+            "shape": list(data.shape),
+        }
+        sidecar = {
+            "arr_meta": arr_meta,
+            "unix_ts": float(unix_ts),
+            "sync_time": sync_time,
+            "corr_acc_cnt": corr_acc_cnt,
+            "wiring": wiring,
+        }
+        return {
+            "data": data.tobytes(),
+            "sidecar": json.dumps(sidecar),
+        }
+
+    def add(
+        self,
+        data,
+        unix_ts,
+        sync_time=None,
+        corr_acc_cnt=None,
+        wiring=None,
+    ):
+        """
+        Publish one ADC snapshot frame.
+
+        Parameters
+        ----------
+        data : np.ndarray
+            Raw ADC samples, shape ``(n_antennas, 2, n_samples)``,
+            dtype ``int8``. Axis 0 is the antenna index as consumed by
+            :meth:`Input.get_adc_snapshot`; axis 1 is pol (0=x, 1=y);
+            axis 2 is the sample time axis.
+        unix_ts : float
+            Wall clock time at which the snapshot was captured.
+        sync_time : float or None
+            The SNAP sync_time at capture, if available. Lets the
+            consumer place the snapshot on the same time axis as the
+            corr stream.
+        corr_acc_cnt : int or None
+            The ``corr_acc_cnt`` register value at capture, if
+            available. Same alignment purpose as ``sync_time``.
+        wiring : dict or None
+            Subset of ``wiring.yaml`` describing the antenna-to-core
+            mapping. Included so the live-status app can label cores
+            without parsing the corr header separately.
+
+        Raises
+        ------
+        ValueError
+            If ``data`` is not a numpy array.
+        """
+        self.publish(
+            data,
+            unix_ts,
+            sync_time=sync_time,
+            corr_acc_cnt=corr_acc_cnt,
+            wiring=wiring,
+        )
+
+
+class AdcSnapshotReader(SingleStreamReader):
+    """
+    Consume ADC snapshot frames from the diagnostic snapshot stream.
+
+    Returns ``(data, sidecar)`` from :meth:`read`; the
+    ``(None, None)`` tuple is returned when the snapshot stream
+    isn't registered yet (producer hasn't started).
+    """
+
+    stream = ADC_SNAPSHOT_STREAM
+    absent_warning = (
+        "No ADC snapshot stream found. Publisher may not have started yet."
+    )
+
+    def _absent_sentinel(self):
+        return None, None
+
+    def _decode(self, entry_id, fields):
+        sidecar = json.loads(fields[b"sidecar"].decode("utf-8"))
+        arr_meta = sidecar["arr_meta"]
+        data = np.frombuffer(
+            fields[b"data"], dtype=np.dtype(arr_meta["dtype"])
+        ).reshape(arr_meta["shape"])
+        return data, sidecar