mesofield 0.3.2b0__py3-none-any.whl

mesofield/processing/__init__.py

@@ -0,0 +1,12 @@
+"""Intermediate processing stages (DLC, mesomap, lab pipelines).
+
+Each processor is a small subclass of :class:`ProcessorRunner` that
+defines ``run(inputs, **params) -> list[Path]``. Calling the runner
+wraps that work in a hashing + manifest-writing harness so every
+processed file lands with a ``<tool_name>.process.json`` sidecar
+recording inputs, parameters, tool version, and upstream provenance.
+"""
+
+from mesofield.processing.runner import ProcessorRunner
+
+__all__ = ["ProcessorRunner"]

mesofield/processing/runner.py

@@ -0,0 +1,237 @@
+"""Base class for intermediate processing stages.
+
+A processor takes raw acquisition files and produces derived files. The
+runner handles the boilerplate: hashing inputs, locating the upstream
+AcquisitionManifest, writing a ProcessingManifest sidecar alongside the
+outputs, and turning declared outputs into ProducerEntry shapes the
+ingest layer can consume.
+
+Typical use:
+
+    from mesofield.processing import ProcessorRunner
+
+    class SpikeSorter(ProcessorRunner):
+        tool_name = "my_lab_spikesort"
+        tool_version = "0.1.0"
+
+        def run(self, inputs, *, sigma=4.0):
+            in_path = inputs[0]
+            out = in_path.parent / "spikes.csv"
+            # ... do the work; write to `out` ...
+            return [out]
+
+    runner = SpikeSorter()
+    runner([recording_path], sigma=5.0)
+    # → spikes.csv written, plus my_lab_spikesort.process.json next to it
+"""
+
+from __future__ import annotations
+
+import inspect
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, ClassVar, Iterable, Optional, Sequence
+
+from mesokit_schema import (
+    AcquisitionManifest,
+    InputRef,
+    ProcessingManifest,
+    ProducerEntry,
+    TimeBasis,
+)
+from mesokit_schema.dataset import hash_file
+
+
+class ProcessorRunner:
+    """Wrap a file-to-file transformation in a ProcessingManifest contract.
+
+    Subclasses must set :attr:`tool_name` and :attr:`tool_version`, and
+    override :meth:`run`. Calling the instance executes the work and
+    writes the sidecar.
+    """
+
+    tool_name: ClassVar[str] = ""
+    tool_version: ClassVar[str] = "0.0.0"
+
+    #: Where the manifest sidecar lands. ``"output_dir"`` writes
+    #: ``<tool_name>.process.json`` in the directory of the first output;
+    #: subclasses can override :meth:`manifest_path` for custom placement.
+    manifest_placement: ClassVar[str] = "output_dir"
+
+    # ------------------------------------------------------------------ user hooks
+
+    def run(self, inputs: Sequence[Path], **params: Any) -> list[Path]:
+        """Do the actual work. Return the list of files written."""
+        raise NotImplementedError("ProcessorRunner subclasses must implement run()")
+
+    def declare_outputs(
+        self,
+        outputs: Sequence[Path],
+        params: dict[str, Any],
+        session_root: Optional[Path],
+    ) -> list[ProducerEntry]:
+        """Turn run() outputs into ProducerEntry instances.
+
+        Default: one entry per output, with data_type=tool_name and
+        bids_type/file_type inferred from the path. Override for richer
+        declarations (e.g. multiple roles, sidecars).
+        """
+        entries: list[ProducerEntry] = []
+        for path in outputs:
+            rel = self._relative_to_session(path, session_root)
+            entries.append(
+                ProducerEntry(
+                    device_id=self.tool_name,
+                    device_type="processor",
+                    data_type=self.tool_name,
+                    bids_type=self._infer_bids_type(path, session_root),
+                    file_type=self._infer_file_type(path),
+                    output_path=rel,
+                    time_basis=TimeBasis(
+                        clock_source="derived",
+                        description=f"Derived by {self.tool_name} v{self.tool_version}",
+                    ),
+                )
+            )
+        return entries
+
+    # ------------------------------------------------------------------ orchestration
+
+    def __call__(
+        self,
+        inputs: Sequence[Path],
+        *,
+        upstream: Optional[AcquisitionManifest | Path] = None,
+        session_root: Optional[Path] = None,
+        **params: Any,
+    ) -> tuple[list[Path], ProcessingManifest]:
+        """Run + emit the sidecar. Returns (outputs, manifest)."""
+        if not self.tool_name:
+            raise ValueError(f"{type(self).__name__} must set tool_name")
+
+        input_paths = [Path(p) for p in inputs]
+        input_refs = [
+            InputRef(path=str(p), content_hash=hash_file(p)) for p in input_paths
+        ]
+
+        upstream_manifest, upstream_hash, resolved_session_root = self._resolve_upstream(
+            upstream, session_root, input_paths
+        )
+        if session_root is None:
+            session_root = resolved_session_root
+
+        outputs = [Path(p) for p in self.run(input_paths, **params)]
+        if not outputs:
+            raise RuntimeError(
+                f"{self.tool_name}.run() returned no outputs; nothing to declare"
+            )
+
+        producer_entries = self.declare_outputs(outputs, params, session_root)
+
+        manifest = ProcessingManifest(
+            tool_name=self.tool_name,
+            tool_version=self.tool_version,
+            tool_invocation=self._invocation(input_paths, params),
+            built_at=datetime.now(timezone.utc),
+            upstream_acquisition_hash=upstream_hash,
+            inputs=input_refs,
+            parameters=self._jsonable_params(params),
+            outputs=producer_entries,
+        )
+        sidecar = self.manifest_path(outputs)
+        sidecar.parent.mkdir(parents=True, exist_ok=True)
+        manifest.write(sidecar)
+        return outputs, manifest
+
+    # ------------------------------------------------------------------ helpers
+
+    def manifest_path(self, outputs: Sequence[Path]) -> Path:
+        if self.manifest_placement == "output_dir":
+            return Path(outputs[0]).parent / f"{self.tool_name}.process.json"
+        raise ValueError(f"Unknown manifest_placement: {self.manifest_placement!r}")
+
+    def _invocation(self, inputs: Sequence[Path], params: dict[str, Any]) -> str:
+        sig = inspect.signature(self.run)
+        bound = ", ".join(
+            [f"inputs={[p.name for p in inputs]}"]
+            + [f"{k}={v!r}" for k, v in params.items()]
+        )
+        return f"{type(self).__name__}().run({bound})"
+
+    def _jsonable_params(self, params: dict[str, Any]) -> dict[str, Any]:
+        out: dict[str, Any] = {}
+        for k, v in params.items():
+            try:
+                import json
+                json.dumps(v)
+                out[k] = v
+            except TypeError:
+                out[k] = repr(v)
+        return out
+
+    @staticmethod
+    def _relative_to_session(path: Path, session_root: Optional[Path]) -> str:
+        if session_root is None:
+            return str(path)
+        try:
+            return str(Path(path).resolve().relative_to(Path(session_root).resolve()))
+        except ValueError:
+            return str(path)
+
+    @staticmethod
+    def _infer_bids_type(path: Path, session_root: Optional[Path]) -> Optional[str]:
+        if session_root is None:
+            return path.parent.name or None
+        try:
+            rel = Path(path).resolve().relative_to(Path(session_root).resolve())
+        except ValueError:
+            return path.parent.name or None
+        # rel is e.g. processed/<bids_type>/<file> or processed/<file>
+        parts = rel.parts
+        if len(parts) >= 3 and parts[0] == "processed":
+            return parts[1]
+        if len(parts) == 2:
+            return parts[0] if parts[0] != "processed" else None
+        return None
+
+    @staticmethod
+    def _infer_file_type(path: Path) -> str:
+        # Preserve multi-dot extensions like ome.tiff.
+        name = path.name
+        if "." not in name:
+            return ""
+        first_dot = name.index(".")
+        return name[first_dot + 1:]
+
+    @staticmethod
+    def _resolve_upstream(
+        upstream: Optional[AcquisitionManifest | Path],
+        session_root: Optional[Path],
+        input_paths: Sequence[Path],
+    ) -> tuple[Optional[AcquisitionManifest], Optional[str], Optional[Path]]:
+        """Locate the AcquisitionManifest for the provenance chain.
+
+        Accepts an already-loaded manifest, an explicit path, or `None`
+        (in which case we walk up from the first input looking for
+        `manifest.json`). Returns (manifest, content_hash, session_root).
+        """
+        if isinstance(upstream, AcquisitionManifest):
+            return upstream, upstream.content_hash(), session_root
+
+        candidate: Optional[Path] = None
+        if isinstance(upstream, (str, Path)):
+            candidate = Path(upstream)
+        elif input_paths:
+            for parent in [input_paths[0].resolve().parent, *input_paths[0].resolve().parents]:
+                if (parent / "manifest.json").exists():
+                    candidate = parent / "manifest.json"
+                    break
+
+        if candidate is None or not candidate.exists():
+            return None, None, session_root
+
+        try:
+            manifest = AcquisitionManifest.read(candidate)
+        except Exception:
+            return None, None, session_root
+        return manifest, manifest.content_hash(), candidate.parent

mesofield/processors/__init__.py

@@ -0,0 +1,13 @@
+"""Real-time, per-frame processors for camera DataProducers.
+
+A :class:`FrameProcessor` subscribes to a camera's ``signals.frame``,
+runs ``compute(img, idx, ts)`` on a daemon worker thread, and emits the
+result on both ``signals.data`` (for the DataQueue / CSV logger) and a
+Qt-compatible ``valueUpdated(time, value)`` signal (for the existing
+:class:`~mesofield.gui.speedplotter.SerialWidget` plotter).
+"""
+
+from .base import FrameProcessor
+from .frame_mean import FrameMean
+
+__all__ = ["FrameProcessor", "FrameMean"]

mesofield/processors/base.py

@@ -0,0 +1,287 @@
+"""Threaded base class for real-time camera frame processors."""
+
+from __future__ import annotations
+
+import queue
+import threading
+import time
+from typing import Any, ClassVar, Dict, Optional, TYPE_CHECKING
+
+from PyQt6.QtCore import QObject, pyqtSignal
+
+from mesofield.signals import DeviceSignals
+from mesofield.utils._logger import get_logger
+
+if TYPE_CHECKING:
+    from mesofield.protocols import DataProducer
+
+
+class _QtAdapter(QObject):
+    """QObject carrier so the processor can expose a pyqtSignal that
+    cross-thread emits are auto-queued to the GUI thread."""
+
+    valueUpdated = pyqtSignal(float, float)
+
+
+class FrameProcessor:
+    """Base class for per-frame, real-time processors.
+
+    Acquisition stays on the camera thread; ``compute`` runs on a daemon
+    worker thread fed by a bounded queue (size 1, drop-oldest replace)
+    so processing latency cannot stall the camera. Subclasses implement
+    one method::
+
+        def compute(self, img, idx, ts) -> float | None: ...
+
+    Returning ``None`` skips emission for that frame. Otherwise the
+    scalar is emitted on:
+
+    * ``self.signals.data(value, ts)`` -- psygnal; pushed onto the
+      DataQueue / CSV logger when the processor is registered with
+      :meth:`DataManager.register_hardware_device`.
+    * ``self.valueUpdated(t, value)`` -- ``pyqtSignal(float, float)``;
+      Qt's cross-thread queued connection makes
+      :class:`~mesofield.gui.speedplotter.SerialWidget` safe to drive
+      from the worker thread.
+    """
+
+    device_type: ClassVar[str] = "processor"
+    data_type: ClassVar[str] = "scalar"
+    file_type: ClassVar[str] = "csv"
+    bids_type: ClassVar[Optional[str]] = None
+
+    # Recognized SerialWidget styling kwargs collected into ``plot_config``.
+    _PLOT_KWARGS = (
+        "label",
+        "value_label",
+        "value_units",
+        "y_range",
+        "value_scale",
+        "max_points",
+    )
+
+    def __init__(
+        self,
+        name: Optional[str] = None,
+        camera: Optional["DataProducer"] = None,
+        sampling_rate: float = 0.0,
+        plot: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        # Default name = lowercased class name so ``FrameMean(camera=cam)`` works.
+        if name is None:
+            name = self.__class__.__name__.lower()
+        self.device_id: str = name
+        self.id: str = name
+        self.name: str = name
+        self.signals = DeviceSignals()
+        self.sampling_rate: float = (
+            sampling_rate
+            or float(getattr(camera, "sampling_rate", 0.0) or 0.0)
+        )
+        self.is_active: bool = False
+        self.output_path: Optional[str] = None
+        self.metadata_path: Optional[str] = None
+        self.logger = get_logger(
+            f"{self.__class__.__module__}.{self.__class__.__name__}[{self.device_id}]"
+        )
+
+        self._qt = _QtAdapter()
+        self.valueUpdated = self._qt.valueUpdated
+
+        self._queue: "queue.Queue[tuple]" = queue.Queue(maxsize=1)
+        self._worker: Optional[threading.Thread] = None
+        self._stop = threading.Event()
+        self._camera: Optional["DataProducer"] = None
+        # Remembered camera reference for no-arg attach() / status reporting.
+        self.camera: Optional["DataProducer"] = camera
+
+        # ---- compute-load counters (see ``status()``) ------------------
+        self.n_enqueued: int = 0
+        self.n_dropped: int = 0
+        self.n_processed: int = 0
+        self.n_errors: int = 0
+        self.compute_ms_ewma: float = 0.0
+        self.compute_ms_max: float = 0.0
+        self._ewma_alpha: float = 0.1
+
+        # ---- GUI plot wiring (opt-in) ----------------------------------
+        self.plot_enabled: bool = bool(plot)
+        self.plot_config: Dict[str, Any] = {
+            k: kwargs.pop(k) for k in list(kwargs) if k in self._PLOT_KWARGS
+        }
+        if kwargs:
+            # Unknown kwargs are usually a typo (e.g. ``y_lim`` for ``y_range``).
+            raise TypeError(
+                f"{type(self).__name__}: unexpected keyword argument(s) "
+                f"{sorted(kwargs)}. Recognized plot kwargs: {self._PLOT_KWARGS}"
+            )
+
+        # ---- auto-attach if a camera was supplied ----------------------
+        if camera is not None:
+            self.attach(camera)
+
+    # ---- subclass contract --------------------------------------------
+    def compute(self, img: Any, idx: Any, ts: Any) -> Optional[float]:
+        """Subclass hook: return one scalar (or ``None``) per frame.
+
+        Called on every frame the attached camera emits. Implementations
+        should be fast — anything heavy will starve the camera buffer.
+        Return ``None`` to skip emitting a sample for this frame.
+
+        Args:
+            img: Frame from the camera (typically a 2-D ``ndarray``).
+            idx: Frame index assigned by the camera (monotonic).
+            ts: Device timestamp for the frame.
+
+        Returns:
+            A single ``float`` to be pushed to the data queue / plot, or
+            ``None`` to skip this frame.
+        """
+        raise NotImplementedError
+
+    # ---- compute-load reporting ----------------------------------------
+    def status(self) -> Dict[str, Any]:
+        """Snapshot of the per-run compute-load counters."""
+        enq = self.n_enqueued or 1
+        return {
+            "device_id": self.device_id,
+            "n_enqueued": self.n_enqueued,
+            "n_processed": self.n_processed,
+            "n_dropped": self.n_dropped,
+            "n_errors": self.n_errors,
+            "drop_ratio": self.n_dropped / enq,
+            "compute_ms_ewma": round(self.compute_ms_ewma, 3),
+            "compute_ms_max": round(self.compute_ms_max, 3),
+        }
+
+    def _log_status(self) -> None:
+        s = self.status()
+        self.logger.info(
+            f"compute-load: processed={s['n_processed']} "
+            f"dropped={s['n_dropped']} ({s['drop_ratio']*100:.1f}%) "
+            f"compute_ms ewma={s['compute_ms_ewma']} max={s['compute_ms_max']} "
+            f"errors={s['n_errors']}"
+        )
+
+    # ---- public API ----------------------------------------------------
+    def attach(self, camera: Optional["DataProducer"] = None) -> None:
+        """Attach to a camera's ``signals.frame`` and start the worker.
+
+        ``camera`` defaults to whatever was passed at construction time
+        (``self.camera``); callers building processors first and attaching
+        later may pass it explicitly.
+        """
+        if self._camera is not None:
+            return
+        if camera is None:
+            camera = self.camera
+        if camera is None:
+            raise ValueError(
+                f"{type(self).__name__}: no camera to attach to "
+                "(pass camera= to attach() or to the constructor)"
+            )
+        self._camera = camera
+        self.camera = camera
+        # Reset counters at the start of each attach session.
+        self.n_enqueued = 0
+        self.n_dropped = 0
+        self.n_processed = 0
+        self.n_errors = 0
+        self.compute_ms_ewma = 0.0
+        self.compute_ms_max = 0.0
+        self._stop.clear()
+        self._worker = threading.Thread(
+            target=self._loop,
+            name=f"FrameProcessor-{self.device_id}",
+            daemon=True,
+        )
+        self._worker.start()
+        self.is_active = True
+        try:
+            camera.signals.frame.connect(self._enqueue)
+        except Exception as exc:
+            self.logger.warning(f"attach: signals.frame.connect failed: {exc}")
+        try:
+            camera.signals.finished.connect(self._log_status)
+        except Exception:
+            pass
+
+    def detach(self) -> None:
+        cam = self._camera
+        if cam is not None:
+            try:
+                cam.signals.frame.disconnect(self._enqueue)
+            except Exception:
+                pass
+            try:
+                cam.signals.finished.disconnect(self._log_status)
+            except Exception:
+                pass
+        self._camera = None
+        self._stop.set()
+        if self._worker is not None:
+            self._worker.join(timeout=1.0)
+            self._worker = None
+        self.is_active = False
+
+    # SerialWidget toggles call device.start()/device.stop().
+    def start(self) -> bool:
+        return True
+
+    def stop(self) -> bool:
+        return True
+
+    # ---- internals -----------------------------------------------------
+    def _enqueue(self, img: Any, idx: Any, ts: Any) -> None:
+        self.n_enqueued += 1
+        try:
+            self._queue.put_nowait((img, idx, ts))
+            return
+        except queue.Full:
+            pass
+        # Drop the stale item so we always work on the freshest frame.
+        self.n_dropped += 1
+        try:
+            self._queue.get_nowait()
+        except queue.Empty:
+            pass
+        try:
+            self._queue.put_nowait((img, idx, ts))
+        except queue.Full:
+            pass
+
+    def _loop(self) -> None:
+        while not self._stop.is_set():
+            try:
+                img, idx, ts = self._queue.get(timeout=0.1)
+            except queue.Empty:
+                continue
+            t0 = time.perf_counter()
+            try:
+                value = self.compute(img, idx, ts)
+            except Exception as exc:
+                self.n_errors += 1
+                self.logger.warning(f"compute failed: {exc}")
+                continue
+            dt_ms = (time.perf_counter() - t0) * 1000.0
+            self.n_processed += 1
+            if dt_ms > self.compute_ms_max:
+                self.compute_ms_max = dt_ms
+            self.compute_ms_ewma = (
+                self._ewma_alpha * dt_ms
+                + (1.0 - self._ewma_alpha) * self.compute_ms_ewma
+            )
+            if value is None:
+                continue
+            try:
+                self.signals.data.emit(value, ts)
+            except Exception:
+                pass
+            try:
+                self._qt.valueUpdated.emit(
+                    float(ts) if ts is not None else 0.0,
+                    float(value),
+                )
+            except Exception:
+                pass

mesofield/processors/frame_mean.py

@@ -0,0 +1,19 @@
+"""Reference :class:`FrameProcessor` that emits per-frame mean intensity."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from .base import FrameProcessor
+
+
+class FrameMean(FrameProcessor):
+    data_type = "frame_mean"
+
+    def compute(self, img: Any, idx: Any, ts: Any) -> Optional[float]:
+        if img is None:
+            return None
+        try:
+            return float(img.mean())
+        except Exception:
+            return None