mesofield 0.3.2b0__py3-none-any.whl

mesofield/scaffold/experiment.py

@@ -0,0 +1,400 @@
+"""Scaffold a new experiment directory.
+
+The `mesofield init` CLI command (see :mod:`mesofield.cli.acquire`) calls
+:func:`scaffold_experiment` to generate a working layout new lab users
+can run, edit, and extend without writing anything from scratch:
+
+    my-experiment/
+      experiment.json   # session / protocol / duration
+      hardware.yaml     # device stanzas
+      procedure.py      # Procedure subclass (Python entry point)
+      devices/
+        __init__.py
+        thermal_example.py   # annotated custom-device template
+
+The `hardware.yaml` is supplied by the caller (``mesofield init``):
+
+  - a ``Path`` to a canonical rig file -> copied in verbatim,
+  - ``"dev"``    -> a mock config (:class:`MockEncoderDevice`) that runs
+    out of the box on any machine,
+  - ``"blank"``  -> a commented real-hardware template to fill out.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+from typing import Optional, Union
+
+
+def scaffold_experiment(
+    target: Path,
+    *,
+    name: Optional[str] = None,
+    force: bool = False,
+    hardware: Union[Path, str] = "blank",
+) -> Path:
+    """Generate a runnable experiment layout at `target`. Returns the dir.
+
+    ``hardware`` selects the ``hardware.yaml`` written into the experiment:
+    a :class:`~pathlib.Path` (copied verbatim from a canonical rig file),
+    ``"dev"`` (mock config), or ``"blank"`` (fill-out template).
+    """
+    target = Path(target).resolve()
+    if target.exists() and any(target.iterdir()):
+        if not force:
+            raise FileExistsError(
+                f"{target} is not empty. Pass --force to overwrite, or pick an empty dir."
+            )
+    target.mkdir(parents=True, exist_ok=True)
+    (target / "devices").mkdir(exist_ok=True)
+
+    protocol = name or _protocol_from_dir(target)
+    files = {
+        target / "experiment.json": _experiment_json(protocol),
+        target / "procedure.py": _procedure_py(protocol),
+        target / "devices" / "__init__.py": "",
+        target / "devices" / "thermal_example.py": _thermal_example_py(),
+        target / "README.md": _readme(protocol, target.name),
+    }
+    for path, content in files.items():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(content, encoding="utf-8")
+
+    # hardware.yaml: copy a canonical rig file, or write a built-in template.
+    hardware_dst = target / "hardware.yaml"
+    if isinstance(hardware, Path):
+        shutil.copyfile(hardware, hardware_dst)
+    elif hardware == "dev":
+        hardware_dst.write_text(_hardware_yaml_mock(), encoding="utf-8")
+    elif hardware == "blank":
+        hardware_dst.write_text(hardware_yaml_template(), encoding="utf-8")
+    else:
+        raise ValueError(
+            f"hardware must be a Path, 'dev', or 'blank'; got {hardware!r}"
+        )
+    return target
+
+
+def _protocol_from_dir(target: Path) -> str:
+    return target.name.upper().replace("-", "_").replace(" ", "_")
+
+
+def _experiment_json(protocol: str) -> str:
+    return (
+        "{\n"
+        '    "Configuration": {\n'
+        '        "experimenter": "your-name",\n'
+        f'        "protocol": "{protocol}",\n'
+        '        "duration": 5,\n'
+        '        "start_on_trigger": false\n'
+        "    },\n"
+        '    "procedure_file": "procedure.py",\n'
+        '    "procedure_class": "MyProcedure",\n'
+        '    "Subjects": {\n'
+        '        "SUBJ01": {\n'
+        '            "session": "01",\n'
+        '            "task": "demo"\n'
+        "        }\n"
+        "    },\n"
+        '    "DisplayKeys": [\n'
+        '        "subject", "session", "task",\n'
+        '        "experimenter", "protocol", "duration"\n'
+        "    ]\n"
+        "}\n"
+    )
+
+
+def _hardware_yaml_mock() -> str:
+    """The ``dev`` hardware config: a mock encoder, runs without hardware."""
+    return (
+        "# Hardware configuration for this experiment (DEV / mock).\n"
+        "#\n"
+        "# Each top-level stanza (other than the reserved keys at the bottom)\n"
+        "# is a device. `type:` selects which DataProducer class instantiates\n"
+        "# it (registered via @DeviceRegistry.register(\"...\")).\n"
+        "#\n"
+        "# Built-in device types include: camera (micromanager), opencv_camera,\n"
+        "# wheel, encoder, psychopy, nidaq, mock_wheel, mock_camera.\n"
+        "#\n"
+        "# Add a lab-specific device by creating devices/<name>.py with a\n"
+        "# DataProducer subclass and a nested `Parser(TimeseriesSource)`,\n"
+        "# then reference its registered `type:` here.\n"
+        "memory_buffer_size: 1000\n"
+        "\n"
+        "# Mock encoder so this runs without real hardware.\n"
+        "# Replace with your real device when ready (`type: wheel` etc.).\n"
+        "wheel:\n"
+        "  type: mock_wheel\n"
+        "  primary: true\n"
+        "  sample_interval_ms: 50\n"
+        "  cpr: 2400\n"
+        "  diameter_mm: 80\n"
+        "  output:\n"
+        "    suffix: wheel\n"
+        "    file_type: csv\n"
+        "    bids_type: beh\n"
+    )
+
+
+def hardware_yaml_template() -> str:
+    """A commented real-hardware skeleton that must be filled out before use.
+
+    Used both for ``mesofield init`` with the ``blank`` choice and for
+    ``mesofield rig new`` (a fresh canonical rig file to edit).
+    """
+    return (
+        "# Canonical hardware configuration for this rig -- FILL THIS OUT.\n"
+        "#\n"
+        "# Each top-level stanza (other than the reserved keys) is a device.\n"
+        "# `type:` selects the DataProducer class (registered via\n"
+        "# @DeviceRegistry.register(\"...\")). Built-in types: camera\n"
+        "# (micromanager), opencv_camera, wheel, encoder, psychopy, nidaq,\n"
+        "# mock_wheel, mock_camera.\n"
+        "#\n"
+        "# Exactly one device must be flagged `primary: true`.\n"
+        "# Replace every <PLACEHOLDER> below with this machine's real values.\n"
+        "memory_buffer_size: 1000\n"
+        "\n"
+        "# --- Running wheel encoder (Arduino/Teensy over USB-serial) --------\n"
+        "wheel:\n"
+        "  type: wheel\n"
+        "  primary: true\n"
+        "  port: \"COM?\"            # <PLACEHOLDER> e.g. COM4 (Win) or /dev/ttyUSB0\n"
+        "  baudrate: 115200\n"
+        "  cpr: 2400\n"
+        "  diameter_mm: 80\n"
+        "  output:\n"
+        "    suffix: wheel\n"
+        "    file_type: csv\n"
+        "    bids_type: beh\n"
+        "\n"
+        "# --- Camera (uncomment and adapt ONE of the following) -------------\n"
+        "# OpenCV / UVC camera:\n"
+        "# mesoscope:\n"
+        "#   type: opencv_camera\n"
+        "#   id: mesoscope\n"
+        "#   name: mesoscope\n"
+        "#   backend: opencv\n"
+        "#   device_index: 0          # <PLACEHOLDER> OpenCV VideoCapture index\n"
+        "#   fps: 30\n"
+        "#   output:\n"
+        "#     suffix: mesoscope\n"
+        "#     file_type: mp4\n"
+        "#     bids_type: func\n"
+        "#\n"
+        "# Micro-Manager camera:\n"
+        "# pupil:\n"
+        "#   type: camera\n"
+        "#   id: pupil\n"
+        "#   name: pupil\n"
+        "#   backend: micromanager\n"
+        "#   configuration_path: \"<PLACEHOLDER>/MMConfig.cfg\"\n"
+        "#   output:\n"
+        "#     suffix: pupil\n"
+        "#     file_type: ome.tiff\n"
+        "#     bids_type: func\n"
+    )
+
+
+def _procedure_py(protocol: str) -> str:
+    return (
+        f'"""Procedure for the `{protocol}` experiment.\n'
+        '\n'
+        'The base mesofield.Procedure handles the AcquisitionManifest contract\n'
+        'for you. Override `prerun`, `on_started`, `on_finished` to add\n'
+        'experiment-specific behavior. Override `manifest_extra()` to attach\n'
+        'session-level metadata to the manifest (LED pattern, model version,\n'
+        'etc).\n'
+        '\n'
+        'To run::\n'
+        '\n'
+        '    mesofield launch experiment.json     # GUI\n'
+        '    python -m experiment_script          # headless (see __main__)\n'
+        '"""\n'
+        '\n'
+        'from __future__ import annotations\n'
+        '\n'
+        'import threading\n'
+        '\n'
+        'from mesofield import DeviceRegistry\n'
+        'from mesofield.base import Procedure\n'
+        'from mesofield.devices.mocks import MockEncoderDevice\n'
+        '\n'
+        '# Register any custom devices you have added under devices/ here.\n'
+        '# Built-in device types (camera, wheel, encoder, psychopy, nidaq)\n'
+        '# are already registered by mesofield itself.\n'
+        'DeviceRegistry._registry.setdefault("mock_wheel", MockEncoderDevice)\n'
+        '\n'
+        '\n'
+        'class MyProcedure(Procedure):\n'
+        '    """Your experiment\'s procedure logic.\n'
+        '\n'
+        '    Lifecycle (subclass hooks in **bold**):\n'
+        '      1. initialize_hardware\n'
+        '      2. **prerun**            -- override for per-run setup\n'
+        '      3. hardware.arm_all\n'
+        '      4. **on_started**        -- start timers, mark events\n'
+        '      5. hardware.start_all\n'
+        '      6. ... (acquisition runs until primary device finishes)\n'
+        '      7. save_data + _write_acquisition_manifest\n'
+        '      8. **on_finished**       -- post-acquisition hook\n'
+        '    """\n'
+        '\n'
+        '    def on_started(self) -> None:\n'
+        '        """Called immediately after start_all. Arm a wall-clock cap."""\n'
+        '        duration = self.config.get("duration")\n'
+        '        if duration:\n'
+        '            self.logger.info(f"Duration cap armed: {duration}s")\n'
+        '            self._duration_timer = threading.Timer(float(duration), self.cleanup)\n'
+        '            self._duration_timer.daemon = True\n'
+        '            self._duration_timer.start()\n'
+        '\n'
+        '    def on_finished(self) -> None:\n'
+        '        super().on_finished()\n'
+        '        timer = getattr(self, "_duration_timer", None)\n'
+        '        if timer is not None:\n'
+        '            timer.cancel()\n'
+        '            self._duration_timer = None\n'
+        '\n'
+        '    def manifest_extra(self) -> dict:\n'
+        '        """Attach session-level metadata to the AcquisitionManifest."""\n'
+        '        return {}\n'
+        '\n'
+        '\n'
+        'def main():\n'
+        '    """Run headless: `python procedure.py`."""\n'
+        '    import sys\n'
+        '    from pathlib import Path\n'
+        '    cfg = Path(__file__).parent / "experiment.json"\n'
+        '    proc = MyProcedure(str(cfg))\n'
+        '    finished = proc.run_until_finished(timeout=30.0)\n'
+        '    sys.exit(0 if finished else 1)\n'
+        '\n'
+        '\n'
+        'if __name__ == "__main__":\n'
+        '    main()\n'
+    )
+
+
+def _thermal_example_py() -> str:
+    return (
+        '"""Example custom device: a thermal sensor over USB-serial.\n'
+        '\n'
+        'This file is the canonical pattern for adding a lab-specific device\n'
+        'to mesofield. One file holds both halves of the contract:\n'
+        '\n'
+        '  - The producer (`ThermalSensor`) writes data during acquisition.\n'
+        '  - The parser (`_ThermalParser`) reads it back during ingest.\n'
+        '\n'
+        'The producer is registered via `@DeviceRegistry.register("thermal")`;\n'
+        'after that the device is referenced in hardware.yaml as\n'
+        '`type: thermal`. The parser is bound to the producer via\n'
+        '`ThermalSensor.Parser = _ThermalParser` so the SOURCE_REGISTRY\n'
+        'resolves it through the producer class.\n'
+        '\n'
+        'To use:\n'
+        '  1. Adapt `parse_line()` for your sensor\'s actual serial protocol.\n'
+        '  2. Import the module from your procedure.py to trigger the\n'
+        '     registration:\n'
+        '         from devices import thermal_example  # noqa: F401\n'
+        '  3. Add a stanza to hardware.yaml:\n'
+        '         thermal:\n'
+        '           type: thermal\n'
+        '           port: /dev/ttyUSB0\n'
+        '           baudrate: 115200\n'
+        '           output:\n'
+        '             suffix: thermal\n'
+        '             file_type: csv\n'
+        '             bids_type: beh\n'
+        '"""\n'
+        '\n'
+        'from __future__ import annotations\n'
+        '\n'
+        'from pathlib import Path\n'
+        'from typing import Optional, Tuple\n'
+        '\n'
+        'import numpy as np\n'
+        'import pandas as pd\n'
+        '\n'
+        'from mesofield import DeviceRegistry\n'
+        'from mesofield.devices.base import BaseSerialDevice\n'
+        'from mesofield.datakit.sources.register import SourceContext, TimeseriesSource\n'
+        '\n'
+        '\n'
+        '@DeviceRegistry.register("thermal")\n'
+        'class ThermalSensor(BaseSerialDevice):\n'
+        '    """USB-serial thermal sensor reading float Celsius values."""\n'
+        '\n'
+        '    file_type = "csv"\n'
+        '    bids_type = "beh"\n'
+        '    data_type = "thermal"\n'
+        '\n'
+        '    def parse_line(self, line: bytes) -> Optional[Tuple[float, Optional[float]]]:\n'
+        '        text = line.decode("utf-8", errors="replace").strip()\n'
+        '        if not text:\n'
+        '            return None\n'
+        '        try:\n'
+        '            return float(text), None\n'
+        '        except ValueError:\n'
+        '            self.logger.debug("Non-float line: %r", text)\n'
+        '            return None\n'
+        '\n'
+        '\n'
+        'class _ThermalParser(TimeseriesSource):\n'
+        '    """Ingest-side parser for ThermalSensor CSV output."""\n'
+        '\n'
+        '    tag = "thermal"\n'
+        '    patterns = ("**/*_thermal.csv",)\n'
+        '\n'
+        '    def build_timeseries(self, path: Path, *, context: SourceContext | None = None):\n'
+        '        df = pd.read_csv(path)\n'
+        '        t = df["timestamp"].to_numpy(dtype=np.float64)\n'
+        '        return t, df, {"source_file": str(path), "n_samples": len(df)}\n'
+        '\n'
+        '\n'
+        '# Bind the parser to the producer so SOURCE_REGISTRY["thermal"]\n'
+        '# resolves through ThermalSensor.Parser.\n'
+        'ThermalSensor.Parser = _ThermalParser\n'
+    )
+
+
+def _readme(protocol: str, dir_name: str) -> str:
+    return (
+        f"# {protocol}\n"
+        "\n"
+        "A mesofield experiment scaffolded by `mesofield init`.\n"
+        "\n"
+        "## Layout\n"
+        "\n"
+        "- `experiment.json` — session/protocol/duration; subjects.\n"
+        "- `hardware.yaml` — device stanzas. Each `type:` value names a\n"
+        "  registered DataProducer class.\n"
+        "- `procedure.py` — the `MyProcedure(Procedure)` subclass. Run with\n"
+        "  `python procedure.py` (headless) or `mesofield launch experiment.json`.\n"
+        "- `devices/thermal_example.py` — annotated template for adding a\n"
+        "  custom hardware device. Delete it when you no longer need it.\n"
+        "\n"
+        "## Quick start\n"
+        "\n"
+        "```sh\n"
+        "python procedure.py\n"
+        "```\n"
+        "\n"
+        f"The acquisition writes a per-session BIDS layout under `data/sub-SUBJ01/ses-01/`,\n"
+        "with an `AcquisitionManifest` (`manifest.json`) at the session root.\n"
+        "Datakit ingests by reading the manifest -- no globbing of filenames.\n"
+        "\n"
+        "## Customizing\n"
+        "\n"
+        f"1. Edit `experiment.json` to set your real subject IDs and duration.\n"
+        "2. Edit `hardware.yaml` to declare your rig's devices. Replace the\n"
+        "   `mock_wheel` stanza with `type: wheel` (Arduino) or `type: camera`\n"
+        "   (micromanager) once you have real hardware connected.\n"
+        "3. Copy `devices/thermal_example.py` as a template for any lab-specific\n"
+        "   device. Adapt `parse_line()` for your serial protocol; register\n"
+        "   the type in `procedure.py` (the `DeviceRegistry._registry.setdefault`\n"
+        "   call).\n"
+        "4. Add stage-specific behavior by overriding `prerun` / `on_started`\n"
+        "   / `on_finished` in `MyProcedure`.\n"
+    )

mesofield/scaffold/rigs.py

@@ -0,0 +1,121 @@
+"""Machine-level store of canonical ``hardware.yaml`` rig configurations.
+
+A ``hardware.yaml`` is rig-specific -- it pins COM ports, camera ids, device
+indices, and Micro-Manager ``.cfg`` paths to one physical computer. Rather
+than hand-copying the right file into every new experiment, each machine keeps
+a small store of named canonical configs in its OS-native config directory
+(via :mod:`platformdirs`). ``mesofield init`` copies a chosen rig into the new
+experiment so the experiment folder stays self-contained.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import platformdirs
+import yaml
+
+from mesofield.scaffold.experiment import hardware_yaml_template
+
+
+def rigs_dir() -> Path:
+    """Return (creating if needed) the directory holding canonical rig files."""
+    path = Path(platformdirs.user_config_dir("mesofield", appauthor=False)) / "rigs"
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def list_rigs() -> list[str]:
+    """Return the sorted names of every rig in the store."""
+    return sorted(
+        p.stem for p in rigs_dir().iterdir()
+        if p.suffix in (".yaml", ".yml") and p.is_file()
+    )
+
+
+def rig_path(name: str) -> Path:
+    """Return the path a rig named ``name`` resolves to (``.yaml``)."""
+    return rigs_dir() / f"{name}.yaml"
+
+
+def _resolve_existing(name: str) -> Path:
+    """Return an existing rig's path, accepting either ``.yaml`` or ``.yml``."""
+    yaml_path = rig_path(name)
+    if yaml_path.is_file():
+        return yaml_path
+    yml_path = rigs_dir() / f"{name}.yml"
+    if yml_path.is_file():
+        return yml_path
+    raise FileNotFoundError(
+        f"No rig named {name!r}. Known rigs: {', '.join(list_rigs()) or '(none)'}"
+    )
+
+
+def add_rig(name: str, source: Path, *, force: bool = False) -> Path:
+    """Copy an existing ``hardware.yaml`` into the store under ``name``.
+
+    The source is parsed with :func:`yaml.safe_load` first so a malformed
+    file is rejected before it lands in the store.
+    """
+    source = Path(source)
+    with open(source, "r", encoding="utf-8") as fh:
+        yaml.safe_load(fh)  # validate it parses; result intentionally unused
+
+    dst = rig_path(name)
+    if dst.exists() and not force:
+        raise FileExistsError(
+            f"Rig {name!r} already exists at {dst}. Pass force=True to overwrite."
+        )
+    shutil.copyfile(source, dst)
+    return dst
+
+
+def new_rig(name: str, *, force: bool = False) -> Path:
+    """Write a blank fill-out hardware template into the store under ``name``."""
+    dst = rig_path(name)
+    if dst.exists() and not force:
+        raise FileExistsError(
+            f"Rig {name!r} already exists at {dst}. Pass force=True to overwrite."
+        )
+    dst.write_text(hardware_yaml_template(), encoding="utf-8")
+    return dst
+
+
+def remove_rig(name: str) -> None:
+    """Delete a rig from the store."""
+    _resolve_existing(name).unlink()
+
+
+# Top-level hardware.yaml keys that configure the rig but are not devices.
+_NON_DEVICE_KEYS = frozenset({
+    "memory_buffer_size", "viewer_type", "widgets",
+    "blue_led_power_mw", "violet_led_power_mw",
+})
+
+
+def rig_devices(name: str) -> list[tuple[str, str]]:
+    """Return ``(device_name, device_type)`` pairs declared by rig ``name``.
+
+    Mirrors how :class:`~mesofield.hardware.HardwareManager` reads the YAML:
+    every top-level mapping (other than scalar config keys) is a device, and
+    a ``cameras:`` list expands to one entry per camera.
+    """
+    with open(_resolve_existing(name), "r", encoding="utf-8") as fh:
+        doc = yaml.safe_load(fh) or {}
+
+    devices: list[tuple[str, str]] = []
+    for key, value in doc.items():
+        if key in _NON_DEVICE_KEYS:
+            continue
+        if key == "cameras" and isinstance(value, list):
+            for cam in value:
+                if not isinstance(cam, dict):
+                    continue
+                cam_name = cam.get("id") or cam.get("name") or "camera"
+                cam_type = cam.get("type") or cam.get("backend") or "camera"
+                devices.append((str(cam_name), str(cam_type)))
+            continue
+        if isinstance(value, dict):
+            devices.append((key, str(value.get("type", key))))
+    return devices

mesofield/signals.py

@@ -0,0 +1,85 @@
+"""Lightweight signaling hub used across mesofield.
+
+Every :class:`~mesofield.protocols.HardwareDevice` exposes a
+:class:`DeviceSignals` instance on ``self.signals`` so the rest of the
+system (e.g. :class:`~mesofield.data.manager.DataManager`,
+:class:`~mesofield.base.Procedure`) can connect uniformly without caring
+which backend the device uses.
+
+Four signals form the standard contract:
+
+``started()``
+    Emitted once the device is actively acquiring / running.
+``finished()``
+    Emitted when the device has stopped on its own (e.g. an MDA sequence
+    completed) *or* in response to ``stop()``.  The ``primary`` device's
+    ``finished`` is what triggers :meth:`Procedure._cleanup_procedure`.
+``data(payload, device_ts)``
+    Emitted for every datum that should land on
+    :class:`~mesofield.data.manager.DataQueue`.  ``payload`` is the raw
+    sample (frame index, encoder click count, NIDAQ count, ...) and
+    ``device_ts`` is the device-side timestamp (float seconds, optional).
+``frame(img, idx, device_ts)``
+    Optional.  Emitted by camera-like producers carrying the raw frame
+    array in addition to the lightweight ``data`` emission.  Subscribers
+    use this for real-time processing (see ``mesofield.processors``).
+    Producers without per-sample raw payloads never emit on this signal.
+
+The implementation wraps :mod:`psygnal` so emission is Qt-free, weakly
+referenced and thread-safe.  GUI code that needs a Qt slot can use
+:func:`qt_bridge` to forward a :class:`psygnal.Signal` into a
+``pyqtSignal``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from psygnal import Signal
+
+__all__ = ["Signal", "DeviceSignals", "qt_bridge"]
+
+
+class DeviceSignals:
+    """Standard bundle of signals carried by every hardware device.
+
+    Defined as instance attributes (not class attributes) so each device
+    owns its own emitters.  ``psygnal.Signal`` instances are descriptors
+    when declared on a class; we instantiate them directly here so they
+    behave as plain emitters on the bundle.
+    """
+
+    __slots__ = ("started", "finished", "data", "frame")
+
+    def __init__(self) -> None:
+        from psygnal import SignalInstance
+
+        # Construct lightweight SignalInstance objects directly so the
+        # bundle is independent of any owning class.
+        self.started: SignalInstance = SignalInstance(())
+        self.finished: SignalInstance = SignalInstance(())
+        self.data: SignalInstance = SignalInstance((object, object))
+        self.frame: SignalInstance = SignalInstance((object, object, object))
+
+    def disconnect_all(self) -> None:
+        for sig in (self.started, self.finished, self.data, self.frame):
+            try:
+                sig.disconnect()
+            except Exception:
+                pass
+
+
+def qt_bridge(signal: Any, qt_signal: Any) -> None:
+    """Forward emissions of a ``psygnal`` signal to a ``pyqtSignal``.
+
+    Use from GUI code only.  Both signals must accept the same argument
+    arity.  The connection is one-way: psygnal -> Qt.
+    """
+
+    def _relay(*args: Any) -> None:
+        try:
+            qt_signal.emit(*args)
+        except Exception:
+            pass
+
+    signal.connect(_relay)