mesofield 0.3.2b0__py3-none-any.whl

experiments/two_cam_demo/load_dataset.py

@@ -0,0 +1,213 @@
+"""Build a pandas pickle from this experiment's session outputs.
+
+Reads each `manifest.json` produced by `procedure.py`, ingests each
+declared producer (`wheel` CSV, `meso` / `pupil` OME-TIFF + frame-metadata
+sidecars), and writes a single `processed/dataset.pkl` you can load with
+``pd.read_pickle(...)``.
+
+The dataset has the shape databench consumes:
+
+  Subject  Session  Task     | Source   Signal
+  -----------------------------------------------------
+  DEMO     01       freeview | wheel    timestamp, payload
+                             | meso     frame_index, TimeReceivedByCore
+                             | pupil    frame_index, TimeReceivedByCore
+
+The mock producers don't write columns that match datakit's real
+SOURCE_REGISTRY parsers (the wheel parser expects ``Clicks, Time,
+Speed``; the mesoscope parser globs ``*_mesoscope.ome.tiff…``). We
+therefore use the AcquisitionManifest as the source of truth and do a
+manifest-driven ingest -- which is the pattern lab programmers will
+write against their own producers.
+
+The script does, however, import freely from ``mesofield.datakit`` and
+``mesokit_schema``: the manifest models, the loader's
+``ExperimentStore`` (only for the final pickle write convention),
+``hash_file`` for provenance, etc. The point is the datakit module is
+the toolkit; the parsers shipped with it are one option among many.
+
+Run::
+
+    python load_dataset.py             # uses ./data
+    python load_dataset.py --root ./   # explicit
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable
+
+import numpy as np
+import pandas as pd
+
+# mesokit-schema: the typed manifest contract.
+from mesokit_schema import (
+    AcquisitionManifest,
+    DatasetManifest,
+    ProducerEntry,
+    SourceVersion,
+    TimeBasis,
+)
+from mesokit_schema.dataset import hash_file
+
+# mesofield.datakit: the ingest toolkit. We don't use SOURCE_REGISTRY's
+# real-hardware parsers here (mock filenames don't match their globs),
+# but we lean on the datakit module's package surface for everything else.
+import mesofield.datakit  # noqa: F401 — ensures datakit's logger is wired
+
+
+def discover_sessions(root: Path) -> list[Path]:
+    """Find every `data/sub-*/ses-*/manifest.json` under `root`."""
+    data_root = root / "data"
+    if not data_root.exists():
+        return []
+    return sorted(data_root.glob("sub-*/ses-*/manifest.json"))
+
+
+def load_producer_table(session_dir: Path, producer: ProducerEntry) -> pd.DataFrame:
+    """Read one producer's output (or its sidecar) into a DataFrame."""
+    if producer.file_type.endswith("csv"):
+        return pd.read_csv(session_dir / producer.output_path)
+
+    if producer.file_type == "ome.tiff":
+        # The pixel data lives in the TIFF (inspect with ImageJ); the
+        # time-axis metadata we merge into the dataset comes from the
+        # frame-metadata sidecar.
+        if not producer.metadata_path:
+            raise RuntimeError(
+                f"Camera producer {producer.device_id!r} has no metadata_path"
+            )
+        sidecar = json.loads((session_dir / producer.metadata_path).read_text())
+        return pd.DataFrame(sidecar.get("p0", []))
+
+    raise RuntimeError(
+        f"load_dataset.py: unknown file_type {producer.file_type!r} for "
+        f"producer {producer.device_id!r}; extend this loader for new types."
+    )
+
+
+def build_session_table(session_dir: Path, manifest: AcquisitionManifest) -> pd.DataFrame:
+    """Multi-source DataFrame for a single session.
+
+    Anchored to the mesoscope (primary) frame count; shorter sources
+    are NaN-padded, longer ones truncated. Trades temporal precision
+    for an inspectable, regular table.
+    """
+    primary = next(
+        (p for p in manifest.producers if p.data_type == "frames" and p.bids_type == "func"),
+        manifest.producers[0],
+    )
+    primary_df = load_producer_table(session_dir, primary)
+    anchor_rows = len(primary_df)
+
+    columns: dict[tuple[str, str], list] = {}
+
+    def _add(source: str, df: pd.DataFrame) -> None:
+        for col in df.columns:
+            values = df[col].iloc[:anchor_rows].tolist()
+            if len(values) < anchor_rows:
+                values = values + [None] * (anchor_rows - len(values))
+            columns[(source, col)] = values
+
+    for producer in manifest.producers:
+        df = load_producer_table(session_dir, producer)
+        # Use device_id as the source key so two cameras (meso / pupil)
+        # appear as distinct sources even though both share data_type=frames.
+        _add(producer.device_id, df)
+
+    table = pd.DataFrame(columns)
+    table.columns = pd.MultiIndex.from_tuples(table.columns, names=["Source", "Signal"])
+
+    session = manifest.session
+    task = session.task or "default"
+    table.index = pd.MultiIndex.from_tuples(
+        [(session.subject, session.session, task)] * len(table),
+        names=["Subject", "Session", "Task"],
+    )
+    return table
+
+
+def build_dataset(root: Path) -> tuple[Path, Path, pd.DataFrame]:
+    """Ingest every session under `root/data/`. Returns (pickle, manifest, df)."""
+    manifest_paths = discover_sessions(root)
+    if not manifest_paths:
+        raise FileNotFoundError(
+            f"No AcquisitionManifests found under {root / 'data'}. "
+            f"Run `python procedure.py` first."
+        )
+
+    tables: list[pd.DataFrame] = []
+    acq_hashes: list[str] = []
+    for mpath in manifest_paths:
+        acq = AcquisitionManifest.read(mpath)
+        if not acq.acquisition_complete:
+            print(f"[skip] {mpath} reports acquisition_complete=False")
+            continue
+        tables.append(build_session_table(mpath.parent, acq))
+        acq_hashes.append(acq.content_hash())
+        print(f"[ok]   {acq.session.subject}/{acq.session.session}: "
+              f"{len(tables[-1])} rows × {len(tables[-1].columns)} cols")
+
+    if not tables:
+        raise RuntimeError("No complete sessions to ingest.")
+
+    dataset = pd.concat(tables, axis=0).sort_index()
+    out_dir = root / "processed"
+    out_dir.mkdir(exist_ok=True)
+    pkl_path = out_dir / "dataset.pkl"
+    dataset.to_pickle(pkl_path)
+
+    ds_manifest = DatasetManifest(
+        datakit_version="manifest-driven-ingest",
+        built_at=datetime.now(timezone.utc),
+        upstream_acquisition_hash=acq_hashes[0] if len(acq_hashes) == 1 else None,
+        data_file="dataset.pkl",
+        data_content_hash=hash_file(pkl_path),
+        time_basis=TimeBasis(
+            clock_source="derived",
+            description="Anchored on mesoscope frame count per session.",
+        ),
+        source_versions=[
+            SourceVersion(
+                tag="multi-source",
+                version="0.1.0",
+                parser_class="load_dataset.build_session_table",
+            )
+        ],
+        columns=[(s, c) for (s, c) in dataset.columns.tolist()],
+    )
+    ds_manifest_path = out_dir / "dataset_manifest.json"
+    ds_manifest.write(ds_manifest_path)
+
+    return pkl_path, ds_manifest_path, dataset
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--root", type=Path, default=Path(__file__).resolve().parent,
+        help="Experiment root containing data/ (default: this file's dir)",
+    )
+    args = parser.parse_args()
+
+    pkl, ds_manifest_path, df = build_dataset(args.root.resolve())
+    print()
+    print(f"Wrote {pkl}")
+    print(f"Wrote {ds_manifest_path}")
+    print()
+    print("--- first 3 rows ---")
+    print(df.head(3))
+    print()
+    print(f"--- columns ({len(df.columns)} total) ---")
+    for col in df.columns:
+        print(f"  {col}")
+    print()
+    print("Load in Python: pd.read_pickle({})".format(repr(str(pkl))))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

experiments/two_cam_demo/procedure.py

@@ -0,0 +1,87 @@
+"""Two-camera + encoder demo: 10-second headless acquisition.
+
+Subclasses :class:`mesofield.base.Procedure` with the standard duration-
+timer pattern so the run terminates cleanly without a primary device's
+hardware finished-signal. The AcquisitionManifest is written
+automatically by the base Procedure's cleanup hook.
+
+Run:
+
+    python procedure.py            # headless (no Qt)
+    mesofield launch experiment.json   # GUI
+
+The headless path is what `mesofield init` scaffolds; cf. TUTORIAL.md.
+"""
+
+from __future__ import annotations
+
+import threading
+
+from mesofield import DeviceRegistry
+from mesofield.base import Procedure
+from mesofield.devices.mocks import MockFrameProducer
+from mesofield.devices.mocks import MockEncoderDevice
+from mesofield.processors import FrameMean
+
+
+DeviceRegistry._registry.setdefault("mock_wheel", MockEncoderDevice)
+DeviceRegistry._registry.setdefault("mock_camera", MockFrameProducer)
+
+
+class TwoCamDemoProcedure(Procedure):
+    """Mock 2-camera + encoder procedure with duration-gated cleanup."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # Worked example of the procedure-authored processor API:
+        # construct, assign, done. Auto-attached, auto-registered with the
+        # DataManager, and auto-plotted in the GUI when `plot=True`.
+        self.frame_mean = FrameMean(
+            camera=self.hardware.primary,
+            plot=True,
+            label="Frame Mean",
+            value_label="Mean intensity",
+            y_range=(0, 255),
+        )
+
+    def on_started(self) -> None:
+        duration = self.config.get("duration")
+        if duration:
+            self.logger.info(f"Duration cap armed: {duration}s")
+            self._duration_timer = threading.Timer(float(duration), self.cleanup)
+            self._duration_timer.daemon = True
+            self._duration_timer.start()
+
+    def on_finished(self) -> None:
+        super().on_finished()
+        timer = getattr(self, "_duration_timer", None)
+        if timer is not None:
+            timer.cancel()
+            self._duration_timer = None
+
+
+def main() -> int:
+    import sys
+    from pathlib import Path
+
+    cfg = Path(__file__).parent / "experiment.json"
+    proc = TwoCamDemoProcedure(str(cfg))
+    finished = proc.run_until_finished(timeout=30.0)
+    if not finished:
+        print("Procedure did not finish in time.", file=sys.stderr)
+        return 1
+
+    session_dir = (
+        Path(proc.data_dir)
+        / f"sub-{proc.config.subject}"
+        / f"ses-{proc.config.session}"
+    )
+    print(f"\nAcquisition complete.")
+    print(f"  Session dir: {session_dir}")
+    print(f"  Manifest:    {session_dir / 'manifest.json'}")
+    print(f"\nNext: python load_dataset.py")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mesofield/__init__.py

@@ -0,0 +1,45 @@
+"""Mesofield top-level package.
+
+This module exposes the :class:`DeviceRegistry` decorator used by hardware
+adapters to declare themselves under a YAML ``type:`` key. The remaining
+subpackages are not auto-imported; pull them in explicitly:
+
+.. code-block:: python
+
+    from mesofield.base import Procedure
+    from mesofield.config import ExperimentConfig
+    from mesofield.hardware import HardwareManager
+"""
+
+from typing import Type, Callable, Dict, Optional, Any, TypeVar
+
+T = TypeVar("T")
+
+class DeviceRegistry:
+    """Registry mapping YAML ``type:`` strings to device classes.
+
+    Hardware adapters register themselves via the
+    :meth:`~DeviceRegistry.register` decorator; the
+    :class:`~mesofield.hardware.HardwareManager` looks them up by string
+    when materialising devices from a YAML file.
+    """
+    
+    _registry: Dict[str, Type[Any]] = {}
+    
+    @classmethod
+    def register(cls, device_type: str) -> Callable[[Type[T]], Type[T]]:
+        """Register a device class for a specific device type.
+
+        The decorator also stamps ``registry_key`` onto the class so any
+        device instance can report its YAML ``type:`` for hardware export.
+        """
+        def decorator(device_class: Type[T]) -> Type[T]:
+            cls._registry[device_type] = device_class
+            device_class.registry_key = device_type
+            return device_class
+        return decorator
+    
+    @classmethod
+    def get_class(cls, device_type: str) -> Optional[Type[Any]]:
+        """Get the device class for a specific device type."""
+        return cls._registry.get(device_type)

mesofield/__main__.py

@@ -0,0 +1,11 @@
+"""``python -m mesofield`` — entry point for the Mesofield CLI.
+
+The CLI itself lives in :mod:`mesofield.cli`; this module just exposes the
+root group so both ``python -m mesofield`` and the ``mesofield`` console
+script resolve to the same place.
+"""
+
+from mesofield.cli import cli
+
+if __name__ == "__main__":
+    cli()

mesofield/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.3.2b0'
+__version_tuple__ = version_tuple = (0, 3, 2, 'b0')
+
+__commit_id__ = commit_id = None