mesofield 0.3.2b0__py3-none-any.whl

mesofield/utils/_logger.py

@@ -0,0 +1,156 @@
+"""
+Centralized logging for Mesofield using loguru.
+
+Usage:
+    from mesofield.utils._logger import get_logger
+    logger = get_logger("MyClass")
+    logger.info("Hello world")
+"""
+
+import functools
+import logging
+import sys
+from os import PathLike
+from pathlib import Path
+from typing import Optional
+
+from loguru import logger
+
+_configured = False
+_log_dir: Optional[Path] = None
+
+# Default extra so the format string always resolves {extra[logger_name]}
+logger.configure(extra={"logger_name": "mesofield"})
+
+_CONSOLE_FORMAT = (
+    "<green>{time:HH:mm:ss}</green> | "
+    "<level>{level: <8}</level> | "
+    "[{extra[logger_name]}] <cyan>{file.name}:{line}</cyan> --> {message}"
+)
+_FILE_FORMAT = "{time:HH:mm:ss} | {level: <8} | [{extra[logger_name]}] {file.name}:{line} --> {message}"
+
+
+class InterceptHandler(logging.Handler):
+    """Route stdlib logging records through loguru."""
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        frame, depth = logging.currentframe(), 0
+        while frame.f_code.co_filename in (logging.__file__, __file__):
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(
+            level, record.getMessage()
+        )
+
+
+def install_excepthook() -> None:
+    """Log uncaught exceptions through loguru with full traceback diagnostics."""
+
+    def _handle(exc_type, exc_value, exc_traceback):
+        if issubclass(exc_type, KeyboardInterrupt):
+            sys.__excepthook__(exc_type, exc_value, exc_traceback)
+            return
+        logger.opt(exception=(exc_type, exc_value, exc_traceback)).error(
+            "Uncaught exception"
+        )
+        if _log_dir is not None:
+            print(f"Uncaught exception logged to {_log_dir / 'mesofield.log'}")
+
+    sys.excepthook = _handle
+
+
+def setup_logging(log_dir: Optional[str] = None, level: str = "INFO") -> None:
+    global _configured, _log_dir
+    if _configured:
+        return
+
+    if log_dir:
+        _log_dir = Path(log_dir)
+    else:
+        project_root = Path(__file__).resolve().parent.parent
+        _log_dir = project_root / "logs"
+
+    _log_dir.mkdir(parents=True, exist_ok=True)
+
+    logger.remove()
+
+    logger.add(
+        sys.stderr,
+        format=_CONSOLE_FORMAT,
+        level=level.upper(),
+        colorize=True,
+        diagnose=True,
+    )
+
+    logger.add(
+        _log_dir / "mesofield.log",
+        format=_FILE_FORMAT,
+        level="DEBUG",
+        rotation="00:00",
+        retention="7 days",
+        encoding="utf-8",
+        diagnose=True,
+        backtrace=True,
+    )
+
+    # Route all stdlib logging through loguru
+    logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True)
+
+    # Suppress noisy third-party stdlib loggers before they reach loguru
+    for lib in ("matplotlib", "asyncio", "traitlets", "pymmcore_plus", "pymmcore-plus",
+                "ipykernel"):
+        logging.getLogger(lib).setLevel(logging.WARNING)
+
+    install_excepthook()
+    _configured = True
+
+
+def log_this_fr(func):
+    """Decorator that logs entry, exit, and exceptions of the function."""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        _log = get_logger(func.__module__)
+        _log.debug(f"Entering {func.__qualname__} args={args!r}, kwargs={kwargs!r}")
+        try:
+            result = func(*args, **kwargs)
+            _log.debug(f"Exiting {func.__qualname__} returned {result!r}")
+            return result
+        except Exception:
+            _log.exception(f"Exception in {func.__qualname__}")
+            raise
+
+    return wrapper
+
+
+def get_logger(name: str):
+    if not _configured:
+        setup_logging()
+    return logger.bind(logger_name=name)
+
+
+def hyperlink(path: str | PathLike[str], text: str) -> str:
+    """Return an OSC-8 terminal hyperlink with custom display text.
+
+    Args:
+        path: Local filesystem path or URI target.
+        text: Link label shown in the log message.
+    """
+    target = str(path)
+    if not target:
+        return text
+
+    try:
+        if target.startswith(("file://", "http://", "https://")):
+            uri = target
+        else:
+            uri = Path(target).expanduser().resolve(strict=False).as_uri()
+        return f"\033]8;;{uri}\033\\{text}\033]8;;\033\\"
+    except Exception:
+        return text

mesofield/utils/retrofit.py

@@ -0,0 +1,309 @@
+"""Reconstruct AcquisitionManifests for legacy mesofield sessions.
+
+Mesofield only started writing a `mesokit_schema.AcquisitionManifest` once
+the `Procedure._write_acquisition_manifest` hook landed. Sessions acquired
+before that have no manifest, which means downstream tools (datakit,
+databench) cannot ingest them through the contract path.
+
+This module walks a BIDS-laid-out session directory (or an experiment
+root containing many sessions) and synthesizes a manifest from what is
+already on disk:
+
+  - subject / session from the BIDS path (`sub-X/ses-Y/`)
+  - task from the filename suffix (`..._task-Z_...`)
+  - per-producer `output_path`, `bids_type`, `file_type` from the file tree
+  - `started_at` / `ended_at` from the session's `*_timestamps.csv`
+  - frame-metadata sidecars (`*_frame_metadata.json`) attached to their tiff
+
+Fields that the legacy filesystem does not carry (hardware calibration,
+mesofield_version, per-producer sampling_rate) are written as their
+empty/default values. The manifest still gates ingest, but downstream
+analyses that need calibration will need it filled in by hand.
+
+Multi-task sessions get one manifest per task, written as
+`manifest_task-<T>.json`; single-task sessions write `manifest.json`.
+"""
+
+from __future__ import annotations
+
+import csv
+import re
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterator, Optional
+
+from mesokit_schema import (
+    AcquisitionManifest,
+    ProducerEntry,
+    SessionIdentity,
+    TimeBasis,
+)
+
+
+_BIDS_FILE_RE = re.compile(
+    r"^(?P<timestamp>\d{8}_\d{6})"
+    r"_sub-(?P<subject>[^_]+)"
+    r"_ses-(?P<session>[^_]+)"
+    r"_task-(?P<task>[^_]+)"
+    r"_(?P<suffix>[^.]+)"
+    r"\.(?P<extension>.+)$"
+)
+
+# Filenames whose BIDS suffix marks them as session metadata, not producer data.
+_SESSION_LEVEL_SUFFIXES: frozenset[str] = frozenset({
+    "configuration", "notes", "timestamps",
+})
+
+_SIDECAR_TAIL = "_frame_metadata.json"
+
+# Default values for fields the legacy filesystem doesn't carry.
+LEGACY_VERSION = "legacy"
+LEGACY_CLOCK_SOURCE = "wall_unix_s"
+
+
+def discover_sessions(root: Path) -> Iterator[Path]:
+    """Yield session directories (`.../sub-X/ses-Y/`) under `root`.
+
+    Accepts either a session dir, an experiment root containing `data/`,
+    or any ancestor that contains BIDS-laid-out sessions.
+    """
+    root = Path(root).resolve()
+    if _looks_like_session(root):
+        yield root
+        return
+
+    # Common layouts: <exp>/data/sub-*/ses-*  or  <exp>/sub-*/ses-*
+    for pattern in ("data/sub-*/ses-*", "sub-*/ses-*", "*/sub-*/ses-*"):
+        matches = sorted(root.glob(pattern))
+        if matches:
+            for path in matches:
+                if _looks_like_session(path):
+                    yield path
+            return
+
+
+def _looks_like_session(path: Path) -> bool:
+    return (
+        path.is_dir()
+        and path.name.startswith("ses-")
+        and path.parent.name.startswith("sub-")
+    )
+
+
+def synthesize_manifests(session_dir: Path) -> dict[str, AcquisitionManifest]:
+    """Build one manifest per task found under `session_dir`.
+
+    Returns a {task: AcquisitionManifest} mapping. Single-task sessions
+    return a one-entry dict; the caller decides the filename.
+    """
+    session_dir = Path(session_dir).resolve()
+    subject = session_dir.parent.name.removeprefix("sub-")
+    session = session_dir.name.removeprefix("ses-")
+
+    files = [p for p in session_dir.rglob("*") if p.is_file()]
+    sidecars = _index_sidecars(files)
+
+    # Partition producer files by task.
+    by_task: dict[str, list[tuple[Path, re.Match[str]]]] = defaultdict(list)
+    for path in files:
+        m = _BIDS_FILE_RE.match(path.name)
+        if not m:
+            continue
+        suffix = m.group("suffix")
+        if suffix in _SESSION_LEVEL_SUFFIXES:
+            continue
+        if path.name.endswith(_SIDECAR_TAIL):
+            continue
+        by_task[m.group("task")].append((path, m))
+
+    if not by_task:
+        return {}
+
+    out: dict[str, AcquisitionManifest] = {}
+    for task, entries in by_task.items():
+        # Session-level files are written per-task too (``..._task-<T>_timestamps.csv``),
+        # so resolve them against *this* task rather than grabbing the first match --
+        # otherwise every task's manifest inherits one arbitrary task's timing/config.
+        timestamps_path = _find_session_file(files, "timestamps", task)
+        started_at, ended_at = (
+            _read_timestamps(timestamps_path) if timestamps_path else (None, None)
+        )
+        config_path = _find_session_file(files, "configuration", task)
+        config = _read_configuration(config_path) if config_path else {}
+        fallback_started_at = started_at or _earliest_mtime(
+            [path for path, _ in entries]
+        )
+        producers = [
+            _producer_for(path, match, session_dir, sidecars)
+            for path, match in sorted(entries, key=lambda pair: pair[0].name)
+        ]
+        out[task] = AcquisitionManifest(
+            mesofield_version=LEGACY_VERSION,
+            acquisition_complete=True,
+            started_at=fallback_started_at,
+            ended_at=ended_at,
+            session=SessionIdentity(
+                subject=subject,
+                session=session,
+                task=task,
+                experimenter=config.get("experimenter"),
+                protocol=config.get("protocol"),
+            ),
+            producers=producers,
+        )
+    return out
+
+
+def manifest_filename(task: str, multi_task: bool) -> str:
+    """Decide the on-disk filename for a synthesized manifest."""
+    return f"manifest_task-{task}.json" if multi_task else "manifest.json"
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+
+
+def _index_sidecars(files: list[Path]) -> dict[Path, Path]:
+    """Map each data file to its sibling `_frame_metadata.json` sidecar, if any.
+
+    Writers name the sidecar by appending ``_frame_metadata.json`` to the data
+    file's *full* name including extension (see ``writer.FRAME_MD_FILENAME``),
+    so stripping the tail yields the data file's exact name -- e.g.
+    ``foo.ome.tiff_frame_metadata.json`` -> ``foo.ome.tiff``. Match that name
+    directly; the previous ``base + "."`` prefix test required a trailing dot
+    that never exists and so attached no sidecars at all.
+    """
+    index: dict[Path, Path] = {}
+    for path in files:
+        if not path.name.endswith(_SIDECAR_TAIL):
+            continue
+        base = path.name[: -len(_SIDECAR_TAIL)]
+        for candidate in path.parent.iterdir():
+            if (
+                candidate.is_file()
+                and candidate != path
+                and (
+                    candidate.name == base
+                    or candidate.name.startswith(base + ".")
+                )
+            ):
+                index[candidate] = path
+                break
+    return index
+
+
+def _find_session_file(
+    files: list[Path], suffix: str, task: Optional[str] = None
+) -> Optional[Path]:
+    """Locate a session-level file by its BIDS suffix (e.g. 'timestamps').
+
+    Session-level files carry a ``_task-<T>_`` token like everything else, so
+    when *task* is given prefer the file written for that task. Fall back to a
+    lone file of that suffix only when there is exactly one (covers sessions
+    whose task token differs slightly from the producer files).
+    """
+    matches: list[tuple[Path, re.Match[str]]] = []
+    for path in files:
+        m = _BIDS_FILE_RE.match(path.name)
+        if m and m.group("suffix") == suffix:
+            matches.append((path, m))
+    if not matches:
+        return None
+    if task is not None:
+        for path, m in matches:
+            if m.group("task") == task:
+                return path
+        if len(matches) == 1:
+            return matches[0][0]
+        return None
+    return matches[0][0]
+
+
+def _read_timestamps(path: Path) -> tuple[Optional[datetime], Optional[datetime]]:
+    """Return (start, stop) UTC-aware datetimes from a legacy timestamps.csv.
+
+    The file is written by `DataSaver.save_timestamps` with columns
+    ``device_id, started, stopped`` (older revisions used ``id, start_time,
+    stop_time`` — both are accepted). The first row is the procedure-level
+    timing; subsequent rows are per-device. Legacy values may be
+    naive-local; we convert to UTC via `astimezone`.
+    """
+    with open(path, "r", newline="", encoding="utf-8") as fh:
+        reader = csv.DictReader(fh)
+        for row in reader:
+            start = row.get("started") or row.get("start_time")
+            stop = row.get("stopped") or row.get("stop_time")
+            return _parse_dt(start), _parse_dt(stop)
+    return None, None
+
+
+def _read_configuration(path: Path) -> dict[str, str]:
+    """Pull a flat `{Parameter: Value}` dict from a session configuration.csv."""
+    out: dict[str, str] = {}
+    with open(path, "r", newline="", encoding="utf-8") as fh:
+        reader = csv.DictReader(fh)
+        for row in reader:
+            key = (row.get("Parameter") or "").strip()
+            val = (row.get("Value") or "").strip()
+            if key:
+                out[key] = val
+    return out
+
+
+def _parse_dt(raw: Optional[str]) -> Optional[datetime]:
+    if not raw:
+        return None
+    raw = raw.strip()
+    try:
+        dt = datetime.fromisoformat(raw)
+    except ValueError:
+        return None
+    if dt.tzinfo is None:
+        # Treat naive as system-local and convert to UTC.
+        dt = dt.astimezone(timezone.utc)
+    return dt
+
+
+def _earliest_mtime(files: list[Path]) -> datetime:
+    """Fallback session start time when timestamps.csv is missing."""
+    if not files:
+        return datetime.now(timezone.utc)
+    earliest = min(p.stat().st_mtime for p in files)
+    return datetime.fromtimestamp(earliest, tz=timezone.utc)
+
+
+def _producer_for(
+    path: Path,
+    match: re.Match[str],
+    session_dir: Path,
+    sidecars: dict[Path, Path],
+) -> ProducerEntry:
+    rel = path.resolve().relative_to(session_dir.resolve())
+    bids_type = rel.parent.name if str(rel.parent) != "." else None
+    sidecar_rel: Optional[str] = None
+    if path in sidecars:
+        sidecar_rel = str(sidecars[path].resolve().relative_to(session_dir.resolve()))
+    suffix = match.group("suffix")
+    return ProducerEntry(
+        device_id=suffix,
+        device_type=_guess_device_type(suffix, match.group("extension")),
+        data_type=suffix,
+        bids_type=bids_type,
+        file_type=match.group("extension"),
+        output_path=str(rel),
+        metadata_path=sidecar_rel,
+        sampling_rate_hz=None,
+        time_basis=TimeBasis(clock_source=LEGACY_CLOCK_SOURCE),
+        calibration={},
+    )
+
+
+def _guess_device_type(suffix: str, extension: str) -> str:
+    if extension.endswith(("tiff", "tif", "mp4", "avi")):
+        return "camera"
+    if any(token in suffix for token in ("encoder", "wheel", "treadmill")):
+        return "encoder"
+    if "nidaq" in suffix:
+        return "nidaq"
+    return "device"

mesofield/utils/utils.py

@@ -0,0 +1,217 @@
+"""Miscellaneous diagnostic helpers.
+
+These utilities are not imported by the runtime; they are convenience
+functions for use in ad-hoc scripts, notebooks, and the embedded IPython
+console. ``nidaqmx`` is imported lazily inside the NI-DAQ helpers so this
+module loads cleanly on machines without NI-DAQmx installed.
+"""
+
+import serial.tools.list_ports  # pip install pyserial
+import requests
+import pymmcore_plus
+from useq import MDASequence
+import pandas as pd
+
+
+def test_arduino_connection():
+    try:
+        arduino = serial.Serial('COM4', 9600)
+        arduino.close()
+        print("Arduino connection successful!")
+    except serial.SerialException:
+        print("Failed to connect to Arduino on COM4.")
+        
+#TODO Save config for each session
+#TODO Add a button to save the current configuration to a JSON file
+#TODO Auto-fps calculation based on the number of frames and duration
+def get_fps(mmc: pymmcore_plus.CMMCorePlus):
+    ## There is a problem when this function is called more than once in succession, freezing the program
+    """
+    Calculate the frames per second (FPS) based on the number of frames and duration of an MDA sequence.
+        - num_frames = num_trials × trial_time(5 seconds) × framerate (45 fps)
+        - num_trials = num_frames / (trial_time * framerate) (255 frames for a 5 seconds trial at 45 fps)
+        - Total duration = num_frames / framerate or num_trials * trial_time
+        - num_frames = num_trials × trial_time(5 seconds) × framerate (45 fps)
+
+    """
+    from pymmcore_plus import Metadata
+    core = mmc
+    frames = 120
+    core.run_mda(
+        MDASequence(time_plan={"interval": 0, "loops": frames}), 
+        block=True
+    )
+    
+    duration = core.mda._time_elapsed()
+    fps = frames / duration
+    return fps
+
+### Utility functions for querying serial ports and USB IDs ###
+
+def load_metadata_from_json(json_file_path) -> pd.DataFrame:
+    """load metadata from a JSON file as a Pandas Dataframe."""
+    try:
+        metadata_df = pd.read_json(json_file_path)
+        return metadata_df
+    except Exception as e:
+        print(f"Error loading metadata from JSON file: {e}")
+        return pd.DataFrame()
+
+def list_serial_ports():
+    """
+    List all available serial ports on the system
+    """
+    # Get a list of all available serial ports
+    ports = serial.tools.list_ports.comports()
+    
+    # Check if there are any ports available
+    if not ports:
+        print("No serial ports found.")
+        return
+    
+    # Print details of each available serial port
+    for port in ports:
+        print(f"Device: {port.device}, Description: {port.description}, HWID: {port.hwid}")
+
+
+
+def download_usb_ids():
+    """
+    Download the USB IDs file from the internet resources 
+    """
+    url = "http://www.linux-usb.org/usb.ids"
+    response = requests.get(url)
+    if response.status_code == 200:
+        return response.text
+    else:
+        print("Failed to download USB IDs")
+        return None
+
+def parse_usb_ids(usb_ids_content):
+    """
+    Parse the USB IDs file and return a dictionary of vendor IDs and product IDs
+    """
+
+    vendor_ids = {}
+    current_vendor_id = ""
+    for line in usb_ids_content.splitlines():
+        if not line or line.startswith("#") or line.startswith("\t\t"):
+            continue
+        if line.startswith("\t"):
+            product_id, product_name = line.strip().split(" ", 1)
+            vendor_ids[current_vendor_id]["products"][product_id] = product_name
+        else:
+            vendor_id, vendor_name = line.split(" ", 1)
+            current_vendor_id = vendor_id
+            vendor_ids[vendor_id] = {"name": vendor_name, "products": {}}
+    return vendor_ids
+
+def identify_device(vendor_id, product_id, usb_ids):
+    """
+    Identify the vendor and product of a USB device using the USB IDs file
+    """
+
+    vendor = usb_ids.get(vendor_id, {}).get("name", "Unknown Vendor")
+    product = usb_ids.get(vendor_id, {}).get("products", {}).get(product_id, "Unknown Product")
+    return vendor, product
+
+def list_serial_ports_with_vendors(usb_ids):
+    """List serial ports and resolve their vendor / product names.
+
+    Args:
+        usb_ids: Dictionary produced by :func:`parse_usb_ids`.
+    """
+    ports = serial.tools.list_ports.comports()
+    if not ports:
+        print("No serial ports found.")
+        return
+    for port in ports:
+        hwid_parts = port.hwid.split()
+        vid_pid = [part for part in hwid_parts if part.startswith("VID:PID=")]
+        if vid_pid:
+            vid, pid = vid_pid[0].replace("VID:PID=", "").split(":")
+            vendor, product = identify_device(vid, pid, usb_ids)
+            print(f"Device: {port.device}, Description: {port.description}, Vendor: {vendor}, Product: {product}")
+        else:
+            print(f"Device: {port.device}, Description: {port.description}, HWID: {port.hwid}")
+
+### NI-DAQ utility functions ###
+
+def list_nidaq_devices():
+    """List all connected NI-DAQ devices."""
+    import nidaqmx  # lazy: not all rigs have NI-DAQmx installed
+    system = nidaqmx.system.System.local()
+    return [device.name for device in system.devices]
+
+
+def read_analog_input(device_name, channel='ai0'):
+    """Read a single analog input from a specified channel."""
+    import nidaqmx
+    with nidaqmx.Task() as task:
+        task.ai_channels.add_ai_voltage_chan(f"{device_name}/{channel}")
+        return task.read()
+
+
+def test_nidaq_connection(device_name):
+    """Test connection to a specified NI-DAQ device."""
+    import nidaqmx
+    try:
+        with nidaqmx.Task() as task:
+            task.ai_channels.add_ai_voltage_chan(f"{device_name}/ai0")
+        return True
+    except nidaqmx.DaqError:
+        return False
+    
+    
+### MMCore configuration-check utility functions (check settings of devices connected to Micro-Manager via loaded configuration.cfg file) ###
+
+def sanity_check(mmc):
+    """
+    Perform a sanity check to ensure that the Core is properly
+    initialized and that the camera is connected. 
+
+    This function lists: 
+    - the loaded devices
+    - configuration groups
+    - current MMCore configuration settings 
+    - camera settings.
+
+    """
+    print("Sanity Check:")
+    print("=============")
+    
+    # ==== List all devices loaded by Micro-Manager ==== #
+    devices = mmc.getLoadedDevices()
+    print("Loaded Devices:")
+    for device in devices:
+        print(f" - {device}: {mmc.getDeviceDescription(device)}")
+    
+    # ==== Display configuration groups ==== #
+    config_groups = mmc.getAvailableConfigGroups()
+    print("\nConfiguration Groups:")
+    for group in config_groups:
+        print(f" - {group}")
+        configs = mmc.getAvailableConfigs(group)
+        for config in configs:
+            print(f"    - {config}")
+    
+    # ==== Display current configuration ==== #
+    print("\nCurrent Configuration:")
+    # Get the current configuration settings for each group
+    for group in config_groups:
+        print(f" - {group}: {mmc.getCurrentConfig(group)}")
+    
+    # ==== Display camera settings ==== #
+    camera_device = mmc.getCameraDevice()
+    if camera_device:
+        print(f"\nCamera Device: {camera_device}")
+        print(f" - Exposure: {mmc.getExposure()} ms")
+        print(f" - Pixel Size: {mmc.getPixelSizeUm()} um")
+    else:
+        print("\nNo camera device found.")
+
+    
+    print("\nOther Information:")
+    print(f" - Image Width: {mmc.getImageWidth()}")
+    print(f" - Image Height: {mmc.getImageHeight()}")
+    print(f" - Bit Depth: {mmc.getImageBitDepth()}")