mesofield 0.3.2b0__py3-none-any.whl

mesofield/datakit/datamodel.py

@@ -0,0 +1,200 @@
+"""Common data structures shared across the ``datakit`` pipeline.
+
+The module groups the frozen dataclasses that represent discovered files,
+aligned timelines, and high-level dataset bundles so that explorers can see
+"what flows through the system" in one place.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Iterable, Mapping, Optional, Sequence, Tuple
+
+import numpy as np
+import pandas as pd
+
+
+StreamKind = str
+
+
+@dataclass(frozen=True)
+class StreamPayload:
+    """Typed wrapper around arbitrary stream values.
+
+    Each payload records its ``kind`` (table/array/mapping/sequence/scalar), the
+    raw ``data`` object, and lightweight ``attrs`` that describe shape or column
+    metadata. Persisting this metadata makes it easier to reconstruct the
+    original Python object on reload.
+    """
+
+    kind: StreamKind
+    data: Any
+    attrs: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "attrs", dict(self.attrs or {}))
+
+    # ------------------------------------------------------------------
+    # Constructors
+    # ------------------------------------------------------------------
+    @classmethod
+    def table(cls, frame: pd.DataFrame, attrs: Optional[Dict[str, Any]] = None) -> "StreamPayload":
+        base_attrs = {
+            "columns": list(frame.columns),
+            "index_name": frame.index.name,
+        }
+        if attrs:
+            base_attrs.update(attrs)
+        return cls(kind="table", data=frame, attrs=base_attrs)
+
+    @classmethod
+    def array(cls, array: np.ndarray, attrs: Optional[Dict[str, Any]] = None) -> "StreamPayload":
+        base_attrs = {
+            "dtype": str(array.dtype),
+            "shape": tuple(int(x) for x in array.shape),
+        }
+        if attrs:
+            base_attrs.update(attrs)
+        return cls(kind="array", data=array, attrs=base_attrs)
+
+    @classmethod
+    def mapping(cls, mapping: Mapping[str, Any], attrs: Optional[Dict[str, Any]] = None) -> "StreamPayload":
+        base_attrs = {
+            "keys": list(mapping.keys()),
+        }
+        if attrs:
+            base_attrs.update(attrs)
+        return cls(kind="mapping", data=dict(mapping), attrs=base_attrs)
+
+    @classmethod
+    def sequence(cls, seq: Sequence[Any], attrs: Optional[Dict[str, Any]] = None) -> "StreamPayload":
+        base_attrs = {
+            "length": len(seq),
+        }
+        if attrs:
+            base_attrs.update(attrs)
+        return cls(kind="sequence", data=list(seq), attrs=base_attrs)
+
+    @classmethod
+    def scalar(cls, value: Any, attrs: Optional[Dict[str, Any]] = None) -> "StreamPayload":
+        return cls(kind="scalar", data=value, attrs=dict(attrs or {}))
+
+    # ------------------------------------------------------------------
+    # Introspection helpers
+    # ------------------------------------------------------------------
+    @property
+    def is_table(self) -> bool:
+        return self.kind == "table"
+
+    @property
+    def is_array(self) -> bool:
+        return self.kind == "array"
+
+    @property
+    def is_mapping(self) -> bool:
+        return self.kind == "mapping"
+
+    @property
+    def is_sequence(self) -> bool:
+        return self.kind == "sequence"
+
+    @classmethod
+    def ensure(cls, value: Any, *, kind: Optional[StreamKind] = None, attrs: Optional[Dict[str, Any]] = None) -> "StreamPayload":
+        if isinstance(value, StreamPayload):
+            if attrs:
+                merged = dict(value.attrs)
+                merged.update(attrs)
+                return StreamPayload(kind=value.kind, data=value.data, attrs=merged)
+            return value
+
+        detected_kind = kind
+
+        if detected_kind is None:
+            if isinstance(value, pd.DataFrame):
+                detected_kind = "table"
+            elif isinstance(value, np.ndarray):
+                detected_kind = "array"
+            elif isinstance(value, Mapping):
+                detected_kind = "mapping"
+            elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+                detected_kind = "sequence"
+            else:
+                detected_kind = "scalar"
+
+        if detected_kind == "table":
+            frame = value if isinstance(value, pd.DataFrame) else pd.DataFrame(value)
+            return cls.table(frame, attrs)
+        if detected_kind == "array":
+            array = value if isinstance(value, np.ndarray) else np.asarray(value)
+            return cls.array(array, attrs)
+        if detected_kind == "mapping":
+            if not isinstance(value, Mapping):
+                raise TypeError("Cannot coerce value to mapping payload")
+            return cls.mapping(dict(value), attrs)
+        if detected_kind == "sequence":
+            sequence = value if isinstance(value, Sequence) else list(value)
+            return cls.sequence(sequence, attrs)
+        return cls.scalar(value, attrs)
+
+    @property
+    def data_view(self) -> Any:
+        """Return the canonical data representation (DataFrame, ndarray, etc)."""
+
+        if self.kind == "table" and not isinstance(self.data, pd.DataFrame):
+            return pd.DataFrame(self.data)
+        if self.kind == "array" and not isinstance(self.data, np.ndarray):
+            return np.asarray(self.data)
+        if self.kind == "mapping" and not isinstance(self.data, dict):
+            return dict(self.data)
+        if self.kind == "sequence" and not isinstance(self.data, list):
+            return list(self.data)
+        return self.data
+
+
+@dataclass(frozen=True)
+class ManifestEntry:
+    """Single file discovered during manifest building."""
+
+    tag: str          # e.g. "meso_metadata"
+    path: str
+    origin: str       # "data" | "processed"
+    subject: str      # e.g. "STREHAB07"
+    session: str      # e.g. "05"
+    task: str | None = None  # e.g. "widefield"
+
+
+@dataclass(frozen=True)
+class LoadedStream:
+    """Hydrated data stream with timestamps and metadata."""
+
+    tag: str
+    t: np.ndarray  # seconds, float64, strictly increasing
+    value: object  # array or domain object
+    meta: dict
+    _payload: StreamPayload = field(init=False, repr=False, compare=False)
+
+    def __post_init__(self) -> None:
+        payload = StreamPayload.ensure(self.value)
+        object.__setattr__(self, "t", np.asarray(self.t, dtype=np.float64))
+        object.__setattr__(self, "meta", dict(self.meta or {}))
+        object.__setattr__(self, "_payload", payload)
+        object.__setattr__(self, "value", payload.data_view)
+
+    @property
+    def payload(self) -> StreamPayload:
+        return self._payload
+
+    @property
+    def data(self) -> Any:
+        return self._payload.data_view
+
+
+@dataclass(frozen=True)
+class Manifest:
+    """Manifest of files discovered for an experiment root."""
+
+    root: Path
+    entries: list[ManifestEntry]
+

mesofield/datakit/discover.py

@@ -0,0 +1,124 @@
+"""File-system discovery helpers that produce machine-readable manifests."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from ._utils._logger import get_logger
+from .datamodel import Manifest, ManifestEntry
+from .sources import SOURCE_REGISTRY
+
+_BIDS_COMPONENT_PATTERN = re.compile(r"(sub|ses|task)-([A-Za-z0-9]+)", re.IGNORECASE)
+
+logger = get_logger(__name__)
+
+
+def discover_manifest(experiment_dir: Path | str) -> Manifest:
+    """Return a :class:`Manifest` for ``experiment_dir`` by scanning registered sources."""
+
+    root = Path(experiment_dir).resolve()
+    entries = _discover_entries(root)
+    if not entries:
+        raise ValueError(f"No files discovered for {root}")
+    entries.sort(key=lambda entry: (entry.tag, entry.path))
+    return Manifest(root=root, entries=entries)
+
+
+def _discover_entries(root: Path) -> list[ManifestEntry]:
+    logger.info("Starting file discovery", extra={"phase": "discover", "experiment_dir": str(root)})
+
+    entries: list[ManifestEntry] = []
+    for tag in sorted(SOURCE_REGISTRY.keys()):
+        logger.debug("Discovering files", extra={"phase": "discover", "tag": tag})
+        source_class = SOURCE_REGISTRY[tag]
+
+        for pattern in getattr(source_class, "patterns", ()):  # type: ignore[arg-type]
+            for origin in ("processed", "data"):
+                search_root = root / origin
+                if not search_root.exists():
+                    continue
+                for file_path in search_root.glob(pattern):
+                    rel_path = file_path.relative_to(root)
+                    subject, session, task = _parse_components(rel_path, tag)
+                    entries.append(
+                        ManifestEntry(
+                            tag=tag,
+                            path=rel_path.as_posix(),
+                            origin=origin,
+                            subject=subject,
+                            session=session,
+                            task=task,
+                        )
+                    )
+                    logger.debug(
+                        "Found file",
+                        extra={
+                            "phase": "discover",
+                            "tag": tag,
+                            "path": rel_path.as_posix(),
+                            "origin": origin,
+                            "task": task,
+                        },
+                    )
+
+    logger.info("Discovery complete", extra={"phase": "discover", "total_files": len(entries)})
+    return entries
+
+
+def _parse_components(relative_path: Path, tag: str) -> tuple[str, str, str | None]:
+    captures: dict[str, str] = {}
+    normalized_path = relative_path.as_posix()
+    for label, value in _BIDS_COMPONENT_PATTERN.findall(normalized_path):
+        captures.setdefault(label.lower(), value)
+
+    parts = list(relative_path.parts)
+    if parts and parts[0] in {"data", "processed"}:
+        parts = parts[1:]
+
+    subject = captures.get("sub")
+    if subject is None and parts:
+        candidate = parts[0]
+        if candidate.startswith("sub-"):
+            _, _, remainder = candidate.partition("-")
+            subject = remainder or None
+        else:
+            subject = candidate
+
+    session_value = captures.get("ses")
+    session_token: str | None = None
+    if session_value is not None:
+        session_token = f"ses-{session_value}"
+    else:
+        for component in parts:
+            if component.startswith("ses-"):
+                session_token = component
+                break
+
+    task_value = captures.get("task")
+    task_token: str | None = None
+    if task_value is not None:
+        task_token = f"task-{task_value}"
+    else:
+        match = re.search(r"task-([A-Za-z0-9]+)", normalized_path)
+        if match:
+            task_token = f"task-{match.group(1)}"
+        else:
+            task_token = "task-session"
+
+    missing: list[str] = []
+    if not subject:
+        missing.append("subject")
+    if not session_token:
+        missing.append("session")
+    if missing:
+        raise ValueError(
+            f"Missing {', '.join(missing)} token(s) in path '{normalized_path}' for tag '{tag}'"
+        )
+
+    assert subject is not None
+    assert session_token is not None
+    return subject, session_token, task_token
+
+
+__all__ = ["discover_manifest"]