mesofield 0.3.2b0__py3-none-any.whl

mesofield/datakit/sources/register.py

@@ -0,0 +1,204 @@
+"""Base classes for loading datakit sources."""
+
+from __future__ import annotations
+
+import inspect
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, ClassVar, Dict, Iterable, Mapping, Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+from mesofield.datakit.config import settings
+from ..datamodel import LoadedStream
+
+
+_DOCUMENT_FLAG = "__datakit_document__"
+
+
+def document(method):
+    """Mark a method so its docstring is captured into stream ``meta['docs']``."""
+    setattr(method, _DOCUMENT_FLAG, True)
+    return method
+
+
+@dataclass(frozen=True)
+class LoadContext:
+    """Context object passed to every ``DataSource.load`` call.
+
+    Carries identity (subject/session/task), the inventory row for that cell
+    (so sources can locate sibling files via :meth:`path_for`), and any
+    upstream sources that were loaded for the same cell as declared on
+    :attr:`DataSource.requires`.
+
+    For backward parity with previous releases, when ``"dataqueue"`` is
+    present in :attr:`dependencies`, the convenience attributes
+    :attr:`dataqueue_frame`, :attr:`dataqueue_meta`, :attr:`master_timeline`,
+    and :attr:`experiment_window` are populated from it. New sources should
+    prefer reading from :attr:`dependencies` directly.
+    """
+
+    subject: str
+    session: str
+    task: str | None
+    inventory_row: Mapping[str, Any]
+    dependencies: Mapping[str, "LoadedStream | None"] = field(default_factory=dict)
+    master_timeline: np.ndarray | None = None
+    experiment_window: tuple[float, float] | None = None
+    dataqueue_frame: pd.DataFrame | None = None
+    dataqueue_meta: Mapping[str, Any] | None = None
+
+    def path_for(self, tag: str) -> Path | None:
+        """Return the path stored in the inventory row for ``tag``, or None."""
+        value = self.inventory_row.get(tag)
+        if value is None:
+            return None
+        try:
+            if pd.isna(value):
+                return None
+        except (TypeError, ValueError):
+            pass
+        return Path(str(value))
+
+    def require_path(self, tag: str) -> Path:
+        """Like :meth:`path_for` but raises ``FileNotFoundError`` when missing."""
+        path = self.path_for(tag)
+        if path is None:
+            raise FileNotFoundError(
+                f"Missing '{tag}' path for ({self.subject}, {self.session}, {self.task})"
+            )
+        return path
+
+    def get_dependency(self, tag: str) -> "LoadedStream | None":
+        """Return a previously-loaded dependency stream, or None if unavailable."""
+        return self.dependencies.get(tag)
+
+    def require_dependency(self, tag: str) -> "LoadedStream":
+        """Like :meth:`get_dependency` but raises if the dependency is missing."""
+        dep = self.dependencies.get(tag)
+        if dep is None:
+            raise RuntimeError(
+                f"Missing required dependency '{tag}' for "
+                f"({self.subject}, {self.session}, {self.task})"
+            )
+        return dep
+
+
+class DataSource:
+    """Base class for a file-backed data source."""
+
+    tag: ClassVar[str]
+    patterns: ClassVar[Iterable[str]]
+    camera_tag: ClassVar[str | None] = None
+    is_timeseries: ClassVar[bool] = True
+    flatten_payload: ClassVar[bool] = True
+    requires: ClassVar[Tuple[str, ...]] = ()
+    """Tag names of upstream sources whose loaded streams should be made
+    available via ``LoadContext.dependencies``. Soft contract: a missing or
+    failed dependency yields ``None`` in ``dependencies[tag]``; sources are
+    responsible for either degrading gracefully or raising."""
+
+    def load(self, path: Path, *, context: LoadContext | None = None) -> LoadedStream:
+        """Load data from the given path."""
+        raise NotImplementedError(f"{self.__class__.__name__} must implement load()")
+
+    def _require_context(self, context: LoadContext | None) -> LoadContext:
+        if context is None:
+            raise ValueError(
+                f"{self.__class__.__name__} requires a LoadContext; call through Dataset"
+            )
+        return context
+
+    def _decorate_meta(self, meta: Optional[Dict[str, Any]] = None, *, is_interval: bool = False) -> Dict[str, Any]:
+        meta_dict: Dict[str, Any] = dict(meta or {})
+        if self.camera_tag is not None:
+            meta_dict.setdefault(settings.sources.meta_camera_key, self.camera_tag)
+        meta_dict.setdefault(settings.sources.meta_timeseries_key, self.is_timeseries)
+        meta_dict.setdefault(settings.sources.meta_source_key, self.tag)
+        if is_interval:
+            meta_dict.setdefault(settings.sources.meta_interval_key, True)
+        docs = {
+            name: inspect.getdoc(attr)
+            for name, attr in inspect.getmembers(type(self), callable)
+            if getattr(attr, _DOCUMENT_FLAG, False)
+        }
+        if docs:
+            meta_dict.setdefault("docs", docs)
+        return meta_dict
+
+
+class TimeseriesSource(DataSource):
+    """Base class for time-indexed sources."""
+
+    is_timeseries: ClassVar[bool] = True
+
+    def build_timeseries(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[np.ndarray, Any, Dict[str, Any]]:
+        """Return (timeline, value, meta)."""
+        raise NotImplementedError(f"{self.__class__.__name__} must implement build_timeseries()")
+
+    def load(self, path: Path, *, context: LoadContext | None = None) -> LoadedStream:
+        t, value, meta = self.build_timeseries(path, context=context)
+        timeline = np.asarray(t, dtype=np.float64)
+        return LoadedStream(tag=self.tag, t=timeline, value=value, meta=self._decorate_meta(meta))
+
+
+class TableSource(DataSource):
+    """Base class for static table sources."""
+
+    is_timeseries: ClassVar[bool] = False
+
+    def build_table(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[np.ndarray, Any, Dict[str, Any]]:
+        """Return (timeline, value, meta)."""
+        raise NotImplementedError(f"{self.__class__.__name__} must implement build_table()")
+
+    def load(self, path: Path, *, context: LoadContext | None = None) -> LoadedStream:
+        t, value, meta = self.build_table(path, context=context)
+        timeline = np.asarray(t, dtype=np.float64)
+        return LoadedStream(tag=self.tag, t=timeline, value=value, meta=self._decorate_meta(meta))
+
+
+class IntervalSeriesSource(DataSource):
+    """Base class for interval-based sources."""
+
+    is_timeseries: ClassVar[bool] = True
+
+    def build_intervals(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[pd.DataFrame, Dict[str, Any]]:
+        """Return an intervals table with start/stop columns and meta."""
+        raise NotImplementedError(f"{self.__class__.__name__} must implement build_intervals()")
+
+    def load(self, path: Path, *, context: LoadContext | None = None) -> LoadedStream:
+        intervals, meta = self.build_intervals(path, context=context)
+        if "start_s" not in intervals.columns or "stop_s" not in intervals.columns:
+            raise ValueError("Intervals must include 'start_s' and 'stop_s' columns")
+        timeline = pd.to_numeric(intervals["start_s"], errors="coerce").to_numpy(dtype=np.float64)
+        return LoadedStream(
+            tag=self.tag,
+            t=timeline,
+            value=intervals,
+            meta=self._decorate_meta(meta, is_interval=True),
+        )
+
+
+__all__ = [
+    "LoadContext",
+    "DataSource",
+    "TimeseriesSource",
+    "TableSource",
+    "IntervalSeriesSource",
+]

mesofield/datakit/sources/session/config.py

@@ -0,0 +1,130 @@
+"""Session configuration data source."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, ClassVar, Dict, Iterable, Mapping
+
+import numpy as np
+import pandas as pd
+
+from mesofield.datakit.sources.register import LoadContext, TableSource
+from mesofield.datakit.config import settings
+
+
+def _clean_config_value(value: Any) -> Any:
+    """Normalize configuration values for metadata storage."""
+
+    if isinstance(value, str):
+        stripped = value.strip()
+        if not stripped:
+            return None
+        lowered = stripped.lower()
+        if lowered in {"nan", "na", "none", "null"}:
+            return None
+        return stripped
+
+    if value is None:
+        return None
+
+    if isinstance(value, (np.floating,)):
+        if np.isnan(value):
+            return None
+        return float(value)
+
+    if isinstance(value, (np.integer,)):
+        return int(value)
+
+    if isinstance(value, float) and np.isnan(value):
+        return None
+
+    return value
+
+
+
+class SessionConfigSource(TableSource):
+    """Load configuration CSVs and surface subject/session attributes."""
+
+    tag = "session_config"
+    patterns = ("**/*_configuration.csv",)
+    camera_tag = None
+    flatten_payload = True
+
+    DEFAULT_VARIABLE_MAPPING: ClassVar[Dict[str, tuple[str, ...]]] = {
+        "subject": ("sex", "genotype", "DOS", "DOB"),
+        "session": ("weight", "XYZ", "PT"),
+    }
+    variable_mapping: ClassVar[Dict[str, tuple[str, ...]]] = DEFAULT_VARIABLE_MAPPING.copy()
+
+
+    def build_table(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[np.ndarray, pd.DataFrame, dict]:
+        raw_df = pd.read_csv(path)
+
+        required_columns = {"Parameter", "Value"}
+        if not required_columns.issubset(raw_df.columns):
+            missing = ", ".join(sorted(required_columns - set(raw_df.columns)))
+            raise ValueError(f"Configuration CSV missing required columns: {missing}")
+
+        df = raw_df.set_index("Parameter").T
+
+        # Config is static - single timepoint
+        t = np.arange(len(df), dtype=float)
+
+        # Normalize parameter values for metadata usage
+        values_row = df.iloc[0] if not df.empty else pd.Series(dtype=object)
+        parameter_values = {
+            str(col).strip(): _clean_config_value(values_row[col])
+            for col in values_row.index
+        }
+
+        mapping = self.variable_mapping or {}
+        subject_keys = mapping.get("subject", ())
+        session_keys = mapping.get("session", ())
+
+        subject_variables = {
+            key: parameter_values[key]
+            for key in subject_keys
+            if key in parameter_values and parameter_values[key] is not None
+        }
+        session_variables = {
+            key: parameter_values[key]
+            for key in session_keys
+            if key in parameter_values and parameter_values[key] is not None
+        }
+
+        dedupe_keys = set(subject_variables) | set(session_variables) | {"subject", "session"}
+        parameter_values = {
+            key: value for key, value in parameter_values.items() if key not in dedupe_keys
+        }
+
+        mapped_keys = set(subject_variables) | set(session_variables)
+        remaining_variables = {
+            key: value
+            for key, value in parameter_values.items()
+            if key not in mapped_keys and value is not None
+        }
+
+        flattened_scope = {
+            **subject_variables,
+            **session_variables,
+            **remaining_variables,
+        }
+
+        meta = {
+            "source_file": str(path),
+            "n_params": len(parameter_values),
+            "parameter_values": parameter_values,
+            "subject_variables": subject_variables,
+            "session_variables": session_variables,
+            "unmapped_variables": remaining_variables,
+            "variable_mapping": {scope: list(keys) for scope, keys in mapping.items()},
+            "scope": settings.dataset.session_scope,
+            **flattened_scope,
+        }
+
+        return t, df, meta

mesofield/datakit/sources/session/notes.py

@@ -0,0 +1,63 @@
+"""Session notes data source.
+
+Parses free-form ``*_notes.txt`` files where each line begins with a timestamp
+followed by a colon and a short message.  The result is a simple dataframe with
+an index-aligned ``time_elapsed_s`` axis so that manual observations can be
+plotted alongside other timeseries.
+"""
+
+import pandas as pd
+import numpy as np
+from pathlib import Path
+import re
+from datetime import datetime
+
+from mesofield.datakit.sources.register import LoadContext, TimeseriesSource
+
+
+class SessionNotesSource(TimeseriesSource):
+    """Load timestamped notes recorded by the experimenter."""
+    tag = "notes"
+    patterns = ("**/*_notes.txt",)
+    camera_tag = None
+    flatten_payload = False
+    line_pattern = r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}):\s*(.*)"
+    timestamp_format = "%Y-%m-%d %H:%M:%S"
+    empty_time_value = 0.0
+    
+    def build_timeseries(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[np.ndarray, pd.DataFrame, dict]:
+        """Parse ``*_notes.txt`` into a timeline-aware dataframe."""
+        with open(path, 'r') as f:
+            notes = f.readlines()
+        
+        timestamps = []
+        note_texts = []
+        
+        for line in notes:
+            if not line.strip():
+                continue
+            
+            match = re.match(self.line_pattern, line.strip())
+            if match:
+                timestamp_str, note_text = match.groups()
+                try:
+                    timestamp = datetime.strptime(timestamp_str, self.timestamp_format)
+                    timestamps.append(timestamp)
+                    note_texts.append(note_text)
+                except ValueError:
+                    continue
+        
+        if timestamps:
+            # Convert to seconds relative to first note
+            t = np.array([(ts - timestamps[0]).total_seconds() for ts in timestamps])
+            df = pd.DataFrame({'timestamp': timestamps, 'note': note_texts})
+        else:
+            t = np.array([self.empty_time_value])
+            df = pd.DataFrame({'timestamp': [], 'note': []})
+        
+        return t.astype(np.float64), df, {"source_file": str(path), "n_notes": len(note_texts)}

mesofield/datakit/sources/session/timestamps.py

@@ -0,0 +1,58 @@
+"""Session timing markers data source.
+
+Transforms a CSV of per-device ``started``/``stopped`` timestamps into a static
+stream that downstream tooling can inspect alongside other metadata sources.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pandas as pd
+
+from mesofield.datakit.sources.register import IntervalSeriesSource, LoadContext
+
+
+class SessionTimestampsSource(IntervalSeriesSource):
+    """Load per-device start/stop timestamps as intervals."""
+
+    tag = "timestamps"
+    patterns = ("**/*_timestamps.csv",)
+    camera_tag = None
+
+    required_columns = ("device_id", "started", "stopped")
+
+    def build_intervals(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[pd.DataFrame, dict]:
+        frame = pd.read_csv(path)
+
+        missing = [col for col in self.required_columns if col not in frame.columns]
+        if missing:
+            raise ValueError(f"Timestamps CSV missing required columns {missing}: {path}")
+
+        normalized = frame.copy()
+        for column in ("started", "stopped"):
+            normalized[column] = pd.to_datetime(normalized[column], errors="coerce")
+
+        if normalized["started"].notna().any():
+            origin = normalized["started"].min()
+        else:
+            origin = pd.Timestamp(0)
+
+        normalized["start_s"] = (normalized["started"] - origin).dt.total_seconds()
+        normalized["stop_s"] = (normalized["stopped"] - origin).dt.total_seconds()
+
+        meta = {
+            "source_file": str(path),
+            "devices": normalized["device_id"].dropna().astype(str).tolist(),
+            "time_basis": "timestamps_relative",
+        }
+
+        return normalized, meta
+
+
+__all__ = ["SessionTimestampsSource"]