mesofield 0.3.2b0__py3-none-any.whl

mesofield/datakit/sources/analysis/mesomap.py

@@ -0,0 +1,194 @@
+"""Mesomap trace loader.
+
+Parses wide-field mesomap traces exported as ``*_mesoscope.ome_traces.csv``
+files. The accompanying ``.mask.npy`` and ``.regions.csv`` files are treated
+as session-level metadata so they can be stored once per subject/session
+instead of repeating for every task.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable, Tuple
+
+import numpy as np
+import pandas as pd
+
+from mesofield.datakit.config import settings
+from mesofield.datakit.sources.register import LoadContext, TimeseriesSource
+
+
+class MesoMapSource(TimeseriesSource):
+	"""Load mesomap traces and attach optional mask/region metadata."""
+
+	tag = "mesomap"
+	patterns = ("**/*_mesoscope.ome_traces.csv",)
+	camera_tag = "meso_metadata"
+	flatten_payload = True
+	requires = ("dataqueue",)
+ 
+	mask_suffix = ".mask.npy"
+	regions_suffix = ".regions.csv"
+
+	frame_column = "frame"
+	time_basis = "mesomap_frame"
+
+	def build_timeseries(
+		self,
+		path: Path,
+		*,
+		context: LoadContext | None = None,
+	) -> tuple[np.ndarray, pd.DataFrame, dict]:
+		trace_df = pd.read_csv(path)
+
+		if self.frame_column not in trace_df.columns:
+			raise ValueError(f"Mesomap traces missing required column '{self.frame_column}' in {path}")
+
+		trace_df = trace_df.copy()
+		trace_df.sort_values(by=self.frame_column, inplace=True)
+
+		trace_columns = [col for col in trace_df.columns if col != self.frame_column]
+		timeline = trace_df[self.frame_column].to_numpy(dtype=np.float64)
+		timeline_meta = {"time_basis": self.time_basis}
+		aligned = self._aligned_timeline(context, len(trace_df))
+		if aligned is not None:
+			timeline, timeline_meta = aligned
+
+		payload_df = trace_df.loc[:, trace_columns].copy()
+		payload_df["time_elapsed_s"] = timeline
+
+		regions_df, roi_to_mask, missing_regions = self._load_regions(path, trace_columns)
+		mask_info = self._load_mask(path)
+
+		meta = {
+			"source_file": str(path),
+			"n_frames": len(trace_df),
+			"n_rois": len(trace_columns),
+			"trace_columns": trace_columns,
+			"frame_column": self.frame_column,
+			"frame_index": trace_df[self.frame_column].to_numpy(dtype=np.float64),
+			"roi_to_mask_label": roi_to_mask,
+			"roi_missing_region_metadata": missing_regions,
+			**timeline_meta,
+			"scope": settings.dataset.session_scope,
+			**mask_info,
+		}
+
+		if regions_df is not None:
+			meta.update(
+				{
+					"regions_path": str(self._regions_path(path)),
+					"region_count": len(regions_df),
+					"regions_table": regions_df,
+				}
+			)
+
+		return timeline, payload_df, meta
+
+	def _aligned_timeline(
+		self,
+		context: LoadContext | None,
+		n_frames: int,
+	) -> tuple[np.ndarray, dict] | None:
+		if context is None:
+			return None
+
+		if context.dataqueue_frame is not None:
+			aligned = self._aligned_from_frame(context.dataqueue_frame, n_frames)
+			if aligned is not None:
+				return aligned
+
+		if context.master_timeline is not None and context.master_timeline.shape[0] == n_frames:
+			return context.master_timeline, {
+				"time_basis": "master_timeline",
+				"dataqueue_alignment": "direct",
+				"dataqueue_anchors": int(n_frames),
+			}
+
+		return None
+
+	def _aligned_from_frame(
+		self,
+		frame: pd.DataFrame,
+		n_frames: int,
+	) -> tuple[np.ndarray, dict] | None:
+		device_col = "device_id"
+		queue_col = settings.timeline.queue_column
+		if device_col not in frame.columns or queue_col not in frame.columns:
+			return None
+
+		device_series = frame[device_col].astype(str)
+		mask = pd.Series(False, index=frame.index)
+		for pattern in settings.timeline.window_device_patterns:
+			mask |= device_series.str.contains(pattern, case=False, na=False, regex=False)
+
+		if not mask.any():
+			return None
+
+		times = pd.to_numeric(frame.loc[mask, queue_col], errors="coerce").dropna().to_numpy(dtype=np.float64)
+		if times.size == 0:
+			return None
+
+		times = times - times[0]
+		n_anchors = int(times.size)
+		if n_anchors == n_frames:
+			return times, {
+				"time_basis": "dataqueue",
+				"dataqueue_alignment": "direct",
+				"dataqueue_anchors": n_anchors,
+			}
+
+		anchor_index = np.arange(n_anchors, dtype=np.float64)
+		frame_index = np.linspace(0.0, float(n_anchors - 1), num=n_frames, dtype=np.float64)
+		aligned = np.interp(frame_index, anchor_index, times)
+		return aligned, {
+			"time_basis": "dataqueue",
+			"dataqueue_alignment": "resampled",
+			"dataqueue_anchors": n_anchors,
+		}
+
+	def _mask_path(self, path: Path) -> Path:
+		return path.with_suffix(self.mask_suffix)
+
+	def _regions_path(self, path: Path) -> Path:
+		return path.with_suffix(self.regions_suffix)
+
+	def _load_mask(self, path: Path) -> dict:
+		mask_path = self._mask_path(path)
+		if not mask_path.exists():
+			return {
+				"mask_path": str(mask_path),
+				"mask_missing": True,
+			}
+
+		mask_array = np.load(mask_path)
+		unique_labels = np.unique(mask_array)
+		return {
+			"mask_path": str(mask_path),
+			"mask_shape": tuple(int(dim) for dim in mask_array.shape),
+			"mask_dtype": str(mask_array.dtype),
+			"mask_labels": unique_labels.tolist(),
+			"mask_missing": False,
+		}
+
+	def _load_regions(
+		self, path: Path, trace_columns: Iterable[str]
+	) -> Tuple[pd.DataFrame | None, dict[str, int], list[str]]:
+		regions_path = self._regions_path(path)
+		if not regions_path.exists():
+			return None, {}, list(trace_columns)
+
+		regions_df = pd.read_csv(regions_path)
+
+		required_cols = {"acronym", "mask_label"}
+		if not required_cols.issubset(regions_df.columns):
+			missing = ", ".join(sorted(required_cols - set(regions_df.columns)))
+			raise ValueError(f"Regions file {regions_path} missing required columns: {missing}")
+
+		region_lookup = regions_df.set_index("acronym")["mask_label"]
+		roi_to_mask = {
+			roi: int(region_lookup[roi]) for roi in trace_columns if roi in region_lookup
+		}
+		missing_regions = sorted(set(trace_columns) - set(region_lookup.index))
+
+		return regions_df, roi_to_mask, missing_regions

mesofield/datakit/sources/analysis/mesoscope.py

@@ -0,0 +1,77 @@
+"""Mesoscope mean fluorescence analysis data source.
+
+This module translates CSV exports from the mesoscope processing pipeline into
+time-indexed :class:`~datakit.datamodel.LoadedStream` objects.  Each record in the
+table represents one slice/ROI with derived dF/F traces, ready for alignment
+against other experiment timelines.
+"""
+
+import pandas as pd
+import numpy as np
+from pathlib import Path
+
+from mesofield.datakit.sources.register import LoadContext, TimeseriesSource, document
+
+
+class MesoMeanSource(TimeseriesSource):
+    """Load mean fluorescence CSV.
+
+    The CSV is expected to contain at least ``Slice`` and ``Mean`` columns.  
+    """
+    tag = "meso"
+    patterns = ("**/*_meso-mean-trace.csv",)
+    camera_tag = "meso_metadata"  # Bind to meso camera
+    required_columns = ("Mean")
+    assumed_frame_rate_hz = 50.0
+    normalization_baseline = "min"
+    
+    def build_timeseries(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[np.ndarray, pd.DataFrame, dict]:
+        """Read a mesoscope CSV and return a normalized table."""
+        df = pd.read_csv(path)
+        #remove the "Slice" column
+        df = df.drop(columns=["Slice"])
+        
+        n_frames = len(df)
+        t = np.arange(n_frames, dtype=np.float64) / float(self.assumed_frame_rate_hz)
+        df = df.copy()
+                
+        return t, df, {"source_file": str(path), "n_slices": len(df)}
+
+    
+class MesoDFFSource(TimeseriesSource):
+    """Load DFF fluorescence CSV.
+
+    The CSV is expected to contain at least ``Slice`` and ``Mean`` columns.
+    """
+    tag = "meso"
+    patterns = ("**/*_meso-collapse_first-trace.csv",)
+    camera_tag = "meso_metadata"  # Bind to meso camera
+    required_columns = ("Mean")
+    assumed_frame_rate_hz = 50.0
+    normalization_baseline = "min"
+    
+    def build_timeseries(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[np.ndarray, pd.DataFrame, dict]:
+        """Read a mesoscope CSV and return a normalized table."""
+        df = pd.read_csv(path)
+        #remove the "Slice" column
+        df = df.drop(columns=["Slice"])
+        #rename the "Mean" column to "dF_F"
+        df = df.rename(columns={"Mean": "dF_F"})
+        
+        n_frames = len(df)
+        t = np.arange(n_frames, dtype=np.float64) / float(self.assumed_frame_rate_hz)
+        df = df.copy()
+                
+        return t, df, {"source_file": str(path), "n_slices": len(df)}
+
+

mesofield/datakit/sources/analysis/pupil.py

@@ -0,0 +1,246 @@
+"""Pupil DeepLabCut analysis data source."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from ..._utils._logger import get_logger
+from mesofield.datakit.sources.camera.pupil import PupilMetadataSource
+from mesofield.datakit.sources.register import LoadContext, TimeseriesSource
+from mesofield.datakit.timeline import DataqueueIndex
+
+logger = get_logger(__name__)
+
+
+class PupilDLCSource(TimeseriesSource):
+    """Load DeepLabCut HDF5 output and compute pupil diameters."""
+
+    tag = "pupil_dlc"
+    patterns = (
+        "**/*_pupilDLC_*.h5",
+        "**/*_pupilDLC_*.hdf5",
+        "**/*_pupil.omeDLC_*.h5"
+    )
+    camera_tag = "pupil_metadata"
+    flatten_payload = True
+    requires = ("dataqueue", "pupil_metadata")
+    default_frame_rate_hz = 20.0
+    confidence_threshold = 0.7
+    pixel_to_mm = 53.6
+    dpi = 300
+    landmark_pairs = ((0, 1), (2, 3), (4, 5), (6, 7))
+    warn_on_low_confidence = True
+    alignment_device_id = "thorcam"
+
+    def build_timeseries(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[np.ndarray, pd.DataFrame, dict]:
+        """Load DeepLabCut output and return a time-indexed table."""
+        context = self._require_context(context)
+        analyzed_df = self._analyze_pupil_h5(path)
+
+        n_frames = len(analyzed_df)
+        timing = self._aligned_timeline(context, n_frames)
+        if timing is None:
+            metadata_path = context.path_for("pupil_metadata")
+            if metadata_path is None:
+                logger.warning(
+                    "PupilDLCSource: ThorCam alignment and pupil metadata missing; falling back to assumed frame-rate timeline."
+                )
+                t = np.arange(n_frames, dtype=np.float64) / float(self.default_frame_rate_hz)
+                timing_meta = {
+                    "time_basis": "assumed_frame_rate",
+                    "assumed_frame_rate_hz": float(self.default_frame_rate_hz),
+                    "pupil_metadata_file": None,
+                    "pupil_metadata_alignment": None,
+                    "pupil_metadata_frames": None,
+                }
+            else:
+                logger.warning(
+                    "PupilDLCSource: ThorCam alignment missing; falling back to metadata timeline."
+                )
+                metadata_stream = PupilMetadataSource().load(metadata_path, context=context)
+                metadata_t = metadata_stream.t
+                if metadata_t.size == 0:
+                    raise ValueError("PupilDLCSource: pupil metadata contains no timeline samples")
+                metadata_t = metadata_t - metadata_t[0]
+                n_meta = int(metadata_t.size)
+                if n_meta == n_frames:
+                    t = metadata_t
+                    method = "metadata_direct"
+                else:
+                    anchor_index = np.arange(n_meta, dtype=np.float64)
+                    frame_index = np.linspace(0.0, float(n_meta - 1), num=n_frames, dtype=np.float64)
+                    t = np.interp(frame_index, anchor_index, metadata_t)
+                    method = "metadata_resampled"
+
+                timing_meta = {
+                    "time_basis": "pupil_metadata",
+                    "pupil_metadata_file": str(metadata_path),
+                    "pupil_metadata_alignment": method,
+                    "pupil_metadata_frames": n_meta,
+                }
+        else:
+            t, timing_meta = timing
+
+        analyzed_df = analyzed_df.copy()
+        analyzed_df["time_elapsed_s"] = t
+
+        meta = {
+            "source_file": str(path),
+            "n_frames": n_frames,
+            "class_name": self.__class__.__name__,
+            "confidence_threshold": self.confidence_threshold,
+            "pixel_to_mm": self.pixel_to_mm,
+            "landmark_pairs": self.landmark_pairs,
+            **timing_meta,
+        }
+
+        return t, analyzed_df, meta
+
+    def _aligned_timeline(self, context: LoadContext, n_frames: int) -> tuple[np.ndarray, dict] | None:
+        if context.dataqueue_frame is not None:
+            times = self._aligned_from_frame(context.dataqueue_frame)
+            dq_path = context.path_for("dataqueue")
+        else:
+            dq_path = context.path_for("dataqueue")
+            if dq_path is None:
+                # No dataqueue available for this session (e.g. older
+                # STREHAB02/03/05). Signal the caller to fall back to the
+                # pupil-metadata timeline instead of raising.
+                return None
+            timeline = DataqueueIndex.from_path(dq_path)
+            device_slice = timeline.slice(
+                lambda ids: ids.str.contains(self.alignment_device_id, case=False, na=False, regex=False)
+            )
+            times = device_slice.queue_elapsed().to_numpy(dtype=np.float64)
+        if times.size == 0:
+            return None
+        n_anchors = int(times.size)
+        if n_anchors == n_frames:
+            aligned = times
+            method = "direct"
+        else:
+            anchor_index = np.arange(n_anchors, dtype=np.float64)
+            frame_index = np.linspace(0.0, float(n_anchors - 1), num=n_frames, dtype=np.float64)
+            aligned = np.interp(frame_index, anchor_index, times)
+            method = "resampled"
+
+        aligned = aligned - aligned[0]
+
+        meta = {
+            "time_basis": "dataqueue",
+            "dataqueue_file": str(dq_path) if dq_path is not None else None,
+            "dataqueue_device": self.alignment_device_id,
+            "dataqueue_anchors": n_anchors,
+            "dataqueue_alignment": method,
+        }
+        return aligned, meta
+
+    def _aligned_from_frame(self, frame: pd.DataFrame) -> np.ndarray:
+        device_col = "device_id"
+        time_col = "queue_elapsed"
+        if device_col not in frame.columns or time_col not in frame.columns:
+            return np.array([], dtype=np.float64)
+        device_series = frame[device_col].astype(str)
+        mask = device_series.str.contains(self.alignment_device_id, case=False, na=False, regex=False)
+        times = pd.to_numeric(frame.loc[mask, time_col], errors="coerce").dropna()
+        return times.to_numpy(dtype=np.float64)
+
+    def _analyze_pupil_h5(
+        self,
+        filepath: Path,
+        *,
+        confidence_threshold: float | None = None,
+        pixel_to_mm: float | None = None,
+        dpi: int | None = None,
+    ) -> pd.DataFrame:
+        """Compute pupil diameters from DeepLabCut HDF5 output."""
+        threshold = self.confidence_threshold if confidence_threshold is None else confidence_threshold
+        px_to_mm = self.pixel_to_mm if pixel_to_mm is None else pixel_to_mm
+        dpi_value = self.dpi if dpi is None else dpi
+
+        with pd.HDFStore(filepath, mode="r") as store:
+            keys = store.keys()
+            if not keys:
+                raise ValueError(f"No keys found in HDF5 file: {filepath}")
+            key = "/df_with_missing" if "/df_with_missing" in keys else keys[0]
+            frame = store.get(key)
+
+        if not isinstance(frame.columns, pd.MultiIndex) or frame.columns.nlevels < 3:
+            raise ValueError(f"Unexpected HDF5 column format for {filepath}")
+
+        scorer = str(frame.columns.get_level_values(0)[0])
+        sub = frame[scorer]
+
+        bodyparts = sorted(set(sub.columns.get_level_values(0)))
+        coords_list: list[np.ndarray] = []
+        conf_list: list[np.ndarray] = []
+        used_bodyparts: list[str] = []
+
+        for bodypart in bodyparts:
+            try:
+                x = sub[(bodypart, "x")].to_numpy(dtype=np.float64)
+                y = sub[(bodypart, "y")].to_numpy(dtype=np.float64)
+                likelihood = sub[(bodypart, "likelihood")].to_numpy(dtype=np.float64)
+            except KeyError:
+                continue
+            coords_list.append(np.stack([x, y], axis=1))
+            conf_list.append(likelihood)
+            used_bodyparts.append(bodypart)
+
+        if not coords_list:
+            raise ValueError(f"No coordinate columns found in {filepath}")
+
+        coords = np.stack(coords_list, axis=1)
+        conf = np.stack(conf_list, axis=1)
+
+        any_confident = np.any(conf >= threshold)
+        if self.warn_on_low_confidence and not any_confident:
+            logger.warning(
+                "PupilDLCSource: no confidence values above threshold",
+                extra={
+                    "phase": "pupil_dlc_analysis",
+                    "threshold": threshold,
+                    "dpi": dpi_value,
+                    "bodyparts": used_bodyparts,
+                },
+            )
+
+        n_points = coords.shape[1]
+        pairs = [(a, b) for a, b in self.landmark_pairs if a < n_points and b < n_points]
+        if not pairs:
+            diameters = np.full(coords.shape[0], np.nan, dtype=np.float64)
+        else:
+            dists = []
+            for a, b in pairs:
+                diff = coords[:, a, :] - coords[:, b, :]
+                dist = np.linalg.norm(diff, axis=1)
+                valid = (conf[:, a] >= threshold) & (conf[:, b] >= threshold)
+                dist = np.where(valid, dist, np.nan)
+                dists.append(dist)
+            stacked = np.vstack(dists).T
+            valid_counts = np.sum(np.isfinite(stacked), axis=1)
+            diameters = np.full(stacked.shape[0], np.nan, dtype=np.float64)
+            valid_mask = valid_counts > 0
+            if not np.any(valid_mask) and self.warn_on_low_confidence:
+                logger.warning(
+                    "PupilDLCSource: no valid landmark pairs after confidence filtering",
+                    extra={
+                        "phase": "pupil_dlc_analysis",
+                        "threshold": threshold,
+                        "bodyparts": used_bodyparts,
+                    },
+                )
+            if np.any(valid_mask):
+                with np.errstate(all="ignore"):
+                    diameters[valid_mask] = np.nanmean(stacked[valid_mask], axis=1)
+
+        pupil_series = pd.Series(diameters, index=frame.index).interpolate() / px_to_mm
+        return pd.DataFrame({"pupil_diameter_mm": pupil_series})