mesofield 0.3.2b0__py3-none-any.whl

mesofield/datakit/sources/behavior/dataqueue.py

@@ -0,0 +1,281 @@
+"""Dataqueue synchronization data source.
+
+This loader ingests the central ``*_dataqueue.csv`` log produced by the rig and
+returns a stream describing every message with absolute and relative timing.
+It is commonly used to align per-device clocks (treadmill, cameras, nidaq) by
+extracting anchor pairs for downstream timeline fitting.
+"""
+
+from typing import Optional
+import pandas as pd
+import numpy as np
+from pathlib import Path
+
+from mesofield.datakit.sources.register import LoadContext, TimeseriesSource
+
+
+class DataqueueSource(TimeseriesSource):
+    """Load the dataqueue CSV as a time-indexed table."""
+    tag = "dataqueue"
+    patterns = ("**/*_dataqueue.csv",)
+    camera_tag = None
+    time_column = "queue_elapsed"
+    device_id_column = "device_id"
+    device_timestamp_column = "device_ts"
+    payload_column = "payload"
+    master_device_priority: tuple[str, ...] = ("dhyana", "mesoscope")
+    device_alias_patterns: tuple[tuple[str, str], ...] = (
+        ("dhyana", "meso"),
+        ("mesoscope", "meso"),
+        ("thorcam", "pupil"),
+    )
+    
+    def build_timeseries(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[np.ndarray, pd.DataFrame, dict]:
+        """Read ``*_dataqueue.csv`` and return a time-indexed table."""
+        raw = pd.read_csv(path, low_memory=False)
+
+        if self.time_column not in raw.columns:
+            raise ValueError(f"Dataqueue file missing required time column '{self.time_column}': {path}")
+
+        queue_numeric = pd.to_numeric(raw[self.time_column], errors="coerce")
+        queue_elapsed = queue_numeric.to_numpy(dtype=np.float64)
+        if queue_elapsed.size == 0:
+            raise ValueError("Dataqueue has no timing samples")
+
+        queue_start = float(queue_elapsed[0])
+        t = queue_elapsed - queue_start
+
+        master_id = self._select_master_device(raw)
+        master_elapsed = self._master_elapsed(raw, master_id, queue_start)
+        if master_elapsed.size == 0:
+            master_elapsed = t
+
+        device_elapsed = self._build_device_elapsed(raw, queue_start)
+        device_ts = self._build_device_ts(raw)
+        device_aliases = self._build_device_aliases(raw)
+        affine = self._fit_master_affine(raw, master_id, queue_start)
+        device_aligned = self._apply_affine(device_elapsed, affine)
+
+        frame = raw.copy()
+        frame["time_elapsed_s"] = t.astype(np.float64)
+        per_device_start_map: dict[str, float] = {}
+        if self.device_id_column in frame.columns:
+            per_device_start = (
+                frame.groupby(self.device_id_column)[self.time_column]
+                .transform("min")
+                .astype(np.float64)
+            )
+            per_device_start_map = {
+                str(device_id): float(value)
+                for device_id, value in (
+                    frame.groupby(self.device_id_column)[self.time_column]
+                    .min()
+                    .items()
+                )
+            }
+            # `device_elapsed_s` is `queue_elapsed - per_device_start`; keep the
+            # offsets in meta and drop the redundant full-length column.
+
+        # Memory optimizations: parse timestamp strings to datetime64, low-cardinality
+        # string columns to categoricals. These columns are stored as full-length
+        # ndarrays per row in the materialized output, so per-value bytes matter.
+        #
+        # ``payload`` is mostly numeric (TTL levels, counts) with a sparse mix of
+        # semantic strings (e.g. ``EncoderData(...)``). The dataqueue itself
+        # treats payloads as opaque — sources that need the semantic content
+        # (such as ``TreadmillSource``) re-read the raw CSV via
+        # ``context.path_for("dataqueue")`` and parse it themselves. Here we
+        # just downcast the whole column to ``float32`` (non-numeric entries
+        # become NaN) to keep the materialized output small.
+        if self.device_timestamp_column in frame.columns:
+            parsed = pd.to_datetime(frame[self.device_timestamp_column], errors="coerce", utc=True)
+            if not parsed.isna().all():
+                frame[self.device_timestamp_column] = parsed
+        if "packet_ts" in frame.columns:
+            parsed = pd.to_datetime(frame["packet_ts"], errors="coerce", utc=True)
+            if not parsed.isna().all():
+                frame["packet_ts"] = parsed
+        if self.device_id_column in frame.columns:
+            frame[self.device_id_column] = frame[self.device_id_column].astype("category")
+        if self.payload_column in frame.columns:
+            frame[self.payload_column] = (
+                pd.to_numeric(frame[self.payload_column], errors="coerce")
+                .astype(np.float32)
+            )
+
+        meta = {
+            "source_file": str(path),
+            "n_entries": int(len(raw)),
+            "master_device_id": master_id,
+            "master_queue_start": float(queue_start),
+            "per_device_queue_start": per_device_start_map,
+            "device_sample_rate_hz": self._estimate_device_rates(device_elapsed),
+            "device_aliases": device_aliases,
+            "affine": affine,
+            "time_basis": self.time_column,
+        }
+
+        return t.astype(np.float64), frame, meta
+
+    def _select_master_device(self, df: pd.DataFrame) -> str:
+        """Pick the device used as the master time basis."""
+
+        if self.device_id_column not in df.columns:
+            return "master"
+
+        ids = df[self.device_id_column].astype(str).fillna("")
+        for pattern in self.master_device_priority:
+            mask = ids.str.contains(pattern, case=False, regex=False)
+            if mask.any():
+                return str(ids[mask].iloc[0])
+
+        non_empty = ids[ids != ""]
+        if not non_empty.empty:
+            return str(non_empty.iloc[0])
+        return "master"
+
+    def _master_elapsed(self, df: pd.DataFrame, master_id: str, queue_start: float) -> np.ndarray:
+        """Return elapsed seconds for the master device."""
+
+        device_series = df.get(self.device_id_column)
+        if device_series is None:
+            return np.array([], dtype=np.float64)
+
+        mask = device_series.astype(str) == master_id
+        master_rows = df.loc[mask]
+        if master_rows.empty:
+            return np.array([], dtype=np.float64)
+
+        master_queue = pd.to_numeric(master_rows[self.time_column], errors="coerce").dropna()
+        if master_queue.empty:
+            return np.array([], dtype=np.float64)
+
+        master_queue = master_queue - float(master_queue.iloc[0])
+        return master_queue.to_numpy(dtype=np.float64)
+
+    def _estimate_rate(self, timeline: np.ndarray) -> float:
+        if timeline.size < 2:
+            return 0.0
+        diffs = np.diff(timeline)
+        diffs = diffs[np.isfinite(diffs) & (diffs > 0)]
+        if diffs.size == 0:
+            return 0.0
+        return float(1.0 / np.median(diffs))
+
+    def _build_device_elapsed(self, df: pd.DataFrame, queue_start: float) -> dict[str, np.ndarray]:
+        """Return per-device elapsed seconds from queue timestamps."""
+
+        device_elapsed: dict[str, np.ndarray] = {}
+        if self.device_id_column not in df.columns:
+            return device_elapsed
+
+        for device_id, group in df.groupby(self.device_id_column):
+            device_key = str(device_id)
+            queue_rel = pd.to_numeric(group[self.time_column], errors="coerce").to_numpy(dtype=np.float64)
+            queue_rel = queue_rel - float(queue_start)
+            order = np.argsort(queue_rel)
+            device_elapsed[device_key] = queue_rel[order]
+        return device_elapsed
+
+    def _build_device_ts(self, df: pd.DataFrame) -> dict[str, np.ndarray]:
+        """Return per-device raw timestamps as ISO strings when possible."""
+
+        device_ts: dict[str, np.ndarray] = {}
+        if self.device_id_column not in df.columns or self.device_timestamp_column not in df.columns:
+            return device_ts
+
+        for device_id, group in df.groupby(self.device_id_column):
+            device_key = str(device_id)
+            parsed = pd.to_datetime(group[self.device_timestamp_column], errors="coerce", utc=True)
+            if parsed.isna().all():
+                device_ts[device_key] = group[self.device_timestamp_column].to_numpy()
+            else:
+                device_ts[device_key] = parsed.dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ").to_numpy()
+        return device_ts
+
+    def _build_device_aliases(self, df: pd.DataFrame) -> dict[str, str]:
+        """Map device identifiers to configured aliases (e.g., Dhyana → meso)."""
+
+        aliases: dict[str, str] = {}
+        if self.device_id_column not in df.columns:
+            return aliases
+
+        ids = df[self.device_id_column].dropna().astype(str).unique().tolist()
+        for device_id in ids:
+            alias = self._alias_for_device(device_id)
+            if alias is not None:
+                aliases[device_id] = alias
+        return aliases
+
+    def _alias_for_device(self, device_id: str) -> Optional[str]:
+        lowered = device_id.lower()
+        for pattern, alias in self.device_alias_patterns:
+            if pattern.lower() in lowered:
+                return alias
+        return None
+
+    def _fit_master_affine(
+        self,
+        df: pd.DataFrame,
+        master_id: str,
+        queue_start: float,
+    ) -> dict[str, float] | None:
+        """Fit affine map from elapsed seconds to absolute time using master device."""
+
+        if self.device_id_column not in df.columns or self.device_timestamp_column not in df.columns:
+            return None
+
+        master_rows = df.loc[df[self.device_id_column].astype(str) == master_id]
+        if master_rows.empty:
+            return None
+
+        elapsed = pd.to_numeric(master_rows[self.time_column], errors="coerce").to_numpy(dtype=np.float64)
+        elapsed = elapsed - float(queue_start)
+        absolute = pd.to_datetime(master_rows[self.device_timestamp_column], errors="coerce", utc=True)
+        if elapsed.size < 2 or absolute.isna().all():
+            return None
+
+        valid_mask = np.isfinite(elapsed) & ~absolute.isna()
+        if valid_mask.sum() < 2:
+            return None
+
+        elapsed = elapsed[valid_mask]
+        absolute = absolute[valid_mask]
+
+        e0 = float(elapsed[0])
+        e1 = float(elapsed[-1])
+        a0 = float(absolute.iloc[0].value) / 1e9
+        a1 = float(absolute.iloc[-1].value) / 1e9
+        if e1 == e0:
+            return None
+
+        a = (a1 - a0) / (e1 - e0)
+        b = a0 - a * e0
+        return {"a": float(a), "b": float(b)}
+
+    def _apply_affine(
+        self,
+        device_elapsed: dict[str, np.ndarray],
+        affine: dict[str, float] | None,
+    ) -> dict[str, np.ndarray]:
+        """Project elapsed times into aligned absolute seconds (UTC)."""
+
+        if affine is None:
+            return {}
+        a = affine.get("a")
+        b = affine.get("b")
+        if a is None or b is None:
+            return {}
+
+        aligned: dict[str, np.ndarray] = {}
+        for device_id, elapsed in device_elapsed.items():
+            aligned[device_id] = (a * elapsed + b).astype(np.float64)
+        return aligned
+
+    def _estimate_device_rates(self, device_elapsed: dict[str, np.ndarray]) -> dict[str, float]:
+        return {device_id: self._estimate_rate(values) for device_id, values in device_elapsed.items()}

mesofield/datakit/sources/behavior/psychopy.py

@@ -0,0 +1,364 @@
+"""Psychopy behavioral event data source.
+
+Explicit assumptions:
+- Psychopy CSV contains at least one display_*.started column with 2+ numeric rows.
+- A single keypress RT column exists (key_resp*.rt) and is used to align time zero.
+- If a dataqueue timeline exists, it must include nidaq payload==1 pulses.
+
+psychopy files have an expStart column that indicates start of the experiment in absolute clock datetime.
+Psychopy files have a Custom_Trigger routine; a spacebar press then triggers a sync pulse to the DAQ system.
+Thus, the true start is the `expStart` + the `key_resp_2.rt` (the key_resp_2.rt is column in the psychopy file, with 1 value, but that value may not be on the first row)
+
+true end can be calculated using the `expStart` + the elapsed value from the last row of the column that precedes `Subject ID` and 
+
+The dataqueue file collects nidaq TTL pulses. The first pulse (payload=1) has a clock time and relative time (queue_elapsed). We can use this to align the psychopy timeline to the dataqueue timeline.
+The experimental window, for sample_experiment3, is between the first nidaq and last nidaq.
+
+The psychopy experiment likely runs longer than the nidaq pulse window, so we can trim the psychopy timeline to be within the nidaq pulse window.
+
+Additional details:
+    - The loader derives its primary timeline from event start columns (e.g. display_*.started).
+    - All time-like columns are aligned by subtracting the keypress RT so time zero is the trigger.
+    - For task-gratings, rows are taken from trial metadata and extended to include the final stopped event.
+    - When dataqueue is available, the timeline is trimmed to the nidaq window but remains zero-based.
+"""
+
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Optional
+import re
+
+import numpy as np
+import pandas as pd
+
+from mesofield.datakit.sources.register import IntervalSeriesSource, LoadContext
+from mesofield.datakit.timeline import DataqueueIndex
+
+
+
+_TASK_PATTERN = re.compile(r"task[-_]?([A-Za-z0-9]+)", re.IGNORECASE)
+_TASK_ALIASES = {
+    "gratings": "task-gratings",
+    "movies": "task-movies",
+}
+_TIME_SUFFIXES = (".started", ".stopped", ".rt", ".t")
+
+
+class Psychopy(IntervalSeriesSource):
+    """Load Psychopy trial windows as an interval table."""
+
+    tag = "psychopy"
+    patterns = ("**/*_psychopy.csv",)
+    requires = ("dataqueue",)
+    task_pattern = _TASK_PATTERN
+    task_aliases = _TASK_ALIASES
+    time_suffixes = _TIME_SUFFIXES
+    keypress_rt_pattern = re.compile(r"key_resp.*\.rt$")
+    time_source_prefix = "display_"
+    time_source_suffix = ".started"
+    gratings_columns = (
+        "trials.thisN",
+        "stim_grayScreen.started",
+        "stim_grayScreen.stopped",
+        "stim_grating.started",
+        "stim_grating.stopped",
+        "display_gratings.stopped",
+    )
+    movies_columns = (
+        "trials_2.thisN",
+        "trials_2.thisIndex",
+        "trials2.thisN",
+        "trials.thisIndex",
+        "description",
+        "thisRow.t",
+        "display_mov.started",
+        "natural_mov.started",
+        "display_mov.stopped",
+        "grey.started",
+        "grey.stopped",
+    )
+    dataqueue_elapsed_column = "queue_elapsed"
+    dataqueue_device_column = "device_id"
+    dataqueue_device_match = "nidaq"
+    dataqueue_payload_column = "payload"
+    dataqueue_payload_value = 1
+    def build_intervals(
+        self,
+        path: Path,
+        *,
+        context: LoadContext | None = None,
+    ) -> tuple[pd.DataFrame, dict]:
+        context = self._require_context(context)
+        # 1) Read raw psychopy CSV.
+        raw = pd.read_csv(path, low_memory=False)
+
+        # 2) Infer task and choose trial table.
+        task = self._infer_task(path)
+        if task == "task-gratings":
+            trial_table = self._extract_gratings(raw)
+            task_branch = "gratings"
+        elif task == "task-movies":
+            trial_table = self._extract_movies(raw)
+            task_branch = "movies"
+        else:
+            trial_table = raw
+            task_branch = "raw"
+
+        # 3) Resolve timeline and keypress offset.
+        time_source = trial_table if any(
+            c.startswith(self.time_source_prefix) and c.endswith(self.time_source_suffix) for c in trial_table.columns
+        ) else raw
+        row_time, row_col, row_meta = self._resolve_row_time(time_source)
+        key_rt, key_col = self._resolve_keypress_rt(raw)
+
+        # 4) Parse experiment start time (optional).
+        exp_start = None
+        exp_start_raw = raw.get("expStart")
+        if exp_start_raw is not None and exp_start_raw.dropna().any():
+            exp_start = self._parse_exp_start(exp_start_raw.dropna().iloc[0])
+
+        # 5) Apply dataqueue offset and align times.
+        dq_start, dq_meta = self._dataqueue_offset(context)
+        exp_start_offset = None
+        exp_start_keypress_offset = None
+        exp_start_offset = self._calc_exp_start_offset(exp_start, dq_meta.get("abs_start_time"))
+        # Offset into psychopy-zero time (exp_start + keypress RT).
+        exp_start_keypress_offset = exp_start_offset - float(key_rt)
+        dq_start = float(exp_start_keypress_offset)
+        aligned_t = self._align_time(row_time, key_rt, dq_start)
+        aligned_table = self._align_table(trial_table, key_rt, dq_start)
+
+        # 6) For gratings, build windows once from the aligned table.
+        if task_branch == "gratings":
+            if "window_grating_start" in aligned_table.columns:
+                aligned_table["gratings_windows"] = [
+                    self._build_window_list(aligned_table, "window_grating_start", "window_grating_stop")
+                ] * len(aligned_table)
+            if "window_gray_start" in aligned_table.columns:
+                aligned_table["gray_windows"] = [
+                    self._build_window_list(aligned_table, "window_gray_start", "window_gray_stop")
+                ] * len(aligned_table)
+
+        # 7) Build interval table and metrics.
+        interval_start = None
+        interval_stop = None
+        if "window_grating_start" in aligned_table.columns and "window_grating_stop" in aligned_table.columns:
+            interval_start = aligned_table["window_grating_start"]
+            interval_stop = aligned_table["window_grating_stop"]
+        elif "window_gray_start" in aligned_table.columns and "window_gray_stop" in aligned_table.columns:
+            interval_start = aligned_table["window_gray_start"]
+            interval_stop = aligned_table["window_gray_stop"]
+        else:
+            interval_start = pd.Series(aligned_t, index=aligned_table.index)
+            interval_stop = pd.Series(aligned_t, index=aligned_table.index)
+
+        intervals = aligned_table.rename(
+            columns={c: f"{task_branch}_{c}" for c in aligned_table.columns}
+        ).copy()
+        intervals["start_s"] = pd.to_numeric(interval_start, errors="coerce")
+        intervals["stop_s"] = pd.to_numeric(interval_stop, errors="coerce")
+        intervals = self._apply_experiment_window(intervals, context, dq_start)
+        metrics = {
+            "source_file": str(path),
+            "n_rows": int(len(raw)),
+            "task_inferred": task,
+            "task_branch": task_branch,
+            "row_time_column": row_col,
+            "key_resp_rt": key_rt,
+            "key_resp_column": key_col,
+            "exp_start": exp_start.isoformat() if exp_start is not None else None,
+            "start_offset_s": dq_start,
+        }
+        if exp_start_offset is not None:
+            metrics["exp_start_offset_s"] = float(exp_start_offset)
+        if exp_start_keypress_offset is not None:
+            metrics["exp_start_keypress_offset_s"] = float(exp_start_keypress_offset)
+        metrics.update(row_meta)
+        metrics.update(dq_meta)
+        metrics["time_basis"] = "psychopy_zero_based" if dq_start is not None else "psychopy_native"
+        metrics["alignment_method"] = "keypress_relative"
+
+        return intervals, metrics
+
+    def _parse_exp_start(self, value: Any) -> Optional[pd.Timestamp]:
+        text = str(value).strip()
+        text = text.replace("h", ":", 1)
+        text = re.sub(r":(\d{2})\.(\d{2}\.\d+)", r":\1:\2", text)
+        parsed = pd.to_datetime(text, errors="coerce", utc=False)
+        return None if pd.isna(parsed) else parsed
+
+    def _strip_tz(self, value: pd.Timestamp) -> pd.Timestamp:
+        return value.tz_localize(None) if getattr(value, "tzinfo", None) else value
+
+    def _calc_exp_start_offset(self, exp_start: pd.Timestamp, abs_start_time: Any) -> Optional[float]:
+        if not abs_start_time:
+            return None
+        abs_start = pd.to_datetime(abs_start_time, errors="coerce", utc=False)
+        if pd.isna(abs_start):
+            return None
+        abs_start = self._strip_tz(abs_start)
+        exp_start = self._strip_tz(exp_start)
+        return (abs_start - exp_start).total_seconds()
+
+    def _infer_task(self, file_path: Path) -> str | None:
+        name = file_path.as_posix().lower()
+        match = self.task_pattern.search(name)
+        if match:
+            return self.task_aliases.get(match.group(1).lower())
+        for key, label in self.task_aliases.items():
+            if key in name:
+                return label
+        return None
+
+    def _select_columns(self, frame: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
+        available = [col for col in columns if col in frame.columns]
+        if not available:
+            raise ValueError("Psychopy required columns are missing from the CSV.")
+        return frame.loc[:, available].copy()
+
+    def _coalesce_column(self, frame: pd.DataFrame, columns: list[str]) -> pd.Series:
+        available = [col for col in columns if col in frame.columns]
+        if not available:
+            raise ValueError(f"Psychopy required columns missing: {columns}")
+        return frame.loc[:, available].bfill(axis=1).iloc[:, 0]
+
+    def _resolve_keypress_rt(self, frame: pd.DataFrame) -> tuple[float, str]:
+        candidates = [col for col in frame.columns if self.keypress_rt_pattern.match(str(col))]
+        for col in candidates:
+            series = pd.to_numeric(frame[col], errors="coerce")
+            if series.notna().any():
+                return float(series.dropna().iloc[0]), str(col)
+        raise ValueError("Psychopy keypress RT column not found (key_resp*.rt).")
+
+    def _resolve_row_time(self, frame: pd.DataFrame) -> tuple[np.ndarray, str, dict]:
+        display_candidates = [
+            c for c in frame.columns if c.startswith(self.time_source_prefix) and c.endswith(self.time_source_suffix)
+        ]
+        for col in display_candidates:
+            series = pd.to_numeric(frame[col], errors="coerce")
+            if series.notna().sum() >= 2:
+                filled = series.interpolate(limit_direction="both")
+                time_cols = [c for c in frame.columns if any(c.endswith(suf) for suf in self.time_suffixes)]
+                max_times = [pd.to_numeric(frame[c], errors="coerce").max() for c in time_cols]
+                max_time = max([t for t in max_times if pd.notna(t)], default=filled.max())
+                if max_time > filled.max():
+                    last_valid_idx = filled.last_valid_index()
+                    if last_valid_idx is not None and last_valid_idx < len(filled) - 1:
+                        filled.loc[last_valid_idx + 1:] = max_time
+                meta = {
+                    "row_time_column": col,
+                    "row_time_extended_to": float(max_time),
+                }
+                return filled.to_numpy(dtype=np.float64), col, meta
+        raise ValueError("Psychopy requires a display_*.started column with at least 2 numeric values.")
+
+    def _dataqueue_offset(self, context: LoadContext) -> tuple[Optional[float], dict]:
+        #TODO: If a dataqueue does not have nidaq payload==1 but may start at nidaq==2,
+        # treat the first pulse as the start and log a warning. 
+        dq_path = context.path_for("dataqueue")
+        if context.dataqueue_frame is not None:
+            frame = context.dataqueue_frame
+            queue_all = pd.to_numeric(frame.get(self.dataqueue_elapsed_column), errors="coerce").dropna()
+            if queue_all.empty:
+                raise ValueError("Dataqueue frame contains no queue_elapsed samples")
+            queue_start = float(queue_all.iloc[0])
+            device_series = frame.get(self.dataqueue_device_column, pd.Series(dtype=str)).astype(str)
+            mask = device_series.str.contains(self.dataqueue_device_match, case=False, na=False, regex=False)
+            device_rows = frame.loc[mask]
+            pulses = pd.to_numeric(device_rows.get(self.dataqueue_elapsed_column), errors="coerce")
+            abs_start = pd.to_datetime(device_rows.get("device_ts"), errors="coerce", utc=True)
+            abs_start = abs_start.dropna().dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ").to_numpy(dtype=str)
+        else:
+            if dq_path is None:
+                raise FileNotFoundError("Psychopy: dataqueue path not available")
+            timeline = DataqueueIndex.from_path(dq_path)
+            queue_all = timeline.queue_series()
+            queue_start = float(queue_all.iloc[0])
+            device_slice = timeline.slice(self.dataqueue_device_match)
+            abs_start = device_slice.packet_absolute().to_numpy(dtype=str)
+            pulses = pd.to_numeric(device_slice.rows.get(self.dataqueue_elapsed_column), errors="coerce")
+        # if self.dataqueue_payload_column in device_slice.rows.columns:
+        #     payload = pd.to_numeric(device_slice.rows[self.dataqueue_payload_column], errors="coerce")
+        #     pulses = pulses.loc[payload == self.dataqueue_payload_value]
+        pulses = pulses.dropna()
+        if pulses.empty:
+            raise ValueError(f"Dataqueue timeline exists {timeline.source_path} but contains no nidaq payload==1 pulses.")
+
+        start_rel = float(pulses.iloc[0]) - 0
+        meta = {
+            "dataqueue_file": str(dq_path) if dq_path is not None else None,
+            "dataqueue_pulses": int(pulses.size),
+            "dataqueue_queue_start": queue_start,
+            "abs_start_time": abs_start[0] if len(abs_start) else None,
+        }
+        return start_rel, meta
+
+    def _apply_experiment_window(
+        self,
+        intervals: pd.DataFrame,
+        context: LoadContext,
+        dq_start: float | None,
+    ) -> pd.DataFrame:
+        if context.experiment_window is None or dq_start is None:
+            return intervals
+        start, stop = context.experiment_window
+        window_start = float(start) - float(dq_start)
+        window_stop = float(stop) - float(dq_start)
+
+        intervals = intervals.copy()
+        intervals["start_s"] = intervals["start_s"].clip(lower=window_start, upper=window_stop)
+        intervals["stop_s"] = intervals["stop_s"].clip(lower=window_start, upper=window_stop)
+        mask = (intervals["stop_s"] >= window_start) & (intervals["start_s"] <= window_stop)
+        return intervals.loc[mask].reset_index(drop=True)
+
+    def _align_time(self, values: np.ndarray, key_rt: float, offset: Optional[float]) -> np.ndarray:
+        aligned = values.astype(np.float64, copy=True) - float(key_rt)
+        if offset is not None:
+            aligned = aligned - float(offset)
+        return aligned
+
+    def _align_table(self, frame: pd.DataFrame, key_rt: float, offset: Optional[float]) -> pd.DataFrame:
+        aligned = frame.copy()
+        for col in aligned.columns:
+            name = str(col)
+            if not (name.lower().endswith(self.time_suffixes) or name.startswith("window_")):
+                continue
+            series = pd.to_numeric(aligned[col], errors="coerce")
+            aligned[col] = self._align_time(series.to_numpy(dtype=np.float64), key_rt, offset)
+        return aligned.dropna(axis=1, how="all")
+
+    def _build_window_list(self, frame: pd.DataFrame, start_col: str, stop_col: str) -> list[tuple[float, float]]:
+        if start_col not in frame.columns or stop_col not in frame.columns:
+            return []
+        starts = pd.to_numeric(frame[start_col], errors="coerce").to_numpy(dtype=np.float64, copy=False)
+        stops = pd.to_numeric(frame[stop_col], errors="coerce").to_numpy(dtype=np.float64, copy=False)
+        return [
+            (float(start), float(stop))
+            for start, stop in zip(starts, stops)
+            if np.isfinite(start) and np.isfinite(stop) and stop > start
+        ]
+
+    def _extract_gratings(self, frame: pd.DataFrame) -> pd.DataFrame:
+        selected = self._select_columns(frame, list(self.gratings_columns))
+        result = selected.loc[selected.get("trials.thisN").notna()].copy()
+        if result.empty:
+            raise ValueError("Psychopy gratings trials are empty.")
+
+        gray_start = self._coalesce_column(result, ["stim_grayScreen.started"])
+        gray_stop = self._coalesce_column(result, ["stim_grating.started"])
+        grating_start = self._coalesce_column(result, ["stim_grating.started"])
+        grating_stop = self._coalesce_column(result, ["display_gratings.stopped"])
+
+        result["window_grating_start"] = pd.to_numeric(grating_start, errors="coerce")
+        result["window_grating_stop"] = pd.to_numeric(grating_stop, errors="coerce")
+        result["window_gray_start"] = pd.to_numeric(gray_start, errors="coerce")
+        result["window_gray_stop"] = pd.to_numeric(gray_stop, errors="coerce")
+
+        return result
+
+    def _extract_movies(self, frame: pd.DataFrame) -> pd.DataFrame:
+        return self._select_columns(frame, list(self.movies_columns))
+
+