TwoPy 0.1.0__py3-none-any.whl

twopy/analysis/response_processing/__init__.py

@@ -0,0 +1,55 @@
+"""Core response post-processing helpers.
+
+Inputs: ROI dF/F traces, grouped trial responses, and explicit processing
+options.
+Outputs: processed response objects plus optional quality-control scores.
+
+This package is GUI-independent. It owns the math and typed option contracts
+for response smoothing, low-pass filtering, and correlation-based response
+quality control. Workflow code decides when to call it; persistence code decides
+how to save the resulting audit trail.
+"""
+
+from twopy.analysis.response_processing.apply import (
+    ResponseProcessingResult,
+    process_grouped_roi_responses,
+    process_roi_delta_f_over_f,
+)
+from twopy.analysis.response_processing.options import (
+    CorrelationFilterOptions,
+    CorrelationFilterReference,
+    CorrelationWindowSeconds,
+    LowPassFilterMethod,
+    LowPassFilterOptions,
+    ResponseProcessingOptions,
+    SmoothingMethod,
+    SmoothingOptions,
+    validate_response_processing_options,
+)
+from twopy.analysis.response_processing.qc import (
+    RoiCorrelationScores,
+    score_roi_correlations,
+)
+from twopy.analysis.response_processing.signals import (
+    butterworth_low_pass,
+    nan_aware_moving_average,
+)
+
+__all__ = [
+    "butterworth_low_pass",
+    "CorrelationFilterOptions",
+    "CorrelationFilterReference",
+    "CorrelationWindowSeconds",
+    "LowPassFilterMethod",
+    "LowPassFilterOptions",
+    "nan_aware_moving_average",
+    "process_grouped_roi_responses",
+    "process_roi_delta_f_over_f",
+    "ResponseProcessingOptions",
+    "ResponseProcessingResult",
+    "RoiCorrelationScores",
+    "score_roi_correlations",
+    "SmoothingMethod",
+    "SmoothingOptions",
+    "validate_response_processing_options",
+]

twopy/analysis/response_processing/apply.py

@@ -0,0 +1,298 @@
+"""Apply response post-processing options to core analysis objects.
+
+Inputs: ROI dF/F traces, grouped trial responses, and typed processing options.
+Outputs: new analysis objects with processed values plus optional QC scores.
+
+This module coordinates the pure signal kernels and QC scoring helpers. It does
+not know how options were selected, where outputs are saved, or how plots are
+drawn.
+"""
+
+from dataclasses import dataclass
+
+import numpy as np
+import numpy.typing as npt
+
+from twopy.analysis.dff import RoiDeltaFOverF
+from twopy.analysis.response_processing.options import (
+    ResponseProcessingOptions,
+    validate_response_processing_options,
+)
+from twopy.analysis.response_processing.qc import (
+    RoiCorrelationScores,
+    score_roi_correlations,
+)
+from twopy.analysis.response_processing.signals import (
+    butterworth_low_pass,
+    nan_aware_moving_average,
+)
+from twopy.analysis.responses import GroupedRoiResponses, RoiResponseTrial
+
+__all__ = [
+    "ResponseProcessingResult",
+    "process_grouped_roi_responses",
+    "process_roi_delta_f_over_f",
+]
+
+
+@dataclass(frozen=True)
+class ResponseProcessingResult:
+    """Processed grouped responses plus optional QC scores.
+
+    Inputs: grouped trial responses and one processing option object.
+    Outputs: grouped responses with derived values, the options used to create
+    them, and optional ROI correlation scores.
+    """
+
+    grouped_responses: GroupedRoiResponses
+    options: ResponseProcessingOptions
+    correlation_scores: RoiCorrelationScores | None
+
+
+def process_roi_delta_f_over_f(
+    dff: RoiDeltaFOverF,
+    *,
+    options: ResponseProcessingOptions,
+    data_rate_hz: float,
+) -> RoiDeltaFOverF:
+    """Apply smoothing and low-pass filtering to continuous ROI dF/F.
+
+    Args:
+        dff: Continuous ROI dF/F result.
+        options: Processing options. Correlation filtering is ignored here
+            because it needs grouped repeated-trial responses.
+        data_rate_hz: Sampling rate in hertz.
+
+    Returns:
+        New ``RoiDeltaFOverF`` with processed ``values`` and original raw audit
+        arrays preserved.
+
+    Raises:
+        ValueError: If options or dF/F shapes are invalid.
+
+    This is the preferred stage for smoothing and low-pass filtering because it
+    operates on continuous traces before trial slicing, avoiding per-trial edge
+    artifacts.
+    """
+    _validate_dff_for_processing(dff)
+    validate_response_processing_options(options, data_rate_hz=data_rate_hz)
+    processed_values = _process_values(
+        dff.values,
+        options=options,
+        data_rate_hz=data_rate_hz,
+    )
+    metadata = dict(dff.metadata)
+    metadata.update(_processing_metadata(options))
+    return RoiDeltaFOverF(
+        fluorescence=dff.fluorescence,
+        baseline=dff.baseline,
+        values=processed_values,
+        labels=dff.labels,
+        start_frame=dff.start_frame,
+        stop_frame=dff.stop_frame,
+        tau=dff.tau,
+        amplitudes=dff.amplitudes,
+        interleave_frame_numbers=dff.interleave_frame_numbers,
+        interleave_fluorescence=dff.interleave_fluorescence,
+        metadata=metadata,
+    )
+
+
+def process_grouped_roi_responses(
+    grouped: GroupedRoiResponses,
+    *,
+    options: ResponseProcessingOptions,
+) -> ResponseProcessingResult:
+    """Apply response post-processing to grouped trial responses.
+
+    Args:
+        grouped: Grouped ROI responses.
+        options: Processing options.
+
+    Returns:
+        Processed grouped responses plus optional correlation scores.
+
+    Raises:
+        ValueError: If options or grouped response shapes are invalid.
+
+    This grouped-response path is useful for saved responses or plot-only
+    previews. New full workflow wiring should prefer continuous dF/F processing
+    before grouping when smoothing or low-pass filtering are enabled.
+    """
+    validate_response_processing_options(options, data_rate_hz=grouped.data_rate_hz)
+    _validate_grouped_for_processing(grouped)
+    processed = _copy_grouped_with_processed_values(grouped, options=options)
+    correlation_scores = None
+    if options.correlation_filter.reference in {"epoch_mean", "epoch_peak"}:
+        correlation_scores = score_roi_correlations(
+            processed,
+            minimum_correlation=options.correlation_filter.minimum_correlation,
+            reference=options.correlation_filter.reference,
+            window_seconds=options.correlation_filter.window_seconds,
+        )
+        processed = _mask_excluded_rois(
+            processed,
+            included_mask=correlation_scores.included_mask,
+        )
+    return ResponseProcessingResult(
+        grouped_responses=processed,
+        options=options,
+        correlation_scores=correlation_scores,
+    )
+
+
+def _process_values(
+    values: npt.NDArray[np.float64],
+    *,
+    options: ResponseProcessingOptions,
+    data_rate_hz: float,
+) -> npt.NDArray[np.float64]:
+    """Apply enabled signal transforms to frame-by-ROI values."""
+    processed = values.astype(np.float64, copy=True)
+    if options.smoothing.method == "moving_average":
+        processed = nan_aware_moving_average(
+            processed,
+            window_frames=options.smoothing.window_frames,
+        )
+    if options.low_pass.method == "butterworth":
+        if options.low_pass.cutoff_hz is None:
+            msg = "enabled low-pass filtering requires cutoff_hz"
+            raise ValueError(msg)
+        processed = butterworth_low_pass(
+            processed,
+            sample_rate_hz=data_rate_hz,
+            cutoff_hz=options.low_pass.cutoff_hz,
+            order=options.low_pass.order,
+        )
+    return processed
+
+
+def _copy_grouped_with_processed_values(
+    grouped: GroupedRoiResponses,
+    *,
+    options: ResponseProcessingOptions,
+) -> GroupedRoiResponses:
+    """Return grouped responses with each trial value matrix processed."""
+    trials = tuple(
+        _copy_trial_with_values(
+            trial,
+            _process_values(
+                trial.values,
+                options=options,
+                data_rate_hz=grouped.data_rate_hz,
+            ),
+        )
+        for trial in grouped.trials
+    )
+    return GroupedRoiResponses(
+        roi_labels=grouped.roi_labels,
+        data_rate_hz=grouped.data_rate_hz,
+        trials=trials,
+    )
+
+
+def _mask_excluded_rois(
+    grouped: GroupedRoiResponses,
+    *,
+    included_mask: npt.NDArray[np.bool_],
+) -> GroupedRoiResponses:
+    """Return grouped responses with excluded ROI columns set to NaN."""
+    if included_mask.shape != (len(grouped.roi_labels),):
+        msg = (
+            "included_mask must match ROI label count; "
+            f"got {included_mask.shape} for {len(grouped.roi_labels)} labels"
+        )
+        raise ValueError(msg)
+    trials: list[RoiResponseTrial] = []
+    for trial in grouped.trials:
+        values = trial.values.copy()
+        values[:, ~included_mask] = np.nan
+        trials.append(_copy_trial_with_values(trial, values))
+    return GroupedRoiResponses(
+        roi_labels=grouped.roi_labels,
+        data_rate_hz=grouped.data_rate_hz,
+        trials=tuple(trials),
+    )
+
+
+def _copy_trial_with_values(
+    trial: RoiResponseTrial,
+    values: npt.NDArray[np.float64],
+) -> RoiResponseTrial:
+    """Return one trial response with replacement values."""
+    return RoiResponseTrial(
+        epoch_number=trial.epoch_number,
+        epoch_name=trial.epoch_name,
+        trial_index=trial.trial_index,
+        window_index=trial.window_index,
+        start_frame=trial.start_frame,
+        stop_frame=trial.stop_frame,
+        frame_numbers=trial.frame_numbers,
+        time_seconds=trial.time_seconds,
+        values=values,
+    )
+
+
+def _validate_dff_for_processing(dff: RoiDeltaFOverF) -> None:
+    """Validate a dF/F object before value-only processing."""
+    if dff.values.ndim != 2:
+        msg = f"dF/F values must have shape (frames, rois); got {dff.values.shape}"
+        raise ValueError(msg)
+    if dff.values.shape[1] != len(dff.labels):
+        msg = (
+            "dF/F values width does not match labels: "
+            f"{dff.values.shape[1]} values, {len(dff.labels)} labels"
+        )
+        raise ValueError(msg)
+
+
+def _validate_grouped_for_processing(grouped: GroupedRoiResponses) -> None:
+    """Validate grouped responses before value-only processing."""
+    if grouped.data_rate_hz <= 0:
+        msg = f"data_rate_hz must be positive; got {grouped.data_rate_hz}"
+        raise ValueError(msg)
+    for trial in grouped.trials:
+        if trial.values.ndim != 2:
+            msg = (
+                f"trial values must have shape (frames, rois); got {trial.values.shape}"
+            )
+            raise ValueError(msg)
+        if trial.values.shape[1] != len(grouped.roi_labels):
+            msg = (
+                "trial response width does not match ROI labels: "
+                f"{trial.values.shape[1]} values, {len(grouped.roi_labels)} labels"
+            )
+            raise ValueError(msg)
+
+
+def _processing_metadata(
+    options: ResponseProcessingOptions,
+) -> dict[str, str | int | float | bool]:
+    """Return flat metadata for a processed dF/F result."""
+    return {
+        "response_processing_smoothing_method": options.smoothing.method,
+        "response_processing_smoothing_window_frames": options.smoothing.window_frames,
+        "response_processing_low_pass_method": options.low_pass.method,
+        "response_processing_low_pass_cutoff_hz": (
+            "none"
+            if options.low_pass.cutoff_hz is None
+            else float(options.low_pass.cutoff_hz)
+        ),
+        "response_processing_low_pass_order": options.low_pass.order,
+        "response_processing_correlation_reference": (
+            options.correlation_filter.reference
+        ),
+        "response_processing_minimum_correlation": (
+            options.correlation_filter.minimum_correlation
+        ),
+        "response_processing_correlation_window_start_seconds": (
+            "none"
+            if options.correlation_filter.window_seconds[0] is None
+            else float(options.correlation_filter.window_seconds[0])
+        ),
+        "response_processing_correlation_window_stop_seconds": (
+            "none"
+            if options.correlation_filter.window_seconds[1] is None
+            else float(options.correlation_filter.window_seconds[1])
+        ),
+    }

twopy/analysis/response_processing/options.py

@@ -0,0 +1,192 @@
+"""Typed options for response post-processing.
+
+Inputs: user or script-selected processing parameters.
+Outputs: immutable option objects validated before math is run.
+
+These objects are intentionally plain dataclasses so GUI controls, scripts, and
+future persistence code can all pass the same explicit contract without pulling
+Qt or HDF5 concerns into the analysis layer.
+"""
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+import numpy as np
+
+__all__ = [
+    "CorrelationFilterOptions",
+    "CorrelationFilterReference",
+    "CorrelationWindowSeconds",
+    "LowPassFilterMethod",
+    "LowPassFilterOptions",
+    "ResponseProcessingOptions",
+    "SmoothingMethod",
+    "SmoothingOptions",
+    "validate_response_processing_options",
+]
+
+SmoothingMethod = Literal["none", "moving_average"]
+LowPassFilterMethod = Literal["none", "butterworth"]
+CorrelationFilterReference = Literal["none", "epoch_mean", "epoch_peak"]
+CorrelationWindowSeconds = tuple[float | None, float | None]
+
+
+@dataclass(frozen=True)
+class SmoothingOptions:
+    """Moving-average smoothing settings.
+
+    Inputs: a smoothing method and window length in imaging frames.
+    Outputs: a typed contract consumed by signal-processing kernels.
+
+    ``method="none"`` leaves values unchanged. ``window_frames`` is still kept
+    explicit so saved options can report exactly what the user selected.
+    """
+
+    method: SmoothingMethod = "none"
+    window_frames: int = 1
+
+
+@dataclass(frozen=True)
+class LowPassFilterOptions:
+    """Low-pass filter settings for ROI response traces.
+
+    Inputs: filter method, cutoff frequency, and Butterworth order.
+    Outputs: a typed contract consumed by low-pass kernels.
+
+    ``cutoff_hz`` is optional because disabled filters do not need a cutoff.
+    Enabled Butterworth filters must provide a positive cutoff below Nyquist.
+    """
+
+    method: LowPassFilterMethod = "none"
+    cutoff_hz: float | None = None
+    order: int = 2
+
+
+@dataclass(frozen=True)
+class CorrelationFilterOptions:
+    """Correlation-based response quality-control settings.
+
+    Inputs: reference strategy, accepted correlation, and relative time window.
+    Outputs: ROI-level scores and inclusion masks when enabled.
+
+    Correlation filtering should produce auditable scores and masks. Downstream
+    callers can hide, mark, or exclude responses, but raw data should remain
+    available elsewhere in the analysis record.
+
+    ``window_seconds`` is a half-open ``(start, stop)`` interval on each trial's
+    epoch-relative time axis. ``None`` on the stop side leaves it unbounded, so
+    the default starts at epoch onset and uses the full stored epoch response.
+    """
+
+    reference: CorrelationFilterReference = "none"
+    minimum_correlation: float = 0.0
+    window_seconds: CorrelationWindowSeconds = (0.0, None)
+
+
+@dataclass(frozen=True)
+class ResponseProcessingOptions:
+    """Complete response post-processing configuration.
+
+    Inputs: optional smoothing, low-pass filtering, and correlation QC settings.
+    Outputs: one immutable object passed through workflow, plotting, and future
+    persistence boundaries.
+    """
+
+    smoothing: SmoothingOptions = field(default_factory=SmoothingOptions)
+    low_pass: LowPassFilterOptions = field(default_factory=LowPassFilterOptions)
+    correlation_filter: CorrelationFilterOptions = field(
+        default_factory=CorrelationFilterOptions
+    )
+
+
+def validate_response_processing_options(
+    options: ResponseProcessingOptions,
+    *,
+    data_rate_hz: float | None = None,
+) -> None:
+    """Validate one response-processing option object.
+
+    Args:
+        options: Candidate processing configuration.
+        data_rate_hz: Optional sampling rate used to validate low-pass Nyquist
+            bounds when the filter is enabled.
+
+    Returns:
+        None.
+
+    Raises:
+        ValueError: If any enabled option is invalid.
+    """
+    _validate_smoothing_options(options.smoothing)
+    _validate_low_pass_options(options.low_pass, data_rate_hz=data_rate_hz)
+    _validate_correlation_options(options.correlation_filter)
+
+
+def _validate_smoothing_options(options: SmoothingOptions) -> None:
+    """Validate smoothing options."""
+    if options.method not in {"none", "moving_average"}:
+        msg = f"Unknown smoothing method {options.method!r}"
+        raise ValueError(msg)
+    if options.window_frames < 1:
+        msg = f"smoothing window_frames must be at least 1; got {options.window_frames}"
+        raise ValueError(msg)
+
+
+def _validate_low_pass_options(
+    options: LowPassFilterOptions,
+    *,
+    data_rate_hz: float | None,
+) -> None:
+    """Validate low-pass filter options."""
+    if options.method not in {"none", "butterworth"}:
+        msg = f"Unknown low-pass method {options.method!r}"
+        raise ValueError(msg)
+    if options.order < 1:
+        msg = f"low-pass order must be at least 1; got {options.order}"
+        raise ValueError(msg)
+    if options.method == "none":
+        return
+    if options.cutoff_hz is None or options.cutoff_hz <= 0:
+        msg = "enabled low-pass filtering requires a positive cutoff_hz"
+        raise ValueError(msg)
+    if data_rate_hz is not None:
+        if data_rate_hz <= 0:
+            msg = f"data_rate_hz must be positive; got {data_rate_hz}"
+            raise ValueError(msg)
+        nyquist_hz = data_rate_hz / 2.0
+        if options.cutoff_hz >= nyquist_hz:
+            msg = (
+                "low-pass cutoff_hz must be below Nyquist; "
+                f"got {options.cutoff_hz} Hz for {data_rate_hz} Hz data"
+            )
+            raise ValueError(msg)
+
+
+def _validate_correlation_options(options: CorrelationFilterOptions) -> None:
+    """Validate correlation-filter options."""
+    if options.reference not in {"none", "epoch_mean", "epoch_peak"}:
+        msg = f"Unknown correlation reference {options.reference!r}"
+        raise ValueError(msg)
+    if not -1.0 <= options.minimum_correlation <= 1.0:
+        msg = (
+            "minimum_correlation must be between -1 and 1; "
+            f"got {options.minimum_correlation}"
+        )
+        raise ValueError(msg)
+    start_seconds, stop_seconds = options.window_seconds
+    if start_seconds is not None and not np.isfinite(start_seconds):
+        msg = f"correlation window start must be finite or None; got {start_seconds}"
+        raise ValueError(msg)
+    if stop_seconds is not None and not np.isfinite(stop_seconds):
+        msg = f"correlation window stop must be finite or None; got {stop_seconds}"
+        raise ValueError(msg)
+    if (
+        start_seconds is not None
+        and stop_seconds is not None
+        and start_seconds >= stop_seconds
+    ):
+        msg = (
+            "correlation window start must be before stop; "
+            f"got ({start_seconds}, {stop_seconds})"
+        )
+        raise ValueError(msg)