TwoPy 0.1.0__py3-none-any.whl

twopy/config.py

@@ -0,0 +1,200 @@
+"""Load twopy configuration from a small YAML file.
+
+Inputs: a YAML config file with lab paths and database access mode.
+Outputs: a typed ``TwopyConfig`` object with normalized values.
+
+The config layer keeps machine-specific paths out of analysis code and makes the
+GUI easy to point at the right database and recording roots.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal, cast
+
+import yaml
+
+from twopy.database.types import DatabaseAccess
+
+__all__ = [
+    "DEFAULT_CONFIG_PATH",
+    "AnalysisOutputMode",
+    "TwopyConfig",
+    "load_config",
+    "resolve_analysis_output_dir",
+]
+
+DEFAULT_CONFIG_PATH = Path("config.yml")
+AnalysisOutputMode = Path | Literal["source"]
+
+
+@dataclass(frozen=True)
+class TwopyConfig:
+    """Machine-local paths used by twopy.
+
+    Inputs: lab paths, DB access mode, and analysis output from ``config.yml``.
+    Outputs: immutable config values that GUI and analysis code can pass around.
+
+    The paths are not required to exist during parsing so tests and machines
+    without mounted lab drives can still load and inspect configuration.
+    """
+
+    database_path: Path
+    data_path: Path
+    database_access: DatabaseAccess
+    analysis_output: AnalysisOutputMode
+
+
+def load_config(path: Path = DEFAULT_CONFIG_PATH) -> TwopyConfig:
+    """Load twopy configuration from YAML.
+
+    Args:
+        path: YAML config file path. Defaults to ``config.yml`` in the current
+            working directory.
+
+    Returns:
+        A typed configuration object with paths, DB access, and output mode.
+
+    Raises:
+        FileNotFoundError: If the config file is absent.
+        ValueError: If the YAML is not a mapping or required keys are missing.
+
+    YAML is used because the file is edited by people and benefits from comments.
+    """
+    config_path = path.expanduser()
+    if not config_path.is_file():
+        msg = f"Missing twopy config file: {config_path}"
+        raise FileNotFoundError(msg)
+
+    with config_path.open("r", encoding="utf-8") as config_file:
+        loaded: object = yaml.safe_load(config_file)
+
+    if not isinstance(loaded, dict):
+        msg = f"twopy config must be a YAML mapping: {config_path}"
+        raise ValueError(msg)
+
+    raw_config = cast(dict[object, object], loaded)
+
+    return TwopyConfig(
+        database_path=_required_path(raw_config, "database_path", config_path),
+        data_path=_required_path(raw_config, "data_path", config_path),
+        database_access=_database_access(raw_config, config_path),
+        analysis_output=_analysis_output(raw_config, config_path),
+    )
+
+
+def resolve_analysis_output_dir(config: TwopyConfig, recording_dir: Path) -> Path:
+    """Resolve the output directory for one recording.
+
+    Args:
+        config: Loaded twopy configuration.
+        recording_dir: Recording folder being analyzed.
+
+    Returns:
+        Directory where twopy should write analysis output for the recording.
+
+    Raises:
+        ValueError: If ``analysis_output`` is a root path and the recording is
+            not inside ``config.data_path``.
+
+    ``analysis_output: source`` writes into ``recording_dir/twopy``. A path
+    value mirrors the recording's path relative to ``data_path`` under that
+    output root.
+    """
+    source_dir = recording_dir.expanduser()
+    if config.analysis_output == "source":
+        return source_dir / "twopy"
+
+    data_root = config.data_path.expanduser()
+    try:
+        relative_recording = source_dir.relative_to(data_root)
+    except ValueError as error:
+        msg = (
+            f"Recording {source_dir} is not inside configured data_path "
+            f"{data_root}; cannot mirror analysis output directory"
+        )
+        raise ValueError(msg) from error
+
+    return config.analysis_output / relative_recording
+
+
+def _required_path(config: dict[object, object], key: str, config_path: Path) -> Path:
+    """Read one required path value from parsed YAML.
+
+    Args:
+        config: Parsed YAML mapping.
+        key: Required key to read.
+        config_path: Source config file path, used for clear errors.
+
+    Returns:
+        Expanded ``Path`` value for the requested key.
+
+    Raises:
+        ValueError: If the key is missing or is not a string.
+
+    This helper keeps config validation explicit and avoids hidden defaults.
+    """
+    value = config.get(key)
+    if not isinstance(value, str) or value == "":
+        msg = f"twopy config key {key!r} must be a non-empty string: {config_path}"
+        raise ValueError(msg)
+
+    return Path(value).expanduser()
+
+
+def _database_access(
+    config: dict[object, object],
+    config_path: Path,
+) -> DatabaseAccess:
+    """Read the optional database access mode from parsed YAML.
+
+    Args:
+        config: Parsed YAML mapping.
+        config_path: Source config file path, used for clear errors.
+
+    Returns:
+        ``copy`` or ``direct``. Missing values default to ``copy``.
+
+    The default favors local DB copies because database queries over the network
+    can be slow, but transferring the DB file locally is usually fast. twopy
+    verifies whether a copy is stale before copying again.
+    """
+    value = config.get("database_access", "copy")
+    if value in {"copy", "direct"}:
+        return cast(DatabaseAccess, value)
+
+    msg = (
+        f"twopy config key 'database_access' must be 'copy' or 'direct': {config_path}"
+    )
+    raise ValueError(msg)
+
+
+def _analysis_output(
+    config: dict[object, object],
+    config_path: Path,
+) -> AnalysisOutputMode:
+    """Read the required analysis output setting from parsed YAML.
+
+    Args:
+        config: Parsed YAML mapping.
+        config_path: Source config file path, used for clear errors.
+
+    Returns:
+        ``source`` or an expanded output root path.
+
+    Raises:
+        ValueError: If the key is absent or not a non-empty string.
+
+    Keeping this explicit avoids accidental writes to unclear locations.
+    """
+    value = config.get("analysis_output")
+    if not isinstance(value, str) or value == "":
+        msg = (
+            "twopy config key 'analysis_output' must be 'source' or a "
+            f"non-empty path string: {config_path}"
+        )
+        raise ValueError(msg)
+
+    if value == "source":
+        return "source"
+
+    return Path(value).expanduser()

twopy/conversion/__init__.py

@@ -0,0 +1,40 @@
+"""Convert microscope source files into twopy-owned HDF5 files.
+
+Inputs: one validated two-photon recording session directory.
+Outputs: HDF5 files containing twopy-owned datasets for later analysis.
+
+This package keeps the public conversion API small while separating source
+loading, HDF5 writing, and data-object definitions into focused modules.
+"""
+
+from twopy.conversion.api import convert_recording_to_twopy
+from twopy.conversion.source_loading import load_source_conversion_inputs
+from twopy.conversion.types import (
+    AcquisitionMetadata,
+    AlignedMovieSource,
+    AlignmentCropAudit,
+    ConvertedRecording,
+    FrameCountAudit,
+    PhotodiodeSignals,
+    RunMetadata,
+    SourceConversionInputs,
+    StimulusCodeMetadata,
+    StimulusData,
+    StimulusParameters,
+)
+
+__all__ = [
+    "AcquisitionMetadata",
+    "AlignmentCropAudit",
+    "AlignedMovieSource",
+    "ConvertedRecording",
+    "FrameCountAudit",
+    "PhotodiodeSignals",
+    "RunMetadata",
+    "SourceConversionInputs",
+    "StimulusCodeMetadata",
+    "StimulusParameters",
+    "StimulusData",
+    "convert_recording_to_twopy",
+    "load_source_conversion_inputs",
+]

twopy/conversion/alignment_crop.py

@@ -0,0 +1,329 @@
+"""Compute alignment-valid spatial crops during source conversion.
+
+Inputs: alignment text offsets, acquisition metadata, movie shape, and
+photodiode signals.
+Outputs: a typed crop audit object saved into converted HDF5 metadata.
+"""
+
+from pathlib import Path
+
+import numpy as np
+import numpy.typing as npt
+
+from twopy.conversion.types import (
+    AcquisitionMetadata,
+    AlignedMovieSource,
+    AlignmentCropAudit,
+    PhotodiodeSignals,
+)
+from twopy.photodiode import stimulus_frame_bounds_from_photodiode
+from twopy.spatial import SpatialCrop
+
+__all__ = ["load_alignment_valid_crop"]
+
+ONE_X_PIXELS_PER_MICRON_AT_256_X = 0.4
+DEFAULT_MOTION_THRESHOLD_PERCENT = 150.0
+DEFAULT_PLANE_MOTION_LIMIT_MICRONS = 5.0
+DEFAULT_LINESCAN_MOTION_LIMIT_MICRONS = 15.0
+
+
+def load_alignment_valid_crop(
+    *,
+    alignment_path: Path,
+    aligned_movie: AlignedMovieSource,
+    acquisition: AcquisitionMetadata,
+    photodiode: PhotodiodeSignals,
+) -> AlignmentCropAudit:
+    """Compute the alignment-valid crop used by crop-domain analysis.
+
+    Args:
+        alignment_path: Text file with per-frame alignment offsets.
+        aligned_movie: Aligned movie source metadata.
+        acquisition: Acquisition metadata with zoom and frame-rate fields.
+        photodiode: Photodiode vectors used to find stimulus-bounded frames.
+
+    Returns:
+        Crop bounds plus audit values.
+
+    The source analysis path estimates this crop from alignment rows inside the
+    stimulus presentation window, then applies background correction inside that
+    crop. twopy saves the same contract while keeping the converted movie
+    full-frame.
+    """
+    alignment = np.asarray(np.loadtxt(alignment_path), dtype=np.float64)
+    if alignment.ndim != 2 or alignment.shape[1] < 2:
+        msg = f"Alignment text must be a 2D table with x/y offsets: {alignment_path}"
+        raise ValueError(msg)
+
+    frame_start, frame_stop = _stimulus_alignment_frame_range(
+        aligned_movie=aligned_movie,
+        acquisition=acquisition,
+        photodiode=photodiode,
+    )
+    if frame_stop > alignment.shape[0]:
+        msg = (
+            "Stimulus-bounded alignment range exceeds alignment rows: "
+            f"[{frame_start}, {frame_stop}) for {alignment.shape[0]} rows"
+        )
+        raise ValueError(msg)
+
+    alignment_window = alignment[frame_start:frame_stop, :]
+    line_scan = _is_line_scan(acquisition)
+    motion_threshold_pixels = _motion_threshold_pixels(
+        acquisition=acquisition,
+        spatial_shape=_movie_spatial_shape(aligned_movie),
+        line_scan=line_scan,
+    )
+    alignment_shift_pixels = _alignment_shift_pixels(
+        alignment[: aligned_movie.shape[0], :],
+        line_scan=line_scan,
+    )
+    motion_artifact_mask = alignment_shift_pixels > motion_threshold_pixels
+    kept_alignment, over_moved_count = _remove_over_moved_alignment_rows(
+        alignment_window,
+        line_scan=line_scan,
+        motion_threshold_pixels=motion_threshold_pixels,
+    )
+    crop, x_cutoff, y_cutoff = _alignment_crop_from_offsets(
+        kept_alignment,
+        spatial_shape=_movie_spatial_shape(aligned_movie),
+        line_scan=line_scan,
+    )
+
+    return AlignmentCropAudit(
+        crop=crop,
+        alignment_frame_start=frame_start,
+        alignment_frame_stop=frame_stop,
+        x_cutoff_pixels=x_cutoff,
+        y_cutoff_pixels=y_cutoff,
+        over_moved_frame_count=over_moved_count,
+        motion_threshold_pixels=motion_threshold_pixels,
+        alignment_shift_pixels=alignment_shift_pixels,
+        motion_artifact_mask=motion_artifact_mask,
+    )
+
+
+def _stimulus_alignment_frame_range(
+    *,
+    aligned_movie: AlignedMovieSource,
+    acquisition: AcquisitionMetadata,
+    photodiode: PhotodiodeSignals,
+) -> tuple[int, int]:
+    """Find the alignment rows that cover stimulus presentation.
+
+    Args:
+        aligned_movie: Movie metadata that gives frame count.
+        acquisition: Acquisition metadata with frame rate and line-scan flag.
+        photodiode: High-resolution photodiode signal.
+
+    Returns:
+        Half-open frame range used for the alignment-valid crop.
+    """
+    if _is_line_scan(acquisition):
+        return (0, aligned_movie.shape[0])
+
+    return stimulus_frame_bounds_from_photodiode(
+        photodiode.high_res_pd,
+        movie_frame_count=aligned_movie.shape[0],
+        frame_rate_hz=_required_float_field(acquisition, "acq.frameRate"),
+    )
+
+
+def _motion_threshold_pixels(
+    *,
+    acquisition: AcquisitionMetadata,
+    spatial_shape: tuple[int, int],
+    line_scan: bool,
+) -> float:
+    """Convert the motion rejection threshold from microns to pixels.
+
+    Args:
+        acquisition: Acquisition metadata with zoom.
+        spatial_shape: Full movie spatial shape in twopy array axes.
+        line_scan: Whether this recording is a line scan.
+
+    Returns:
+        Maximum allowed alignment movement in pixels.
+    """
+    micron_limit = (
+        DEFAULT_LINESCAN_MOTION_LIMIT_MICRONS
+        if line_scan
+        else DEFAULT_PLANE_MOTION_LIMIT_MICRONS
+    )
+    zoom = _required_float_field(acquisition, "acq.zoomFactor")
+    x_axis_conversion = 256.0 / float(spatial_shape[0])
+    return micron_limit * ONE_X_PIXELS_PER_MICRON_AT_256_X * zoom * x_axis_conversion
+
+
+def _remove_over_moved_alignment_rows(
+    alignment: npt.NDArray[np.float64],
+    *,
+    line_scan: bool,
+    motion_threshold_pixels: float,
+) -> tuple[npt.NDArray[np.float64], int]:
+    """Drop alignment rows with motion beyond the allowed threshold.
+
+    Args:
+        alignment: Stimulus-bounded alignment table.
+        line_scan: Whether this recording is a line scan.
+        motion_threshold_pixels: Maximum allowed movement in pixels.
+
+    Returns:
+        Filtered alignment rows and the number of dropped rows.
+    """
+    full_shift = _alignment_shift_pixels(alignment, line_scan=line_scan)
+    over_moved = full_shift > motion_threshold_pixels
+    over_moved_percent = (
+        100.0 * float(np.count_nonzero(over_moved)) / alignment.shape[0]
+    )
+    if over_moved_percent > DEFAULT_MOTION_THRESHOLD_PERCENT:
+        return alignment, 0
+
+    kept = alignment[~over_moved, :]
+    if kept.size == 0:
+        msg = (
+            "Alignment-valid crop cannot be computed; all alignment rows moved too far"
+        )
+        raise ValueError(msg)
+    return kept, int(np.count_nonzero(over_moved))
+
+
+def _alignment_shift_pixels(
+    alignment: npt.NDArray[np.float64],
+    *,
+    line_scan: bool,
+) -> npt.NDArray[np.float64]:
+    """Return per-frame alignment shift magnitudes in pixels.
+
+    Args:
+        alignment: Alignment table with x/y offset columns.
+        line_scan: Whether this recording is a line scan.
+
+    Returns:
+        One shift magnitude per alignment row.
+    """
+    if line_scan:
+        return np.abs(alignment[:, 0])
+    return np.sqrt(alignment[:, 0] ** 2 + alignment[:, 1] ** 2)
+
+
+def _alignment_crop_from_offsets(
+    alignment: npt.NDArray[np.float64],
+    *,
+    spatial_shape: tuple[int, int],
+    line_scan: bool,
+) -> tuple[SpatialCrop, int, int]:
+    """Convert alignment offsets into twopy movie-axis crop bounds.
+
+    Args:
+        alignment: Filtered alignment table.
+        spatial_shape: Full movie spatial shape in twopy array axes.
+        line_scan: Whether this recording is a line scan.
+
+    Returns:
+        Spatial crop plus x/y cutoff values.
+
+    Source alignment columns are x/y motion offsets. twopy arrays store x along
+    spatial axis 0 and y along spatial axis 1, so the crop is mapped directly to
+    those axes.
+    """
+    x_cutoff = _ceil_abs_offset(alignment[:, 0])
+    y_cutoff = 1 if line_scan else _ceil_abs_offset(alignment[:, 1])
+    axis0_start, axis0_stop = _crop_bounds_from_cutoff(
+        length=spatial_shape[0],
+        cutoff=x_cutoff,
+    )
+    axis1_start, axis1_stop = _crop_bounds_from_cutoff(
+        length=spatial_shape[1],
+        cutoff=y_cutoff,
+    )
+    return (
+        SpatialCrop(
+            axis0_start=axis0_start,
+            axis0_stop=axis0_stop,
+            axis1_start=axis1_start,
+            axis1_stop=axis1_stop,
+            original_shape=spatial_shape,
+            source="alignment_valid_crop",
+        ),
+        x_cutoff,
+        y_cutoff,
+    )
+
+
+def _ceil_abs_offset(values: npt.NDArray[np.float64]) -> int:
+    """Return the ceiling of the largest absolute alignment offset.
+
+    Args:
+        values: One alignment offset column.
+
+    Returns:
+        Nonnegative integer cutoff.
+    """
+    return int(np.ceil(float(np.nanmax(np.abs(values)))))
+
+
+def _crop_bounds_from_cutoff(*, length: int, cutoff: int) -> tuple[int, int]:
+    """Convert a source cutoff value into Python half-open bounds.
+
+    Args:
+        length: Axis length.
+        cutoff: One-based cutoff convention from the source crop code.
+
+    Returns:
+        ``(start, stop)`` bounds.
+    """
+    if cutoff <= 1:
+        return (0, length)
+    start = cutoff - 1
+    stop = length - cutoff + 1
+    if start >= stop:
+        msg = f"Alignment cutoff {cutoff} leaves no valid pixels for length {length}"
+        raise ValueError(msg)
+    return start, stop
+
+
+def _movie_spatial_shape(aligned_movie: AlignedMovieSource) -> tuple[int, int]:
+    """Return the two spatial dimensions of the aligned source movie.
+
+    Args:
+        aligned_movie: Source movie metadata.
+
+    Returns:
+        ``(axis0, axis1)`` spatial shape.
+    """
+    if len(aligned_movie.shape) != 3:
+        msg = f"Aligned movie must be 3D; got {aligned_movie.shape}"
+        raise ValueError(msg)
+    return (aligned_movie.shape[1], aligned_movie.shape[2])
+
+
+def _required_float_field(acquisition: AcquisitionMetadata, key: str) -> float:
+    """Read one required numeric acquisition metadata field.
+
+    Args:
+        acquisition: Loaded acquisition metadata.
+        key: Field name to read.
+
+    Returns:
+        Field value as a float.
+    """
+    value = acquisition.fields[key]
+    if isinstance(value, str | bytes | int | float | np.generic):
+        return float(value)
+
+    value_type = type(value).__name__
+    msg = f"Acquisition field {key!r} must be numeric, got {value_type}"
+    raise ValueError(msg)
+
+
+def _is_line_scan(acquisition: AcquisitionMetadata) -> bool:
+    """Return whether acquisition metadata describes a line scan.
+
+    Args:
+        acquisition: Loaded acquisition metadata.
+
+    Returns:
+        ``True`` for line-scan recordings.
+    """
+    return _required_float_field(acquisition, "acq.scanAngleMultiplierSlow") == 0.0

twopy/conversion/api.py

@@ -0,0 +1,113 @@
+"""Public conversion workflow from microscope source files to twopy files.
+
+Inputs: one source recording folder plus one output folder.
+Outputs: a ``ConvertedRecording`` pointing at twopy-owned HDF5 files.
+
+The workflow reads MATLAB/TIFF-derived source data only during conversion.
+Analysis code should operate on the converted HDF5 files.
+"""
+
+from pathlib import Path
+
+from twopy.config import DEFAULT_CONFIG_PATH, load_config, resolve_analysis_output_dir
+from twopy.conversion.frame_ranges import normalize_frame_range
+from twopy.conversion.hdf5_writing import (
+    CONVERTED_ALIGNED_MOVIE_FILENAME,
+    CONVERTED_RECORDING_FILENAME,
+    write_aligned_movie_file,
+    write_recording_data_file,
+)
+from twopy.conversion.source_loading import load_source_conversion_inputs
+from twopy.conversion.types import ConvertedRecording
+
+__all__ = ["convert_recording_to_twopy"]
+
+
+def convert_recording_to_twopy(
+    session_dir: Path,
+    output_dir: Path | None = None,
+    *,
+    mean_start_frame: int | None = None,
+    mean_stop_frame: int | None = None,
+    config_path: Path = DEFAULT_CONFIG_PATH,
+) -> ConvertedRecording:
+    """Convert one recording folder into twopy HDF5 files.
+
+    Args:
+        session_dir: Source recording folder.
+        output_dir: Optional directory that receives converted twopy HDF5 files.
+            When omitted, twopy uses ``analysis_output`` from config.
+        mean_start_frame: Optional first frame for the mean image.
+        mean_stop_frame: Optional exclusive stop frame for the mean image.
+        config_path: YAML config file used when ``output_dir`` is omitted.
+
+    Returns:
+        Summary of the converted HDF5 files.
+
+    The conversion copies source data into twopy-owned datasets. The aligned
+    movie gets its own file because it dominates size, while the recording data
+    file keeps metadata, stimulus, photodiode, and the quick-look mean image.
+    """
+    inputs = load_source_conversion_inputs(session_dir)
+    start_frame, stop_frame = normalize_frame_range(
+        frame_count=inputs.aligned_movie.shape[0],
+        start_frame=mean_start_frame,
+        stop_frame=mean_stop_frame,
+    )
+    mean_image = inputs.aligned_movie.mean_image(
+        start_frame=start_frame,
+        stop_frame=stop_frame,
+    )
+
+    destination_dir = _resolve_conversion_output_dir(
+        session_dir=inputs.session_files.session_dir,
+        output_dir=output_dir,
+        config_path=config_path,
+    )
+    destination_dir.mkdir(parents=True, exist_ok=True)
+    recording_data_path = destination_dir / CONVERTED_RECORDING_FILENAME
+    movie_path = destination_dir / CONVERTED_ALIGNED_MOVIE_FILENAME
+
+    write_aligned_movie_file(inputs, movie_path)
+    write_recording_data_file(
+        inputs=inputs,
+        output_path=recording_data_path,
+        mean_image=mean_image,
+        mean_start_frame=start_frame,
+        mean_stop_frame=stop_frame,
+    )
+
+    return ConvertedRecording(
+        path=recording_data_path,
+        movie_path=movie_path,
+        source_session_dir=inputs.session_files.session_dir,
+        movie_shape=inputs.aligned_movie.shape,
+        mean_image_start_frame=start_frame,
+        mean_image_stop_frame=stop_frame,
+    )
+
+
+def _resolve_conversion_output_dir(
+    *,
+    session_dir: Path,
+    output_dir: Path | None,
+    config_path: Path,
+) -> Path:
+    """Choose where conversion output should be written.
+
+    Args:
+        session_dir: Source recording folder.
+        output_dir: Caller-provided output directory, if any.
+        config_path: Config file used when ``output_dir`` is omitted.
+
+    Returns:
+        Expanded output directory path.
+
+    Passing ``output_dir`` is an explicit override. Otherwise conversion follows
+    the same ``analysis_output`` routing that the rest of twopy uses.
+    """
+    if output_dir is not None:
+        return output_dir.expanduser()
+
+    config = load_config(config_path)
+    return resolve_analysis_output_dir(config, session_dir)