aind-ecephys-s3-to-phy 0.4.0__py3-none-any.whl

aind_ecephys_s3_to_phy/__init__.py

@@ -0,0 +1,8 @@
+"""Tools to get phy-ready data from AIND ecephys data on S3."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("aind-ecephys-s3-to-phy")
+except PackageNotFoundError:
+    __version__ = "0.0.0.dev0"

aind_ecephys_s3_to_phy/analyzer.py

@@ -0,0 +1,236 @@
+"""Load SortingAnalyzers and discover recordings in S3 assets."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import s3fs
+import spikeinterface as si
+from spikeinterface.core.baserecording import BaseRecording
+from spikeinterface.core.sortinganalyzer import SortingAnalyzer
+
+from aind_ecephys_s3_to_phy.recording import match_analyzer_to_raw
+
+
+def _is_s3(uri: str) -> bool:
+    """Check whether a URI is an S3 path."""
+    return uri.startswith("s3://")
+
+
+def load_analyzer(
+    analyzer_path: str,
+    storage_options: dict[str, Any] | None = None,
+) -> SortingAnalyzer:
+    """Load a SortingAnalyzer and ensure the ``templates`` extension is available.
+
+    Parameters
+    ----------
+    analyzer_path : str
+        Path or S3 URI to the SortingAnalyzer ``.zarr`` directory.
+    storage_options : dict or None
+        fsspec storage options.
+
+    Returns
+    -------
+    SortingAnalyzer
+        The loaded analyzer with ``templates`` extension loaded.
+
+    Raises
+    ------
+    ValueError
+        If the ``templates`` extension is not available.
+    """
+    backend_options = {"storage_options": storage_options or {}}
+    analyzer: SortingAnalyzer = si.load_sorting_analyzer(analyzer_path, backend_options=backend_options)
+
+    # Extensions are lazy for remote paths — explicitly load templates
+    if analyzer.get_extension("templates") is None:
+        analyzer.load_extension("templates")
+    if analyzer.get_extension("templates") is None:
+        raise ValueError(f"Analyzer at {analyzer_path} has no 'templates' extension (required for phy export).")
+
+    return analyzer
+
+
+def attach_recording(analyzer: SortingAnalyzer, recording: BaseRecording) -> None:
+    """Attach a preprocessed recording to a SortingAnalyzer.
+
+    Validates that the recording attributes match the analyzer's expectations,
+    then sets both the temporary and internal recording references. The
+    internal ``_recording`` assignment is needed because ``export_to_phy``
+    checks ``has_recording()`` which only inspects ``_recording``.
+
+    Parameters
+    ----------
+    analyzer : SortingAnalyzer
+        The analyzer to attach the recording to.
+    recording : BaseRecording
+        The preprocessed recording (must match analyzer's channel_ids, etc.).
+
+    Raises
+    ------
+    ValueError
+        If the recording attributes don't match the analyzer.
+    """
+    # set_temporary_recording validates attributes and channel locations
+    analyzer.set_temporary_recording(recording)
+    # Work around: export_to_phy checks has_recording() which only looks at
+    # _recording, not _temporary_recording.
+    analyzer._recording = recording
+
+
+# --- Discovery ---
+
+
+def list_recordings(
+    sorted_asset: str,
+    storage_options: dict[str, Any] | None = None,
+) -> list[tuple[str, int]]:
+    """List postprocessed SortingAnalyzer Zarrs in a sorted asset.
+
+    Parameters
+    ----------
+    sorted_asset : str
+        S3 URI or local path to the sorted asset.
+    storage_options : dict or None
+        fsspec storage options (defaults to anonymous access for S3).
+
+    Returns
+    -------
+    list of (name, size_bytes)
+        Analyzer Zarr names and their total sizes.
+    """
+    subdir = sorted_asset.rstrip("/") + "/postprocessed"
+    return _list_zarrs(subdir, storage_options)
+
+
+def list_raw_recordings(
+    raw_asset: str,
+    storage_options: dict[str, Any] | None = None,
+) -> list[tuple[str, int]]:
+    """List compressed raw recording Zarrs in a raw asset.
+
+    Filters to Neuropixels recordings only (excludes NI-DAQ).
+
+    Parameters
+    ----------
+    raw_asset : str
+        S3 URI or local path to the raw asset.
+    storage_options : dict or None
+        fsspec storage options.
+
+    Returns
+    -------
+    list of (name, size_bytes)
+        Raw recording Zarr names and their total sizes.
+    """
+    subdir = raw_asset.rstrip("/") + "/ecephys/ecephys_compressed"
+    all_zarrs = _list_zarrs(subdir, storage_options)
+    return [(name, size) for name, size in all_zarrs if "Neuropix" in name]
+
+
+def discover_pairs(
+    sorted_asset_uri: str,
+    raw_asset_uri: str,
+    storage_options: dict[str, Any] | None = None,
+) -> list[tuple[str, str]]:
+    """Match postprocessed analyzers to their raw recordings.
+
+    Parameters
+    ----------
+    sorted_asset_uri : str
+        S3 URI of the sorted asset.
+    raw_asset_uri : str
+        S3 URI of the raw asset.
+    storage_options : dict or None
+        fsspec storage options.
+
+    Returns
+    -------
+    list of (analyzer_name, raw_name)
+        Matched pairs. Unmatched analyzers are excluded.
+    """
+    analyzers = list_recordings(sorted_asset_uri, storage_options)
+    raw_recs = list_raw_recordings(raw_asset_uri, storage_options)
+    raw_names = {name for name, _ in raw_recs}
+
+    pairs: list[tuple[str, str]] = []
+    for az_name, _ in analyzers:
+        raw_candidate = match_analyzer_to_raw(az_name)
+        if raw_candidate in raw_names:
+            pairs.append((az_name, raw_candidate))
+
+    return pairs
+
+
+# --- Filtering ---
+
+
+def filter_recordings(
+    names: list[str],
+    *,
+    probe: str | None = None,
+    experiment: str | None = None,
+) -> list[str]:
+    """Filter recording names by probe ID and/or experiment.
+
+    Parameters
+    ----------
+    names : list of str
+        Recording names (e.g. analyzer zarr filenames).
+    probe : str or None
+        If given, keep only names containing this substring (e.g. ``"46101"``).
+    experiment : str or None
+        If given, keep only names starting with this prefix (e.g.
+        ``"experiment1"``).
+
+    Returns
+    -------
+    list of str
+        Filtered names, in original order.
+    """
+    result = names
+    if experiment is not None:
+        result = [n for n in result if n.startswith(experiment)]
+    if probe is not None:
+        result = [n for n in result if probe in n]
+    return result
+
+
+# --- Internal helpers ---
+
+
+def _list_zarrs(directory: str, storage_options: dict[str, Any] | None = None) -> list[tuple[str, int]]:
+    """List ``.zarr`` directories under a path (S3 or local)."""
+    if _is_s3(directory):
+        return _list_zarrs_s3(directory, storage_options)
+    return _list_zarrs_local(directory)
+
+
+def _list_zarrs_s3(directory: str, storage_options: dict[str, Any] | None) -> list[tuple[str, int]]:
+    """List ``.zarr`` directories on S3."""
+    opts = storage_options or {"anon": True}
+    fs = s3fs.S3FileSystem(**opts)
+    prefix = directory.replace("s3://", "")
+    entries = fs.ls(prefix, detail=True)
+    results: list[tuple[str, int]] = []
+    for entry in entries:
+        name = entry["name"].split("/")[-1]
+        if name.endswith(".zarr"):
+            size: int = fs.du(entry["name"])
+            results.append((name, size))
+    return results
+
+
+def _list_zarrs_local(directory: str) -> list[tuple[str, int]]:
+    """List ``.zarr`` directories on the local filesystem."""
+    parent = Path(directory)
+    if not parent.is_dir():
+        return []
+    results: list[tuple[str, int]] = []
+    for entry in sorted(parent.iterdir()):
+        if entry.name.endswith(".zarr") and entry.is_dir():
+            size = sum(f.stat().st_size for f in entry.rglob("*") if f.is_file())
+            results.append((entry.name, size))
+    return results

aind_ecephys_s3_to_phy/cli.py

@@ -0,0 +1,392 @@
+"""CLI entry point for s3-to-phy.
+
+Usage::
+
+    s3-to-phy convert --sorted-uri s3://... --raw-uri s3://...
+    s3-to-phy convert --sorted-uri s3://... --raw-uri s3://... --probe 46101
+    s3-to-phy convert --sorted-uri s3://... --raw-uri s3://... --lazy
+    s3-to-phy list --sorted-uri s3://...
+    s3-to-phy extract-recording --analyzer-uri s3://... --raw-uri s3://... --output out.dat
+    s3-to-phy compress --phy-dir ./phy_output
+"""
+
+from __future__ import annotations
+
+import shutil
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated
+
+if TYPE_CHECKING:
+    from spikeinterface.core.baserecording import BaseRecording
+
+import numpy as np
+import tyro
+from aind_ephys_assets import correct_recording, load_motion
+
+from aind_ecephys_s3_to_phy.analyzer import (
+    attach_recording,
+    discover_pairs,
+    filter_recordings,
+    list_raw_recordings,
+    list_recordings,
+    load_analyzer,
+)
+from aind_ecephys_s3_to_phy.export import compress_binary, export_phy, export_phy_lazy
+from aind_ecephys_s3_to_phy.recording import lazy_recording_dict, match_analyzer_to_raw, reconstruct_recording
+
+
+@dataclass
+class Convert:
+    """Full pipeline: export spike data, write binary traces, compress."""
+
+    sorted_uri: str
+    """S3 URI or local path to the sorted asset."""
+
+    raw_uri: str
+    """S3 URI or local path to the raw asset."""
+
+    output_dir: Path = Path(".")
+    """Base output directory."""
+
+    recordings: list[str] = field(default_factory=list)
+    """Recording names to export (default: all matched pairs)."""
+
+    probe: str | None = None
+    """Filter recordings by probe ID substring (e.g. 46101)."""
+
+    experiment: str | None = None
+    """Filter recordings by experiment prefix (e.g. experiment1)."""
+
+    compress: bool = True
+    """Compress recording.dat to .cbin."""
+
+    skip_check: bool = False
+    """Skip mtscomp compression verification."""
+
+    n_jobs: int = 1
+    """Parallel jobs for binary writing and PC extraction."""
+
+    chunk_duration: str = "1s"
+    """Chunk duration for binary writing."""
+
+    motion_correct: bool = True
+    """Apply KS4 motion correction if a motion estimate exists (default: True)."""
+
+    lazy: bool = False
+    """Write no traces: phy reads them from the raw asset when viewed, through recording.json.
+    Needs phylib with SpikeInterfaceEphysReader. Traces are not motion-corrected."""
+
+
+@dataclass
+class List:
+    """List available recordings in a sorted asset."""
+
+    sorted_uri: str
+    """S3 URI or local path to the sorted asset."""
+
+    raw_uri: str | None = None
+    """S3 URI or local path to the raw asset (shows matched pairs)."""
+
+
+@dataclass
+class ExtractRecording:
+    """Write preprocessed recording to a flat binary file.
+
+    Reconstructs the full preprocessing chain from a SortingAnalyzer
+    (including channel selection) and writes the result as a flat binary.
+    With --compress, produces a .cbin (mtscomp) file instead.
+    """
+
+    analyzer_uri: str
+    """S3 URI or path to a postprocessed SortingAnalyzer .zarr."""
+
+    raw_uri: str
+    """S3 URI or path to the raw recording .zarr."""
+
+    output: Path
+    """Output file path (e.g. preprocessed.dat)."""
+
+    compress: bool = False
+    """Compress output to .cbin (mtscomp)."""
+
+    dtype: str | None = None
+    """Output dtype (default: same as recording, typically int16)."""
+
+    n_jobs: int = 1
+    """Parallel jobs for binary writing."""
+
+    chunk_duration: str = "1s"
+    """Chunk duration for binary writing."""
+
+    motion_correct: bool = True
+    """Apply KS4 motion correction if a motion estimate exists (default: True)."""
+
+    sorted_uri: str | None = None
+    """S3 URI or local path to the sorted asset (needed for motion correction)."""
+
+
+@dataclass
+class Compress:
+    """Compress an existing recording.dat to mtscomp .cbin format."""
+
+    phy_dir: Path
+    """Path to a phy output directory containing recording.dat."""
+
+    skip_check: bool = False
+    """Skip compression verification."""
+
+
+Cmd = Annotated[
+    Convert | List | ExtractRecording | Compress,
+    tyro.conf.arg(prefix_name=False),
+]
+
+
+def main() -> None:
+    """CLI entry point."""
+    cmd = tyro.cli(Cmd, prog="s3-to-phy")
+    match cmd:
+        case Convert():
+            _run_convert(cmd)
+        case List():
+            _run_list(cmd)
+        case ExtractRecording():
+            _run_extract_recording(cmd)
+        case Compress():
+            _run_compress(cmd)
+
+
+# --- Subcommand implementations ---
+
+
+def _run_convert(cmd: Convert) -> None:
+    """Run the full conversion pipeline."""
+    recordings = _resolve_recordings(cmd)
+    sorted_name = cmd.sorted_uri.rstrip("/").split("/")[-1]
+
+    n_total = len(recordings)
+    n_ok = 0
+    n_fail = 0
+
+    for i, rec_name in enumerate(recordings, 1):
+        raw_name = match_analyzer_to_raw(rec_name)
+        analyzer_uri = f"{cmd.sorted_uri.rstrip('/')}/postprocessed/{rec_name}"
+        raw_uri = f"{cmd.raw_uri.rstrip('/')}/ecephys/ecephys_compressed/{raw_name}"
+        rec_stem = rec_name.removesuffix(".zarr")
+        phy_dir = cmd.output_dir / sorted_name / rec_stem
+
+        print(f"\n[{i}/{n_total}] {rec_name}")
+        print(f"  Output: {phy_dir}")
+
+        try:
+            _convert_one(cmd, analyzer_uri, raw_uri, phy_dir, rec_name)
+            n_ok += 1
+            print(f"  Done: {phy_dir}")
+        except KeyboardInterrupt:
+            print("\n  Interrupted — cleaning up partial output...", file=sys.stderr)
+            _cleanup_partial(phy_dir)
+            sys.exit(130)
+        except Exception as exc:  # noqa: BLE001 - one bad recording must not stop the batch
+            n_fail += 1
+            print(f"  FAILED: {exc}", file=sys.stderr)
+
+    print(f"\nFinished: {n_ok} succeeded, {n_fail} failed out of {n_total}.")
+    if n_fail > 0:
+        sys.exit(1)
+
+
+def _resolve_recordings(cmd: Convert) -> list[str]:
+    """Determine which recordings to export based on CLI flags."""
+    recordings = cmd.recordings
+
+    if not recordings:
+        print("Discovering recordings...")
+        pairs = discover_pairs(cmd.sorted_uri, cmd.raw_uri)
+        if not pairs:
+            print("No matching analyzer/raw recording pairs found.", file=sys.stderr)
+            sys.exit(1)
+        recordings = [az for az, _ in pairs]
+
+    recordings = filter_recordings(recordings, probe=cmd.probe, experiment=cmd.experiment)
+
+    if not recordings:
+        print("No recordings match the given filters.", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Selected {len(recordings)} recording(s):")
+    for name in recordings:
+        print(f"  {name}")
+
+    return recordings
+
+
+def _convert_one(cmd: Convert, analyzer_uri: str, raw_uri: str, phy_dir: Path, rec_name: str) -> None:
+    """Convert a single recording to phy."""
+    print("  Loading analyzer...")
+    analyzer = load_analyzer(analyzer_uri)
+
+    if cmd.lazy:
+        print("  Exporting to phy with lazy traces (no motion correction)...")
+        export_phy_lazy(analyzer, lazy_recording_dict(analyzer_uri, raw_uri), phy_dir)
+        return
+
+    print("  Reconstructing preprocessed recording...")
+    recording = reconstruct_recording(analyzer_uri, raw_uri)
+
+    recording = _maybe_motion_correct(recording, cmd.sorted_uri, rec_name, cmd.motion_correct)
+
+    print("  Attaching recording to analyzer...")
+    attach_recording(analyzer, recording)
+
+    print("  Exporting to phy...")
+    export_phy(
+        analyzer,
+        recording,
+        phy_dir,
+        compress=cmd.compress,
+        check_compression=not cmd.skip_check,
+        n_jobs=cmd.n_jobs,
+        chunk_duration=cmd.chunk_duration,
+    )
+
+
+def _maybe_motion_correct(
+    recording: BaseRecording,
+    sorted_uri: str | None,
+    recording_name: str,
+    motion_correct: bool,
+) -> BaseRecording:
+    """Apply motion correction if requested and a motion estimate exists."""
+    if not motion_correct:
+        print("  Motion correction: disabled")
+        return recording
+    if sorted_uri is None:
+        print("  Motion correction: skipped (no sorted URI provided)")
+        return recording
+
+    storage_options = {"anon": True} if sorted_uri.startswith("s3://") else None
+    try:
+        motion, info = load_motion(sorted_uri, recording_name, recording, storage_options=storage_options)
+    except FileNotFoundError:
+        print("  Motion correction: the sort saved no KS4 estimate — writing uncorrected traces")
+        return recording
+
+    print(f"  Applying KS4 motion correction from {info['folder']} ...")
+    from spikeinterface.preprocessing import astype
+
+    corrected = correct_recording(recording, motion)
+    # Interpolation runs in float32; integer traces go back to their dtype so .dat/.cbin keep their size.
+    if np.issubdtype(recording.get_dtype(), np.integer):
+        corrected = astype(corrected, recording.get_dtype(), round=True)
+    return corrected
+
+
+def _run_list(cmd: List) -> None:
+    """List recordings and optionally match to raw recordings."""
+    analyzers = list_recordings(cmd.sorted_uri)
+
+    if cmd.raw_uri is not None:
+        raw_recs = list_raw_recordings(cmd.raw_uri)
+        raw_names = {name for name, _ in raw_recs}
+        raw_sizes = {name: size for name, size in raw_recs}
+
+        print(f"{'Analyzer':<80} {'Size':>10}  {'Raw Match':<60} {'Raw Size':>10}")
+        print("-" * 165)
+        for az_name, az_size in sorted(analyzers):
+            raw_candidate = match_analyzer_to_raw(az_name)
+            if raw_candidate in raw_names:
+                raw_s = raw_sizes[raw_candidate]
+                print(f"{az_name:<80} {az_size / 1e6:>9.1f}M  {raw_candidate:<60} {raw_s / 1e6:>9.1f}M")
+            else:
+                print(f"{az_name:<80} {az_size / 1e6:>9.1f}M  (no match)")
+    else:
+        print(f"{'Analyzer':<80} {'Size':>10}")
+        print("-" * 92)
+        for az_name, az_size in sorted(analyzers):
+            print(f"{az_name:<80} {az_size / 1e6:>9.1f}M")
+
+
+def _run_extract_recording(cmd: ExtractRecording) -> None:
+    """Reconstruct the preprocessed recording and write to binary."""
+    import mtscomp
+    from spikeinterface.core.recording_tools import write_binary_recording
+
+    print("Reconstructing preprocessed recording...")
+    print(f"  Analyzer: {cmd.analyzer_uri}")
+    print(f"  Raw:      {cmd.raw_uri}")
+    recording = reconstruct_recording(cmd.analyzer_uri, cmd.raw_uri)
+
+    # Derive recording name from analyzer URI for motion lookup
+    rec_name = cmd.analyzer_uri.rstrip("/").split("/")[-1]
+    recording = _maybe_motion_correct(recording, cmd.sorted_uri, rec_name, cmd.motion_correct)
+
+    n_ch = recording.get_num_channels()
+    n_frames = recording.get_num_frames()
+    dtype = cmd.dtype or str(recording.get_dtype())
+    sample_rate = recording.sampling_frequency
+    est_size_gb = n_frames * n_ch * recording.get_dtype().itemsize / 1e9
+    print(f"  Channels: {n_ch}, Frames: {n_frames}, Duration: {recording.get_total_duration():.1f}s")
+    print(f"  Estimated raw size: {est_size_gb:.2f} GB")
+
+    cmd.output.parent.mkdir(parents=True, exist_ok=True)
+
+    if cmd.compress:
+        dat_path = cmd.output.with_suffix(".dat")
+        print(f"Writing intermediate {dat_path}...")
+        write_binary_recording(
+            recording,
+            file_paths=dat_path,
+            dtype=dtype,
+            n_jobs=cmd.n_jobs,
+            chunk_duration=cmd.chunk_duration,
+            progress_bar=True,
+            pool_engine="thread",
+        )
+        cbin_path = cmd.output.with_suffix(".cbin")
+        ch_path = cmd.output.with_suffix(".ch")
+        print(f"Compressing to {cbin_path}...")
+        mtscomp.compress(
+            str(dat_path),
+            out=str(cbin_path),
+            outmeta=str(ch_path),
+            sample_rate=sample_rate,
+            n_channels=n_ch,
+            dtype=dtype,
+        )
+        dat_path.unlink()
+        print(f"Done: {cbin_path} ({cbin_path.stat().st_size / 1e9:.2f} GB)")
+    else:
+        print(f"Writing to {cmd.output}...")
+        write_binary_recording(
+            recording,
+            file_paths=cmd.output,
+            dtype=dtype,
+            n_jobs=cmd.n_jobs,
+            chunk_duration=cmd.chunk_duration,
+            progress_bar=True,
+            pool_engine="thread",
+        )
+        print(f"Done: {cmd.output} ({cmd.output.stat().st_size / 1e9:.2f} GB)")
+
+
+def _run_compress(cmd: Compress) -> None:
+    """Compress recording.dat in the given phy directory."""
+    print(f"Compressing {cmd.phy_dir / 'recording.dat'}...")
+    compress_binary(cmd.phy_dir, check=not cmd.skip_check)
+    print(f"Done: {cmd.phy_dir / 'recording.cbin'}")
+
+
+def _cleanup_partial(phy_dir: Path) -> None:
+    """Remove a partial phy output directory after interruption."""
+    dat_path = phy_dir / "recording.dat"
+    if dat_path.exists():
+        dat_path.unlink()
+        print(f"  Removed partial {dat_path}", file=sys.stderr)
+    elif phy_dir.exists() and not any(phy_dir.iterdir()):
+        shutil.rmtree(phy_dir)
+
+
+if __name__ == "__main__":
+    main()

aind_ecephys_s3_to_phy/export.py

@@ -0,0 +1,362 @@
+"""Export a SortingAnalyzer to a phy directory, with traces written to disk or read lazily."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import mtscomp
+import numpy as np
+import pandas as pd
+import spikeinterface as si
+from spikeinterface.core.baserecording import BaseRecording
+from spikeinterface.core.core_tools import SIJsonEncoder
+from spikeinterface.core.recording_tools import write_binary_recording
+from spikeinterface.core.sortinganalyzer import SortingAnalyzer
+from spikeinterface.exporters import export_to_phy
+
+
+def export_phy(
+    analyzer: SortingAnalyzer,
+    recording: BaseRecording,
+    output_dir: str | Path,
+    *,
+    compress: bool = True,
+    check_compression: bool = True,
+    n_jobs: int = 1,
+    chunk_duration: str = "1s",
+) -> Path:
+    """Export a SortingAnalyzer to a phy directory.
+
+    Writes spike data via SpikeInterface's ``export_to_phy``, then streams
+    the preprocessed recording to a flat binary and optionally compresses it
+    with mtscomp.
+
+    Parameters
+    ----------
+    analyzer : SortingAnalyzer
+        Analyzer with recording attached (see :func:`~.analyzer.attach_recording`).
+    recording : BaseRecording
+        The preprocessed recording to write as binary traces.
+    output_dir : str or Path
+        Directory to write phy files into.
+    compress : bool
+        If True (default), compress ``recording.dat`` to ``.cbin`` format.
+    check_compression : bool
+        If True (default), verify compression by decompressing and comparing.
+    n_jobs : int
+        Number of parallel jobs for binary writing.
+    chunk_duration : str
+        Chunk duration for binary writing (e.g. ``"1s"``).
+
+    Returns
+    -------
+    Path
+        The output directory.
+    """
+    output_dir = Path(output_dir)
+    dat_path = output_dir / "recording.dat"
+    cbin_path = output_dir / "recording.cbin"
+    ch_path = output_dir / "recording.ch"
+
+    # 1. Write the binary first (most expensive step, skip if already done).
+    #    This gives us local data for PC extraction in step 2.
+    if cbin_path.exists():
+        print(f"  Skipping binary write — {cbin_path} already exists")
+    elif dat_path.exists():
+        print(f"  Skipping binary write — {dat_path} already exists")
+    else:
+        output_dir.mkdir(parents=True, exist_ok=True)
+        write_binary_recording(
+            recording,
+            file_paths=dat_path,
+            dtype=recording.get_dtype(),
+            n_jobs=n_jobs,
+            chunk_duration=chunk_duration,
+            progress_bar=True,
+            pool_engine="thread",
+        )
+
+    # 2. Export spike data (templates, amplitudes, PCs, metrics, etc.).
+    #    If the .dat exists, swap the analyzer's recording to a local
+    #    BinaryRecordingExtractor so PC extraction reads from disk, not S3.
+    if dat_path.exists():
+        from spikeinterface.core import BinaryRecordingExtractor
+
+        local_rec = BinaryRecordingExtractor(
+            file_paths=dat_path,
+            sampling_frequency=recording.sampling_frequency,
+            num_channels=recording.get_num_channels(),
+            dtype=recording.get_dtype(),
+        )
+        local_rec.set_channel_locations(recording.get_channel_locations())
+        analyzer._recording = local_rec
+
+    #    Stash binary files so export_to_phy can have a clean directory.
+    #    Use try/finally to guarantee restoration — these files are expensive.
+    _binary_files = [cbin_path, ch_path, dat_path]
+    _preserved: list[tuple[Path, Path]] = []
+    for f in _binary_files:
+        if f.exists():
+            stash = output_dir.parent / f"._{output_dir.name}_{f.name}.stash"
+            if stash.exists():
+                raise RuntimeError(
+                    f"Stash file {stash} already exists from a previous interrupted run. "
+                    f"Restore it manually to {f} before re-running."
+                )
+            f.rename(stash)
+            _preserved.append((stash, f))
+
+    try:
+        export_to_phy(
+            analyzer,
+            output_dir,
+            copy_binary=False,
+            compute_pc_features=True,
+            compute_amplitudes=True,
+            use_relative_path=True,
+            remove_if_exists=True,
+            n_jobs=n_jobs,
+            pool_engine="thread",
+        )
+    finally:
+        # Restore binary files even if export_to_phy fails
+        for stash, orig in _preserved:
+            if stash.exists():
+                orig.parent.mkdir(parents=True, exist_ok=True)
+                stash.rename(orig)
+
+    # Rewrite cluster_group.tsv using sorting labels
+    _write_cluster_labels(analyzer, output_dir)
+
+    # 3. Update params.py and optionally compress to .cbin
+    if cbin_path.exists():
+        _update_params_dat_path(output_dir, "recording.cbin")
+    elif compress and dat_path.exists():
+        _update_params_dat_path(output_dir, "recording.dat")
+        compress_binary(output_dir, check=check_compression)
+    elif dat_path.exists():
+        _update_params_dat_path(output_dir, "recording.dat")
+
+    return output_dir
+
+
+def export_phy_lazy(
+    analyzer: SortingAnalyzer,
+    recording_dict: dict[str, Any],
+    output_dir: str | Path,
+) -> Path:
+    """Export a SortingAnalyzer to a phy directory that reads its traces lazily.
+
+    No traces are written. ``recording.json`` holds *recording_dict*, which phylib opens with
+    SpikeInterface when ``dat_path`` names a ``.json`` file (phylib with ``SpikeInterfaceEphysReader``).
+    Waveforms come from the analyzer's stored subset (``_phy_spikes_subset.*``), so phy reads traces
+    only in the trace view, and needs no PC features.
+
+    Parameters
+    ----------
+    analyzer : SortingAnalyzer
+        Analyzer with the ``random_spikes`` and ``waveforms`` extensions; no recording needed.
+    recording_dict : dict
+        Provenance dict of the analyzer's preprocessed recording, from
+        :func:`~.recording.lazy_recording_dict`.
+    output_dir : str or Path
+        Directory to write phy files into.
+
+    Returns
+    -------
+    Path
+        The output directory.
+
+    Raises
+    ------
+    ValueError
+        If the analyzer has empty units, lacks the waveform extensions, or the recording does not
+        match the analyzer's channels and samples.
+    """
+    output_dir = Path(output_dir)
+    counts = analyzer.sorting.count_num_spikes_per_unit(outputs="array")
+    # export_to_phy drops empty units, which would renumber clusters away from the spike vector's unit_index
+    if counts.min() == 0:
+        raise ValueError(f"{int((counts == 0).sum())} empty units; phy cluster ids would not match the analyzer")
+    for ext in ("random_spikes", "waveforms"):
+        if analyzer.get_extension(ext) is None:
+            analyzer.load_extension(ext)
+        if analyzer.get_extension(ext) is None:
+            raise ValueError(f"Analyzer has no '{ext}' extension (required for lazy traces).")
+
+    export_to_phy(
+        analyzer,
+        output_dir,
+        copy_binary=False,
+        compute_pc_features=False,
+        use_relative_path=True,
+        remove_if_exists=True,
+        verbose=False,
+    )
+
+    (output_dir / "recording.json").write_text(json.dumps(recording_dict, cls=SIJsonEncoder, indent=1))
+    recording = si.load(output_dir / "recording.json")
+    if list(recording.channel_ids) != list(analyzer.channel_ids) or (
+        recording.get_num_samples() != analyzer.get_num_samples()
+    ):
+        raise ValueError("recording.json does not match the analyzer's channels or sample count")
+
+    _update_params_dat_path(output_dir, "recording.json")
+    _update_params_line(output_dir, "dtype", f"'{np.dtype(recording.get_dtype()).name}'")
+    _write_waveform_subset(analyzer, output_dir)
+    _write_cluster_labels(analyzer, output_dir)
+    return output_dir
+
+
+def compress_binary(phy_dir: str | Path, *, check: bool = True) -> None:
+    """Compress ``recording.dat`` to mtscomp ``.cbin`` format.
+
+    Reads recording metadata from ``params.py``, compresses the ``.dat``
+    file, updates ``params.py`` to point at the ``.cbin``, and deletes
+    the original ``.dat``.
+
+    Parameters
+    ----------
+    phy_dir : str or Path
+        Path to a phy output directory containing ``recording.dat`` and
+        ``params.py``.
+    check : bool
+        If True (default), verify the compressed file by decompressing and
+        comparing against the original. Doubles I/O time but guarantees
+        lossless compression.
+
+    Raises
+    ------
+    FileNotFoundError
+        If ``recording.dat`` or ``params.py`` is missing.
+    """
+    phy_dir = Path(phy_dir)
+    dat_path = phy_dir / "recording.dat"
+    cbin_path = phy_dir / "recording.cbin"
+    ch_path = phy_dir / "recording.ch"
+
+    if not dat_path.exists():
+        raise FileNotFoundError(f"No recording.dat found in {phy_dir}")
+
+    params = _read_params(phy_dir)
+
+    mtscomp.compress(
+        str(dat_path),
+        out=str(cbin_path),
+        outmeta=str(ch_path),
+        sample_rate=params["sample_rate"],
+        n_channels=params["n_channels_dat"],
+        dtype=params["dtype"],
+        check_after_compress=check,
+    )
+
+    _update_params_dat_path(phy_dir, "recording.cbin")
+    dat_path.unlink()
+
+
+# --- Internal helpers ---
+
+
+def _read_params(phy_dir: Path) -> dict[str, Any]:
+    """Parse ``params.py`` in a phy directory."""
+    params_path = phy_dir / "params.py"
+    if not params_path.exists():
+        raise FileNotFoundError(f"No params.py found in {phy_dir}")
+    namespace: dict[str, Any] = {"r": str, "np": np}
+    exec(params_path.read_text(), namespace)  # noqa: S102 - params.py is Python by phy's format
+    return namespace
+
+
+def _write_cluster_labels(analyzer: SortingAnalyzer, output_dir: Path) -> None:
+    """Rewrite ``cluster_group.tsv`` using sorting labels and write individual label columns.
+
+    Assigns phy cluster groups based on a combined heuristic:
+
+    - **good**: KSLabel ``good`` AND decoder_label ``sua``
+    - **mua**: everything else that isn't noise
+    - **noise**: decoder_label ``noise`` OR default_qc ``False``
+
+    Also writes each available label (KSLabel, decoder_label, default_qc,
+    decoder_probability) as a separate ``cluster_*.tsv`` for visibility
+    in phy's cluster view.
+    """
+    sorting = analyzer.sorting
+    unit_ids = sorting.unit_ids
+    n_units = len(unit_ids)
+    cluster_ids = list(range(n_units))
+
+    # Determine combined cluster group
+    ks_label = sorting.get_property("KSLabel")
+    decoder_label = sorting.get_property("decoder_label")
+    default_qc = sorting.get_property("default_qc")
+
+    groups: list[str] = ["unsorted"] * n_units
+    for i in range(n_units):
+        is_noise = False
+        if decoder_label is not None and decoder_label[i] == "noise":
+            is_noise = True
+        if default_qc is not None and not default_qc[i]:
+            is_noise = True
+
+        if is_noise:
+            groups[i] = "noise"
+        elif ks_label is not None and ks_label[i] == "good" and decoder_label is not None and decoder_label[i] == "sua":
+            groups[i] = "good"
+        else:
+            groups[i] = "mua"
+
+    # Overwrite cluster_group.tsv
+    cluster_group = pd.DataFrame({"cluster_id": cluster_ids, "group": groups})
+    cluster_group.to_csv(output_dir / "cluster_group.tsv", sep="\t", index=False)
+
+    # Write individual label columns as cluster_*.tsv
+    _label_properties = ["KSLabel", "decoder_label", "default_qc", "decoder_probability"]
+    for prop_name in _label_properties:
+        values = sorting.get_property(prop_name)
+        if values is not None:
+            df = pd.DataFrame({"cluster_id": cluster_ids, prop_name: values})
+            df.to_csv(output_dir / f"cluster_{prop_name}.tsv", sep="\t", index=False)
+
+
+def _write_waveform_subset(analyzer: SortingAnalyzer, output_dir: Path) -> None:
+    """Write phylib's ``_phy_spikes_subset.*`` files from the analyzer's stored waveforms.
+
+    ``random_spikes`` indexes the spike vector, whose order is also that of ``spike_times.npy``.
+    """
+    spike_ids = analyzer.get_extension("random_spikes").get_data()
+    waveforms = analyzer.get_extension("waveforms").get_data()
+    unit_index = analyzer.sorting.to_spike_vector()["unit_index"][spike_ids]
+    mask = (
+        analyzer.sparsity.mask
+        if analyzer.sparsity is not None
+        else np.ones((len(analyzer.unit_ids), analyzer.get_num_channels()), dtype=bool)
+    )
+    unit_channels = -np.ones((len(analyzer.unit_ids), waveforms.shape[2]), dtype=np.int64)
+    for u in range(len(analyzer.unit_ids)):
+        channels = np.flatnonzero(mask[u])
+        unit_channels[u, : len(channels)] = channels
+    np.save(output_dir / "_phy_spikes_subset.spikes.npy", spike_ids.astype(np.int64))
+    np.save(output_dir / "_phy_spikes_subset.channels.npy", unit_channels[unit_index])
+    np.save(output_dir / "_phy_spikes_subset.waveforms.npy", waveforms)
+
+
+def _update_params_line(phy_dir: Path, key: str, value: str) -> None:
+    """Rewrite the ``key = ...`` line in ``params.py``."""
+    params_path = phy_dir / "params.py"
+    lines = params_path.read_text().splitlines(keepends=True)
+    params_path.write_text("".join(f"{key} = {value}\n" if line.startswith(f"{key} ") else line for line in lines))
+
+
+def _update_params_dat_path(phy_dir: Path, dat_filename: str) -> None:
+    """Rewrite the ``dat_path`` line in ``params.py``."""
+    params_path = phy_dir / "params.py"
+    lines = params_path.read_text().splitlines(keepends=True)
+    new_lines: list[str] = []
+    for line in lines:
+        if line.startswith("dat_path"):
+            new_lines.append(f"dat_path = r'{dat_filename}'\n")
+        else:
+            new_lines.append(line)
+    params_path.write_text("".join(new_lines))

aind_ecephys_s3_to_phy/py.typed

@@ -0,0 +1 @@
+

aind_ecephys_s3_to_phy/recording.py

@@ -0,0 +1,196 @@
+"""Reconstruct a SpikeInterface preprocessing chain from a SortingAnalyzer.
+
+The SortingAnalyzer stores the full recording provenance (preprocessing chain)
+as a nested dictionary in its Zarr store. This module extracts that dict,
+substitutes the raw recording path at the leaf, and reconstructs the full
+recording object including ChannelSlice steps that
+``get_preprocessing_dict_from_analyzer`` intentionally skips.
+"""
+
+from __future__ import annotations
+
+import copy
+import re
+from typing import Any
+
+from spikeinterface.core.base import _load_extractor_from_dict
+from spikeinterface.core.baserecording import BaseRecording
+from spikeinterface.core.zarrextractors import super_zarr_open
+
+
+def get_recording_dict(
+    analyzer_path: str,
+    storage_options: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Extract the recording provenance dict from a SortingAnalyzer Zarr.
+
+    Parameters
+    ----------
+    analyzer_path : str
+        Path or S3 URI to the SortingAnalyzer ``.zarr`` directory.
+    storage_options : dict or None
+        fsspec storage options (e.g. ``{"anon": True}``).
+
+    Returns
+    -------
+    dict
+        Nested dictionary representing the full preprocessing chain.
+
+    Raises
+    ------
+    ValueError
+        If the analyzer Zarr has no ``recording`` dataset.
+    """
+    storage_options = storage_options or {}
+    zarr_root = super_zarr_open(analyzer_path, mode="r", storage_options=storage_options)
+    rec_field = zarr_root.get("recording")
+    if rec_field is None:
+        raise ValueError(f"No recording dict found in analyzer: {analyzer_path}")
+    result: dict[str, Any] = rec_field[0]
+    return result
+
+
+def fix_recording_dict(rec_dict: dict[str, Any], raw_path: str) -> dict[str, Any]:
+    """Substitute the raw recording path in a recording provenance dict.
+
+    Deep-copies the dict and replaces the ``folder_path`` of the innermost
+    (leaf) extractor — typically a ``ZarrRecordingExtractor`` — with
+    *raw_path*. Also clears ``relative_paths`` flags throughout the chain.
+
+    Parameters
+    ----------
+    rec_dict : dict
+        Recording provenance dict from :func:`get_recording_dict`.
+    raw_path : str
+        S3 URI or local path to the raw recording Zarr.
+
+    Returns
+    -------
+    dict
+        A new dict with the leaf path substituted.
+    """
+    rec_dict = copy.deepcopy(rec_dict)
+    _fix_leaf_path(rec_dict, raw_path)
+    _clear_relative_paths(rec_dict)
+    return rec_dict
+
+
+def reconstruct_recording(
+    analyzer_path: str,
+    raw_path: str,
+    storage_options: dict[str, Any] | None = None,
+) -> BaseRecording:
+    """Reconstruct the full preprocessed recording from a SortingAnalyzer.
+
+    Extracts the recording chain from the analyzer Zarr, substitutes the raw
+    recording path, and instantiates the full SpikeInterface preprocessing
+    pipeline (including ChannelSlice steps).
+
+    Parameters
+    ----------
+    analyzer_path : str
+        Path or S3 URI to the SortingAnalyzer ``.zarr`` directory.
+    raw_path : str
+        S3 URI or local path to the raw recording Zarr.
+    storage_options : dict or None
+        fsspec storage options for accessing the analyzer Zarr.
+
+    Returns
+    -------
+    BaseRecording
+        The reconstructed preprocessed recording.
+    """
+    rec_dict = get_recording_dict(analyzer_path, storage_options)
+    fixed = fix_recording_dict(rec_dict, raw_path)
+    return _load_extractor_from_dict(fixed)
+
+
+def lazy_recording_dict(
+    analyzer_path: str,
+    raw_path: str,
+    storage_options: dict[str, Any] | None = None,
+    raw_storage_options: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build a recording provenance dict that reads the raw data from *raw_path* when loaded.
+
+    Parameters
+    ----------
+    analyzer_path : str
+        Path or S3 URI to the SortingAnalyzer ``.zarr`` directory.
+    raw_path : str
+        S3 URI or local path to the raw recording Zarr.
+    storage_options : dict or None
+        fsspec storage options for accessing the analyzer Zarr.
+    raw_storage_options : dict or None
+        fsspec storage options stored with the raw leaf. Defaults to ``{"anon": True}`` for an S3
+        *raw_path*, since a stale AWS profile otherwise fails before unsigned access is tried.
+
+    Returns
+    -------
+    dict
+        A JSON-serializable dict that ``spikeinterface.load`` turns into the preprocessed recording.
+    """
+    rec_dict = fix_recording_dict(get_recording_dict(analyzer_path, storage_options), raw_path)
+    if raw_storage_options is None and raw_path.startswith("s3://"):
+        raw_storage_options = {"anon": True}
+    if raw_storage_options is not None:
+        _leaf(rec_dict)["kwargs"]["storage_options"] = raw_storage_options
+    return rec_dict
+
+
+# --- Name matching ---
+
+
+def match_analyzer_to_raw(analyzer_name: str) -> str:
+    """Derive the raw recording Zarr name from an analyzer Zarr name.
+
+    Analyzer names have a ``_recordingN`` suffix that raw names lack:
+
+    - ``experiment1_...50213-1_recording1.zarr`` → ``experiment1_...50213-1.zarr``
+    - ``foo.zarr`` → ``foo.zarr`` (no suffix, returned as-is)
+
+    Parameters
+    ----------
+    analyzer_name : str
+        Filename of the postprocessed SortingAnalyzer Zarr.
+
+    Returns
+    -------
+    str
+        Corresponding raw recording Zarr filename.
+    """
+    return re.sub(r"_recording\d+\.zarr$", ".zarr", analyzer_name)
+
+
+# --- Internal helpers ---
+
+
+def _fix_leaf_path(d: dict[str, Any], new_path: str) -> None:
+    """Recursively find the leaf extractor and replace its ``folder_path``."""
+    for key in ("recording", "parent_recording"):
+        child = d.get("kwargs", {}).get(key)
+        if isinstance(child, dict) and "class" in child:
+            _fix_leaf_path(child, new_path)
+            return
+    # Leaf node — replace the path
+    d["kwargs"]["folder_path"] = new_path
+    d["relative_paths"] = False
+
+
+def _leaf(d: dict[str, Any]) -> dict[str, Any]:
+    """Return the innermost extractor dict of a recording chain."""
+    for key in ("recording", "parent_recording"):
+        child = d.get("kwargs", {}).get(key)
+        if isinstance(child, dict) and "class" in child:
+            return _leaf(child)
+    return d
+
+
+def _clear_relative_paths(d: dict[str, Any]) -> None:
+    """Set ``relative_paths`` to False throughout the chain."""
+    if "relative_paths" in d:
+        d["relative_paths"] = False
+    for key in ("recording", "parent_recording"):
+        child = d.get("kwargs", {}).get(key)
+        if isinstance(child, dict) and "class" in child:
+            _clear_relative_paths(child)

aind_ecephys_s3_to_phy-0.4.0.dist-info/METADATA

@@ -0,0 +1,133 @@
+Metadata-Version: 2.5
+Name: aind-ecephys-s3-to-phy
+Version: 0.4.0
+Summary: Tools to get phy-ready data from AIND ecephys data on S3
+Project-URL: Repository, https://github.com/AllenNeuralDynamics/aind-ecephys-s3-to-phy/
+Project-URL: Issues, https://github.com/AllenNeuralDynamics/aind-ecephys-s3-to-phy/issues
+Project-URL: Changelog, https://github.com/AllenNeuralDynamics/aind-ecephys-s3-to-phy/blob/main/CHANGELOG.md
+Author: Allen Institute for Neural Dynamics
+Author-email: Galen Lynch <galen@galenlynch.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Requires-Dist: aind-ephys-assets[si]>=0.5
+Requires-Dist: mtscomp
+Requires-Dist: s3fs
+Requires-Dist: spikeinterface[full,streaming-extractors]
+Requires-Dist: tyro
+Requires-Dist: wavpack-numcodecs
+Description-Content-Type: text/markdown
+
+# aind-ecephys-s3-to-phy
+
+![CI](https://github.com/AllenNeuralDynamics/aind-ecephys-s3-to-phy/actions/workflows/ci-call.yml/badge.svg)
+[![PyPI - Version](https://img.shields.io/pypi/v/aind-ecephys-s3-to-phy)](https://pypi.org/project/aind-ecephys-s3-to-phy/)
+[![semantic-release: angular](https://img.shields.io/badge/semantic--release-angular-e10079?logo=semantic-release)](https://github.com/semantic-release/semantic-release)
+[![License](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE)
+[![ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
+[![Copier](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/copier-org/copier/master/img/badge/badge-grayscale-inverted-border.json)](https://github.com/copier-org/copier)
+
+Convert AIND ecephys sorted recordings from S3 into [phy](https://github.com/cortex-lab/phy) directories for manual curation, with raw trace access via compressed binary ([mtscomp](https://github.com/int-brain-lab/mtscomp)).
+
+Two S3 assets are needed per session: a **sorted asset** (postprocessed SortingAnalyzer Zarrs) and its corresponding **raw asset** (compressed Zarr recordings). The tool reconstructs the full SpikeInterface preprocessing chain — including bad-channel removal — from the analyzer, streams traces from S3, and compresses them to `.cbin` for phy's trace viewer.
+
+## Installation
+
+```bash
+pip install aind-ecephys-s3-to-phy
+```
+
+## Quick start
+
+```bash
+SORTED=s3://aind-open-data/ecephys_786867_2025-09-25_12-43-56_sorted_2025-12-24_08-46-14
+RAW=s3://aind-open-data/ecephys_786867_2025-09-25_12-43-56
+
+# List available recordings (and matched raw files)
+s3-to-phy list --sorted-uri $SORTED --raw-uri $RAW
+
+# Convert all recordings to phy (streams from S3, compresses to .cbin)
+s3-to-phy convert --sorted-uri $SORTED --raw-uri $RAW
+
+# Convert only recordings for a specific probe
+s3-to-phy convert --sorted-uri $SORTED --raw-uri $RAW --probe 46101
+
+# Convert only a specific experiment
+s3-to-phy convert --sorted-uri $SORTED --raw-uri $RAW --experiment experiment1
+
+# Write no traces: phy reads them from the raw asset on S3 when you view them
+s3-to-phy convert --sorted-uri $SORTED --raw-uri $RAW --probe 46101 --lazy
+
+# Extract preprocessed traces as flat binary (for non-phy pipelines, e.g. mipmap)
+s3-to-phy extract-recording \
+  --analyzer-uri $SORTED/postprocessed/experiment1_...46101_recording1.zarr \
+  --raw-uri $RAW/ecephys/ecephys_compressed/experiment1_...46101.zarr \
+  --output preprocessed.dat
+
+# Compress an existing recording.dat to .cbin
+s3-to-phy compress --phy-dir ./phy_output
+```
+
+### Motion correction
+
+`convert` and `extract-recording` correct the traces with the drift estimate Kilosort 4 applied, read through
+`aind-ephys-assets` from the sort's `spikesorted/motion/<recording>/`. Sorts made before sorting capsule
+`cdef667` (2026-04-16), the example asset above among them, did not save it; those export uncorrected, and
+preprocessing's never-applied `dredge_fast` estimate is not used as a substitute. Integer traces stay integer.
+Pass `--no-motion-correct` to skip it.
+
+### Lazy traces
+
+`--lazy` exports in about a minute per probe instead of hours, with no trace file:
+
+- `recording.json` holds the analyzer's preprocessing chain over the raw Zarr on S3, and `params.py`'s
+  `dat_path` names it. phylib opens it through SpikeInterface: this needs a phylib with
+  `SpikeInterfaceEphysReader` (cortex-lab/phylib#68).
+- Waveforms come from the analyzer's stored subset, written as `_phy_spikes_subset.*` (most of the folder's
+  size, a few GB per probe). Phy reads traces only in the trace view, or for a waveform outside the subset
+  (about 0.5 s per spike from S3). No PC features are computed.
+- Traces are read on demand: a jump to a new window costs about a second, and scrolling is cached.
+- Traces are not motion-corrected, matching the analyzer's waveforms. SpikeInterface's motion
+  interpolation does not survive a round trip through JSON.
+
+### Python API
+
+```python
+from aind_ecephys_s3_to_phy.analyzer import load_analyzer, attach_recording, discover_pairs
+from aind_ecephys_s3_to_phy.recording import reconstruct_recording
+from aind_ecephys_s3_to_phy.export import export_phy
+
+# Discover and convert
+pairs = discover_pairs(sorted_uri, raw_uri)
+analyzer_name, raw_name = pairs[0]
+
+analyzer_zarr = f"{sorted_uri}/postprocessed/{analyzer_name}"
+raw_zarr = f"{raw_uri}/ecephys/ecephys_compressed/{raw_name}"
+
+analyzer = load_analyzer(analyzer_zarr)
+recording = reconstruct_recording(analyzer_zarr, raw_zarr)
+attach_recording(analyzer, recording)
+export_phy(analyzer, recording, "./phy_output", compress=True)
+
+# Or just get the preprocessed recording for other pipelines
+recording = reconstruct_recording(analyzer_zarr, raw_zarr)
+# → SpikeInterface BaseRecording (filtered, bad channels removed, 373 ch)
+```
+
+### S3 credentials
+
+Public buckets (like `aind-open-data`) work without credentials. For private buckets, configure AWS credentials via the standard methods (`~/.aws/credentials`, environment variables, IAM roles). SpikeInterface uses `s3fs`/`fsspec` which follows the standard AWS credential chain.
+
+## Development
+
+```bash
+uv sync                                # Set up environment
+./scripts/run_linters_and_checks.sh -c  # Full lint + test suite
+uv run --frozen pytest -m integration   # Integration tests (requires S3)
+```
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

aind_ecephys_s3_to_phy-0.4.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+aind_ecephys_s3_to_phy/__init__.py,sha256=fhg7W6K43gwRJAjDtdvoUIhxTGRpEpGp-ssD0lJyA5M,244
+aind_ecephys_s3_to_phy/analyzer.py,sha256=TQ7ev7BkbZcjOS44xB5SdkVDv7cfre0MMC0mPbEaGYE,7257
+aind_ecephys_s3_to_phy/cli.py,sha256=-OuNjFsaTc_92OS7v9t3Wz7-9L2qYUwZY8BYPMAofDo,12999
+aind_ecephys_s3_to_phy/export.py,sha256=kUXmSq6A1vi143kJ9l78DnYwv32GDRfgBgubbErTHMY,13701
+aind_ecephys_s3_to_phy/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+aind_ecephys_s3_to_phy/recording.py,sha256=3UsXBrN6xxQra5mYn8DdWQnF_L3XVL4J44R6UYHifP8,6576
+aind_ecephys_s3_to_phy-0.4.0.dist-info/METADATA,sha256=BoR6UJmo2MOzkjDuxKSm0kV-uAc4QTORg4sbTVlb6iY,6581
+aind_ecephys_s3_to_phy-0.4.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+aind_ecephys_s3_to_phy-0.4.0.dist-info/entry_points.txt,sha256=7rptLSo15ORZCbilUz5A3-U0nHzDWSta8ehCIkD2qVY,62
+aind_ecephys_s3_to_phy-0.4.0.dist-info/licenses/LICENSE,sha256=zMZxYbR1i_XxgyJP_rPSWKcxY7QtyxUOzbIpowHNjCs,1093
+aind_ecephys_s3_to_phy-0.4.0.dist-info/RECORD,,

aind_ecephys_s3_to_phy-0.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

aind_ecephys_s3_to_phy-0.4.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+s3-to-phy = aind_ecephys_s3_to_phy.cli:main

aind_ecephys_s3_to_phy-0.4.0.dist-info/licenses/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Allen Institute for Neural Dynamics
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+