lbm_caiman_python 0.2.0__py3-none-any.whl

lbm_caiman_python/__init__.py

@@ -0,0 +1,63 @@
+from . import _version
+
+# core pipeline api
+from .default_ops import default_ops, mcorr_ops, cnmf_ops
+from .run_lcp import (
+    pipeline,
+    run_volume,
+    run_plane,
+    add_processing_step,
+    generate_plane_dirname,
+)
+from .postprocessing import (
+    load_ops,
+    load_planar_results,
+    dff_rolling_percentile,
+    compute_roi_stats,
+    get_accepted_cells,
+    get_contours,
+)
+
+# existing utilities
+from .collation import combine_z_planes
+from .util.transform import vectorize, unvectorize, calculate_centers
+from .util.quality import get_noise_fft, greedyROI
+from .util.signal import smooth_data, norm_minmax
+from .helpers import (
+    generate_patch_view,
+    get_single_patch_coords,
+    extract_center_square,
+)
+
+__version__ = _version.get_versions()['version']
+
+__all__ = [
+    # core pipeline
+    "pipeline",
+    "run_volume",
+    "run_plane",
+    "default_ops",
+    "mcorr_ops",
+    "cnmf_ops",
+    "add_processing_step",
+    "generate_plane_dirname",
+
+    # postprocessing
+    "load_ops",
+    "load_planar_results",
+    "dff_rolling_percentile",
+    "compute_roi_stats",
+    "get_accepted_cells",
+    "get_contours",
+
+    # existing utilities
+    "combine_z_planes",
+    "generate_patch_view",
+    "get_noise_fft",
+    "calculate_centers",
+    "greedyROI",
+    "get_single_patch_coords",
+    "extract_center_square",
+    "smooth_data",
+    "norm_minmax",
+]

lbm_caiman_python/__main__.py

@@ -0,0 +1,302 @@
+"""
+cli entry point for lbm-caiman-python.
+
+usage:
+    lcp <input> <output> [options]
+    python -m lbm_caiman_python <input> <output> [options]
+"""
+
+import argparse
+import sys
+from functools import partial
+from pathlib import Path
+
+print = partial(print, flush=True)
+
+
+def add_args(parser: argparse.ArgumentParser):
+    """add command-line arguments to the parser."""
+    from lbm_caiman_python.default_ops import default_ops
+
+    defaults = default_ops()
+
+    # positional arguments
+    parser.add_argument(
+        "input",
+        type=str,
+        nargs="?",
+        default=None,
+        help="Input file or directory",
+    )
+    parser.add_argument(
+        "output",
+        type=str,
+        nargs="?",
+        default=None,
+        help="Output directory",
+    )
+
+    # processing flags
+    parser.add_argument(
+        "--planes",
+        type=int,
+        nargs="+",
+        default=None,
+        help="Planes to process (1-based)",
+    )
+    parser.add_argument(
+        "--no-mcorr",
+        action="store_true",
+        help="Skip motion correction",
+    )
+    parser.add_argument(
+        "--no-cnmf",
+        action="store_true",
+        help="Skip CNMF",
+    )
+    parser.add_argument(
+        "--force-mcorr",
+        action="store_true",
+        help="Force re-run motion correction",
+    )
+    parser.add_argument(
+        "--force-cnmf",
+        action="store_true",
+        help="Force re-run CNMF",
+    )
+    parser.add_argument(
+        "--num-frames",
+        type=int,
+        default=None,
+        help="Limit number of frames to process",
+    )
+
+    # motion correction parameters
+    parser.add_argument(
+        "--max-shifts",
+        type=int,
+        nargs=2,
+        default=None,
+        help=f"Maximum rigid shifts (default: {defaults['max_shifts']})",
+    )
+    parser.add_argument(
+        "--strides",
+        type=int,
+        nargs=2,
+        default=None,
+        help=f"Patch strides for piecewise-rigid (default: {defaults['strides']})",
+    )
+    parser.add_argument(
+        "--overlaps",
+        type=int,
+        nargs=2,
+        default=None,
+        help=f"Patch overlaps (default: {defaults['overlaps']})",
+    )
+    parser.add_argument(
+        "--gSig-filt",
+        type=int,
+        nargs=2,
+        default=None,
+        help=f"Gaussian filter size (default: {defaults['gSig_filt']})",
+    )
+    parser.add_argument(
+        "--no-pw-rigid",
+        action="store_true",
+        help="Disable piecewise-rigid correction (use rigid only)",
+    )
+
+    # cnmf parameters
+    parser.add_argument(
+        "--K",
+        type=int,
+        default=None,
+        help=f"Expected number of neurons (default: {defaults['K']})",
+    )
+    parser.add_argument(
+        "--gSig",
+        type=int,
+        nargs=2,
+        default=None,
+        help=f"Expected neuron half-width (default: {defaults['gSig']})",
+    )
+    parser.add_argument(
+        "--min-SNR",
+        type=float,
+        default=None,
+        help=f"Minimum SNR threshold (default: {defaults['min_SNR']})",
+    )
+    parser.add_argument(
+        "--rval-thr",
+        type=float,
+        default=None,
+        help=f"Correlation threshold (default: {defaults['rval_thr']})",
+    )
+    parser.add_argument(
+        "--merge-thresh",
+        type=float,
+        default=None,
+        help=f"Merging threshold (default: {defaults['merge_thresh']})",
+    )
+
+    # general parameters
+    parser.add_argument(
+        "--fr",
+        type=float,
+        default=None,
+        help=f"Frame rate in Hz (default: {defaults['fr']})",
+    )
+    parser.add_argument(
+        "--n-processes",
+        type=int,
+        default=None,
+        help="Number of parallel processes (default: auto)",
+    )
+
+    # utility flags
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="Show version information",
+    )
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        help="Enable debug mode",
+    )
+
+    return parser
+
+
+def build_ops_from_args(args) -> dict:
+    """build ops dictionary from command-line arguments."""
+    from lbm_caiman_python.default_ops import default_ops
+
+    ops = default_ops()
+
+    # map cli args to ops keys
+    arg_to_ops = {
+        "max_shifts": "max_shifts",
+        "strides": "strides",
+        "overlaps": "overlaps",
+        "gSig_filt": "gSig_filt",
+        "K": "K",
+        "gSig": "gSig",
+        "min_SNR": "min_SNR",
+        "rval_thr": "rval_thr",
+        "merge_thresh": "merge_thresh",
+        "fr": "fr",
+        "n_processes": "n_processes",
+    }
+
+    for arg_name, ops_key in arg_to_ops.items():
+        value = getattr(args, arg_name.replace("-", "_"), None)
+        if value is not None:
+            if isinstance(value, list) and len(value) == 2:
+                ops[ops_key] = tuple(value)
+            else:
+                ops[ops_key] = value
+
+    # handle boolean flags
+    if args.no_mcorr:
+        ops["do_motion_correction"] = False
+    if args.no_cnmf:
+        ops["do_cnmf"] = False
+    if args.no_pw_rigid:
+        ops["pw_rigid"] = False
+
+    return ops
+
+
+def main():
+    """main cli entry point."""
+    print("\n")
+    print("--- LBM-CaImAn Pipeline ---")
+    print("\n")
+
+    parser = argparse.ArgumentParser(
+        description="LBM-CaImAn processing pipeline",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+examples:
+  lcp data.tiff output/
+  lcp data/ results/ --planes 1 2 3
+  lcp movie.tif results/ --K 100 --gSig 5 5
+  lcp data.tiff output/ --no-mcorr  # skip motion correction
+        """,
+    )
+    parser = add_args(parser)
+    args = parser.parse_args()
+
+    # handle version
+    if args.version:
+        import lbm_caiman_python
+        print(f"lbm_caiman_python v{lbm_caiman_python.__version__}")
+        return
+
+    # check required arguments
+    if args.input is None:
+        parser.print_help()
+        return
+
+    # setup logging
+    if args.debug:
+        import logging
+        logging.basicConfig(level=logging.DEBUG)
+        print("Debug mode enabled.")
+
+    # validate input
+    input_path = Path(args.input).expanduser().resolve()
+    if not input_path.exists():
+        print(f"Error: Input path does not exist: {input_path}")
+        sys.exit(1)
+
+    # setup output
+    if args.output is None:
+        output_path = input_path.parent / (input_path.stem + "_caiman_results")
+    else:
+        output_path = Path(args.output).expanduser().resolve()
+
+    print(f"Input: {input_path}")
+    print(f"Output: {output_path}")
+
+    # build ops from arguments
+    ops = build_ops_from_args(args)
+
+    # run pipeline
+    from lbm_caiman_python.run_lcp import pipeline
+
+    try:
+        results = pipeline(
+            input_data=input_path,
+            save_path=str(output_path),
+            ops=ops,
+            planes=args.planes,
+            force_mcorr=args.force_mcorr,
+            force_cnmf=args.force_cnmf,
+            num_timepoints=args.num_frames,
+        )
+
+        print("\n")
+        print("--- Processing Complete ---")
+        print(f"Results saved to: {output_path}")
+        print(f"Processed {len(results)} plane(s)")
+
+        # print summary
+        for ops_path in results:
+            from lbm_caiman_python.postprocessing import load_ops
+            ops_result = load_ops(ops_path)
+            plane = ops_result.get("plane", "?")
+            n_cells = ops_result.get("n_cells", 0)
+            print(f"  Plane {plane}: {n_cells} cells")
+
+    except Exception as e:
+        print(f"\nError: {e}")
+        if args.debug:
+            import traceback
+            traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

lbm_caiman_python/_version.py

@@ -0,0 +1,8 @@
+"""version information for lbm_caiman_python."""
+
+__version__ = "0.1.0"
+
+
+def get_versions():
+    """return version dict for backwards compatibility."""
+    return {"version": __version__}

lbm_caiman_python/batch.py

@@ -0,0 +1,188 @@
+import re as regex
+from pathlib import Path
+from typing import Union
+from contextlib import contextmanager
+import pathlib
+import lbm_mc as mc
+
+COMPUTE_BACKEND_SUBPROCESS = "subprocess"  #: subprocess backend
+COMPUTE_BACKEND_SLURM = "slurm"  #: SLURM backend
+COMPUTE_BACKEND_LOCAL = "local"
+
+COMPUTE_BACKENDS = [
+    COMPUTE_BACKEND_SUBPROCESS,
+    COMPUTE_BACKEND_SLURM,
+    COMPUTE_BACKEND_LOCAL,
+]
+
+DATAFRAME_COLUMNS = [
+    "algo",
+    "item_name",
+    "input_movie_path",
+    "params",
+    "outputs",
+    "added_time",
+    "ran_time",
+    "algo_duration",
+    "comments",
+    "uuid",
+]
+
+
+@contextmanager
+def _set_posix_windows():
+    posix_backup = pathlib.PosixPath
+    try:
+        pathlib.PosixPath = pathlib.WindowsPath
+        yield
+    finally:
+        pathlib.PosixPath = posix_backup
+
+
+@contextmanager
+def _set_windows_posix():
+    """
+    Set the Path class to WindowsPath on a POSIX system.
+    """
+    windows_backup = pathlib.WindowsPath
+    try:
+        pathlib.WindowsPath = pathlib.PosixPath
+        yield
+    finally:
+        pathlib.WindowsPath = windows_backup
+
+
+def load_batch(batch_path: str | Path):
+    """
+    Load a batch after transfering it from a Windows to a POSIX system or vice versa.
+
+    Parameters
+    ----------
+    batch_path : str, Path
+        The path to the batch file.
+
+    Returns
+    -------
+    pandas.DataFrame
+        The loaded batch.
+    """
+    try:
+        with _set_windows_posix():
+            return mc.load_batch(batch_path)
+    except Exception:
+        with _set_posix_windows():
+            return mc.load_batch(batch_path)
+
+
+def clean_batch(df):
+    """
+        Clean a batch of DataFrame entries by removing unsuccessful df from storage.
+
+        This function iterates over the df of the given DataFrame, identifies
+        df where the 'outputs' column is either `None` or a dictionary containing
+        a 'success' key with a `False` value. For each such row, the corresponding
+        item is removed using the `df.caiman.remove_item()` method, and the removal
+        is saved to disk.
+
+        Parameters
+        ----------
+        df : pandas.DataFrame
+            The DataFrame to be cleaned. It must have a 'uuid' column for identification
+            and an 'outputs' column containing a dictionary with a 'success' key.
+
+        Returns
+        -------
+        pandas.DataFrame
+            The DataFrame reloaded from disk after unsuccessful items have been removed.
+
+        Notes
+        -----
+        - If 'outputs' is None or does not contain 'success' as a key with a value of
+          `False`, the row will be removed.
+
+        Examples
+        --------
+        >>> import pandas as pd
+        >>> df = pd.DataFrame({
+        ...     'uuid': ['123', '456', '789'],
+        ...     'outputs': [{'success': True}, {'success': False}, None]
+        ... })
+        >>> cleaned_df = clean_batch(df)
+        Removing unsuccessful batch row 1.
+        Row 1 deleted.
+        Removing unsuccessful batch row 2.
+        Row 2 deleted.
+        """
+    for index, row in df.iterrows():
+        # Check if 'outputs' is a dictionary and has 'success' key with value False
+        if isinstance(row["outputs"], dict) and row["outputs"].get("success") is False or row["outputs"] is None:
+            uuid = row["uuid"]
+            print(f"Removing unsuccessful batch row {row.index}.")
+            df.caiman.remove_item(uuid, remove_data=True, safe_removal=False)
+            print(f"Row {row.index} deleted.")
+    df.caiman.save_to_disk()
+    return df.caiman.reload_from_disk()
+
+
+def delete_batch_rows(df, rows_delete, remove_data=False, safe_removal=True):
+    rows_delete = [rows_delete] if isinstance(rows_delete, int) else rows_delete
+    uuids_delete = [row.uuid for i, row in df.iterrows() if i in rows_delete]
+    for uuid in uuids_delete:
+        df.caiman.remove_item(uuid, remove_data=remove_data, safe_removal=safe_removal)
+    df.caiman.save_to_disk()
+    return df
+
+
+def validate_path(path: Union[str, Path]):
+    if not regex.match("^[A-Za-z0-9@\/\\\:._-]*$", str(path)):
+        raise ValueError(
+            "Paths must only contain alphanumeric characters, "
+            "hyphens ( - ), underscores ( _ ) or periods ( . )"
+        )
+    return path
+
+
+def drop_duplicates(df):
+    """
+    Remove duplicate items from a batch DataFrame.
+
+    Parameters
+    ----------
+    df : pandas.DataFrame
+        The batch DataFrame to remove duplicates from.
+
+    Returns
+    -------
+    None
+
+    """
+    import hashlib
+    df["hash"] = df.apply(lambda row: hashlib.sha256(row.mcorr.get_output().tobytes()).hexdigest(), axis=1)
+    uuids_to_remove = []
+    for _, group in df.groupby("hash"):
+        if len(group) > 1:
+            for idx in group.index[1:]:
+                uuid = df.loc[idx, "uuid"]
+                uuids_to_remove.append(uuid)
+    if not uuids_to_remove:
+        print("No duplicates found.")
+        return
+    for uuid in uuids_to_remove:
+        print(f"Removing duplicate item {uuid}.")
+        df.caiman.remove_item(uuid, remove_data=True, safe_removal=False)
+    df.drop(columns="hash", inplace=True)
+    df.caiman.save_to_disk()
+    return df
+
+
+def get_batch_from_path(batch_path):
+    """
+    Load or create a batch at the given batch_path.
+    """
+    try:
+        df = mc.load_batch(batch_path)
+        print(f"Batch found at {batch_path}")
+    except (IsADirectoryError, FileNotFoundError):
+        print(f"Creating batch at {batch_path}")
+        df = mc.create_batch(batch_path)
+    return df

lbm_caiman_python/collation.py

@@ -0,0 +1,125 @@
+from pathlib import Path
+import numpy as np
+from scipy.sparse import hstack
+import copy
+import scipy.signal
+import json
+
+
+def combine_z_planes(results: dict):
+    """
+    Combines all z-planes in the results dictionary into a single estimates object.
+
+    Parameters
+    ----------
+    results (dict): Dictionary with estimates for each z-plane.
+
+    Returns
+    -------
+    estimates.Estimates: Combined estimates for all z-planes.
+    """
+    from caiman.source_extraction.cnmf import estimates
+    keys = sorted(results.keys())
+    e_list = [results[k].estimates for k in keys]
+
+    # Initialize lists to collect components
+    A_list = []
+    b_list = []
+    C_list = []
+    f_list = []
+    R_list = []
+
+    for e in e_list:
+        A_list.append(e.A)
+        b_list.append(e.b)
+        C_list.append(e.C)
+        f_list.append(e.f)
+        R_list.append(e.R)
+
+    # Combine the components
+    A_new = hstack(A_list).tocsr()
+    b_new = np.concatenate(b_list, axis=0)
+    C_new = np.concatenate(C_list, axis=0)
+    f_new = np.concatenate(f_list, axis=0)
+    R_new = np.concatenate(R_list, axis=0)
+
+    # Assuming all z-planes have the same spatial dimensions
+    dims_new = e_list[0].dims  # e.g., (height, width)
+
+    # Create new estimates object
+    e_new = estimates.Estimates(
+        A=A_new,
+        C=C_new,
+        b=b_new,
+        f=f_new,
+        R=R_new,
+        dims=dims_new
+    )
+
+    return e_new
+
+
+def calculate_interplane_shifts(volume, n_planes, params, json_logger=None):
+    """
+    """
+    interplane_shifts = np.zeros((n_planes - 1, 2), dtype=int)
+    accumulated_shifts = np.zeros((n_planes - 1, 2), dtype=int)
+
+    for i_plane in range(n_planes - 1):
+        im1_copy = copy.deepcopy(volume[0, :, :, i_plane])
+        im2_copy = copy.deepcopy(volume[0, :, :, i_plane + 1])
+
+        # Removing NaNs
+        nonan_mask = np.stack(
+            (~np.isnan(im1_copy), ~np.isnan(im2_copy)), axis=0
+        )
+        nonan_mask = np.all(nonan_mask, axis=0)
+        coord_nonan_pixels = np.where(nonan_mask)
+        min_x, max_x = np.min(coord_nonan_pixels[0]), np.max(coord_nonan_pixels[0])
+        min_y, max_y = np.min(coord_nonan_pixels[1]), np.max(coord_nonan_pixels[1])
+
+        im1_nonan = im1_copy[min_x:max_x + 1, min_y:max_y + 1]
+        im2_nonan = im2_copy[min_x:max_x + 1, min_y:max_y + 1]
+
+        # Normalize intensities
+        im1_nonan -= np.min(im1_nonan)
+        im2_nonan -= np.min(im2_nonan)
+
+        # Cross-correlation
+        cross_corr_img = scipy.signal.fftconvolve(
+            im1_nonan, im2_nonan[::-1, ::-1], mode="same"
+        )
+
+        # Find correlation peak
+        corr_img_peak_x, corr_img_peak_y = np.unravel_index(
+            np.argmax(cross_corr_img), cross_corr_img.shape
+        )
+        self_corr_peak_x, self_corr_peak_y = [
+            dim / 2 for dim in cross_corr_img.shape
+        ]
+        interplane_shift = [
+            corr_img_peak_x - self_corr_peak_x,
+            corr_img_peak_y - self_corr_peak_y,
+        ]
+
+        interplane_shifts[i_plane] = copy.deepcopy(interplane_shift)
+        accumulated_shifts[i_plane] = np.sum(
+            interplane_shifts, axis=0, dtype=int
+        )
+
+    # Normalize shifts to start at zero
+    min_accumulated_shift = np.min(accumulated_shifts, axis=0)
+    for xy in range(2):
+        accumulated_shifts[:, xy] -= min_accumulated_shift[xy]
+
+    # Optional JSON logging
+    if params.get("json_logging") and json_logger:
+        json_logger.info(
+            json.dumps({"accumulated_shifts": accumulated_shifts.tolist()})
+        )
+
+    return accumulated_shifts
+
+
+if __name__ == "__main__":
+    pass