nanoobjectdetection 2.0.0__py3-none-any.whl

nanoobjectdetection/legacy_config.py

@@ -0,0 +1,157 @@
+"""1.0-compatible configuration for the 2.0 chunked/GPU pipeline."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from .config import AnalysisConfig, Roi
+
+
+def _number(value, default):
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _flag(value) -> bool:
+    return str(value).strip().lower() in {"1", "1.0", "true", "auto", "manual"}
+
+
+def _kernel_sigma(value):
+    """1.0 KernelSize: "auto" (PSF width) or the Gaussian sigma in pixels."""
+    if str(value).strip().lower() == "auto":
+        return None
+    return _number(value, None)
+
+
+def _saturation(value):
+    """Return the saturation threshold, or None for 'No Saturation'."""
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def load_parameter_file(parameter_file: str | Path) -> tuple[AnalysisConfig, dict]:
+    """Translate a 1.0 ``parameter.json`` into an equivalent 2.0 config."""
+    parameter_path = Path(parameter_file)
+    settings = json.loads(parameter_path.read_text(encoding="utf-8"))
+    return load_parameter_settings(settings, parameter_path)
+
+
+def load_parameter_settings(
+    settings: dict, parameter_path: str | Path = "."
+) -> tuple[AnalysisConfig, dict]:
+    """Translate in-memory 1.0 settings into an equivalent 2.0 config."""
+    parameter_path = Path(parameter_path)
+    exp = settings["Exp"]
+    fiber = settings["Fiber"]
+    files = settings["File"]
+    roi_settings = settings["ROI"]
+    preprocessing = settings["PreProcessing"]
+    find = settings["Find"]
+    link = settings["Link"]
+    stationary = settings["StationaryObjects"]
+    split = settings["Split"]
+    drift = settings["Drift"]
+    msd = settings["MSD"]
+    help_settings = settings.get("Help", {})
+    single = settings.get("SingleParticle", {})
+    post = settings["PostProcessing"]
+
+    frame_stop = None
+    if roi_settings.get("frame_max", 0) and roi_settings["frame_max"] < 10**6:
+        frame_stop = int(roi_settings["frame_max"])
+    roi = Roi(
+        x_min=int(roi_settings.get("x_min", 0)),
+        x_max=None if roi_settings.get("x_max", 10**9) >= 10**9 else int(roi_settings["x_max"]),
+        y_min=int(roi_settings.get("y_min", 0)),
+        y_max=None if roi_settings.get("y_max", 10**9) >= 10**9 else int(roi_settings["y_max"]),
+    )
+    hindrance_aware = bool(msd.get("HindrancePositionAware", 0))
+    hindrance_mode = "position" if hindrance_aware else (
+        "legacy" if bool(msd.get("EstimateHindranceFactor", 0)) else "none"
+    )
+    max_diameter = _number(post.get("MaxDiameter", 0), 0.0) or None
+    max_trajectory = _number(split.get("Max_traj_length", 0), 0.0) or None
+    if max_trajectory and max_trajectory >= 10**8:
+        max_trajectory = None
+
+    input_file = Path(files["data_file_name"])
+    if not input_file.is_file():
+        candidate = parameter_path.parent / input_file.name
+        if candidate.is_file():
+            input_file = candidate
+    output_dir = Path(settings["Plot"].get("SaveFolder", parameter_path.parent))
+    if not output_dir.is_absolute() or not output_dir.exists():
+        output_dir = parameter_path.parent / "analysis" / "pipeline_2_0"
+
+    config = AnalysisConfig(
+        input_file=input_file,
+        output_dir=output_dir,
+        frame_start=int(roi_settings.get("frame_min", 0)),
+        frame_stop=frame_stop,
+        roi=roi,
+        fps=_number(exp["fps"], 100.0),
+        pixel_size_um=_number(exp["Microns_per_pixel"], 1.0),
+        wavelength_nm=_number(exp.get("lambda"), 532.0),
+        numerical_aperture=_number(exp.get("NA"), 0.25),
+        exposure_time_s=_number(exp.get("ExposureTime"), 0.0007),
+        temperature_k=_number(exp.get("Temperature"), 293.15),
+        viscosity_pa_s=_number(exp.get("Viscosity"), 0.001) * 1e12,
+        channel_diameter_nm=_number(fiber.get("TubeDiameter_nm"), 0.0) or None,
+        channel_center_y_px=(
+            _number(fiber.get("Center_y_px"), 0.0)
+            if str(fiber.get("Center_y_px", "")).lower() != "unknown"
+            else None
+        ),
+        auto_channel_center=str(fiber.get("Center_y_px", "")).lower() == "unknown",
+        hindrance_mode=hindrance_mode,
+        remove_camera_offset=bool(preprocessing.get("Remove_CameraOffset", 1)),
+        remove_laser_fluctuation=bool(
+            preprocessing.get("Remove_Laserfluctuation", 0)
+        ),
+        remove_static_background=bool(preprocessing.get("Remove_StaticBackground", 1)),
+        enhance_snr=bool(preprocessing.get("EnhanceSNR", 0)),
+        snr_kernel_sigma_px=_kernel_sigma(preprocessing.get("KernelSize", "auto")),
+        zncc_min=_number(preprocessing.get("ZNCC_min"), 0.7),
+        auto_minmass=_flag(help_settings.get("Bead brightness", 0)),
+        auto_diameter=_flag(help_settings.get("Bead size", 0)),
+        minmass_try_frames=int(_number(help_settings.get("TryFrames"), 30)),
+        red_x_mode=str(msd.get("Estimate_X", "Exp")).lower(),
+        single_particle=bool(single.get("Enabled", 0)),
+        single_particle_keep=str(single.get("Keep", "longest")).lower(),
+        stitch_max_gap_frames=int(single.get("Max gap frames", 300)),
+        stitch_max_distance_px=float(single.get("Max distance px", 15.0)),
+        stitch_distance_growth_px=float(single.get("Distance growth px per sqrt frame", 0.5)),
+        stitch_max_mass_ratio=float(single.get("Max mass ratio", 3.0)),
+        stitch_min_fragment_frames=int(single.get("Min fragment frames", 3)),
+        background_method="legacy_trimmed_mean",
+        intensity_normalization="global",
+        particle_diameter=int(find.get("tp_diameter", 7)),
+        particle_minmass=_number(find.get("tp_minmass"), 358.0),
+        particle_separation=int(find.get("tp_separation", 20)),
+        particle_percentile=_number(find.get("tp_percentile"), 20.0),
+        minimum_trajectory_frames=int(link.get("Min_tracking_frames", 5)),
+        minimum_trajectory_frames_before_drift=int(link.get("Min tracking frames before drift", 10)),
+        drift_enabled=bool(drift.get("Apply", 1)),
+        msd_max_lag_frames=max(2, int(msd.get("lagtimes_max", 3))),
+        msd_lag_auto=bool(msd.get("Amount lagtimes auto", 0)),
+        analyze_stationary_particles=bool(stationary.get("Analyze fixed spots", 1)),
+        stationary_search_range=_number(link.get("Max displacement fix"), 0.3),
+        stationary_minimum_frames=int(link.get("Dwell time stationary objects", 100)),
+        no_go_distance=_number(stationary.get("Min distance to stationary object"), 20.0),
+        split_intensity_jumps=bool(split.get("IntensityJump", 0)),
+        max_relative_intensity_step=_number(split.get("Max rel median intensity step"), 0.906),
+        maximum_trajectory_frames=None if max_trajectory is None else int(max_trajectory),
+        keep_short_trajectory_tail=bool(split.get("Max_traj_length_keep_tail", 0)),
+        ks_method=str(msd.get("KSMethod", "legacy")).lower(),
+        minimum_ks_significance=_number(msd.get("KSMinSignificance"), 0.01),
+        saturated_pixel_value=_saturation(find.get("SaturatedPixelValue")),
+        apply_quality_filter=bool(msd.get("CheckBrownianMotion", 1)),
+        maximum_diameter_nm=max_diameter,
+        force_ultra_uniform=bool(post.get("ForceUltraUniform", 0)),
+    )
+    return config, settings

nanoobjectdetection/legacy_outputs.py

@@ -0,0 +1,121 @@
+"""Write CSV files with the file names, column order and columns used by 1.0."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+OBJECT_COLUMNS = [
+    "y", "x", "mass", "size", "ecc", "signal", "raw_mass", "ep", "frame",
+    "abstime", "saturated",
+]
+TRAJECTORY_COLUMNS = [
+    "frame", "y", "x", "mass", "size", "ecc", "signal", "raw_mass", "ep",
+    "abstime", "saturated", "particle",
+]
+SIZE_COLUMNS = [
+    "particle", "diameter", "diffusion", "diffusion std", "ep", "red_x",
+    "signal", "mass", "rawmass_mean", "rawmass_median", "rawmass_max",
+    "max step", "first frame", "traj length", "valid frames", "size", "ecc",
+    "stat_sign", "true_particle", "x in first frame", "y in first frame",
+]
+
+
+def _ordered(frame: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
+    return frame.reindex(columns=columns)
+
+
+def write_legacy_table(frame: pd.DataFrame, path: Path, columns: list[str]) -> None:
+    _ordered(frame, columns).to_csv(path, index=False)
+
+
+def legacy_trajectories(corrected: pd.DataFrame) -> pd.DataFrame:
+    """Return a trajectory table whose x/y are the drift-corrected positions."""
+    result = corrected.copy()
+    if "x_corrected" in result:
+        result["x"] = result["x_corrected"]
+        result["y"] = result["y_corrected"]
+    return _ordered(result, TRAJECTORY_COLUMNS)
+
+
+def legacy_sizes_table(
+    corrected: pd.DataFrame,
+    moving: pd.DataFrame,
+    estimates: pd.DataFrame,
+    quality: pd.DataFrame,
+    fps: float,
+) -> pd.DataFrame:
+    """Build the per-particle table written as ``sizes_df_lin_*.csv`` by 1.0."""
+    valid = estimates[estimates["diameter_nm"].notna()].copy()
+    if valid.empty:
+        return pd.DataFrame(columns=SIZE_COLUMNS)
+    stat = {}
+    if len(quality) and "ks_significance" in quality:
+        stat = dict(zip(quality["particle"], quality["ks_significance"]))
+    start = (
+        moving.sort_values("frame").groupby("particle").first()
+        if len(moving)
+        else pd.DataFrame()
+    )
+    tracks = {particle: group for particle, group in corrected.groupby("particle")}
+    lagtime = 1.0 / fps
+    rows = []
+    for _, row in valid.iterrows():
+        particle = row["particle"]
+        track = tracks.get(particle)
+        if track is None:
+            continue
+        track = track.sort_values("frame")
+        red_x = row["intercept_um2"] / (lagtime * row["slope_um2_s"])
+        red_x = max(red_x, -0.5)
+        valid_frames = row["valid_frames"]
+        error = np.sqrt(
+            2.0 / (valid_frames - 1) * (1.0 + 2.0 * np.sqrt(1.0 + 2.0 * red_x))
+        )
+        first = start.loc[particle] if particle in start.index else track.iloc[0]
+        rows.append(
+            {
+                "particle": int(particle),
+                "diameter": row["diameter_nm"],
+                "diffusion": row["diffusion_um2_s"],
+                "diffusion std": row["diffusion_um2_s"] * error,
+                "ep": track["ep"].mean(),
+                "red_x": red_x,
+                "signal": track["signal"].mean(),
+                "mass": track["mass"].mean(),
+                "rawmass_mean": track["raw_mass"].mean(),
+                "rawmass_median": track["raw_mass"].median(),
+                "rawmass_max": track["raw_mass"].max(),
+                "max step": 0.0,
+                "first frame": float(track["frame"].iloc[0]),
+                "traj length": float(track["frame"].iloc[-1] - track["frame"].iloc[0] + 1),
+                "valid frames": float(valid_frames),
+                "size": track["size"].mean(),
+                "ecc": track["ecc"].mean(),
+                "stat_sign": stat.get(particle, np.nan),
+                "true_particle": float(particle),
+                "x in first frame": first["x"],
+                "y in first frame": first["y"],
+            }
+        )
+    return pd.DataFrame(rows, columns=SIZE_COLUMNS)
+
+
+def export_legacy_aliases(output_dir: Path) -> None:
+    """Write 1.0-compatible detection and trajectory CSVs without removing 2.0 files."""
+    detections = output_dir / "detections.csv"
+    if detections.is_file():
+        table = pd.read_csv(detections)
+        write_legacy_table(table, output_dir / "obj_all.csv", OBJECT_COLUMNS)
+        write_legacy_table(
+            table, output_dir / "obj_moving_no_go_area.csv", OBJECT_COLUMNS
+        )
+    for source, target in (
+        ("stationary_trajectories.csv", "traj_fixed.csv"),
+        ("trajectories.csv", "traj_moving.csv"),
+    ):
+        path = output_dir / source
+        if path.is_file():
+            write_legacy_table(pd.read_csv(path), output_dir / target, TRAJECTORY_COLUMNS)

nanoobjectdetection/localization.py

@@ -0,0 +1,73 @@
+"""Particle localization using a direct, optional Trackpy dependency."""
+
+from __future__ import annotations
+
+import pandas as pd
+import numpy as np
+import os
+import sys
+from concurrent.futures import ProcessPoolExecutor
+
+def _locate_frame(args):
+    import trackpy as tp
+
+    frame, diameter, minmass, separation, percentile, preprocess = args
+    return tp.locate(
+        frame,
+        diameter,
+        minmass=minmass,
+        separation=separation,
+        percentile=percentile,
+        preprocess=preprocess,
+        engine="python",
+    )
+
+
+def locate(
+    frames: np.ndarray,
+    diameter: int,
+    minmass: float,
+    separation: int,
+    percentile: float,
+    preprocess: bool = True,
+    workers: int = 1,
+    executor: ProcessPoolExecutor | None = None,
+) -> pd.DataFrame:
+    """Locate particles in one block and preserve local frame numbers.
+
+    ``workers`` > 1 spreads frames over processes; 0 uses all cores.
+    """
+    try:
+        import trackpy as tp  # noqa: F401
+    except ImportError as exc:
+        raise RuntimeError(
+            "Particle localization requires 'trackpy', but it is not "
+            f"installed in the active interpreter ({sys.executable}). "
+            "Install it with: python -m pip install trackpy"
+        ) from exc
+
+    frame_list = np.asarray(frames)
+    jobs = [
+        (frame, diameter, minmass, separation, percentile, preprocess)
+        for frame in frame_list
+    ]
+    workers = (os.cpu_count() or 1) if workers == 0 else workers
+    if workers > 1 and len(jobs) > 1:
+        # A few large batches per worker keep process start-up and pickling overhead low
+        batch = max(1, -(-len(jobs) // (workers * 4)))
+        if executor is not None:
+            results = list(executor.map(_locate_frame, jobs, chunksize=batch))
+        else:
+            with ProcessPoolExecutor(max_workers=workers) as pool:
+                results = list(pool.map(_locate_frame, jobs, chunksize=batch))
+    else:
+        results = [_locate_frame(job) for job in jobs]
+    detections = []
+    for frame_number, found in enumerate(results):
+        if not found.empty:
+            found = found.copy()
+            found["frame"] = frame_number
+            detections.append(found)
+    if not detections:
+        return pd.DataFrame(columns=["x", "y", "frame"])
+    return pd.concat(detections, ignore_index=True)

nanoobjectdetection/msd.py

@@ -0,0 +1,215 @@
+"""Mean-squared-displacement and diffusion estimation."""
+
+from __future__ import annotations
+
+import warnings
+
+import numpy as np
+import pandas as pd
+
+from .progress import progress
+
+
+def calculate_msd(
+    trajectory: pd.DataFrame,
+    dimension: str = "2d",
+    max_lag: int | None = None,
+    pixel_size_um: float = 1.0,
+    fps: float = 300.0,
+) -> pd.DataFrame:
+    """Calculate MSD in µm² for one trajectory."""
+    if dimension not in {"x", "y", "2d"}:
+        raise ValueError("dimension must be 'x', 'y', or '2d'")
+    x_col = "x_corrected" if "x_corrected" in trajectory else "x"
+    y_col = "y_corrected" if "y_corrected" in trajectory else "y"
+    values = trajectory.sort_values("frame").drop_duplicates("frame")
+    frames = values["frame"].to_numpy(int)
+    x = values[x_col].to_numpy(float) * pixel_size_um
+    y = values[y_col].to_numpy(float) * pixel_size_um
+    n = len(values)
+    # Short lags are less affected by channel confinement and residual drift.
+    limit = min(
+        int(frames[-1] - frames[0]) if n else 0,
+        max_lag or min(max(1, n // 4), 20),
+    )
+    rows = []
+    for lag in range(1, limit + 1):
+        right = np.searchsorted(frames, frames + lag)
+        valid = right < n
+        valid &= frames[np.minimum(right, n - 1)] == frames + lag
+        left = np.nonzero(valid)[0]
+        if not len(left):
+            continue
+        dx = x[right[left]] - x[left]
+        dy = y[right[left]] - y[left]
+        squared = dx * dx if dimension == "x" else dy * dy if dimension == "y" else dx * dx + dy * dy
+        rows.append({"lag": lag, "time_s": lag / fps, "msd_um2": float(np.mean(squared)), "pairs": len(squared)})
+    return pd.DataFrame(rows)
+
+
+def estimate_diffusion(msd: pd.DataFrame, dimension: str = "2d") -> dict[str, float]:
+    """Fit MSD = intercept + 2*d*D*time and return D in µm²/s."""
+    if len(msd) < 2:
+        raise ValueError("At least two MSD points are required")
+    time = msd["time_s"].to_numpy(dtype=float)
+    values = msd["msd_um2"].to_numpy(dtype=float)
+    centered_time = time - float(np.mean(time))
+    centered_values = values - float(np.mean(values))
+    denominator = float(np.dot(centered_time, centered_time))
+    if denominator == 0:
+        raise ValueError("MSD fit requires at least two distinct lag times")
+    slope = float(np.dot(centered_time, centered_values) / denominator)
+    intercept = float(np.mean(values) - slope * np.mean(time))
+    factor = 4.0 if dimension == "2d" else 2.0
+    return {
+        "diffusion_um2_s": float(slope / factor),
+        "slope_um2_s": float(slope),
+        "intercept_um2": float(intercept),
+        "fit_points": float(len(msd)),
+    }
+
+
+def _ols(time: np.ndarray, values: np.ndarray) -> tuple[float, float]:
+    slope, intercept = np.polyfit(time, values, 1)
+    return float(slope), float(intercept)
+
+
+def _block_msd(positions: list[np.ndarray], lag: int) -> tuple[float, int]:
+    """Legacy MSD at one lag: mean of non-overlapping block means of the squared displacements."""
+    total = len(positions[0])
+    squared = np.zeros(total)
+    for pos in positions:
+        displacement = np.full(total, np.nan)
+        displacement[lag:] = pos[lag:] - pos[:-lag]
+        squared = squared + displacement**2
+    pairs = int(np.count_nonzero(~np.isnan(squared)))
+    blocks = (total - lag) // lag + 1
+    if blocks < 1:
+        return float("nan"), pairs
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", category=RuntimeWarning)
+        block_means = np.nanmean(squared[: blocks * lag].reshape(blocks, lag), axis=1)
+        return float(np.nanmean(block_means)), pairs
+
+
+def estimate_diffusion_auto_lag(
+    trajectory: pd.DataFrame,
+    dimension: str,
+    pixel_size_um: float,
+    fps: float,
+    max_iterations: int = 10,
+    fixed_red_x: float | None = None,
+) -> dict[str, float] | None:
+    """Reproduce the 1.0 MSD fit with automatic lag selection (Michalet 2012, appendix B).
+
+    Follows CalcDiameter.OptimizeMSD of 1.0 step by step, including its details:
+    block-averaged MSD, the slope being fitted with the second entry of the lag pair
+    returned by UpdateP_Min (the offset with the first), and convergence on that entry.
+    Returns None if the track is too short, has too many holes or does not converge.
+    """
+    values = trajectory.sort_values("frame").drop_duplicates("frame")
+    n_rows = len(values)
+    frames = values["frame"].to_numpy(int)
+    x_col = "x_corrected" if "x_corrected" in values else "x"
+    y_col = "y_corrected" if "y_corrected" in values else "y"
+    total = int(frames[-1] - frames[0]) + 1 if n_rows else 0
+    positions = []
+    for use, column in ((dimension in ("x", "2d"), x_col), (dimension in ("y", "2d"), y_col)):
+        if use:
+            pos = np.full(total, np.nan)
+            pos[frames - frames[0]] = values[column].to_numpy(float) * pixel_size_um
+            positions.append(pos)
+    if not positions:
+        raise ValueError("dimension must be 'x', 'y', or '2d'")
+    start = min(max(int(np.ceil(n_rows / 10)), 2), 100)
+    lags = [start, start]
+    factor = 4.0 if dimension == "2d" else 2.0
+    t_frame = 1.0 / fps
+    for counter in range(max_iterations + 1):
+        lag_max = max(lags)
+        if total < lag_max:
+            return None
+        msd, pairs = zip(*(_block_msd(positions, lag) for lag in range(1, lag_max + 1)))
+        if pairs[lag_max - 1] < lag_max + 1 or np.isnan(msd).any():
+            return None
+        pairs_lag1 = pairs[0]
+        time = np.arange(1, lag_max + 1) / fps
+        msd = np.asarray(msd)
+        slope, _ = _ols(time[: lags[1]], msd[: lags[1]])
+        _, intercept = _ols(time[: lags[0]], msd[: lags[0]])
+        l_a = 3 + max(4.5 * pairs_lag1**0.4 - 8.5, 0.0) ** 1.2
+        l_b = 0.8 + 0.564 * pairs_lag1
+        if slope < 0:
+            new = [int(l_a), int(l_b)]
+        else:
+            if fixed_red_x is not None:
+                red_x = max(fixed_red_x, 0.0)
+            else:
+                red_x = max(intercept / (t_frame * slope), 0.0) if slope else 0.0
+            f_a = 2 + 1.6 * red_x**0.51
+            p_a = max(np.floor(f_a * l_a / (f_a**3 + l_a**3) ** (1 / 3)), 2)
+            f_b = 2 + 1.35 * red_x**0.6
+            p_b = max(np.floor(min(l_b, f_b * l_b / (f_b**3 + l_b**3) ** (1 / 3))), 2)
+            new = [int(p_b), int(p_a)]
+        if counter == max_iterations:
+            return None
+        if new[1] == lags[1]:
+            return {
+                "diffusion_um2_s": slope / factor,
+                "slope_um2_s": slope,
+                "intercept_um2": intercept,
+                "fit_points": float(lags[1]),
+                "msd_pairs": int(sum(pairs)),
+                "valid_frames": int(pairs_lag1),
+            }
+        lags = new
+    return None
+
+
+def estimate_all(
+    trajectories: pd.DataFrame,
+    auto_lag: bool = False,
+    fixed_red_x: float | None = None,
+    **kwargs,
+) -> pd.DataFrame:
+    rows = []
+    dimension = kwargs.get("dimension", "2d")
+    groups = trajectories.groupby("particle")
+    for particle, trajectory in progress(
+        groups,
+        desc="Estimating MSD and particle size",
+        total=trajectories["particle"].nunique(),
+        unit="particle",
+    ):
+        if auto_lag:
+            row = estimate_diffusion_auto_lag(
+                trajectory,
+                dimension,
+                kwargs.get("pixel_size_um", 1.0),
+                kwargs.get("fps", 300.0),
+                fixed_red_x=fixed_red_x,
+            )
+            if row is None:
+                continue
+        else:
+            msd = calculate_msd(trajectory, **kwargs)
+            if len(msd) < 2:
+                continue
+            row = estimate_diffusion(msd, dimension)
+            row["msd_pairs"] = int(msd["pairs"].sum())
+            row["valid_frames"] = int(msd["pairs"].iloc[0])
+        row.update(
+            {
+                "particle": particle,
+                "trajectory_frames": len(trajectory),
+                "mean_brightness": float(
+                    trajectory["mass"].mean()
+                    if "mass" in trajectory
+                    else trajectory["signal"].mean()
+                    if "signal" in trajectory
+                    else float("nan")
+                ),
+            }
+        )
+        rows.append(row)
+    return pd.DataFrame(rows)