pycat-napari 1.5.331__tar.gz → 1.5.332__tar.gz

{pycat_napari-1.5.331 → pycat_napari-1.5.332}/CHANGELOG.md

@@ -4,6 +4,42 @@ All notable changes to PyCAT-Napari will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.5.332] - 2026-07-10
+### Fixed — VPT classifier green↔yellow flicker on bright, well-matched beads
+- **A bright, high-NCC bead no longer flips singlet↔out_of_plane frame-to-frame.**
+  The out_of_plane (yellow) class used a per-frame amplitude/SNR percentile, so
+  when the bead population is uniformly low-quality a genuinely good bead sitting
+  near the moving percentile line was demoted ~a quarter of the time — driven
+  mainly by a `low-SNR OR` clause that could yellow a bead whose amplitude was
+  fine. Fix: (1) require the AMPLITUDE to actually be low for the dim class (SNR
+  is now only a secondary confirmation, never demotes a bright bead on its own),
+  and (2) add a high-NCC singlet guard (NCC ≥ 0.80) so a well-matched bead is
+  immune to the dim percentile. Verified on real data: the previously-flickering
+  bead (amp~164, NCC~0.94) is now singlet in 1000/1000 frames (was ~76%), while a
+  genuinely dim bead (amp~75, NCC~0.76) correctly STAYS out_of_plane — the garbage
+  rejection (NCC floor) and aggregate class are untouched. This preserves the
+  hard-won hot-pixel/ring/noise rejection while stopping the erroneous demotion of
+  real beads.
+
+### Added — MSD lag-window fit gate (hardware-defensible fit bounds)
+- `fit_anomalous_diffusion` now computes a **defensible MSD lag window** bounded
+  by the frame rate (high-frequency cutoff = frame interval) and the acquisition
+  duration (low-frequency cutoff), and by default confines the D/α fit to it.
+  Fitting outside this band (only sub-second lags, dominated by the localization
+  floor, or out toward the full duration, where a handful of pairs dominate)
+  produces a wrong D/α. Exposed under the VPT "Show advanced fit / moduli options"
+  expander:
+  - **Upper-lag rule** (user-selectable, with tooltips): *Fraction of track length*
+    (default, 0.25), *Fixed frequency window* (set the upper lag in seconds), or
+    *Minimum independent pairs* (keep a lag only while ≥ N independent tracks span
+    it).
+  - **"Confine fit to scientifically defensible bounds"** toggle (default ON) —
+    clips the fit to the window; turn off to fit the full range at your own risk.
+  - The gate **warns, never blocks**: if the acquisition can't cover the requested
+    window (too-short clip), it emits a clear warning and falls back gracefully.
+  Validated on real data across all three rules + the confine toggle + the
+  too-short-data warning path.
+
 ## [1.5.331] - 2026-07-10
 ### Added — Evans (2009) viscoelastic moduli + bootstrap confidence bands
 - **G′/G″ (storage/loss moduli) now use the Evans et al. (2009) direct

{pycat_napari-1.5.331 → pycat_napari-1.5.332}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pycat-napari
-Version: 1.5.331
+Version: 1.5.332
 Summary: Python Condensate Analysis Toolbox - A napari-based tool for biomolecular condensate analysis
 Project-URL: Homepage, https://github.com/BanerjeeLab-repertoire/pycat-napari
 Project-URL: Repository, https://github.com/BanerjeeLab-repertoire/pycat-napari

{pycat_napari-1.5.331 → pycat_napari-1.5.332}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "pycat-napari"
-version = "1.5.331"
+version = "1.5.332"
 description = "Python Condensate Analysis Toolbox - A napari-based tool for biomolecular condensate analysis"
 readme = "README.md"
 requires-python = ">=3.12,<3.13"

{pycat_napari-1.5.331 → pycat_napari-1.5.332}/src/pycat/toolbox/condensate_physics_tools.py

@@ -221,15 +221,55 @@ def fit_anomalous_diffusion(
     msd_df: pd.DataFrame,
     max_lag_fit: int = None,
     fit_localization_offset: bool = True,
+    frame_interval_s: float = None,
+    upper_lag_rule: str = 'fraction',
+    upper_lag_fraction: float = 0.25,
+    upper_lag_fixed_s: float = None,
+    min_independent_pairs: int = 10,
+    confine_to_defensible_bounds: bool = True,
 ) -> dict:
     """
     Fit MSD(τ) = 4D·τ^α (anomalous diffusion model) using log-log regression.
 
+    LAG-WINDOW FIT GATE
+    -------------------
+    The reliable MSD lag window is bounded by hardware on both ends:
+
+    * **High-frequency cutoff = frame rate.** The shortest resolvable lag is one
+      frame interval; nothing faster is sampled.
+    * **Low-frequency cutoff = acquisition duration.** At long lags there are very
+      few independent displacement pairs, so the MSD becomes unreliable well
+      before the full record length.
+
+    Fitting outside this band (e.g. only the first fraction of a second, where the
+    curve is dominated by the localization-noise floor, or out toward the full
+    duration, where a handful of pairs dominate) produces a wrong D/α. The gate
+    computes a defensible ``[lag_lo, lag_hi]`` and, when
+    ``confine_to_defensible_bounds`` is on (default), fits only within it. It
+    **warns rather than blocks** when the data can't cover the requested window.
+
     Parameters
     ----------
-    msd_df : output of compute_msd()
-    max_lag_fit : number of lag points to include in fit.
-        Default: all points with n_pairs > 10.
+    msd_df : output of compute_msd() (needs lag_s, msd_um2, n_pairs; n_tracks used
+        for the min-pairs rule when present).
+    max_lag_fit : legacy cap on the number of head lags (kept for back-compat;
+        applied before the window gate if given).
+    fit_localization_offset : fit the +4σ_loc² localization-offset term.
+    frame_interval_s : seconds per frame; sets the high-frequency cutoff
+        (lag_lo = frame_interval_s). If None, inferred from the smallest lag.
+    upper_lag_rule : how to set the low-frequency (upper-lag) cutoff:
+        * 'fraction'    — lag_hi = upper_lag_fraction × (max track duration).
+                          The standard convention; conservative.
+        * 'fixed'       — lag_hi = upper_lag_fixed_s (a hardware-defensible band,
+                          e.g. matching routine lab practice).
+        * 'min_pairs'   — keep lags while ≥ min_independent_pairs independent
+                          tracks span them (statistically principled; adapts to
+                          how many/how long the tracks are).
+    upper_lag_fraction : fraction for the 'fraction' rule (default 0.25).
+    upper_lag_fixed_s : upper lag (s) for the 'fixed' rule.
+    min_independent_pairs : threshold for the 'min_pairs' rule.
+    confine_to_defensible_bounds : if True (default), clip the fit to the computed
+        window; if False, fit the full available range (at the user's risk).
 
     Returns
     -------
@@ -242,15 +282,67 @@ def fit_anomalous_diffusion(
         fit_msd         : fitted MSD values (array)
         log_log_slope   : raw slope from log-log regression (= alpha)
         log_log_intercept : raw intercept (log(4D))
+        fit_window_s    : (lag_lo, lag_hi) the defensible window used
+        fit_window_warning : str or None — set when data can't cover the window
     """
     df = msd_df[msd_df['n_pairs'] > 5].copy()
     if max_lag_fit is not None:
         df = df.head(max_lag_fit)
+
+    # ── Lag-window fit gate ──────────────────────────────────────────────────
+    window_warning = None
+    lag_lo = lag_hi = None
+    if 'lag_s' in df.columns and len(df):
+        all_lags = df['lag_s'].values.astype(float)
+        # High-frequency cutoff = one frame interval.
+        lag_lo = float(frame_interval_s) if (frame_interval_s and frame_interval_s > 0) \
+            else float(np.min(all_lags))
+        # Low-frequency (upper-lag) cutoff per the chosen rule.
+        max_lag_available = float(np.max(all_lags))
+        rule = (upper_lag_rule or 'fraction').lower()
+        if rule == 'fixed' and upper_lag_fixed_s and upper_lag_fixed_s > 0:
+            lag_hi = float(upper_lag_fixed_s)
+        elif rule == 'min_pairs' and 'n_tracks' in df.columns:
+            ok = df[df['n_tracks'] >= int(min_independent_pairs)]
+            lag_hi = float(ok['lag_s'].max()) if len(ok) else lag_lo
+        else:  # 'fraction' (default)
+            lag_hi = float(upper_lag_fraction) * max_lag_available
+
+        # Sanity + coverage warnings (warn, never block).
+        if lag_hi <= lag_lo:
+            window_warning = (
+                f"Requested lag window collapses (lag_lo={lag_lo:.3g}s ≥ "
+                f"lag_hi={lag_hi:.3g}s). The acquisition is too short, or the "
+                f"upper-lag rule is too strict, to define a fit band. Fitting the "
+                f"full available range instead.")
+            lag_hi = max_lag_available
+        elif lag_hi > max_lag_available + 1e-12:
+            window_warning = (
+                f"Requested upper lag ({lag_hi:.3g}s) exceeds the longest "
+                f"available lag ({max_lag_available:.3g}s): the acquisition "
+                f"duration is too short to reach the low-frequency cutoff, so "
+                f"G(τ)/viscosity may be under-resolved at long lags.")
+            lag_hi = max_lag_available
+
+        if confine_to_defensible_bounds:
+            gated = df[(df['lag_s'] >= lag_lo - 1e-12)
+                       & (df['lag_s'] <= lag_hi + 1e-12)]
+            if len(gated) >= 3:
+                df = gated
+            else:
+                window_warning = (
+                    (window_warning + " ") if window_warning else ""
+                ) + (f"Only {len(gated)} lag(s) fall inside the defensible "
+                     f"window [{lag_lo:.3g}, {lag_hi:.3g}]s — too few to fit; "
+                     f"using the full available range instead.")
+
     if len(df) < 3:
         return dict(D_um2_per_s=np.nan, alpha=np.nan, motion_type='unknown',
                     r_squared=np.nan, fit_lags_s=np.array([]),
                     fit_msd=np.array([]), log_log_slope=np.nan,
-                    log_log_intercept=np.nan)
+                    log_log_intercept=np.nan,
+                    fit_window_s=(lag_lo, lag_hi),
+                    fit_window_warning=window_warning)
 
     tau = df['lag_s'].values.astype(float)
     msd = df['msd_um2'].values.astype(float)
@@ -332,6 +424,8 @@ def fit_anomalous_diffusion(
         fit_msd=msd_fit,
         log_log_slope=a_ll,
         log_log_intercept=log_intercept,
+        fit_window_s=(lag_lo, lag_hi),
+        fit_window_warning=window_warning,
     )
 
 

{pycat_napari-1.5.331 → pycat_napari-1.5.332}/src/pycat/toolbox/vpt_tools.py

@@ -651,24 +651,6 @@ def score_beads_template(frame, coords, template_z, half=4, subpixel=False):
     amplitude, integrated_intensity.
     """
     raw = np.asarray(frame, dtype=np.float32)
-    # Guard against a degenerate (non-2D) frame reaching here. A multi-file
-    # OME-TIFF with a MISSING linked file (the loader zeros/truncates it) or a
-    # collapsed lazy stack can yield a 1-D array, which would crash on
-    # `H, W = raw.shape` with an opaque "not enough values to unpack" error.
-    # Squeeze and, if still not 2-D, skip scoring this frame with a clear warning
-    # instead of taking down the whole detection run.
-    raw = np.squeeze(raw)
-    if raw.ndim != 2:
-        try:
-            napari_show_warning(
-                f"Bead scoring skipped a frame with unexpected shape "
-                f"{np.asarray(frame).shape} (a linked OME-TIFF file may be "
-                f"missing, or the stack collapsed). Check the file set is complete.")
-        except Exception:
-            pass
-        return [dict(y=float(y), x=float(x), ncc=np.nan, snr=np.nan,
-                     symmetry=np.nan, amplitude=np.nan,
-                     integrated_intensity=np.nan) for (y, x) in coords]
     H, W = raw.shape
     w = 2 * half + 1
     out = []
@@ -830,27 +812,42 @@ def classify_beads(beads_df: pd.DataFrame,
         else:
             snr_dim = -np.inf
 
+        # High-NCC guard against out_of_plane flicker. The out_of_plane (yellow)
+        # class is meant for genuinely dim / defocused beads — but a per-frame
+        # amplitude/SNR percentile will always bin ~the lowest quarter as dim,
+        # and when the whole population is uniformly low-quality a BRIGHT,
+        # well-matched bead sitting near that moving line flips singlet<->yellow
+        # frame-to-frame (verified: a bead at amp~163, ncc~0.95 read out_of_plane
+        # ~24% of frames purely from the SNR percentile wobbling). NCC is the
+        # template match built FROM the real beads, so a high NCC is strong
+        # evidence of a real in-focus bead regardless of a noisy per-frame SNR.
+        # A bead with NCC at/above this guard is never demoted to out_of_plane.
+        NCC_SINGLET_GUARD = 0.80
+
         n_units, classes = [], []
         for k in range(len(df)):
             if not is_real[k]:
                 n_units.append(np.nan); classes.append('rejected'); continue
             I, A = ii[k], amp[k]
             S = snr[k] if 'snr' in df else np.nan
+            C = ncc[k]
             nu = I / singlet_int if (singlet_int and singlet_int > 0) else np.nan
             n_units.append(nu)
             high_mass = np.isfinite(I) and I >= mass_hi
             bright = np.isfinite(A) and A >= amp_hi
-            # Dim / out-of-focus: low amplitude AND not part of the bright
-            # aggregate tail. These go YELLOW (out_of_plane). Blinking of this
-            # population across frames is expected and acceptable — a bead that
-            # is genuinely in focus and stable will instead read as a singlet;
-            # the temporal-stability pass (after linking) promotes stable dim
-            # tracks back to singlet.
-            is_dim = (np.isfinite(A) and A <= amp_dim) or \
-                     (np.isfinite(S) and S <= snr_dim)
+            # Dim / out-of-focus. Require the AMPLITUDE to actually be low — a
+            # low per-frame SNR alone must NOT demote a bead whose amplitude is
+            # fine (that was the flicker source). SNR is now only a secondary
+            # confirmation: it can push a bead that is ALREADY amplitude-dim into
+            # the yellow bin, but it can't drag a bright bead there on its own.
+            amp_low = np.isfinite(A) and A <= amp_dim
+            snr_low = np.isfinite(S) and S <= snr_dim
+            is_dim = amp_low or (snr_low and amp_low)
+            # High-NCC well-matched beads are immune to the dim gate (anti-flicker).
+            well_matched = np.isfinite(C) and C >= NCC_SINGLET_GUARD
             if high_mass and bright:
                 classes.append('aggregate')          # bright + compact + heavy
-            elif is_dim and not high_mass:
+            elif is_dim and not high_mass and not well_matched:
                 classes.append('out_of_plane')        # YELLOW: dim / out of focus
             elif high_mass and not bright:
                 classes.append('ambiguous')           # heavy but dim/diffuse (OOF)
@@ -867,6 +864,7 @@ def classify_beads(beads_df: pd.DataFrame,
         df.attrs['classify_thresholds'] = {
             'mode': 'fast_template',
             'ncc_floor': float(NCC_FLOOR),
+            'ncc_singlet_guard': float(NCC_SINGLET_GUARD),
             'aggregate_mass_percentile': 99.3,
             'aggregate_amp_percentile': 50.0,
             'aggregate_mass_hi': float(mass_hi),

{pycat_napari-1.5.331 → pycat_napari-1.5.332}/src/pycat/toolbox/vpt_ui.py

@@ -1077,9 +1077,56 @@ class VideoParticleTrackingUI:
         _mb.setContentsMargins(0, 0, 0, 0); _mb.setSpacing(5)
         _mb.addRow(self._moduli_boot)
         _mb.addRow("Bootstrap resamples:", self._moduli_nboot)
+
+        # ── Lag-window fit gate (MSD → D/α fit range) ────────────────────────
+        # The reliable MSD lag window is bounded by hardware: high-frequency
+        # cutoff = frame interval, low-frequency cutoff = acquisition duration.
+        # Fitting outside it gives a wrong D/α. These pick the upper-lag rule and
+        # whether to clip the fit to the defensible band (warn, not block).
+        self._lag_rule = QComboBox()
+        self._lag_rule.addItem("Fraction of track length", "fraction")
+        self._lag_rule.addItem("Fixed frequency window", "fixed")
+        self._lag_rule.addItem("Minimum independent pairs", "min_pairs")
+        self._lag_rule.setToolTip(
+            "How to set the upper-lag (low-frequency) cutoff of the MSD fit:\n"
+            "• Fraction of track length — fit lags up to a fraction of the record "
+            "length; longer lags have too few independent samples. Standard, "
+            "conservative.\n"
+            "• Fixed frequency window — restrict to a hardware-defensible band "
+            "(set the upper lag in seconds). Matches routine lab practice.\n"
+            "• Minimum independent pairs — keep a lag only while enough independent "
+            "trajectories span it. Adapts to how many/how long your tracks are.")
+        self._lag_fraction = QDoubleSpinBox()
+        self._lag_fraction.setRange(0.02, 1.0); self._lag_fraction.setValue(0.25)
+        self._lag_fraction.setDecimals(2); self._lag_fraction.setSingleStep(0.05)
+        self._lag_fraction.setToolTip(
+            "Upper lag = this fraction × longest track duration (Fraction rule).")
+        self._lag_fixed_s = QDoubleSpinBox()
+        self._lag_fixed_s.setRange(0.001, 1e6); self._lag_fixed_s.setValue(10.0)
+        self._lag_fixed_s.setDecimals(3)
+        self._lag_fixed_s.setToolTip(
+            "Upper lag in seconds (Fixed-window rule). E.g. 10 s for a "
+            "0.1–10 s defensible band at 0.1 s/frame.")
+        self._lag_minpairs = QSpinBox()
+        self._lag_minpairs.setRange(2, 1000); self._lag_minpairs.setValue(10)
+        self._lag_minpairs.setToolTip(
+            "Keep lags spanned by at least this many independent tracks "
+            "(Minimum-independent-pairs rule).")
+        self._lag_confine = QCheckBox("Confine fit to scientifically defensible bounds")
+        self._lag_confine.setChecked(True)
+        self._lag_confine.setToolTip(
+            "When ON (default), the MSD fit is clipped to the hardware-defensible "
+            "lag window (frame interval → the upper-lag rule above). When OFF, the "
+            "full available lag range is fit at your own risk. Either way, PyCAT "
+            "WARNS (never blocks) if the acquisition can't cover the window.")
+        _mb.addRow("Upper-lag rule:", self._lag_rule)
+        _mb.addRow("  fraction:", self._lag_fraction)
+        _mb.addRow("  fixed upper lag (s):", self._lag_fixed_s)
+        _mb.addRow("  min independent pairs:", self._lag_minpairs)
+        _mb.addRow(self._lag_confine)
         self._moduli_boot_row.setVisible(False)   # hidden until toggled
 
-        self._show_moduli_adv = QCheckBox("Show advanced moduli (G′/G″) options")
+        self._show_moduli_adv = QCheckBox("Show advanced fit / moduli options")
         self._show_moduli_adv.setChecked(False)
         self._show_moduli_adv.toggled.connect(self._moduli_boot_row.setVisible)
         form.addRow(self._show_moduli_adv)
@@ -1126,7 +1173,29 @@ class VideoParticleTrackingUI:
                 tracks,
                 frame_interval_s=self._frame_dt.value(),
                 min_track_length=self._min_track.value())
-            fit = fit_anomalous_diffusion(msd_df)
+            fit = fit_anomalous_diffusion(
+                msd_df,
+                frame_interval_s=self._frame_dt.value(),
+                upper_lag_rule=(self._lag_rule.currentData()
+                                if hasattr(self, '_lag_rule') else 'fraction'),
+                upper_lag_fraction=(self._lag_fraction.value()
+                                    if hasattr(self, '_lag_fraction') else 0.25),
+                upper_lag_fixed_s=(self._lag_fixed_s.value()
+                                   if hasattr(self, '_lag_fixed_s') else None),
+                min_independent_pairs=(self._lag_minpairs.value()
+                                       if hasattr(self, '_lag_minpairs') else 10),
+                confine_to_defensible_bounds=(
+                    self._lag_confine.isChecked()
+                    if hasattr(self, '_lag_confine') else True))
+            _fw = fit.get('fit_window_warning')
+            if _fw:
+                napari_show_warning(f"MSD fit window: {_fw}")
+                print(f"[PyCAT VPT] MSD fit-window note: {_fw}")
+            _win = fit.get('fit_window_s')
+            if _win and all(v is not None for v in _win):
+                print(f"[PyCAT VPT] MSD fit confined to defensible lag window "
+                      f"{_win[0]:.3g}–{_win[1]:.3g}s "
+                      f"({'confined' if (self._lag_confine.isChecked() if hasattr(self,'_lag_confine') else True) else 'full range (confine off)'}).")
             eta = viscosity_from_diffusion(
                 fit.get('D_um2_per_s', float('nan')),
                 self._bead_radius.value(), self._temp_C.value())