microstructure-tpw 0.3.0__py3-none-any.whl

odas_tpw/scor160/despike.py

@@ -0,0 +1,227 @@
+# Mar-2026, Claude and Pat Welch, pat@mousebrains.com
+"""Spike removal for shear probe and micro-conductivity signals.
+
+Port of despike.m from the ODAS MATLAB library.
+"""
+
+import warnings
+from typing import NamedTuple
+
+import numpy as np
+import numpy.typing as npt
+from scipy.signal import butter, filtfilt
+
+
+def _matlab_padlen(b: np.ndarray, a: np.ndarray) -> int:
+    """Return the ``filtfilt`` *padlen* matching MATLAB's convention.
+
+    MATLAB pads with ``3*(nfilt-1)`` reflected samples where
+    ``nfilt = max(len(a), len(b))``; scipy defaults to ``3*nfilt``.
+    For a 1st-order Butterworth (nfilt=2) this is padlen=3 vs scipy's 6.
+    Replicated locally (rather than imported from ``l2``) to avoid a
+    circular import, mirroring ``l2._matlab_padlen``.
+    """
+    return 3 * (max(len(a), len(b)) - 1)
+
+
+class DespikeResult(NamedTuple):
+    """Result of iterative spike removal."""
+
+    y: np.ndarray
+    spike_indices: np.ndarray
+    n_passes: int
+    fraction: float
+
+
+def despike(
+    signal_in: npt.ArrayLike,
+    fs: float,
+    thresh: float = 8,
+    smooth: float = 0.5,
+    N: int | None = None,
+    max_passes: int = 10,
+) -> DespikeResult:
+    """Iterative spike removal.
+
+    Identifies spikes by comparing the instantaneous rectified HP-filtered
+    signal against its smoothed envelope. Replaces spike neighborhoods with
+    local mean.
+
+    Parameters
+    ----------
+    signal_in : array_like
+        Signal to despike.
+    fs : float
+        Sampling rate [Hz].
+    thresh : float
+        Ratio threshold for spike detection. Default: 8.
+    smooth : float
+        Low-pass cutoff [Hz] for envelope smoothing. Default: 0.5.
+    N : int or None
+        Spike removal half-width [samples]. A total of ~1.5*N points are
+        removed around each spike. Default: round(0.04 * fs) (~40 ms).
+    max_passes : int
+        Maximum despiking iterations. Default: 10.
+
+    Returns
+    -------
+    y : ndarray
+        Despiked signal.
+    spike_indices : ndarray
+        Indices of detected spikes in original signal.
+    n_passes : int
+        Number of iterations performed.
+    fraction : float
+        Fraction of data replaced.
+    """
+    if N is None:
+        N = round(0.04 * fs)
+
+    y = np.array(signal_in, dtype=np.float64).ravel()
+    # Non-finite samples propagate through filtfilt and turn the entire
+    # detection ratio into NaN, silently disabling ALL spike removal so
+    # genuine spikes survive. ODAS removes/interpolates NaNs upstream; this
+    # port does not, so at minimum warn to make the silent no-op observable.
+    if y.size and not np.all(np.isfinite(y)):
+        warnings.warn(
+            "despike: input contains non-finite samples; filtfilt will "
+            "propagate them and spike detection is disabled for this array. "
+            "Remove or interpolate NaN/Inf before despiking.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+    original = y.copy()
+    all_spikes: set[int] = set()
+    n_passes = 0
+
+    for _ in range(max_passes):
+        y_new, spikes = _single_despike(y, thresh, smooth, fs, N)
+        if len(spikes) == 0:
+            break
+        y = y_new
+        all_spikes.update(spikes)
+        n_passes += 1
+
+    spike_indices = np.sort(np.array(list(all_spikes), dtype=np.intp))
+    # NaN-safe change count: ``y != original`` reports True at unchanged NaN
+    # positions (NaN != NaN), so a no-op pass on NaN-containing input used to
+    # report a spurious nonzero fraction. Exclude the NaN==NaN positions.
+    changed = (y != original) & ~(np.isnan(y) & np.isnan(original))
+    fraction = float(np.sum(changed)) / len(y) if len(y) else 0.0
+    return DespikeResult(y, spike_indices, n_passes, fraction)
+
+
+def _single_despike(
+    dv: np.ndarray, thresh: float, smooth: float, fs: float, N: int
+) -> tuple[np.ndarray, np.ndarray]:
+    """Single pass of spike detection and removal."""
+    # Port note: MATLAB despike.m uses a float half-width (N/2) and rounds the
+    # resulting window boundaries, so for odd N its bad-window can differ from
+    # this floored N//2 by ~1 sample on each side. The physical effect is
+    # sub-sample and the default N=round(0.04*fs)=20 (fs=512) is even, so the
+    # default path matches MATLAB exactly; documented rather than changed.
+    N_half = N // 2
+    dv = dv.copy()
+    length = len(dv)
+
+    # Zero-pad with reflected ends to handle edge effects
+    pad_len = min(length, 2 * int(fs / smooth))
+    dv_padded = np.concatenate(
+        [
+            dv[pad_len - 1 :: -1],
+            dv,
+            dv[-1 : -pad_len - 1 : -1],
+        ]
+    )
+    rng = slice(pad_len, pad_len + length)
+
+    # High-pass filter at 0.5 Hz, rectify.
+    # Use MATLAB's padlen (3) not scipy's default (6): restores port
+    # faithfulness and avoids a ValueError crash on short (e.g. length-2,
+    # padded-to-6) sections where scipy requires len(x) > padlen. Guard the
+    # still-too-short remainder (e.g. length-1, padded-to-3) so despike
+    # degrades to a no-op instead of crashing.
+    b_hp, a_hp = butter(1, 0.5 / (fs / 2), btype="high")
+    scipy_min_len = _matlab_padlen(b_hp, a_hp) + 1
+    if len(dv_padded) < scipy_min_len:
+        return dv, np.array([], dtype=np.intp)
+    dv_hp = np.abs(filtfilt(b_hp, a_hp, dv_padded, padlen=_matlab_padlen(b_hp, a_hp)))
+
+    # Smooth envelope
+    b_lp, a_lp = butter(1, smooth / (fs / 2))
+    dv_lp = filtfilt(b_lp, a_lp, dv_hp, padlen=_matlab_padlen(b_lp, a_lp))
+
+    # Detect spikes (avoid divide-by-zero)
+    with np.errstate(divide="ignore", invalid="ignore"):
+        ratio = dv_hp / dv_lp
+    spike_idx = np.where(ratio > thresh)[0]
+
+    # Ignore spikes in padding
+    spike_idx = spike_idx[(spike_idx >= rng.start) & (spike_idx < rng.stop)]
+    if len(spike_idx) == 0:
+        return dv, np.array([], dtype=np.intp)
+
+    # Mark bad points: N_half before spike, 2*N_half after spike
+    good = np.ones(len(dv_padded), dtype=bool)
+    R = round(fs / (4 * smooth))  # averaging region size
+
+    for s in spike_idx:
+        lo = max(0, s - N_half)
+        hi = min(len(good), s + 2 * N_half + 1)
+        good[lo:hi] = False
+
+    # Find contiguous bad regions and replace with local mean
+    transitions = np.diff(good.astype(np.int8))
+    starts = np.where(transitions == -1)[0] + 1
+    ends = np.where(transitions == 1)[0] + 1
+
+    # Handle edge cases
+    if not good[0]:
+        starts = np.concatenate([[0], starts])
+    if not good[-1]:
+        ends = np.concatenate([ends, [len(good)]])
+
+    boundary_nan = False
+    for s_bad, e_bad in zip(starts, ends):
+        # Average from region R before start
+        before_idx = np.arange(max(0, s_bad - R), s_bad)
+        before_vals = (
+            dv_padded[before_idx[good[before_idx]]] if len(before_idx) > 0 else np.array([])
+        )
+
+        # Average from region R after end
+        after_idx = np.arange(e_bad, min(len(good), e_bad + R))
+        after_vals = dv_padded[after_idx[good[after_idx]]] if len(after_idx) > 0 else np.array([])
+
+        # Match MATLAB despike.m: start_value/stop_value are sum/length, so an
+        # empty region gives 0/0 = NaN and the average (NaN + x)/2 = NaN. The
+        # prior port fabricated 0.0 (and one-sided fallbacks), silently
+        # injecting plausible-looking data instead of a detectable NaN.
+        start_value = np.mean(before_vals) if len(before_vals) > 0 else np.nan
+        stop_value = np.mean(after_vals) if len(after_vals) > 0 else np.nan
+        replacement = (start_value + stop_value) / 2
+
+        # A spike region touching a boundary can have no clean samples on one
+        # side, so the MATLAB-parity replacement above is NaN. Keep that NaN
+        # (never fabricate), but make it observable rather than silent —
+        # mirroring the non-finite-input warning. Only reachable for very short
+        # sections; a full-length record never trips it.
+        if not np.isfinite(replacement):
+            boundary_nan = True
+
+        dv_padded[s_bad:e_bad] = replacement
+
+    if boundary_nan:
+        warnings.warn(
+            "despike: a spike region adjacent to the array boundary had no "
+            "clean neighboring samples on one side; those samples are set to "
+            "NaN (matching ODAS despike.m) rather than fabricated. This "
+            "typically affects only very short sections.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
+    # Remove padding, return spike indices in original coordinates
+    result = dv_padded[rng]
+    spikes_orig = spike_idx - pad_len
+    return result, spikes_orig

odas_tpw/scor160/goodman.py

@@ -0,0 +1,288 @@
+# Mar-2026, Claude and Pat Welch, pat@mousebrains.com
+"""Goodman coherent noise removal for shear probe spectra.
+
+Port of clean_shear_spec.m from the ODAS MATLAB library.
+Removes vibration-coherent noise from shear spectra using
+accelerometer cross-spectra.
+"""
+
+from typing import NamedTuple
+
+import numpy as np
+
+from odas_tpw.scor160.spectral import csd_matrix, csd_matrix_batch
+
+
+def _sanitize_autospectra(clean_UU: np.ndarray) -> np.ndarray:
+    """Sanitize cleaned auto-spectrum (diagonal) bins.
+
+    Two corrections, both on the diagonal auto-spectra:
+
+    * Non-finite -> NaN. A singular accelerometer matrix AA cannot be cleaned.
+      np.linalg.solve raises LinAlgError on some LAPACK builds (handled by the
+      per-frequency fallback, which NaNs the bin) but on others returns inf/NaN
+      WITHOUT raising -- so the cleaned bin is inf/NaN here instead. Map any
+      non-finite bin to NaN so l4's bad-bin handling excludes it *portably*,
+      rather than the result depending on whether solve happened to raise.
+    * Finite negative -> 0. An ill-conditioned (not exactly singular) AA can
+      over-subtract and drive a cleaned auto-spectrum slightly negative, which
+      is physically impossible (a PSD cannot be < 0). Floor to 0 (matching the
+      variance-method polyfit's zero-bin handling) so the epsilon integral / FOM
+      see consistent non-negative power.
+
+    Operates on the last two (n_sh, n_sh) axes, so it works for both the single
+    (n_freq, ...) and batched (n_win, n_freq, ...) shapes.  Only the diagonal is
+    touched (and taken real — an auto-spectrum is real by definition), so a
+    complex ``clean_UU`` keeps its off-diagonal cleaned cross-spectra intact.
+    """
+    n_sh = clean_UU.shape[-1]
+    for i in range(n_sh):
+        diag_i = np.real(clean_UU[..., i, i])
+        clean_UU[..., i, i] = np.where(
+            ~np.isfinite(diag_i), np.nan, np.where(diag_i < 0, 0.0, diag_i)
+        )
+    return clean_UU
+
+
+def _nan_singular_bins(clean_UU: np.ndarray, AA: np.ndarray) -> np.ndarray:
+    """NaN whole frequency bins whose accelerometer matrix AA is singular.
+
+    ``np.linalg.solve``'s output for a singular AA is platform-dependent -- it
+    raises ``LinAlgError`` on some LAPACK builds (handled by the per-frequency
+    fallback) but returns inf/NaN or even finite garbage on others. Detect the
+    singular bins portably via the condition number (cond == inf or astronomical
+    for an exactly rank-deficient AA, e.g. duplicate/collinear accelerometer
+    channels) and NaN those bins so the result does not depend on the solver's
+    behavior. Merely ill-conditioned bins (cond <= 1e13) are left to
+    :func:`_sanitize_autospectra`'s floor. clean_UU: (..., n_sh, n_sh); AA:
+    (..., n_ac, n_ac) with matching leading dims.
+    """
+    # A NaN/inf in AA (a vibration dropout flows into the accel auto-spectrum)
+    # makes np.linalg.cond's SVD raise LinAlgError ("SVD did not converge"),
+    # which the errstate does NOT catch -- it would abort the whole .p file.
+    # Exclude non-finite AA matrices from the cond/SVD and treat them as bad
+    # bins directly, preserving l4's graceful NaN-bin handling.
+    finite_AA = np.all(np.isfinite(AA), axis=(-2, -1))
+    cond = np.full(AA.shape[:-2], np.inf)
+    if np.any(finite_AA):
+        with np.errstate(invalid="ignore", divide="ignore"):
+            cond[finite_AA] = np.linalg.cond(AA[finite_AA])
+    bad = ~np.isfinite(cond) | (cond > 1.0e13)
+    if np.any(bad):
+        clean_UU[bad] = np.nan
+    return clean_UU
+
+
+class CleanShearResult(NamedTuple):
+    """Result of Goodman coherent noise removal."""
+
+    clean_UU: np.ndarray
+    AA: np.ndarray
+    UU: np.ndarray
+    UA: np.ndarray
+    F: np.ndarray
+
+
+def _bias_correction(n_samples: int, nfft: int, n_accel: int) -> float:
+    """Goodman bias correction factor (ODAS TN-061, Eq. 3).
+
+    Factor 1.02 accounts for effective degrees-of-freedom reduction
+    from overlapping FFT segments (Goodman 2006, ODAS TN-061).
+    """
+    fft_segments = 2 * n_samples // nfft - 1
+    if fft_segments <= 1.02 * n_accel:
+        import warnings
+
+        warnings.warn(
+            f"Insufficient FFT segments ({fft_segments}) for Goodman bias "
+            f"correction with {n_accel} accelerometers; skipping correction",
+            stacklevel=3,
+        )
+        return 1.0
+    return 1.0 / (1.0 - 1.02 * n_accel / fft_segments)
+
+
+def clean_shear_spec(
+    accel: np.ndarray,
+    shear: np.ndarray,
+    nfft: int,
+    rate: float,
+) -> CleanShearResult:
+    """Remove acceleration-coherent noise from shear spectra.
+
+    Parameters
+    ----------
+    accel : ndarray, shape (N, n_accel)
+        Accelerometer signals (columns).
+    shear : ndarray, shape (N, n_shear)
+        Shear probe signals (columns).
+    nfft : int
+        FFT segment length.
+    rate : float
+        Sampling rate [Hz].
+
+    Returns
+    -------
+    clean_UU : ndarray, shape (n_freq, n_shear, n_shear)
+        Cleaned shear spectral matrix. Diagonal elements are the
+        cleaned auto-spectra.
+    AA : ndarray, shape (n_freq, n_accel, n_accel)
+        Accelerometer spectral matrix.
+    UU : ndarray, shape (n_freq, n_shear, n_shear)
+        Original (uncleaned) shear spectral matrix.
+    UA : ndarray, shape (n_freq, n_shear, n_accel)
+        Shear-accelerometer cross-spectral matrix.
+    F : ndarray
+        Frequency vector [Hz].
+    """
+    accel = np.atleast_2d(np.asarray(accel, dtype=np.float64))
+    shear = np.atleast_2d(np.asarray(shear, dtype=np.float64))
+    if accel.shape[0] < accel.shape[1]:
+        accel = accel.T
+    if shear.shape[0] < shear.shape[1]:
+        shear = shear.T
+    if accel.shape[0] != shear.shape[0]:
+        raise ValueError("accel and shear must have same number of rows")
+
+    n_accel = accel.shape[1]
+
+    # Compute all spectra in one call: x=shear, y=accel
+    # Returns Cxy=(n_freq, n_sh, n_ac), Cxx=(n_freq, n_sh, n_sh), Cyy=(n_freq, n_ac, n_ac)
+    try:
+        UA, F, UU, AA = csd_matrix(shear, accel, nfft, rate, overlap=nfft // 2, detrend="linear")
+    except ValueError:
+        # Signal too short for cross-spectrum — fall back to shear auto-spectrum
+        import warnings
+
+        warnings.warn(
+            f"Insufficient FFT segments for Goodman cleaning "
+            f"(signal length {shear.shape[0]} < 2*nfft {2 * nfft}); "
+            f"returning uncleaned spectra",
+            stacklevel=2,
+        )
+        n_sh = shear.shape[1]
+        n_ac = accel.shape[1]
+        try:
+            # Compute the uncleaned shear auto-spectrum so callers still
+            # get a usable spectrum.  Subject to the same 2*nfft length
+            # check as the cross-spectrum, so this succeeds only when the
+            # failure above was not a length error.
+            UU, F, _, _ = csd_matrix(shear, None, nfft, rate, overlap=nfft // 2, detrend="linear")
+        except ValueError:
+            # Even auto-spectrum fails — return zeros
+            n_freq_fb = nfft // 2 + 1
+            F = np.arange(n_freq_fb) * rate / nfft
+            UU = np.zeros((n_freq_fb, n_sh, n_sh), dtype=np.complex128)
+        n_freq_fb = len(F)
+        AA = np.zeros((n_freq_fb, n_ac, n_ac), dtype=np.complex128)
+        UA = np.zeros((n_freq_fb, n_sh, n_ac), dtype=np.complex128)
+        # Keep the fallback consistent with the cleaned path (Gem-D-c): return a
+        # complex matrix with a sanitized real diagonal rather than np.real(UU),
+        # so clean_UU has one dtype across code paths and the off-diagonal is not
+        # silently dropped here.  .copy() so we don't mutate the raw UU field.
+        return CleanShearResult(_sanitize_autospectra(UU.copy()), AA, UU, UA, F)
+    assert UU is not None and AA is not None  # always returned when y is provided
+
+    # UU, AA, UA are complex; extract real diagonal for auto-spectra
+    n_freq = len(F)
+
+    # Batched linear solve: np.linalg.solve supports batch dimensions
+    # clean = UU - UA @ inv(AA) @ conj(UA).T
+    try:
+        ua_H = np.conj(np.swapaxes(UA, -2, -1))  # (n_freq, n_ac, n_sh)
+        solved = np.linalg.solve(AA, ua_H)  # (n_freq, n_ac, n_sh)
+        clean_UU = UU - np.matmul(UA, solved)  # (n_freq, n_sh, n_sh)
+    except np.linalg.LinAlgError:
+        # Fallback: per-frequency with singular handling. A singular AA[f]
+        # cannot be cleaned; NaN that bin so downstream treats it as bad rather
+        # than passing the UNCLEANED raw UU through (and then bias-correcting it
+        # as if it had been cleaned).
+        clean_UU = np.copy(UU)
+        for f in range(n_freq):
+            try:
+                clean_UU[f] = UU[f] - UA[f] @ np.linalg.solve(AA[f], np.conj(UA[f]).T)
+            except np.linalg.LinAlgError:
+                clean_UU[f] = np.nan
+
+    # Keep clean_UU complex: sanitize only the (real) diagonal auto-spectra, so
+    # the off-diagonal cleaned cross-spectra are preserved for callers that want
+    # them (e.g. cross-probe coherence). Current consumers take np.real of the
+    # diagonal, which is numerically unchanged.  (2026-07-03 review, Gem-D-c.)
+    clean_UU = _sanitize_autospectra(clean_UU)
+    clean_UU = _nan_singular_bins(clean_UU, AA)
+
+    # Bias correction (ODAS Technical Note 61, Eq. 3)
+    clean_UU *= _bias_correction(shear.shape[0], nfft, n_accel)
+
+    return CleanShearResult(clean_UU, AA, UU, UA, F)
+
+
+def clean_shear_spec_batch(
+    accel_windows: np.ndarray,
+    shear_windows: np.ndarray,
+    nfft: int,
+    rate: float,
+) -> tuple[np.ndarray, np.ndarray]:
+    """Batched Goodman noise removal for all dissipation windows at once.
+
+    Parameters
+    ----------
+    accel_windows : ndarray, shape (n_windows, diss_length, n_accel)
+    shear_windows : ndarray, shape (n_windows, diss_length, n_shear)
+    nfft : int
+    rate : float
+
+    Returns
+    -------
+    clean_UU : ndarray, shape (n_windows, n_freq, n_shear, n_shear)
+        Cleaned shear spectral matrices. Complex: the diagonal auto-spectra are
+        real (and non-negative), the off-diagonal cleaned cross-spectra retain
+        their imaginary part. Consumers of the auto-spectra take ``np.real`` of
+        the diagonal.
+    F : ndarray, shape (n_freq,)
+        Frequency vector [Hz].
+    """
+    n_accel = accel_windows.shape[2]
+
+    # Batched CSD: x=shear, y=accel
+    # UA: (n_win, n_freq, n_sh, n_ac)
+    # UU: (n_win, n_freq, n_sh, n_sh)
+    # AA: (n_win, n_freq, n_ac, n_ac)
+    UA, F, UU, AA = csd_matrix_batch(
+        shear_windows,
+        accel_windows,
+        nfft,
+        rate,
+        overlap=nfft // 2,
+        detrend="linear",
+    )
+    assert UU is not None and AA is not None  # guaranteed when y is not None
+
+    # Goodman cleaning: clean = UU - UA @ inv(AA) @ conj(UA).T
+    # np.linalg.solve broadcasts over leading dims (n_win, n_freq)
+    ua_H = np.conj(np.swapaxes(UA, -2, -1))  # (n_win, n_freq, n_ac, n_sh)
+    try:
+        solved = np.linalg.solve(AA, ua_H)  # (n_win, n_freq, n_ac, n_sh)
+        clean_UU = UU - np.matmul(UA, solved)
+    except np.linalg.LinAlgError:
+        clean_UU = UU.copy()
+        n_win, n_freq = UU.shape[:2]
+        for w in range(n_win):
+            for fi in range(n_freq):
+                try:
+                    clean_UU[w, fi] = UU[w, fi] - UA[w, fi] @ np.linalg.solve(
+                        AA[w, fi], np.conj(UA[w, fi]).T
+                    )
+                except np.linalg.LinAlgError:
+                    # Singular AA: NaN the bin rather than keep the raw UU.
+                    clean_UU[w, fi] = np.nan
+
+    # Sanitize only the real diagonal; off-diagonal cleaned cross-spectra stay
+    # complex for future cross-probe consumers. (2026-07-03 review, Gem-D-c.)
+    clean_UU = _sanitize_autospectra(clean_UU)
+    clean_UU = _nan_singular_bins(clean_UU, AA)
+
+    # Bias correction
+    clean_UU *= _bias_correction(shear_windows.shape[1], nfft, n_accel)
+
+    return clean_UU, F