microstructure-tpw 0.3.0__py3-none-any.whl

odas_tpw/processing/top_trim.py

@@ -0,0 +1,274 @@
+# Mar-2026, Claude and Pat Welch, pat@mousebrains.com
+"""Top trimming — drop initial instabilities from a profile.
+
+Caller supplies one or more fast-rate motion proxies as a dict. The
+algorithm bins each by depth, computes per-bin std, and reports the depth
+below which the instrument's motion has settled. The settling depth is
+located as the first bin beneath the *surface-attached* elevated
+(prop-wash) run — the run of elevated bins anchored to the top of the
+search range, bridging quiet lulls up to ``max_gap`` bins. Bridging lulls
+keeps a momentarily quiet near-surface bin from ending the search early;
+anchoring to the surface keeps an isolated *deep* transient from
+over-trimming the quiet band above it. Channels are combined with the
+median (fully robust to one bad channel only for three or more voters; the
+VMP caller feeds two accelerometers, where median == mean).
+
+The instrument-specific question of which channels to feed lives in the
+caller. They must reflect the *instrument's* state, not the ocean: on a
+VMP the accelerometers settle at the true prop-wash exit, whereas shear
+probes, inclinometers, and fall rate stay elevated through deep ocean
+turbulence and would over-trim.
+
+Reference: Code/trim_top_profiles.m (85 lines)
+"""
+
+import numpy as np
+import numpy.typing as npt
+
+
+def _bin_std(values: np.ndarray, depth: np.ndarray, bin_edges: np.ndarray) -> np.ndarray:
+    """Compute standard deviation per depth bin (NaN-aware, ddof=0)."""
+    n_bins = len(bin_edges) - 1
+    # Bin assignment via searchsorted gives the same [edges[i], edges[i+1])
+    # half-open bins as the original mask comparison.
+    idx = np.searchsorted(bin_edges, depth, side="right") - 1
+    in_range = (idx >= 0) & (idx < n_bins) & np.isfinite(values)
+    idx_v = idx[in_range]
+    vals_v = values[in_range]
+    counts = np.bincount(idx_v, minlength=n_bins)
+    sums = np.bincount(idx_v, weights=vals_v, minlength=n_bins)
+    sums_sq = np.bincount(idx_v, weights=vals_v * vals_v, minlength=n_bins)
+    with np.errstate(invalid="ignore", divide="ignore"):
+        means = np.where(counts > 0, sums / np.maximum(counts, 1), 0.0)
+        var = sums_sq / np.maximum(counts, 1) - means * means
+        var = np.maximum(var, 0.0)
+        result = np.where(counts > 1, np.sqrt(var), np.nan)
+    return result
+
+
+def _surface_run_end(elevated: np.ndarray, max_gap: int) -> int | None:
+    """Index of the deepest bin of the surface-attached elevated run.
+
+    Prop wash is a *surface-attached* transient: a run of elevated bins that
+    begins at (or within ``max_gap`` bins of) the top of the search range and
+    extends downward. Quiet lulls of up to ``max_gap`` bins inside the run are
+    bridged so a momentarily quiet near-surface bin does not end the search
+    early (audit #66). A wider quiet band separates the surface wash from any
+    deeper *isolated* elevated bin (a cable snap-load or mid-column turbulence
+    patch), which is contamination — not prop wash — and must not drive the
+    trim (audit r1-2).
+
+    ``elevated`` is a boolean array over a channel's valid bins, shallow→deep.
+    Returns the index (into ``elevated``) of the deepest bin in the
+    surface-attached run, or None when there is no surface-attached prop wash
+    (no elevated bins, or the only elevated bins are detached from the
+    surface).
+    """
+    idx = np.flatnonzero(elevated)
+    if idx.size == 0:
+        return None
+    first = int(idx[0])
+    if first > max_gap:
+        # Elevated bins exist, but none within max_gap of the surface: the
+        # descent was already settled at the top, so these are detached
+        # mid-column transients, not prop wash.
+        return None
+    last = first
+    gap = 0
+    for pos in range(first + 1, len(elevated)):
+        if elevated[pos]:
+            last = pos
+            gap = 0
+        else:
+            gap += 1
+            if gap > max_gap:
+                break
+    return last
+
+
+def compute_trim_depth(
+    depth_fast: npt.ArrayLike,
+    channels: dict[str, np.ndarray],
+    *,
+    dz: float = 0.5,
+    min_depth: float = 1.0,
+    max_depth: float = 50.0,
+    quantile: float = 0.6,
+    noise_factor: float = 2.0,
+    max_gap: int = 3,
+) -> float | None:
+    """Compute the trim depth for a single profile.
+
+    For each channel the per-bin standard deviation is compared against a
+    *settled background* level — the ``quantile`` of that channel's per-bin
+    std (a robust estimate of the quiet level, valid while the prop wash
+    spans less than ``1 - quantile`` of the binned range). A bin is treated
+    as still inside the prop wash when its std exceeds ``noise_factor``
+    times that background. The channel's prop-wash exit is the first bin
+    *below the surface-attached elevated run* — the run of elevated bins
+    that starts within ``max_gap`` bins of the top of the search range and
+    extends down, bridging quiet lulls of up to ``max_gap`` bins. Bridging
+    lulls prevents a momentarily quiet near-surface bin from ending the
+    search early (audit #66); anchoring the run to the surface prevents an
+    isolated *deep* transient (a cable snap-load or mid-column turbulence
+    patch) from over-trimming the entire quiet band above it (audit r1-2).
+    A channel whose only elevated bins are detached from the surface sees
+    no prop wash and abstains. The profile trim depth is the **median**
+    exit across the channels that detected prop wash. Note the median is
+    fully robust to one bad channel only for *three or more* voters; the
+    production VMP caller feeds exactly two accelerometers (``Ax``/``Ay``),
+    for which the median equals the mean, so a disagreeing channel pulls
+    the trim halfway and a lone surface-detecting channel sets it outright.
+    The surface-attachment rule above (not the median) is what rejects a
+    single channel's spurious *deep* transient. A flat / zero-variance
+    channel carries no settling information and is dropped before voting.
+
+    The caller chooses which channels best mark the instrument's settling.
+    On VMP data the accelerometers are the right choice: they capture the
+    mechanical entry transient. Shear probes, inclinometers, and fall rate
+    respond to the *ocean* turbulence the instrument falls through, so their
+    per-bin std stays elevated at depth and would over-trim.
+
+    Parameters
+    ----------
+    depth_fast : array_like
+        Depth (positive downward, fast rate) [m].
+    channels : dict
+        Fast-rate channel data, name -> 1-D array. Each channel must
+        match ``depth_fast`` in length. Use instrument-motion proxies that
+        settle once the descent stabilizes (accelerometers); avoid channels
+        driven by ocean turbulence (shear).
+    dz : float
+        Depth bin size [m].
+    min_depth : float
+        Minimum search depth [m].
+    max_depth : float
+        Maximum search depth [m].
+    quantile : float
+        Quantile of per-bin std taken as the settled background level;
+        must be in ``(0, 1)``. Trimming is robust while the prop wash
+        spans less than ``1 - quantile`` of the search range.
+    noise_factor : float
+        A bin counts as still in the prop wash when its std exceeds
+        ``noise_factor`` times the settled background. Must be > 1 so the
+        background's own bin-to-bin scatter is not mistaken for prop wash.
+    max_gap : int
+        Maximum run of quiet bins bridged within the surface-attached
+        prop-wash run, and the maximum offset of the first elevated bin
+        from the surface for the run to count as surface-attached [bins].
+        Must be >= 0; ``>= 1`` preserves the audit-#66 momentary-lull
+        tolerance. At the default ``dz=0.5`` m, ``max_gap=3`` bridges lulls
+        up to ~1.5 m.
+
+    Returns
+    -------
+    float or None
+        Trim depth [m], or None if no trim point found.
+
+    Raises
+    ------
+    ValueError
+        If ``quantile`` is not in ``(0, 1)`` or ``noise_factor`` is not
+        greater than 1.
+    """
+    if not 0.0 < quantile < 1.0:
+        raise ValueError(f"quantile must be in (0, 1), got {quantile}")
+    if not noise_factor > 1.0:
+        raise ValueError(f"noise_factor must be > 1, got {noise_factor}")
+    if max_gap < 0:
+        raise ValueError(f"max_gap must be >= 0, got {max_gap}")
+    depth = np.asarray(depth_fast, dtype=np.float64)
+    bin_edges = np.arange(min_depth - dz / 2, max_depth + dz, dz)
+    bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2.0
+
+    if len(bin_edges) < 2:
+        return None
+
+    # Compute std per bin for each channel that carries a real signal.
+    all_stds = []
+    for name, data in channels.items():
+        data = np.asarray(data, dtype=np.float64)
+        if len(data) != len(depth):
+            continue
+        finite = np.isfinite(data)
+        if int(finite.sum()) < 2 or np.ptp(data[finite]) == 0:
+            # All-NaN or constant: a dead / non-functional sensor carries no
+            # settling information, so it must not vote (it would otherwise drag
+            # the median toward a minimal trim).
+            continue
+        all_stds.append(_bin_std(data, depth, bin_edges))
+
+    if not all_stds:
+        return None
+
+    # Per channel, locate the prop-wash exit: the bin just below the deepest
+    # bin of the surface-attached elevated run. Anchoring the run to the
+    # surface (rather than taking the globally deepest elevated bin) bridges a
+    # momentary near-surface lull (audit #66) while rejecting isolated deep
+    # transients that would otherwise over-trim the whole quiet band above them
+    # (audit r1-2).
+    exits = []
+    any_live = False
+    for std_arr in all_stds:
+        valid = np.isfinite(std_arr)
+        if np.sum(valid) < 3:
+            continue
+        any_live = True
+        valid_pos = np.flatnonzero(valid)
+        background = np.nanquantile(std_arr[valid], quantile)
+        elevated = std_arr[valid_pos] > noise_factor * background
+        run_end = _surface_run_end(elevated, max_gap)
+        if run_end is None:
+            # No surface-attached prop wash (none, or only detached deep
+            # contamination): abstain rather than voting a trim — a shallow
+            # vote would pull the median up, a deep vote to a transient would
+            # discard valid near-surface data.
+            continue
+        # Bin just below the deepest surface-run bin. If that is the deepest
+        # search bin the profile never settled within range; fall back to it
+        # (the most conservative, deepest trim).
+        deepest = int(valid_pos[run_end])
+        exit_idx = min(deepest + 1, len(bin_centers) - 1)
+        exits.append(bin_centers[exit_idx])
+
+    if exits:
+        # Median exit across the channels that detected prop wash: robust to a
+        # single misbehaving channel (a `max` lets one bad channel drag the
+        # trim arbitrarily deep).
+        trim = float(np.median(exits))
+        # A never-settled cast (the surface run reaches the deepest populated
+        # bin) yields an exit bin center in the empty tail of the search range,
+        # deeper than every real sample; the caller's ``P >= trim_depth`` would
+        # then be empty and apply NO trim, silently keeping the whole
+        # wash-contaminated column. Clamp to the deepest observed in-range
+        # sample so the trim is always applicable (the cast is then trimmed to
+        # its bottom, i.e. flagged as all-wash, rather than passed untouched).
+        in_range = depth[(depth >= min_depth) & (depth <= max_depth) & np.isfinite(depth)]
+        if in_range.size:
+            trim = min(trim, float(np.max(in_range)))
+        return trim
+    if any_live:
+        # Live channels but none saw prop wash: trim minimally (top of range).
+        return float(bin_centers[0])
+    return None
+
+
+def compute_trim_depths(
+    profiles_data: list[dict],
+    **params,
+) -> list[float | None]:
+    """Compute trim depths for multiple profiles.
+
+    Parameters
+    ----------
+    profiles_data : list of dict
+        Each dict has keys 'depth_fast' and 'channels' (dict of arrays).
+    **params
+        Keyword arguments passed to :func:`compute_trim_depth`.
+
+    Returns
+    -------
+    list of (float or None)
+        Per-profile trim depths.
+    """
+    return [compute_trim_depth(pd["depth_fast"], pd["channels"], **params) for pd in profiles_data]

odas_tpw/pyturb/__init__.py

@@ -0,0 +1,3 @@
+"""pyturb-cli: Jesse-compatible CLI backed by odas_tpw processing."""
+
+__version__ = "0.1.0"

odas_tpw/pyturb/__main__.py

@@ -0,0 +1,5 @@
+"""Allow ``python -m odas_tpw.pyturb`` invocation."""
+
+from odas_tpw.pyturb.cli import main
+
+main()

odas_tpw/pyturb/_compat.py

@@ -0,0 +1,204 @@
+"""Compatibility helpers for pyturb-cli."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import xarray as xr
+
+
+def seconds_to_samples(seconds: float, fs: float) -> int:
+    """Convert a time duration to a sample count (made even).
+
+    Matches Jesse's pyturb: ``n = int(seconds * fs)`` rounded to the nearest
+    even number (no power-of-2 rounding).
+    """
+    n = int(seconds * fs)
+    if n < 2:
+        return 2
+    # Make even
+    return n if n % 2 == 0 else n + 1
+
+
+def compute_window_parameters(
+    fft_len_sec: float, diss_len_sec: float, fs: float
+) -> tuple[int, int, int, int]:
+    """Compute (n_fft, n_diss, fft_overlap, diss_overlap) matching Jesse's pyturb.
+
+    Jesse's algorithm:
+      n_fft  = int(fft_len * fs), made even
+      n_diss = round(diss_len * fs), rounded UP to nearest multiple of n_fft
+      fft_overlap = n_fft // 2
+      diss_overlap = n_fft // 2
+    """
+    n_fft = seconds_to_samples(fft_len_sec, fs)
+    n_diss_raw = round(diss_len_sec * fs)
+    # Round up to nearest multiple of n_fft
+    n_diss = n_fft if n_diss_raw <= n_fft else ((n_diss_raw + n_fft - 1) // n_fft) * n_fft
+    fft_overlap = n_fft // 2
+    diss_overlap = n_fft // 2
+    return n_fft, n_diss, fft_overlap, diss_overlap
+
+
+def check_overwrite(path: Path, overwrite: bool) -> bool:
+    """Return True if *path* should be written (does not exist, or overwrite is set)."""
+    if not path.exists():
+        return True
+    return overwrite
+
+
+def load_auxiliary(
+    path: str | Path,
+    *,
+    lat_var: str = "lat",
+    lon_var: str = "lon",
+    temp_var: str = "temperature",
+    sal_var: str = "salinity",
+    dens_var: str = "density",
+) -> xr.Dataset:
+    """Load an auxiliary CTD NetCDF and validate required variables exist."""
+    ds = xr.open_dataset(path)
+    for name, var in [
+        ("lat", lat_var),
+        ("lon", lon_var),
+        ("temp", temp_var),
+        ("sal", sal_var),
+        ("dens", dens_var),
+    ]:
+        if var not in ds:
+            raise KeyError(f"Auxiliary file missing variable '{var}' (for {name})")
+    return ds
+
+
+def rename_eps_dataset(
+    l4,
+    l3,
+    l3_chi,
+    *,
+    aux_data: xr.Dataset | None = None,
+    fs_fast: float = 512.0,
+    salinity: float = 35.0,
+) -> xr.Dataset:
+    """Build an output Dataset with Jesse's variable naming convention.
+
+    Parameters
+    ----------
+    l4 : L4Data
+        Level-4 dissipation estimates.
+    l3 : L3Data
+        Level-3 shear wavenumber spectra.
+    l3_chi : L3ChiData or None
+        Level-3 temperature gradient spectra (may be None).
+    aux_data : xr.Dataset, optional
+        Auxiliary CTD data to merge.
+    fs_fast : float
+        Fast sampling rate for metadata.
+    salinity : float
+        Default salinity for viscosity computation.
+    """
+    from odas_tpw.scor160.ocean import density as calc_density
+    from odas_tpw.scor160.ocean import visc35
+
+    n_spec = l4.n_spectra
+    if n_spec == 0:
+        return xr.Dataset()
+
+    # Scalar arrays
+    pres = l4.pres
+    speed = l4.pspd_rel
+    temp = l3.temp
+
+    # Viscosity and density
+    nu = np.array([visc35(t) for t in temp])
+    sal = np.full(n_spec, salinity)
+    dens = np.array([calc_density(temp[i], sal[i], pres[i]) for i in range(n_spec)])
+
+    # Frequency and wavenumber vectors (from first window)
+    n_freq = l3.kcyc.shape[0]
+    freq = np.arange(n_freq) * fs_fast / ((n_freq - 1) * 2)
+    # Mean wavenumber across windows for coordinate
+    k_mean = np.mean(l3.kcyc, axis=1)
+
+    data_vars: dict = {
+        "pressure": (["time"], pres, {"units": "dbar", "long_name": "mean pressure"}),
+        "W": (["time"], speed, {"units": "m/s", "long_name": "profiling speed"}),
+        "temperature": (
+            ["time"], temp, {"units": "degree_Celsius", "long_name": "mean temperature"},
+        ),
+        "nu": (["time"], nu, {"units": "m2/s", "long_name": "kinematic viscosity"}),
+        "salinity": (["time"], sal, {"units": "PSU", "long_name": "practical salinity"}),
+        "density": (["time"], dens, {"units": "kg/m3", "long_name": "seawater density"}),
+    }
+
+    # Epsilon per probe
+    for i in range(l4.n_shear):
+        data_vars[f"eps_{i + 1}"] = (
+            ["time"],
+            l4.epsi[i],
+            {"units": "W/kg", "long_name": f"TKE dissipation rate (probe {i + 1})"},
+        )
+        data_vars[f"k_max_{i + 1}"] = (
+            ["time"],
+            l4.kmax[i],
+            {"units": "cpm", "long_name": f"upper integration limit (probe {i + 1})"},
+        )
+        data_vars[f"fom_{i + 1}"] = (
+            ["time"],
+            l4.fom[i],
+            {"long_name": f"figure of merit (probe {i + 1})"},
+        )
+        data_vars[f"mad_{i + 1}"] = (
+            ["time"],
+            l4.mad[i],
+            {"long_name": f"mean absolute deviation (probe {i + 1})"},
+        )
+        data_vars[f"method_{i + 1}"] = (
+            ["time"],
+            l4.method[i],
+            {"long_name": f"estimation method (probe {i + 1}): 0=variance, 1=ISR"},
+        )
+
+    data_vars["eps_final"] = (
+        ["time"],
+        l4.epsi_final,
+        {"units": "W/kg", "long_name": "combined TKE dissipation rate"},
+    )
+
+    # Shear spectra per probe
+    for i in range(l3.n_shear):
+        data_vars[f"S_sh{i + 1}"] = (
+            ["frequency", "time"],
+            l3.sh_spec_clean[i],
+            {
+                "units": "s-2/cpm",
+                "long_name": (
+                    f"shear spectrum (probe {i + 1}); Goodman-cleaned when "
+                    "--goodman is set, otherwise the raw spectrum"
+                ),
+            },
+        )
+
+    # Temperature gradient spectra
+    if l3_chi is not None and l3_chi.n_spectra > 0:
+        for i in range(l3_chi.n_gradt):
+            data_vars[f"S_gradT{i + 1}"] = (
+                ["frequency", "time"],
+                l3_chi.gradt_spec[i],
+                {"units": "K2/m2/cpm", "long_name": f"gradT spectrum (probe {i + 1})"},
+            )
+
+    coords: dict = {
+        "frequency": freq,
+        "k": (["frequency"], k_mean, {"units": "cpm", "long_name": "wavenumber"}),
+    }
+
+    ds = xr.Dataset(data_vars, coords=coords)
+
+    # Merge auxiliary data if provided
+    if aux_data is not None:
+        for var in aux_data.data_vars:
+            if var not in ds:
+                ds[var] = aux_data[var]
+
+    return ds

odas_tpw/pyturb/_profind.py

@@ -0,0 +1,124 @@
+"""Peak-finding profile detection (Jesse's algorithm).
+
+Uses scipy.signal.find_peaks on smoothed pressure to identify
+up-cast and down-cast segments.  Does not modify existing
+rsi/profile.py or scor160/profile.py.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from scipy.signal import butter, filtfilt, find_peaks
+
+
+def find_profiles_peaks(
+    pressure: np.ndarray,
+    fs: float,
+    *,
+    direction: str = "down",
+    min_pressure: float = 0.0,
+    peaks_height: float = 25.0,
+    peaks_distance: int = 200,
+    peaks_prominence: float = 25.0,
+    min_speed: float = 0.2,
+    smoothing_tau: float = 2.0,
+) -> list[tuple[int, int]]:
+    """Find profiles using scipy peak-finding on smoothed pressure.
+
+    Parameters
+    ----------
+    pressure : ndarray
+        Pressure time series [dbar] at slow rate.
+    fs : float
+        Sampling rate of *pressure* [Hz].
+    direction : str
+        'down', 'up', or 'both'.
+    min_pressure : float
+        Minimum pressure for valid profile segments [dbar].
+    peaks_height : float
+        Minimum peak height for ``find_peaks`` [dbar].
+    peaks_distance : int
+        Minimum distance between peaks [samples].
+    peaks_prominence : float
+        Minimum peak prominence [dbar].
+    min_speed : float
+        Minimum vertical speed to include a sample [m/s].
+    smoothing_tau : float
+        Time constant for Butterworth low-pass smoothing [s].
+
+    Returns
+    -------
+    list of (start_idx, end_idx)
+        Start and end indices (inclusive) in slow-rate samples.
+    """
+    pressure = np.asarray(pressure, dtype=np.float64)
+    if len(pressure) < 10:
+        return []
+
+    # Low-pass smooth pressure
+    f_c = 0.68 / max(smoothing_tau, 0.1)
+    nyquist = fs / 2.0
+    if f_c >= nyquist:
+        f_c = nyquist * 0.9
+    b, a = butter(2, f_c / nyquist)
+    p_smooth = filtfilt(b, a, pressure)
+
+    # Find pressure maxima (bottom of downcasts)
+    maxima, _ = find_peaks(
+        p_smooth,
+        height=peaks_height,
+        distance=peaks_distance,
+        prominence=peaks_prominence,
+    )
+
+    # Find pressure minima (top of casts) — peaks of inverted pressure
+    minima, _ = find_peaks(
+        -p_smooth,
+        distance=peaks_distance,
+    )
+
+    if len(maxima) == 0:
+        return []
+
+    # Build sorted array of all extrema with type labels
+    events: list[tuple[int, str]] = []
+    for idx in maxima:
+        events.append((int(idx), "max"))
+    for idx in minima:
+        events.append((int(idx), "min"))
+    events.sort(key=lambda x: x[0])
+
+    profiles: list[tuple[int, int]] = []
+
+    if direction in ("down", "both"):
+        # Downcasts: from min to max
+        for i, (idx, typ) in enumerate(events):
+            if typ != "max":
+                continue
+            # Find preceding minimum
+            prev_min = 0
+            for j in range(i - 1, -1, -1):
+                if events[j][1] == "min":
+                    prev_min = events[j][0]
+                    break
+            if p_smooth[idx] - p_smooth[prev_min] >= min_pressure:
+                profiles.append((prev_min, idx))
+
+    if direction in ("up", "both"):
+        # Upcasts: from max to min
+        for i, (idx, typ) in enumerate(events):
+            if typ != "max":
+                continue
+            # Find following minimum
+            next_min = len(pressure) - 1
+            for j in range(i + 1, len(events)):
+                if events[j][1] == "min":
+                    next_min = events[j][0]
+                    break
+            if p_smooth[idx] - p_smooth[next_min] >= min_pressure:
+                profiles.append((idx, next_min))
+
+    # Sort by start index
+    profiles.sort(key=lambda x: x[0])
+
+    return profiles