microstructure-tpw 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- microstructure_tpw-0.3.0.dist-info/METADATA +234 -0
- microstructure_tpw-0.3.0.dist-info/RECORD +119 -0
- microstructure_tpw-0.3.0.dist-info/WHEEL +5 -0
- microstructure_tpw-0.3.0.dist-info/entry_points.txt +7 -0
- microstructure_tpw-0.3.0.dist-info/licenses/LICENSE +674 -0
- microstructure_tpw-0.3.0.dist-info/top_level.txt +1 -0
- odas_tpw/__init__.py +18 -0
- odas_tpw/chi/__init__.py +48 -0
- odas_tpw/chi/batchelor.py +189 -0
- odas_tpw/chi/chi.py +857 -0
- odas_tpw/chi/fp07.py +649 -0
- odas_tpw/chi/l2_chi.py +153 -0
- odas_tpw/chi/l3_chi.py +388 -0
- odas_tpw/chi/l4_chi.py +391 -0
- odas_tpw/config_base.py +423 -0
- odas_tpw/perturb/__init__.py +18 -0
- odas_tpw/perturb/adcp.py +274 -0
- odas_tpw/perturb/atomic_io.py +47 -0
- odas_tpw/perturb/binning.py +761 -0
- odas_tpw/perturb/cli.py +591 -0
- odas_tpw/perturb/combo.py +346 -0
- odas_tpw/perturb/config.py +1046 -0
- odas_tpw/perturb/ctd.py +424 -0
- odas_tpw/perturb/diag/__init__.py +12 -0
- odas_tpw/perturb/diag/_common.py +140 -0
- odas_tpw/perturb/diag/chi.py +42 -0
- odas_tpw/perturb/diag/cli.py +67 -0
- odas_tpw/perturb/diag/data.py +662 -0
- odas_tpw/perturb/diag/epsilon.py +45 -0
- odas_tpw/perturb/diag/inspector.py +342 -0
- odas_tpw/perturb/diag/mixing.py +94 -0
- odas_tpw/perturb/diag/render.py +543 -0
- odas_tpw/perturb/discover.py +40 -0
- odas_tpw/perturb/fp07_cal.py +354 -0
- odas_tpw/perturb/gps.py +458 -0
- odas_tpw/perturb/hotel.py +572 -0
- odas_tpw/perturb/logging_setup.py +181 -0
- odas_tpw/perturb/merge.py +320 -0
- odas_tpw/perturb/mk_sections.py +310 -0
- odas_tpw/perturb/netcdf_schema.py +520 -0
- odas_tpw/perturb/pipeline.py +2654 -0
- odas_tpw/perturb/plot/__init__.py +15 -0
- odas_tpw/perturb/plot/cli.py +107 -0
- odas_tpw/perturb/plot/diagnostics.py +195 -0
- odas_tpw/perturb/plot/eps_chi.py +524 -0
- odas_tpw/perturb/plot/figure.py +679 -0
- odas_tpw/perturb/plot/gamma_scaling.py +1097 -0
- odas_tpw/perturb/plot/grid.py +112 -0
- odas_tpw/perturb/plot/layout.py +361 -0
- odas_tpw/perturb/plot/overview.py +547 -0
- odas_tpw/perturb/plot/presets/eps-chi.yaml +16 -0
- odas_tpw/perturb/plot/presets/profiles.yaml +32 -0
- odas_tpw/perturb/plot/presets/scalar.yaml +21 -0
- odas_tpw/perturb/plot/profiles.py +652 -0
- odas_tpw/perturb/plot/scalar.py +391 -0
- odas_tpw/perturb/plot/sections.py +506 -0
- odas_tpw/perturb/plot/xaxis.py +396 -0
- odas_tpw/perturb/qc_gate.py +177 -0
- odas_tpw/perturb/qc_rules.py +373 -0
- odas_tpw/perturb/resolve.py +306 -0
- odas_tpw/perturb/seawater.py +76 -0
- odas_tpw/perturb/speed.py +14 -0
- odas_tpw/perturb/trim.py +313 -0
- odas_tpw/processing/__init__.py +62 -0
- odas_tpw/processing/bottom.py +215 -0
- odas_tpw/processing/chi_combine.py +300 -0
- odas_tpw/processing/ct_align.py +172 -0
- odas_tpw/processing/epsilon_combine.py +210 -0
- odas_tpw/processing/mixing.py +630 -0
- odas_tpw/processing/thorpe.py +458 -0
- odas_tpw/processing/top_trim.py +274 -0
- odas_tpw/pyturb/__init__.py +3 -0
- odas_tpw/pyturb/__main__.py +5 -0
- odas_tpw/pyturb/_compat.py +204 -0
- odas_tpw/pyturb/_profind.py +124 -0
- odas_tpw/pyturb/bin.py +113 -0
- odas_tpw/pyturb/cli.py +175 -0
- odas_tpw/pyturb/eps.py +353 -0
- odas_tpw/pyturb/merge.py +101 -0
- odas_tpw/pyturb/p2nc.py +76 -0
- odas_tpw/rsi/__init__.py +31 -0
- odas_tpw/rsi/adapter.py +314 -0
- odas_tpw/rsi/bench.py +904 -0
- odas_tpw/rsi/binning.py +193 -0
- odas_tpw/rsi/channels.py +376 -0
- odas_tpw/rsi/chi_io.py +980 -0
- odas_tpw/rsi/cli.py +1411 -0
- odas_tpw/rsi/combine.py +120 -0
- odas_tpw/rsi/config.py +126 -0
- odas_tpw/rsi/config_patch.py +790 -0
- odas_tpw/rsi/convert.py +651 -0
- odas_tpw/rsi/deconvolve.py +119 -0
- odas_tpw/rsi/diss_look.py +343 -0
- odas_tpw/rsi/dissipation.py +669 -0
- odas_tpw/rsi/helpers.py +527 -0
- odas_tpw/rsi/mixing_look.py +509 -0
- odas_tpw/rsi/p_file.py +771 -0
- odas_tpw/rsi/pipeline.py +924 -0
- odas_tpw/rsi/profile.py +624 -0
- odas_tpw/rsi/quick_look.py +758 -0
- odas_tpw/rsi/sensor_inventory.py +684 -0
- odas_tpw/rsi/shear_noise.py +128 -0
- odas_tpw/rsi/speed.py +221 -0
- odas_tpw/rsi/vehicle.py +74 -0
- odas_tpw/rsi/viewer_base.py +1071 -0
- odas_tpw/rsi/window.py +459 -0
- odas_tpw/scor160/__init__.py +36 -0
- odas_tpw/scor160/cli.py +242 -0
- odas_tpw/scor160/compare.py +484 -0
- odas_tpw/scor160/despike.py +227 -0
- odas_tpw/scor160/goodman.py +288 -0
- odas_tpw/scor160/io.py +690 -0
- odas_tpw/scor160/l2.py +311 -0
- odas_tpw/scor160/l3.py +279 -0
- odas_tpw/scor160/l4.py +850 -0
- odas_tpw/scor160/nasmyth.py +208 -0
- odas_tpw/scor160/ocean.py +287 -0
- odas_tpw/scor160/profile.py +231 -0
- odas_tpw/scor160/spectral.py +482 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Mar-2026, Claude and Pat Welch, pat@mousebrains.com
|
|
2
|
+
"""Seawater properties via TEOS-10 (gsw).
|
|
3
|
+
|
|
4
|
+
Reference: Code/add_seawater_properties.m
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
import numpy.typing as npt
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def add_seawater_properties(
|
|
12
|
+
T: npt.ArrayLike,
|
|
13
|
+
C: npt.ArrayLike,
|
|
14
|
+
P: npt.ArrayLike,
|
|
15
|
+
lat: npt.ArrayLike,
|
|
16
|
+
lon: npt.ArrayLike,
|
|
17
|
+
) -> dict[str, np.ndarray]:
|
|
18
|
+
"""Compute seawater properties from T, C, P, lat, lon.
|
|
19
|
+
|
|
20
|
+
Parameters
|
|
21
|
+
----------
|
|
22
|
+
T : array_like
|
|
23
|
+
In-situ temperature [degC] (from slow CT sensor, e.g. JAC_T).
|
|
24
|
+
C : array_like
|
|
25
|
+
Conductivity [mS/cm].
|
|
26
|
+
P : array_like
|
|
27
|
+
Pressure [dbar].
|
|
28
|
+
lat, lon : array_like
|
|
29
|
+
Latitude and longitude [degrees].
|
|
30
|
+
|
|
31
|
+
Returns
|
|
32
|
+
-------
|
|
33
|
+
dict with keys: SP, SA, CT, sigma0, rho, depth
|
|
34
|
+
SP = practical salinity [PSU]
|
|
35
|
+
SA = absolute salinity [g/kg]
|
|
36
|
+
CT = conservative temperature [degC]
|
|
37
|
+
sigma0 = potential density anomaly [kg/m^3]
|
|
38
|
+
rho = in-situ density - 1000 [kg/m^3]
|
|
39
|
+
depth = depth (positive downward) [m]
|
|
40
|
+
"""
|
|
41
|
+
import gsw
|
|
42
|
+
|
|
43
|
+
# Fill masked cells with NaN (do NOT np.asarray a masked array — that drops
|
|
44
|
+
# the mask and exposes raw _FillValue, e.g. -999, as if it were real T/C/P).
|
|
45
|
+
def _filled(x):
|
|
46
|
+
return np.ma.filled(np.ma.asarray(x).astype(np.float64), np.nan)
|
|
47
|
+
|
|
48
|
+
T = _filled(T)
|
|
49
|
+
C = _filled(C)
|
|
50
|
+
P = _filled(P)
|
|
51
|
+
lat = _filled(lat)
|
|
52
|
+
lon = _filled(lon)
|
|
53
|
+
|
|
54
|
+
# Replace NaN lat/lon with 0 for gsw (matches Matlab default)
|
|
55
|
+
lat_safe = np.where(np.isnan(lat), 0.0, lat)
|
|
56
|
+
lon_safe = np.where(np.isnan(lon), 0.0, lon)
|
|
57
|
+
|
|
58
|
+
SP = gsw.SP_from_C(C, T, P)
|
|
59
|
+
SA = gsw.SA_from_SP(SP, P, lon_safe, lat_safe)
|
|
60
|
+
CT = gsw.CT_from_t(SA, T, P)
|
|
61
|
+
sigma0 = gsw.sigma0(SA, CT)
|
|
62
|
+
rho = gsw.rho(SA, CT, P) - 1000.0
|
|
63
|
+
# gsw.z_from_p returns height Z (m, <=0 below the surface); positive-down
|
|
64
|
+
# depth is -Z. Using -Z (not |Z|) keeps above-surface samples (P<0, e.g.
|
|
65
|
+
# deck calibrations) as negative depths instead of folding them back down.
|
|
66
|
+
z = gsw.z_from_p(P, lat_safe)
|
|
67
|
+
depth = -z
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
"SP": np.asarray(SP),
|
|
71
|
+
"SA": np.asarray(SA),
|
|
72
|
+
"CT": np.asarray(CT),
|
|
73
|
+
"sigma0": np.asarray(sigma0),
|
|
74
|
+
"rho": np.asarray(rho),
|
|
75
|
+
"depth": np.asarray(depth),
|
|
76
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# May-2026, Pat Welch, pat@mousebrains.com
|
|
2
|
+
"""Through-water speed for the perturb pipeline.
|
|
3
|
+
|
|
4
|
+
The implementation now lives in :mod:`odas_tpw.rsi.speed` so the lower-level
|
|
5
|
+
``rsi`` pipeline can share the exact same speed model (pressure / em / flight /
|
|
6
|
+
constant) without a backwards dependency on ``perturb``. This module re-exports
|
|
7
|
+
it for backward compatibility.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from odas_tpw.rsi.speed import compute_speed_for_pfile
|
|
13
|
+
|
|
14
|
+
__all__ = ["compute_speed_for_pfile"]
|
odas_tpw/perturb/trim.py
ADDED
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
# Mar-2026, Claude and Pat Welch, pat@mousebrains.com
|
|
2
|
+
"""Corrupt record trimming for .p files.
|
|
3
|
+
|
|
4
|
+
If the last record is incomplete (fractional), copy only complete records
|
|
5
|
+
to a new file.
|
|
6
|
+
|
|
7
|
+
Reference: Code/trim_P_files.m
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import contextlib
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import struct
|
|
14
|
+
import tempfile
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import NamedTuple
|
|
17
|
+
|
|
18
|
+
from odas_tpw.rsi.p_file import _H, HEADER_BYTES, HEADER_WORDS, _detect_endian
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TrimResult(NamedTuple):
|
|
22
|
+
"""Outcome of a single :func:`trim_p_file` call.
|
|
23
|
+
|
|
24
|
+
Attributes
|
|
25
|
+
----------
|
|
26
|
+
dest : Path
|
|
27
|
+
The path downstream stages should read. For ``"trimmed"`` /
|
|
28
|
+
``"skipped"`` this is the rewritten file under the trim directory;
|
|
29
|
+
for ``"referenced"`` it is the *original* source path, left in place.
|
|
30
|
+
action : str
|
|
31
|
+
One of:
|
|
32
|
+
|
|
33
|
+
* ``"trimmed"`` — a fractional final record was dropped; a new
|
|
34
|
+
truncated file was written to the trim directory.
|
|
35
|
+
* ``"referenced"`` — the file was already complete, so it is used
|
|
36
|
+
in place from its original location (no copy, no link).
|
|
37
|
+
* ``"skipped"`` — an up-to-date trimmed output already existed;
|
|
38
|
+
no work done.
|
|
39
|
+
bytes_removed : int
|
|
40
|
+
Bytes dropped from the incomplete final record. 0 unless
|
|
41
|
+
*action* is ``"trimmed"``.
|
|
42
|
+
record_size : int
|
|
43
|
+
The .p record size in bytes, read from the header. 0 when the file
|
|
44
|
+
was skipped (the header is not read on the skip path).
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
dest: Path
|
|
48
|
+
action: str
|
|
49
|
+
bytes_removed: int
|
|
50
|
+
record_size: int
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def trim_destination(source: Path, output_dir: Path, root: Path | str | None = None) -> Path:
|
|
54
|
+
"""Return the trim output path for *source*.
|
|
55
|
+
|
|
56
|
+
When *root* is supplied and *source* lives underneath it, the relative
|
|
57
|
+
directory structure is preserved under *output_dir*. This prevents files
|
|
58
|
+
such as ``SN001/cast.p`` and ``SN002/cast.p`` from overwriting each other.
|
|
59
|
+
"""
|
|
60
|
+
source = Path(source)
|
|
61
|
+
output_dir = Path(output_dir)
|
|
62
|
+
if root is not None:
|
|
63
|
+
try:
|
|
64
|
+
rel = source.resolve().relative_to(Path(root).resolve())
|
|
65
|
+
except ValueError:
|
|
66
|
+
rel = Path(source.name)
|
|
67
|
+
return output_dir / rel
|
|
68
|
+
return output_dir / source.name
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ---------------------------------------------------------------------------
|
|
72
|
+
# Trim decision cache (incremental re-runs / slow mounts)
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
# A re-run otherwise re-opens and header-reads EVERY source .p to decide it
|
|
75
|
+
# needs no trimming — cheap locally, but minutes over an SMB/network mount where
|
|
76
|
+
# each open is a round-trip. The cache records, per source, a ``size + 2 s-mtime``
|
|
77
|
+
# fingerprint and whether the file was complete (referenced) or trimmed, so an
|
|
78
|
+
# unchanged file is decided from a single ``stat`` with no header read. Keyed on
|
|
79
|
+
# the source's path *relative to the trim dir* (portable across machines/mounts),
|
|
80
|
+
# and only trusted when its physical output is still present. ``force`` / a
|
|
81
|
+
# changed fingerprint re-reads the header. mtime is binned to 2 s so the cache
|
|
82
|
+
# survives a copy to exFAT (2 s mtime granularity) — the same file-identity model
|
|
83
|
+
# as the per-file processing cache (pipeline._file_fingerprint). A same-size
|
|
84
|
+
# header-geometry change (e.g. record_size) landing in the same 2 s mtime bin is
|
|
85
|
+
# therefore not re-detected without ``force``; this is field-unreachable for raw
|
|
86
|
+
# acquisition files.
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _trim_fingerprint(source: Path) -> dict | None:
|
|
90
|
+
try:
|
|
91
|
+
st = source.stat()
|
|
92
|
+
except OSError:
|
|
93
|
+
return None
|
|
94
|
+
return {"size": st.st_size, "mtime_2s": int(st.st_mtime) // 2 * 2}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _trim_marker(cache_dir: Path, dest: Path, output_dir: Path) -> Path:
|
|
98
|
+
"""Marker path mirroring the source's relative layout (collision-free,
|
|
99
|
+
portable). *dest* is ``trim_destination(source, output_dir, root)``."""
|
|
100
|
+
try:
|
|
101
|
+
rel = dest.relative_to(output_dir).as_posix()
|
|
102
|
+
except ValueError:
|
|
103
|
+
rel = dest.name
|
|
104
|
+
return cache_dir / "trim" / f"{rel}.json"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _trim_cache_lookup(
|
|
108
|
+
cache_dir: Path, source: Path, dest: Path, output_dir: Path
|
|
109
|
+
) -> "TrimResult | None":
|
|
110
|
+
marker = _trim_marker(cache_dir, dest, output_dir)
|
|
111
|
+
# Any unreadable/malformed marker is a cache MISS, never a hard failure: the
|
|
112
|
+
# cache only accelerates the header-read decision, so a corrupted / manually
|
|
113
|
+
# edited / future-schema-bad payload must fall through to the real trim.
|
|
114
|
+
try:
|
|
115
|
+
data = json.loads(marker.read_text())
|
|
116
|
+
if not isinstance(data, dict) or data.get("fp") != _trim_fingerprint(source):
|
|
117
|
+
return None
|
|
118
|
+
if data.get("referenced"):
|
|
119
|
+
# Complete file, unchanged — reference the original, no header read.
|
|
120
|
+
return TrimResult(source, "referenced", 0, int(data.get("record_size", 0)))
|
|
121
|
+
# Previously trimmed — reuse the existing trimmed output only if it is
|
|
122
|
+
# still present AND the right size. The size check preserves the original
|
|
123
|
+
# guard against a truncated/corrupted/externally-replaced trimmed output
|
|
124
|
+
# (the source fingerprint matched, but the *output* could have moved).
|
|
125
|
+
if dest.stat().st_size == data.get("dest_size"):
|
|
126
|
+
return TrimResult(dest, "skipped", 0, 0)
|
|
127
|
+
except (OSError, ValueError, TypeError):
|
|
128
|
+
return None
|
|
129
|
+
return None
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _trim_cache_store(
|
|
133
|
+
cache_dir: Path,
|
|
134
|
+
source: Path,
|
|
135
|
+
dest: Path,
|
|
136
|
+
output_dir: Path,
|
|
137
|
+
result: "TrimResult",
|
|
138
|
+
fp: dict | None,
|
|
139
|
+
) -> None:
|
|
140
|
+
# ``fp`` is the source fingerprint snapshot taken BEFORE the trim read (see
|
|
141
|
+
# trim_p_file). Re-stat'ing the source here instead would fold in any records
|
|
142
|
+
# appended during the trim (live acquisition / an rsync completing mid-trim),
|
|
143
|
+
# so a later run would match the grown source against a short trimmed output
|
|
144
|
+
# and silently drop the appended records.
|
|
145
|
+
if fp is None:
|
|
146
|
+
return
|
|
147
|
+
marker = _trim_marker(cache_dir, dest, output_dir)
|
|
148
|
+
payload = {
|
|
149
|
+
"fp": fp,
|
|
150
|
+
"referenced": result.action == "referenced",
|
|
151
|
+
"record_size": result.record_size,
|
|
152
|
+
}
|
|
153
|
+
if result.action != "referenced":
|
|
154
|
+
# Record the trimmed output's size so a later run can verify it wasn't
|
|
155
|
+
# truncated/replaced before reusing it (see _trim_cache_lookup).
|
|
156
|
+
try:
|
|
157
|
+
payload["dest_size"] = dest.stat().st_size
|
|
158
|
+
except OSError:
|
|
159
|
+
return # can't verify the output later -> don't cache it
|
|
160
|
+
try:
|
|
161
|
+
marker.parent.mkdir(parents=True, exist_ok=True)
|
|
162
|
+
tmp = marker.with_name(f"{marker.name}.{os.getpid()}.tmp")
|
|
163
|
+
tmp.write_text(json.dumps(payload))
|
|
164
|
+
os.replace(tmp, marker)
|
|
165
|
+
except OSError:
|
|
166
|
+
pass # caching is best-effort; never fail a trim over a marker write
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def trim_p_file(
|
|
170
|
+
source: Path,
|
|
171
|
+
output_dir: Path,
|
|
172
|
+
root: Path | str | None = None,
|
|
173
|
+
*,
|
|
174
|
+
force: bool = False,
|
|
175
|
+
cache_dir: Path | None = None,
|
|
176
|
+
) -> TrimResult:
|
|
177
|
+
"""Trim an incomplete final record from a .p file, or reference it in place.
|
|
178
|
+
|
|
179
|
+
Complete files are *not* copied: the original is used in place
|
|
180
|
+
(``action="referenced"``) so the trim directory only ever holds files
|
|
181
|
+
that were genuinely truncated. This avoids duplicating large raw files
|
|
182
|
+
and needs no filesystem-specific support (hard/soft links, reflinks),
|
|
183
|
+
so it works identically on local disks, SMB/network mounts, and Windows.
|
|
184
|
+
|
|
185
|
+
Parameters
|
|
186
|
+
----------
|
|
187
|
+
source : Path
|
|
188
|
+
Path to the original .p file.
|
|
189
|
+
output_dir : Path
|
|
190
|
+
Directory for any trimmed output file. Untouched for complete files.
|
|
191
|
+
root : Path, optional
|
|
192
|
+
Source root used to preserve relative paths under *output_dir*.
|
|
193
|
+
force : bool, keyword-only, default False
|
|
194
|
+
Re-trim even when an up-to-date trimmed output already exists. By
|
|
195
|
+
default an existing trimmed file at least as new as *source* is
|
|
196
|
+
reused (``action="skipped"``).
|
|
197
|
+
|
|
198
|
+
Returns
|
|
199
|
+
-------
|
|
200
|
+
TrimResult
|
|
201
|
+
See :class:`TrimResult`. ``dest`` is the original source for
|
|
202
|
+
``"referenced"`` files and the trim-directory path otherwise.
|
|
203
|
+
|
|
204
|
+
Notes
|
|
205
|
+
-----
|
|
206
|
+
Skip guard (trimmed files only): the header is read every run, and a
|
|
207
|
+
trimmed output is reused only when its size equals the freshly-computed
|
|
208
|
+
trimmed size AND ``dest_mtime >= source_mtime`` (the trim-write path
|
|
209
|
+
stamps a "now" mtime newer than the older source). A source modified
|
|
210
|
+
after its trimmed output (newer mtime), repaired to completeness, or
|
|
211
|
+
given a changed header geometry therefore re-trims/re-references rather
|
|
212
|
+
than reusing a stale truncated copy. Complete files write nothing — they
|
|
213
|
+
are re-referenced each run at the cost of a single 128-byte header read.
|
|
214
|
+
"""
|
|
215
|
+
dest = trim_destination(source, output_dir, root=root)
|
|
216
|
+
|
|
217
|
+
# Snapshot the source fingerprint BEFORE any read, so a source that grows
|
|
218
|
+
# during the trim (appended records) fails the fingerprint match on the next
|
|
219
|
+
# run and is re-trimmed, rather than matching against a short trimmed copy.
|
|
220
|
+
source_fp = _trim_fingerprint(source)
|
|
221
|
+
|
|
222
|
+
# Incremental skip: an unchanged source (fingerprint match) reuses last
|
|
223
|
+
# run's decision from a single stat — no per-file header read/open, which is
|
|
224
|
+
# what costs minutes over a slow mount. force_trim re-reads.
|
|
225
|
+
if not force and cache_dir is not None:
|
|
226
|
+
cached = _trim_cache_lookup(cache_dir, source, dest, output_dir)
|
|
227
|
+
if cached is not None:
|
|
228
|
+
return cached
|
|
229
|
+
|
|
230
|
+
def _finish(result: TrimResult) -> TrimResult:
|
|
231
|
+
if cache_dir is not None:
|
|
232
|
+
_trim_cache_store(cache_dir, source, dest, output_dir, result, source_fp)
|
|
233
|
+
return result
|
|
234
|
+
|
|
235
|
+
# Read the source header first (the same 128-byte read the 'referenced'
|
|
236
|
+
# path already pays) so the skip decision is content-aware rather than
|
|
237
|
+
# mtime-only. A previously-incomplete source that was later repaired (now
|
|
238
|
+
# complete, or with changed header geometry) but carries an equal/older
|
|
239
|
+
# mtime than the stale trimmed output must NOT be skipped.
|
|
240
|
+
with open(source, "rb") as f:
|
|
241
|
+
raw_hdr = f.read(HEADER_BYTES)
|
|
242
|
+
if len(raw_hdr) < HEADER_BYTES:
|
|
243
|
+
raise ValueError(f"{source.name}: file too small for header")
|
|
244
|
+
|
|
245
|
+
endian = _detect_endian(raw_hdr, source)
|
|
246
|
+
fmt = f"{endian}{HEADER_WORDS}H"
|
|
247
|
+
words = struct.unpack(fmt, raw_hdr)
|
|
248
|
+
header = {name: words[idx] for name, idx in _H.items()}
|
|
249
|
+
|
|
250
|
+
header_size = header["header_size"]
|
|
251
|
+
config_size = header["config_size"]
|
|
252
|
+
record_size = header["record_size"]
|
|
253
|
+
|
|
254
|
+
first_record_size = header_size + config_size
|
|
255
|
+
|
|
256
|
+
f.seek(0, 2)
|
|
257
|
+
file_size = f.tell()
|
|
258
|
+
|
|
259
|
+
data_bytes = file_size - first_record_size
|
|
260
|
+
if data_bytes < 0:
|
|
261
|
+
raise ValueError(f"{source.name}: file smaller than first record")
|
|
262
|
+
|
|
263
|
+
if record_size <= 0:
|
|
264
|
+
raise ValueError(f"{source.name}: invalid record_size={record_size}")
|
|
265
|
+
|
|
266
|
+
remainder = data_bytes % record_size
|
|
267
|
+
if remainder == 0:
|
|
268
|
+
# Complete file — reference the original in place, no copy. Reached
|
|
269
|
+
# also when a once-incomplete source has since been completed, so its
|
|
270
|
+
# stale trimmed output is deliberately not reused.
|
|
271
|
+
return _finish(TrimResult(source, "referenced", 0, record_size))
|
|
272
|
+
|
|
273
|
+
# Trim the fractional final record into the trim directory.
|
|
274
|
+
n_complete = data_bytes // record_size
|
|
275
|
+
trimmed_size = first_record_size + n_complete * record_size
|
|
276
|
+
|
|
277
|
+
# Skip-if-up-to-date: a prior run wrote this exact trimmed output and the
|
|
278
|
+
# source is unchanged. Require the dest SIZE to equal the freshly-computed
|
|
279
|
+
# trimmed size (so a repaired or geometry-changed source re-trims rather
|
|
280
|
+
# than reusing a stale truncated copy) on top of the nanosecond-mtime
|
|
281
|
+
# guard. The trim-write path stamps a "now" mtime, so dest_mtime >=
|
|
282
|
+
# source_mtime holds for an unchanged source.
|
|
283
|
+
if (
|
|
284
|
+
not force
|
|
285
|
+
and dest.exists()
|
|
286
|
+
and dest.stat().st_size == trimmed_size
|
|
287
|
+
and dest.stat().st_mtime_ns >= source.stat().st_mtime_ns
|
|
288
|
+
):
|
|
289
|
+
return _finish(TrimResult(dest, "skipped", 0, 0))
|
|
290
|
+
|
|
291
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
292
|
+
# Write to a temp file then atomically replace. If the dest resolves to the
|
|
293
|
+
# source (files already under <root>/trimmed/, or a basename collision),
|
|
294
|
+
# opening dest "wb" directly would truncate the source to 0 bytes BEFORE the
|
|
295
|
+
# read — destroying it. A separate temp fd + os.replace keeps the source
|
|
296
|
+
# intact until the atomic swap.
|
|
297
|
+
fd, tmp = tempfile.mkstemp(dir=str(dest.parent), prefix=".trim-", suffix=".tmp")
|
|
298
|
+
try:
|
|
299
|
+
with open(source, "rb") as src_f, os.fdopen(fd, "wb") as dst_f:
|
|
300
|
+
remaining = trimmed_size
|
|
301
|
+
while remaining > 0:
|
|
302
|
+
chunk = src_f.read(min(remaining, 1 << 20))
|
|
303
|
+
if not chunk:
|
|
304
|
+
break
|
|
305
|
+
dst_f.write(chunk)
|
|
306
|
+
remaining -= len(chunk)
|
|
307
|
+
os.replace(tmp, dest)
|
|
308
|
+
except BaseException:
|
|
309
|
+
with contextlib.suppress(OSError):
|
|
310
|
+
os.unlink(tmp)
|
|
311
|
+
raise
|
|
312
|
+
|
|
313
|
+
return _finish(TrimResult(dest, "trimmed", remainder, record_size))
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# May-2026, Claude and Pat Welch, pat@mousebrains.com
|
|
2
|
+
"""Instrument-agnostic profile-level processing.
|
|
3
|
+
|
|
4
|
+
Operates on raw numpy arrays (and a thin layer of xarray for
|
|
5
|
+
mk_epsilon_mean) — no PFile, no NetCDF I/O, no perturb-specific config.
|
|
6
|
+
Suitable for any vertical profiler whose pipeline can hand over the
|
|
7
|
+
right arrays.
|
|
8
|
+
|
|
9
|
+
- :func:`top_trim.compute_trim_depth` — depth where shear / acceleration
|
|
10
|
+
variance settles below a quantile threshold (prop-wash exit).
|
|
11
|
+
- :func:`bottom.detect_bottom_crash` — depth where vibration spikes near
|
|
12
|
+
the deepest bin (seafloor impact).
|
|
13
|
+
- :func:`ct_align.ct_align` — cross-correlate diff(T) vs diff(C) and
|
|
14
|
+
shift C by the median lag across profiles.
|
|
15
|
+
- :func:`epsilon_combine.mk_epsilon_mean` — Lueck (2022) iterative 95% CI
|
|
16
|
+
geometric mean of multi-probe dissipation estimates.
|
|
17
|
+
- :func:`chi_combine.mk_chi_mean` — same iterative 95% CI machinery for
|
|
18
|
+
multi-probe thermal-variance dissipation chi.
|
|
19
|
+
- :func:`mixing.window_stratification` / :func:`mixing.mixing_coefficients`
|
|
20
|
+
— background N² and dT/dz per dissipation window, and the derived
|
|
21
|
+
mixing quantities K_T (Osborn-Cox), Gamma (Oakey), K_rho (Osborn).
|
|
22
|
+
- :func:`thorpe.window_thorpe` / :func:`thorpe.patch_n2` — per-window
|
|
23
|
+
Thorpe displacements/scale L_T and the overturn-weighted "patch"
|
|
24
|
+
stratification (Smyth et al. 2001), plus Ozmidov/R_OT/Re_b/Cox helpers.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from odas_tpw.processing.bottom import detect_bottom_crash
|
|
28
|
+
from odas_tpw.processing.chi_combine import mk_chi_mean
|
|
29
|
+
from odas_tpw.processing.ct_align import ct_align
|
|
30
|
+
from odas_tpw.processing.epsilon_combine import mk_epsilon_mean
|
|
31
|
+
from odas_tpw.processing.mixing import (
|
|
32
|
+
mixing_coefficients,
|
|
33
|
+
pair_nearest,
|
|
34
|
+
window_stratification,
|
|
35
|
+
)
|
|
36
|
+
from odas_tpw.processing.thorpe import (
|
|
37
|
+
cox_number,
|
|
38
|
+
ozmidov,
|
|
39
|
+
patch_n2,
|
|
40
|
+
r_ot,
|
|
41
|
+
reynolds_buoyancy,
|
|
42
|
+
window_thorpe,
|
|
43
|
+
)
|
|
44
|
+
from odas_tpw.processing.top_trim import compute_trim_depth, compute_trim_depths
|
|
45
|
+
|
|
46
|
+
__all__ = [
|
|
47
|
+
"compute_trim_depth",
|
|
48
|
+
"compute_trim_depths",
|
|
49
|
+
"cox_number",
|
|
50
|
+
"ct_align",
|
|
51
|
+
"detect_bottom_crash",
|
|
52
|
+
"mixing_coefficients",
|
|
53
|
+
"mk_chi_mean",
|
|
54
|
+
"mk_epsilon_mean",
|
|
55
|
+
"ozmidov",
|
|
56
|
+
"pair_nearest",
|
|
57
|
+
"patch_n2",
|
|
58
|
+
"r_ot",
|
|
59
|
+
"reynolds_buoyancy",
|
|
60
|
+
"window_stratification",
|
|
61
|
+
"window_thorpe",
|
|
62
|
+
]
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# Mar-2026, Claude and Pat Welch, pat@mousebrains.com
|
|
2
|
+
"""Bottom-crash detection — find seafloor impact from a vibration signal.
|
|
3
|
+
|
|
4
|
+
Caller supplies any number of fast-rate vibration channels as a dict
|
|
5
|
+
(e.g. ``{"Ax": …, "Ay": …}`` for two-axis accel, or just
|
|
6
|
+
``{"vibration": …}`` for a pre-computed RMS). The algorithm forms the
|
|
7
|
+
elementwise root-sum-square magnitude across all supplied channels, bins
|
|
8
|
+
its std by depth, and flags the deepest bin whose std exceeds
|
|
9
|
+
``vibration_factor`` times the median — but only within a few bins of the
|
|
10
|
+
deepest sampled depth, since a real seafloor strike is at the cast bottom.
|
|
11
|
+
A supra-threshold spike far above the bottom is mid-column vibration, not a
|
|
12
|
+
crash, and is reported via a warning rather than truncating the cast.
|
|
13
|
+
|
|
14
|
+
Reference: Code/bottom_crash_profiles.m (currently a stub in Matlab)
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
import warnings
|
|
18
|
+
from collections.abc import Mapping
|
|
19
|
+
|
|
20
|
+
import numpy as np
|
|
21
|
+
import numpy.typing as npt
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def detect_bottom_crash(
|
|
25
|
+
depth_fast: npt.ArrayLike,
|
|
26
|
+
vibration_channels: Mapping[str, npt.ArrayLike],
|
|
27
|
+
fs: float,
|
|
28
|
+
*,
|
|
29
|
+
depth_window: float = 4.0,
|
|
30
|
+
depth_minimum: float = 10.0,
|
|
31
|
+
speed_factor: float = 0.3,
|
|
32
|
+
median_factor: float = 1.0,
|
|
33
|
+
vibration_frequency: int = 16,
|
|
34
|
+
vibration_factor: float = 4.0,
|
|
35
|
+
proximity_bins: int = 2,
|
|
36
|
+
) -> float | None:
|
|
37
|
+
"""Detect bottom crash from a set of vibration channels.
|
|
38
|
+
|
|
39
|
+
Parameters
|
|
40
|
+
----------
|
|
41
|
+
depth_fast : array_like
|
|
42
|
+
Depth array (positive downward, fast rate) [m].
|
|
43
|
+
vibration_channels : mapping of name -> 1-D array
|
|
44
|
+
Fast-rate vibration / motion channels. Two-axis accel
|
|
45
|
+
``{"Ax": …, "Ay": …}`` reproduces the historical 2-axis
|
|
46
|
+
magnitude. A single pre-aggregated channel
|
|
47
|
+
(e.g. ``{"vibration_rms": rms}``) is also fine. All arrays
|
|
48
|
+
must match ``depth_fast`` in length.
|
|
49
|
+
fs : float
|
|
50
|
+
Fast sampling rate [Hz]. Currently UNUSED — accepted for
|
|
51
|
+
backward compatibility; the algorithm operates on depth bins,
|
|
52
|
+
not time.
|
|
53
|
+
depth_window : float
|
|
54
|
+
Depth window for variance computation [m].
|
|
55
|
+
depth_minimum : float
|
|
56
|
+
Minimum depth to start searching [m].
|
|
57
|
+
speed_factor : float
|
|
58
|
+
Currently UNUSED — accepted for backward compatibility (the
|
|
59
|
+
speed-drop confirmation from the original MATLAB design is not
|
|
60
|
+
implemented). Tuning it has no effect.
|
|
61
|
+
median_factor : float
|
|
62
|
+
Currently UNUSED — accepted for backward compatibility.
|
|
63
|
+
Tuning it has no effect.
|
|
64
|
+
vibration_frequency : int
|
|
65
|
+
Currently UNUSED — accepted for backward compatibility.
|
|
66
|
+
Tuning it has no effect.
|
|
67
|
+
vibration_factor : float
|
|
68
|
+
Vibration std dev acceptance factor: the deepest bin whose
|
|
69
|
+
magnitude std exceeds ``vibration_factor`` times the median
|
|
70
|
+
marks the crash.
|
|
71
|
+
proximity_bins : int
|
|
72
|
+
A crash is accepted only in a bin within ``proximity_bins`` of the
|
|
73
|
+
deepest sampled bin — a real seafloor strike is at the cast bottom,
|
|
74
|
+
so a supra-threshold spike higher up is mid-column vibration, not a
|
|
75
|
+
crash (audit r1-3). Must be >= 0; at the default ``depth_window=4``
|
|
76
|
+
m, ``proximity_bins=2`` accepts a strike within ~8 m of the deepest
|
|
77
|
+
sample. This is the geometric stand-in for the unimplemented
|
|
78
|
+
``speed_factor`` fall-rate-collapse confirmation.
|
|
79
|
+
|
|
80
|
+
Returns
|
|
81
|
+
-------
|
|
82
|
+
float or None
|
|
83
|
+
Bottom depth [m], or None if no crash detected.
|
|
84
|
+
|
|
85
|
+
Notes
|
|
86
|
+
-----
|
|
87
|
+
The reported depth is the *mean* depth of the samples in the flagged
|
|
88
|
+
``depth_window``-wide bin. When the crash onset falls early in that bin the
|
|
89
|
+
mean lies below the onset, so up to roughly half a bin width (~``depth_window
|
|
90
|
+
/ 2``) of contaminated data can remain *above* the reported depth and be left
|
|
91
|
+
un-trimmed. This is a fundamental limit of the bin resolution; size
|
|
92
|
+
``depth_window`` accordingly if tighter bottom trimming is required.
|
|
93
|
+
"""
|
|
94
|
+
# speed_factor / median_factor / vibration_frequency are accepted for
|
|
95
|
+
# backward compatibility but are not yet wired into the algorithm (see the
|
|
96
|
+
# parameter docs above). They are kept as valid config keys — strict
|
|
97
|
+
# validate_config would reject a YAML that still lists them — but warn if a
|
|
98
|
+
# caller tunes one off its default so it is no longer a SILENT no-op.
|
|
99
|
+
# (#104 U4-F2.)
|
|
100
|
+
_unwired = {
|
|
101
|
+
"speed_factor": (speed_factor, 0.3),
|
|
102
|
+
"median_factor": (median_factor, 1.0),
|
|
103
|
+
"vibration_frequency": (vibration_frequency, 16),
|
|
104
|
+
}
|
|
105
|
+
_tuned = [name for name, (val, default) in _unwired.items() if val != default]
|
|
106
|
+
if _tuned:
|
|
107
|
+
warnings.warn(
|
|
108
|
+
"detect_bottom_crash: parameter(s) "
|
|
109
|
+
f"{', '.join(_tuned)} are accepted but not yet implemented; the "
|
|
110
|
+
"supplied value(s) have no effect on bottom detection",
|
|
111
|
+
stacklevel=2,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
depth = np.asarray(depth_fast, dtype=np.float64)
|
|
115
|
+
|
|
116
|
+
if not vibration_channels:
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
# Elementwise root-sum-square across all supplied channels. For two-axis
|
|
120
|
+
# ``{Ax, Ay}`` this matches the previous ``sqrt(Ax**2 + Ay**2)`` exactly.
|
|
121
|
+
mag_sq: np.ndarray | None = None
|
|
122
|
+
for arr in vibration_channels.values():
|
|
123
|
+
a = np.asarray(arr, dtype=np.float64)
|
|
124
|
+
if len(a) != len(depth):
|
|
125
|
+
return None
|
|
126
|
+
sq = a * a
|
|
127
|
+
mag_sq = sq if mag_sq is None else mag_sq + sq
|
|
128
|
+
assert mag_sq is not None # unreachable: guarded by `if not vibration_channels`
|
|
129
|
+
accel_mag = np.sqrt(mag_sq)
|
|
130
|
+
|
|
131
|
+
# An all-NaN depth segment makes nanmax NaN; np.arange(..., NaN, ...) below
|
|
132
|
+
# then raises. Bail out cleanly instead of crashing the profile.
|
|
133
|
+
#
|
|
134
|
+
# Audit fix: short-circuit BEFORE nanmax on an all-NaN depth. np.errstate
|
|
135
|
+
# only governs IEEE FP flags (invalid/divide/...), NOT numpy's Python-level
|
|
136
|
+
# "All-NaN slice encountered" RuntimeWarning — so the old errstate guard let
|
|
137
|
+
# that warning escape (spurious log noise; promotable to an error by callers
|
|
138
|
+
# that filter warnings). Guarding on np.isfinite up front avoids the warning
|
|
139
|
+
# and the NaN max_depth entirely.
|
|
140
|
+
if not np.any(np.isfinite(depth)):
|
|
141
|
+
return None
|
|
142
|
+
max_depth = np.nanmax(depth)
|
|
143
|
+
if not np.isfinite(max_depth) or max_depth < depth_minimum:
|
|
144
|
+
return None
|
|
145
|
+
|
|
146
|
+
# Bin acceleration variance by depth
|
|
147
|
+
bin_size = depth_window
|
|
148
|
+
bins = np.arange(depth_minimum, max_depth + bin_size, bin_size)
|
|
149
|
+
if len(bins) < 2:
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
n_bins = len(bins) - 1
|
|
153
|
+
idx = np.searchsorted(bins, depth, side="right") - 1
|
|
154
|
+
in_range = (idx >= 0) & (idx < n_bins) & np.isfinite(accel_mag)
|
|
155
|
+
idx_v = idx[in_range]
|
|
156
|
+
vals_v = accel_mag[in_range]
|
|
157
|
+
counts = np.bincount(idx_v, minlength=n_bins)
|
|
158
|
+
sums = np.bincount(idx_v, weights=vals_v, minlength=n_bins)
|
|
159
|
+
sums_sq = np.bincount(idx_v, weights=vals_v * vals_v, minlength=n_bins)
|
|
160
|
+
with np.errstate(invalid="ignore", divide="ignore"):
|
|
161
|
+
means = np.where(counts > 0, sums / np.maximum(counts, 1), 0.0)
|
|
162
|
+
var = sums_sq / np.maximum(counts, 1) - means * means
|
|
163
|
+
var = np.maximum(var, 0.0)
|
|
164
|
+
bin_std = np.where(counts > 1, np.sqrt(var), np.nan)
|
|
165
|
+
|
|
166
|
+
valid = np.isfinite(bin_std)
|
|
167
|
+
if np.sum(valid) < 3:
|
|
168
|
+
return None
|
|
169
|
+
|
|
170
|
+
med_std = np.nanmedian(bin_std[valid])
|
|
171
|
+
threshold = med_std * vibration_factor
|
|
172
|
+
|
|
173
|
+
# A real bottom crash is at the seafloor — the deepest point of the cast.
|
|
174
|
+
# Restrict crash candidates to the near-bottom zone (within `proximity_bins`
|
|
175
|
+
# of the deepest sampled bin). A supra-threshold spike far above the bottom
|
|
176
|
+
# is mid-column vibration (a cable transient or a turbulence patch), not a
|
|
177
|
+
# crash; truncating the cast there would silently delete valid deep data
|
|
178
|
+
# (audit r1-3). This proximity gate is the geometric stand-in for the
|
|
179
|
+
# unimplemented MATLAB speed-drop (`speed_factor`) confirmation.
|
|
180
|
+
valid_idx = np.flatnonzero(valid)
|
|
181
|
+
deepest_valid = int(valid_idx[-1])
|
|
182
|
+
zone_top = deepest_valid - max(proximity_bins, 0)
|
|
183
|
+
|
|
184
|
+
def _bin_mean_depth(i: int) -> float:
|
|
185
|
+
# MEAN depth of the samples in bin i, not its geometric center: the
|
|
186
|
+
# deepest bin's right edge overhangs nanmax(depth) by up to one bin
|
|
187
|
+
# width, so its center can lie BELOW the deepest real sample and the
|
|
188
|
+
# caller's `P >= bottom_depth` would match nothing. The sample mean is
|
|
189
|
+
# guaranteed to lie within the bin's observed depths (robust to the
|
|
190
|
+
# fast/slow rate mismatch against the slow pressure the caller trims).
|
|
191
|
+
return float(np.nanmean(depth[in_range & (idx == i)]))
|
|
192
|
+
|
|
193
|
+
# Search the near-bottom zone from the deepest bin upward for the spike.
|
|
194
|
+
for i in range(deepest_valid, zone_top - 1, -1):
|
|
195
|
+
if valid[i] and bin_std[i] > threshold:
|
|
196
|
+
return _bin_mean_depth(i)
|
|
197
|
+
|
|
198
|
+
# No crash at the bottom. Surface any supra-threshold spike higher up —
|
|
199
|
+
# what the pre-fix deepest-upward scan would have mis-reported as a crash —
|
|
200
|
+
# so genuine mid-column contamination is visible rather than silently
|
|
201
|
+
# ignored, without truncating the (valid) deep cast.
|
|
202
|
+
for i in range(zone_top - 1, -1, -1):
|
|
203
|
+
if valid[i] and bin_std[i] > threshold:
|
|
204
|
+
spike_depth = _bin_mean_depth(i)
|
|
205
|
+
deepest_depth = float(np.nanmax(depth[in_range]))
|
|
206
|
+
warnings.warn(
|
|
207
|
+
f"detect_bottom_crash: vibration spike at ~{spike_depth:.1f} m "
|
|
208
|
+
f"is {deepest_depth - spike_depth:.1f} m above the deepest "
|
|
209
|
+
"sample; treated as mid-column contamination, not a bottom "
|
|
210
|
+
"crash (cast not truncated)",
|
|
211
|
+
stacklevel=2,
|
|
212
|
+
)
|
|
213
|
+
break
|
|
214
|
+
|
|
215
|
+
return None
|