oceanarray 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanarray/__init__.py +20 -0
- oceanarray/_version.py +1 -0
- oceanarray/cli.py +1343 -0
- oceanarray/clock_offset.py +675 -0
- oceanarray/config/OS1_sensor_attrs.yaml +49 -0
- oceanarray/config/OS1_var_names.yaml +5 -0
- oceanarray/config/OS1_vocab_attrs.yaml +84 -0
- oceanarray/config/legacy/project_RAPID.yaml +69 -0
- oceanarray/config/legacy/rodb_keys.txt +206 -0
- oceanarray/config/legacy/rodb_keys.yaml +1012 -0
- oceanarray/config/logging.yaml +28 -0
- oceanarray/legacy/__init__.py +47 -0
- oceanarray/legacy/caldip.py +753 -0
- oceanarray/legacy/convertOS.py +527 -0
- oceanarray/legacy/find_deployment.py +489 -0
- oceanarray/legacy/mooring_rodb.py +429 -0
- oceanarray/legacy/process_rodb.py +369 -0
- oceanarray/legacy/rodb.py +330 -0
- oceanarray/logger.py +192 -0
- oceanarray/logsheet/__init__.py +60 -0
- oceanarray/logsheet/_columns.py +97 -0
- oceanarray/logsheet/_config.py +245 -0
- oceanarray/logsheet/_firmware.py +43 -0
- oceanarray/logsheet/_latex.py +128 -0
- oceanarray/logsheet/_render.py +126 -0
- oceanarray/logsheet/builders/__init__.py +13 -0
- oceanarray/logsheet/builders/caldip_download.py +195 -0
- oceanarray/logsheet/builders/caldip_setup.py +188 -0
- oceanarray/logsheet/builders/deployment_setup.py +218 -0
- oceanarray/logsheet/builders/mooring_download.py +243 -0
- oceanarray/logsheet/builders/recovery.py +157 -0
- oceanarray/logsheet/data/column_defs.yaml +686 -0
- oceanarray/logsheet/data/latex_templates/_base_instrument.tex.j2 +42 -0
- oceanarray/logsheet/data/latex_templates/caldip_download.tex.j2 +47 -0
- oceanarray/logsheet/data/latex_templates/caldip_setup.tex.j2 +70 -0
- oceanarray/logsheet/data/latex_templates/deployment_setup.tex.j2 +97 -0
- oceanarray/logsheet/data/latex_templates/mooring_download.tex.j2 +54 -0
- oceanarray/logsheet/data/latex_templates/recovery.tex.j2 +103 -0
- oceanarray/mooring_level.py +1368 -0
- oceanarray/oceanarray.mplstyle +18 -0
- oceanarray/parameters.py +270 -0
- oceanarray/plotter.py +942 -0
- oceanarray/plotters/__init__.py +97 -0
- oceanarray/plotters/_animation.py +261 -0
- oceanarray/plotters/_current.py +705 -0
- oceanarray/plotters/_diagnostic.py +667 -0
- oceanarray/plotters/_helpers.py +57 -0
- oceanarray/plotters/_primitives.py +101 -0
- oceanarray/plotters/_section.py +16 -0
- oceanarray/plotters/_timeseries.py +11 -0
- oceanarray/rapid_interp.py +739 -0
- oceanarray/read_rbr_hex.py +541 -0
- oceanarray/readers.py +303 -0
- oceanarray/report/__init__.py +5 -0
- oceanarray/report/_array.py +386 -0
- oceanarray/report/_grid.py +539 -0
- oceanarray/report/_html_helpers.py +1004 -0
- oceanarray/report/_instrument.py +853 -0
- oceanarray/report/_mooring.py +1385 -0
- oceanarray/report/_plots.py +4284 -0
- oceanarray/report/_stack.py +941 -0
- oceanarray/stage1.py +1959 -0
- oceanarray/stage2.py +1160 -0
- oceanarray/stage3.py +2494 -0
- oceanarray/time_gridding.py +958 -0
- oceanarray/tools.py +400 -0
- oceanarray/transports.py +1 -0
- oceanarray/utilities.py +538 -0
- oceanarray/validation.py +389 -0
- oceanarray/writers.py +99 -0
- oceanarray-0.1.0.dist-info/METADATA +503 -0
- oceanarray-0.1.0.dist-info/RECORD +76 -0
- oceanarray-0.1.0.dist-info/WHEEL +5 -0
- oceanarray-0.1.0.dist-info/entry_points.txt +2 -0
- oceanarray-0.1.0.dist-info/licenses/LICENSE +21 -0
- oceanarray-0.1.0.dist-info/top_level.txt +1 -0
oceanarray/stage3.py
ADDED
|
@@ -0,0 +1,2494 @@
|
|
|
1
|
+
"""Stage 3: pressure interpolation + QARTOD QC (gross-range and spike tests).
|
|
2
|
+
|
|
3
|
+
Processing order per instrument
|
|
4
|
+
---------------------------------
|
|
5
|
+
1. Load ``_stage2.nc``.
|
|
6
|
+
2. **Pressure interpolation** (targets only — instruments lacking pressure or
|
|
7
|
+
whose pressure is flagged bad in the YAML):
|
|
8
|
+
a. Near-neighbour if any source has |Δhab| ≤ ``HAB_THRESHOLD``.
|
|
9
|
+
b. Weighted bracketing from the closest source above and below.
|
|
10
|
+
c. Extrapolation (with WARNING) when target is outside all source habs.
|
|
11
|
+
Interpolated pressure gets ``pressure_qc = 8`` (interpolated_value).
|
|
12
|
+
3. **QARTOD gross-range test** on temperature, conductivity, pressure, and
|
|
13
|
+
velocity components. Flags 4 (bad) or 3 (suspect) based on thresholds in
|
|
14
|
+
``parameters.QC_GROSS_RANGE`` (overrideable per mooring / per instrument in
|
|
15
|
+
YAML via a ``qc_ranges`` key).
|
|
16
|
+
4. **QARTOD spike test** on the same variables. Flags from
|
|
17
|
+
``parameters.QC_SPIKE`` (overrideable via ``qc_spike`` in YAML).
|
|
18
|
+
5. Merge all QC flags using priority order: 9 > 4 > 3 > 8 > 2 > 1.
|
|
19
|
+
6. Write ``_stage3.nc`` for **all** instruments (not only pressure targets).
|
|
20
|
+
|
|
21
|
+
Flag combination priority
|
|
22
|
+
--------------------------
|
|
23
|
+
Missing (9) > Bad (4) > Suspect (3) > Interpolated (8) > Prob-good (2) > Good (1)
|
|
24
|
+
|
|
25
|
+
This means that if interpolated pressure also fails the range test it is
|
|
26
|
+
flagged 4 (bad), not 8 (interpolated).
|
|
27
|
+
|
|
28
|
+
YAML configuration keys
|
|
29
|
+
------------------------
|
|
30
|
+
Top-level (mooring-wide):
|
|
31
|
+
``qc_ranges`` : mapping of variable → {fail_span, suspect_span}
|
|
32
|
+
``qc_spike`` : mapping of variable → {suspect_threshold, fail_threshold}
|
|
33
|
+
|
|
34
|
+
Per-instrument (in a clamp entry):
|
|
35
|
+
``qc_ranges`` : same structure; overrides the mooring-level setting for
|
|
36
|
+
the variables listed (others fall back to mooring/global defaults)
|
|
37
|
+
``qc_spike`` : same structure
|
|
38
|
+
``pressure_qc`` : int — mark this instrument's own pressure as bad (≥3) so
|
|
39
|
+
stage3 replaces it with an interpolated value.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import copy
|
|
45
|
+
import datetime
|
|
46
|
+
import hashlib
|
|
47
|
+
import re
|
|
48
|
+
from pathlib import Path
|
|
49
|
+
from typing import Any, Dict, List, Optional
|
|
50
|
+
|
|
51
|
+
import numpy as np
|
|
52
|
+
import xarray as xr
|
|
53
|
+
import yaml
|
|
54
|
+
|
|
55
|
+
from .utilities import (
|
|
56
|
+
cast_output_dtypes,
|
|
57
|
+
drop_all_zero_vars,
|
|
58
|
+
extract_inline_instruments,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
HAB_THRESHOLD = 2.0 # metres — use near-neighbour below this Δhab
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
# BEAM → ENU coordinate transform helpers
|
|
67
|
+
# ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _xyz_to_enu(
|
|
71
|
+
vx: np.ndarray,
|
|
72
|
+
vy: np.ndarray,
|
|
73
|
+
vz: np.ndarray,
|
|
74
|
+
heading_deg: np.ndarray,
|
|
75
|
+
pitch_deg: np.ndarray,
|
|
76
|
+
roll_deg: np.ndarray,
|
|
77
|
+
declination_deg: float = 0.0,
|
|
78
|
+
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
|
79
|
+
"""Vectorised XYZ → ENU rotation per Nortek Support reference script.
|
|
80
|
+
|
|
81
|
+
R = H @ P where (with hdg = heading - 90 + declination):
|
|
82
|
+
H = [[cos(hdg), sin(hdg), 0], [-sin(hdg), cos(hdg), 0], [0, 0, 1]]
|
|
83
|
+
P = [[cos(p), -sin(p)*sin(r), -cos(r)*sin(p)],
|
|
84
|
+
[0, cos(r), -sin(r)],
|
|
85
|
+
[sin(p), sin(r)*cos(p), cos(p)*cos(r)]]
|
|
86
|
+
|
|
87
|
+
The -90 offset accounts for the Nortek Aquadopp instrument frame where
|
|
88
|
+
heading=90° aligns X→East, Y→North (standard geography at zero tilt).
|
|
89
|
+
Magnetic declination is added to convert magnetic heading to true north.
|
|
90
|
+
|
|
91
|
+
Returns (east, north, up) arrays of the same shape as the inputs.
|
|
92
|
+
"""
|
|
93
|
+
h = np.radians(heading_deg - 90.0 + declination_deg)
|
|
94
|
+
p = np.radians(pitch_deg)
|
|
95
|
+
r = np.radians(roll_deg)
|
|
96
|
+
|
|
97
|
+
ch = np.cos(h)
|
|
98
|
+
sh = np.sin(h)
|
|
99
|
+
cp = np.cos(p)
|
|
100
|
+
sp = np.sin(p)
|
|
101
|
+
cr = np.cos(r)
|
|
102
|
+
sr = np.sin(r)
|
|
103
|
+
|
|
104
|
+
# Expanded R = H @ P (verified against Nortek Support reference script)
|
|
105
|
+
east = (
|
|
106
|
+
ch * cp * vx + (-ch * sp * sr + sh * cr) * vy + (-ch * sp * cr - sh * sr) * vz
|
|
107
|
+
)
|
|
108
|
+
north = (
|
|
109
|
+
-sh * cp * vx + (sh * sp * sr + ch * cr) * vy + (sh * sp * cr - ch * sr) * vz
|
|
110
|
+
)
|
|
111
|
+
up = sp * vx + sr * cp * vy + cp * cr * vz
|
|
112
|
+
return east, north, up
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _apply_beam_to_enu(
|
|
116
|
+
ds: "xr.Dataset",
|
|
117
|
+
entry: Dict[str, Any],
|
|
118
|
+
lat: float,
|
|
119
|
+
lon: float,
|
|
120
|
+
latlon_source: str = "unknown",
|
|
121
|
+
log_fn: Any = None,
|
|
122
|
+
) -> "xr.Dataset":
|
|
123
|
+
"""Transform BEAM or XYZ Nortek velocities to ENU geographic coordinates.
|
|
124
|
+
|
|
125
|
+
Adds east_velocity, north_velocity, up_velocity, current_speed,
|
|
126
|
+
current_direction. Updates coordinate_system attr to 'ENU'.
|
|
127
|
+
No-ops for instruments already in ENU or with unknown coordinate system.
|
|
128
|
+
Requires normalized variable names (heading, pitch, roll) — re-run stage1
|
|
129
|
+
if these are absent.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
def _warn(msg: str) -> None:
|
|
133
|
+
if log_fn:
|
|
134
|
+
log_fn(msg)
|
|
135
|
+
else:
|
|
136
|
+
print(msg)
|
|
137
|
+
|
|
138
|
+
coord_sys = ds.attrs.get("coordinate_system", "ENU")
|
|
139
|
+
if coord_sys not in ("BEAM", "XYZ"):
|
|
140
|
+
return ds
|
|
141
|
+
|
|
142
|
+
# Require normalized heading/pitch/roll (stage1 normalization must have run)
|
|
143
|
+
for vname in ("heading", "pitch", "roll"):
|
|
144
|
+
if vname not in ds.data_vars:
|
|
145
|
+
_warn(
|
|
146
|
+
f" WARNING: BEAM→ENU skipped — '{vname}' not in dataset. "
|
|
147
|
+
"Re-run stage1 to get normalized variable names."
|
|
148
|
+
)
|
|
149
|
+
return ds
|
|
150
|
+
|
|
151
|
+
heading = ds["heading"].values.astype(float)
|
|
152
|
+
pitch = ds["pitch"].values.astype(float)
|
|
153
|
+
roll = ds["roll"].values.astype(float)
|
|
154
|
+
|
|
155
|
+
# Magnetic declination via ppigrf
|
|
156
|
+
declination = 0.0
|
|
157
|
+
if lat == 0.0 and lon == 0.0 and "unknown" in latlon_source.lower():
|
|
158
|
+
_warn(
|
|
159
|
+
f" WARNING: BEAM→ENU — lat/lon unknown for serial {entry.get('serial', '?')}. "
|
|
160
|
+
"Skipping magnetic declination to avoid silently applying the Gulf of Guinea value. "
|
|
161
|
+
"Set lat/lon in the mooring YAML to get a correct declination correction."
|
|
162
|
+
)
|
|
163
|
+
ds.attrs["magnetic_declination"] = "UNK"
|
|
164
|
+
ds.attrs["magnetic_declination_latlon_source"] = (
|
|
165
|
+
f"mooring YAML ({latlon_source})"
|
|
166
|
+
)
|
|
167
|
+
else:
|
|
168
|
+
try:
|
|
169
|
+
import ppigrf
|
|
170
|
+
import datetime as _dt
|
|
171
|
+
|
|
172
|
+
time_vals = ds["time"].values
|
|
173
|
+
t_mid = time_vals[len(time_vals) // 2]
|
|
174
|
+
t_mid_s = int(t_mid.astype("datetime64[s]").astype("int64"))
|
|
175
|
+
t_mid_dt = _dt.datetime.utcfromtimestamp(t_mid_s)
|
|
176
|
+
Be, Bn, _ = ppigrf.igrf(float(lon), float(lat), 0.0, t_mid_dt)
|
|
177
|
+
declination = float(
|
|
178
|
+
np.degrees(
|
|
179
|
+
np.arctan2(float(np.atleast_1d(Be)[0]), float(np.atleast_1d(Bn)[0]))
|
|
180
|
+
)
|
|
181
|
+
)
|
|
182
|
+
ds.attrs["magnetic_declination"] = declination
|
|
183
|
+
ds.attrs["magnetic_declination_units"] = "degrees_east"
|
|
184
|
+
ds.attrs["magnetic_declination_method"] = (
|
|
185
|
+
"ppigrf IGRF at deployment midpoint"
|
|
186
|
+
)
|
|
187
|
+
ds.attrs["magnetic_declination_lat"] = lat
|
|
188
|
+
ds.attrs["magnetic_declination_lon"] = lon
|
|
189
|
+
ds.attrs["magnetic_declination_latlon_source"] = (
|
|
190
|
+
f"mooring YAML ({latlon_source})"
|
|
191
|
+
)
|
|
192
|
+
_warn(f" BEAM→ENU: magnetic declination = {declination:.2f}°")
|
|
193
|
+
except Exception as e: # noqa: BLE001 — ppigrf optional; proceed with 0° declination
|
|
194
|
+
_warn(f" WARNING: magnetic declination unavailable ({e}) — using 0°")
|
|
195
|
+
|
|
196
|
+
if coord_sys == "BEAM":
|
|
197
|
+
# Stage1 could not apply the T matrix (header file missing or unparseable).
|
|
198
|
+
# Re-run stage1 with the correct header file to produce velocity_x/y/z.
|
|
199
|
+
_warn(
|
|
200
|
+
f" SKIPPING BEAM→ENU for serial {entry.get('serial', '?')}: "
|
|
201
|
+
"data are still in BEAM coordinates — re-run stage1 with the instrument "
|
|
202
|
+
"header file so the T matrix can be extracted and velocity_x/y/z produced."
|
|
203
|
+
)
|
|
204
|
+
return ds
|
|
205
|
+
|
|
206
|
+
else: # XYZ — stage3 only needs to do XYZ→ENU
|
|
207
|
+
if "velocity_x" in ds.data_vars:
|
|
208
|
+
# stage1 applied the T matrix (BEAM→XYZ) and stored instrument-frame XYZ
|
|
209
|
+
T_source = "applied in stage1 (BEAM→XYZ)"
|
|
210
|
+
vx = ds["velocity_x"].values.astype(float)
|
|
211
|
+
vy = ds["velocity_y"].values.astype(float)
|
|
212
|
+
vz = ds["velocity_z"].values.astype(float)
|
|
213
|
+
elif "x_velocity" in ds.data_vars:
|
|
214
|
+
# Instrument natively reports XYZ; T matrix applied in firmware
|
|
215
|
+
T_source = "not applicable (instrument-native XYZ)"
|
|
216
|
+
vx = ds["x_velocity"].values.astype(float)
|
|
217
|
+
vy = ds["y_velocity"].values.astype(float)
|
|
218
|
+
vz = ds["z_velocity"].values.astype(float)
|
|
219
|
+
else:
|
|
220
|
+
# Legacy fallback: XYZ stored in beam variable slots
|
|
221
|
+
T_source = "not applicable (legacy XYZ fallback)"
|
|
222
|
+
vx = ds["velocity_beam1"].values.astype(float)
|
|
223
|
+
vy = ds["velocity_beam2"].values.astype(float)
|
|
224
|
+
vz = ds["velocity_beam3"].values.astype(float)
|
|
225
|
+
|
|
226
|
+
# XYZ → ENU
|
|
227
|
+
valid_all = (
|
|
228
|
+
np.isfinite(vx)
|
|
229
|
+
& np.isfinite(vy)
|
|
230
|
+
& np.isfinite(vz)
|
|
231
|
+
& np.isfinite(heading)
|
|
232
|
+
& np.isfinite(pitch)
|
|
233
|
+
& np.isfinite(roll)
|
|
234
|
+
)
|
|
235
|
+
east = np.full_like(vx, np.nan)
|
|
236
|
+
north = np.full_like(vx, np.nan)
|
|
237
|
+
up = np.full_like(vx, np.nan)
|
|
238
|
+
if valid_all.any():
|
|
239
|
+
east[valid_all], north[valid_all], up[valid_all] = _xyz_to_enu(
|
|
240
|
+
vx[valid_all],
|
|
241
|
+
vy[valid_all],
|
|
242
|
+
vz[valid_all],
|
|
243
|
+
heading[valid_all],
|
|
244
|
+
pitch[valid_all],
|
|
245
|
+
roll[valid_all],
|
|
246
|
+
declination,
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
speed = np.where(
|
|
250
|
+
np.isfinite(east) & np.isfinite(north), np.sqrt(east**2 + north**2), np.nan
|
|
251
|
+
)
|
|
252
|
+
direction = np.where(
|
|
253
|
+
np.isfinite(east) & np.isfinite(north),
|
|
254
|
+
np.degrees(np.arctan2(east, north)) % 360.0,
|
|
255
|
+
np.nan,
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
time_dim = ds["heading"].dims[0]
|
|
259
|
+
for name, arr, cf, units, long_name in [
|
|
260
|
+
(
|
|
261
|
+
"east_velocity",
|
|
262
|
+
east,
|
|
263
|
+
"eastward_sea_water_velocity",
|
|
264
|
+
"m s-1",
|
|
265
|
+
"Eastward sea water velocity",
|
|
266
|
+
),
|
|
267
|
+
(
|
|
268
|
+
"north_velocity",
|
|
269
|
+
north,
|
|
270
|
+
"northward_sea_water_velocity",
|
|
271
|
+
"m s-1",
|
|
272
|
+
"Northward sea water velocity",
|
|
273
|
+
),
|
|
274
|
+
(
|
|
275
|
+
"up_velocity",
|
|
276
|
+
up,
|
|
277
|
+
"upward_sea_water_velocity",
|
|
278
|
+
"m s-1",
|
|
279
|
+
"Upward sea water velocity",
|
|
280
|
+
),
|
|
281
|
+
(
|
|
282
|
+
"current_speed",
|
|
283
|
+
speed,
|
|
284
|
+
"sea_water_speed",
|
|
285
|
+
"m s-1",
|
|
286
|
+
"Horizontal current speed",
|
|
287
|
+
),
|
|
288
|
+
(
|
|
289
|
+
"current_direction",
|
|
290
|
+
direction,
|
|
291
|
+
"direction_of_sea_water_velocity",
|
|
292
|
+
"degrees",
|
|
293
|
+
"Current direction (0=N, clockwise)",
|
|
294
|
+
),
|
|
295
|
+
]:
|
|
296
|
+
ds[name] = xr.Variable(
|
|
297
|
+
time_dim,
|
|
298
|
+
arr,
|
|
299
|
+
attrs={"units": units, "standard_name": cf, "long_name": long_name},
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
ds.attrs["coordinate_system"] = "ENU"
|
|
303
|
+
ds.attrs["coordinate_system_source"] = (
|
|
304
|
+
f"rotated from {coord_sys} by oceanarray stage3"
|
|
305
|
+
)
|
|
306
|
+
_warn(
|
|
307
|
+
f" BEAM→ENU: produced east/north/up_velocity, current_speed, current_direction "
|
|
308
|
+
f"(coord_sys was {coord_sys}, T matrix: {T_source})"
|
|
309
|
+
)
|
|
310
|
+
return ds
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _apply_declination_to_enu(
|
|
314
|
+
ds: "xr.Dataset",
|
|
315
|
+
lat: float,
|
|
316
|
+
lon: float,
|
|
317
|
+
latlon_source: str = "unknown",
|
|
318
|
+
log_fn: Any = None,
|
|
319
|
+
) -> "xr.Dataset":
|
|
320
|
+
"""Apply magnetic declination rotation to velocities already in ENU frame.
|
|
321
|
+
|
|
322
|
+
When a Nortek instrument is configured to output ENU coordinates internally,
|
|
323
|
+
the heading reference used is magnetic north. This function rotates
|
|
324
|
+
east_velocity and north_velocity by the declination angle so that north
|
|
325
|
+
aligns with true (geographic) north.
|
|
326
|
+
|
|
327
|
+
Rotation (D = declination, positive = east):
|
|
328
|
+
u_true = u_mag * cos(D) + v_mag * sin(D)
|
|
329
|
+
v_true = -u_mag * sin(D) + v_mag * cos(D)
|
|
330
|
+
|
|
331
|
+
No-ops if east_velocity or north_velocity are absent, or if magnetic
|
|
332
|
+
declination has already been applied (``magnetic_declination`` attr present).
|
|
333
|
+
"""
|
|
334
|
+
|
|
335
|
+
def _warn(msg: str) -> None:
|
|
336
|
+
if log_fn:
|
|
337
|
+
log_fn(msg)
|
|
338
|
+
else:
|
|
339
|
+
print(msg)
|
|
340
|
+
|
|
341
|
+
if "magnetic_declination" in ds.attrs:
|
|
342
|
+
return ds # already applied
|
|
343
|
+
|
|
344
|
+
if "east_velocity" not in ds.data_vars or "north_velocity" not in ds.data_vars:
|
|
345
|
+
return ds
|
|
346
|
+
|
|
347
|
+
if lat == 0.0 and lon == 0.0 and "unknown" in latlon_source.lower():
|
|
348
|
+
_warn(
|
|
349
|
+
" WARNING: ENU declination correction skipped — lat/lon unknown. "
|
|
350
|
+
"Applying declination from (0°N, 0°E) would silently rotate velocities by "
|
|
351
|
+
"the Gulf of Guinea value. Set lat/lon in the mooring YAML."
|
|
352
|
+
)
|
|
353
|
+
ds.attrs["magnetic_declination"] = "UNK"
|
|
354
|
+
ds.attrs["magnetic_declination_latlon_source"] = (
|
|
355
|
+
f"mooring YAML ({latlon_source})"
|
|
356
|
+
)
|
|
357
|
+
return ds
|
|
358
|
+
|
|
359
|
+
try:
|
|
360
|
+
import ppigrf
|
|
361
|
+
import datetime as _dt
|
|
362
|
+
|
|
363
|
+
time_vals = ds["time"].values
|
|
364
|
+
t_mid = time_vals[len(time_vals) // 2]
|
|
365
|
+
t_mid_s = int(t_mid.astype("datetime64[s]").astype("int64"))
|
|
366
|
+
t_mid_dt = _dt.datetime.utcfromtimestamp(t_mid_s)
|
|
367
|
+
Be, Bn, _ = ppigrf.igrf(float(lon), float(lat), 0.0, t_mid_dt)
|
|
368
|
+
declination = float(
|
|
369
|
+
np.degrees(
|
|
370
|
+
np.arctan2(float(np.atleast_1d(Be)[0]), float(np.atleast_1d(Bn)[0]))
|
|
371
|
+
)
|
|
372
|
+
)
|
|
373
|
+
|
|
374
|
+
D = np.radians(declination)
|
|
375
|
+
u = ds["east_velocity"].values.astype(float)
|
|
376
|
+
v = ds["north_velocity"].values.astype(float)
|
|
377
|
+
ds["east_velocity"].values[:] = u * np.cos(D) + v * np.sin(D)
|
|
378
|
+
ds["north_velocity"].values[:] = -u * np.sin(D) + v * np.cos(D)
|
|
379
|
+
|
|
380
|
+
ds.attrs["magnetic_declination"] = declination
|
|
381
|
+
ds.attrs["magnetic_declination_units"] = "degrees_east"
|
|
382
|
+
ds.attrs["magnetic_declination_method"] = "ppigrf IGRF at deployment midpoint"
|
|
383
|
+
ds.attrs["magnetic_declination_lat"] = lat
|
|
384
|
+
ds.attrs["magnetic_declination_lon"] = lon
|
|
385
|
+
ds.attrs["magnetic_declination_latlon_source"] = (
|
|
386
|
+
f"mooring YAML ({latlon_source})"
|
|
387
|
+
)
|
|
388
|
+
ds.attrs["coordinate_system_source"] = (
|
|
389
|
+
"ENU from instrument; declination-corrected by oceanarray stage3"
|
|
390
|
+
)
|
|
391
|
+
_warn(
|
|
392
|
+
f" ENU declination correction: {declination:+.2f}° applied to "
|
|
393
|
+
"east_velocity / north_velocity"
|
|
394
|
+
)
|
|
395
|
+
except Exception as e: # noqa: BLE001 — ppigrf optional
|
|
396
|
+
_warn(f" WARNING: declination correction unavailable ({e}) — ENU unchanged")
|
|
397
|
+
|
|
398
|
+
return ds
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
# Priority order for merging QC flags (higher priority = worse data quality).
|
|
402
|
+
# 9=missing, 4=bad, 3=suspect, 8=interpolated, 2=prob-good, 1=good
|
|
403
|
+
_QC_PRIORITY: Dict[int, int] = {9: 6, 4: 5, 3: 4, 8: 3, 2: 2, 1: 1, 0: 0}
|
|
404
|
+
|
|
405
|
+
# CF standard names and canonical long_names for known physics variables.
|
|
406
|
+
# Applied as a normalization pass just before writing _stage3.nc so that all
|
|
407
|
+
# output files have consistent metadata regardless of which reader produced them.
|
|
408
|
+
_CF_ATTRS: Dict[str, Dict[str, str]] = {
|
|
409
|
+
"temperature": {
|
|
410
|
+
"standard_name": "sea_water_temperature",
|
|
411
|
+
"long_name": "Temperature",
|
|
412
|
+
"units": "degrees_C",
|
|
413
|
+
},
|
|
414
|
+
"conductivity": {
|
|
415
|
+
"standard_name": "sea_water_electrical_conductivity",
|
|
416
|
+
"long_name": "Conductivity",
|
|
417
|
+
},
|
|
418
|
+
"salinity": {
|
|
419
|
+
"standard_name": "sea_water_practical_salinity",
|
|
420
|
+
"long_name": "Salinity",
|
|
421
|
+
"units": "1",
|
|
422
|
+
},
|
|
423
|
+
"pressure": {
|
|
424
|
+
"standard_name": "sea_water_pressure",
|
|
425
|
+
"long_name": "Pressure",
|
|
426
|
+
"units": "dbar",
|
|
427
|
+
},
|
|
428
|
+
"turbidity": {
|
|
429
|
+
"standard_name": "sea_water_turbidity",
|
|
430
|
+
"long_name": "Turbidity",
|
|
431
|
+
},
|
|
432
|
+
"dissolved_oxygen": {
|
|
433
|
+
"standard_name": "mole_concentration_of_dissolved_molecular_oxygen_in_sea_water",
|
|
434
|
+
"long_name": "Dissolved Oxygen",
|
|
435
|
+
"units": "umol/L",
|
|
436
|
+
},
|
|
437
|
+
"oxygen_saturation_pct": {
|
|
438
|
+
"long_name": "Dissolved Oxygen Percent Saturation",
|
|
439
|
+
"units": "%",
|
|
440
|
+
},
|
|
441
|
+
"apparent_oxygen_utilization": {
|
|
442
|
+
"standard_name": "apparent_oxygen_utilization",
|
|
443
|
+
"long_name": "Apparent Oxygen Utilization",
|
|
444
|
+
"units": "umol/kg",
|
|
445
|
+
},
|
|
446
|
+
"dissolved_oxygen_ml_l": {
|
|
447
|
+
"standard_name": "mole_concentration_of_dissolved_molecular_oxygen_in_sea_water",
|
|
448
|
+
"long_name": "Dissolved Oxygen",
|
|
449
|
+
"units": "mL/L",
|
|
450
|
+
},
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _safe_serial(serial: Any) -> str:
|
|
455
|
+
return re.sub(r"[^\w\-]", "", str(serial))
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def _times_to_float(time_values: np.ndarray) -> np.ndarray:
|
|
459
|
+
"""Convert numpy datetime64 array to float64 nanoseconds (for numpy.interp)."""
|
|
460
|
+
return time_values.astype("datetime64[ns]").astype(np.float64)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _is_burst_mode(time_values: np.ndarray, burst_ratio: float = 5.0) -> bool:
|
|
464
|
+
"""Return True when the time series has a bimodal Δt (burst sampling).
|
|
465
|
+
|
|
466
|
+
Burst-mode instruments (e.g. Nortek Aquadopp) take N pings at 1 Hz then
|
|
467
|
+
wait ~120 s. The within-burst interval (p50) is << the burst interval
|
|
468
|
+
(p90). The QARTOD spike test compares adjacent samples and generates false
|
|
469
|
+
positives at every burst boundary, so callers skip it when this is True.
|
|
470
|
+
"""
|
|
471
|
+
if len(time_values) < 10:
|
|
472
|
+
return False
|
|
473
|
+
dt = np.diff(time_values).astype("datetime64[s]").astype(float)
|
|
474
|
+
dt = dt[dt > 0]
|
|
475
|
+
if len(dt) == 0:
|
|
476
|
+
return False
|
|
477
|
+
p50 = float(np.percentile(dt, 50))
|
|
478
|
+
p90 = float(np.percentile(dt, 90))
|
|
479
|
+
return p50 > 0 and p90 > burst_ratio * p50
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _interp_pressure(
|
|
483
|
+
source_time: np.ndarray,
|
|
484
|
+
source_pressure: np.ndarray,
|
|
485
|
+
target_time: np.ndarray,
|
|
486
|
+
) -> np.ndarray:
|
|
487
|
+
"""Linearly interpolate source pressure onto target time axis.
|
|
488
|
+
|
|
489
|
+
Nearest-point extrapolation at the edges (no NaN fill outside range).
|
|
490
|
+
"""
|
|
491
|
+
src_t = _times_to_float(source_time)
|
|
492
|
+
tgt_t = _times_to_float(target_time)
|
|
493
|
+
valid = np.isfinite(source_pressure)
|
|
494
|
+
if valid.sum() < 2:
|
|
495
|
+
return np.full(len(target_time), np.nan)
|
|
496
|
+
return np.interp(
|
|
497
|
+
tgt_t,
|
|
498
|
+
src_t[valid],
|
|
499
|
+
source_pressure[valid],
|
|
500
|
+
left=source_pressure[valid][0],
|
|
501
|
+
right=source_pressure[valid][-1],
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _merge_flags(*flag_arrays: np.ndarray) -> np.ndarray:
|
|
506
|
+
"""Merge multiple int8 flag arrays using priority ordering.
|
|
507
|
+
|
|
508
|
+
Returns element-wise flag with highest priority (worst quality).
|
|
509
|
+
Works on arrays of any shape (1-D time series or 2-D time×N_BINS).
|
|
510
|
+
"""
|
|
511
|
+
# Lookup table: priority[flag] for flags 0-9
|
|
512
|
+
# _QC_PRIORITY: {9:6, 4:5, 3:4, 8:3, 2:2, 1:1, 0:0}; unlisted flags → 0
|
|
513
|
+
_priority = np.array([0, 1, 2, 4, 5, 0, 0, 0, 3, 6], dtype=np.int8)
|
|
514
|
+
result = np.asarray(flag_arrays[0], dtype=np.int8).copy()
|
|
515
|
+
for fa in flag_arrays[1:]:
|
|
516
|
+
fa = np.asarray(fa, dtype=np.int8)
|
|
517
|
+
replace = _priority[fa.clip(0, 9)] > _priority[result.clip(0, 9)]
|
|
518
|
+
result = np.where(replace, fa, result).astype(np.int8)
|
|
519
|
+
return result
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def _deep_merge(base: Dict, override: Dict) -> Dict:
|
|
523
|
+
"""Merge two nested dicts; override values win at the variable level."""
|
|
524
|
+
merged = copy.deepcopy(base)
|
|
525
|
+
for k, v in override.items():
|
|
526
|
+
if isinstance(v, dict) and k in merged and isinstance(merged[k], dict):
|
|
527
|
+
merged[k] = {**merged[k], **v}
|
|
528
|
+
else:
|
|
529
|
+
merged[k] = copy.deepcopy(v)
|
|
530
|
+
return merged
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _tilt_from_span(qc_ranges: Dict[str, Any]) -> Dict[str, float]:
|
|
534
|
+
"""Extract tilt thresholds from a qc_ranges block if 'tilt' is present.
|
|
535
|
+
|
|
536
|
+
Converts the symmetric span format (matching all other variables) into the
|
|
537
|
+
threshold format used internally by _apply_tilt_qc. The upper bound of
|
|
538
|
+
each span is the threshold — tilt is always non-negative so the lower bound
|
|
539
|
+
is typically negative and only the upper bound matters:
|
|
540
|
+
suspect_span: [-20, 20] → suspect_threshold: 20
|
|
541
|
+
fail_span: [-30, 30] → fail_threshold: 30
|
|
542
|
+
"""
|
|
543
|
+
out: Dict[str, float] = {}
|
|
544
|
+
tilt_cfg = qc_ranges.get("tilt", {})
|
|
545
|
+
if "suspect_span" in tilt_cfg:
|
|
546
|
+
out["suspect_threshold"] = float(tilt_cfg["suspect_span"][1])
|
|
547
|
+
if "fail_span" in tilt_cfg:
|
|
548
|
+
out["fail_threshold"] = float(tilt_cfg["fail_span"][1])
|
|
549
|
+
return out
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _load_qc_config(
|
|
553
|
+
mooring_cfg: Dict[str, Any],
|
|
554
|
+
entry: Dict[str, Any],
|
|
555
|
+
) -> tuple[Dict, Dict, Dict, Dict]:
|
|
556
|
+
"""Return QC threshold dicts for one instrument.
|
|
557
|
+
|
|
558
|
+
Returns a 4-tuple ``(gross_range, spike, tilt, flat_line)`` built from
|
|
559
|
+
package defaults with mooring-level and then instrument-level YAML
|
|
560
|
+
overrides applied (later values win).
|
|
561
|
+
|
|
562
|
+
``gross_range`` and ``spike`` map variable names (e.g. ``"temperature"``,
|
|
563
|
+
``"pressure"``) to threshold dicts understood by ``_apply_qc_tests``.
|
|
564
|
+
``tilt`` holds ``suspect_threshold`` / ``fail_threshold`` in degrees,
|
|
565
|
+
used to flag Aquadopp velocity data when the instrument is tilted beyond
|
|
566
|
+
acceptable limits. ``flat_line`` maps variable names to
|
|
567
|
+
``{suspect_n, fail_n, tolerance}`` dicts for the stuck-sensor test.
|
|
568
|
+
|
|
569
|
+
YAML override keys:
|
|
570
|
+
|
|
571
|
+
* ``qc_ranges`` (mooring or instrument level) — gross-range and tilt spans
|
|
572
|
+
* ``qc_spike`` — spike thresholds
|
|
573
|
+
* ``tilt_qc`` — tilt thresholds (takes precedence over ``qc_ranges.tilt``)
|
|
574
|
+
* ``qc_flat_line`` — stuck-sensor thresholds
|
|
575
|
+
|
|
576
|
+
``tilt`` is stripped from the gross-range dict before returning so it is
|
|
577
|
+
not passed to the QARTOD gross-range test runner.
|
|
578
|
+
"""
|
|
579
|
+
from . import parameters as P
|
|
580
|
+
|
|
581
|
+
gr = copy.deepcopy(P.QC_GROSS_RANGE)
|
|
582
|
+
sp = copy.deepcopy(P.QC_SPIKE)
|
|
583
|
+
tilt = copy.deepcopy(P.QC_TILT)
|
|
584
|
+
fl = copy.deepcopy(P.QC_FLAT_LINE)
|
|
585
|
+
|
|
586
|
+
# Mooring-level overrides
|
|
587
|
+
if "qc_ranges" in mooring_cfg:
|
|
588
|
+
gr = _deep_merge(gr, mooring_cfg["qc_ranges"])
|
|
589
|
+
tilt.update(_tilt_from_span(mooring_cfg["qc_ranges"]))
|
|
590
|
+
if "qc_spike" in mooring_cfg:
|
|
591
|
+
sp = _deep_merge(sp, mooring_cfg["qc_spike"])
|
|
592
|
+
if "tilt_qc" in mooring_cfg:
|
|
593
|
+
tilt.update(mooring_cfg["tilt_qc"])
|
|
594
|
+
if "qc_flat_line" in mooring_cfg:
|
|
595
|
+
fl = _deep_merge(fl, mooring_cfg["qc_flat_line"])
|
|
596
|
+
|
|
597
|
+
# Instrument-level overrides
|
|
598
|
+
if "qc_ranges" in entry:
|
|
599
|
+
gr = _deep_merge(gr, entry["qc_ranges"])
|
|
600
|
+
tilt.update(_tilt_from_span(entry["qc_ranges"]))
|
|
601
|
+
if "qc_spike" in entry:
|
|
602
|
+
sp = _deep_merge(sp, entry["qc_spike"])
|
|
603
|
+
if "tilt_qc" in entry:
|
|
604
|
+
tilt.update(entry["tilt_qc"])
|
|
605
|
+
if "qc_flat_line" in entry:
|
|
606
|
+
fl = _deep_merge(fl, entry["qc_flat_line"])
|
|
607
|
+
|
|
608
|
+
# Remove tilt from the gross-range dict — it is handled separately and
|
|
609
|
+
# is not a valid QARTOD gross-range variable.
|
|
610
|
+
gr.pop("tilt", None)
|
|
611
|
+
|
|
612
|
+
return gr, sp, tilt, fl
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
# Velocity variable names in both beam and ENU coordinate systems.
|
|
616
|
+
_VELOCITY_VARS = (
|
|
617
|
+
"velocity_beam1",
|
|
618
|
+
"velocity_beam2",
|
|
619
|
+
"velocity_beam3",
|
|
620
|
+
"east_velocity",
|
|
621
|
+
"north_velocity",
|
|
622
|
+
"up_velocity",
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def _apply_tilt_qc(
|
|
627
|
+
ds: xr.Dataset,
|
|
628
|
+
tilt_cfg: Dict[str, Any],
|
|
629
|
+
qc_attrs: Dict[str, Any],
|
|
630
|
+
) -> tuple[xr.Dataset, int, int]:
|
|
631
|
+
"""Flag velocity variables when pitch or roll exceeds QC thresholds.
|
|
632
|
+
|
|
633
|
+
Primary path — pitch_qc / roll_qc already exist (created by the gross-range
|
|
634
|
+
QC step when pitch and/or roll appear in ``qc_ranges`` in the YAML):
|
|
635
|
+
The two flag arrays are merged element-wise (worst flag wins) and the
|
|
636
|
+
result is propagated to every velocity variable. Any time step where
|
|
637
|
+
pitch OR roll is flagged suspect (3) or bad (4) will flag the velocities
|
|
638
|
+
with the same severity.
|
|
639
|
+
|
|
640
|
+
Fallback path — neither pitch_qc nor roll_qc exist:
|
|
641
|
+
tilt is computed as max(|pitch|, |roll|) and compared against
|
|
642
|
+
tilt_cfg thresholds (``suspect_threshold`` / ``fail_threshold``).
|
|
643
|
+
|
|
644
|
+
In both paths ``tilt_suspect_threshold`` and ``tilt_fail_threshold`` are
|
|
645
|
+
written to global attrs so the report can draw reference lines on the tilt
|
|
646
|
+
time-series panel.
|
|
647
|
+
|
|
648
|
+
Returns (ds, n_suspect, n_bad). No-ops when both pitch and roll are absent.
|
|
649
|
+
"""
|
|
650
|
+
has_pitch = "pitch" in ds.data_vars
|
|
651
|
+
has_roll = "roll" in ds.data_vars
|
|
652
|
+
if not has_pitch and not has_roll:
|
|
653
|
+
return ds, 0, 0
|
|
654
|
+
|
|
655
|
+
has_pitch_qc = "pitch_qc" in ds.data_vars
|
|
656
|
+
has_roll_qc = "roll_qc" in ds.data_vars
|
|
657
|
+
|
|
658
|
+
if has_pitch_qc or has_roll_qc:
|
|
659
|
+
# Primary path: merge already-computed pitch/roll QC flags.
|
|
660
|
+
n_time = ds["time"].size
|
|
661
|
+
combined = np.ones(n_time, dtype=np.int8)
|
|
662
|
+
if has_pitch_qc:
|
|
663
|
+
combined = _merge_flags(combined, ds["pitch_qc"].values.astype(np.int8))
|
|
664
|
+
if has_roll_qc:
|
|
665
|
+
combined = _merge_flags(combined, ds["roll_qc"].values.astype(np.int8))
|
|
666
|
+
tilt_flags = combined
|
|
667
|
+
|
|
668
|
+
# Read thresholds from whichever QC variable is available so the
|
|
669
|
+
# report can draw consistent reference lines on the tilt panel.
|
|
670
|
+
ref_attrs = ds["pitch_qc" if has_pitch_qc else "roll_qc"].attrs
|
|
671
|
+
suspect_thresh = float(
|
|
672
|
+
ref_attrs.get(
|
|
673
|
+
"qc_gross_range_suspect_max", tilt_cfg.get("suspect_threshold", 20.0)
|
|
674
|
+
)
|
|
675
|
+
)
|
|
676
|
+
fail_thresh = float(
|
|
677
|
+
ref_attrs.get(
|
|
678
|
+
"qc_gross_range_fail_max", tilt_cfg.get("fail_threshold", 30.0)
|
|
679
|
+
)
|
|
680
|
+
)
|
|
681
|
+
else:
|
|
682
|
+
# Fallback: compute from raw pitch/roll values.
|
|
683
|
+
n_time = ds["time"].size
|
|
684
|
+
pitch = ds["pitch"].values.astype(float) if has_pitch else np.zeros(n_time)
|
|
685
|
+
roll = ds["roll"].values.astype(float) if has_roll else np.zeros(n_time)
|
|
686
|
+
|
|
687
|
+
tilt = np.maximum(np.abs(pitch), np.abs(roll))
|
|
688
|
+
tilt[~np.isfinite(pitch) | ~np.isfinite(roll)] = np.nan
|
|
689
|
+
|
|
690
|
+
suspect_thresh = float(tilt_cfg.get("suspect_threshold", 20.0))
|
|
691
|
+
fail_thresh = float(tilt_cfg.get("fail_threshold", 30.0))
|
|
692
|
+
|
|
693
|
+
tilt_flags = np.where(
|
|
694
|
+
~np.isfinite(tilt),
|
|
695
|
+
np.int8(9),
|
|
696
|
+
np.where(
|
|
697
|
+
tilt >= fail_thresh,
|
|
698
|
+
np.int8(4),
|
|
699
|
+
np.where(tilt >= suspect_thresh, np.int8(3), np.int8(1)),
|
|
700
|
+
),
|
|
701
|
+
).astype(np.int8)
|
|
702
|
+
|
|
703
|
+
n_suspect = int(np.sum(tilt_flags == 3))
|
|
704
|
+
n_bad = int(np.sum(tilt_flags == 4))
|
|
705
|
+
|
|
706
|
+
# Persist thresholds so the report can draw reference lines on the tilt panel.
|
|
707
|
+
ds.attrs["tilt_suspect_threshold"] = suspect_thresh
|
|
708
|
+
ds.attrs["tilt_fail_threshold"] = fail_thresh
|
|
709
|
+
|
|
710
|
+
if n_suspect == 0 and n_bad == 0:
|
|
711
|
+
return ds, 0, 0
|
|
712
|
+
|
|
713
|
+
vel_vars = [v for v in _VELOCITY_VARS if v in ds.data_vars]
|
|
714
|
+
for varname in vel_vars:
|
|
715
|
+
qc_varname = f"{varname}_qc"
|
|
716
|
+
# For ADCP, velocity is 2-D (time, N_BINS); broadcast (time,) → (time, N_BINS).
|
|
717
|
+
if ds[varname].values.ndim > 1:
|
|
718
|
+
flags_for_var = np.broadcast_to(
|
|
719
|
+
tilt_flags[:, np.newaxis], ds[varname].shape
|
|
720
|
+
).copy()
|
|
721
|
+
else:
|
|
722
|
+
flags_for_var = tilt_flags.copy()
|
|
723
|
+
if qc_varname in ds:
|
|
724
|
+
existing = ds[qc_varname].values.astype(np.int8)
|
|
725
|
+
new_flags = _merge_flags(existing, flags_for_var)
|
|
726
|
+
else:
|
|
727
|
+
new_flags = flags_for_var
|
|
728
|
+
ds[qc_varname] = xr.Variable(
|
|
729
|
+
ds[varname].dims,
|
|
730
|
+
new_flags,
|
|
731
|
+
attrs={"long_name": f"quality flag for {varname}", **qc_attrs},
|
|
732
|
+
)
|
|
733
|
+
|
|
734
|
+
return ds, n_suspect, n_bad
|
|
735
|
+
|
|
736
|
+
|
|
737
|
+
def _ensure_conductivity_units(
|
|
738
|
+
ds: xr.Dataset,
|
|
739
|
+
log_fn: Any = None,
|
|
740
|
+
) -> xr.Dataset:
|
|
741
|
+
"""Convert conductivity from S/m → mS/cm if needed.
|
|
742
|
+
|
|
743
|
+
QC thresholds in parameters.QC_GROSS_RANGE are in mS/cm. Some readers
|
|
744
|
+
(notably sbe-ascii) write S/m; this normalises before QC is applied so
|
|
745
|
+
thresholds are always compared against values in the same unit.
|
|
746
|
+
"""
|
|
747
|
+
if "conductivity" not in ds.data_vars:
|
|
748
|
+
return ds
|
|
749
|
+
units = ds["conductivity"].attrs.get("units", "")
|
|
750
|
+
if units.lower() == "s/m":
|
|
751
|
+
if log_fn:
|
|
752
|
+
log_fn(" WARNING: conductivity is in S/m — converting to mS/cm before QC")
|
|
753
|
+
new_data = ds["conductivity"].values * 10.0
|
|
754
|
+
new_attrs = dict(ds["conductivity"].attrs)
|
|
755
|
+
new_attrs["units"] = "mS/cm"
|
|
756
|
+
ds["conductivity"] = xr.Variable(
|
|
757
|
+
ds["conductivity"].dims, new_data, attrs=new_attrs
|
|
758
|
+
)
|
|
759
|
+
return ds
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def _compute_salinity_data(
|
|
763
|
+
ds: xr.Dataset,
|
|
764
|
+
log_fn: Any = None,
|
|
765
|
+
) -> xr.Dataset:
|
|
766
|
+
"""Compute Practical Salinity (SP) data values only — no QC flags yet.
|
|
767
|
+
|
|
768
|
+
Call this BEFORE ``_apply_qc_tests`` so that salinity participates in the
|
|
769
|
+
gross-range QC pass and gets its threshold attrs written to ``salinity_qc``.
|
|
770
|
+
Call ``_merge_salinity_parent_qc`` afterward to fold in T/C/P parent flags.
|
|
771
|
+
"""
|
|
772
|
+
required = {"temperature", "conductivity", "pressure"}
|
|
773
|
+
if not required.issubset(ds.data_vars):
|
|
774
|
+
return ds
|
|
775
|
+
if ds["conductivity"].attrs.get("units", "").lower() == "s/m":
|
|
776
|
+
if log_fn:
|
|
777
|
+
log_fn(" WARNING: salinity skipped — conductivity still in S/m")
|
|
778
|
+
return ds
|
|
779
|
+
try:
|
|
780
|
+
import gsw
|
|
781
|
+
except ImportError:
|
|
782
|
+
if log_fn:
|
|
783
|
+
log_fn(" WARNING: gsw not installed — salinity not computed")
|
|
784
|
+
return ds
|
|
785
|
+
|
|
786
|
+
C = ds["conductivity"].values.astype(float) # mS/cm
|
|
787
|
+
t = ds["temperature"].values.astype(float) # °C ITS-90
|
|
788
|
+
p = ds["pressure"].values.astype(float) # dbar
|
|
789
|
+
|
|
790
|
+
SP = gsw.SP_from_C(C, t, p)
|
|
791
|
+
ds["salinity"] = xr.Variable(
|
|
792
|
+
"time",
|
|
793
|
+
SP,
|
|
794
|
+
attrs={
|
|
795
|
+
"units": "1",
|
|
796
|
+
"long_name": "Practical Salinity",
|
|
797
|
+
"standard_name": "sea_water_practical_salinity",
|
|
798
|
+
"coverage_content_type": "physicalMeasurement",
|
|
799
|
+
"comment": "Derived from conductivity, temperature, pressure via gsw.SP_from_C",
|
|
800
|
+
},
|
|
801
|
+
)
|
|
802
|
+
return ds
|
|
803
|
+
|
|
804
|
+
|
|
805
|
+
def _merge_salinity_parent_qc(
|
|
806
|
+
ds: xr.Dataset,
|
|
807
|
+
qc_attrs: Dict[str, Any],
|
|
808
|
+
) -> xr.Dataset:
|
|
809
|
+
"""Merge parent (T/C/P) QC flags into salinity_qc after QC tests have run.
|
|
810
|
+
|
|
811
|
+
``_apply_qc_tests`` sets salinity_qc from the gross-range test and stores
|
|
812
|
+
the threshold attrs needed by the report histogram. Here we additionally
|
|
813
|
+
fold in the worst flag from temperature_qc, conductivity_qc, and
|
|
814
|
+
pressure_qc so that a bad/suspect input propagates to salinity.
|
|
815
|
+
"""
|
|
816
|
+
if "salinity" not in ds.data_vars:
|
|
817
|
+
return ds
|
|
818
|
+
parent_flags = []
|
|
819
|
+
for varname in ("temperature", "conductivity", "pressure"):
|
|
820
|
+
qv = f"{varname}_qc"
|
|
821
|
+
if qv not in ds.data_vars:
|
|
822
|
+
continue
|
|
823
|
+
f = ds[qv].values.astype(np.int8).copy()
|
|
824
|
+
if varname == "pressure":
|
|
825
|
+
# Interpolated pressure (flag 8) does not degrade salinity: T and C
|
|
826
|
+
# are still directly measured, so salinity is valid. Treat flag 8
|
|
827
|
+
# as good (1) when propagating from pressure to salinity.
|
|
828
|
+
f[f == 8] = np.int8(1)
|
|
829
|
+
parent_flags.append(f)
|
|
830
|
+
if not parent_flags:
|
|
831
|
+
return ds
|
|
832
|
+
|
|
833
|
+
parent_merged = _merge_flags(*parent_flags)
|
|
834
|
+
if "salinity_qc" in ds:
|
|
835
|
+
existing_attrs = dict(ds["salinity_qc"].attrs)
|
|
836
|
+
new_flags = _merge_flags(
|
|
837
|
+
ds["salinity_qc"].values.astype(np.int8), parent_merged
|
|
838
|
+
)
|
|
839
|
+
else:
|
|
840
|
+
existing_attrs = {}
|
|
841
|
+
new_flags = parent_merged
|
|
842
|
+
|
|
843
|
+
ds["salinity_qc"] = xr.Variable(
|
|
844
|
+
"time",
|
|
845
|
+
new_flags,
|
|
846
|
+
attrs={
|
|
847
|
+
**existing_attrs,
|
|
848
|
+
"long_name": "quality flag for salinity",
|
|
849
|
+
"comment": "QARTOD gross-range + worst of T/C/P parent flags",
|
|
850
|
+
**qc_attrs,
|
|
851
|
+
},
|
|
852
|
+
)
|
|
853
|
+
return ds
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def _apply_qc_tests(
|
|
857
|
+
ds: xr.Dataset,
|
|
858
|
+
gross_range: Dict[str, Any],
|
|
859
|
+
spike: Dict[str, Any],
|
|
860
|
+
qc_attrs: Dict[str, Any],
|
|
861
|
+
flat_line: Optional[Dict[str, Any]] = None,
|
|
862
|
+
) -> xr.Dataset:
|
|
863
|
+
"""Apply QARTOD gross-range, spike, and flat-line tests, writing ``*_qc`` variables.
|
|
864
|
+
|
|
865
|
+
Three QARTOD tests are applied in sequence; the worst flag across all
|
|
866
|
+
tests wins for each sample:
|
|
867
|
+
|
|
868
|
+
* **Gross-range**: flags values outside physically plausible bounds as
|
|
869
|
+
SUSPECT (3) or BAD (4) — e.g. temperature below -2.5 °C or above 40 °C.
|
|
870
|
+
* **Spike**: flags isolated outliers that deviate from the surrounding
|
|
871
|
+
record by more than a threshold — e.g. a brief salinity glitch from
|
|
872
|
+
biofouling or a pressure transient.
|
|
873
|
+
* **Flat-line** (stuck-sensor): flags runs of consecutive samples where
|
|
874
|
+
the value does not change by more than ``tolerance`` — e.g. a pressure
|
|
875
|
+
sensor frozen at 0 dbar after a failure. Threshold is expressed in
|
|
876
|
+
sample counts (``suspect_n``, ``fail_n``) and converted to seconds
|
|
877
|
+
using the median sample interval. Applied by default to pressure only
|
|
878
|
+
(see ``parameters.QC_FLAT_LINE``).
|
|
879
|
+
|
|
880
|
+
Pre-existing ``*_qc`` values (e.g. ``pressure_qc = 8`` set by the
|
|
881
|
+
pressure-interpolation step in the stack) are preserved: the worst flag
|
|
882
|
+
across the incoming value and the new test flags is written to the output.
|
|
883
|
+
|
|
884
|
+
Provenance: the thresholds actually applied are stored as attributes on
|
|
885
|
+
each ``{var}_qc`` variable so the treatment can be reconstructed from
|
|
886
|
+
the NetCDF file alone, without re-reading the YAML:
|
|
887
|
+
|
|
888
|
+
* ``qc_gross_range_fail_min`` / ``qc_gross_range_fail_max``
|
|
889
|
+
* ``qc_gross_range_suspect_min`` / ``qc_gross_range_suspect_max``
|
|
890
|
+
* ``qc_spike_suspect_threshold`` / ``qc_spike_fail_threshold``
|
|
891
|
+
* ``qc_flat_line_suspect_count`` / ``qc_flat_line_fail_count``
|
|
892
|
+
"""
|
|
893
|
+
from ioos_qc import qartod
|
|
894
|
+
|
|
895
|
+
_flat = flat_line or {}
|
|
896
|
+
test_vars = [
|
|
897
|
+
v
|
|
898
|
+
for v in ds.data_vars
|
|
899
|
+
if (v in gross_range or v in spike or v in _flat) and not v.endswith("_qc")
|
|
900
|
+
]
|
|
901
|
+
|
|
902
|
+
for varname in test_vars:
|
|
903
|
+
data = ds[varname].values.copy().astype(float)
|
|
904
|
+
# Start with flag 1 (good) for all non-NaN, 9 (missing) for NaN
|
|
905
|
+
base_flags = np.where(np.isfinite(data), np.int8(1), np.int8(9))
|
|
906
|
+
flags_list = [base_flags]
|
|
907
|
+
|
|
908
|
+
if varname in gross_range:
|
|
909
|
+
cfg = gross_range[varname]
|
|
910
|
+
gr_flags = (
|
|
911
|
+
qartod.gross_range_test(
|
|
912
|
+
inp=data,
|
|
913
|
+
fail_span=tuple(cfg["fail_span"]),
|
|
914
|
+
suspect_span=tuple(cfg.get("suspect_span", cfg["fail_span"])),
|
|
915
|
+
)
|
|
916
|
+
.filled(9)
|
|
917
|
+
.astype(np.int8)
|
|
918
|
+
)
|
|
919
|
+
flags_list.append(gr_flags)
|
|
920
|
+
|
|
921
|
+
if varname in spike:
|
|
922
|
+
cfg = spike[varname]
|
|
923
|
+
sp_flags = (
|
|
924
|
+
qartod.spike_test(
|
|
925
|
+
inp=data,
|
|
926
|
+
suspect_threshold=cfg.get("suspect_threshold"),
|
|
927
|
+
fail_threshold=cfg.get("fail_threshold"),
|
|
928
|
+
)
|
|
929
|
+
.filled(9)
|
|
930
|
+
.astype(np.int8)
|
|
931
|
+
)
|
|
932
|
+
flags_list.append(sp_flags)
|
|
933
|
+
|
|
934
|
+
if varname in _flat and "time" in ds:
|
|
935
|
+
fl_cfg = _flat[varname]
|
|
936
|
+
time_vals = ds["time"].values
|
|
937
|
+
# Convert datetime64 → float seconds since epoch for ioos_qc
|
|
938
|
+
time_s = time_vals.astype("datetime64[s]").astype(float)
|
|
939
|
+
dt_s = float(np.nanmedian(np.diff(time_s))) if len(time_s) > 1 else 60.0
|
|
940
|
+
if dt_s <= 0:
|
|
941
|
+
# All timestamps are identical (or single record); ioos_qc would
|
|
942
|
+
# divide by zero inside flat_line_test — skip the test.
|
|
943
|
+
import warnings
|
|
944
|
+
|
|
945
|
+
warnings.warn(
|
|
946
|
+
f"flat_line_test skipped for '{varname}': median time interval is "
|
|
947
|
+
f"{dt_s:.1f} s (all timestamps identical?)",
|
|
948
|
+
RuntimeWarning,
|
|
949
|
+
stacklevel=2,
|
|
950
|
+
)
|
|
951
|
+
continue
|
|
952
|
+
suspect_n = int(fl_cfg.get("suspect_n", 3))
|
|
953
|
+
fail_n = int(fl_cfg.get("fail_n", 10))
|
|
954
|
+
try:
|
|
955
|
+
_fl_result = qartod.flat_line_test(
|
|
956
|
+
inp=data,
|
|
957
|
+
tinp=time_s,
|
|
958
|
+
suspect_threshold=int(suspect_n * dt_s),
|
|
959
|
+
fail_threshold=int(fail_n * dt_s),
|
|
960
|
+
tolerance=float(fl_cfg.get("tolerance", 0.0)),
|
|
961
|
+
)
|
|
962
|
+
except (ValueError, OverflowError) as _fl_err:
|
|
963
|
+
import warnings
|
|
964
|
+
|
|
965
|
+
warnings.warn(
|
|
966
|
+
f"flat_line_test skipped for '{varname}': {_fl_err}",
|
|
967
|
+
RuntimeWarning,
|
|
968
|
+
stacklevel=2,
|
|
969
|
+
)
|
|
970
|
+
continue
|
|
971
|
+
# flat_line_test may return a masked array or plain ndarray depending
|
|
972
|
+
# on the ioos_qc version; handle both.
|
|
973
|
+
fl_flags = (
|
|
974
|
+
_fl_result.filled(9)
|
|
975
|
+
if hasattr(_fl_result, "filled")
|
|
976
|
+
else np.where(np.isnan(_fl_result.astype(float)), 9, _fl_result)
|
|
977
|
+
).astype(np.int8)
|
|
978
|
+
flags_list.append(fl_flags)
|
|
979
|
+
|
|
980
|
+
new_flags = _merge_flags(*flags_list)
|
|
981
|
+
|
|
982
|
+
qc_varname = f"{varname}_qc"
|
|
983
|
+
if qc_varname in ds:
|
|
984
|
+
existing = ds[qc_varname].values.astype(np.int8)
|
|
985
|
+
new_flags = _merge_flags(existing, new_flags)
|
|
986
|
+
|
|
987
|
+
# Store the actual thresholds applied so downstream tools (e.g. the
|
|
988
|
+
# report histogram) can show exactly what was used without re-reading YAML.
|
|
989
|
+
threshold_attrs: Dict[str, Any] = {}
|
|
990
|
+
if varname in gross_range:
|
|
991
|
+
gcfg = gross_range[varname]
|
|
992
|
+
if "fail_span" in gcfg:
|
|
993
|
+
threshold_attrs["qc_gross_range_fail_min"] = float(gcfg["fail_span"][0])
|
|
994
|
+
threshold_attrs["qc_gross_range_fail_max"] = float(gcfg["fail_span"][1])
|
|
995
|
+
if "suspect_span" in gcfg:
|
|
996
|
+
threshold_attrs["qc_gross_range_suspect_min"] = float(
|
|
997
|
+
gcfg["suspect_span"][0]
|
|
998
|
+
)
|
|
999
|
+
threshold_attrs["qc_gross_range_suspect_max"] = float(
|
|
1000
|
+
gcfg["suspect_span"][1]
|
|
1001
|
+
)
|
|
1002
|
+
if varname in spike:
|
|
1003
|
+
scfg = spike[varname]
|
|
1004
|
+
if "suspect_threshold" in scfg:
|
|
1005
|
+
threshold_attrs["qc_spike_suspect_threshold"] = float(
|
|
1006
|
+
scfg["suspect_threshold"]
|
|
1007
|
+
)
|
|
1008
|
+
if "fail_threshold" in scfg:
|
|
1009
|
+
threshold_attrs["qc_spike_fail_threshold"] = float(
|
|
1010
|
+
scfg["fail_threshold"]
|
|
1011
|
+
)
|
|
1012
|
+
if varname in _flat:
|
|
1013
|
+
fl_cfg = _flat[varname]
|
|
1014
|
+
threshold_attrs["qc_flat_line_suspect_count"] = int(
|
|
1015
|
+
fl_cfg.get("suspect_n", 3)
|
|
1016
|
+
)
|
|
1017
|
+
threshold_attrs["qc_flat_line_fail_count"] = int(fl_cfg.get("fail_n", 10))
|
|
1018
|
+
|
|
1019
|
+
ds[qc_varname] = xr.Variable(
|
|
1020
|
+
ds[varname].dims,
|
|
1021
|
+
new_flags,
|
|
1022
|
+
attrs={
|
|
1023
|
+
"long_name": f"quality flag for {varname}",
|
|
1024
|
+
**qc_attrs,
|
|
1025
|
+
**threshold_attrs,
|
|
1026
|
+
},
|
|
1027
|
+
)
|
|
1028
|
+
|
|
1029
|
+
return ds
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def _apply_enu_velocity_qc(
|
|
1033
|
+
ds: xr.Dataset,
|
|
1034
|
+
gr_cfg: Dict[str, Any],
|
|
1035
|
+
qc_attrs: Dict[str, Any],
|
|
1036
|
+
) -> xr.Dataset:
|
|
1037
|
+
"""Apply QARTOD gross-range QC to ENU velocity vars and propagate w flags.
|
|
1038
|
+
|
|
1039
|
+
Must be called after _apply_beam_to_enu (east/north/up_velocity must exist).
|
|
1040
|
+
No spike test is applied to velocity (burst-mode Aquadopps generate false
|
|
1041
|
+
positives at every burst boundary).
|
|
1042
|
+
|
|
1043
|
+
Propagates up_velocity_qc (flag 3 or 4) to east_velocity_qc and
|
|
1044
|
+
north_velocity_qc: if vertical velocity is implausibly large the whole
|
|
1045
|
+
3-D velocity measurement is suspect/bad. Full bidirectional unification
|
|
1046
|
+
(large east/north flags → up_velocity_qc) is done by _unify_velocity_qc,
|
|
1047
|
+
which must be called after _apply_tilt_qc.
|
|
1048
|
+
"""
|
|
1049
|
+
enu_gr = {
|
|
1050
|
+
k: v
|
|
1051
|
+
for k, v in gr_cfg.items()
|
|
1052
|
+
if k in ("east_velocity", "north_velocity", "up_velocity")
|
|
1053
|
+
}
|
|
1054
|
+
if enu_gr:
|
|
1055
|
+
ds = _apply_qc_tests(ds, enu_gr, {}, qc_attrs)
|
|
1056
|
+
|
|
1057
|
+
if "up_velocity_qc" in ds.data_vars:
|
|
1058
|
+
up_flags = ds["up_velocity_qc"].values.astype(np.int8)
|
|
1059
|
+
for vel_var in ("east_velocity", "north_velocity"):
|
|
1060
|
+
qc_varname = f"{vel_var}_qc"
|
|
1061
|
+
if qc_varname in ds.data_vars:
|
|
1062
|
+
existing = ds[qc_varname].values.astype(np.int8)
|
|
1063
|
+
merged = _merge_flags(existing, up_flags)
|
|
1064
|
+
ds[qc_varname] = xr.Variable(
|
|
1065
|
+
ds[vel_var].dims, merged, attrs=dict(ds[qc_varname].attrs)
|
|
1066
|
+
)
|
|
1067
|
+
return ds
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
def _unify_velocity_qc(ds: xr.Dataset) -> xr.Dataset:
|
|
1071
|
+
"""Unify ENU velocity QC flags so all three components share the worst flag.
|
|
1072
|
+
|
|
1073
|
+
After gross-range, up-velocity propagation, and tilt QC, each component may
|
|
1074
|
+
carry different flags. A large east/north velocity flags only that component;
|
|
1075
|
+
tilt flags all three; up-velocity flags east and north but not itself from the
|
|
1076
|
+
east/north gross-range test. This function computes the element-wise worst flag
|
|
1077
|
+
across east_velocity_qc, north_velocity_qc, and up_velocity_qc, then writes
|
|
1078
|
+
that combined flag back to all three. The result: masking on any single
|
|
1079
|
+
component produces an identical, consistent velocity mask.
|
|
1080
|
+
"""
|
|
1081
|
+
keys = [
|
|
1082
|
+
v
|
|
1083
|
+
for v in ("east_velocity_qc", "north_velocity_qc", "up_velocity_qc")
|
|
1084
|
+
if v in ds.data_vars
|
|
1085
|
+
]
|
|
1086
|
+
if not keys:
|
|
1087
|
+
return ds
|
|
1088
|
+
combined = np.ones(ds[keys[0]].shape, dtype=np.int8)
|
|
1089
|
+
for k in keys:
|
|
1090
|
+
combined = _merge_flags(combined, ds[k].values.astype(np.int8))
|
|
1091
|
+
for k in keys:
|
|
1092
|
+
ds[k] = xr.Variable(ds[k].dims, combined.copy(), attrs=dict(ds[k].attrs))
|
|
1093
|
+
return ds
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
def _compute_adcp_bin_pressure(
|
|
1097
|
+
ds: xr.Dataset,
|
|
1098
|
+
lat: float,
|
|
1099
|
+
log_fn: Any = None,
|
|
1100
|
+
) -> xr.Dataset:
|
|
1101
|
+
"""Compute pressure at each ADCP bin from transducer pressure and along-beam range.
|
|
1102
|
+
|
|
1103
|
+
For each time step, adds a pressure offset per bin based on the bin's distance
|
|
1104
|
+
from the transducer. The offset is ``gsw.p_from_z(-range_m, lat)``, which
|
|
1105
|
+
approximates the hydrostatic pressure contribution of *range_m* metres of
|
|
1106
|
+
seawater at the given latitude.
|
|
1107
|
+
|
|
1108
|
+
**Note on ``range`` units**: the RDI WorkHorse firmware already converts slant range
|
|
1109
|
+
to vertical distance before writing to the raw output (using the nominal beam angle;
|
|
1110
|
+
see RDI ADCP Coordinate Transformation manual §4.2, Equation 8). Dolfyn reads these
|
|
1111
|
+
vertical distances directly, so ``range`` is already in metres of vertical depth —
|
|
1112
|
+
no beam-angle correction is needed.
|
|
1113
|
+
|
|
1114
|
+
**Approximation** (fixable post-OdB; see ``.claude/refactor-plan-postOdB-20260721.md``):
|
|
1115
|
+
``gsw.p_from_z(-range_m, lat)`` computes the pressure of a water column of depth
|
|
1116
|
+
*range_m* measured from the *sea surface*, not the true pressure increment at the
|
|
1117
|
+
ADCP's actual depth. Error at 300 m range from a 500 m transducer is ~2–3 dbar —
|
|
1118
|
+
within the seabed-QC margin (~20 dbar) for current moorings.
|
|
1119
|
+
|
|
1120
|
+
The sign follows instrument orientation:
|
|
1121
|
+
|
|
1122
|
+
- Downward-looking (``orientation == "down"``): bins are deeper than the
|
|
1123
|
+
transducer → pressure increases with bin index.
|
|
1124
|
+
- Upward-looking (``orientation == "up"``): bins are shallower than the
|
|
1125
|
+
transducer → pressure decreases with bin index.
|
|
1126
|
+
|
|
1127
|
+
If orientation cannot be determined from dataset attributes a WARNING is emitted
|
|
1128
|
+
and ``"down"`` is assumed.
|
|
1129
|
+
|
|
1130
|
+
Parameters
|
|
1131
|
+
----------
|
|
1132
|
+
ds : xr.Dataset
|
|
1133
|
+
Stage 3 ADCP dataset. Must contain ``pressure(time)`` in dbar (pressure at
|
|
1134
|
+
the transducer head) and ``range(N_BINS)`` in metres (bin centre distance from
|
|
1135
|
+
the transducer face). Orientation is read from the ``orientation_yaml`` global
|
|
1136
|
+
attribute (preferred) or ``orientation_instrument`` (fallback).
|
|
1137
|
+
lat : float
|
|
1138
|
+
Mooring latitude in decimal degrees (positive North), used by
|
|
1139
|
+
``gsw.p_from_z`` for the gravitational/centrifugal correction.
|
|
1140
|
+
log_fn : callable, optional
|
|
1141
|
+
Logging callback (e.g. ``logger.info``).
|
|
1142
|
+
|
|
1143
|
+
Returns
|
|
1144
|
+
-------
|
|
1145
|
+
xr.Dataset
|
|
1146
|
+
Input dataset with ``bin_pressure(time, N_BINS)`` added in dbar.
|
|
1147
|
+
The variable's ``comment`` attribute records the formula and orientation used.
|
|
1148
|
+
Returns *ds* unchanged if ``pressure`` or ``range`` are absent.
|
|
1149
|
+
|
|
1150
|
+
"""
|
|
1151
|
+
import gsw
|
|
1152
|
+
from . import parameters as _P
|
|
1153
|
+
|
|
1154
|
+
if "pressure" not in ds.data_vars or "range" not in ds.coords:
|
|
1155
|
+
if log_fn:
|
|
1156
|
+
log_fn(" ADCP bin_pressure: skipped (no pressure or range coord)")
|
|
1157
|
+
return ds
|
|
1158
|
+
|
|
1159
|
+
orientation = ds.attrs.get("orientation_yaml") or ds.attrs.get(
|
|
1160
|
+
"orientation_instrument"
|
|
1161
|
+
)
|
|
1162
|
+
if not orientation:
|
|
1163
|
+
orientation = "down"
|
|
1164
|
+
if log_fn:
|
|
1165
|
+
log_fn(
|
|
1166
|
+
" WARNING: ADCP orientation unknown (no orientation_yaml or "
|
|
1167
|
+
"orientation_instrument attr) — assuming downward-looking. "
|
|
1168
|
+
"bin_pressure signs may be wrong for upward-looking instruments."
|
|
1169
|
+
)
|
|
1170
|
+
sign = 1.0 if str(orientation).lower() == "down" else -1.0
|
|
1171
|
+
|
|
1172
|
+
p_trans = ds["pressure"].values.astype(float) # (time,)
|
|
1173
|
+
range_m = ds["range"].values.astype(float) # (N_BINS,)
|
|
1174
|
+
|
|
1175
|
+
# gsw.p_from_z(z, lat): z is depth in m (negative = below surface)
|
|
1176
|
+
dp = gsw.p_from_z(-range_m, lat) # (N_BINS,) pressure offset per bin
|
|
1177
|
+
|
|
1178
|
+
bin_p = p_trans[:, np.newaxis] + sign * dp[np.newaxis, :] # (time, N_BINS)
|
|
1179
|
+
|
|
1180
|
+
ds["bin_pressure"] = xr.Variable(
|
|
1181
|
+
("time", _P.ADCP_BIN_DIM),
|
|
1182
|
+
bin_p.astype(np.float32),
|
|
1183
|
+
attrs={
|
|
1184
|
+
"long_name": "pressure at ADCP bin",
|
|
1185
|
+
"units": "dbar",
|
|
1186
|
+
"comment": (
|
|
1187
|
+
f"transducer pressure + gsw.p_from_z(-range, lat={lat:.2f}°)"
|
|
1188
|
+
f" × sign({orientation}-looking)"
|
|
1189
|
+
),
|
|
1190
|
+
},
|
|
1191
|
+
)
|
|
1192
|
+
if log_fn:
|
|
1193
|
+
log_fn(
|
|
1194
|
+
f" ADCP bin_pressure: {ds['range'].size} bins, "
|
|
1195
|
+
f"orientation={orientation}, lat={lat:.2f}°"
|
|
1196
|
+
)
|
|
1197
|
+
return ds
|
|
1198
|
+
|
|
1199
|
+
|
|
1200
|
+
def _apply_adcp_seabed_qc(
|
|
1201
|
+
ds: xr.Dataset,
|
|
1202
|
+
water_depth_m: float,
|
|
1203
|
+
lat: float,
|
|
1204
|
+
qc_attrs: Dict[str, Any],
|
|
1205
|
+
fail_margin_m: float = 20.0,
|
|
1206
|
+
log_fn: Any = None,
|
|
1207
|
+
) -> xr.Dataset:
|
|
1208
|
+
"""Flag ADCP bins that are at or below the seabed.
|
|
1209
|
+
|
|
1210
|
+
Bins whose ``bin_pressure`` exceeds the estimated seabed pressure are flagged
|
|
1211
|
+
suspect (3); bins more than *fail_margin_m* metres below the seabed are flagged
|
|
1212
|
+
bad (4). The flag is stored as the standalone variable ``seabed_qc(time, N_BINS)``.
|
|
1213
|
+
|
|
1214
|
+
**Flag scale** (OceanSITES / QARTOD convention):
|
|
1215
|
+
1 = good, 3 = suspect, 4 = bad, 9 = missing.
|
|
1216
|
+
|
|
1217
|
+
**Standalone flag**: ``seabed_qc`` is *not* merged into ``east_velocity_qc``,
|
|
1218
|
+
``north_velocity_qc``, or ``up_velocity_qc``. Downstream code that needs clean
|
|
1219
|
+
velocity data must explicitly include all relevant flags, for example::
|
|
1220
|
+
|
|
1221
|
+
clean = (ds.east_velocity_qc == 1) & (ds.seabed_qc == 1)
|
|
1222
|
+
|
|
1223
|
+
The seabed pressure threshold is computed via ``gsw.p_from_z(-water_depth_m, lat)``,
|
|
1224
|
+
consistent with ``_compute_adcp_bin_pressure``.
|
|
1225
|
+
|
|
1226
|
+
Parameters
|
|
1227
|
+
----------
|
|
1228
|
+
ds : xr.Dataset
|
|
1229
|
+
Stage 3 ADCP dataset containing ``bin_pressure(time, N_BINS)`` in dbar.
|
|
1230
|
+
water_depth_m : float
|
|
1231
|
+
Water depth at the mooring site in metres (from YAML ``waterdepth`` key).
|
|
1232
|
+
If ≤ 0 the function is a no-op.
|
|
1233
|
+
lat : float
|
|
1234
|
+
Mooring latitude in decimal degrees, used by ``gsw.p_from_z``.
|
|
1235
|
+
qc_attrs : dict
|
|
1236
|
+
OceanSITES flag attribute dict written to ``seabed_qc``.
|
|
1237
|
+
fail_margin_m : float
|
|
1238
|
+
Margin below the seabed (in metres) that separates suspect (3) from bad (4).
|
|
1239
|
+
Default 20 m. Bins between 0 and *fail_margin_m* below the seabed receive
|
|
1240
|
+
flag 3; bins more than *fail_margin_m* below receive flag 4.
|
|
1241
|
+
log_fn : callable, optional
|
|
1242
|
+
Logging callback.
|
|
1243
|
+
|
|
1244
|
+
Returns
|
|
1245
|
+
-------
|
|
1246
|
+
xr.Dataset
|
|
1247
|
+
Input dataset with ``seabed_qc(time, N_BINS)`` added.
|
|
1248
|
+
Returns *ds* unchanged if ``water_depth_m <= 0`` or ``bin_pressure`` is absent.
|
|
1249
|
+
|
|
1250
|
+
"""
|
|
1251
|
+
if water_depth_m <= 0:
|
|
1252
|
+
if log_fn:
|
|
1253
|
+
log_fn(" seabed QC: skipped (waterdepth not set in mooring YAML)")
|
|
1254
|
+
return ds
|
|
1255
|
+
if "bin_pressure" not in ds.data_vars:
|
|
1256
|
+
if log_fn:
|
|
1257
|
+
log_fn(" seabed QC: skipped (bin_pressure not in dataset)")
|
|
1258
|
+
return ds
|
|
1259
|
+
|
|
1260
|
+
import gsw
|
|
1261
|
+
from . import parameters as _P
|
|
1262
|
+
|
|
1263
|
+
p_seabed = float(gsw.p_from_z(-water_depth_m, lat))
|
|
1264
|
+
p_fail = float(gsw.p_from_z(-(water_depth_m + fail_margin_m), lat))
|
|
1265
|
+
|
|
1266
|
+
bin_p = ds["bin_pressure"].values # (time, N_BINS)
|
|
1267
|
+
seabed_flags = np.ones(bin_p.shape, dtype=np.int8) # 1 = good
|
|
1268
|
+
seabed_flags = np.where(bin_p > p_seabed, np.int8(3), seabed_flags) # suspect
|
|
1269
|
+
seabed_flags = np.where(bin_p > p_fail, np.int8(4), seabed_flags) # bad
|
|
1270
|
+
seabed_flags = seabed_flags.astype(np.int8)
|
|
1271
|
+
|
|
1272
|
+
n_suspect = int(np.sum(seabed_flags == 3))
|
|
1273
|
+
n_bad = int(np.sum(seabed_flags == 4))
|
|
1274
|
+
|
|
1275
|
+
ds["seabed_qc"] = xr.Variable(
|
|
1276
|
+
("time", _P.ADCP_BIN_DIM),
|
|
1277
|
+
seabed_flags,
|
|
1278
|
+
{
|
|
1279
|
+
"long_name": "Seabed proximity QC flag",
|
|
1280
|
+
"comment": (
|
|
1281
|
+
f"Bins at or below seabed ({water_depth_m:.0f} m, "
|
|
1282
|
+
f"p_seabed={p_seabed:.1f} dbar): flag 3 (suspect); "
|
|
1283
|
+
f"bins >{fail_margin_m:.0f} m below seabed "
|
|
1284
|
+
f"(p={p_fail:.1f} dbar): flag 4 (bad). "
|
|
1285
|
+
"Standalone — not merged into velocity_qc."
|
|
1286
|
+
),
|
|
1287
|
+
**qc_attrs,
|
|
1288
|
+
},
|
|
1289
|
+
)
|
|
1290
|
+
|
|
1291
|
+
if log_fn:
|
|
1292
|
+
log_fn(
|
|
1293
|
+
f" seabed QC: water_depth={water_depth_m:.0f} m, "
|
|
1294
|
+
f"p_seabed={p_seabed:.1f} dbar, "
|
|
1295
|
+
f"p_fail={p_fail:.1f} dbar (+{fail_margin_m:.0f} m), "
|
|
1296
|
+
f"suspect={n_suspect}, bad={n_bad}"
|
|
1297
|
+
)
|
|
1298
|
+
return ds
|
|
1299
|
+
|
|
1300
|
+
|
|
1301
|
+
def _apply_adcp_surface_qc(
|
|
1302
|
+
ds: xr.Dataset,
|
|
1303
|
+
lat: float,
|
|
1304
|
+
qc_attrs: Dict[str, Any],
|
|
1305
|
+
suspect_margin_m: float = 20.0,
|
|
1306
|
+
log_fn: Any = None,
|
|
1307
|
+
) -> xr.Dataset:
|
|
1308
|
+
"""Flag ADCP bins that are at or above the sea surface.
|
|
1309
|
+
|
|
1310
|
+
Bins whose ``bin_pressure`` is ≤ 0 dbar are flagged bad (4); bins within
|
|
1311
|
+
*suspect_margin_m* metres of the surface (0 < bin_pressure < p_suspect) are
|
|
1312
|
+
flagged suspect (3). The result is stored as the standalone variable
|
|
1313
|
+
``surface_qc(time, N_BINS)``.
|
|
1314
|
+
|
|
1315
|
+
This catches upward-looking ADCP bins that extend above the water surface
|
|
1316
|
+
when the instrument range exceeds the distance to the surface. Symmetric
|
|
1317
|
+
counterpart to ``_apply_adcp_seabed_qc``.
|
|
1318
|
+
|
|
1319
|
+
**Flag scale** (OceanSITES / QARTOD convention):
|
|
1320
|
+
1 = good, 3 = suspect, 4 = bad.
|
|
1321
|
+
|
|
1322
|
+
**Standalone flag**: ``surface_qc`` is *not* merged into the velocity QC
|
|
1323
|
+
variables. Downstream consumers must combine flags explicitly::
|
|
1324
|
+
|
|
1325
|
+
clean = (ds.east_velocity_qc == 1) & (ds.surface_qc == 1) & (ds.seabed_qc == 1)
|
|
1326
|
+
|
|
1327
|
+
Parameters
|
|
1328
|
+
----------
|
|
1329
|
+
ds : xr.Dataset
|
|
1330
|
+
Stage 3 ADCP dataset containing ``bin_pressure(time, N_BINS)`` in dbar.
|
|
1331
|
+
lat : float
|
|
1332
|
+
Mooring latitude in decimal degrees, used by ``gsw.p_from_z``.
|
|
1333
|
+
qc_attrs : dict
|
|
1334
|
+
OceanSITES flag attribute dict written to ``surface_qc``.
|
|
1335
|
+
suspect_margin_m : float
|
|
1336
|
+
Depth (m) below the surface that defines the suspect zone. Bins with
|
|
1337
|
+
0 < bin_pressure < p_from_z(-suspect_margin_m) receive flag 3; bins
|
|
1338
|
+
with bin_pressure ≤ 0 receive flag 4. Default 20 m.
|
|
1339
|
+
log_fn : callable, optional
|
|
1340
|
+
Logging callback.
|
|
1341
|
+
|
|
1342
|
+
Returns
|
|
1343
|
+
-------
|
|
1344
|
+
xr.Dataset
|
|
1345
|
+
Input dataset with ``surface_qc(time, N_BINS)`` added.
|
|
1346
|
+
Returns *ds* unchanged if ``bin_pressure`` is absent.
|
|
1347
|
+
|
|
1348
|
+
"""
|
|
1349
|
+
if "bin_pressure" not in ds.data_vars:
|
|
1350
|
+
if log_fn:
|
|
1351
|
+
log_fn(" surface QC: skipped (bin_pressure not in dataset)")
|
|
1352
|
+
return ds
|
|
1353
|
+
|
|
1354
|
+
import gsw
|
|
1355
|
+
from . import parameters as _P
|
|
1356
|
+
|
|
1357
|
+
p_suspect = float(gsw.p_from_z(-suspect_margin_m, lat))
|
|
1358
|
+
|
|
1359
|
+
bin_p = ds["bin_pressure"].values # (time, N_BINS)
|
|
1360
|
+
surface_flags = np.ones(bin_p.shape, dtype=np.int8) # 1 = good
|
|
1361
|
+
surface_flags = np.where(bin_p < p_suspect, np.int8(3), surface_flags) # suspect
|
|
1362
|
+
surface_flags = np.where(bin_p <= 0.0, np.int8(4), surface_flags) # bad
|
|
1363
|
+
|
|
1364
|
+
n_suspect = int(np.sum(surface_flags == 3))
|
|
1365
|
+
n_bad = int(np.sum(surface_flags == 4))
|
|
1366
|
+
|
|
1367
|
+
ds["surface_qc"] = xr.Variable(
|
|
1368
|
+
("time", _P.ADCP_BIN_DIM),
|
|
1369
|
+
surface_flags,
|
|
1370
|
+
{
|
|
1371
|
+
"long_name": "Sea surface proximity QC flag",
|
|
1372
|
+
"comment": (
|
|
1373
|
+
f"Bins within {suspect_margin_m:.0f} m of sea surface "
|
|
1374
|
+
f"(bin_pressure < {p_suspect:.1f} dbar): flag 3 (suspect); "
|
|
1375
|
+
f"bins at or above surface (bin_pressure ≤ 0 dbar): flag 4 (bad). "
|
|
1376
|
+
"Standalone — not merged into velocity_qc."
|
|
1377
|
+
),
|
|
1378
|
+
**qc_attrs,
|
|
1379
|
+
},
|
|
1380
|
+
)
|
|
1381
|
+
|
|
1382
|
+
if log_fn:
|
|
1383
|
+
log_fn(
|
|
1384
|
+
f" surface QC: p_suspect={p_suspect:.1f} dbar ({suspect_margin_m:.0f} m), "
|
|
1385
|
+
f"suspect={n_suspect}, bad={n_bad}"
|
|
1386
|
+
)
|
|
1387
|
+
return ds
|
|
1388
|
+
|
|
1389
|
+
|
|
1390
|
+
def _apply_adcp_velocity_qc(
|
|
1391
|
+
ds: xr.Dataset,
|
|
1392
|
+
gr_cfg: Dict[str, Any],
|
|
1393
|
+
prcnt_gd_bad: float,
|
|
1394
|
+
prcnt_gd_suspect: float,
|
|
1395
|
+
error_vel_threshold: float,
|
|
1396
|
+
qc_attrs: Dict[str, Any],
|
|
1397
|
+
log_fn: Any = None,
|
|
1398
|
+
) -> xr.Dataset:
|
|
1399
|
+
"""Apply QC to ADCP 2D velocity variables (time × N_BINS).
|
|
1400
|
+
|
|
1401
|
+
Each QC criterion produces its **own standalone variable**; flags are
|
|
1402
|
+
**not** merged across variables. Downstream users combine them to mask
|
|
1403
|
+
data, e.g.::
|
|
1404
|
+
|
|
1405
|
+
good = (
|
|
1406
|
+
(ds.east_velocity_qc == 1)
|
|
1407
|
+
& (ds.percent_good_qc == 1)
|
|
1408
|
+
& (ds.error_velocity_qc == 1)
|
|
1409
|
+
& (ds.seabed_qc == 1)
|
|
1410
|
+
& (ds.surface_qc == 1)
|
|
1411
|
+
)
|
|
1412
|
+
|
|
1413
|
+
QC variables produced
|
|
1414
|
+
---------------------
|
|
1415
|
+
``east/north/up_velocity_qc``
|
|
1416
|
+
Gross-range flag on the velocity component value itself (same
|
|
1417
|
+
fail_span / suspect_span thresholds as point instruments). Flag 1
|
|
1418
|
+
means the velocity value is within the accepted range; it says nothing
|
|
1419
|
+
about acoustic quality.
|
|
1420
|
+
|
|
1421
|
+
``percent_good_qc(time, N_BINS)``
|
|
1422
|
+
RDI ADCPs write four percent-good columns per ensemble per bin:
|
|
1423
|
+
|
|
1424
|
+
======= =============================================================
|
|
1425
|
+
Col 0 % pings accepted as 3-beam solutions (one beam rejected)
|
|
1426
|
+
Col 1 % pings rejected by the error-velocity threshold
|
|
1427
|
+
Col 2 % pings rejected by low correlation or low amplitude
|
|
1428
|
+
**Col 3** **% pings accepted as 4-beam solutions ← quality metric**
|
|
1429
|
+
======= =============================================================
|
|
1430
|
+
|
|
1431
|
+
Column 3 (4-beam solutions) is the relevant quality indicator.
|
|
1432
|
+
Averaging all four columns is wrong: when data is perfect, col 3 ≈
|
|
1433
|
+
100 % and cols 0–2 ≈ 0 %, giving a mean of ~25 %, which falls below
|
|
1434
|
+
any reasonable suspect threshold and flags everything. This function
|
|
1435
|
+
therefore uses column 3 alone (falling back to the column mean for
|
|
1436
|
+
non-4-beam ADCPs that store fewer than 4 columns).
|
|
1437
|
+
|
|
1438
|
+
Flags: col3 < *prcnt_gd_bad* → bad (4);
|
|
1439
|
+
col3 < *prcnt_gd_suspect* → suspect (3); otherwise good (1).
|
|
1440
|
+
|
|
1441
|
+
``error_velocity_qc(time, N_BINS)``
|
|
1442
|
+
For a 4-beam ADCP the error velocity is the difference between two
|
|
1443
|
+
independent estimates of vertical velocity from opposite beam pairs.
|
|
1444
|
+
It is zero for a perfect measurement; large values indicate beam
|
|
1445
|
+
decorrelation (e.g. fish, bubbles, mooring motion).
|
|
1446
|
+
|error_velocity| > *error_vel_threshold* → bad (4).
|
|
1447
|
+
|
|
1448
|
+
Parameters
|
|
1449
|
+
----------
|
|
1450
|
+
ds : xr.Dataset
|
|
1451
|
+
Stage 2 dataset with 2-D ADCP velocity variables.
|
|
1452
|
+
gr_cfg : dict
|
|
1453
|
+
Gross-range config (same format as ``_load_qc_config`` returns).
|
|
1454
|
+
prcnt_gd_bad : float
|
|
1455
|
+
4-beam percent good below this → flag 4 (bad). Percent.
|
|
1456
|
+
prcnt_gd_suspect : float
|
|
1457
|
+
4-beam percent good below this (but above prcnt_gd_bad) → flag 3
|
|
1458
|
+
(suspect). Percent.
|
|
1459
|
+
error_vel_threshold : float
|
|
1460
|
+
|error_velocity| above this → flag 4 (bad). m s⁻¹.
|
|
1461
|
+
qc_attrs : dict
|
|
1462
|
+
OceanSITES flag attribute dict written to every ``*_qc`` variable.
|
|
1463
|
+
log_fn : callable, optional
|
|
1464
|
+
Logging callback.
|
|
1465
|
+
|
|
1466
|
+
Returns
|
|
1467
|
+
-------
|
|
1468
|
+
xr.Dataset
|
|
1469
|
+
Input dataset with the following standalone QC variables added (where their
|
|
1470
|
+
parent data variables are present):
|
|
1471
|
+
|
|
1472
|
+
- ``east_velocity_qc(time, N_BINS)`` — gross-range flag on east velocity
|
|
1473
|
+
- ``north_velocity_qc(time, N_BINS)`` — gross-range flag on north velocity
|
|
1474
|
+
- ``up_velocity_qc(time, N_BINS)`` — gross-range flag on up velocity
|
|
1475
|
+
- ``percent_good_qc(time, N_BINS)`` — 4-beam percent-good flag
|
|
1476
|
+
- ``error_velocity_qc(time, N_BINS)`` — error velocity magnitude flag
|
|
1477
|
+
|
|
1478
|
+
All flags use the OceanSITES scale: 1 = good, 3 = suspect, 4 = bad, 9 = missing.
|
|
1479
|
+
Variables are standalone; no merging across criteria is performed.
|
|
1480
|
+
|
|
1481
|
+
"""
|
|
1482
|
+
# 1. Gross-range on each ENU velocity component independently.
|
|
1483
|
+
# Flags only reflect whether that component's value is out of range.
|
|
1484
|
+
for varname in ("east_velocity", "north_velocity", "up_velocity"):
|
|
1485
|
+
if varname not in ds.data_vars:
|
|
1486
|
+
continue
|
|
1487
|
+
cfg = gr_cfg.get(varname)
|
|
1488
|
+
if not cfg:
|
|
1489
|
+
continue
|
|
1490
|
+
data = ds[varname].values.astype(float)
|
|
1491
|
+
fail_min, fail_max = cfg["fail_span"]
|
|
1492
|
+
susp_min, susp_max = cfg.get("suspect_span", cfg["fail_span"])
|
|
1493
|
+
flags = np.where(
|
|
1494
|
+
~np.isfinite(data),
|
|
1495
|
+
np.int8(9),
|
|
1496
|
+
np.where(
|
|
1497
|
+
(data < fail_min) | (data > fail_max),
|
|
1498
|
+
np.int8(4),
|
|
1499
|
+
np.where((data < susp_min) | (data > susp_max), np.int8(3), np.int8(1)),
|
|
1500
|
+
),
|
|
1501
|
+
).astype(np.int8)
|
|
1502
|
+
qc_var = f"{varname}_qc"
|
|
1503
|
+
threshold_attrs: Dict[str, Any] = {
|
|
1504
|
+
"qc_gross_range_fail_min": float(fail_min),
|
|
1505
|
+
"qc_gross_range_fail_max": float(fail_max),
|
|
1506
|
+
"qc_gross_range_suspect_min": float(susp_min),
|
|
1507
|
+
"qc_gross_range_suspect_max": float(susp_max),
|
|
1508
|
+
}
|
|
1509
|
+
ds[qc_var] = xr.Variable(
|
|
1510
|
+
ds[varname].dims,
|
|
1511
|
+
flags,
|
|
1512
|
+
attrs={
|
|
1513
|
+
"long_name": f"quality flag for {varname}",
|
|
1514
|
+
**qc_attrs,
|
|
1515
|
+
**threshold_attrs,
|
|
1516
|
+
},
|
|
1517
|
+
)
|
|
1518
|
+
|
|
1519
|
+
# 2. percent_good QC — standalone variable, NOT merged into velocity_qc.
|
|
1520
|
+
# RDI ADCPs store four percent-good columns per bin:
|
|
1521
|
+
# [0] 3-beam solutions [1] rejected (error vel)
|
|
1522
|
+
# [2] rejected (low corr/amp) [3] 4-beam solutions ← the useful one
|
|
1523
|
+
# Using column 3 only; averaging all columns gives ~25% even for
|
|
1524
|
+
# perfect data (100% 4-beam, 0% others), which would flag everything.
|
|
1525
|
+
if "percent_good" in ds.data_vars:
|
|
1526
|
+
pg = ds["percent_good"].values.astype(float) # (time, N_BINS, beam)
|
|
1527
|
+
if pg.ndim == 3 and pg.shape[2] >= 4:
|
|
1528
|
+
pg_col3 = pg[:, :, 3] # 4-beam solutions column
|
|
1529
|
+
else:
|
|
1530
|
+
pg_col3 = np.nanmean(pg, axis=-1) # fallback for non-4-beam ADCPs
|
|
1531
|
+
pg_flags = np.where(
|
|
1532
|
+
~np.isfinite(pg_col3),
|
|
1533
|
+
np.int8(9),
|
|
1534
|
+
np.where(
|
|
1535
|
+
pg_col3 < prcnt_gd_bad,
|
|
1536
|
+
np.int8(4),
|
|
1537
|
+
np.where(pg_col3 < prcnt_gd_suspect, np.int8(3), np.int8(1)),
|
|
1538
|
+
),
|
|
1539
|
+
).astype(np.int8)
|
|
1540
|
+
n_bad_pg = int(np.sum(pg_flags == 4))
|
|
1541
|
+
n_susp_pg = int(np.sum(pg_flags == 3))
|
|
1542
|
+
if log_fn:
|
|
1543
|
+
log_fn(
|
|
1544
|
+
f" ADCP percent_good QC (col-3 4-beam): bad={n_bad_pg}, suspect={n_susp_pg} "
|
|
1545
|
+
f"(thresholds: bad<{prcnt_gd_bad}%, suspect<{prcnt_gd_suspect}%)"
|
|
1546
|
+
)
|
|
1547
|
+
from . import parameters as _P
|
|
1548
|
+
|
|
1549
|
+
ds["percent_good_qc"] = xr.Variable(
|
|
1550
|
+
("time", _P.ADCP_BIN_DIM),
|
|
1551
|
+
pg_flags,
|
|
1552
|
+
attrs={
|
|
1553
|
+
"long_name": "QC flag for ADCP percent good (4-beam solutions, column 3)",
|
|
1554
|
+
"comment": (
|
|
1555
|
+
f"Based on RDI percent_good column 3 (4-beam solutions). "
|
|
1556
|
+
f"bad<{prcnt_gd_bad}%, suspect<{prcnt_gd_suspect}%. "
|
|
1557
|
+
"Standalone — not merged into velocity_qc."
|
|
1558
|
+
),
|
|
1559
|
+
**qc_attrs,
|
|
1560
|
+
},
|
|
1561
|
+
)
|
|
1562
|
+
|
|
1563
|
+
# 3. error_velocity QC — standalone variable, NOT merged into velocity_qc.
|
|
1564
|
+
if "error_velocity" in ds.data_vars:
|
|
1565
|
+
ev = ds["error_velocity"].values.astype(float) # (time, N_BINS)
|
|
1566
|
+
ev_flags = np.where(
|
|
1567
|
+
~np.isfinite(ev),
|
|
1568
|
+
np.int8(9),
|
|
1569
|
+
np.where(np.abs(ev) > error_vel_threshold, np.int8(4), np.int8(1)),
|
|
1570
|
+
).astype(np.int8)
|
|
1571
|
+
n_ev_bad = int(np.sum(ev_flags == 4))
|
|
1572
|
+
if log_fn:
|
|
1573
|
+
log_fn(
|
|
1574
|
+
f" ADCP error_velocity QC: bad={n_ev_bad} "
|
|
1575
|
+
f"(threshold: |ev|>{error_vel_threshold:.2f} m s-1). Standalone."
|
|
1576
|
+
)
|
|
1577
|
+
ds["error_velocity_qc"] = xr.Variable(
|
|
1578
|
+
ds["error_velocity"].dims,
|
|
1579
|
+
ev_flags,
|
|
1580
|
+
attrs={
|
|
1581
|
+
"long_name": "QC flag for error velocity",
|
|
1582
|
+
"comment": (
|
|
1583
|
+
f"|error_velocity| > {error_vel_threshold:.2f} m s-1 → bad (4). "
|
|
1584
|
+
"Standalone — not merged into velocity_qc."
|
|
1585
|
+
),
|
|
1586
|
+
"qc_threshold_fail_m_s": float(error_vel_threshold),
|
|
1587
|
+
**qc_attrs,
|
|
1588
|
+
},
|
|
1589
|
+
)
|
|
1590
|
+
|
|
1591
|
+
return ds
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
def _derive_oxygen_saturation(ds: "xr.Dataset") -> "xr.Dataset":
|
|
1595
|
+
"""Compute O2 % saturation and AOU from dissolved_oxygen, temperature, salinity, pressure.
|
|
1596
|
+
|
|
1597
|
+
Requires dissolved_oxygen (µmol/L), temperature (°C), salinity (PSU), and pressure
|
|
1598
|
+
(dbar) all present in *ds*. Returns *ds* unchanged when any variable is missing or
|
|
1599
|
+
when gsw is unavailable.
|
|
1600
|
+
|
|
1601
|
+
Unit conversion uses in-situ **seawater** density from ``gsw.rho(SA, CT, p)``
|
|
1602
|
+
(kg m⁻³) — NOT freshwater density (~1000 kg m⁻³). Seawater density at typical
|
|
1603
|
+
mooring conditions is ~1025–1028 kg m⁻³, giving a ~2.5 % correction relative to
|
|
1604
|
+
freshwater. This correction is oceanographically significant and must not be skipped.
|
|
1605
|
+
|
|
1606
|
+
lon/lat for ``gsw.SA_from_SP`` are taken from global attrs (default 0.0 if absent;
|
|
1607
|
+
the SA error from a wrong position is typically < 0.05 g kg⁻¹ at open-ocean sites).
|
|
1608
|
+
|
|
1609
|
+
Derived variables stored
|
|
1610
|
+
------------------------
|
|
1611
|
+
oxygen_saturation_pct : %
|
|
1612
|
+
100 × O2_measured(µmol kg⁻¹) / O2sol(µmol kg⁻¹)
|
|
1613
|
+
where O2sol is from ``gsw.O2sol_SP_pt(SP, pt)``.
|
|
1614
|
+
apparent_oxygen_utilization : µmol kg⁻¹
|
|
1615
|
+
O2sol − O2_measured (positive = oxygen-depleted water).
|
|
1616
|
+
"""
|
|
1617
|
+
required = {"dissolved_oxygen", "temperature", "salinity", "pressure"}
|
|
1618
|
+
if not required.issubset(ds.data_vars):
|
|
1619
|
+
return ds
|
|
1620
|
+
try:
|
|
1621
|
+
import gsw
|
|
1622
|
+
except ImportError:
|
|
1623
|
+
return ds
|
|
1624
|
+
|
|
1625
|
+
O2_L = ds["dissolved_oxygen"].values.astype(float)
|
|
1626
|
+
t = ds["temperature"].values.astype(float)
|
|
1627
|
+
SP = ds["salinity"].values.astype(float)
|
|
1628
|
+
p = ds["pressure"].values.astype(float)
|
|
1629
|
+
try:
|
|
1630
|
+
lon = float(ds.attrs.get("longitude", 0.0))
|
|
1631
|
+
except (ValueError, TypeError):
|
|
1632
|
+
lon = 0.0
|
|
1633
|
+
try:
|
|
1634
|
+
lat = float(ds.attrs.get("latitude", 0.0))
|
|
1635
|
+
except (ValueError, TypeError):
|
|
1636
|
+
lat = 0.0
|
|
1637
|
+
|
|
1638
|
+
# Mask already-flagged dissolved oxygen so bad values produce NaN saturation
|
|
1639
|
+
# rather than spurious negative percentages that would then back-propagate.
|
|
1640
|
+
if "dissolved_oxygen_qc" in ds.data_vars:
|
|
1641
|
+
_do_qc = ds["dissolved_oxygen_qc"].values.astype(np.int8)
|
|
1642
|
+
O2_L = O2_L.copy()
|
|
1643
|
+
O2_L[np.isin(_do_qc, [4, 9])] = np.nan
|
|
1644
|
+
|
|
1645
|
+
SA = gsw.SA_from_SP(SP, p, lon, lat)
|
|
1646
|
+
CT = gsw.CT_from_t(SA, t, p)
|
|
1647
|
+
pt = gsw.pt0_from_t(SA, t, p)
|
|
1648
|
+
|
|
1649
|
+
O2sol = gsw.O2sol_SP_pt(SP, pt) # µmol/kg at equilibrium
|
|
1650
|
+
rho_sw = gsw.rho(SA, CT, p) # kg/m³ in-situ seawater density
|
|
1651
|
+
O2_kg = O2_L / (rho_sw / 1000.0) # µmol/kg measured (divides by kg/L)
|
|
1652
|
+
|
|
1653
|
+
pct_sat = 100.0 * O2_kg / O2sol
|
|
1654
|
+
aou = O2sol - O2_kg
|
|
1655
|
+
|
|
1656
|
+
# Back-flag: pct_sat < 0 is physically impossible (negative dissolved oxygen).
|
|
1657
|
+
# Set saturation to NaN and flag the underlying dissolved_oxygen as bad (4).
|
|
1658
|
+
neg_sat = np.isfinite(pct_sat) & (pct_sat < 0.0)
|
|
1659
|
+
n_neg = int(np.sum(neg_sat))
|
|
1660
|
+
if n_neg > 0:
|
|
1661
|
+
pct_sat[neg_sat] = np.nan
|
|
1662
|
+
aou[neg_sat] = np.nan
|
|
1663
|
+
if "dissolved_oxygen_qc" in ds.data_vars:
|
|
1664
|
+
do_qc = ds["dissolved_oxygen_qc"].values.astype(np.int8).copy()
|
|
1665
|
+
do_qc[neg_sat] = np.int8(4)
|
|
1666
|
+
ds["dissolved_oxygen_qc"] = xr.Variable(
|
|
1667
|
+
ds["dissolved_oxygen_qc"].dims,
|
|
1668
|
+
do_qc,
|
|
1669
|
+
attrs=dict(ds["dissolved_oxygen_qc"].attrs),
|
|
1670
|
+
)
|
|
1671
|
+
|
|
1672
|
+
ds = ds.assign(
|
|
1673
|
+
oxygen_saturation_pct=xr.Variable(
|
|
1674
|
+
"time",
|
|
1675
|
+
pct_sat.astype(np.float32),
|
|
1676
|
+
{
|
|
1677
|
+
"long_name": "Dissolved Oxygen Percent Saturation",
|
|
1678
|
+
"units": "%",
|
|
1679
|
+
"comment": (
|
|
1680
|
+
"100 × O2_meas / O2sol. O2_meas converted from µmol/L to µmol/kg "
|
|
1681
|
+
"using in-situ seawater density gsw.rho(SA,CT,p); "
|
|
1682
|
+
"O2sol = gsw.O2sol_SP_pt(SP, pt0). "
|
|
1683
|
+
f"Back-flagged {n_neg} sample(s) where pct_sat < 0 as bad (flag 4)."
|
|
1684
|
+
if n_neg
|
|
1685
|
+
else (
|
|
1686
|
+
"100 × O2_meas / O2sol. O2_meas converted from µmol/L to µmol/kg "
|
|
1687
|
+
"using in-situ seawater density gsw.rho(SA,CT,p); "
|
|
1688
|
+
"O2sol = gsw.O2sol_SP_pt(SP, pt0)."
|
|
1689
|
+
)
|
|
1690
|
+
),
|
|
1691
|
+
},
|
|
1692
|
+
),
|
|
1693
|
+
apparent_oxygen_utilization=xr.Variable(
|
|
1694
|
+
"time",
|
|
1695
|
+
aou.astype(np.float32),
|
|
1696
|
+
{
|
|
1697
|
+
"long_name": "Apparent Oxygen Utilization",
|
|
1698
|
+
"standard_name": "apparent_oxygen_utilization",
|
|
1699
|
+
"units": "umol/kg",
|
|
1700
|
+
"comment": (
|
|
1701
|
+
"AOU = O2sol - O2_meas (µmol/kg); positive = undersaturated "
|
|
1702
|
+
"(oxygen consumed). Uses in-situ seawater density for µmol/L → µmol/kg."
|
|
1703
|
+
),
|
|
1704
|
+
},
|
|
1705
|
+
),
|
|
1706
|
+
)
|
|
1707
|
+
return ds
|
|
1708
|
+
|
|
1709
|
+
|
|
1710
|
+
class Stage3Processor:
|
|
1711
|
+
"""Pressure interpolation + QARTOD QC for all mooring instruments."""
|
|
1712
|
+
|
|
1713
|
+
def __init__(
|
|
1714
|
+
self,
|
|
1715
|
+
base_dir: Optional[str] = None,
|
|
1716
|
+
*,
|
|
1717
|
+
proc_dir: Optional[str] = None,
|
|
1718
|
+
) -> None:
|
|
1719
|
+
"""Apply QC flags, coordinate rotation, and derived variables to mooring data.
|
|
1720
|
+
|
|
1721
|
+
Reads stage2 CF-NetCDF files and applies: QARTOD gross-range and spike QC
|
|
1722
|
+
flags for scalar sensors; pressure interpolation for instruments that lack a
|
|
1723
|
+
pressure sensor; BEAM→XYZ→ENU coordinate rotation for Aquadopp current meters
|
|
1724
|
+
(using the instrument T matrix and heading/pitch/roll); and magnetic declination
|
|
1725
|
+
correction so velocities are referenced to true north. Writes
|
|
1726
|
+
``{mooring}_{serial}_stage3.nc``.
|
|
1727
|
+
|
|
1728
|
+
Parameters
|
|
1729
|
+
----------
|
|
1730
|
+
base_dir : str, optional
|
|
1731
|
+
Legacy: cruise-level base directory containing a ``proc/`` subdirectory.
|
|
1732
|
+
proc_dir : str, optional
|
|
1733
|
+
Cruise-level processed output directory. Pipeline appends ``/{mooring}/``.
|
|
1734
|
+
|
|
1735
|
+
"""
|
|
1736
|
+
if base_dir is not None:
|
|
1737
|
+
self.base_dir: Optional[Path] = Path(base_dir)
|
|
1738
|
+
self._proc_dir: Optional[Path] = None
|
|
1739
|
+
self._legacy = True
|
|
1740
|
+
else:
|
|
1741
|
+
self.base_dir = None
|
|
1742
|
+
self._proc_dir = Path(proc_dir) if proc_dir else None
|
|
1743
|
+
self._legacy = False
|
|
1744
|
+
self.log_file = None
|
|
1745
|
+
|
|
1746
|
+
def _setup_logging(self, mooring_name: str, output_path: Path) -> None:
|
|
1747
|
+
from .logger import setup_stage_logging
|
|
1748
|
+
|
|
1749
|
+
self.log_file = setup_stage_logging(mooring_name, "stage3", output_path)
|
|
1750
|
+
|
|
1751
|
+
def _log(self, *args: Any, **kwargs: Any) -> None:
|
|
1752
|
+
print(*args, **kwargs)
|
|
1753
|
+
if self.log_file:
|
|
1754
|
+
try:
|
|
1755
|
+
with open(self.log_file, "a") as f:
|
|
1756
|
+
print(*args, **kwargs, file=f)
|
|
1757
|
+
except OSError:
|
|
1758
|
+
pass
|
|
1759
|
+
|
|
1760
|
+
def _get_proc_dir(self, mooring_name: str) -> Path:
|
|
1761
|
+
if not self._legacy and self._proc_dir is not None:
|
|
1762
|
+
return self._proc_dir / mooring_name
|
|
1763
|
+
proc = self.base_dir / "proc"
|
|
1764
|
+
if not proc.is_dir():
|
|
1765
|
+
legacy = self.base_dir / "moor" / "proc"
|
|
1766
|
+
proc = legacy if legacy.is_dir() else proc
|
|
1767
|
+
return proc / mooring_name
|
|
1768
|
+
|
|
1769
|
+
# ------------------------------------------------------------------
|
|
1770
|
+
def process_mooring(
|
|
1771
|
+
self,
|
|
1772
|
+
mooring_name: str,
|
|
1773
|
+
serials: Optional[List[str]] = None,
|
|
1774
|
+
force: bool = False,
|
|
1775
|
+
dry_run: bool = False,
|
|
1776
|
+
) -> bool:
|
|
1777
|
+
"""Run Stage 3 QC and pressure interpolation for all instruments on a mooring."""
|
|
1778
|
+
proc_dir = self._get_proc_dir(mooring_name)
|
|
1779
|
+
if not proc_dir.exists():
|
|
1780
|
+
print(f"ERROR: Processing directory not found: {proc_dir}")
|
|
1781
|
+
return False
|
|
1782
|
+
|
|
1783
|
+
self._setup_logging(mooring_name, proc_dir)
|
|
1784
|
+
mode = "DRY RUN — " if dry_run else ""
|
|
1785
|
+
self._log(f"{mode}Stage 3 (pressure interpolation + QC) for: {mooring_name}")
|
|
1786
|
+
|
|
1787
|
+
config_file = proc_dir / f"{mooring_name}.mooring.yaml"
|
|
1788
|
+
if not config_file.exists():
|
|
1789
|
+
self._log(f"ERROR: Config not found: {config_file}")
|
|
1790
|
+
return False
|
|
1791
|
+
|
|
1792
|
+
with open(config_file) as f:
|
|
1793
|
+
mooring_config = yaml.safe_load(f)
|
|
1794
|
+
|
|
1795
|
+
instrument_list = list(
|
|
1796
|
+
mooring_config.get("clamp", mooring_config.get("instruments", []))
|
|
1797
|
+
)
|
|
1798
|
+
instrument_list += extract_inline_instruments(mooring_config.get("inline", []))
|
|
1799
|
+
|
|
1800
|
+
# ── Mooring location for BEAM→ENU declination ──────────────────
|
|
1801
|
+
from .mooring_level import _parse_latlon_with_source
|
|
1802
|
+
|
|
1803
|
+
_mooring_lat, _mooring_lon, _latlon_source = _parse_latlon_with_source(
|
|
1804
|
+
mooring_config
|
|
1805
|
+
)
|
|
1806
|
+
|
|
1807
|
+
_water_depth_m = float(mooring_config.get("waterdepth") or 0.0)
|
|
1808
|
+
|
|
1809
|
+
# ── Build instrument table ──────────────────────────────────────
|
|
1810
|
+
instruments: List[Dict[str, Any]] = []
|
|
1811
|
+
for entry in instrument_list:
|
|
1812
|
+
if not isinstance(entry, dict):
|
|
1813
|
+
continue
|
|
1814
|
+
serial = _safe_serial(entry.get("serial", ""))
|
|
1815
|
+
instr_type = entry.get("instrument", "unknown")
|
|
1816
|
+
if entry.get("skip"):
|
|
1817
|
+
reason = entry.get("skip_reason", "marked skip:true in YAML")
|
|
1818
|
+
self._log(f" SKIP {instr_type} {serial}: {reason}")
|
|
1819
|
+
continue
|
|
1820
|
+
hab = entry.get("hab")
|
|
1821
|
+
if hab is None:
|
|
1822
|
+
self._log(f" WARNING: serial {serial} has no 'hab' — skipping")
|
|
1823
|
+
continue
|
|
1824
|
+
nc_path = proc_dir / instr_type / f"{mooring_name}_{serial}_stage2.nc"
|
|
1825
|
+
if not nc_path.exists():
|
|
1826
|
+
continue
|
|
1827
|
+
qc_flags = {
|
|
1828
|
+
key[:-3]: int(val)
|
|
1829
|
+
for key, val in entry.items()
|
|
1830
|
+
if key.endswith("_qc") and isinstance(val, int)
|
|
1831
|
+
}
|
|
1832
|
+
gr_cfg, sp_cfg, tilt_cfg, fl_cfg = _load_qc_config(mooring_config, entry)
|
|
1833
|
+
# Parse optional hab_segments: list of {from: ISO, hab: float}
|
|
1834
|
+
# Stored as sorted list of (np.datetime64, float) breakpoints.
|
|
1835
|
+
raw_segs = entry.get("hab_segments", [])
|
|
1836
|
+
hab_segments: List[tuple] = []
|
|
1837
|
+
for seg in raw_segs:
|
|
1838
|
+
try:
|
|
1839
|
+
bp = np.datetime64(seg["from"], "ns")
|
|
1840
|
+
hab_segments.append((bp, float(seg["hab"])))
|
|
1841
|
+
except (KeyError, ValueError) as _e:
|
|
1842
|
+
self._log(
|
|
1843
|
+
f" WARNING: serial {serial} hab_segments entry invalid"
|
|
1844
|
+
f" ({seg}): {_e} — skipped"
|
|
1845
|
+
)
|
|
1846
|
+
hab_segments.sort(key=lambda x: x[0])
|
|
1847
|
+
|
|
1848
|
+
instruments.append(
|
|
1849
|
+
{
|
|
1850
|
+
"serial": serial,
|
|
1851
|
+
"instrument": instr_type,
|
|
1852
|
+
"hab": float(hab),
|
|
1853
|
+
"hab_segments": hab_segments,
|
|
1854
|
+
"nc_path": nc_path,
|
|
1855
|
+
"qc_flags": qc_flags,
|
|
1856
|
+
"gross_range": gr_cfg,
|
|
1857
|
+
"spike": sp_cfg,
|
|
1858
|
+
"tilt": tilt_cfg,
|
|
1859
|
+
"flat_line": fl_cfg,
|
|
1860
|
+
"entry": entry,
|
|
1861
|
+
"lat": _mooring_lat,
|
|
1862
|
+
"lon": _mooring_lon,
|
|
1863
|
+
"latlon_source": _latlon_source,
|
|
1864
|
+
"water_depth_m": _water_depth_m,
|
|
1865
|
+
}
|
|
1866
|
+
)
|
|
1867
|
+
|
|
1868
|
+
if not instruments:
|
|
1869
|
+
self._log("ERROR: No _stage2.nc files found with hab values")
|
|
1870
|
+
return False
|
|
1871
|
+
|
|
1872
|
+
# Filter by serial early for targeted reruns
|
|
1873
|
+
if serials:
|
|
1874
|
+
safe_req = {_safe_serial(s) for s in serials}
|
|
1875
|
+
instruments = [i for i in instruments if i["serial"] in safe_req]
|
|
1876
|
+
if not instruments:
|
|
1877
|
+
self._log(
|
|
1878
|
+
f" No instruments found matching serial(s): {', '.join(serials)}"
|
|
1879
|
+
)
|
|
1880
|
+
return True
|
|
1881
|
+
|
|
1882
|
+
# ── Scan what pressure variables each instrument has ────────────
|
|
1883
|
+
def _find_pressure_var(data_vars: set) -> Optional[str]:
|
|
1884
|
+
if "pressure" in data_vars:
|
|
1885
|
+
return "pressure"
|
|
1886
|
+
for cand in sorted(
|
|
1887
|
+
v
|
|
1888
|
+
for v in data_vars
|
|
1889
|
+
if v.startswith("pressure_") and not v.endswith("_qc")
|
|
1890
|
+
):
|
|
1891
|
+
return cand
|
|
1892
|
+
return None
|
|
1893
|
+
|
|
1894
|
+
for info in instruments:
|
|
1895
|
+
try:
|
|
1896
|
+
with xr.open_dataset(info["nc_path"], decode_timedelta=False) as _ds:
|
|
1897
|
+
info["data_vars"] = set(_ds.data_vars)
|
|
1898
|
+
except Exception as e: # noqa: BLE001 — missing file skipped; mooring continues
|
|
1899
|
+
self._log(f" WARNING: Could not open {info['nc_path'].name}: {e}")
|
|
1900
|
+
info["data_vars"] = set()
|
|
1901
|
+
info["pressure_var"] = _find_pressure_var(info["data_vars"])
|
|
1902
|
+
info["has_pressure"] = info["pressure_var"] is not None
|
|
1903
|
+
info["ds"] = None
|
|
1904
|
+
|
|
1905
|
+
for varname, qc_val in info["qc_flags"].items():
|
|
1906
|
+
if varname not in info["data_vars"]:
|
|
1907
|
+
self._log(
|
|
1908
|
+
f" WARNING: serial {info['serial']}: "
|
|
1909
|
+
f"'{varname}_qc: {qc_val}' in YAML but '{varname}' "
|
|
1910
|
+
f"not found in {info['nc_path'].name} — ignored"
|
|
1911
|
+
)
|
|
1912
|
+
|
|
1913
|
+
def pressure_bad(info: Dict[str, Any]) -> bool:
|
|
1914
|
+
return info["qc_flags"].get("pressure", 0) >= 3
|
|
1915
|
+
|
|
1916
|
+
sources = [i for i in instruments if i["has_pressure"] and not pressure_bad(i)]
|
|
1917
|
+
targets = [i for i in instruments if not i["has_pressure"] or pressure_bad(i)]
|
|
1918
|
+
|
|
1919
|
+
if not sources and targets:
|
|
1920
|
+
self._log(
|
|
1921
|
+
"WARNING: No reliable pressure sources found — pressure interpolation skipped"
|
|
1922
|
+
)
|
|
1923
|
+
|
|
1924
|
+
self._log(
|
|
1925
|
+
f" Pressure sources ({len(sources)}): "
|
|
1926
|
+
+ ", ".join(
|
|
1927
|
+
f"{s['instrument']} {s['serial']} hab={s['hab']:.1f}m" for s in sources
|
|
1928
|
+
)
|
|
1929
|
+
)
|
|
1930
|
+
if targets:
|
|
1931
|
+
self._log(
|
|
1932
|
+
f" Pressure targets ({len(targets)}): "
|
|
1933
|
+
+ ", ".join(
|
|
1934
|
+
f"{t['instrument']} {t['serial']} hab={t['hab']:.1f}m"
|
|
1935
|
+
+ (
|
|
1936
|
+
f" [pressure_qc={t['qc_flags']['pressure']}→replace]"
|
|
1937
|
+
if pressure_bad(t)
|
|
1938
|
+
else " [no pressure]"
|
|
1939
|
+
)
|
|
1940
|
+
for t in targets
|
|
1941
|
+
)
|
|
1942
|
+
)
|
|
1943
|
+
|
|
1944
|
+
if dry_run:
|
|
1945
|
+
for info in instruments:
|
|
1946
|
+
is_target = info in targets
|
|
1947
|
+
l3 = info["nc_path"].with_name(
|
|
1948
|
+
info["nc_path"].name.replace("_stage2.nc", "_stage3.nc")
|
|
1949
|
+
)
|
|
1950
|
+
action = "interpolate pressure + QC" if is_target else "QC only"
|
|
1951
|
+
self._log(
|
|
1952
|
+
f" {info['serial']} ({info['instrument']}): {action} → {l3.name}"
|
|
1953
|
+
)
|
|
1954
|
+
self._log("DRY RUN complete")
|
|
1955
|
+
return True
|
|
1956
|
+
|
|
1957
|
+
# ── Load source datasets for pressure interpolation ─────────────
|
|
1958
|
+
for src in sources:
|
|
1959
|
+
try:
|
|
1960
|
+
src["ds"] = xr.open_dataset(
|
|
1961
|
+
src["nc_path"], decode_timedelta=False
|
|
1962
|
+
).load()
|
|
1963
|
+
except Exception as e: # noqa: BLE001 — one bad source must not abort pressure interp
|
|
1964
|
+
self._log(
|
|
1965
|
+
f" WARNING: Could not load source {src['nc_path'].name}: {e}"
|
|
1966
|
+
)
|
|
1967
|
+
src["ds"] = None
|
|
1968
|
+
|
|
1969
|
+
# ── Process every instrument ────────────────────────────────────
|
|
1970
|
+
from . import parameters as P
|
|
1971
|
+
|
|
1972
|
+
_qc_attrs = {
|
|
1973
|
+
"flag_values": P.QC_FLAG_VALUES,
|
|
1974
|
+
"flag_meanings": P.QC_FLAG_MEANINGS,
|
|
1975
|
+
"conventions": P.QC_CONVENTION,
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
success_count = 0
|
|
1979
|
+
for info in instruments:
|
|
1980
|
+
ok = self._process_instrument(
|
|
1981
|
+
info, sources, targets, _qc_attrs, force=force
|
|
1982
|
+
)
|
|
1983
|
+
if ok:
|
|
1984
|
+
success_count += 1
|
|
1985
|
+
|
|
1986
|
+
for info in instruments:
|
|
1987
|
+
if info.get("ds") is not None:
|
|
1988
|
+
info["ds"].close()
|
|
1989
|
+
|
|
1990
|
+
self._log(
|
|
1991
|
+
f"Stage 3 complete: {success_count}/{len(instruments)} instruments written"
|
|
1992
|
+
)
|
|
1993
|
+
return success_count == len(instruments)
|
|
1994
|
+
|
|
1995
|
+
# ------------------------------------------------------------------
|
|
1996
|
+
def _process_instrument(
|
|
1997
|
+
self,
|
|
1998
|
+
info: Dict[str, Any],
|
|
1999
|
+
sources: List[Dict[str, Any]],
|
|
2000
|
+
targets: List[Dict[str, Any]],
|
|
2001
|
+
qc_attrs: Dict[str, Any],
|
|
2002
|
+
force: bool = False,
|
|
2003
|
+
) -> bool:
|
|
2004
|
+
"""Apply Stage 3 processing to one instrument's Stage 2 NetCDF.
|
|
2005
|
+
|
|
2006
|
+
Steps applied (where applicable to the instrument type):
|
|
2007
|
+
|
|
2008
|
+
1. Pressure interpolation — targets without a reliable pressure sensor
|
|
2009
|
+
have pressure interpolated from *sources* (neighbours on the mooring).
|
|
2010
|
+
2. Conductivity unit normalisation and practical salinity computation
|
|
2011
|
+
(CTD/microcat instruments only).
|
|
2012
|
+
3. QARTOD QC tests (gross-range, spike) using thresholds from *qc_attrs*.
|
|
2013
|
+
4. BEAM→ENU or XYZ→ENU coordinate transformation with magnetic declination
|
|
2014
|
+
correction (current meters / Aquadopp).
|
|
2015
|
+
5. Tilt QC — velocity flagged suspect/bad when pitch or roll exceed
|
|
2016
|
+
configured thresholds.
|
|
2017
|
+
6. History attribute updated with all processing steps applied.
|
|
2018
|
+
|
|
2019
|
+
Writes ``{mooring}_{serial}_stage3.nc`` alongside the Stage 2 file.
|
|
2020
|
+
Skips if the output already exists and *force* is False.
|
|
2021
|
+
|
|
2022
|
+
Returns True on success or skip, False on error.
|
|
2023
|
+
"""
|
|
2024
|
+
nc_path = info["nc_path"]
|
|
2025
|
+
serial = info["serial"]
|
|
2026
|
+
l3_path = nc_path.with_name(nc_path.name.replace("_stage2.nc", "_stage3.nc"))
|
|
2027
|
+
|
|
2028
|
+
if l3_path.exists():
|
|
2029
|
+
if not force:
|
|
2030
|
+
self._log(f" SKIP (exists): {l3_path.name} (--force to overwrite)")
|
|
2031
|
+
return True
|
|
2032
|
+
try:
|
|
2033
|
+
l3_path.unlink()
|
|
2034
|
+
except OSError as e:
|
|
2035
|
+
self._log(f" ERROR: cannot remove existing {l3_path.name}: {e}")
|
|
2036
|
+
return False
|
|
2037
|
+
|
|
2038
|
+
is_target = info in targets
|
|
2039
|
+
pressure_bad_flag = info["qc_flags"].get("pressure", 0) >= 3
|
|
2040
|
+
is_adcp = info.get("instrument", "").lower() == "adcp"
|
|
2041
|
+
|
|
2042
|
+
self._log(
|
|
2043
|
+
f"--> Processing {info.get('instrument', 'unknown')}: {nc_path.name}"
|
|
2044
|
+
)
|
|
2045
|
+
|
|
2046
|
+
try:
|
|
2047
|
+
from . import parameters as P
|
|
2048
|
+
|
|
2049
|
+
ds = xr.open_dataset(nc_path, decode_timedelta=False).load()
|
|
2050
|
+
|
|
2051
|
+
# Record stage2 source-file provenance as named global attrs
|
|
2052
|
+
_st2_stat = nc_path.stat()
|
|
2053
|
+
_st2_mtime = datetime.datetime.utcfromtimestamp(
|
|
2054
|
+
_st2_stat.st_mtime
|
|
2055
|
+
).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
2056
|
+
with open(nc_path, "rb") as _f:
|
|
2057
|
+
_st2_sha = hashlib.sha256(_f.read()).hexdigest()[:8]
|
|
2058
|
+
ds.attrs["stage2_source_file"] = nc_path.name
|
|
2059
|
+
ds.attrs["stage2_source_mtime"] = _st2_mtime
|
|
2060
|
+
ds.attrs["stage2_source_sha256"] = _st2_sha
|
|
2061
|
+
|
|
2062
|
+
target_time = ds["time"].values
|
|
2063
|
+
history_notes = []
|
|
2064
|
+
|
|
2065
|
+
# ── Pressure interpolation (targets only) ──────────────────
|
|
2066
|
+
if is_target and sources:
|
|
2067
|
+
ds, method = self._interpolate_pressure(
|
|
2068
|
+
ds, info, sources, target_time, pressure_bad_flag, qc_attrs
|
|
2069
|
+
)
|
|
2070
|
+
history_notes.append(
|
|
2071
|
+
f"stage3 pressure interpolated (pressure_qc=8) — {method}"
|
|
2072
|
+
)
|
|
2073
|
+
|
|
2074
|
+
# ── Ensure conductivity is in mS/cm before QC ─────────────
|
|
2075
|
+
ds = _ensure_conductivity_units(ds, log_fn=self._log)
|
|
2076
|
+
|
|
2077
|
+
# ── Compute salinity data before QC so it gets range-tested ─
|
|
2078
|
+
ds = _compute_salinity_data(ds, log_fn=self._log)
|
|
2079
|
+
|
|
2080
|
+
# ── QARTOD QC tests (temperature, conductivity, salinity, …) ─
|
|
2081
|
+
gr_cfg = info["gross_range"]
|
|
2082
|
+
sp_cfg = info["spike"]
|
|
2083
|
+
fl_cfg = info["flat_line"]
|
|
2084
|
+
# For ADCP, exclude 2-D velocity vars from the ioos_qc path
|
|
2085
|
+
# (gross_range_test expects 1-D input); velocity QC is handled
|
|
2086
|
+
# below by _apply_adcp_velocity_qc.
|
|
2087
|
+
if is_adcp:
|
|
2088
|
+
_ENU_KEYS = {"east_velocity", "north_velocity", "up_velocity"}
|
|
2089
|
+
gr_cfg_scalar = {k: v for k, v in gr_cfg.items() if k not in _ENU_KEYS}
|
|
2090
|
+
ds = _apply_qc_tests(
|
|
2091
|
+
ds, gr_cfg_scalar, sp_cfg, qc_attrs, flat_line=fl_cfg
|
|
2092
|
+
)
|
|
2093
|
+
else:
|
|
2094
|
+
ds = _apply_qc_tests(ds, gr_cfg, sp_cfg, qc_attrs, flat_line=fl_cfg)
|
|
2095
|
+
|
|
2096
|
+
# ── Post-QC pressure check ─────────────────────────────────
|
|
2097
|
+
# If QC (e.g. flat-line test) just flagged ALL pressure values as
|
|
2098
|
+
# bad and this instrument was not already a pressure target, treat
|
|
2099
|
+
# it as one now: interpolate from the other sources on the mooring.
|
|
2100
|
+
# This catches stuck-sensor faults that only become visible after
|
|
2101
|
+
# the flat-line / gross-range tests run — which is why the pre-scan
|
|
2102
|
+
# source/target classification cannot catch them.
|
|
2103
|
+
# Note: if this instrument was already used as a source for a
|
|
2104
|
+
# previously processed target, those targets will have received bad
|
|
2105
|
+
# pressure values. To fix that, re-run stage 3 on the full mooring
|
|
2106
|
+
# once this instrument's fault is known (e.g. add pressure_qc: 4 to
|
|
2107
|
+
# the YAML so it is excluded from sources at the outset).
|
|
2108
|
+
if (
|
|
2109
|
+
not is_target
|
|
2110
|
+
and "pressure_qc" in ds.data_vars
|
|
2111
|
+
and "pressure" in ds.data_vars
|
|
2112
|
+
):
|
|
2113
|
+
_pqc = ds["pressure_qc"].values.astype(np.int8)
|
|
2114
|
+
_n_total = len(_pqc)
|
|
2115
|
+
_n_bad = int(np.sum(np.isin(_pqc, [4, 9])))
|
|
2116
|
+
_frac_bad = _n_bad / _n_total if _n_total > 0 else 0.0
|
|
2117
|
+
# Trigger interpolation when > 70% of values are bad. The flat-line
|
|
2118
|
+
# test leaves the first fail_n samples as flag 1/3 (window startup),
|
|
2119
|
+
# so strict all-bad checking would never fire for a fully stuck sensor.
|
|
2120
|
+
if _frac_bad > 0.70:
|
|
2121
|
+
_other_sources = [s for s in sources if s["serial"] != serial]
|
|
2122
|
+
if _other_sources:
|
|
2123
|
+
self._log(
|
|
2124
|
+
f" WARNING: {serial} — {_frac_bad:.0%} of pressure values "
|
|
2125
|
+
f"flagged bad by QC (flat-line or gross-range); replacing "
|
|
2126
|
+
f"entire pressure record with interpolation from "
|
|
2127
|
+
f"{len(_other_sources)} source(s). "
|
|
2128
|
+
f"Re-run stage 3 on the full mooring with "
|
|
2129
|
+
f"'pressure_qc: 4' in the YAML entry for {serial} "
|
|
2130
|
+
f"so it is excluded from sources from the start."
|
|
2131
|
+
)
|
|
2132
|
+
# Temporarily set YAML pressure flag to 4 so that
|
|
2133
|
+
# _interpolate_pressure stores pressure_orig_qc = 4
|
|
2134
|
+
# (the actual QC reason), not the YAML default of 0.
|
|
2135
|
+
info["qc_flags"]["pressure"] = 4
|
|
2136
|
+
ds, method = self._interpolate_pressure(
|
|
2137
|
+
ds,
|
|
2138
|
+
info,
|
|
2139
|
+
_other_sources,
|
|
2140
|
+
target_time,
|
|
2141
|
+
pressure_bad_flag=True,
|
|
2142
|
+
qc_attrs=qc_attrs,
|
|
2143
|
+
)
|
|
2144
|
+
history_notes.append(
|
|
2145
|
+
f"stage3 pressure interpolated post-QC "
|
|
2146
|
+
f"({_frac_bad:.0%} pressure_qc=4/9 after QC) — {method}"
|
|
2147
|
+
)
|
|
2148
|
+
else:
|
|
2149
|
+
self._log(
|
|
2150
|
+
f" WARNING: {serial} — {_frac_bad:.0%} of pressure flagged "
|
|
2151
|
+
f"bad but no other pressure sources on this mooring; "
|
|
2152
|
+
f"pressure remains bad-flagged."
|
|
2153
|
+
)
|
|
2154
|
+
|
|
2155
|
+
# ── Fold T/C/P parent QC into salinity_qc ─────────────────
|
|
2156
|
+
ds = _merge_salinity_parent_qc(ds, qc_attrs)
|
|
2157
|
+
|
|
2158
|
+
# ── BEAM / XYZ → ENU coordinate transform ─────────────────
|
|
2159
|
+
# Must run before tilt QC so east/north/up_velocity exist to be flagged.
|
|
2160
|
+
coord_sys_before = ds.attrs.get("coordinate_system", "ENU")
|
|
2161
|
+
ds = _apply_beam_to_enu(
|
|
2162
|
+
ds,
|
|
2163
|
+
info["entry"],
|
|
2164
|
+
info["lat"],
|
|
2165
|
+
info["lon"],
|
|
2166
|
+
latlon_source=info.get("latlon_source", "unknown"),
|
|
2167
|
+
log_fn=self._log,
|
|
2168
|
+
)
|
|
2169
|
+
# If instrument was already in ENU (e.g. configured internally),
|
|
2170
|
+
# still apply the magnetic declination rotation.
|
|
2171
|
+
if coord_sys_before == "ENU":
|
|
2172
|
+
ds = _apply_declination_to_enu(
|
|
2173
|
+
ds,
|
|
2174
|
+
info["lat"],
|
|
2175
|
+
info["lon"],
|
|
2176
|
+
latlon_source=info.get("latlon_source", "unknown"),
|
|
2177
|
+
log_fn=self._log,
|
|
2178
|
+
)
|
|
2179
|
+
|
|
2180
|
+
# ── ENU velocity QC + up→east/north flag propagation ──────
|
|
2181
|
+
if ds.attrs.get("coordinate_system") == "ENU":
|
|
2182
|
+
if is_adcp:
|
|
2183
|
+
adcp_qc = P.QC_ADCP
|
|
2184
|
+
ds = _apply_adcp_velocity_qc(
|
|
2185
|
+
ds,
|
|
2186
|
+
gr_cfg=gr_cfg,
|
|
2187
|
+
prcnt_gd_bad=adcp_qc["percent_good_bad"],
|
|
2188
|
+
prcnt_gd_suspect=adcp_qc["percent_good_suspect"],
|
|
2189
|
+
error_vel_threshold=adcp_qc["error_velocity_threshold"],
|
|
2190
|
+
qc_attrs=qc_attrs,
|
|
2191
|
+
log_fn=self._log,
|
|
2192
|
+
)
|
|
2193
|
+
ds = _compute_adcp_bin_pressure(ds, info["lat"], log_fn=self._log)
|
|
2194
|
+
ds = _apply_adcp_seabed_qc(
|
|
2195
|
+
ds,
|
|
2196
|
+
water_depth_m=info.get("water_depth_m", 0.0),
|
|
2197
|
+
lat=info["lat"],
|
|
2198
|
+
qc_attrs=qc_attrs,
|
|
2199
|
+
log_fn=self._log,
|
|
2200
|
+
)
|
|
2201
|
+
ds = _apply_adcp_surface_qc(
|
|
2202
|
+
ds,
|
|
2203
|
+
lat=info["lat"],
|
|
2204
|
+
qc_attrs=qc_attrs,
|
|
2205
|
+
log_fn=self._log,
|
|
2206
|
+
)
|
|
2207
|
+
else:
|
|
2208
|
+
ds = _apply_enu_velocity_qc(ds, gr_cfg, qc_attrs)
|
|
2209
|
+
|
|
2210
|
+
# ── Tilt QC ────────────────────────────────────────────────
|
|
2211
|
+
# Flags all velocity variables when pitch+roll tilt exceeds threshold.
|
|
2212
|
+
# Skipped for ADCP: percent_good (col 3) and error_velocity capture
|
|
2213
|
+
# ensemble-level quality without a separate tilt threshold step.
|
|
2214
|
+
# Note: this does NOT mean tilt is checked per-bin — it is not.
|
|
2215
|
+
tilt_cfg = info["tilt"]
|
|
2216
|
+
if is_adcp:
|
|
2217
|
+
n_tilt_susp, n_tilt_bad = 0, 0
|
|
2218
|
+
history_notes.append(
|
|
2219
|
+
"tilt QC skipped for ADCP (percent_good+error_velocity QC applied instead)"
|
|
2220
|
+
)
|
|
2221
|
+
else:
|
|
2222
|
+
ds, n_tilt_susp, n_tilt_bad = _apply_tilt_qc(ds, tilt_cfg, qc_attrs)
|
|
2223
|
+
if n_tilt_susp or n_tilt_bad:
|
|
2224
|
+
history_notes.append(
|
|
2225
|
+
f"tilt QC (tilt≥{tilt_cfg['suspect_threshold']}°→suspect, "
|
|
2226
|
+
f"tilt≥{tilt_cfg['fail_threshold']}°→bad): "
|
|
2227
|
+
f"suspect={n_tilt_susp}, bad={n_tilt_bad}"
|
|
2228
|
+
)
|
|
2229
|
+
|
|
2230
|
+
# ── Unify ENU velocity QC ──────────────────────────────────
|
|
2231
|
+
# After tilt + gross-range QC, combine worst flag across all three
|
|
2232
|
+
# ENU components and write it back to all three so masking on any
|
|
2233
|
+
# single component (east, north, or up) gives an identical result.
|
|
2234
|
+
# This catches e.g. large east/north currents (>3 m/s suspect) that
|
|
2235
|
+
# would otherwise leave up_velocity_qc unflagged.
|
|
2236
|
+
if ds.attrs.get("coordinate_system") == "ENU":
|
|
2237
|
+
ds = _unify_velocity_qc(ds)
|
|
2238
|
+
|
|
2239
|
+
if ds.attrs.get("coordinate_system") == "ENU" and coord_sys_before != "ENU":
|
|
2240
|
+
_ba = ds.attrs.get("nortek_beam_angle", "?")
|
|
2241
|
+
_ba_src = ds.attrs.get("nortek_beam_angle_source", "")
|
|
2242
|
+
_assumed = "ASSUMED DEFAULT" in _ba_src
|
|
2243
|
+
history_notes.append(
|
|
2244
|
+
f"BEAM→ENU rotation applied: "
|
|
2245
|
+
f"beam_angle={_ba}° "
|
|
2246
|
+
f"({'ASSUMED DEFAULT — not from datasheet' if _assumed else 'from YAML'}), "
|
|
2247
|
+
f"declination={ds.attrs.get('magnetic_declination', 0.0):.2f}° "
|
|
2248
|
+
f"({'ppigrf IGRF' if 'magnetic_declination' in ds.attrs else 'assumed 0'})"
|
|
2249
|
+
)
|
|
2250
|
+
|
|
2251
|
+
# Report which variables got QC flags and their flag counts
|
|
2252
|
+
qc_summary = []
|
|
2253
|
+
for v in sorted(ds.data_vars):
|
|
2254
|
+
if v.endswith("_qc") and not v.endswith("_orig_qc"):
|
|
2255
|
+
counts = np.bincount(
|
|
2256
|
+
ds[v].values.astype(np.int8).clip(0, 9).ravel(), minlength=10
|
|
2257
|
+
)
|
|
2258
|
+
n_good = int(counts[1])
|
|
2259
|
+
n_susp = int(counts[3])
|
|
2260
|
+
n_bad = int(counts[4])
|
|
2261
|
+
n_interp = int(counts[8])
|
|
2262
|
+
n_miss = int(counts[9])
|
|
2263
|
+
parts = [f"good={n_good}"]
|
|
2264
|
+
if n_susp:
|
|
2265
|
+
parts.append(f"suspect={n_susp}")
|
|
2266
|
+
if n_bad:
|
|
2267
|
+
parts.append(f"bad={n_bad}")
|
|
2268
|
+
if n_interp:
|
|
2269
|
+
parts.append(f"interp={n_interp}")
|
|
2270
|
+
if n_miss:
|
|
2271
|
+
parts.append(f"missing={n_miss}")
|
|
2272
|
+
qc_summary.append(f"{v}=[{', '.join(parts)}]")
|
|
2273
|
+
if qc_summary:
|
|
2274
|
+
history_notes.append(
|
|
2275
|
+
"QARTOD gross-range+spike: " + "; ".join(qc_summary)
|
|
2276
|
+
)
|
|
2277
|
+
|
|
2278
|
+
# ── History ────────────────────────────────────────────────
|
|
2279
|
+
stamp = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%MZ")
|
|
2280
|
+
entry = f"{stamp}: " + " | ".join(history_notes)
|
|
2281
|
+
existing = ds.attrs.get("history", "")
|
|
2282
|
+
ds.attrs["history"] = f"{existing}; {entry}" if existing else entry
|
|
2283
|
+
|
|
2284
|
+
# Derive O2 % saturation and AOU when dissolved_oxygen + T/S/P are present.
|
|
2285
|
+
ds = _derive_oxygen_saturation(ds)
|
|
2286
|
+
|
|
2287
|
+
# Normalize CF standard_name and long_name for known physics variables.
|
|
2288
|
+
# Only fills in missing attrs — never overwrites values already set.
|
|
2289
|
+
for _vname, _cf in _CF_ATTRS.items():
|
|
2290
|
+
if _vname in ds.data_vars:
|
|
2291
|
+
for _k, _v in _cf.items():
|
|
2292
|
+
if _k not in ds[_vname].attrs:
|
|
2293
|
+
ds[_vname].attrs[_k] = _v
|
|
2294
|
+
|
|
2295
|
+
ds = drop_all_zero_vars(ds, ["amplitude_beam", "analog_input_"])
|
|
2296
|
+
cast_output_dtypes(ds).to_netcdf(l3_path)
|
|
2297
|
+
ds.close()
|
|
2298
|
+
self._log(
|
|
2299
|
+
f" Creating output file: {l3_path.name} ({'; '.join(qc_summary)})"
|
|
2300
|
+
)
|
|
2301
|
+
|
|
2302
|
+
except Exception as e: # noqa: BLE001 — log and continue; one instrument must not abort mooring
|
|
2303
|
+
self._log(f" ERROR processing {serial}: {e}")
|
|
2304
|
+
import traceback
|
|
2305
|
+
|
|
2306
|
+
self._log(traceback.format_exc())
|
|
2307
|
+
return False
|
|
2308
|
+
return True
|
|
2309
|
+
|
|
2310
|
+
# ------------------------------------------------------------------
|
|
2311
|
+
def _interp_pressure_for_hab(
|
|
2312
|
+
self,
|
|
2313
|
+
hab_t: float,
|
|
2314
|
+
sorted_sources: List[Dict[str, Any]],
|
|
2315
|
+
target_time: np.ndarray,
|
|
2316
|
+
serial: str,
|
|
2317
|
+
) -> tuple[np.ndarray, str]:
|
|
2318
|
+
"""Interpolate pressure for a single nominal HAB; return (p_array, method_str)."""
|
|
2319
|
+
habs = np.array([s["hab"] for s in sorted_sources])
|
|
2320
|
+
diffs = np.abs(habs - hab_t)
|
|
2321
|
+
nearest_idx = int(np.argmin(diffs))
|
|
2322
|
+
|
|
2323
|
+
if diffs[nearest_idx] <= HAB_THRESHOLD:
|
|
2324
|
+
src = sorted_sources[nearest_idx]
|
|
2325
|
+
hab_offset_dbar = src["hab"] - hab_t
|
|
2326
|
+
p = (
|
|
2327
|
+
_interp_pressure(
|
|
2328
|
+
src["ds"]["time"].values,
|
|
2329
|
+
src["ds"][src["pressure_var"]].values,
|
|
2330
|
+
target_time,
|
|
2331
|
+
)
|
|
2332
|
+
+ hab_offset_dbar
|
|
2333
|
+
)
|
|
2334
|
+
method = (
|
|
2335
|
+
f"near-neighbour from {src['instrument']} {src['serial']} "
|
|
2336
|
+
f"(Δhab={diffs[nearest_idx]:.1f}m, "
|
|
2337
|
+
f"static offset={hab_offset_dbar:+.1f} dbar)"
|
|
2338
|
+
)
|
|
2339
|
+
self._log(f" {serial} (hab={hab_t:.1f}m): {method}")
|
|
2340
|
+
else:
|
|
2341
|
+
below = [(s, h) for s, h in zip(sorted_sources, habs) if h < hab_t]
|
|
2342
|
+
above = [(s, h) for s, h in zip(sorted_sources, habs) if h > hab_t]
|
|
2343
|
+
|
|
2344
|
+
if below and above:
|
|
2345
|
+
src_below, h_below = below[-1]
|
|
2346
|
+
src_above, h_above = above[0]
|
|
2347
|
+
w_above = (hab_t - h_below) / (h_above - h_below)
|
|
2348
|
+
w_below = 1.0 - w_above
|
|
2349
|
+
p_below = _interp_pressure(
|
|
2350
|
+
src_below["ds"]["time"].values,
|
|
2351
|
+
src_below["ds"][src_below["pressure_var"]].values,
|
|
2352
|
+
target_time,
|
|
2353
|
+
)
|
|
2354
|
+
p_above = _interp_pressure(
|
|
2355
|
+
src_above["ds"]["time"].values,
|
|
2356
|
+
src_above["ds"][src_above["pressure_var"]].values,
|
|
2357
|
+
target_time,
|
|
2358
|
+
)
|
|
2359
|
+
p = w_below * p_below + w_above * p_above
|
|
2360
|
+
method = (
|
|
2361
|
+
f"bracketed: {w_below:.2f}×{src_below['instrument']} "
|
|
2362
|
+
f"{src_below['serial']} (hab={h_below:.1f}m) + "
|
|
2363
|
+
f"{w_above:.2f}×{src_above['instrument']} "
|
|
2364
|
+
f"{src_above['serial']} (hab={h_above:.1f}m)"
|
|
2365
|
+
)
|
|
2366
|
+
self._log(f" {serial} (hab={hab_t:.1f}m): {method}")
|
|
2367
|
+
else:
|
|
2368
|
+
src = below[-1][0] if below else above[0][0]
|
|
2369
|
+
hab_offset_dbar = src["hab"] - hab_t
|
|
2370
|
+
p = (
|
|
2371
|
+
_interp_pressure(
|
|
2372
|
+
src["ds"]["time"].values,
|
|
2373
|
+
src["ds"][src["pressure_var"]].values,
|
|
2374
|
+
target_time,
|
|
2375
|
+
)
|
|
2376
|
+
+ hab_offset_dbar
|
|
2377
|
+
)
|
|
2378
|
+
method = (
|
|
2379
|
+
f"extrapolated from {src['instrument']} {src['serial']} "
|
|
2380
|
+
f"(hab={src['hab']:.1f}m, "
|
|
2381
|
+
f"static offset={hab_offset_dbar:+.1f} dbar) — WARNING: out of range"
|
|
2382
|
+
)
|
|
2383
|
+
self._log(f" WARNING: {serial} (hab={hab_t:.1f}m): {method}")
|
|
2384
|
+
|
|
2385
|
+
return p, method
|
|
2386
|
+
|
|
2387
|
+
def _interpolate_pressure(
|
|
2388
|
+
self,
|
|
2389
|
+
ds: xr.Dataset,
|
|
2390
|
+
target_info: Dict[str, Any],
|
|
2391
|
+
sources: List[Dict[str, Any]],
|
|
2392
|
+
target_time: np.ndarray,
|
|
2393
|
+
pressure_bad_flag: bool,
|
|
2394
|
+
qc_attrs: Dict[str, Any],
|
|
2395
|
+
) -> tuple[xr.Dataset, str]:
|
|
2396
|
+
"""Interpolate pressure from sources onto target; return (ds, method_str).
|
|
2397
|
+
|
|
2398
|
+
If *target_info* contains ``hab_segments`` (a sorted list of
|
|
2399
|
+
``(breakpoint_datetime64, hab_float)`` pairs), the record is split into
|
|
2400
|
+
time segments and each segment is interpolated with its own HAB, allowing
|
|
2401
|
+
for instruments that physically moved during the deployment.
|
|
2402
|
+
|
|
2403
|
+
HAB convention for segmented deployments
|
|
2404
|
+
-----------------------------------------
|
|
2405
|
+
``target_info["hab"]`` is the **initial** HAB — the height above bottom
|
|
2406
|
+
at which the instrument was **deployed**. Segment 0 runs from the start
|
|
2407
|
+
of the record up to (but not including) the first breakpoint, using this
|
|
2408
|
+
value. Each entry in ``hab_segments`` gives the new HAB **at and after**
|
|
2409
|
+
the breakpoint timestamp (i.e. records with ``T >= breakpoint`` use the
|
|
2410
|
+
new HAB; the breakpoint itself belongs to the new segment). This matches
|
|
2411
|
+
the YAML convention where ``hab:`` records the as-deployed position and
|
|
2412
|
+
``hab_segments:`` records subsequent slides, with ``from:`` marking the
|
|
2413
|
+
first timestamp that belongs to the post-slide position.
|
|
2414
|
+
"""
|
|
2415
|
+
if len(target_time) == 0:
|
|
2416
|
+
return ds, "no data"
|
|
2417
|
+
|
|
2418
|
+
pressure_qc_val = target_info["qc_flags"].get("pressure", 0)
|
|
2419
|
+
serial = target_info["serial"]
|
|
2420
|
+
|
|
2421
|
+
sorted_sources = sorted(
|
|
2422
|
+
[s for s in sources if s.get("ds") is not None],
|
|
2423
|
+
key=lambda s: s["hab"],
|
|
2424
|
+
)
|
|
2425
|
+
if not sorted_sources:
|
|
2426
|
+
return ds, "no sources available"
|
|
2427
|
+
|
|
2428
|
+
hab_segments = target_info.get("hab_segments", [])
|
|
2429
|
+
|
|
2430
|
+
if hab_segments:
|
|
2431
|
+
# Build list of (start_ns, end_ns, hab) covering the full record.
|
|
2432
|
+
# Segment 0: record start → first breakpoint, using nominal HAB.
|
|
2433
|
+
# Segment k: breakpoint[k-1] → breakpoint[k] (or record end), using
|
|
2434
|
+
# the HAB specified at breakpoint[k-1].
|
|
2435
|
+
T = target_time
|
|
2436
|
+
seg_bounds: List[tuple] = []
|
|
2437
|
+
seg_hab = target_info["hab"]
|
|
2438
|
+
seg_start = T[0]
|
|
2439
|
+
for bp_ns, next_hab in hab_segments:
|
|
2440
|
+
seg_bounds.append((seg_start, bp_ns, seg_hab))
|
|
2441
|
+
seg_start = bp_ns
|
|
2442
|
+
seg_hab = next_hab
|
|
2443
|
+
seg_bounds.append((seg_start, T[-1] + np.timedelta64(1, "ns"), seg_hab))
|
|
2444
|
+
|
|
2445
|
+
p_interp = np.full(len(T), np.nan)
|
|
2446
|
+
method_parts = []
|
|
2447
|
+
for seg_start_ns, seg_end_ns, hab_t in seg_bounds:
|
|
2448
|
+
idx = np.where((T >= seg_start_ns) & (T < seg_end_ns))[0]
|
|
2449
|
+
if len(idx) == 0:
|
|
2450
|
+
continue
|
|
2451
|
+
p_seg, meth = self._interp_pressure_for_hab(
|
|
2452
|
+
hab_t, sorted_sources, T[idx], serial
|
|
2453
|
+
)
|
|
2454
|
+
p_interp[idx] = p_seg
|
|
2455
|
+
bp_str = str(seg_end_ns)[:16]
|
|
2456
|
+
method_parts.append(f"hab={hab_t:.1f}m until {bp_str}: {meth}")
|
|
2457
|
+
method = "; ".join(method_parts)
|
|
2458
|
+
else:
|
|
2459
|
+
p_interp, method = self._interp_pressure_for_hab(
|
|
2460
|
+
target_info["hab"], sorted_sources, target_time, serial
|
|
2461
|
+
)
|
|
2462
|
+
|
|
2463
|
+
# Preserve bad original pressure
|
|
2464
|
+
if pressure_bad_flag and "pressure" in ds.data_vars:
|
|
2465
|
+
orig_attrs = dict(ds["pressure"].attrs)
|
|
2466
|
+
orig_attrs["comment"] = (
|
|
2467
|
+
orig_attrs.get("comment", "") + " [original; flagged bad in YAML]"
|
|
2468
|
+
).strip()
|
|
2469
|
+
ds["pressure_orig"] = ds["pressure"].copy()
|
|
2470
|
+
ds["pressure_orig"].attrs = orig_attrs
|
|
2471
|
+
ds["pressure_orig_qc"] = xr.Variable(
|
|
2472
|
+
"time",
|
|
2473
|
+
np.full(len(ds["time"]), pressure_qc_val, dtype=np.int8),
|
|
2474
|
+
attrs={"long_name": "quality flag for pressure_orig", **qc_attrs},
|
|
2475
|
+
)
|
|
2476
|
+
ds = ds.drop_vars("pressure")
|
|
2477
|
+
|
|
2478
|
+
ds["pressure"] = xr.Variable(
|
|
2479
|
+
"time",
|
|
2480
|
+
p_interp,
|
|
2481
|
+
attrs={
|
|
2482
|
+
"units": "dbar",
|
|
2483
|
+
"long_name": "sea water pressure",
|
|
2484
|
+
"standard_name": "sea_water_pressure",
|
|
2485
|
+
"pressure_source": method,
|
|
2486
|
+
"comment": "interpolated from neighbouring instrument(s)",
|
|
2487
|
+
},
|
|
2488
|
+
)
|
|
2489
|
+
ds["pressure_qc"] = xr.Variable(
|
|
2490
|
+
"time",
|
|
2491
|
+
np.full(len(ds["time"]), 8, dtype=np.int8),
|
|
2492
|
+
attrs={"long_name": "quality flag for pressure", **qc_attrs},
|
|
2493
|
+
)
|
|
2494
|
+
return ds, method
|