oceanarray 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. oceanarray/__init__.py +20 -0
  2. oceanarray/_version.py +1 -0
  3. oceanarray/cli.py +1343 -0
  4. oceanarray/clock_offset.py +675 -0
  5. oceanarray/config/OS1_sensor_attrs.yaml +49 -0
  6. oceanarray/config/OS1_var_names.yaml +5 -0
  7. oceanarray/config/OS1_vocab_attrs.yaml +84 -0
  8. oceanarray/config/legacy/project_RAPID.yaml +69 -0
  9. oceanarray/config/legacy/rodb_keys.txt +206 -0
  10. oceanarray/config/legacy/rodb_keys.yaml +1012 -0
  11. oceanarray/config/logging.yaml +28 -0
  12. oceanarray/legacy/__init__.py +47 -0
  13. oceanarray/legacy/caldip.py +753 -0
  14. oceanarray/legacy/convertOS.py +527 -0
  15. oceanarray/legacy/find_deployment.py +489 -0
  16. oceanarray/legacy/mooring_rodb.py +429 -0
  17. oceanarray/legacy/process_rodb.py +369 -0
  18. oceanarray/legacy/rodb.py +330 -0
  19. oceanarray/logger.py +192 -0
  20. oceanarray/logsheet/__init__.py +60 -0
  21. oceanarray/logsheet/_columns.py +97 -0
  22. oceanarray/logsheet/_config.py +245 -0
  23. oceanarray/logsheet/_firmware.py +43 -0
  24. oceanarray/logsheet/_latex.py +128 -0
  25. oceanarray/logsheet/_render.py +126 -0
  26. oceanarray/logsheet/builders/__init__.py +13 -0
  27. oceanarray/logsheet/builders/caldip_download.py +195 -0
  28. oceanarray/logsheet/builders/caldip_setup.py +188 -0
  29. oceanarray/logsheet/builders/deployment_setup.py +218 -0
  30. oceanarray/logsheet/builders/mooring_download.py +243 -0
  31. oceanarray/logsheet/builders/recovery.py +157 -0
  32. oceanarray/logsheet/data/column_defs.yaml +686 -0
  33. oceanarray/logsheet/data/latex_templates/_base_instrument.tex.j2 +42 -0
  34. oceanarray/logsheet/data/latex_templates/caldip_download.tex.j2 +47 -0
  35. oceanarray/logsheet/data/latex_templates/caldip_setup.tex.j2 +70 -0
  36. oceanarray/logsheet/data/latex_templates/deployment_setup.tex.j2 +97 -0
  37. oceanarray/logsheet/data/latex_templates/mooring_download.tex.j2 +54 -0
  38. oceanarray/logsheet/data/latex_templates/recovery.tex.j2 +103 -0
  39. oceanarray/mooring_level.py +1368 -0
  40. oceanarray/oceanarray.mplstyle +18 -0
  41. oceanarray/parameters.py +270 -0
  42. oceanarray/plotter.py +942 -0
  43. oceanarray/plotters/__init__.py +97 -0
  44. oceanarray/plotters/_animation.py +261 -0
  45. oceanarray/plotters/_current.py +705 -0
  46. oceanarray/plotters/_diagnostic.py +667 -0
  47. oceanarray/plotters/_helpers.py +57 -0
  48. oceanarray/plotters/_primitives.py +101 -0
  49. oceanarray/plotters/_section.py +16 -0
  50. oceanarray/plotters/_timeseries.py +11 -0
  51. oceanarray/rapid_interp.py +739 -0
  52. oceanarray/read_rbr_hex.py +541 -0
  53. oceanarray/readers.py +303 -0
  54. oceanarray/report/__init__.py +5 -0
  55. oceanarray/report/_array.py +386 -0
  56. oceanarray/report/_grid.py +539 -0
  57. oceanarray/report/_html_helpers.py +1004 -0
  58. oceanarray/report/_instrument.py +853 -0
  59. oceanarray/report/_mooring.py +1385 -0
  60. oceanarray/report/_plots.py +4284 -0
  61. oceanarray/report/_stack.py +941 -0
  62. oceanarray/stage1.py +1959 -0
  63. oceanarray/stage2.py +1160 -0
  64. oceanarray/stage3.py +2494 -0
  65. oceanarray/time_gridding.py +958 -0
  66. oceanarray/tools.py +400 -0
  67. oceanarray/transports.py +1 -0
  68. oceanarray/utilities.py +538 -0
  69. oceanarray/validation.py +389 -0
  70. oceanarray/writers.py +99 -0
  71. oceanarray-0.1.0.dist-info/METADATA +503 -0
  72. oceanarray-0.1.0.dist-info/RECORD +76 -0
  73. oceanarray-0.1.0.dist-info/WHEEL +5 -0
  74. oceanarray-0.1.0.dist-info/entry_points.txt +2 -0
  75. oceanarray-0.1.0.dist-info/licenses/LICENSE +21 -0
  76. oceanarray-0.1.0.dist-info/top_level.txt +1 -0
oceanarray/stage3.py ADDED
@@ -0,0 +1,2494 @@
1
+ """Stage 3: pressure interpolation + QARTOD QC (gross-range and spike tests).
2
+
3
+ Processing order per instrument
4
+ ---------------------------------
5
+ 1. Load ``_stage2.nc``.
6
+ 2. **Pressure interpolation** (targets only — instruments lacking pressure or
7
+ whose pressure is flagged bad in the YAML):
8
+ a. Near-neighbour if any source has |Δhab| ≤ ``HAB_THRESHOLD``.
9
+ b. Weighted bracketing from the closest source above and below.
10
+ c. Extrapolation (with WARNING) when target is outside all source habs.
11
+ Interpolated pressure gets ``pressure_qc = 8`` (interpolated_value).
12
+ 3. **QARTOD gross-range test** on temperature, conductivity, pressure, and
13
+ velocity components. Flags 4 (bad) or 3 (suspect) based on thresholds in
14
+ ``parameters.QC_GROSS_RANGE`` (overrideable per mooring / per instrument in
15
+ YAML via a ``qc_ranges`` key).
16
+ 4. **QARTOD spike test** on the same variables. Flags from
17
+ ``parameters.QC_SPIKE`` (overrideable via ``qc_spike`` in YAML).
18
+ 5. Merge all QC flags using priority order: 9 > 4 > 3 > 8 > 2 > 1.
19
+ 6. Write ``_stage3.nc`` for **all** instruments (not only pressure targets).
20
+
21
+ Flag combination priority
22
+ --------------------------
23
+ Missing (9) > Bad (4) > Suspect (3) > Interpolated (8) > Prob-good (2) > Good (1)
24
+
25
+ This means that if interpolated pressure also fails the range test it is
26
+ flagged 4 (bad), not 8 (interpolated).
27
+
28
+ YAML configuration keys
29
+ ------------------------
30
+ Top-level (mooring-wide):
31
+ ``qc_ranges`` : mapping of variable → {fail_span, suspect_span}
32
+ ``qc_spike`` : mapping of variable → {suspect_threshold, fail_threshold}
33
+
34
+ Per-instrument (in a clamp entry):
35
+ ``qc_ranges`` : same structure; overrides the mooring-level setting for
36
+ the variables listed (others fall back to mooring/global defaults)
37
+ ``qc_spike`` : same structure
38
+ ``pressure_qc`` : int — mark this instrument's own pressure as bad (≥3) so
39
+ stage3 replaces it with an interpolated value.
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import copy
45
+ import datetime
46
+ import hashlib
47
+ import re
48
+ from pathlib import Path
49
+ from typing import Any, Dict, List, Optional
50
+
51
+ import numpy as np
52
+ import xarray as xr
53
+ import yaml
54
+
55
+ from .utilities import (
56
+ cast_output_dtypes,
57
+ drop_all_zero_vars,
58
+ extract_inline_instruments,
59
+ )
60
+
61
+
62
+ HAB_THRESHOLD = 2.0 # metres — use near-neighbour below this Δhab
63
+
64
+
65
+ # ---------------------------------------------------------------------------
66
+ # BEAM → ENU coordinate transform helpers
67
+ # ---------------------------------------------------------------------------
68
+
69
+
70
+ def _xyz_to_enu(
71
+ vx: np.ndarray,
72
+ vy: np.ndarray,
73
+ vz: np.ndarray,
74
+ heading_deg: np.ndarray,
75
+ pitch_deg: np.ndarray,
76
+ roll_deg: np.ndarray,
77
+ declination_deg: float = 0.0,
78
+ ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
79
+ """Vectorised XYZ → ENU rotation per Nortek Support reference script.
80
+
81
+ R = H @ P where (with hdg = heading - 90 + declination):
82
+ H = [[cos(hdg), sin(hdg), 0], [-sin(hdg), cos(hdg), 0], [0, 0, 1]]
83
+ P = [[cos(p), -sin(p)*sin(r), -cos(r)*sin(p)],
84
+ [0, cos(r), -sin(r)],
85
+ [sin(p), sin(r)*cos(p), cos(p)*cos(r)]]
86
+
87
+ The -90 offset accounts for the Nortek Aquadopp instrument frame where
88
+ heading=90° aligns X→East, Y→North (standard geography at zero tilt).
89
+ Magnetic declination is added to convert magnetic heading to true north.
90
+
91
+ Returns (east, north, up) arrays of the same shape as the inputs.
92
+ """
93
+ h = np.radians(heading_deg - 90.0 + declination_deg)
94
+ p = np.radians(pitch_deg)
95
+ r = np.radians(roll_deg)
96
+
97
+ ch = np.cos(h)
98
+ sh = np.sin(h)
99
+ cp = np.cos(p)
100
+ sp = np.sin(p)
101
+ cr = np.cos(r)
102
+ sr = np.sin(r)
103
+
104
+ # Expanded R = H @ P (verified against Nortek Support reference script)
105
+ east = (
106
+ ch * cp * vx + (-ch * sp * sr + sh * cr) * vy + (-ch * sp * cr - sh * sr) * vz
107
+ )
108
+ north = (
109
+ -sh * cp * vx + (sh * sp * sr + ch * cr) * vy + (sh * sp * cr - ch * sr) * vz
110
+ )
111
+ up = sp * vx + sr * cp * vy + cp * cr * vz
112
+ return east, north, up
113
+
114
+
115
+ def _apply_beam_to_enu(
116
+ ds: "xr.Dataset",
117
+ entry: Dict[str, Any],
118
+ lat: float,
119
+ lon: float,
120
+ latlon_source: str = "unknown",
121
+ log_fn: Any = None,
122
+ ) -> "xr.Dataset":
123
+ """Transform BEAM or XYZ Nortek velocities to ENU geographic coordinates.
124
+
125
+ Adds east_velocity, north_velocity, up_velocity, current_speed,
126
+ current_direction. Updates coordinate_system attr to 'ENU'.
127
+ No-ops for instruments already in ENU or with unknown coordinate system.
128
+ Requires normalized variable names (heading, pitch, roll) — re-run stage1
129
+ if these are absent.
130
+ """
131
+
132
+ def _warn(msg: str) -> None:
133
+ if log_fn:
134
+ log_fn(msg)
135
+ else:
136
+ print(msg)
137
+
138
+ coord_sys = ds.attrs.get("coordinate_system", "ENU")
139
+ if coord_sys not in ("BEAM", "XYZ"):
140
+ return ds
141
+
142
+ # Require normalized heading/pitch/roll (stage1 normalization must have run)
143
+ for vname in ("heading", "pitch", "roll"):
144
+ if vname not in ds.data_vars:
145
+ _warn(
146
+ f" WARNING: BEAM→ENU skipped — '{vname}' not in dataset. "
147
+ "Re-run stage1 to get normalized variable names."
148
+ )
149
+ return ds
150
+
151
+ heading = ds["heading"].values.astype(float)
152
+ pitch = ds["pitch"].values.astype(float)
153
+ roll = ds["roll"].values.astype(float)
154
+
155
+ # Magnetic declination via ppigrf
156
+ declination = 0.0
157
+ if lat == 0.0 and lon == 0.0 and "unknown" in latlon_source.lower():
158
+ _warn(
159
+ f" WARNING: BEAM→ENU — lat/lon unknown for serial {entry.get('serial', '?')}. "
160
+ "Skipping magnetic declination to avoid silently applying the Gulf of Guinea value. "
161
+ "Set lat/lon in the mooring YAML to get a correct declination correction."
162
+ )
163
+ ds.attrs["magnetic_declination"] = "UNK"
164
+ ds.attrs["magnetic_declination_latlon_source"] = (
165
+ f"mooring YAML ({latlon_source})"
166
+ )
167
+ else:
168
+ try:
169
+ import ppigrf
170
+ import datetime as _dt
171
+
172
+ time_vals = ds["time"].values
173
+ t_mid = time_vals[len(time_vals) // 2]
174
+ t_mid_s = int(t_mid.astype("datetime64[s]").astype("int64"))
175
+ t_mid_dt = _dt.datetime.utcfromtimestamp(t_mid_s)
176
+ Be, Bn, _ = ppigrf.igrf(float(lon), float(lat), 0.0, t_mid_dt)
177
+ declination = float(
178
+ np.degrees(
179
+ np.arctan2(float(np.atleast_1d(Be)[0]), float(np.atleast_1d(Bn)[0]))
180
+ )
181
+ )
182
+ ds.attrs["magnetic_declination"] = declination
183
+ ds.attrs["magnetic_declination_units"] = "degrees_east"
184
+ ds.attrs["magnetic_declination_method"] = (
185
+ "ppigrf IGRF at deployment midpoint"
186
+ )
187
+ ds.attrs["magnetic_declination_lat"] = lat
188
+ ds.attrs["magnetic_declination_lon"] = lon
189
+ ds.attrs["magnetic_declination_latlon_source"] = (
190
+ f"mooring YAML ({latlon_source})"
191
+ )
192
+ _warn(f" BEAM→ENU: magnetic declination = {declination:.2f}°")
193
+ except Exception as e: # noqa: BLE001 — ppigrf optional; proceed with 0° declination
194
+ _warn(f" WARNING: magnetic declination unavailable ({e}) — using 0°")
195
+
196
+ if coord_sys == "BEAM":
197
+ # Stage1 could not apply the T matrix (header file missing or unparseable).
198
+ # Re-run stage1 with the correct header file to produce velocity_x/y/z.
199
+ _warn(
200
+ f" SKIPPING BEAM→ENU for serial {entry.get('serial', '?')}: "
201
+ "data are still in BEAM coordinates — re-run stage1 with the instrument "
202
+ "header file so the T matrix can be extracted and velocity_x/y/z produced."
203
+ )
204
+ return ds
205
+
206
+ else: # XYZ — stage3 only needs to do XYZ→ENU
207
+ if "velocity_x" in ds.data_vars:
208
+ # stage1 applied the T matrix (BEAM→XYZ) and stored instrument-frame XYZ
209
+ T_source = "applied in stage1 (BEAM→XYZ)"
210
+ vx = ds["velocity_x"].values.astype(float)
211
+ vy = ds["velocity_y"].values.astype(float)
212
+ vz = ds["velocity_z"].values.astype(float)
213
+ elif "x_velocity" in ds.data_vars:
214
+ # Instrument natively reports XYZ; T matrix applied in firmware
215
+ T_source = "not applicable (instrument-native XYZ)"
216
+ vx = ds["x_velocity"].values.astype(float)
217
+ vy = ds["y_velocity"].values.astype(float)
218
+ vz = ds["z_velocity"].values.astype(float)
219
+ else:
220
+ # Legacy fallback: XYZ stored in beam variable slots
221
+ T_source = "not applicable (legacy XYZ fallback)"
222
+ vx = ds["velocity_beam1"].values.astype(float)
223
+ vy = ds["velocity_beam2"].values.astype(float)
224
+ vz = ds["velocity_beam3"].values.astype(float)
225
+
226
+ # XYZ → ENU
227
+ valid_all = (
228
+ np.isfinite(vx)
229
+ & np.isfinite(vy)
230
+ & np.isfinite(vz)
231
+ & np.isfinite(heading)
232
+ & np.isfinite(pitch)
233
+ & np.isfinite(roll)
234
+ )
235
+ east = np.full_like(vx, np.nan)
236
+ north = np.full_like(vx, np.nan)
237
+ up = np.full_like(vx, np.nan)
238
+ if valid_all.any():
239
+ east[valid_all], north[valid_all], up[valid_all] = _xyz_to_enu(
240
+ vx[valid_all],
241
+ vy[valid_all],
242
+ vz[valid_all],
243
+ heading[valid_all],
244
+ pitch[valid_all],
245
+ roll[valid_all],
246
+ declination,
247
+ )
248
+
249
+ speed = np.where(
250
+ np.isfinite(east) & np.isfinite(north), np.sqrt(east**2 + north**2), np.nan
251
+ )
252
+ direction = np.where(
253
+ np.isfinite(east) & np.isfinite(north),
254
+ np.degrees(np.arctan2(east, north)) % 360.0,
255
+ np.nan,
256
+ )
257
+
258
+ time_dim = ds["heading"].dims[0]
259
+ for name, arr, cf, units, long_name in [
260
+ (
261
+ "east_velocity",
262
+ east,
263
+ "eastward_sea_water_velocity",
264
+ "m s-1",
265
+ "Eastward sea water velocity",
266
+ ),
267
+ (
268
+ "north_velocity",
269
+ north,
270
+ "northward_sea_water_velocity",
271
+ "m s-1",
272
+ "Northward sea water velocity",
273
+ ),
274
+ (
275
+ "up_velocity",
276
+ up,
277
+ "upward_sea_water_velocity",
278
+ "m s-1",
279
+ "Upward sea water velocity",
280
+ ),
281
+ (
282
+ "current_speed",
283
+ speed,
284
+ "sea_water_speed",
285
+ "m s-1",
286
+ "Horizontal current speed",
287
+ ),
288
+ (
289
+ "current_direction",
290
+ direction,
291
+ "direction_of_sea_water_velocity",
292
+ "degrees",
293
+ "Current direction (0=N, clockwise)",
294
+ ),
295
+ ]:
296
+ ds[name] = xr.Variable(
297
+ time_dim,
298
+ arr,
299
+ attrs={"units": units, "standard_name": cf, "long_name": long_name},
300
+ )
301
+
302
+ ds.attrs["coordinate_system"] = "ENU"
303
+ ds.attrs["coordinate_system_source"] = (
304
+ f"rotated from {coord_sys} by oceanarray stage3"
305
+ )
306
+ _warn(
307
+ f" BEAM→ENU: produced east/north/up_velocity, current_speed, current_direction "
308
+ f"(coord_sys was {coord_sys}, T matrix: {T_source})"
309
+ )
310
+ return ds
311
+
312
+
313
+ def _apply_declination_to_enu(
314
+ ds: "xr.Dataset",
315
+ lat: float,
316
+ lon: float,
317
+ latlon_source: str = "unknown",
318
+ log_fn: Any = None,
319
+ ) -> "xr.Dataset":
320
+ """Apply magnetic declination rotation to velocities already in ENU frame.
321
+
322
+ When a Nortek instrument is configured to output ENU coordinates internally,
323
+ the heading reference used is magnetic north. This function rotates
324
+ east_velocity and north_velocity by the declination angle so that north
325
+ aligns with true (geographic) north.
326
+
327
+ Rotation (D = declination, positive = east):
328
+ u_true = u_mag * cos(D) + v_mag * sin(D)
329
+ v_true = -u_mag * sin(D) + v_mag * cos(D)
330
+
331
+ No-ops if east_velocity or north_velocity are absent, or if magnetic
332
+ declination has already been applied (``magnetic_declination`` attr present).
333
+ """
334
+
335
+ def _warn(msg: str) -> None:
336
+ if log_fn:
337
+ log_fn(msg)
338
+ else:
339
+ print(msg)
340
+
341
+ if "magnetic_declination" in ds.attrs:
342
+ return ds # already applied
343
+
344
+ if "east_velocity" not in ds.data_vars or "north_velocity" not in ds.data_vars:
345
+ return ds
346
+
347
+ if lat == 0.0 and lon == 0.0 and "unknown" in latlon_source.lower():
348
+ _warn(
349
+ " WARNING: ENU declination correction skipped — lat/lon unknown. "
350
+ "Applying declination from (0°N, 0°E) would silently rotate velocities by "
351
+ "the Gulf of Guinea value. Set lat/lon in the mooring YAML."
352
+ )
353
+ ds.attrs["magnetic_declination"] = "UNK"
354
+ ds.attrs["magnetic_declination_latlon_source"] = (
355
+ f"mooring YAML ({latlon_source})"
356
+ )
357
+ return ds
358
+
359
+ try:
360
+ import ppigrf
361
+ import datetime as _dt
362
+
363
+ time_vals = ds["time"].values
364
+ t_mid = time_vals[len(time_vals) // 2]
365
+ t_mid_s = int(t_mid.astype("datetime64[s]").astype("int64"))
366
+ t_mid_dt = _dt.datetime.utcfromtimestamp(t_mid_s)
367
+ Be, Bn, _ = ppigrf.igrf(float(lon), float(lat), 0.0, t_mid_dt)
368
+ declination = float(
369
+ np.degrees(
370
+ np.arctan2(float(np.atleast_1d(Be)[0]), float(np.atleast_1d(Bn)[0]))
371
+ )
372
+ )
373
+
374
+ D = np.radians(declination)
375
+ u = ds["east_velocity"].values.astype(float)
376
+ v = ds["north_velocity"].values.astype(float)
377
+ ds["east_velocity"].values[:] = u * np.cos(D) + v * np.sin(D)
378
+ ds["north_velocity"].values[:] = -u * np.sin(D) + v * np.cos(D)
379
+
380
+ ds.attrs["magnetic_declination"] = declination
381
+ ds.attrs["magnetic_declination_units"] = "degrees_east"
382
+ ds.attrs["magnetic_declination_method"] = "ppigrf IGRF at deployment midpoint"
383
+ ds.attrs["magnetic_declination_lat"] = lat
384
+ ds.attrs["magnetic_declination_lon"] = lon
385
+ ds.attrs["magnetic_declination_latlon_source"] = (
386
+ f"mooring YAML ({latlon_source})"
387
+ )
388
+ ds.attrs["coordinate_system_source"] = (
389
+ "ENU from instrument; declination-corrected by oceanarray stage3"
390
+ )
391
+ _warn(
392
+ f" ENU declination correction: {declination:+.2f}° applied to "
393
+ "east_velocity / north_velocity"
394
+ )
395
+ except Exception as e: # noqa: BLE001 — ppigrf optional
396
+ _warn(f" WARNING: declination correction unavailable ({e}) — ENU unchanged")
397
+
398
+ return ds
399
+
400
+
401
+ # Priority order for merging QC flags (higher priority = worse data quality).
402
+ # 9=missing, 4=bad, 3=suspect, 8=interpolated, 2=prob-good, 1=good
403
+ _QC_PRIORITY: Dict[int, int] = {9: 6, 4: 5, 3: 4, 8: 3, 2: 2, 1: 1, 0: 0}
404
+
405
+ # CF standard names and canonical long_names for known physics variables.
406
+ # Applied as a normalization pass just before writing _stage3.nc so that all
407
+ # output files have consistent metadata regardless of which reader produced them.
408
+ _CF_ATTRS: Dict[str, Dict[str, str]] = {
409
+ "temperature": {
410
+ "standard_name": "sea_water_temperature",
411
+ "long_name": "Temperature",
412
+ "units": "degrees_C",
413
+ },
414
+ "conductivity": {
415
+ "standard_name": "sea_water_electrical_conductivity",
416
+ "long_name": "Conductivity",
417
+ },
418
+ "salinity": {
419
+ "standard_name": "sea_water_practical_salinity",
420
+ "long_name": "Salinity",
421
+ "units": "1",
422
+ },
423
+ "pressure": {
424
+ "standard_name": "sea_water_pressure",
425
+ "long_name": "Pressure",
426
+ "units": "dbar",
427
+ },
428
+ "turbidity": {
429
+ "standard_name": "sea_water_turbidity",
430
+ "long_name": "Turbidity",
431
+ },
432
+ "dissolved_oxygen": {
433
+ "standard_name": "mole_concentration_of_dissolved_molecular_oxygen_in_sea_water",
434
+ "long_name": "Dissolved Oxygen",
435
+ "units": "umol/L",
436
+ },
437
+ "oxygen_saturation_pct": {
438
+ "long_name": "Dissolved Oxygen Percent Saturation",
439
+ "units": "%",
440
+ },
441
+ "apparent_oxygen_utilization": {
442
+ "standard_name": "apparent_oxygen_utilization",
443
+ "long_name": "Apparent Oxygen Utilization",
444
+ "units": "umol/kg",
445
+ },
446
+ "dissolved_oxygen_ml_l": {
447
+ "standard_name": "mole_concentration_of_dissolved_molecular_oxygen_in_sea_water",
448
+ "long_name": "Dissolved Oxygen",
449
+ "units": "mL/L",
450
+ },
451
+ }
452
+
453
+
454
+ def _safe_serial(serial: Any) -> str:
455
+ return re.sub(r"[^\w\-]", "", str(serial))
456
+
457
+
458
+ def _times_to_float(time_values: np.ndarray) -> np.ndarray:
459
+ """Convert numpy datetime64 array to float64 nanoseconds (for numpy.interp)."""
460
+ return time_values.astype("datetime64[ns]").astype(np.float64)
461
+
462
+
463
+ def _is_burst_mode(time_values: np.ndarray, burst_ratio: float = 5.0) -> bool:
464
+ """Return True when the time series has a bimodal Δt (burst sampling).
465
+
466
+ Burst-mode instruments (e.g. Nortek Aquadopp) take N pings at 1 Hz then
467
+ wait ~120 s. The within-burst interval (p50) is << the burst interval
468
+ (p90). The QARTOD spike test compares adjacent samples and generates false
469
+ positives at every burst boundary, so callers skip it when this is True.
470
+ """
471
+ if len(time_values) < 10:
472
+ return False
473
+ dt = np.diff(time_values).astype("datetime64[s]").astype(float)
474
+ dt = dt[dt > 0]
475
+ if len(dt) == 0:
476
+ return False
477
+ p50 = float(np.percentile(dt, 50))
478
+ p90 = float(np.percentile(dt, 90))
479
+ return p50 > 0 and p90 > burst_ratio * p50
480
+
481
+
482
+ def _interp_pressure(
483
+ source_time: np.ndarray,
484
+ source_pressure: np.ndarray,
485
+ target_time: np.ndarray,
486
+ ) -> np.ndarray:
487
+ """Linearly interpolate source pressure onto target time axis.
488
+
489
+ Nearest-point extrapolation at the edges (no NaN fill outside range).
490
+ """
491
+ src_t = _times_to_float(source_time)
492
+ tgt_t = _times_to_float(target_time)
493
+ valid = np.isfinite(source_pressure)
494
+ if valid.sum() < 2:
495
+ return np.full(len(target_time), np.nan)
496
+ return np.interp(
497
+ tgt_t,
498
+ src_t[valid],
499
+ source_pressure[valid],
500
+ left=source_pressure[valid][0],
501
+ right=source_pressure[valid][-1],
502
+ )
503
+
504
+
505
+ def _merge_flags(*flag_arrays: np.ndarray) -> np.ndarray:
506
+ """Merge multiple int8 flag arrays using priority ordering.
507
+
508
+ Returns element-wise flag with highest priority (worst quality).
509
+ Works on arrays of any shape (1-D time series or 2-D time×N_BINS).
510
+ """
511
+ # Lookup table: priority[flag] for flags 0-9
512
+ # _QC_PRIORITY: {9:6, 4:5, 3:4, 8:3, 2:2, 1:1, 0:0}; unlisted flags → 0
513
+ _priority = np.array([0, 1, 2, 4, 5, 0, 0, 0, 3, 6], dtype=np.int8)
514
+ result = np.asarray(flag_arrays[0], dtype=np.int8).copy()
515
+ for fa in flag_arrays[1:]:
516
+ fa = np.asarray(fa, dtype=np.int8)
517
+ replace = _priority[fa.clip(0, 9)] > _priority[result.clip(0, 9)]
518
+ result = np.where(replace, fa, result).astype(np.int8)
519
+ return result
520
+
521
+
522
+ def _deep_merge(base: Dict, override: Dict) -> Dict:
523
+ """Merge two nested dicts; override values win at the variable level."""
524
+ merged = copy.deepcopy(base)
525
+ for k, v in override.items():
526
+ if isinstance(v, dict) and k in merged and isinstance(merged[k], dict):
527
+ merged[k] = {**merged[k], **v}
528
+ else:
529
+ merged[k] = copy.deepcopy(v)
530
+ return merged
531
+
532
+
533
+ def _tilt_from_span(qc_ranges: Dict[str, Any]) -> Dict[str, float]:
534
+ """Extract tilt thresholds from a qc_ranges block if 'tilt' is present.
535
+
536
+ Converts the symmetric span format (matching all other variables) into the
537
+ threshold format used internally by _apply_tilt_qc. The upper bound of
538
+ each span is the threshold — tilt is always non-negative so the lower bound
539
+ is typically negative and only the upper bound matters:
540
+ suspect_span: [-20, 20] → suspect_threshold: 20
541
+ fail_span: [-30, 30] → fail_threshold: 30
542
+ """
543
+ out: Dict[str, float] = {}
544
+ tilt_cfg = qc_ranges.get("tilt", {})
545
+ if "suspect_span" in tilt_cfg:
546
+ out["suspect_threshold"] = float(tilt_cfg["suspect_span"][1])
547
+ if "fail_span" in tilt_cfg:
548
+ out["fail_threshold"] = float(tilt_cfg["fail_span"][1])
549
+ return out
550
+
551
+
552
+ def _load_qc_config(
553
+ mooring_cfg: Dict[str, Any],
554
+ entry: Dict[str, Any],
555
+ ) -> tuple[Dict, Dict, Dict, Dict]:
556
+ """Return QC threshold dicts for one instrument.
557
+
558
+ Returns a 4-tuple ``(gross_range, spike, tilt, flat_line)`` built from
559
+ package defaults with mooring-level and then instrument-level YAML
560
+ overrides applied (later values win).
561
+
562
+ ``gross_range`` and ``spike`` map variable names (e.g. ``"temperature"``,
563
+ ``"pressure"``) to threshold dicts understood by ``_apply_qc_tests``.
564
+ ``tilt`` holds ``suspect_threshold`` / ``fail_threshold`` in degrees,
565
+ used to flag Aquadopp velocity data when the instrument is tilted beyond
566
+ acceptable limits. ``flat_line`` maps variable names to
567
+ ``{suspect_n, fail_n, tolerance}`` dicts for the stuck-sensor test.
568
+
569
+ YAML override keys:
570
+
571
+ * ``qc_ranges`` (mooring or instrument level) — gross-range and tilt spans
572
+ * ``qc_spike`` — spike thresholds
573
+ * ``tilt_qc`` — tilt thresholds (takes precedence over ``qc_ranges.tilt``)
574
+ * ``qc_flat_line`` — stuck-sensor thresholds
575
+
576
+ ``tilt`` is stripped from the gross-range dict before returning so it is
577
+ not passed to the QARTOD gross-range test runner.
578
+ """
579
+ from . import parameters as P
580
+
581
+ gr = copy.deepcopy(P.QC_GROSS_RANGE)
582
+ sp = copy.deepcopy(P.QC_SPIKE)
583
+ tilt = copy.deepcopy(P.QC_TILT)
584
+ fl = copy.deepcopy(P.QC_FLAT_LINE)
585
+
586
+ # Mooring-level overrides
587
+ if "qc_ranges" in mooring_cfg:
588
+ gr = _deep_merge(gr, mooring_cfg["qc_ranges"])
589
+ tilt.update(_tilt_from_span(mooring_cfg["qc_ranges"]))
590
+ if "qc_spike" in mooring_cfg:
591
+ sp = _deep_merge(sp, mooring_cfg["qc_spike"])
592
+ if "tilt_qc" in mooring_cfg:
593
+ tilt.update(mooring_cfg["tilt_qc"])
594
+ if "qc_flat_line" in mooring_cfg:
595
+ fl = _deep_merge(fl, mooring_cfg["qc_flat_line"])
596
+
597
+ # Instrument-level overrides
598
+ if "qc_ranges" in entry:
599
+ gr = _deep_merge(gr, entry["qc_ranges"])
600
+ tilt.update(_tilt_from_span(entry["qc_ranges"]))
601
+ if "qc_spike" in entry:
602
+ sp = _deep_merge(sp, entry["qc_spike"])
603
+ if "tilt_qc" in entry:
604
+ tilt.update(entry["tilt_qc"])
605
+ if "qc_flat_line" in entry:
606
+ fl = _deep_merge(fl, entry["qc_flat_line"])
607
+
608
+ # Remove tilt from the gross-range dict — it is handled separately and
609
+ # is not a valid QARTOD gross-range variable.
610
+ gr.pop("tilt", None)
611
+
612
+ return gr, sp, tilt, fl
613
+
614
+
615
+ # Velocity variable names in both beam and ENU coordinate systems.
616
+ _VELOCITY_VARS = (
617
+ "velocity_beam1",
618
+ "velocity_beam2",
619
+ "velocity_beam3",
620
+ "east_velocity",
621
+ "north_velocity",
622
+ "up_velocity",
623
+ )
624
+
625
+
626
+ def _apply_tilt_qc(
627
+ ds: xr.Dataset,
628
+ tilt_cfg: Dict[str, Any],
629
+ qc_attrs: Dict[str, Any],
630
+ ) -> tuple[xr.Dataset, int, int]:
631
+ """Flag velocity variables when pitch or roll exceeds QC thresholds.
632
+
633
+ Primary path — pitch_qc / roll_qc already exist (created by the gross-range
634
+ QC step when pitch and/or roll appear in ``qc_ranges`` in the YAML):
635
+ The two flag arrays are merged element-wise (worst flag wins) and the
636
+ result is propagated to every velocity variable. Any time step where
637
+ pitch OR roll is flagged suspect (3) or bad (4) will flag the velocities
638
+ with the same severity.
639
+
640
+ Fallback path — neither pitch_qc nor roll_qc exist:
641
+ tilt is computed as max(|pitch|, |roll|) and compared against
642
+ tilt_cfg thresholds (``suspect_threshold`` / ``fail_threshold``).
643
+
644
+ In both paths ``tilt_suspect_threshold`` and ``tilt_fail_threshold`` are
645
+ written to global attrs so the report can draw reference lines on the tilt
646
+ time-series panel.
647
+
648
+ Returns (ds, n_suspect, n_bad). No-ops when both pitch and roll are absent.
649
+ """
650
+ has_pitch = "pitch" in ds.data_vars
651
+ has_roll = "roll" in ds.data_vars
652
+ if not has_pitch and not has_roll:
653
+ return ds, 0, 0
654
+
655
+ has_pitch_qc = "pitch_qc" in ds.data_vars
656
+ has_roll_qc = "roll_qc" in ds.data_vars
657
+
658
+ if has_pitch_qc or has_roll_qc:
659
+ # Primary path: merge already-computed pitch/roll QC flags.
660
+ n_time = ds["time"].size
661
+ combined = np.ones(n_time, dtype=np.int8)
662
+ if has_pitch_qc:
663
+ combined = _merge_flags(combined, ds["pitch_qc"].values.astype(np.int8))
664
+ if has_roll_qc:
665
+ combined = _merge_flags(combined, ds["roll_qc"].values.astype(np.int8))
666
+ tilt_flags = combined
667
+
668
+ # Read thresholds from whichever QC variable is available so the
669
+ # report can draw consistent reference lines on the tilt panel.
670
+ ref_attrs = ds["pitch_qc" if has_pitch_qc else "roll_qc"].attrs
671
+ suspect_thresh = float(
672
+ ref_attrs.get(
673
+ "qc_gross_range_suspect_max", tilt_cfg.get("suspect_threshold", 20.0)
674
+ )
675
+ )
676
+ fail_thresh = float(
677
+ ref_attrs.get(
678
+ "qc_gross_range_fail_max", tilt_cfg.get("fail_threshold", 30.0)
679
+ )
680
+ )
681
+ else:
682
+ # Fallback: compute from raw pitch/roll values.
683
+ n_time = ds["time"].size
684
+ pitch = ds["pitch"].values.astype(float) if has_pitch else np.zeros(n_time)
685
+ roll = ds["roll"].values.astype(float) if has_roll else np.zeros(n_time)
686
+
687
+ tilt = np.maximum(np.abs(pitch), np.abs(roll))
688
+ tilt[~np.isfinite(pitch) | ~np.isfinite(roll)] = np.nan
689
+
690
+ suspect_thresh = float(tilt_cfg.get("suspect_threshold", 20.0))
691
+ fail_thresh = float(tilt_cfg.get("fail_threshold", 30.0))
692
+
693
+ tilt_flags = np.where(
694
+ ~np.isfinite(tilt),
695
+ np.int8(9),
696
+ np.where(
697
+ tilt >= fail_thresh,
698
+ np.int8(4),
699
+ np.where(tilt >= suspect_thresh, np.int8(3), np.int8(1)),
700
+ ),
701
+ ).astype(np.int8)
702
+
703
+ n_suspect = int(np.sum(tilt_flags == 3))
704
+ n_bad = int(np.sum(tilt_flags == 4))
705
+
706
+ # Persist thresholds so the report can draw reference lines on the tilt panel.
707
+ ds.attrs["tilt_suspect_threshold"] = suspect_thresh
708
+ ds.attrs["tilt_fail_threshold"] = fail_thresh
709
+
710
+ if n_suspect == 0 and n_bad == 0:
711
+ return ds, 0, 0
712
+
713
+ vel_vars = [v for v in _VELOCITY_VARS if v in ds.data_vars]
714
+ for varname in vel_vars:
715
+ qc_varname = f"{varname}_qc"
716
+ # For ADCP, velocity is 2-D (time, N_BINS); broadcast (time,) → (time, N_BINS).
717
+ if ds[varname].values.ndim > 1:
718
+ flags_for_var = np.broadcast_to(
719
+ tilt_flags[:, np.newaxis], ds[varname].shape
720
+ ).copy()
721
+ else:
722
+ flags_for_var = tilt_flags.copy()
723
+ if qc_varname in ds:
724
+ existing = ds[qc_varname].values.astype(np.int8)
725
+ new_flags = _merge_flags(existing, flags_for_var)
726
+ else:
727
+ new_flags = flags_for_var
728
+ ds[qc_varname] = xr.Variable(
729
+ ds[varname].dims,
730
+ new_flags,
731
+ attrs={"long_name": f"quality flag for {varname}", **qc_attrs},
732
+ )
733
+
734
+ return ds, n_suspect, n_bad
735
+
736
+
737
+ def _ensure_conductivity_units(
738
+ ds: xr.Dataset,
739
+ log_fn: Any = None,
740
+ ) -> xr.Dataset:
741
+ """Convert conductivity from S/m → mS/cm if needed.
742
+
743
+ QC thresholds in parameters.QC_GROSS_RANGE are in mS/cm. Some readers
744
+ (notably sbe-ascii) write S/m; this normalises before QC is applied so
745
+ thresholds are always compared against values in the same unit.
746
+ """
747
+ if "conductivity" not in ds.data_vars:
748
+ return ds
749
+ units = ds["conductivity"].attrs.get("units", "")
750
+ if units.lower() == "s/m":
751
+ if log_fn:
752
+ log_fn(" WARNING: conductivity is in S/m — converting to mS/cm before QC")
753
+ new_data = ds["conductivity"].values * 10.0
754
+ new_attrs = dict(ds["conductivity"].attrs)
755
+ new_attrs["units"] = "mS/cm"
756
+ ds["conductivity"] = xr.Variable(
757
+ ds["conductivity"].dims, new_data, attrs=new_attrs
758
+ )
759
+ return ds
760
+
761
+
762
+ def _compute_salinity_data(
763
+ ds: xr.Dataset,
764
+ log_fn: Any = None,
765
+ ) -> xr.Dataset:
766
+ """Compute Practical Salinity (SP) data values only — no QC flags yet.
767
+
768
+ Call this BEFORE ``_apply_qc_tests`` so that salinity participates in the
769
+ gross-range QC pass and gets its threshold attrs written to ``salinity_qc``.
770
+ Call ``_merge_salinity_parent_qc`` afterward to fold in T/C/P parent flags.
771
+ """
772
+ required = {"temperature", "conductivity", "pressure"}
773
+ if not required.issubset(ds.data_vars):
774
+ return ds
775
+ if ds["conductivity"].attrs.get("units", "").lower() == "s/m":
776
+ if log_fn:
777
+ log_fn(" WARNING: salinity skipped — conductivity still in S/m")
778
+ return ds
779
+ try:
780
+ import gsw
781
+ except ImportError:
782
+ if log_fn:
783
+ log_fn(" WARNING: gsw not installed — salinity not computed")
784
+ return ds
785
+
786
+ C = ds["conductivity"].values.astype(float) # mS/cm
787
+ t = ds["temperature"].values.astype(float) # °C ITS-90
788
+ p = ds["pressure"].values.astype(float) # dbar
789
+
790
+ SP = gsw.SP_from_C(C, t, p)
791
+ ds["salinity"] = xr.Variable(
792
+ "time",
793
+ SP,
794
+ attrs={
795
+ "units": "1",
796
+ "long_name": "Practical Salinity",
797
+ "standard_name": "sea_water_practical_salinity",
798
+ "coverage_content_type": "physicalMeasurement",
799
+ "comment": "Derived from conductivity, temperature, pressure via gsw.SP_from_C",
800
+ },
801
+ )
802
+ return ds
803
+
804
+
805
+ def _merge_salinity_parent_qc(
806
+ ds: xr.Dataset,
807
+ qc_attrs: Dict[str, Any],
808
+ ) -> xr.Dataset:
809
+ """Merge parent (T/C/P) QC flags into salinity_qc after QC tests have run.
810
+
811
+ ``_apply_qc_tests`` sets salinity_qc from the gross-range test and stores
812
+ the threshold attrs needed by the report histogram. Here we additionally
813
+ fold in the worst flag from temperature_qc, conductivity_qc, and
814
+ pressure_qc so that a bad/suspect input propagates to salinity.
815
+ """
816
+ if "salinity" not in ds.data_vars:
817
+ return ds
818
+ parent_flags = []
819
+ for varname in ("temperature", "conductivity", "pressure"):
820
+ qv = f"{varname}_qc"
821
+ if qv not in ds.data_vars:
822
+ continue
823
+ f = ds[qv].values.astype(np.int8).copy()
824
+ if varname == "pressure":
825
+ # Interpolated pressure (flag 8) does not degrade salinity: T and C
826
+ # are still directly measured, so salinity is valid. Treat flag 8
827
+ # as good (1) when propagating from pressure to salinity.
828
+ f[f == 8] = np.int8(1)
829
+ parent_flags.append(f)
830
+ if not parent_flags:
831
+ return ds
832
+
833
+ parent_merged = _merge_flags(*parent_flags)
834
+ if "salinity_qc" in ds:
835
+ existing_attrs = dict(ds["salinity_qc"].attrs)
836
+ new_flags = _merge_flags(
837
+ ds["salinity_qc"].values.astype(np.int8), parent_merged
838
+ )
839
+ else:
840
+ existing_attrs = {}
841
+ new_flags = parent_merged
842
+
843
+ ds["salinity_qc"] = xr.Variable(
844
+ "time",
845
+ new_flags,
846
+ attrs={
847
+ **existing_attrs,
848
+ "long_name": "quality flag for salinity",
849
+ "comment": "QARTOD gross-range + worst of T/C/P parent flags",
850
+ **qc_attrs,
851
+ },
852
+ )
853
+ return ds
854
+
855
+
856
+ def _apply_qc_tests(
857
+ ds: xr.Dataset,
858
+ gross_range: Dict[str, Any],
859
+ spike: Dict[str, Any],
860
+ qc_attrs: Dict[str, Any],
861
+ flat_line: Optional[Dict[str, Any]] = None,
862
+ ) -> xr.Dataset:
863
+ """Apply QARTOD gross-range, spike, and flat-line tests, writing ``*_qc`` variables.
864
+
865
+ Three QARTOD tests are applied in sequence; the worst flag across all
866
+ tests wins for each sample:
867
+
868
+ * **Gross-range**: flags values outside physically plausible bounds as
869
+ SUSPECT (3) or BAD (4) — e.g. temperature below -2.5 °C or above 40 °C.
870
+ * **Spike**: flags isolated outliers that deviate from the surrounding
871
+ record by more than a threshold — e.g. a brief salinity glitch from
872
+ biofouling or a pressure transient.
873
+ * **Flat-line** (stuck-sensor): flags runs of consecutive samples where
874
+ the value does not change by more than ``tolerance`` — e.g. a pressure
875
+ sensor frozen at 0 dbar after a failure. Threshold is expressed in
876
+ sample counts (``suspect_n``, ``fail_n``) and converted to seconds
877
+ using the median sample interval. Applied by default to pressure only
878
+ (see ``parameters.QC_FLAT_LINE``).
879
+
880
+ Pre-existing ``*_qc`` values (e.g. ``pressure_qc = 8`` set by the
881
+ pressure-interpolation step in the stack) are preserved: the worst flag
882
+ across the incoming value and the new test flags is written to the output.
883
+
884
+ Provenance: the thresholds actually applied are stored as attributes on
885
+ each ``{var}_qc`` variable so the treatment can be reconstructed from
886
+ the NetCDF file alone, without re-reading the YAML:
887
+
888
+ * ``qc_gross_range_fail_min`` / ``qc_gross_range_fail_max``
889
+ * ``qc_gross_range_suspect_min`` / ``qc_gross_range_suspect_max``
890
+ * ``qc_spike_suspect_threshold`` / ``qc_spike_fail_threshold``
891
+ * ``qc_flat_line_suspect_count`` / ``qc_flat_line_fail_count``
892
+ """
893
+ from ioos_qc import qartod
894
+
895
+ _flat = flat_line or {}
896
+ test_vars = [
897
+ v
898
+ for v in ds.data_vars
899
+ if (v in gross_range or v in spike or v in _flat) and not v.endswith("_qc")
900
+ ]
901
+
902
+ for varname in test_vars:
903
+ data = ds[varname].values.copy().astype(float)
904
+ # Start with flag 1 (good) for all non-NaN, 9 (missing) for NaN
905
+ base_flags = np.where(np.isfinite(data), np.int8(1), np.int8(9))
906
+ flags_list = [base_flags]
907
+
908
+ if varname in gross_range:
909
+ cfg = gross_range[varname]
910
+ gr_flags = (
911
+ qartod.gross_range_test(
912
+ inp=data,
913
+ fail_span=tuple(cfg["fail_span"]),
914
+ suspect_span=tuple(cfg.get("suspect_span", cfg["fail_span"])),
915
+ )
916
+ .filled(9)
917
+ .astype(np.int8)
918
+ )
919
+ flags_list.append(gr_flags)
920
+
921
+ if varname in spike:
922
+ cfg = spike[varname]
923
+ sp_flags = (
924
+ qartod.spike_test(
925
+ inp=data,
926
+ suspect_threshold=cfg.get("suspect_threshold"),
927
+ fail_threshold=cfg.get("fail_threshold"),
928
+ )
929
+ .filled(9)
930
+ .astype(np.int8)
931
+ )
932
+ flags_list.append(sp_flags)
933
+
934
+ if varname in _flat and "time" in ds:
935
+ fl_cfg = _flat[varname]
936
+ time_vals = ds["time"].values
937
+ # Convert datetime64 → float seconds since epoch for ioos_qc
938
+ time_s = time_vals.astype("datetime64[s]").astype(float)
939
+ dt_s = float(np.nanmedian(np.diff(time_s))) if len(time_s) > 1 else 60.0
940
+ if dt_s <= 0:
941
+ # All timestamps are identical (or single record); ioos_qc would
942
+ # divide by zero inside flat_line_test — skip the test.
943
+ import warnings
944
+
945
+ warnings.warn(
946
+ f"flat_line_test skipped for '{varname}': median time interval is "
947
+ f"{dt_s:.1f} s (all timestamps identical?)",
948
+ RuntimeWarning,
949
+ stacklevel=2,
950
+ )
951
+ continue
952
+ suspect_n = int(fl_cfg.get("suspect_n", 3))
953
+ fail_n = int(fl_cfg.get("fail_n", 10))
954
+ try:
955
+ _fl_result = qartod.flat_line_test(
956
+ inp=data,
957
+ tinp=time_s,
958
+ suspect_threshold=int(suspect_n * dt_s),
959
+ fail_threshold=int(fail_n * dt_s),
960
+ tolerance=float(fl_cfg.get("tolerance", 0.0)),
961
+ )
962
+ except (ValueError, OverflowError) as _fl_err:
963
+ import warnings
964
+
965
+ warnings.warn(
966
+ f"flat_line_test skipped for '{varname}': {_fl_err}",
967
+ RuntimeWarning,
968
+ stacklevel=2,
969
+ )
970
+ continue
971
+ # flat_line_test may return a masked array or plain ndarray depending
972
+ # on the ioos_qc version; handle both.
973
+ fl_flags = (
974
+ _fl_result.filled(9)
975
+ if hasattr(_fl_result, "filled")
976
+ else np.where(np.isnan(_fl_result.astype(float)), 9, _fl_result)
977
+ ).astype(np.int8)
978
+ flags_list.append(fl_flags)
979
+
980
+ new_flags = _merge_flags(*flags_list)
981
+
982
+ qc_varname = f"{varname}_qc"
983
+ if qc_varname in ds:
984
+ existing = ds[qc_varname].values.astype(np.int8)
985
+ new_flags = _merge_flags(existing, new_flags)
986
+
987
+ # Store the actual thresholds applied so downstream tools (e.g. the
988
+ # report histogram) can show exactly what was used without re-reading YAML.
989
+ threshold_attrs: Dict[str, Any] = {}
990
+ if varname in gross_range:
991
+ gcfg = gross_range[varname]
992
+ if "fail_span" in gcfg:
993
+ threshold_attrs["qc_gross_range_fail_min"] = float(gcfg["fail_span"][0])
994
+ threshold_attrs["qc_gross_range_fail_max"] = float(gcfg["fail_span"][1])
995
+ if "suspect_span" in gcfg:
996
+ threshold_attrs["qc_gross_range_suspect_min"] = float(
997
+ gcfg["suspect_span"][0]
998
+ )
999
+ threshold_attrs["qc_gross_range_suspect_max"] = float(
1000
+ gcfg["suspect_span"][1]
1001
+ )
1002
+ if varname in spike:
1003
+ scfg = spike[varname]
1004
+ if "suspect_threshold" in scfg:
1005
+ threshold_attrs["qc_spike_suspect_threshold"] = float(
1006
+ scfg["suspect_threshold"]
1007
+ )
1008
+ if "fail_threshold" in scfg:
1009
+ threshold_attrs["qc_spike_fail_threshold"] = float(
1010
+ scfg["fail_threshold"]
1011
+ )
1012
+ if varname in _flat:
1013
+ fl_cfg = _flat[varname]
1014
+ threshold_attrs["qc_flat_line_suspect_count"] = int(
1015
+ fl_cfg.get("suspect_n", 3)
1016
+ )
1017
+ threshold_attrs["qc_flat_line_fail_count"] = int(fl_cfg.get("fail_n", 10))
1018
+
1019
+ ds[qc_varname] = xr.Variable(
1020
+ ds[varname].dims,
1021
+ new_flags,
1022
+ attrs={
1023
+ "long_name": f"quality flag for {varname}",
1024
+ **qc_attrs,
1025
+ **threshold_attrs,
1026
+ },
1027
+ )
1028
+
1029
+ return ds
1030
+
1031
+
1032
+ def _apply_enu_velocity_qc(
1033
+ ds: xr.Dataset,
1034
+ gr_cfg: Dict[str, Any],
1035
+ qc_attrs: Dict[str, Any],
1036
+ ) -> xr.Dataset:
1037
+ """Apply QARTOD gross-range QC to ENU velocity vars and propagate w flags.
1038
+
1039
+ Must be called after _apply_beam_to_enu (east/north/up_velocity must exist).
1040
+ No spike test is applied to velocity (burst-mode Aquadopps generate false
1041
+ positives at every burst boundary).
1042
+
1043
+ Propagates up_velocity_qc (flag 3 or 4) to east_velocity_qc and
1044
+ north_velocity_qc: if vertical velocity is implausibly large the whole
1045
+ 3-D velocity measurement is suspect/bad. Full bidirectional unification
1046
+ (large east/north flags → up_velocity_qc) is done by _unify_velocity_qc,
1047
+ which must be called after _apply_tilt_qc.
1048
+ """
1049
+ enu_gr = {
1050
+ k: v
1051
+ for k, v in gr_cfg.items()
1052
+ if k in ("east_velocity", "north_velocity", "up_velocity")
1053
+ }
1054
+ if enu_gr:
1055
+ ds = _apply_qc_tests(ds, enu_gr, {}, qc_attrs)
1056
+
1057
+ if "up_velocity_qc" in ds.data_vars:
1058
+ up_flags = ds["up_velocity_qc"].values.astype(np.int8)
1059
+ for vel_var in ("east_velocity", "north_velocity"):
1060
+ qc_varname = f"{vel_var}_qc"
1061
+ if qc_varname in ds.data_vars:
1062
+ existing = ds[qc_varname].values.astype(np.int8)
1063
+ merged = _merge_flags(existing, up_flags)
1064
+ ds[qc_varname] = xr.Variable(
1065
+ ds[vel_var].dims, merged, attrs=dict(ds[qc_varname].attrs)
1066
+ )
1067
+ return ds
1068
+
1069
+
1070
+ def _unify_velocity_qc(ds: xr.Dataset) -> xr.Dataset:
1071
+ """Unify ENU velocity QC flags so all three components share the worst flag.
1072
+
1073
+ After gross-range, up-velocity propagation, and tilt QC, each component may
1074
+ carry different flags. A large east/north velocity flags only that component;
1075
+ tilt flags all three; up-velocity flags east and north but not itself from the
1076
+ east/north gross-range test. This function computes the element-wise worst flag
1077
+ across east_velocity_qc, north_velocity_qc, and up_velocity_qc, then writes
1078
+ that combined flag back to all three. The result: masking on any single
1079
+ component produces an identical, consistent velocity mask.
1080
+ """
1081
+ keys = [
1082
+ v
1083
+ for v in ("east_velocity_qc", "north_velocity_qc", "up_velocity_qc")
1084
+ if v in ds.data_vars
1085
+ ]
1086
+ if not keys:
1087
+ return ds
1088
+ combined = np.ones(ds[keys[0]].shape, dtype=np.int8)
1089
+ for k in keys:
1090
+ combined = _merge_flags(combined, ds[k].values.astype(np.int8))
1091
+ for k in keys:
1092
+ ds[k] = xr.Variable(ds[k].dims, combined.copy(), attrs=dict(ds[k].attrs))
1093
+ return ds
1094
+
1095
+
1096
+ def _compute_adcp_bin_pressure(
1097
+ ds: xr.Dataset,
1098
+ lat: float,
1099
+ log_fn: Any = None,
1100
+ ) -> xr.Dataset:
1101
+ """Compute pressure at each ADCP bin from transducer pressure and along-beam range.
1102
+
1103
+ For each time step, adds a pressure offset per bin based on the bin's distance
1104
+ from the transducer. The offset is ``gsw.p_from_z(-range_m, lat)``, which
1105
+ approximates the hydrostatic pressure contribution of *range_m* metres of
1106
+ seawater at the given latitude.
1107
+
1108
+ **Note on ``range`` units**: the RDI WorkHorse firmware already converts slant range
1109
+ to vertical distance before writing to the raw output (using the nominal beam angle;
1110
+ see RDI ADCP Coordinate Transformation manual §4.2, Equation 8). Dolfyn reads these
1111
+ vertical distances directly, so ``range`` is already in metres of vertical depth —
1112
+ no beam-angle correction is needed.
1113
+
1114
+ **Approximation** (fixable post-OdB; see ``.claude/refactor-plan-postOdB-20260721.md``):
1115
+ ``gsw.p_from_z(-range_m, lat)`` computes the pressure of a water column of depth
1116
+ *range_m* measured from the *sea surface*, not the true pressure increment at the
1117
+ ADCP's actual depth. Error at 300 m range from a 500 m transducer is ~2–3 dbar —
1118
+ within the seabed-QC margin (~20 dbar) for current moorings.
1119
+
1120
+ The sign follows instrument orientation:
1121
+
1122
+ - Downward-looking (``orientation == "down"``): bins are deeper than the
1123
+ transducer → pressure increases with bin index.
1124
+ - Upward-looking (``orientation == "up"``): bins are shallower than the
1125
+ transducer → pressure decreases with bin index.
1126
+
1127
+ If orientation cannot be determined from dataset attributes a WARNING is emitted
1128
+ and ``"down"`` is assumed.
1129
+
1130
+ Parameters
1131
+ ----------
1132
+ ds : xr.Dataset
1133
+ Stage 3 ADCP dataset. Must contain ``pressure(time)`` in dbar (pressure at
1134
+ the transducer head) and ``range(N_BINS)`` in metres (bin centre distance from
1135
+ the transducer face). Orientation is read from the ``orientation_yaml`` global
1136
+ attribute (preferred) or ``orientation_instrument`` (fallback).
1137
+ lat : float
1138
+ Mooring latitude in decimal degrees (positive North), used by
1139
+ ``gsw.p_from_z`` for the gravitational/centrifugal correction.
1140
+ log_fn : callable, optional
1141
+ Logging callback (e.g. ``logger.info``).
1142
+
1143
+ Returns
1144
+ -------
1145
+ xr.Dataset
1146
+ Input dataset with ``bin_pressure(time, N_BINS)`` added in dbar.
1147
+ The variable's ``comment`` attribute records the formula and orientation used.
1148
+ Returns *ds* unchanged if ``pressure`` or ``range`` are absent.
1149
+
1150
+ """
1151
+ import gsw
1152
+ from . import parameters as _P
1153
+
1154
+ if "pressure" not in ds.data_vars or "range" not in ds.coords:
1155
+ if log_fn:
1156
+ log_fn(" ADCP bin_pressure: skipped (no pressure or range coord)")
1157
+ return ds
1158
+
1159
+ orientation = ds.attrs.get("orientation_yaml") or ds.attrs.get(
1160
+ "orientation_instrument"
1161
+ )
1162
+ if not orientation:
1163
+ orientation = "down"
1164
+ if log_fn:
1165
+ log_fn(
1166
+ " WARNING: ADCP orientation unknown (no orientation_yaml or "
1167
+ "orientation_instrument attr) — assuming downward-looking. "
1168
+ "bin_pressure signs may be wrong for upward-looking instruments."
1169
+ )
1170
+ sign = 1.0 if str(orientation).lower() == "down" else -1.0
1171
+
1172
+ p_trans = ds["pressure"].values.astype(float) # (time,)
1173
+ range_m = ds["range"].values.astype(float) # (N_BINS,)
1174
+
1175
+ # gsw.p_from_z(z, lat): z is depth in m (negative = below surface)
1176
+ dp = gsw.p_from_z(-range_m, lat) # (N_BINS,) pressure offset per bin
1177
+
1178
+ bin_p = p_trans[:, np.newaxis] + sign * dp[np.newaxis, :] # (time, N_BINS)
1179
+
1180
+ ds["bin_pressure"] = xr.Variable(
1181
+ ("time", _P.ADCP_BIN_DIM),
1182
+ bin_p.astype(np.float32),
1183
+ attrs={
1184
+ "long_name": "pressure at ADCP bin",
1185
+ "units": "dbar",
1186
+ "comment": (
1187
+ f"transducer pressure + gsw.p_from_z(-range, lat={lat:.2f}°)"
1188
+ f" × sign({orientation}-looking)"
1189
+ ),
1190
+ },
1191
+ )
1192
+ if log_fn:
1193
+ log_fn(
1194
+ f" ADCP bin_pressure: {ds['range'].size} bins, "
1195
+ f"orientation={orientation}, lat={lat:.2f}°"
1196
+ )
1197
+ return ds
1198
+
1199
+
1200
+ def _apply_adcp_seabed_qc(
1201
+ ds: xr.Dataset,
1202
+ water_depth_m: float,
1203
+ lat: float,
1204
+ qc_attrs: Dict[str, Any],
1205
+ fail_margin_m: float = 20.0,
1206
+ log_fn: Any = None,
1207
+ ) -> xr.Dataset:
1208
+ """Flag ADCP bins that are at or below the seabed.
1209
+
1210
+ Bins whose ``bin_pressure`` exceeds the estimated seabed pressure are flagged
1211
+ suspect (3); bins more than *fail_margin_m* metres below the seabed are flagged
1212
+ bad (4). The flag is stored as the standalone variable ``seabed_qc(time, N_BINS)``.
1213
+
1214
+ **Flag scale** (OceanSITES / QARTOD convention):
1215
+ 1 = good, 3 = suspect, 4 = bad, 9 = missing.
1216
+
1217
+ **Standalone flag**: ``seabed_qc`` is *not* merged into ``east_velocity_qc``,
1218
+ ``north_velocity_qc``, or ``up_velocity_qc``. Downstream code that needs clean
1219
+ velocity data must explicitly include all relevant flags, for example::
1220
+
1221
+ clean = (ds.east_velocity_qc == 1) & (ds.seabed_qc == 1)
1222
+
1223
+ The seabed pressure threshold is computed via ``gsw.p_from_z(-water_depth_m, lat)``,
1224
+ consistent with ``_compute_adcp_bin_pressure``.
1225
+
1226
+ Parameters
1227
+ ----------
1228
+ ds : xr.Dataset
1229
+ Stage 3 ADCP dataset containing ``bin_pressure(time, N_BINS)`` in dbar.
1230
+ water_depth_m : float
1231
+ Water depth at the mooring site in metres (from YAML ``waterdepth`` key).
1232
+ If ≤ 0 the function is a no-op.
1233
+ lat : float
1234
+ Mooring latitude in decimal degrees, used by ``gsw.p_from_z``.
1235
+ qc_attrs : dict
1236
+ OceanSITES flag attribute dict written to ``seabed_qc``.
1237
+ fail_margin_m : float
1238
+ Margin below the seabed (in metres) that separates suspect (3) from bad (4).
1239
+ Default 20 m. Bins between 0 and *fail_margin_m* below the seabed receive
1240
+ flag 3; bins more than *fail_margin_m* below receive flag 4.
1241
+ log_fn : callable, optional
1242
+ Logging callback.
1243
+
1244
+ Returns
1245
+ -------
1246
+ xr.Dataset
1247
+ Input dataset with ``seabed_qc(time, N_BINS)`` added.
1248
+ Returns *ds* unchanged if ``water_depth_m <= 0`` or ``bin_pressure`` is absent.
1249
+
1250
+ """
1251
+ if water_depth_m <= 0:
1252
+ if log_fn:
1253
+ log_fn(" seabed QC: skipped (waterdepth not set in mooring YAML)")
1254
+ return ds
1255
+ if "bin_pressure" not in ds.data_vars:
1256
+ if log_fn:
1257
+ log_fn(" seabed QC: skipped (bin_pressure not in dataset)")
1258
+ return ds
1259
+
1260
+ import gsw
1261
+ from . import parameters as _P
1262
+
1263
+ p_seabed = float(gsw.p_from_z(-water_depth_m, lat))
1264
+ p_fail = float(gsw.p_from_z(-(water_depth_m + fail_margin_m), lat))
1265
+
1266
+ bin_p = ds["bin_pressure"].values # (time, N_BINS)
1267
+ seabed_flags = np.ones(bin_p.shape, dtype=np.int8) # 1 = good
1268
+ seabed_flags = np.where(bin_p > p_seabed, np.int8(3), seabed_flags) # suspect
1269
+ seabed_flags = np.where(bin_p > p_fail, np.int8(4), seabed_flags) # bad
1270
+ seabed_flags = seabed_flags.astype(np.int8)
1271
+
1272
+ n_suspect = int(np.sum(seabed_flags == 3))
1273
+ n_bad = int(np.sum(seabed_flags == 4))
1274
+
1275
+ ds["seabed_qc"] = xr.Variable(
1276
+ ("time", _P.ADCP_BIN_DIM),
1277
+ seabed_flags,
1278
+ {
1279
+ "long_name": "Seabed proximity QC flag",
1280
+ "comment": (
1281
+ f"Bins at or below seabed ({water_depth_m:.0f} m, "
1282
+ f"p_seabed={p_seabed:.1f} dbar): flag 3 (suspect); "
1283
+ f"bins >{fail_margin_m:.0f} m below seabed "
1284
+ f"(p={p_fail:.1f} dbar): flag 4 (bad). "
1285
+ "Standalone — not merged into velocity_qc."
1286
+ ),
1287
+ **qc_attrs,
1288
+ },
1289
+ )
1290
+
1291
+ if log_fn:
1292
+ log_fn(
1293
+ f" seabed QC: water_depth={water_depth_m:.0f} m, "
1294
+ f"p_seabed={p_seabed:.1f} dbar, "
1295
+ f"p_fail={p_fail:.1f} dbar (+{fail_margin_m:.0f} m), "
1296
+ f"suspect={n_suspect}, bad={n_bad}"
1297
+ )
1298
+ return ds
1299
+
1300
+
1301
+ def _apply_adcp_surface_qc(
1302
+ ds: xr.Dataset,
1303
+ lat: float,
1304
+ qc_attrs: Dict[str, Any],
1305
+ suspect_margin_m: float = 20.0,
1306
+ log_fn: Any = None,
1307
+ ) -> xr.Dataset:
1308
+ """Flag ADCP bins that are at or above the sea surface.
1309
+
1310
+ Bins whose ``bin_pressure`` is ≤ 0 dbar are flagged bad (4); bins within
1311
+ *suspect_margin_m* metres of the surface (0 < bin_pressure < p_suspect) are
1312
+ flagged suspect (3). The result is stored as the standalone variable
1313
+ ``surface_qc(time, N_BINS)``.
1314
+
1315
+ This catches upward-looking ADCP bins that extend above the water surface
1316
+ when the instrument range exceeds the distance to the surface. Symmetric
1317
+ counterpart to ``_apply_adcp_seabed_qc``.
1318
+
1319
+ **Flag scale** (OceanSITES / QARTOD convention):
1320
+ 1 = good, 3 = suspect, 4 = bad.
1321
+
1322
+ **Standalone flag**: ``surface_qc`` is *not* merged into the velocity QC
1323
+ variables. Downstream consumers must combine flags explicitly::
1324
+
1325
+ clean = (ds.east_velocity_qc == 1) & (ds.surface_qc == 1) & (ds.seabed_qc == 1)
1326
+
1327
+ Parameters
1328
+ ----------
1329
+ ds : xr.Dataset
1330
+ Stage 3 ADCP dataset containing ``bin_pressure(time, N_BINS)`` in dbar.
1331
+ lat : float
1332
+ Mooring latitude in decimal degrees, used by ``gsw.p_from_z``.
1333
+ qc_attrs : dict
1334
+ OceanSITES flag attribute dict written to ``surface_qc``.
1335
+ suspect_margin_m : float
1336
+ Depth (m) below the surface that defines the suspect zone. Bins with
1337
+ 0 < bin_pressure < p_from_z(-suspect_margin_m) receive flag 3; bins
1338
+ with bin_pressure ≤ 0 receive flag 4. Default 20 m.
1339
+ log_fn : callable, optional
1340
+ Logging callback.
1341
+
1342
+ Returns
1343
+ -------
1344
+ xr.Dataset
1345
+ Input dataset with ``surface_qc(time, N_BINS)`` added.
1346
+ Returns *ds* unchanged if ``bin_pressure`` is absent.
1347
+
1348
+ """
1349
+ if "bin_pressure" not in ds.data_vars:
1350
+ if log_fn:
1351
+ log_fn(" surface QC: skipped (bin_pressure not in dataset)")
1352
+ return ds
1353
+
1354
+ import gsw
1355
+ from . import parameters as _P
1356
+
1357
+ p_suspect = float(gsw.p_from_z(-suspect_margin_m, lat))
1358
+
1359
+ bin_p = ds["bin_pressure"].values # (time, N_BINS)
1360
+ surface_flags = np.ones(bin_p.shape, dtype=np.int8) # 1 = good
1361
+ surface_flags = np.where(bin_p < p_suspect, np.int8(3), surface_flags) # suspect
1362
+ surface_flags = np.where(bin_p <= 0.0, np.int8(4), surface_flags) # bad
1363
+
1364
+ n_suspect = int(np.sum(surface_flags == 3))
1365
+ n_bad = int(np.sum(surface_flags == 4))
1366
+
1367
+ ds["surface_qc"] = xr.Variable(
1368
+ ("time", _P.ADCP_BIN_DIM),
1369
+ surface_flags,
1370
+ {
1371
+ "long_name": "Sea surface proximity QC flag",
1372
+ "comment": (
1373
+ f"Bins within {suspect_margin_m:.0f} m of sea surface "
1374
+ f"(bin_pressure < {p_suspect:.1f} dbar): flag 3 (suspect); "
1375
+ f"bins at or above surface (bin_pressure ≤ 0 dbar): flag 4 (bad). "
1376
+ "Standalone — not merged into velocity_qc."
1377
+ ),
1378
+ **qc_attrs,
1379
+ },
1380
+ )
1381
+
1382
+ if log_fn:
1383
+ log_fn(
1384
+ f" surface QC: p_suspect={p_suspect:.1f} dbar ({suspect_margin_m:.0f} m), "
1385
+ f"suspect={n_suspect}, bad={n_bad}"
1386
+ )
1387
+ return ds
1388
+
1389
+
1390
+ def _apply_adcp_velocity_qc(
1391
+ ds: xr.Dataset,
1392
+ gr_cfg: Dict[str, Any],
1393
+ prcnt_gd_bad: float,
1394
+ prcnt_gd_suspect: float,
1395
+ error_vel_threshold: float,
1396
+ qc_attrs: Dict[str, Any],
1397
+ log_fn: Any = None,
1398
+ ) -> xr.Dataset:
1399
+ """Apply QC to ADCP 2D velocity variables (time × N_BINS).
1400
+
1401
+ Each QC criterion produces its **own standalone variable**; flags are
1402
+ **not** merged across variables. Downstream users combine them to mask
1403
+ data, e.g.::
1404
+
1405
+ good = (
1406
+ (ds.east_velocity_qc == 1)
1407
+ & (ds.percent_good_qc == 1)
1408
+ & (ds.error_velocity_qc == 1)
1409
+ & (ds.seabed_qc == 1)
1410
+ & (ds.surface_qc == 1)
1411
+ )
1412
+
1413
+ QC variables produced
1414
+ ---------------------
1415
+ ``east/north/up_velocity_qc``
1416
+ Gross-range flag on the velocity component value itself (same
1417
+ fail_span / suspect_span thresholds as point instruments). Flag 1
1418
+ means the velocity value is within the accepted range; it says nothing
1419
+ about acoustic quality.
1420
+
1421
+ ``percent_good_qc(time, N_BINS)``
1422
+ RDI ADCPs write four percent-good columns per ensemble per bin:
1423
+
1424
+ ======= =============================================================
1425
+ Col 0 % pings accepted as 3-beam solutions (one beam rejected)
1426
+ Col 1 % pings rejected by the error-velocity threshold
1427
+ Col 2 % pings rejected by low correlation or low amplitude
1428
+ **Col 3** **% pings accepted as 4-beam solutions ← quality metric**
1429
+ ======= =============================================================
1430
+
1431
+ Column 3 (4-beam solutions) is the relevant quality indicator.
1432
+ Averaging all four columns is wrong: when data is perfect, col 3 ≈
1433
+ 100 % and cols 0–2 ≈ 0 %, giving a mean of ~25 %, which falls below
1434
+ any reasonable suspect threshold and flags everything. This function
1435
+ therefore uses column 3 alone (falling back to the column mean for
1436
+ non-4-beam ADCPs that store fewer than 4 columns).
1437
+
1438
+ Flags: col3 < *prcnt_gd_bad* → bad (4);
1439
+ col3 < *prcnt_gd_suspect* → suspect (3); otherwise good (1).
1440
+
1441
+ ``error_velocity_qc(time, N_BINS)``
1442
+ For a 4-beam ADCP the error velocity is the difference between two
1443
+ independent estimates of vertical velocity from opposite beam pairs.
1444
+ It is zero for a perfect measurement; large values indicate beam
1445
+ decorrelation (e.g. fish, bubbles, mooring motion).
1446
+ |error_velocity| > *error_vel_threshold* → bad (4).
1447
+
1448
+ Parameters
1449
+ ----------
1450
+ ds : xr.Dataset
1451
+ Stage 2 dataset with 2-D ADCP velocity variables.
1452
+ gr_cfg : dict
1453
+ Gross-range config (same format as ``_load_qc_config`` returns).
1454
+ prcnt_gd_bad : float
1455
+ 4-beam percent good below this → flag 4 (bad). Percent.
1456
+ prcnt_gd_suspect : float
1457
+ 4-beam percent good below this (but above prcnt_gd_bad) → flag 3
1458
+ (suspect). Percent.
1459
+ error_vel_threshold : float
1460
+ |error_velocity| above this → flag 4 (bad). m s⁻¹.
1461
+ qc_attrs : dict
1462
+ OceanSITES flag attribute dict written to every ``*_qc`` variable.
1463
+ log_fn : callable, optional
1464
+ Logging callback.
1465
+
1466
+ Returns
1467
+ -------
1468
+ xr.Dataset
1469
+ Input dataset with the following standalone QC variables added (where their
1470
+ parent data variables are present):
1471
+
1472
+ - ``east_velocity_qc(time, N_BINS)`` — gross-range flag on east velocity
1473
+ - ``north_velocity_qc(time, N_BINS)`` — gross-range flag on north velocity
1474
+ - ``up_velocity_qc(time, N_BINS)`` — gross-range flag on up velocity
1475
+ - ``percent_good_qc(time, N_BINS)`` — 4-beam percent-good flag
1476
+ - ``error_velocity_qc(time, N_BINS)`` — error velocity magnitude flag
1477
+
1478
+ All flags use the OceanSITES scale: 1 = good, 3 = suspect, 4 = bad, 9 = missing.
1479
+ Variables are standalone; no merging across criteria is performed.
1480
+
1481
+ """
1482
+ # 1. Gross-range on each ENU velocity component independently.
1483
+ # Flags only reflect whether that component's value is out of range.
1484
+ for varname in ("east_velocity", "north_velocity", "up_velocity"):
1485
+ if varname not in ds.data_vars:
1486
+ continue
1487
+ cfg = gr_cfg.get(varname)
1488
+ if not cfg:
1489
+ continue
1490
+ data = ds[varname].values.astype(float)
1491
+ fail_min, fail_max = cfg["fail_span"]
1492
+ susp_min, susp_max = cfg.get("suspect_span", cfg["fail_span"])
1493
+ flags = np.where(
1494
+ ~np.isfinite(data),
1495
+ np.int8(9),
1496
+ np.where(
1497
+ (data < fail_min) | (data > fail_max),
1498
+ np.int8(4),
1499
+ np.where((data < susp_min) | (data > susp_max), np.int8(3), np.int8(1)),
1500
+ ),
1501
+ ).astype(np.int8)
1502
+ qc_var = f"{varname}_qc"
1503
+ threshold_attrs: Dict[str, Any] = {
1504
+ "qc_gross_range_fail_min": float(fail_min),
1505
+ "qc_gross_range_fail_max": float(fail_max),
1506
+ "qc_gross_range_suspect_min": float(susp_min),
1507
+ "qc_gross_range_suspect_max": float(susp_max),
1508
+ }
1509
+ ds[qc_var] = xr.Variable(
1510
+ ds[varname].dims,
1511
+ flags,
1512
+ attrs={
1513
+ "long_name": f"quality flag for {varname}",
1514
+ **qc_attrs,
1515
+ **threshold_attrs,
1516
+ },
1517
+ )
1518
+
1519
+ # 2. percent_good QC — standalone variable, NOT merged into velocity_qc.
1520
+ # RDI ADCPs store four percent-good columns per bin:
1521
+ # [0] 3-beam solutions [1] rejected (error vel)
1522
+ # [2] rejected (low corr/amp) [3] 4-beam solutions ← the useful one
1523
+ # Using column 3 only; averaging all columns gives ~25% even for
1524
+ # perfect data (100% 4-beam, 0% others), which would flag everything.
1525
+ if "percent_good" in ds.data_vars:
1526
+ pg = ds["percent_good"].values.astype(float) # (time, N_BINS, beam)
1527
+ if pg.ndim == 3 and pg.shape[2] >= 4:
1528
+ pg_col3 = pg[:, :, 3] # 4-beam solutions column
1529
+ else:
1530
+ pg_col3 = np.nanmean(pg, axis=-1) # fallback for non-4-beam ADCPs
1531
+ pg_flags = np.where(
1532
+ ~np.isfinite(pg_col3),
1533
+ np.int8(9),
1534
+ np.where(
1535
+ pg_col3 < prcnt_gd_bad,
1536
+ np.int8(4),
1537
+ np.where(pg_col3 < prcnt_gd_suspect, np.int8(3), np.int8(1)),
1538
+ ),
1539
+ ).astype(np.int8)
1540
+ n_bad_pg = int(np.sum(pg_flags == 4))
1541
+ n_susp_pg = int(np.sum(pg_flags == 3))
1542
+ if log_fn:
1543
+ log_fn(
1544
+ f" ADCP percent_good QC (col-3 4-beam): bad={n_bad_pg}, suspect={n_susp_pg} "
1545
+ f"(thresholds: bad<{prcnt_gd_bad}%, suspect<{prcnt_gd_suspect}%)"
1546
+ )
1547
+ from . import parameters as _P
1548
+
1549
+ ds["percent_good_qc"] = xr.Variable(
1550
+ ("time", _P.ADCP_BIN_DIM),
1551
+ pg_flags,
1552
+ attrs={
1553
+ "long_name": "QC flag for ADCP percent good (4-beam solutions, column 3)",
1554
+ "comment": (
1555
+ f"Based on RDI percent_good column 3 (4-beam solutions). "
1556
+ f"bad<{prcnt_gd_bad}%, suspect<{prcnt_gd_suspect}%. "
1557
+ "Standalone — not merged into velocity_qc."
1558
+ ),
1559
+ **qc_attrs,
1560
+ },
1561
+ )
1562
+
1563
+ # 3. error_velocity QC — standalone variable, NOT merged into velocity_qc.
1564
+ if "error_velocity" in ds.data_vars:
1565
+ ev = ds["error_velocity"].values.astype(float) # (time, N_BINS)
1566
+ ev_flags = np.where(
1567
+ ~np.isfinite(ev),
1568
+ np.int8(9),
1569
+ np.where(np.abs(ev) > error_vel_threshold, np.int8(4), np.int8(1)),
1570
+ ).astype(np.int8)
1571
+ n_ev_bad = int(np.sum(ev_flags == 4))
1572
+ if log_fn:
1573
+ log_fn(
1574
+ f" ADCP error_velocity QC: bad={n_ev_bad} "
1575
+ f"(threshold: |ev|>{error_vel_threshold:.2f} m s-1). Standalone."
1576
+ )
1577
+ ds["error_velocity_qc"] = xr.Variable(
1578
+ ds["error_velocity"].dims,
1579
+ ev_flags,
1580
+ attrs={
1581
+ "long_name": "QC flag for error velocity",
1582
+ "comment": (
1583
+ f"|error_velocity| > {error_vel_threshold:.2f} m s-1 → bad (4). "
1584
+ "Standalone — not merged into velocity_qc."
1585
+ ),
1586
+ "qc_threshold_fail_m_s": float(error_vel_threshold),
1587
+ **qc_attrs,
1588
+ },
1589
+ )
1590
+
1591
+ return ds
1592
+
1593
+
1594
+ def _derive_oxygen_saturation(ds: "xr.Dataset") -> "xr.Dataset":
1595
+ """Compute O2 % saturation and AOU from dissolved_oxygen, temperature, salinity, pressure.
1596
+
1597
+ Requires dissolved_oxygen (µmol/L), temperature (°C), salinity (PSU), and pressure
1598
+ (dbar) all present in *ds*. Returns *ds* unchanged when any variable is missing or
1599
+ when gsw is unavailable.
1600
+
1601
+ Unit conversion uses in-situ **seawater** density from ``gsw.rho(SA, CT, p)``
1602
+ (kg m⁻³) — NOT freshwater density (~1000 kg m⁻³). Seawater density at typical
1603
+ mooring conditions is ~1025–1028 kg m⁻³, giving a ~2.5 % correction relative to
1604
+ freshwater. This correction is oceanographically significant and must not be skipped.
1605
+
1606
+ lon/lat for ``gsw.SA_from_SP`` are taken from global attrs (default 0.0 if absent;
1607
+ the SA error from a wrong position is typically < 0.05 g kg⁻¹ at open-ocean sites).
1608
+
1609
+ Derived variables stored
1610
+ ------------------------
1611
+ oxygen_saturation_pct : %
1612
+ 100 × O2_measured(µmol kg⁻¹) / O2sol(µmol kg⁻¹)
1613
+ where O2sol is from ``gsw.O2sol_SP_pt(SP, pt)``.
1614
+ apparent_oxygen_utilization : µmol kg⁻¹
1615
+ O2sol − O2_measured (positive = oxygen-depleted water).
1616
+ """
1617
+ required = {"dissolved_oxygen", "temperature", "salinity", "pressure"}
1618
+ if not required.issubset(ds.data_vars):
1619
+ return ds
1620
+ try:
1621
+ import gsw
1622
+ except ImportError:
1623
+ return ds
1624
+
1625
+ O2_L = ds["dissolved_oxygen"].values.astype(float)
1626
+ t = ds["temperature"].values.astype(float)
1627
+ SP = ds["salinity"].values.astype(float)
1628
+ p = ds["pressure"].values.astype(float)
1629
+ try:
1630
+ lon = float(ds.attrs.get("longitude", 0.0))
1631
+ except (ValueError, TypeError):
1632
+ lon = 0.0
1633
+ try:
1634
+ lat = float(ds.attrs.get("latitude", 0.0))
1635
+ except (ValueError, TypeError):
1636
+ lat = 0.0
1637
+
1638
+ # Mask already-flagged dissolved oxygen so bad values produce NaN saturation
1639
+ # rather than spurious negative percentages that would then back-propagate.
1640
+ if "dissolved_oxygen_qc" in ds.data_vars:
1641
+ _do_qc = ds["dissolved_oxygen_qc"].values.astype(np.int8)
1642
+ O2_L = O2_L.copy()
1643
+ O2_L[np.isin(_do_qc, [4, 9])] = np.nan
1644
+
1645
+ SA = gsw.SA_from_SP(SP, p, lon, lat)
1646
+ CT = gsw.CT_from_t(SA, t, p)
1647
+ pt = gsw.pt0_from_t(SA, t, p)
1648
+
1649
+ O2sol = gsw.O2sol_SP_pt(SP, pt) # µmol/kg at equilibrium
1650
+ rho_sw = gsw.rho(SA, CT, p) # kg/m³ in-situ seawater density
1651
+ O2_kg = O2_L / (rho_sw / 1000.0) # µmol/kg measured (divides by kg/L)
1652
+
1653
+ pct_sat = 100.0 * O2_kg / O2sol
1654
+ aou = O2sol - O2_kg
1655
+
1656
+ # Back-flag: pct_sat < 0 is physically impossible (negative dissolved oxygen).
1657
+ # Set saturation to NaN and flag the underlying dissolved_oxygen as bad (4).
1658
+ neg_sat = np.isfinite(pct_sat) & (pct_sat < 0.0)
1659
+ n_neg = int(np.sum(neg_sat))
1660
+ if n_neg > 0:
1661
+ pct_sat[neg_sat] = np.nan
1662
+ aou[neg_sat] = np.nan
1663
+ if "dissolved_oxygen_qc" in ds.data_vars:
1664
+ do_qc = ds["dissolved_oxygen_qc"].values.astype(np.int8).copy()
1665
+ do_qc[neg_sat] = np.int8(4)
1666
+ ds["dissolved_oxygen_qc"] = xr.Variable(
1667
+ ds["dissolved_oxygen_qc"].dims,
1668
+ do_qc,
1669
+ attrs=dict(ds["dissolved_oxygen_qc"].attrs),
1670
+ )
1671
+
1672
+ ds = ds.assign(
1673
+ oxygen_saturation_pct=xr.Variable(
1674
+ "time",
1675
+ pct_sat.astype(np.float32),
1676
+ {
1677
+ "long_name": "Dissolved Oxygen Percent Saturation",
1678
+ "units": "%",
1679
+ "comment": (
1680
+ "100 × O2_meas / O2sol. O2_meas converted from µmol/L to µmol/kg "
1681
+ "using in-situ seawater density gsw.rho(SA,CT,p); "
1682
+ "O2sol = gsw.O2sol_SP_pt(SP, pt0). "
1683
+ f"Back-flagged {n_neg} sample(s) where pct_sat < 0 as bad (flag 4)."
1684
+ if n_neg
1685
+ else (
1686
+ "100 × O2_meas / O2sol. O2_meas converted from µmol/L to µmol/kg "
1687
+ "using in-situ seawater density gsw.rho(SA,CT,p); "
1688
+ "O2sol = gsw.O2sol_SP_pt(SP, pt0)."
1689
+ )
1690
+ ),
1691
+ },
1692
+ ),
1693
+ apparent_oxygen_utilization=xr.Variable(
1694
+ "time",
1695
+ aou.astype(np.float32),
1696
+ {
1697
+ "long_name": "Apparent Oxygen Utilization",
1698
+ "standard_name": "apparent_oxygen_utilization",
1699
+ "units": "umol/kg",
1700
+ "comment": (
1701
+ "AOU = O2sol - O2_meas (µmol/kg); positive = undersaturated "
1702
+ "(oxygen consumed). Uses in-situ seawater density for µmol/L → µmol/kg."
1703
+ ),
1704
+ },
1705
+ ),
1706
+ )
1707
+ return ds
1708
+
1709
+
1710
+ class Stage3Processor:
1711
+ """Pressure interpolation + QARTOD QC for all mooring instruments."""
1712
+
1713
+ def __init__(
1714
+ self,
1715
+ base_dir: Optional[str] = None,
1716
+ *,
1717
+ proc_dir: Optional[str] = None,
1718
+ ) -> None:
1719
+ """Apply QC flags, coordinate rotation, and derived variables to mooring data.
1720
+
1721
+ Reads stage2 CF-NetCDF files and applies: QARTOD gross-range and spike QC
1722
+ flags for scalar sensors; pressure interpolation for instruments that lack a
1723
+ pressure sensor; BEAM→XYZ→ENU coordinate rotation for Aquadopp current meters
1724
+ (using the instrument T matrix and heading/pitch/roll); and magnetic declination
1725
+ correction so velocities are referenced to true north. Writes
1726
+ ``{mooring}_{serial}_stage3.nc``.
1727
+
1728
+ Parameters
1729
+ ----------
1730
+ base_dir : str, optional
1731
+ Legacy: cruise-level base directory containing a ``proc/`` subdirectory.
1732
+ proc_dir : str, optional
1733
+ Cruise-level processed output directory. Pipeline appends ``/{mooring}/``.
1734
+
1735
+ """
1736
+ if base_dir is not None:
1737
+ self.base_dir: Optional[Path] = Path(base_dir)
1738
+ self._proc_dir: Optional[Path] = None
1739
+ self._legacy = True
1740
+ else:
1741
+ self.base_dir = None
1742
+ self._proc_dir = Path(proc_dir) if proc_dir else None
1743
+ self._legacy = False
1744
+ self.log_file = None
1745
+
1746
+ def _setup_logging(self, mooring_name: str, output_path: Path) -> None:
1747
+ from .logger import setup_stage_logging
1748
+
1749
+ self.log_file = setup_stage_logging(mooring_name, "stage3", output_path)
1750
+
1751
+ def _log(self, *args: Any, **kwargs: Any) -> None:
1752
+ print(*args, **kwargs)
1753
+ if self.log_file:
1754
+ try:
1755
+ with open(self.log_file, "a") as f:
1756
+ print(*args, **kwargs, file=f)
1757
+ except OSError:
1758
+ pass
1759
+
1760
+ def _get_proc_dir(self, mooring_name: str) -> Path:
1761
+ if not self._legacy and self._proc_dir is not None:
1762
+ return self._proc_dir / mooring_name
1763
+ proc = self.base_dir / "proc"
1764
+ if not proc.is_dir():
1765
+ legacy = self.base_dir / "moor" / "proc"
1766
+ proc = legacy if legacy.is_dir() else proc
1767
+ return proc / mooring_name
1768
+
1769
+ # ------------------------------------------------------------------
1770
+ def process_mooring(
1771
+ self,
1772
+ mooring_name: str,
1773
+ serials: Optional[List[str]] = None,
1774
+ force: bool = False,
1775
+ dry_run: bool = False,
1776
+ ) -> bool:
1777
+ """Run Stage 3 QC and pressure interpolation for all instruments on a mooring."""
1778
+ proc_dir = self._get_proc_dir(mooring_name)
1779
+ if not proc_dir.exists():
1780
+ print(f"ERROR: Processing directory not found: {proc_dir}")
1781
+ return False
1782
+
1783
+ self._setup_logging(mooring_name, proc_dir)
1784
+ mode = "DRY RUN — " if dry_run else ""
1785
+ self._log(f"{mode}Stage 3 (pressure interpolation + QC) for: {mooring_name}")
1786
+
1787
+ config_file = proc_dir / f"{mooring_name}.mooring.yaml"
1788
+ if not config_file.exists():
1789
+ self._log(f"ERROR: Config not found: {config_file}")
1790
+ return False
1791
+
1792
+ with open(config_file) as f:
1793
+ mooring_config = yaml.safe_load(f)
1794
+
1795
+ instrument_list = list(
1796
+ mooring_config.get("clamp", mooring_config.get("instruments", []))
1797
+ )
1798
+ instrument_list += extract_inline_instruments(mooring_config.get("inline", []))
1799
+
1800
+ # ── Mooring location for BEAM→ENU declination ──────────────────
1801
+ from .mooring_level import _parse_latlon_with_source
1802
+
1803
+ _mooring_lat, _mooring_lon, _latlon_source = _parse_latlon_with_source(
1804
+ mooring_config
1805
+ )
1806
+
1807
+ _water_depth_m = float(mooring_config.get("waterdepth") or 0.0)
1808
+
1809
+ # ── Build instrument table ──────────────────────────────────────
1810
+ instruments: List[Dict[str, Any]] = []
1811
+ for entry in instrument_list:
1812
+ if not isinstance(entry, dict):
1813
+ continue
1814
+ serial = _safe_serial(entry.get("serial", ""))
1815
+ instr_type = entry.get("instrument", "unknown")
1816
+ if entry.get("skip"):
1817
+ reason = entry.get("skip_reason", "marked skip:true in YAML")
1818
+ self._log(f" SKIP {instr_type} {serial}: {reason}")
1819
+ continue
1820
+ hab = entry.get("hab")
1821
+ if hab is None:
1822
+ self._log(f" WARNING: serial {serial} has no 'hab' — skipping")
1823
+ continue
1824
+ nc_path = proc_dir / instr_type / f"{mooring_name}_{serial}_stage2.nc"
1825
+ if not nc_path.exists():
1826
+ continue
1827
+ qc_flags = {
1828
+ key[:-3]: int(val)
1829
+ for key, val in entry.items()
1830
+ if key.endswith("_qc") and isinstance(val, int)
1831
+ }
1832
+ gr_cfg, sp_cfg, tilt_cfg, fl_cfg = _load_qc_config(mooring_config, entry)
1833
+ # Parse optional hab_segments: list of {from: ISO, hab: float}
1834
+ # Stored as sorted list of (np.datetime64, float) breakpoints.
1835
+ raw_segs = entry.get("hab_segments", [])
1836
+ hab_segments: List[tuple] = []
1837
+ for seg in raw_segs:
1838
+ try:
1839
+ bp = np.datetime64(seg["from"], "ns")
1840
+ hab_segments.append((bp, float(seg["hab"])))
1841
+ except (KeyError, ValueError) as _e:
1842
+ self._log(
1843
+ f" WARNING: serial {serial} hab_segments entry invalid"
1844
+ f" ({seg}): {_e} — skipped"
1845
+ )
1846
+ hab_segments.sort(key=lambda x: x[0])
1847
+
1848
+ instruments.append(
1849
+ {
1850
+ "serial": serial,
1851
+ "instrument": instr_type,
1852
+ "hab": float(hab),
1853
+ "hab_segments": hab_segments,
1854
+ "nc_path": nc_path,
1855
+ "qc_flags": qc_flags,
1856
+ "gross_range": gr_cfg,
1857
+ "spike": sp_cfg,
1858
+ "tilt": tilt_cfg,
1859
+ "flat_line": fl_cfg,
1860
+ "entry": entry,
1861
+ "lat": _mooring_lat,
1862
+ "lon": _mooring_lon,
1863
+ "latlon_source": _latlon_source,
1864
+ "water_depth_m": _water_depth_m,
1865
+ }
1866
+ )
1867
+
1868
+ if not instruments:
1869
+ self._log("ERROR: No _stage2.nc files found with hab values")
1870
+ return False
1871
+
1872
+ # Filter by serial early for targeted reruns
1873
+ if serials:
1874
+ safe_req = {_safe_serial(s) for s in serials}
1875
+ instruments = [i for i in instruments if i["serial"] in safe_req]
1876
+ if not instruments:
1877
+ self._log(
1878
+ f" No instruments found matching serial(s): {', '.join(serials)}"
1879
+ )
1880
+ return True
1881
+
1882
+ # ── Scan what pressure variables each instrument has ────────────
1883
+ def _find_pressure_var(data_vars: set) -> Optional[str]:
1884
+ if "pressure" in data_vars:
1885
+ return "pressure"
1886
+ for cand in sorted(
1887
+ v
1888
+ for v in data_vars
1889
+ if v.startswith("pressure_") and not v.endswith("_qc")
1890
+ ):
1891
+ return cand
1892
+ return None
1893
+
1894
+ for info in instruments:
1895
+ try:
1896
+ with xr.open_dataset(info["nc_path"], decode_timedelta=False) as _ds:
1897
+ info["data_vars"] = set(_ds.data_vars)
1898
+ except Exception as e: # noqa: BLE001 — missing file skipped; mooring continues
1899
+ self._log(f" WARNING: Could not open {info['nc_path'].name}: {e}")
1900
+ info["data_vars"] = set()
1901
+ info["pressure_var"] = _find_pressure_var(info["data_vars"])
1902
+ info["has_pressure"] = info["pressure_var"] is not None
1903
+ info["ds"] = None
1904
+
1905
+ for varname, qc_val in info["qc_flags"].items():
1906
+ if varname not in info["data_vars"]:
1907
+ self._log(
1908
+ f" WARNING: serial {info['serial']}: "
1909
+ f"'{varname}_qc: {qc_val}' in YAML but '{varname}' "
1910
+ f"not found in {info['nc_path'].name} — ignored"
1911
+ )
1912
+
1913
+ def pressure_bad(info: Dict[str, Any]) -> bool:
1914
+ return info["qc_flags"].get("pressure", 0) >= 3
1915
+
1916
+ sources = [i for i in instruments if i["has_pressure"] and not pressure_bad(i)]
1917
+ targets = [i for i in instruments if not i["has_pressure"] or pressure_bad(i)]
1918
+
1919
+ if not sources and targets:
1920
+ self._log(
1921
+ "WARNING: No reliable pressure sources found — pressure interpolation skipped"
1922
+ )
1923
+
1924
+ self._log(
1925
+ f" Pressure sources ({len(sources)}): "
1926
+ + ", ".join(
1927
+ f"{s['instrument']} {s['serial']} hab={s['hab']:.1f}m" for s in sources
1928
+ )
1929
+ )
1930
+ if targets:
1931
+ self._log(
1932
+ f" Pressure targets ({len(targets)}): "
1933
+ + ", ".join(
1934
+ f"{t['instrument']} {t['serial']} hab={t['hab']:.1f}m"
1935
+ + (
1936
+ f" [pressure_qc={t['qc_flags']['pressure']}→replace]"
1937
+ if pressure_bad(t)
1938
+ else " [no pressure]"
1939
+ )
1940
+ for t in targets
1941
+ )
1942
+ )
1943
+
1944
+ if dry_run:
1945
+ for info in instruments:
1946
+ is_target = info in targets
1947
+ l3 = info["nc_path"].with_name(
1948
+ info["nc_path"].name.replace("_stage2.nc", "_stage3.nc")
1949
+ )
1950
+ action = "interpolate pressure + QC" if is_target else "QC only"
1951
+ self._log(
1952
+ f" {info['serial']} ({info['instrument']}): {action} → {l3.name}"
1953
+ )
1954
+ self._log("DRY RUN complete")
1955
+ return True
1956
+
1957
+ # ── Load source datasets for pressure interpolation ─────────────
1958
+ for src in sources:
1959
+ try:
1960
+ src["ds"] = xr.open_dataset(
1961
+ src["nc_path"], decode_timedelta=False
1962
+ ).load()
1963
+ except Exception as e: # noqa: BLE001 — one bad source must not abort pressure interp
1964
+ self._log(
1965
+ f" WARNING: Could not load source {src['nc_path'].name}: {e}"
1966
+ )
1967
+ src["ds"] = None
1968
+
1969
+ # ── Process every instrument ────────────────────────────────────
1970
+ from . import parameters as P
1971
+
1972
+ _qc_attrs = {
1973
+ "flag_values": P.QC_FLAG_VALUES,
1974
+ "flag_meanings": P.QC_FLAG_MEANINGS,
1975
+ "conventions": P.QC_CONVENTION,
1976
+ }
1977
+
1978
+ success_count = 0
1979
+ for info in instruments:
1980
+ ok = self._process_instrument(
1981
+ info, sources, targets, _qc_attrs, force=force
1982
+ )
1983
+ if ok:
1984
+ success_count += 1
1985
+
1986
+ for info in instruments:
1987
+ if info.get("ds") is not None:
1988
+ info["ds"].close()
1989
+
1990
+ self._log(
1991
+ f"Stage 3 complete: {success_count}/{len(instruments)} instruments written"
1992
+ )
1993
+ return success_count == len(instruments)
1994
+
1995
+ # ------------------------------------------------------------------
1996
+ def _process_instrument(
1997
+ self,
1998
+ info: Dict[str, Any],
1999
+ sources: List[Dict[str, Any]],
2000
+ targets: List[Dict[str, Any]],
2001
+ qc_attrs: Dict[str, Any],
2002
+ force: bool = False,
2003
+ ) -> bool:
2004
+ """Apply Stage 3 processing to one instrument's Stage 2 NetCDF.
2005
+
2006
+ Steps applied (where applicable to the instrument type):
2007
+
2008
+ 1. Pressure interpolation — targets without a reliable pressure sensor
2009
+ have pressure interpolated from *sources* (neighbours on the mooring).
2010
+ 2. Conductivity unit normalisation and practical salinity computation
2011
+ (CTD/microcat instruments only).
2012
+ 3. QARTOD QC tests (gross-range, spike) using thresholds from *qc_attrs*.
2013
+ 4. BEAM→ENU or XYZ→ENU coordinate transformation with magnetic declination
2014
+ correction (current meters / Aquadopp).
2015
+ 5. Tilt QC — velocity flagged suspect/bad when pitch or roll exceed
2016
+ configured thresholds.
2017
+ 6. History attribute updated with all processing steps applied.
2018
+
2019
+ Writes ``{mooring}_{serial}_stage3.nc`` alongside the Stage 2 file.
2020
+ Skips if the output already exists and *force* is False.
2021
+
2022
+ Returns True on success or skip, False on error.
2023
+ """
2024
+ nc_path = info["nc_path"]
2025
+ serial = info["serial"]
2026
+ l3_path = nc_path.with_name(nc_path.name.replace("_stage2.nc", "_stage3.nc"))
2027
+
2028
+ if l3_path.exists():
2029
+ if not force:
2030
+ self._log(f" SKIP (exists): {l3_path.name} (--force to overwrite)")
2031
+ return True
2032
+ try:
2033
+ l3_path.unlink()
2034
+ except OSError as e:
2035
+ self._log(f" ERROR: cannot remove existing {l3_path.name}: {e}")
2036
+ return False
2037
+
2038
+ is_target = info in targets
2039
+ pressure_bad_flag = info["qc_flags"].get("pressure", 0) >= 3
2040
+ is_adcp = info.get("instrument", "").lower() == "adcp"
2041
+
2042
+ self._log(
2043
+ f"--> Processing {info.get('instrument', 'unknown')}: {nc_path.name}"
2044
+ )
2045
+
2046
+ try:
2047
+ from . import parameters as P
2048
+
2049
+ ds = xr.open_dataset(nc_path, decode_timedelta=False).load()
2050
+
2051
+ # Record stage2 source-file provenance as named global attrs
2052
+ _st2_stat = nc_path.stat()
2053
+ _st2_mtime = datetime.datetime.utcfromtimestamp(
2054
+ _st2_stat.st_mtime
2055
+ ).strftime("%Y-%m-%dT%H:%M:%SZ")
2056
+ with open(nc_path, "rb") as _f:
2057
+ _st2_sha = hashlib.sha256(_f.read()).hexdigest()[:8]
2058
+ ds.attrs["stage2_source_file"] = nc_path.name
2059
+ ds.attrs["stage2_source_mtime"] = _st2_mtime
2060
+ ds.attrs["stage2_source_sha256"] = _st2_sha
2061
+
2062
+ target_time = ds["time"].values
2063
+ history_notes = []
2064
+
2065
+ # ── Pressure interpolation (targets only) ──────────────────
2066
+ if is_target and sources:
2067
+ ds, method = self._interpolate_pressure(
2068
+ ds, info, sources, target_time, pressure_bad_flag, qc_attrs
2069
+ )
2070
+ history_notes.append(
2071
+ f"stage3 pressure interpolated (pressure_qc=8) — {method}"
2072
+ )
2073
+
2074
+ # ── Ensure conductivity is in mS/cm before QC ─────────────
2075
+ ds = _ensure_conductivity_units(ds, log_fn=self._log)
2076
+
2077
+ # ── Compute salinity data before QC so it gets range-tested ─
2078
+ ds = _compute_salinity_data(ds, log_fn=self._log)
2079
+
2080
+ # ── QARTOD QC tests (temperature, conductivity, salinity, …) ─
2081
+ gr_cfg = info["gross_range"]
2082
+ sp_cfg = info["spike"]
2083
+ fl_cfg = info["flat_line"]
2084
+ # For ADCP, exclude 2-D velocity vars from the ioos_qc path
2085
+ # (gross_range_test expects 1-D input); velocity QC is handled
2086
+ # below by _apply_adcp_velocity_qc.
2087
+ if is_adcp:
2088
+ _ENU_KEYS = {"east_velocity", "north_velocity", "up_velocity"}
2089
+ gr_cfg_scalar = {k: v for k, v in gr_cfg.items() if k not in _ENU_KEYS}
2090
+ ds = _apply_qc_tests(
2091
+ ds, gr_cfg_scalar, sp_cfg, qc_attrs, flat_line=fl_cfg
2092
+ )
2093
+ else:
2094
+ ds = _apply_qc_tests(ds, gr_cfg, sp_cfg, qc_attrs, flat_line=fl_cfg)
2095
+
2096
+ # ── Post-QC pressure check ─────────────────────────────────
2097
+ # If QC (e.g. flat-line test) just flagged ALL pressure values as
2098
+ # bad and this instrument was not already a pressure target, treat
2099
+ # it as one now: interpolate from the other sources on the mooring.
2100
+ # This catches stuck-sensor faults that only become visible after
2101
+ # the flat-line / gross-range tests run — which is why the pre-scan
2102
+ # source/target classification cannot catch them.
2103
+ # Note: if this instrument was already used as a source for a
2104
+ # previously processed target, those targets will have received bad
2105
+ # pressure values. To fix that, re-run stage 3 on the full mooring
2106
+ # once this instrument's fault is known (e.g. add pressure_qc: 4 to
2107
+ # the YAML so it is excluded from sources at the outset).
2108
+ if (
2109
+ not is_target
2110
+ and "pressure_qc" in ds.data_vars
2111
+ and "pressure" in ds.data_vars
2112
+ ):
2113
+ _pqc = ds["pressure_qc"].values.astype(np.int8)
2114
+ _n_total = len(_pqc)
2115
+ _n_bad = int(np.sum(np.isin(_pqc, [4, 9])))
2116
+ _frac_bad = _n_bad / _n_total if _n_total > 0 else 0.0
2117
+ # Trigger interpolation when > 70% of values are bad. The flat-line
2118
+ # test leaves the first fail_n samples as flag 1/3 (window startup),
2119
+ # so strict all-bad checking would never fire for a fully stuck sensor.
2120
+ if _frac_bad > 0.70:
2121
+ _other_sources = [s for s in sources if s["serial"] != serial]
2122
+ if _other_sources:
2123
+ self._log(
2124
+ f" WARNING: {serial} — {_frac_bad:.0%} of pressure values "
2125
+ f"flagged bad by QC (flat-line or gross-range); replacing "
2126
+ f"entire pressure record with interpolation from "
2127
+ f"{len(_other_sources)} source(s). "
2128
+ f"Re-run stage 3 on the full mooring with "
2129
+ f"'pressure_qc: 4' in the YAML entry for {serial} "
2130
+ f"so it is excluded from sources from the start."
2131
+ )
2132
+ # Temporarily set YAML pressure flag to 4 so that
2133
+ # _interpolate_pressure stores pressure_orig_qc = 4
2134
+ # (the actual QC reason), not the YAML default of 0.
2135
+ info["qc_flags"]["pressure"] = 4
2136
+ ds, method = self._interpolate_pressure(
2137
+ ds,
2138
+ info,
2139
+ _other_sources,
2140
+ target_time,
2141
+ pressure_bad_flag=True,
2142
+ qc_attrs=qc_attrs,
2143
+ )
2144
+ history_notes.append(
2145
+ f"stage3 pressure interpolated post-QC "
2146
+ f"({_frac_bad:.0%} pressure_qc=4/9 after QC) — {method}"
2147
+ )
2148
+ else:
2149
+ self._log(
2150
+ f" WARNING: {serial} — {_frac_bad:.0%} of pressure flagged "
2151
+ f"bad but no other pressure sources on this mooring; "
2152
+ f"pressure remains bad-flagged."
2153
+ )
2154
+
2155
+ # ── Fold T/C/P parent QC into salinity_qc ─────────────────
2156
+ ds = _merge_salinity_parent_qc(ds, qc_attrs)
2157
+
2158
+ # ── BEAM / XYZ → ENU coordinate transform ─────────────────
2159
+ # Must run before tilt QC so east/north/up_velocity exist to be flagged.
2160
+ coord_sys_before = ds.attrs.get("coordinate_system", "ENU")
2161
+ ds = _apply_beam_to_enu(
2162
+ ds,
2163
+ info["entry"],
2164
+ info["lat"],
2165
+ info["lon"],
2166
+ latlon_source=info.get("latlon_source", "unknown"),
2167
+ log_fn=self._log,
2168
+ )
2169
+ # If instrument was already in ENU (e.g. configured internally),
2170
+ # still apply the magnetic declination rotation.
2171
+ if coord_sys_before == "ENU":
2172
+ ds = _apply_declination_to_enu(
2173
+ ds,
2174
+ info["lat"],
2175
+ info["lon"],
2176
+ latlon_source=info.get("latlon_source", "unknown"),
2177
+ log_fn=self._log,
2178
+ )
2179
+
2180
+ # ── ENU velocity QC + up→east/north flag propagation ──────
2181
+ if ds.attrs.get("coordinate_system") == "ENU":
2182
+ if is_adcp:
2183
+ adcp_qc = P.QC_ADCP
2184
+ ds = _apply_adcp_velocity_qc(
2185
+ ds,
2186
+ gr_cfg=gr_cfg,
2187
+ prcnt_gd_bad=adcp_qc["percent_good_bad"],
2188
+ prcnt_gd_suspect=adcp_qc["percent_good_suspect"],
2189
+ error_vel_threshold=adcp_qc["error_velocity_threshold"],
2190
+ qc_attrs=qc_attrs,
2191
+ log_fn=self._log,
2192
+ )
2193
+ ds = _compute_adcp_bin_pressure(ds, info["lat"], log_fn=self._log)
2194
+ ds = _apply_adcp_seabed_qc(
2195
+ ds,
2196
+ water_depth_m=info.get("water_depth_m", 0.0),
2197
+ lat=info["lat"],
2198
+ qc_attrs=qc_attrs,
2199
+ log_fn=self._log,
2200
+ )
2201
+ ds = _apply_adcp_surface_qc(
2202
+ ds,
2203
+ lat=info["lat"],
2204
+ qc_attrs=qc_attrs,
2205
+ log_fn=self._log,
2206
+ )
2207
+ else:
2208
+ ds = _apply_enu_velocity_qc(ds, gr_cfg, qc_attrs)
2209
+
2210
+ # ── Tilt QC ────────────────────────────────────────────────
2211
+ # Flags all velocity variables when pitch+roll tilt exceeds threshold.
2212
+ # Skipped for ADCP: percent_good (col 3) and error_velocity capture
2213
+ # ensemble-level quality without a separate tilt threshold step.
2214
+ # Note: this does NOT mean tilt is checked per-bin — it is not.
2215
+ tilt_cfg = info["tilt"]
2216
+ if is_adcp:
2217
+ n_tilt_susp, n_tilt_bad = 0, 0
2218
+ history_notes.append(
2219
+ "tilt QC skipped for ADCP (percent_good+error_velocity QC applied instead)"
2220
+ )
2221
+ else:
2222
+ ds, n_tilt_susp, n_tilt_bad = _apply_tilt_qc(ds, tilt_cfg, qc_attrs)
2223
+ if n_tilt_susp or n_tilt_bad:
2224
+ history_notes.append(
2225
+ f"tilt QC (tilt≥{tilt_cfg['suspect_threshold']}°→suspect, "
2226
+ f"tilt≥{tilt_cfg['fail_threshold']}°→bad): "
2227
+ f"suspect={n_tilt_susp}, bad={n_tilt_bad}"
2228
+ )
2229
+
2230
+ # ── Unify ENU velocity QC ──────────────────────────────────
2231
+ # After tilt + gross-range QC, combine worst flag across all three
2232
+ # ENU components and write it back to all three so masking on any
2233
+ # single component (east, north, or up) gives an identical result.
2234
+ # This catches e.g. large east/north currents (>3 m/s suspect) that
2235
+ # would otherwise leave up_velocity_qc unflagged.
2236
+ if ds.attrs.get("coordinate_system") == "ENU":
2237
+ ds = _unify_velocity_qc(ds)
2238
+
2239
+ if ds.attrs.get("coordinate_system") == "ENU" and coord_sys_before != "ENU":
2240
+ _ba = ds.attrs.get("nortek_beam_angle", "?")
2241
+ _ba_src = ds.attrs.get("nortek_beam_angle_source", "")
2242
+ _assumed = "ASSUMED DEFAULT" in _ba_src
2243
+ history_notes.append(
2244
+ f"BEAM→ENU rotation applied: "
2245
+ f"beam_angle={_ba}° "
2246
+ f"({'ASSUMED DEFAULT — not from datasheet' if _assumed else 'from YAML'}), "
2247
+ f"declination={ds.attrs.get('magnetic_declination', 0.0):.2f}° "
2248
+ f"({'ppigrf IGRF' if 'magnetic_declination' in ds.attrs else 'assumed 0'})"
2249
+ )
2250
+
2251
+ # Report which variables got QC flags and their flag counts
2252
+ qc_summary = []
2253
+ for v in sorted(ds.data_vars):
2254
+ if v.endswith("_qc") and not v.endswith("_orig_qc"):
2255
+ counts = np.bincount(
2256
+ ds[v].values.astype(np.int8).clip(0, 9).ravel(), minlength=10
2257
+ )
2258
+ n_good = int(counts[1])
2259
+ n_susp = int(counts[3])
2260
+ n_bad = int(counts[4])
2261
+ n_interp = int(counts[8])
2262
+ n_miss = int(counts[9])
2263
+ parts = [f"good={n_good}"]
2264
+ if n_susp:
2265
+ parts.append(f"suspect={n_susp}")
2266
+ if n_bad:
2267
+ parts.append(f"bad={n_bad}")
2268
+ if n_interp:
2269
+ parts.append(f"interp={n_interp}")
2270
+ if n_miss:
2271
+ parts.append(f"missing={n_miss}")
2272
+ qc_summary.append(f"{v}=[{', '.join(parts)}]")
2273
+ if qc_summary:
2274
+ history_notes.append(
2275
+ "QARTOD gross-range+spike: " + "; ".join(qc_summary)
2276
+ )
2277
+
2278
+ # ── History ────────────────────────────────────────────────
2279
+ stamp = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%MZ")
2280
+ entry = f"{stamp}: " + " | ".join(history_notes)
2281
+ existing = ds.attrs.get("history", "")
2282
+ ds.attrs["history"] = f"{existing}; {entry}" if existing else entry
2283
+
2284
+ # Derive O2 % saturation and AOU when dissolved_oxygen + T/S/P are present.
2285
+ ds = _derive_oxygen_saturation(ds)
2286
+
2287
+ # Normalize CF standard_name and long_name for known physics variables.
2288
+ # Only fills in missing attrs — never overwrites values already set.
2289
+ for _vname, _cf in _CF_ATTRS.items():
2290
+ if _vname in ds.data_vars:
2291
+ for _k, _v in _cf.items():
2292
+ if _k not in ds[_vname].attrs:
2293
+ ds[_vname].attrs[_k] = _v
2294
+
2295
+ ds = drop_all_zero_vars(ds, ["amplitude_beam", "analog_input_"])
2296
+ cast_output_dtypes(ds).to_netcdf(l3_path)
2297
+ ds.close()
2298
+ self._log(
2299
+ f" Creating output file: {l3_path.name} ({'; '.join(qc_summary)})"
2300
+ )
2301
+
2302
+ except Exception as e: # noqa: BLE001 — log and continue; one instrument must not abort mooring
2303
+ self._log(f" ERROR processing {serial}: {e}")
2304
+ import traceback
2305
+
2306
+ self._log(traceback.format_exc())
2307
+ return False
2308
+ return True
2309
+
2310
+ # ------------------------------------------------------------------
2311
+ def _interp_pressure_for_hab(
2312
+ self,
2313
+ hab_t: float,
2314
+ sorted_sources: List[Dict[str, Any]],
2315
+ target_time: np.ndarray,
2316
+ serial: str,
2317
+ ) -> tuple[np.ndarray, str]:
2318
+ """Interpolate pressure for a single nominal HAB; return (p_array, method_str)."""
2319
+ habs = np.array([s["hab"] for s in sorted_sources])
2320
+ diffs = np.abs(habs - hab_t)
2321
+ nearest_idx = int(np.argmin(diffs))
2322
+
2323
+ if diffs[nearest_idx] <= HAB_THRESHOLD:
2324
+ src = sorted_sources[nearest_idx]
2325
+ hab_offset_dbar = src["hab"] - hab_t
2326
+ p = (
2327
+ _interp_pressure(
2328
+ src["ds"]["time"].values,
2329
+ src["ds"][src["pressure_var"]].values,
2330
+ target_time,
2331
+ )
2332
+ + hab_offset_dbar
2333
+ )
2334
+ method = (
2335
+ f"near-neighbour from {src['instrument']} {src['serial']} "
2336
+ f"(Δhab={diffs[nearest_idx]:.1f}m, "
2337
+ f"static offset={hab_offset_dbar:+.1f} dbar)"
2338
+ )
2339
+ self._log(f" {serial} (hab={hab_t:.1f}m): {method}")
2340
+ else:
2341
+ below = [(s, h) for s, h in zip(sorted_sources, habs) if h < hab_t]
2342
+ above = [(s, h) for s, h in zip(sorted_sources, habs) if h > hab_t]
2343
+
2344
+ if below and above:
2345
+ src_below, h_below = below[-1]
2346
+ src_above, h_above = above[0]
2347
+ w_above = (hab_t - h_below) / (h_above - h_below)
2348
+ w_below = 1.0 - w_above
2349
+ p_below = _interp_pressure(
2350
+ src_below["ds"]["time"].values,
2351
+ src_below["ds"][src_below["pressure_var"]].values,
2352
+ target_time,
2353
+ )
2354
+ p_above = _interp_pressure(
2355
+ src_above["ds"]["time"].values,
2356
+ src_above["ds"][src_above["pressure_var"]].values,
2357
+ target_time,
2358
+ )
2359
+ p = w_below * p_below + w_above * p_above
2360
+ method = (
2361
+ f"bracketed: {w_below:.2f}×{src_below['instrument']} "
2362
+ f"{src_below['serial']} (hab={h_below:.1f}m) + "
2363
+ f"{w_above:.2f}×{src_above['instrument']} "
2364
+ f"{src_above['serial']} (hab={h_above:.1f}m)"
2365
+ )
2366
+ self._log(f" {serial} (hab={hab_t:.1f}m): {method}")
2367
+ else:
2368
+ src = below[-1][0] if below else above[0][0]
2369
+ hab_offset_dbar = src["hab"] - hab_t
2370
+ p = (
2371
+ _interp_pressure(
2372
+ src["ds"]["time"].values,
2373
+ src["ds"][src["pressure_var"]].values,
2374
+ target_time,
2375
+ )
2376
+ + hab_offset_dbar
2377
+ )
2378
+ method = (
2379
+ f"extrapolated from {src['instrument']} {src['serial']} "
2380
+ f"(hab={src['hab']:.1f}m, "
2381
+ f"static offset={hab_offset_dbar:+.1f} dbar) — WARNING: out of range"
2382
+ )
2383
+ self._log(f" WARNING: {serial} (hab={hab_t:.1f}m): {method}")
2384
+
2385
+ return p, method
2386
+
2387
+ def _interpolate_pressure(
2388
+ self,
2389
+ ds: xr.Dataset,
2390
+ target_info: Dict[str, Any],
2391
+ sources: List[Dict[str, Any]],
2392
+ target_time: np.ndarray,
2393
+ pressure_bad_flag: bool,
2394
+ qc_attrs: Dict[str, Any],
2395
+ ) -> tuple[xr.Dataset, str]:
2396
+ """Interpolate pressure from sources onto target; return (ds, method_str).
2397
+
2398
+ If *target_info* contains ``hab_segments`` (a sorted list of
2399
+ ``(breakpoint_datetime64, hab_float)`` pairs), the record is split into
2400
+ time segments and each segment is interpolated with its own HAB, allowing
2401
+ for instruments that physically moved during the deployment.
2402
+
2403
+ HAB convention for segmented deployments
2404
+ -----------------------------------------
2405
+ ``target_info["hab"]`` is the **initial** HAB — the height above bottom
2406
+ at which the instrument was **deployed**. Segment 0 runs from the start
2407
+ of the record up to (but not including) the first breakpoint, using this
2408
+ value. Each entry in ``hab_segments`` gives the new HAB **at and after**
2409
+ the breakpoint timestamp (i.e. records with ``T >= breakpoint`` use the
2410
+ new HAB; the breakpoint itself belongs to the new segment). This matches
2411
+ the YAML convention where ``hab:`` records the as-deployed position and
2412
+ ``hab_segments:`` records subsequent slides, with ``from:`` marking the
2413
+ first timestamp that belongs to the post-slide position.
2414
+ """
2415
+ if len(target_time) == 0:
2416
+ return ds, "no data"
2417
+
2418
+ pressure_qc_val = target_info["qc_flags"].get("pressure", 0)
2419
+ serial = target_info["serial"]
2420
+
2421
+ sorted_sources = sorted(
2422
+ [s for s in sources if s.get("ds") is not None],
2423
+ key=lambda s: s["hab"],
2424
+ )
2425
+ if not sorted_sources:
2426
+ return ds, "no sources available"
2427
+
2428
+ hab_segments = target_info.get("hab_segments", [])
2429
+
2430
+ if hab_segments:
2431
+ # Build list of (start_ns, end_ns, hab) covering the full record.
2432
+ # Segment 0: record start → first breakpoint, using nominal HAB.
2433
+ # Segment k: breakpoint[k-1] → breakpoint[k] (or record end), using
2434
+ # the HAB specified at breakpoint[k-1].
2435
+ T = target_time
2436
+ seg_bounds: List[tuple] = []
2437
+ seg_hab = target_info["hab"]
2438
+ seg_start = T[0]
2439
+ for bp_ns, next_hab in hab_segments:
2440
+ seg_bounds.append((seg_start, bp_ns, seg_hab))
2441
+ seg_start = bp_ns
2442
+ seg_hab = next_hab
2443
+ seg_bounds.append((seg_start, T[-1] + np.timedelta64(1, "ns"), seg_hab))
2444
+
2445
+ p_interp = np.full(len(T), np.nan)
2446
+ method_parts = []
2447
+ for seg_start_ns, seg_end_ns, hab_t in seg_bounds:
2448
+ idx = np.where((T >= seg_start_ns) & (T < seg_end_ns))[0]
2449
+ if len(idx) == 0:
2450
+ continue
2451
+ p_seg, meth = self._interp_pressure_for_hab(
2452
+ hab_t, sorted_sources, T[idx], serial
2453
+ )
2454
+ p_interp[idx] = p_seg
2455
+ bp_str = str(seg_end_ns)[:16]
2456
+ method_parts.append(f"hab={hab_t:.1f}m until {bp_str}: {meth}")
2457
+ method = "; ".join(method_parts)
2458
+ else:
2459
+ p_interp, method = self._interp_pressure_for_hab(
2460
+ target_info["hab"], sorted_sources, target_time, serial
2461
+ )
2462
+
2463
+ # Preserve bad original pressure
2464
+ if pressure_bad_flag and "pressure" in ds.data_vars:
2465
+ orig_attrs = dict(ds["pressure"].attrs)
2466
+ orig_attrs["comment"] = (
2467
+ orig_attrs.get("comment", "") + " [original; flagged bad in YAML]"
2468
+ ).strip()
2469
+ ds["pressure_orig"] = ds["pressure"].copy()
2470
+ ds["pressure_orig"].attrs = orig_attrs
2471
+ ds["pressure_orig_qc"] = xr.Variable(
2472
+ "time",
2473
+ np.full(len(ds["time"]), pressure_qc_val, dtype=np.int8),
2474
+ attrs={"long_name": "quality flag for pressure_orig", **qc_attrs},
2475
+ )
2476
+ ds = ds.drop_vars("pressure")
2477
+
2478
+ ds["pressure"] = xr.Variable(
2479
+ "time",
2480
+ p_interp,
2481
+ attrs={
2482
+ "units": "dbar",
2483
+ "long_name": "sea water pressure",
2484
+ "standard_name": "sea_water_pressure",
2485
+ "pressure_source": method,
2486
+ "comment": "interpolated from neighbouring instrument(s)",
2487
+ },
2488
+ )
2489
+ ds["pressure_qc"] = xr.Variable(
2490
+ "time",
2491
+ np.full(len(ds["time"]), 8, dtype=np.int8),
2492
+ attrs={"long_name": "quality flag for pressure", **qc_attrs},
2493
+ )
2494
+ return ds, method