oceanarray 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. oceanarray/__init__.py +20 -0
  2. oceanarray/_version.py +1 -0
  3. oceanarray/cli.py +1343 -0
  4. oceanarray/clock_offset.py +675 -0
  5. oceanarray/config/OS1_sensor_attrs.yaml +49 -0
  6. oceanarray/config/OS1_var_names.yaml +5 -0
  7. oceanarray/config/OS1_vocab_attrs.yaml +84 -0
  8. oceanarray/config/legacy/project_RAPID.yaml +69 -0
  9. oceanarray/config/legacy/rodb_keys.txt +206 -0
  10. oceanarray/config/legacy/rodb_keys.yaml +1012 -0
  11. oceanarray/config/logging.yaml +28 -0
  12. oceanarray/legacy/__init__.py +47 -0
  13. oceanarray/legacy/caldip.py +753 -0
  14. oceanarray/legacy/convertOS.py +527 -0
  15. oceanarray/legacy/find_deployment.py +489 -0
  16. oceanarray/legacy/mooring_rodb.py +429 -0
  17. oceanarray/legacy/process_rodb.py +369 -0
  18. oceanarray/legacy/rodb.py +330 -0
  19. oceanarray/logger.py +192 -0
  20. oceanarray/logsheet/__init__.py +60 -0
  21. oceanarray/logsheet/_columns.py +97 -0
  22. oceanarray/logsheet/_config.py +245 -0
  23. oceanarray/logsheet/_firmware.py +43 -0
  24. oceanarray/logsheet/_latex.py +128 -0
  25. oceanarray/logsheet/_render.py +126 -0
  26. oceanarray/logsheet/builders/__init__.py +13 -0
  27. oceanarray/logsheet/builders/caldip_download.py +195 -0
  28. oceanarray/logsheet/builders/caldip_setup.py +188 -0
  29. oceanarray/logsheet/builders/deployment_setup.py +218 -0
  30. oceanarray/logsheet/builders/mooring_download.py +243 -0
  31. oceanarray/logsheet/builders/recovery.py +157 -0
  32. oceanarray/logsheet/data/column_defs.yaml +686 -0
  33. oceanarray/logsheet/data/latex_templates/_base_instrument.tex.j2 +42 -0
  34. oceanarray/logsheet/data/latex_templates/caldip_download.tex.j2 +47 -0
  35. oceanarray/logsheet/data/latex_templates/caldip_setup.tex.j2 +70 -0
  36. oceanarray/logsheet/data/latex_templates/deployment_setup.tex.j2 +97 -0
  37. oceanarray/logsheet/data/latex_templates/mooring_download.tex.j2 +54 -0
  38. oceanarray/logsheet/data/latex_templates/recovery.tex.j2 +103 -0
  39. oceanarray/mooring_level.py +1368 -0
  40. oceanarray/oceanarray.mplstyle +18 -0
  41. oceanarray/parameters.py +270 -0
  42. oceanarray/plotter.py +942 -0
  43. oceanarray/plotters/__init__.py +97 -0
  44. oceanarray/plotters/_animation.py +261 -0
  45. oceanarray/plotters/_current.py +705 -0
  46. oceanarray/plotters/_diagnostic.py +667 -0
  47. oceanarray/plotters/_helpers.py +57 -0
  48. oceanarray/plotters/_primitives.py +101 -0
  49. oceanarray/plotters/_section.py +16 -0
  50. oceanarray/plotters/_timeseries.py +11 -0
  51. oceanarray/rapid_interp.py +739 -0
  52. oceanarray/read_rbr_hex.py +541 -0
  53. oceanarray/readers.py +303 -0
  54. oceanarray/report/__init__.py +5 -0
  55. oceanarray/report/_array.py +386 -0
  56. oceanarray/report/_grid.py +539 -0
  57. oceanarray/report/_html_helpers.py +1004 -0
  58. oceanarray/report/_instrument.py +853 -0
  59. oceanarray/report/_mooring.py +1385 -0
  60. oceanarray/report/_plots.py +4284 -0
  61. oceanarray/report/_stack.py +941 -0
  62. oceanarray/stage1.py +1959 -0
  63. oceanarray/stage2.py +1160 -0
  64. oceanarray/stage3.py +2494 -0
  65. oceanarray/time_gridding.py +958 -0
  66. oceanarray/tools.py +400 -0
  67. oceanarray/transports.py +1 -0
  68. oceanarray/utilities.py +538 -0
  69. oceanarray/validation.py +389 -0
  70. oceanarray/writers.py +99 -0
  71. oceanarray-0.1.0.dist-info/METADATA +503 -0
  72. oceanarray-0.1.0.dist-info/RECORD +76 -0
  73. oceanarray-0.1.0.dist-info/WHEEL +5 -0
  74. oceanarray-0.1.0.dist-info/entry_points.txt +2 -0
  75. oceanarray-0.1.0.dist-info/licenses/LICENSE +21 -0
  76. oceanarray-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1368 @@
1
+ """Mooring-level processing: instrument stacking and vertical gridding.
2
+
3
+ Step 1 — stack (``oceanarray stack``):
4
+ Read all instruments from a single deployment and stack them onto a common
5
+ 60 s (or user-specified) time axis. Output: ``{mooring}_stack.nc`` with
6
+ dimensions ``(instrument, time)``. --> change to (time, instrument)!
7
+
8
+ Step 2 — grid (``oceanarray grid``):
9
+ Vertically interpolate the stacked dataset onto a standard pressure grid.
10
+ Output: ``{mooring}_grid.nc`` with dimensions ``(pressure, time)``.
11
+ """
12
+
13
+ from pathlib import Path
14
+ from typing import Any, Dict, List, Optional
15
+ import re
16
+
17
+ import numpy as np
18
+ import xarray as xr
19
+ import yaml
20
+ from . import parameters as P
21
+ from .utilities import _status, cast_output_dtypes, drop_all_zero_vars
22
+
23
+ # Canonical allowed values for the ``instrument:`` field in mooring YAML files.
24
+ # Defined in parameters.py alongside the instrument→file_type mapping.
25
+ KNOWN_INSTRUMENT_TYPES: frozenset = P.KNOWN_INSTRUMENT_TYPES
26
+
27
+ STACK_VARS = [
28
+ "temperature",
29
+ "temperature_qc",
30
+ "salinity",
31
+ "salinity_qc",
32
+ "conductivity",
33
+ "pressure",
34
+ "pressure_qc",
35
+ "east_velocity",
36
+ "north_velocity",
37
+ "up_velocity",
38
+ "current_speed",
39
+ "current_direction",
40
+ "east_velocity_qc",
41
+ "north_velocity_qc",
42
+ "up_velocity_qc",
43
+ "pitch_qc",
44
+ "roll_qc",
45
+ "velocity_beam1",
46
+ "velocity_beam2",
47
+ "velocity_beam3",
48
+ "heading",
49
+ "pitch",
50
+ "roll",
51
+ "analog_input_1",
52
+ "analog_input_2",
53
+ "percent_good_qc",
54
+ "error_velocity_qc",
55
+ "seabed_qc",
56
+ "turbidity",
57
+ "turbidity_qc",
58
+ "dissolved_oxygen",
59
+ "dissolved_oxygen_qc",
60
+ "oxygen_saturation_pct",
61
+ "apparent_oxygen_utilization",
62
+ ]
63
+
64
+ # Variables passed through without QC masking at the stack step.
65
+ # Velocity and orientation are kept at full precision so downstream users
66
+ # (grid step, analysis) can apply their own masking via velocity_flag.
67
+ # QC flag arrays never need masking (there is no companion *_qc_qc* variable).
68
+ _STACK_RAW: frozenset = frozenset(
69
+ {
70
+ "east_velocity",
71
+ "north_velocity",
72
+ "up_velocity",
73
+ "current_speed",
74
+ "current_direction",
75
+ "velocity_beam1",
76
+ "velocity_beam2",
77
+ "velocity_beam3",
78
+ "heading",
79
+ "pitch",
80
+ "roll",
81
+ "east_velocity_qc",
82
+ "north_velocity_qc",
83
+ "up_velocity_qc",
84
+ "pitch_qc",
85
+ "roll_qc",
86
+ "percent_good_qc",
87
+ "error_velocity_qc",
88
+ "seabed_qc",
89
+ }
90
+ )
91
+
92
+
93
+ def _safe_serial(serial: Any) -> str:
94
+ """Sanitise a serial number string for use in filenames.
95
+
96
+ If the raw YAML value contains a comma (e.g. ``16430, R01-024``), only
97
+ the first comma-separated token is used — the remainder is a beacon ID
98
+ or annotation and is not part of the filename.
99
+ """
100
+ s = str(serial).split(",")[0]
101
+ return re.sub(r"[^\w\-]", "", s)
102
+
103
+
104
+ def _dms_to_deg(s: str) -> float:
105
+ """Parse 'DD MM.mmm N/S/E/W' or a plain float string to decimal degrees."""
106
+ s = s.strip()
107
+ try:
108
+ return float(s)
109
+ except ValueError:
110
+ pass
111
+ parts = s.upper().split()
112
+ hemi = parts[-1] if parts[-1] in ("N", "S", "E", "W") else None
113
+ nums = parts[:-1] if hemi else parts
114
+ val = sum(float(x) / 60.0**i for i, x in enumerate(nums))
115
+ if hemi in ("S", "W"):
116
+ val = -val
117
+ return val
118
+
119
+
120
+ def _parse_latlon(cfg: dict) -> tuple[float, float]:
121
+ """Return (lat, lon) in decimal degrees from a mooring config dict."""
122
+ lat, lon, _ = _parse_latlon_with_source(cfg)
123
+ return lat, lon
124
+
125
+
126
+ def _parse_latlon_with_source(cfg: dict) -> tuple[float, float, str]:
127
+ """Return (lat, lon, source_key) from a mooring config dict.
128
+
129
+ source_key is the YAML key pair used, e.g. 'seabed_latitude/seabed_longitude',
130
+ or 'unknown (defaulting to 0, 0)' if none found.
131
+
132
+ Zero values (lat == 0 and lon == 0) are treated as unfilled placeholders and
133
+ skipped so that a later key with a real location is used instead. This handles
134
+ YAML templates where fields like ``seabed_latitude: 00 00.000 N`` are present
135
+ but not yet populated.
136
+ """
137
+ for lat_key, lon_key in [
138
+ ("seabed_latitude", "seabed_longitude"),
139
+ ("deployment_latitude", "deployment_longitude"),
140
+ ("planned_latitude", "planned_longitude"),
141
+ ("latitude", "longitude"),
142
+ ]:
143
+ lat_s = cfg.get(lat_key)
144
+ lon_s = cfg.get(lon_key)
145
+ if lat_s is not None and lon_s is not None:
146
+ try:
147
+ lat_v = _dms_to_deg(str(lat_s))
148
+ lon_v = _dms_to_deg(str(lon_s))
149
+ except Exception: # noqa: BLE001 — _dms_to_deg parse failure; try next key
150
+ continue
151
+ if lat_v == 0.0 and lon_v == 0.0:
152
+ continue # treat (0, 0) as an unfilled placeholder
153
+ return lat_v, lon_v, f"{lat_key}/{lon_key}"
154
+ return 0.0, 0.0, "unknown (defaulting to 0, 0)"
155
+
156
+
157
+ _SIGMA_META = {
158
+ 0: (
159
+ "sigma0",
160
+ "Potential density anomaly referenced to surface (sigma-0)",
161
+ "sea_water_sigma_t",
162
+ ),
163
+ 1000: (
164
+ "sigma1",
165
+ "Potential density anomaly referenced to 1000 dbar (sigma-1)",
166
+ "sea_water_sigma_1",
167
+ ),
168
+ 2000: (
169
+ "sigma2",
170
+ "Potential density anomaly referenced to 2000 dbar (sigma-2)",
171
+ "sea_water_sigma_2",
172
+ ),
173
+ 3000: (
174
+ "sigma3",
175
+ "Potential density anomaly referenced to 3000 dbar (sigma-3)",
176
+ "sea_water_sigma_3",
177
+ ),
178
+ 4000: (
179
+ "sigma4",
180
+ "Potential density anomaly referenced to 4000 dbar (sigma-4)",
181
+ "sea_water_sigma_4",
182
+ ),
183
+ }
184
+
185
+
186
+ def _apply_qc_mask(src_v: np.ndarray, ds: "xr.Dataset", vname: str) -> np.ndarray:
187
+ """Return src_v with flagged values replaced by NaN.
188
+
189
+ Flags kept as valid:
190
+ 0 (no QC performed), 1 (good), 2 (probably good)
191
+ + flag 8 (interpolated) for pressure only.
192
+ Flags masked (→ NaN):
193
+ 3 (suspect), 4 (bad), 9 (missing value).
194
+ If no companion *_qc* variable exists the array is returned unchanged.
195
+ """
196
+ qc_name = f"{vname}_qc"
197
+ if qc_name not in ds.data_vars:
198
+ return src_v
199
+ qc = ds[qc_name].values
200
+ if vname == "pressure":
201
+ keep = np.isin(qc, [0, 1, 2, 8])
202
+ else:
203
+ keep = np.isin(qc, [0, 1, 2])
204
+ out = src_v.copy()
205
+ out[~keep] = np.nan
206
+ return out
207
+
208
+
209
+ def _worst_flag(a: np.ndarray, b: np.ndarray) -> np.ndarray:
210
+ """Element-wise worst QC flag (9 > 4 > 3 > 8 > 2 > 1 > 0).
211
+
212
+ NaN inputs are treated as flag 9 (missing value) so that levels with no
213
+ data are never silently promoted to flag 0 ("no QC performed").
214
+ """
215
+ # rank[flag_value] gives priority; higher rank = worse flag
216
+ _rank = np.array([0, 1, 2, 4, 5, 6, 6, 6, 3, 7], dtype=np.int8)
217
+ a = np.where(np.isfinite(a), a, 9.0)
218
+ b = np.where(np.isfinite(b), b, 9.0)
219
+ ai = np.clip(np.round(a).astype(np.int8), 0, 9)
220
+ bi = np.clip(np.round(b).astype(np.int8), 0, 9)
221
+ take_b = _rank[bi] > _rank[ai]
222
+ out = ai.copy()
223
+ out[take_b] = bi[take_b]
224
+ return out
225
+
226
+
227
+ def _times_to_float(t: np.ndarray) -> np.ndarray:
228
+ return t.astype("datetime64[ns]").astype(np.float64)
229
+
230
+
231
+ def _get_proc_dir(base_dir: Path, mooring_name: str) -> Path:
232
+ proc = base_dir / "proc"
233
+ if not proc.is_dir():
234
+ legacy = base_dir / "moor" / "proc"
235
+ proc = legacy if legacy.is_dir() else proc
236
+ return proc / mooring_name
237
+
238
+
239
+ def _best_nc(
240
+ proc_dir: Path, instr_type: str, mooring_name: str, serial: str
241
+ ) -> Optional[Path]:
242
+ """Return best available stage3 or stage2 file for one instrument; None if only stage1."""
243
+ base = proc_dir / instr_type / f"{mooring_name}_{serial}"
244
+ for suffix in ("_stage3.nc", "_stage2.nc"):
245
+ p = Path(str(base) + suffix)
246
+ if p.exists():
247
+ return p
248
+ return None
249
+
250
+
251
+ def _detect_interval_s(time_vals: np.ndarray) -> float:
252
+ if len(time_vals) < 2:
253
+ return 60.0
254
+ dt = np.diff(time_vals.astype("datetime64[s]").astype(np.float64))
255
+ return float(np.median(dt))
256
+
257
+
258
+ # Variables measured at the ADCP transducer head (point measurements, not per-bin).
259
+ # These must NOT appear in per-bin datasets — they belong only in the head entry.
260
+ _ADCP_HEAD_VARS: frozenset = frozenset(
261
+ {
262
+ "temperature",
263
+ "temperature_qc",
264
+ "heading",
265
+ "pitch",
266
+ "roll",
267
+ "battery_voltage",
268
+ "speed_of_sound",
269
+ "analog_input_1",
270
+ "analog_input_2",
271
+ }
272
+ )
273
+
274
+
275
+ def _make_adcp_head_ds(ds_parent: xr.Dataset) -> xr.Dataset:
276
+ """Create a 1-D (time-only) view of the ADCP transducer head for stack insertion.
277
+
278
+ The ADCP transducer head is treated as a distinct instrument position in the mooring
279
+ stack. It carries instrument-diagnostic variables that describe the sensor, not the
280
+ water column:
281
+
282
+ - ``temperature`` (°C) — internal electronics temperature; useful for diagnosing
283
+ warm-up drift but not a reliable water temperature measurement.
284
+ - ``heading``, ``pitch``, ``roll`` (°) — instrument orientation.
285
+ - ``pressure`` (dbar) — pressure at the transducer face, i.e. the instrument depth.
286
+
287
+ Per-bin velocity and ``bin_pressure`` are deliberately excluded. Each velocity bin
288
+ gets its own stack entry at the pressure appropriate to that bin
289
+ (see ``_make_adcp_bin_ds``), rather than the transducer pressure.
290
+
291
+ Parameters
292
+ ----------
293
+ ds_parent : xr.Dataset
294
+ Full ADCP stage-3 dataset, including 2-D ``(time, N_BINS)`` variables.
295
+
296
+ Returns
297
+ -------
298
+ xr.Dataset
299
+ Dataset containing only time-series (1-D) variables from ``_ADCP_HEAD_VARS``
300
+ plus the transducer ``pressure``. Suitable for merging into a point-instrument
301
+ stack.
302
+
303
+ """
304
+ keep = [
305
+ v
306
+ for v in ds_parent.data_vars
307
+ if v in _ADCP_HEAD_VARS and ds_parent[v].dims == ("time",)
308
+ ]
309
+ # Also keep the instrument-head pressure if present (1-D, not bin_pressure)
310
+ if "pressure" in ds_parent.data_vars and ds_parent["pressure"].dims == ("time",):
311
+ if "pressure" not in keep:
312
+ keep.append("pressure")
313
+ return ds_parent[keep]
314
+
315
+
316
+ def _make_adcp_bin_ds(ds_parent: xr.Dataset, bin_idx: int) -> xr.Dataset:
317
+ """Create a 1-D (time-only) point-instrument view of one ADCP range bin.
318
+
319
+ Slices the parent dataset at *bin_idx* along ``P.ADCP_BIN_DIM`` and replaces the
320
+ transducer ``pressure`` with ``bin_pressure[:, bin_idx]`` so that this entry sits at
321
+ the correct depth in the stack. Computes derived scalar velocity quantities::
322
+
323
+ current_speed = sqrt(east_velocity² + north_velocity²) [m s⁻¹, always ≥ 0]
324
+ current_direction = atan2(east, north) mod 360 [degrees, 0 = N CW]
325
+
326
+ **Direction convention**: the direction *toward which* the water flows, clockwise
327
+ from True North. This is the oceanographic convention (opposite to meteorological
328
+ "wind from"). 0° = northward flow, 90° = eastward flow.
329
+
330
+ Instrument-head variables (``temperature``, ``heading``, ``pitch``, ``roll`` — see
331
+ ``_ADCP_HEAD_VARS``) are dropped because they belong to the transducer location, not
332
+ to an individual velocity bin. Variables retaining non-time dimensions (e.g.
333
+ beam-dimension arrays ``amplitude``, ``correlation``, ``percent_good``) are also
334
+ dropped; they cannot be represented as point-instrument time series without
335
+ ambiguity about which beam to keep.
336
+
337
+ The result is compatible with ``_nearest_subsample`` and ``_linear_interp`` because
338
+ every remaining data variable has dimension ``("time",)`` or is scalar.
339
+
340
+ Parameters
341
+ ----------
342
+ ds_parent : xr.Dataset
343
+ Full ADCP stage-3 dataset.
344
+ bin_idx : int
345
+ Zero-based index along the bin dimension (``P.ADCP_BIN_DIM``). Index 0 is
346
+ the bin nearest the transducer face.
347
+
348
+ Returns
349
+ -------
350
+ xr.Dataset
351
+ Single-bin time-series dataset with ``pressure`` set to ``bin_pressure``
352
+ at this bin and ``current_speed`` / ``current_direction`` added.
353
+
354
+ """
355
+ bin_dim = P.ADCP_BIN_DIM
356
+ ds_bin = ds_parent.isel({bin_dim: bin_idx}, drop=True)
357
+
358
+ # Drop head-only variables — these belong in the separate head entry
359
+ head_to_drop = [v for v in _ADCP_HEAD_VARS if v in ds_bin.data_vars]
360
+ if head_to_drop:
361
+ ds_bin = ds_bin.drop_vars(head_to_drop)
362
+
363
+ if "bin_pressure" in ds_bin.data_vars:
364
+ ds_bin = ds_bin.assign(
365
+ pressure=xr.Variable(
366
+ "time",
367
+ ds_bin["bin_pressure"].values.astype(np.float32),
368
+ {"units": "dbar", "long_name": "Pressure at ADCP bin"},
369
+ )
370
+ ).drop_vars("bin_pressure")
371
+
372
+ if "east_velocity" in ds_bin.data_vars and "north_velocity" in ds_bin.data_vars:
373
+ e = ds_bin["east_velocity"].values.astype(float)
374
+ n = ds_bin["north_velocity"].values.astype(float)
375
+ ds_bin = ds_bin.assign(
376
+ current_speed=xr.Variable(
377
+ "time",
378
+ np.hypot(e, n).astype(np.float32),
379
+ {"units": "m s-1", "long_name": "Current speed"},
380
+ ),
381
+ current_direction=xr.Variable(
382
+ "time",
383
+ (np.degrees(np.arctan2(e, n)) % 360.0).astype(np.float32),
384
+ {
385
+ "units": "degrees",
386
+ "long_name": "Current direction (oceanographic, 0=N clockwise)",
387
+ },
388
+ ),
389
+ )
390
+
391
+ to_drop = [v for v in ds_bin.data_vars if any(d != "time" for d in ds_bin[v].dims)]
392
+ if to_drop:
393
+ ds_bin = ds_bin.drop_vars(to_drop)
394
+
395
+ return ds_bin
396
+
397
+
398
+ def _nearest_subsample(
399
+ ds: xr.Dataset,
400
+ common_time: np.ndarray,
401
+ half_window_s: float,
402
+ ) -> Dict[str, np.ndarray]:
403
+ """Return nearest-neighbour values on common_time within ±half_window_s seconds.
404
+
405
+ Returns a dict {varname: array} with NaN where no sample falls within the window.
406
+ """
407
+ src_t = _times_to_float(ds["time"].values)
408
+ tgt_t = _times_to_float(common_time)
409
+ n = len(common_time)
410
+ half_ns = half_window_s * 1e9 # ns
411
+
412
+ idx = np.searchsorted(src_t, tgt_t)
413
+ result: Dict[str, np.ndarray] = {}
414
+ for vname in STACK_VARS:
415
+ if vname not in ds.data_vars:
416
+ result[vname] = np.full(n, np.nan)
417
+ continue
418
+ src_v = ds[vname].values.astype(np.float64)
419
+ if vname not in _STACK_RAW:
420
+ src_v = _apply_qc_mask(src_v, ds, vname)
421
+ out = np.full(n, np.nan)
422
+ for i, (t_tgt, k) in enumerate(zip(tgt_t, idx)):
423
+ # Check candidates at k-1 and k
424
+ best_dt = np.inf
425
+ best_v = np.nan
426
+ for j in [k - 1, k]:
427
+ if 0 <= j < len(src_t):
428
+ dt = abs(src_t[j] - t_tgt)
429
+ if dt < best_dt and dt <= half_ns:
430
+ best_dt = dt
431
+ best_v = src_v[j]
432
+ out[i] = best_v
433
+ result[vname] = out
434
+ return result
435
+
436
+
437
+ def _linear_interp(
438
+ ds: xr.Dataset,
439
+ common_time: np.ndarray,
440
+ ) -> Dict[str, np.ndarray]:
441
+ """Linearly interpolate all stack variables onto common_time; NaN outside data range."""
442
+ src_t = _times_to_float(ds["time"].values)
443
+ tgt_t = _times_to_float(common_time)
444
+ result: Dict[str, np.ndarray] = {}
445
+ for vname in STACK_VARS:
446
+ if vname not in ds.data_vars:
447
+ result[vname] = np.full(len(common_time), np.nan)
448
+ continue
449
+ src_v = ds[vname].values.astype(np.float64)
450
+ if vname not in _STACK_RAW:
451
+ src_v = _apply_qc_mask(src_v, ds, vname)
452
+ valid = np.isfinite(src_v)
453
+ if valid.sum() < 2:
454
+ result[vname] = np.full(len(common_time), np.nan)
455
+ continue
456
+ if vname.endswith("_qc"):
457
+ # QC flag arrays must not be linearly interpolated — use nearest valid
458
+ src_t_v = src_t[valid]
459
+ src_v_v = src_v[valid]
460
+ nn_idx = np.clip(np.searchsorted(src_t_v, tgt_t), 0, len(src_t_v) - 1)
461
+ result[vname] = src_v_v[nn_idx]
462
+ else:
463
+ result[vname] = np.interp(
464
+ tgt_t, src_t[valid], src_v[valid], left=np.nan, right=np.nan
465
+ )
466
+ return result
467
+
468
+
469
+ class MooringStacker:
470
+ """Step 1: stack all instruments onto a common time axis → ``_stack.nc``."""
471
+
472
+ def __init__(
473
+ self,
474
+ base_dir: Optional[str] = None,
475
+ *,
476
+ proc_dir: Optional[str] = None,
477
+ ) -> None:
478
+ """Resample all instruments from a mooring onto a common time axis.
479
+
480
+ Reads ``{mooring}_{serial}_stage3.nc`` (or ``_stage2.nc`` as fallback) for
481
+ every instrument listed in the mooring YAML, resamples each to a uniform
482
+ sampling interval, and writes a single multi-instrument file
483
+ ``{mooring}_stack.nc`` in the mooring proc directory.
484
+
485
+ Parameters
486
+ ----------
487
+ base_dir : str, optional
488
+ Legacy: cruise-level base directory containing a ``proc/`` subdirectory.
489
+ proc_dir : str, optional
490
+ Cruise-level processed output directory. Pipeline appends ``/{mooring}/``.
491
+
492
+ """
493
+ if base_dir is not None:
494
+ self.base_dir: Optional[Path] = Path(base_dir)
495
+ self._proc_dir: Optional[Path] = None
496
+ self._legacy = True
497
+ else:
498
+ self.base_dir = None
499
+ self._proc_dir = Path(proc_dir) if proc_dir else None
500
+ self._legacy = False
501
+
502
+ def _resolve_proc_dir(self, mooring_name: str) -> Path:
503
+ """Return the mooring-level proc directory."""
504
+ if not self._legacy and self._proc_dir is not None:
505
+ return self._proc_dir / mooring_name
506
+ return _get_proc_dir(self.base_dir, mooring_name)
507
+
508
+ def _rel(self, path: Path) -> str:
509
+ """Return a short display path relative to base_dir or proc_dir."""
510
+ for root in (r for r in (self.base_dir, self._proc_dir) if r):
511
+ try:
512
+ return str(path.relative_to(root))
513
+ except ValueError:
514
+ continue
515
+ return path.name
516
+
517
+ def stack(
518
+ self,
519
+ mooring_name: str,
520
+ dt_seconds: int = 60,
521
+ force: bool = False,
522
+ ) -> bool:
523
+ """Stack all processed instruments for *mooring_name* onto a common time grid.
524
+
525
+ Reads ``_stage3.nc`` (falling back to ``_stage2.nc``) for every instrument
526
+ listed in the mooring YAML, resamples each to *dt_seconds* resolution, and
527
+ writes ``{mooring}_stack.nc`` under the mooring proc directory.
528
+
529
+ Returns True on success, False if no instruments could be loaded or an
530
+ unrecoverable error occurs.
531
+ """
532
+ proc_dir = self._resolve_proc_dir(mooring_name)
533
+ try:
534
+ proc_dir_exists = proc_dir.exists()
535
+ except (TimeoutError, OSError) as exc:
536
+ print("ERROR: Cannot access data drive — is it connected?")
537
+ print(f" Path: {proc_dir}")
538
+ print(f" ({type(exc).__name__}: {exc})")
539
+ return False
540
+
541
+ if not proc_dir_exists:
542
+ print(f"ERROR: Processing directory not found: {proc_dir}")
543
+ return False
544
+
545
+ output_path = proc_dir / f"{mooring_name}_stack.nc"
546
+ if output_path.exists() and not force:
547
+ _status("skip", self._rel(output_path))
548
+ return True
549
+
550
+ config_file = proc_dir / f"{mooring_name}.mooring.yaml"
551
+ if not config_file.exists():
552
+ print(f"ERROR: Config not found: {config_file}")
553
+ return False
554
+
555
+ with open(config_file) as f:
556
+ mooring_config = yaml.safe_load(f)
557
+
558
+ deploy_time = np.datetime64(mooring_config["deployment_time"], "ns")
559
+ recover_time = np.datetime64(mooring_config["recovery_time"], "ns")
560
+
561
+ common_time = np.arange(
562
+ deploy_time.astype("datetime64[s]"),
563
+ recover_time.astype("datetime64[s]") + np.timedelta64(1, "s"),
564
+ np.timedelta64(dt_seconds, "s"),
565
+ ).astype("datetime64[ns]")
566
+ n_time = len(common_time)
567
+
568
+ from .utilities import extract_inline_instruments
569
+
570
+ instrument_list = list(
571
+ mooring_config.get("clamp", mooring_config.get("instruments", []))
572
+ )
573
+ instrument_list += extract_inline_instruments(mooring_config.get("inline", []))
574
+
575
+ # Collect instruments with known hab and an available stage2/stage3 file
576
+ instruments = []
577
+ for entry in instrument_list:
578
+ if not isinstance(entry, dict):
579
+ continue
580
+ serial = _safe_serial(entry.get("serial", ""))
581
+ instr_type = entry.get("instrument", "unknown")
582
+ if instr_type not in KNOWN_INSTRUMENT_TYPES:
583
+ print(
584
+ f" WARNING: instrument '{instr_type}' (s/n {serial}) is not in "
585
+ f"KNOWN_INSTRUMENT_TYPES {sorted(KNOWN_INSTRUMENT_TYPES)}. "
586
+ "Aquadopp-specific plots and processing will be skipped. "
587
+ "Allowed values: microcat, aquadopp."
588
+ )
589
+ hab = entry.get("hab")
590
+ if hab is None:
591
+ continue
592
+ yaml_interval = entry.get("sample_interval_seconds")
593
+ nc_path = _best_nc(proc_dir, instr_type, mooring_name, serial)
594
+ if nc_path is None:
595
+ print(f" SKIP {instr_type} s/n {serial}: no stage2/stage3 file")
596
+ continue
597
+ instruments.append(
598
+ {
599
+ "serial": serial,
600
+ "instrument": instr_type,
601
+ "hab": float(hab),
602
+ "yaml_interval": yaml_interval,
603
+ "nc_path": nc_path,
604
+ }
605
+ )
606
+
607
+ if not instruments:
608
+ print("ERROR: No instruments found with stage2/stage3 files and hab values")
609
+ return False
610
+
611
+ # Sort deep-first (ascending hab: smallest hab = nearest bottom = deepest)
612
+ instruments.sort(key=lambda x: x["hab"])
613
+
614
+ # ── Expand ADCP instruments into per-bin pseudo-instruments ───────
615
+ # Each ADCP bin becomes a separate row in the stack, with its own
616
+ # bin_pressure, velocity, and QC time series sliced from the 2-D
617
+ # (time, N_BINS) arrays in the parent stage3 file.
618
+ # The parent file is loaded once and cached; _make_adcp_bin_ds slices it.
619
+ _adcp_parent_datasets: Dict[str, xr.Dataset] = {}
620
+ expanded: List[Dict] = []
621
+ for info in instruments:
622
+ if info["instrument"].lower() != "adcp":
623
+ expanded.append(info)
624
+ continue
625
+ nc_key = str(info["nc_path"])
626
+ if nc_key not in _adcp_parent_datasets:
627
+ try:
628
+ _adcp_parent_datasets[nc_key] = xr.open_dataset(
629
+ info["nc_path"], decode_timedelta=False
630
+ ).load()
631
+ except Exception as e: # noqa: BLE001 — bad ADCP file must not abort stack
632
+ print(f" WARNING: Could not load ADCP {info['nc_path'].name}: {e}")
633
+ expanded.append(info)
634
+ continue
635
+ ds_adcp = _adcp_parent_datasets[nc_key]
636
+ bin_dim = P.ADCP_BIN_DIM
637
+ if bin_dim not in ds_adcp.dims:
638
+ print(
639
+ f" WARNING: ADCP s/n {info['serial']} has no {bin_dim} dim "
640
+ "— treating as point instrument"
641
+ )
642
+ expanded.append(info)
643
+ continue
644
+ n_bins = ds_adcp.sizes[bin_dim]
645
+ range_vals = (
646
+ ds_adcp["range"].values
647
+ if "range" in ds_adcp.coords
648
+ else np.arange(n_bins, dtype=float)
649
+ )
650
+ orientation = ds_adcp.attrs.get("orientation_yaml") or ds_adcp.attrs.get(
651
+ "orientation_instrument"
652
+ )
653
+ if not orientation:
654
+ orientation = "down"
655
+ print(
656
+ f" WARNING: ADCP {info['serial']} has no orientation attr — "
657
+ "assuming downward-looking. HAB offsets may be wrong for "
658
+ "upward-looking instruments."
659
+ )
660
+ looking_down = str(orientation).lower() == "down"
661
+ # Pre-compute seabed mask so always-submerged bins can be skipped.
662
+ # seabed_qc has dims (time, N_BINS); a bin is permanently below the
663
+ # seabed when every time step is flagged suspect or worse (>= 3).
664
+ _seabed_qc_vals = (
665
+ ds_adcp["seabed_qc"].values
666
+ if "seabed_qc" in ds_adcp.data_vars
667
+ else None
668
+ )
669
+
670
+ n_skipped = 0
671
+ valid_bins = []
672
+ for i in range(n_bins):
673
+ if _seabed_qc_vals is not None and np.all(_seabed_qc_vals[:, i] >= 3):
674
+ n_skipped += 1
675
+ else:
676
+ valid_bins.append(i)
677
+
678
+ print(
679
+ f" ADCP s/n {info['serial']}: expanding {len(valid_bins)} of {n_bins} bins "
680
+ f"({orientation}-looking) into stack + head entry"
681
+ + (
682
+ f" [{n_skipped} bins always below seabed, skipped]"
683
+ if n_skipped
684
+ else ""
685
+ )
686
+ )
687
+
688
+ # Head entry — transducer location carries temperature and orientation.
689
+ # Uses the instrument HAB directly (range = 0 from the transducer).
690
+ expanded.append(
691
+ {
692
+ **info,
693
+ "serial": f"{info['serial']}_hd",
694
+ "hab": float(info["hab"]),
695
+ "_adcp_head": True,
696
+ "_adcp_nc_key": nc_key,
697
+ }
698
+ )
699
+ for i in valid_bins:
700
+ r = float(range_vals[i])
701
+ bin_hab = (
702
+ float(info["hab"]) - r if looking_down else float(info["hab"]) + r
703
+ )
704
+ expanded.append(
705
+ {
706
+ **info,
707
+ "serial": f"{info['serial']}_b{i:02d}",
708
+ "hab": bin_hab,
709
+ "_adcp_bin_idx": i,
710
+ "_adcp_nc_key": nc_key,
711
+ }
712
+ )
713
+ instruments = expanded
714
+ # Re-sort after expansion so ADCP bins interleave with other instruments by depth
715
+ instruments.sort(key=lambda x: x["hab"])
716
+
717
+ n_instr = len(instruments)
718
+
719
+ print(
720
+ f"Merging {n_instr} instruments onto {n_time}-point {dt_seconds}s time grid "
721
+ f"({mooring_name})"
722
+ )
723
+
724
+ stacked: Dict[str, np.ndarray] = {
725
+ v: np.full((n_instr, n_time), np.nan) for v in STACK_VARS
726
+ }
727
+ var_attrs: Dict[str, dict] = {v: {} for v in STACK_VARS}
728
+ serials: List[str] = []
729
+ habs: List[float] = []
730
+ instr_types: List[str] = []
731
+ stage_labels: List[str] = []
732
+ # Per-instrument scalar metadata: {varname: [value_for_instr0, value_for_instr1, ...]}
733
+ scalar_meta: Dict[str, list] = {} # populated during loop
734
+ scalar_attrs: Dict[str, dict] = {}
735
+
736
+ for i, info in enumerate(instruments):
737
+ serials.append(info["serial"])
738
+ habs.append(info["hab"])
739
+ instr_types.append(info["instrument"])
740
+ stage_labels.append(info["nc_path"].stem.split("_")[-1])
741
+
742
+ try:
743
+ if "_adcp_bin_idx" in info:
744
+ ds_parent = _adcp_parent_datasets[info["_adcp_nc_key"]]
745
+ ds = _make_adcp_bin_ds(ds_parent, info["_adcp_bin_idx"])
746
+ elif "_adcp_head" in info:
747
+ ds_parent = _adcp_parent_datasets[info["_adcp_nc_key"]]
748
+ ds = _make_adcp_head_ds(ds_parent)
749
+ else:
750
+ ds = xr.open_dataset(info["nc_path"], decode_timedelta=False).load()
751
+ ds.close()
752
+ except Exception as e: # noqa: BLE001 — one bad instrument must not abort the stack
753
+ print(f" WARNING: Could not load {info['nc_path'].name}: {e}")
754
+ # Ensure scalar_meta lists stay length-consistent
755
+ for lst in scalar_meta.values():
756
+ lst.append(None)
757
+ continue
758
+
759
+ interval_s = (
760
+ float(info["yaml_interval"])
761
+ if info["yaml_interval"]
762
+ else _detect_interval_s(ds["time"].values)
763
+ )
764
+ print(
765
+ f" [{i:2d}] {info['instrument']:10s} s/n {info['serial']:<8} "
766
+ f"hab={info['hab']:6.1f} m dt={interval_s:.0f}s "
767
+ f"({stage_labels[-1]})"
768
+ )
769
+
770
+ half_window = dt_seconds / 2.0
771
+ if interval_s <= dt_seconds:
772
+ values = _nearest_subsample(ds, common_time, half_window)
773
+ else:
774
+ values = _linear_interp(ds, common_time)
775
+
776
+ for vname in STACK_VARS:
777
+ arr = values[vname]
778
+ n = min(len(arr), n_time)
779
+ stacked[vname][i, :n] = arr[:n]
780
+ if not var_attrs[vname] and vname in ds.data_vars:
781
+ var_attrs[vname] = dict(ds[vname].attrs)
782
+
783
+ # Collect scalar (0-D) metadata variables — keep all instruments consistent
784
+ for vname, da in ds.data_vars.items():
785
+ if da.dims: # skip time-series variables
786
+ continue
787
+ if vname not in scalar_meta:
788
+ # Back-fill with None for instruments processed before this variable appeared
789
+ scalar_meta[vname] = [None] * i
790
+ scalar_attrs[vname] = dict(da.attrs)
791
+ scalar_meta[vname].append(da.values.item())
792
+
793
+ # Collect magnetic_declination from global attrs (set by stage3 BEAM→ENU).
794
+ # Must come BEFORE the fill-None loop so the length is already i+1 when
795
+ # the loop checks.
796
+ if "magnetic_declination" not in scalar_meta:
797
+ scalar_meta["magnetic_declination"] = [np.nan] * i
798
+ scalar_attrs["magnetic_declination"] = {
799
+ "units": "degrees_east",
800
+ "long_name": "Magnetic declination (IGRF)",
801
+ }
802
+ decl_val = ds.attrs.get("magnetic_declination")
803
+ scalar_meta["magnetic_declination"].append(
804
+ float(decl_val) if decl_val is not None else np.nan
805
+ )
806
+
807
+ # Fill None for variables not present in this instrument
808
+ for vname in scalar_meta:
809
+ if len(scalar_meta[vname]) < i + 1:
810
+ scalar_meta[vname].append(None)
811
+
812
+ # Release cached ADCP parent datasets
813
+ for _ds_adcp in _adcp_parent_datasets.values():
814
+ _ds_adcp.close()
815
+
816
+ # Build output dataset; skip physics variables that are entirely NaN
817
+ data_vars: Dict = {}
818
+ for vname in STACK_VARS:
819
+ if not np.all(np.isnan(stacked[vname])):
820
+ data_vars[vname] = xr.Variable(
821
+ ("N_LEVELS", "time"), stacked[vname], attrs=var_attrs[vname]
822
+ )
823
+
824
+ # Compute potential density from stacked T, S, P
825
+ ref_p = int(mooring_config.get("density_reference", P.DENSITY_REFERENCE))
826
+ if (
827
+ "temperature" in data_vars
828
+ and "salinity" in data_vars
829
+ and "pressure" in data_vars
830
+ ):
831
+ try:
832
+ import gsw
833
+
834
+ lat, lon = _parse_latlon(mooring_config)
835
+ T_arr = stacked["temperature"]
836
+ SP_arr = stacked["salinity"]
837
+ P_arr = stacked["pressure"]
838
+ SA = gsw.SA_from_SP(SP_arr, P_arr, lon, lat)
839
+ CT = gsw.CT_from_t(SA, T_arr, P_arr)
840
+ _sigma_fn = {
841
+ 0: gsw.sigma0,
842
+ 1000: gsw.sigma1,
843
+ 2000: gsw.sigma2,
844
+ 3000: gsw.sigma3,
845
+ 4000: gsw.sigma4,
846
+ }
847
+ if ref_p in _sigma_fn:
848
+ sigma_vals = _sigma_fn[ref_p](SA, CT)
849
+ else:
850
+ sigma_vals = gsw.pot_rho_t_exact(SA, T_arr, P_arr, ref_p) - 1000.0
851
+ if not np.all(np.isnan(sigma_vals)):
852
+ vname, long_name, std_name = _SIGMA_META.get(
853
+ ref_p,
854
+ (
855
+ f"sigma_{ref_p}",
856
+ f"Potential density anomaly referenced to {ref_p} dbar",
857
+ "",
858
+ ),
859
+ )
860
+ data_vars[vname] = xr.Variable(
861
+ ("N_LEVELS", "time"),
862
+ sigma_vals,
863
+ {
864
+ "units": "kg m-3",
865
+ "long_name": long_name,
866
+ "standard_name": std_name,
867
+ "reference_pressure_dbar": ref_p,
868
+ },
869
+ )
870
+ except Exception as exc: # noqa: BLE001 — gsw failure must not abort stack
871
+ print(f" WARNING: could not compute potential density: {exc}")
872
+
873
+ # velocity_flag: element-wise worst QC flag across east/north/up velocity.
874
+ # Velocity is stored unmasked in the stack; this combined flag is what the
875
+ # grid step (and users) should apply before using velocity values.
876
+ _vel_qc_keys = [
877
+ v
878
+ for v in ("east_velocity_qc", "north_velocity_qc", "up_velocity_qc")
879
+ if v in data_vars
880
+ ]
881
+ if _vel_qc_keys:
882
+ n_lev_v, n_t_v = (
883
+ stacked["east_velocity_qc"].shape
884
+ if "east_velocity_qc" in stacked
885
+ else (0, 0)
886
+ )
887
+ if n_lev_v > 0:
888
+ vel_flag = np.ones((n_lev_v, n_t_v), dtype=np.float64) # 1 = good
889
+ for _qk in _vel_qc_keys:
890
+ vel_flag = _worst_flag(vel_flag, stacked[_qk]).astype(np.float64)
891
+ data_vars["velocity_flag"] = xr.Variable(
892
+ ("N_LEVELS", "time"),
893
+ vel_flag,
894
+ {
895
+ "long_name": "Combined velocity QC flag (worst of east/north/up)",
896
+ "comment": (
897
+ "OceanSITES flag: 1=good, 2=prob_good, 3=suspect, 4=bad, "
898
+ "9=missing. Apply to east/north/up_velocity before use."
899
+ ),
900
+ "flag_values": "0 1 2 3 4 9",
901
+ "flag_meanings": (
902
+ "no_qc_performed good_data probably_good_data "
903
+ "probably_bad_data bad_data missing_value"
904
+ ),
905
+ },
906
+ )
907
+
908
+ # Tilt estimated from pressure difference between two instrument levels.
909
+ # For each level i, find the nearest level j above it that is ≥10 m away
910
+ # in hab AND has at least some finite pressure data. Instruments close
911
+ # together (e.g. a microcat strapped to an Aquadopp frame) are skipped.
912
+ # rope_length = hab[j] - hab[i] (m, from YAML)
913
+ # ΔP = pressure[i,:] - pressure[j,:] (dbar ≈ m, >0 when upright)
914
+ # tilt = arccos(ΔP / rope_length) (degrees from vertical; 0 = upright)
915
+ _MIN_HAB_SEP = 10.0 # minimum hab separation (m) to use as reference
916
+ if "pressure" in data_vars and len(habs) > 1:
917
+ try:
918
+ p_arr = stacked["pressure"] # (N_LEVELS, time) numpy array
919
+ n_lev = len(habs)
920
+ tilt_p = np.full_like(p_arr, np.nan)
921
+ ref_hab_arr = np.full(n_lev, np.nan) # hab of the reference level
922
+ ref_serial_arr = np.array([""] * n_lev, dtype=object)
923
+ # pre-compute which levels have any valid pressure
924
+ _has_p = np.array(
925
+ [np.any(np.isfinite(p_arr[k, :])) for k in range(n_lev)]
926
+ )
927
+ _is_adcp = [t.lower() == "adcp" for t in instr_types]
928
+ for i in range(n_lev):
929
+ if not _has_p[i]:
930
+ continue
931
+ if _is_adcp[i]:
932
+ continue # ADCP transducer pressure is not suitable for tilt
933
+ # Find nearest level above i that is ≥_MIN_HAB_SEP away with pressure,
934
+ # excluding ADCP levels as reference (their bin pressures shift with tilt).
935
+ ref_j = None
936
+ for j in range(i + 1, n_lev):
937
+ if float(habs[j]) - float(habs[i]) < _MIN_HAB_SEP:
938
+ continue
939
+ if not _has_p[j]:
940
+ continue
941
+ if _is_adcp[j]:
942
+ continue
943
+ ref_j = j
944
+ break # first valid j = nearest ≥10 m above
945
+ if ref_j is None:
946
+ continue
947
+ rope = float(habs[ref_j]) - float(habs[i])
948
+ ref_hab_arr[i] = float(habs[ref_j])
949
+ ref_serial_arr[i] = str(serials[ref_j])
950
+ delta_p = p_arr[i, :] - p_arr[ref_j, :]
951
+ ratio = np.clip(delta_p / rope, 0.0, 1.0)
952
+ tilt_p[i, :] = np.degrees(np.arccos(ratio))
953
+ tilt_p[i, ~np.isfinite(delta_p)] = np.nan
954
+ print(
955
+ f" tilt_from_pressure[{i}] s/n {serials[i]} ({habs[i]:.0f} m): "
956
+ f"ref s/n {serials[ref_j]} ({habs[ref_j]:.0f} m, "
957
+ f"rope={rope:.0f} m)"
958
+ )
959
+ data_vars["tilt_from_pressure"] = xr.Variable(
960
+ ("N_LEVELS", "time"),
961
+ tilt_p,
962
+ {
963
+ "units": "degrees",
964
+ "long_name": "Tilt estimated from pressure difference",
965
+ "comment": (
966
+ "arccos(ΔP / rope_length) where ΔP = pressure[i] - pressure[ref] "
967
+ "and rope_length = hab[ref] - hab[i] from mooring YAML; "
968
+ "ref is the nearest instrument ≥10 m above i with valid pressure."
969
+ ),
970
+ },
971
+ )
972
+ data_vars["tilt_pressure_ref_hab"] = xr.Variable(
973
+ ("N_LEVELS",),
974
+ ref_hab_arr,
975
+ {
976
+ "units": "m",
977
+ "long_name": "Height above bottom of the pressure reference instrument used for tilt",
978
+ },
979
+ )
980
+ data_vars["tilt_pressure_ref_serial"] = xr.Variable(
981
+ ("N_LEVELS",),
982
+ ref_serial_arr,
983
+ {
984
+ "long_name": "Serial number of the pressure reference instrument used for tilt"
985
+ },
986
+ )
987
+ except Exception as exc: # noqa: BLE001 — optional derived var; must not abort stack
988
+ print(f" WARNING: could not compute tilt_from_pressure: {exc}")
989
+
990
+ # Coordinate names — exclude these from scalar metadata to avoid name conflicts
991
+ _coord_names = {"serial", "hab", "instrument_type", "instrument", "time"}
992
+
993
+ # Add scalar metadata — collapse to 0-D when all non-NaN values are identical,
994
+ # otherwise store as (N_LEVELS,) with one entry per instrument.
995
+ for vname, values_list in scalar_meta.items():
996
+ if vname in _coord_names:
997
+ continue
998
+ try:
999
+ arr = np.array(
1000
+ [v if v is not None else np.nan for v in values_list],
1001
+ dtype=np.float64,
1002
+ )
1003
+ unique_finite = np.unique(arr[np.isfinite(arr)])
1004
+ if len(unique_finite) <= 1:
1005
+ val = float(unique_finite[0]) if len(unique_finite) == 1 else np.nan
1006
+ data_vars[vname] = xr.Variable((), val, attrs=scalar_attrs[vname])
1007
+ else:
1008
+ data_vars[vname] = xr.Variable(
1009
+ ("N_LEVELS",), arr, attrs=scalar_attrs[vname]
1010
+ )
1011
+ except (ValueError, TypeError):
1012
+ arr = np.array([str(v) if v is not None else "" for v in values_list])
1013
+ unique_vals = np.unique(arr)
1014
+ if len(unique_vals) == 1:
1015
+ data_vars[vname] = xr.Variable(
1016
+ (), unique_vals[0], attrs=scalar_attrs[vname]
1017
+ )
1018
+ else:
1019
+ data_vars[vname] = xr.Variable(
1020
+ ("N_LEVELS",), arr, attrs=scalar_attrs[vname]
1021
+ )
1022
+
1023
+ ds_out = xr.Dataset(
1024
+ data_vars,
1025
+ coords={
1026
+ "time": xr.Variable(
1027
+ "time",
1028
+ common_time,
1029
+ {"long_name": "time", "axis": "T", "standard_name": "time"},
1030
+ ),
1031
+ "serial": xr.Variable(
1032
+ "N_LEVELS",
1033
+ np.array(serials),
1034
+ {"long_name": "instrument serial number"},
1035
+ ),
1036
+ "hab": xr.Variable(
1037
+ "N_LEVELS",
1038
+ np.array(habs),
1039
+ {"units": "m", "long_name": "height above bottom"},
1040
+ ),
1041
+ "instrument_type": xr.Variable(
1042
+ "N_LEVELS",
1043
+ np.array(instr_types),
1044
+ {"long_name": "instrument type"},
1045
+ ),
1046
+ },
1047
+ )
1048
+ _lat_str = (
1049
+ mooring_config.get("seabed_latitude")
1050
+ or mooring_config.get("deployment_latitude")
1051
+ or mooring_config.get("planned_latitude")
1052
+ or mooring_config.get("latitude")
1053
+ or ""
1054
+ )
1055
+ _lon_str = (
1056
+ mooring_config.get("seabed_longitude")
1057
+ or mooring_config.get("deployment_longitude")
1058
+ or mooring_config.get("planned_longitude")
1059
+ or mooring_config.get("longitude")
1060
+ or ""
1061
+ )
1062
+ ds_out.attrs.update(
1063
+ {
1064
+ "mooring_name": mooring_name,
1065
+ "waterdepth": str(mooring_config.get("waterdepth", "")),
1066
+ "latitude": str(_lat_str),
1067
+ "longitude": str(_lon_str),
1068
+ "deployment_time": str(deploy_time),
1069
+ "recovery_time": str(recover_time),
1070
+ "dt_seconds": dt_seconds,
1071
+ "Conventions": "CF-1.13",
1072
+ "history": (
1073
+ f"Step 1 stack: {n_instr} instruments onto {dt_seconds}s grid; "
1074
+ f"fast instruments (dt<={dt_seconds}s) subsampled (nearest-neighbour in time), "
1075
+ f"slow instruments interpolated (linear in time)"
1076
+ ),
1077
+ }
1078
+ )
1079
+
1080
+ if output_path.exists():
1081
+ output_path.unlink()
1082
+ # Beam velocities are superseded by ENU components in the stacked file.
1083
+ beam_vel_vars = [v for v in ds_out.data_vars if v.startswith("velocity_beam")]
1084
+ if beam_vel_vars:
1085
+ ds_out = ds_out.drop_vars(beam_vel_vars)
1086
+ ds_out = drop_all_zero_vars(ds_out, ["amplitude_beam", "analog_input_"])
1087
+ # OceanSITES convention: time is the unlimited (first) dimension.
1088
+ ds_out = ds_out.transpose("time", "N_LEVELS")
1089
+ ds_out = cast_output_dtypes(ds_out)
1090
+ _enc = {
1091
+ v: {"zlib": True, "complevel": 5}
1092
+ for v in ds_out.data_vars
1093
+ if ds_out[v].dtype.kind not in ("O", "U", "S")
1094
+ }
1095
+ ds_out.to_netcdf(output_path, encoding=_enc)
1096
+ _status("file", self._rel(output_path))
1097
+ return True
1098
+
1099
+
1100
+ class MooringGridder:
1101
+ """Step 2: vertically interpolate stacked instruments onto a pressure grid → ``_grid.nc``."""
1102
+
1103
+ def __init__(
1104
+ self,
1105
+ base_dir: Optional[str] = None,
1106
+ *,
1107
+ proc_dir: Optional[str] = None,
1108
+ ) -> None:
1109
+ """Interpolate a mooring stack onto a regular pressure grid.
1110
+
1111
+ Reads ``{mooring}_stack.nc`` (produced by :class:`MooringStacker`) and
1112
+ interpolates all scalar variables onto a uniform pressure axis defined by
1113
+ ``p_start``, ``p_end``, and ``dp`` (in dbar). Writes
1114
+ ``{mooring}_grid.nc`` in the mooring proc directory.
1115
+
1116
+ Parameters
1117
+ ----------
1118
+ base_dir : str, optional
1119
+ Legacy: cruise-level base directory containing a ``proc/`` subdirectory.
1120
+ proc_dir : str, optional
1121
+ Cruise-level processed output directory. Pipeline appends ``/{mooring}/``.
1122
+
1123
+ """
1124
+ if base_dir is not None:
1125
+ self.base_dir: Optional[Path] = Path(base_dir)
1126
+ self._proc_dir: Optional[Path] = None
1127
+ self._legacy = True
1128
+ else:
1129
+ self.base_dir = None
1130
+ self._proc_dir = Path(proc_dir) if proc_dir else None
1131
+ self._legacy = False
1132
+
1133
+ def _resolve_proc_dir(self, mooring_name: str) -> Path:
1134
+ """Return the mooring-level proc directory."""
1135
+ if not self._legacy and self._proc_dir is not None:
1136
+ return self._proc_dir / mooring_name
1137
+ return _get_proc_dir(self.base_dir, mooring_name)
1138
+
1139
+ def _rel(self, path: Path) -> str:
1140
+ """Return a short display path relative to base_dir or proc_dir."""
1141
+ for root in (r for r in (self.base_dir, self._proc_dir) if r):
1142
+ try:
1143
+ return str(path.relative_to(root))
1144
+ except ValueError:
1145
+ continue
1146
+ return path.name
1147
+
1148
+ def grid(
1149
+ self,
1150
+ mooring_name: str,
1151
+ p_start: float = 200.0,
1152
+ p_end: float = 1000.0,
1153
+ dp: float = 20.0,
1154
+ force: bool = False,
1155
+ ) -> bool:
1156
+ """Interpolate the stacked mooring dataset onto a regular pressure grid.
1157
+
1158
+ Reads ``{mooring}_stack.nc``, interpolates all variables onto a pressure
1159
+ axis from *p_start* to *p_end* in steps of *dp* dbar, and writes
1160
+ ``{mooring}_grid.nc``.
1161
+
1162
+ Returns True on success, False on error.
1163
+ """
1164
+ proc_dir = self._resolve_proc_dir(mooring_name)
1165
+ merge_path = proc_dir / f"{mooring_name}_stack.nc"
1166
+ output_path = proc_dir / f"{mooring_name}_grid.nc"
1167
+
1168
+ try:
1169
+ stack_found = merge_path.exists()
1170
+ output_found = output_path.exists()
1171
+ except (TimeoutError, OSError) as exc:
1172
+ print("ERROR: Cannot access data drive — is it connected?")
1173
+ print(f" Path: {merge_path.parent}")
1174
+ print(f" ({type(exc).__name__}: {exc})")
1175
+ return False
1176
+
1177
+ if not stack_found:
1178
+ print(f"ERROR: Stack file not found: {merge_path}")
1179
+ print(" Run 'oceanarray stack' first.")
1180
+ return False
1181
+
1182
+ if output_found and not force:
1183
+ _status("skip", self._rel(output_path))
1184
+ return True
1185
+
1186
+ p_grid = np.arange(p_start, p_end + dp * 0.5, dp)
1187
+ n_p = len(p_grid)
1188
+
1189
+ try:
1190
+ ds = xr.open_dataset(merge_path).load()
1191
+ except (TimeoutError, OSError) as exc:
1192
+ print("ERROR: Cannot read stack file — is the data drive connected?")
1193
+ print(f" ({type(exc).__name__}: {exc})")
1194
+ return False
1195
+ n_time = ds.sizes["time"]
1196
+ n_instr = ds.sizes["N_LEVELS"]
1197
+
1198
+ print(
1199
+ f"Gridding {n_instr} instruments → {n_p} pressure levels "
1200
+ f"({p_start:.0f}:{dp:.0f}:{p_end:.0f} dbar) × {n_time} time steps"
1201
+ )
1202
+
1203
+ if "pressure" not in ds.data_vars:
1204
+ print("ERROR: 'pressure' not found in stack file — cannot grid vertically")
1205
+ ds.close()
1206
+ return False
1207
+
1208
+ pressure = ds["pressure"].values.astype(np.float64) # (time, N_LEVELS)
1209
+ # Variables that are meaningful at the stacked per-instrument level but
1210
+ # should not be interpolated onto the pressure grid. Instrument-frame
1211
+ # and beam-frame velocities are excluded because the XYZ→ENU rotation
1212
+ # has already been applied; gridding the pre-rotation components would
1213
+ # be misleading. Diagnostic quantities (amplitude, correlation,
1214
+ # battery) are per-sensor and do not have a physical meaning on a
1215
+ # spatially-interpolated grid.
1216
+ _GRID_EXCLUDE: frozenset = frozenset(
1217
+ {
1218
+ "velocity_x",
1219
+ "velocity_y",
1220
+ "velocity_z",
1221
+ "velocity_beam1",
1222
+ "velocity_beam2",
1223
+ "velocity_beam3",
1224
+ "amplitude",
1225
+ "correlation",
1226
+ "battery_voltage",
1227
+ "velocity_flag", # flag array, not a gridded physics variable
1228
+ "tilt_from_pressure", # per-instrument diagnostic, not gridded
1229
+ "tilt_pressure_ref_hab",
1230
+ "heading", # instrument-frame orientation — not meaningful on a pressure grid
1231
+ "pitch",
1232
+ "roll",
1233
+ }
1234
+ )
1235
+ grid_vars = [
1236
+ v
1237
+ for v in ds.data_vars
1238
+ if v != "pressure"
1239
+ and ds[v].dims == ("time", "N_LEVELS")
1240
+ and not v.endswith("_qc")
1241
+ and v not in _GRID_EXCLUDE
1242
+ ]
1243
+
1244
+ stacked: Dict[str, np.ndarray] = {
1245
+ v: np.full((n_p, n_time), np.nan) for v in grid_vars
1246
+ }
1247
+ var_data = {v: ds[v].values.astype(np.float64) for v in grid_vars}
1248
+
1249
+ # QC masking before vertical interpolation.
1250
+ # T/S/P: NaN any non-finite values (already masked at stack time).
1251
+ # Velocity: use velocity_flag from the stack to NaN suspect/bad samples.
1252
+ # Flag 3 (suspect), 4 (bad), 9 (missing) → NaN before interpolation.
1253
+ _GRIDDER_TSQC = {"temperature", "conductivity", "salinity"}
1254
+ for _v in grid_vars:
1255
+ if _v in _GRIDDER_TSQC:
1256
+ var_data[_v][~np.isfinite(var_data[_v])] = np.nan
1257
+
1258
+ _vel_vars = {
1259
+ "east_velocity",
1260
+ "north_velocity",
1261
+ "up_velocity",
1262
+ "current_speed",
1263
+ "current_direction",
1264
+ }
1265
+ if "velocity_flag" in ds.data_vars:
1266
+ _vflag = ds["velocity_flag"].values.astype(np.float64)
1267
+ _bad_vel = np.isin(np.round(_vflag).astype(np.int8), [3, 4, 9])
1268
+ for _v in _vel_vars:
1269
+ if _v in var_data:
1270
+ var_data[_v][_bad_vel] = np.nan
1271
+ else:
1272
+ for _v in _vel_vars:
1273
+ if _v in var_data:
1274
+ var_data[_v][~np.isfinite(var_data[_v])] = np.nan
1275
+
1276
+ # ADCP standalone QC variables — mask velocity before gridding.
1277
+ # seabed_qc removes bins at/below the seafloor; percent_good_qc and
1278
+ # error_velocity_qc remove poor-quality pings. All three use flag >= 3
1279
+ # (suspect or worse) as the NaN threshold, consistent with how the
1280
+ # rose diagram and stack report mask these variables.
1281
+ for _adcp_qc in ("seabed_qc", "percent_good_qc", "error_velocity_qc"):
1282
+ if _adcp_qc not in ds.data_vars:
1283
+ continue
1284
+ _qc = ds[_adcp_qc].values # (time, N_LEVELS) after stack transpose
1285
+ _bad = np.isin(np.round(_qc).astype(np.int8), [3, 4, 9])
1286
+ for _v in _vel_vars:
1287
+ if _v in var_data:
1288
+ var_data[_v][_bad] = np.nan
1289
+
1290
+ for t in range(n_time):
1291
+ p_col = pressure[t, :]
1292
+ p_valid_mask = np.isfinite(p_col)
1293
+ if p_valid_mask.sum() < 2:
1294
+ continue
1295
+
1296
+ for vname in grid_vars:
1297
+ v_col = var_data[vname][t, :]
1298
+ both_valid = p_valid_mask & np.isfinite(v_col)
1299
+ if both_valid.sum() < 2:
1300
+ continue
1301
+ p_v = p_col[both_valid]
1302
+ v_v = v_col[both_valid]
1303
+ sort_idx = np.argsort(p_v)
1304
+ stacked[vname][:, t] = np.interp(
1305
+ p_grid,
1306
+ p_v[sort_idx],
1307
+ v_v[sort_idx],
1308
+ left=np.nan,
1309
+ right=np.nan,
1310
+ )
1311
+
1312
+ # Build output dataset; skip variables that are entirely NaN
1313
+ data_vars: Dict = {}
1314
+ for vname in grid_vars:
1315
+ if not np.all(np.isnan(stacked[vname])):
1316
+ a = dict(ds[vname].attrs)
1317
+ a["vertical_interpolation"] = (
1318
+ f"linear in pressure; no extrapolation outside {p_start:.0f}–{p_end:.0f} dbar"
1319
+ )
1320
+ data_vars[vname] = xr.Variable(
1321
+ ("time", "pressure"), stacked[vname].T, attrs=a
1322
+ )
1323
+
1324
+ ds_out = xr.Dataset(
1325
+ data_vars,
1326
+ coords={
1327
+ "time": ds["time"],
1328
+ "pressure": xr.Variable(
1329
+ "pressure",
1330
+ p_grid,
1331
+ {
1332
+ "units": "dbar",
1333
+ "long_name": "sea water pressure",
1334
+ "standard_name": "sea_water_pressure",
1335
+ "axis": "Z",
1336
+ "positive": "down",
1337
+ },
1338
+ ),
1339
+ },
1340
+ )
1341
+
1342
+ prior_history = ds.attrs.get("history", "")
1343
+ ds_out.attrs.update(
1344
+ {
1345
+ **{k: v for k, v in ds.attrs.items() if k not in ("history",)},
1346
+ "p_start_dbar": p_start,
1347
+ "p_end_dbar": p_end,
1348
+ "dp_dbar": dp,
1349
+ "history": (
1350
+ prior_history
1351
+ + f"; Step 2 grid: linear interpolation in pressure onto {dp:.0f} dbar pressure grid "
1352
+ f"({p_start:.0f}–{p_end:.0f} dbar); no extrapolation"
1353
+ ),
1354
+ }
1355
+ )
1356
+
1357
+ ds.close()
1358
+ if output_path.exists():
1359
+ output_path.unlink()
1360
+ ds_out = cast_output_dtypes(ds_out)
1361
+ _enc = {
1362
+ v: {"zlib": True, "complevel": 5}
1363
+ for v in ds_out.data_vars
1364
+ if ds_out[v].dtype.kind not in ("O", "U", "S")
1365
+ }
1366
+ ds_out.to_netcdf(output_path, encoding=_enc)
1367
+ _status("file", self._rel(output_path))
1368
+ return True