oceanarray 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oceanarray/__init__.py +20 -0
- oceanarray/_version.py +1 -0
- oceanarray/cli.py +1343 -0
- oceanarray/clock_offset.py +675 -0
- oceanarray/config/OS1_sensor_attrs.yaml +49 -0
- oceanarray/config/OS1_var_names.yaml +5 -0
- oceanarray/config/OS1_vocab_attrs.yaml +84 -0
- oceanarray/config/legacy/project_RAPID.yaml +69 -0
- oceanarray/config/legacy/rodb_keys.txt +206 -0
- oceanarray/config/legacy/rodb_keys.yaml +1012 -0
- oceanarray/config/logging.yaml +28 -0
- oceanarray/legacy/__init__.py +47 -0
- oceanarray/legacy/caldip.py +753 -0
- oceanarray/legacy/convertOS.py +527 -0
- oceanarray/legacy/find_deployment.py +489 -0
- oceanarray/legacy/mooring_rodb.py +429 -0
- oceanarray/legacy/process_rodb.py +369 -0
- oceanarray/legacy/rodb.py +330 -0
- oceanarray/logger.py +192 -0
- oceanarray/logsheet/__init__.py +60 -0
- oceanarray/logsheet/_columns.py +97 -0
- oceanarray/logsheet/_config.py +245 -0
- oceanarray/logsheet/_firmware.py +43 -0
- oceanarray/logsheet/_latex.py +128 -0
- oceanarray/logsheet/_render.py +126 -0
- oceanarray/logsheet/builders/__init__.py +13 -0
- oceanarray/logsheet/builders/caldip_download.py +195 -0
- oceanarray/logsheet/builders/caldip_setup.py +188 -0
- oceanarray/logsheet/builders/deployment_setup.py +218 -0
- oceanarray/logsheet/builders/mooring_download.py +243 -0
- oceanarray/logsheet/builders/recovery.py +157 -0
- oceanarray/logsheet/data/column_defs.yaml +686 -0
- oceanarray/logsheet/data/latex_templates/_base_instrument.tex.j2 +42 -0
- oceanarray/logsheet/data/latex_templates/caldip_download.tex.j2 +47 -0
- oceanarray/logsheet/data/latex_templates/caldip_setup.tex.j2 +70 -0
- oceanarray/logsheet/data/latex_templates/deployment_setup.tex.j2 +97 -0
- oceanarray/logsheet/data/latex_templates/mooring_download.tex.j2 +54 -0
- oceanarray/logsheet/data/latex_templates/recovery.tex.j2 +103 -0
- oceanarray/mooring_level.py +1368 -0
- oceanarray/oceanarray.mplstyle +18 -0
- oceanarray/parameters.py +270 -0
- oceanarray/plotter.py +942 -0
- oceanarray/plotters/__init__.py +97 -0
- oceanarray/plotters/_animation.py +261 -0
- oceanarray/plotters/_current.py +705 -0
- oceanarray/plotters/_diagnostic.py +667 -0
- oceanarray/plotters/_helpers.py +57 -0
- oceanarray/plotters/_primitives.py +101 -0
- oceanarray/plotters/_section.py +16 -0
- oceanarray/plotters/_timeseries.py +11 -0
- oceanarray/rapid_interp.py +739 -0
- oceanarray/read_rbr_hex.py +541 -0
- oceanarray/readers.py +303 -0
- oceanarray/report/__init__.py +5 -0
- oceanarray/report/_array.py +386 -0
- oceanarray/report/_grid.py +539 -0
- oceanarray/report/_html_helpers.py +1004 -0
- oceanarray/report/_instrument.py +853 -0
- oceanarray/report/_mooring.py +1385 -0
- oceanarray/report/_plots.py +4284 -0
- oceanarray/report/_stack.py +941 -0
- oceanarray/stage1.py +1959 -0
- oceanarray/stage2.py +1160 -0
- oceanarray/stage3.py +2494 -0
- oceanarray/time_gridding.py +958 -0
- oceanarray/tools.py +400 -0
- oceanarray/transports.py +1 -0
- oceanarray/utilities.py +538 -0
- oceanarray/validation.py +389 -0
- oceanarray/writers.py +99 -0
- oceanarray-0.1.0.dist-info/METADATA +503 -0
- oceanarray-0.1.0.dist-info/RECORD +76 -0
- oceanarray-0.1.0.dist-info/WHEEL +5 -0
- oceanarray-0.1.0.dist-info/entry_points.txt +2 -0
- oceanarray-0.1.0.dist-info/licenses/LICENSE +21 -0
- oceanarray-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1368 @@
|
|
|
1
|
+
"""Mooring-level processing: instrument stacking and vertical gridding.
|
|
2
|
+
|
|
3
|
+
Step 1 — stack (``oceanarray stack``):
|
|
4
|
+
Read all instruments from a single deployment and stack them onto a common
|
|
5
|
+
60 s (or user-specified) time axis. Output: ``{mooring}_stack.nc`` with
|
|
6
|
+
dimensions ``(instrument, time)``. --> change to (time, instrument)!
|
|
7
|
+
|
|
8
|
+
Step 2 — grid (``oceanarray grid``):
|
|
9
|
+
Vertically interpolate the stacked dataset onto a standard pressure grid.
|
|
10
|
+
Output: ``{mooring}_grid.nc`` with dimensions ``(pressure, time)``.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Dict, List, Optional
|
|
15
|
+
import re
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
import xarray as xr
|
|
19
|
+
import yaml
|
|
20
|
+
from . import parameters as P
|
|
21
|
+
from .utilities import _status, cast_output_dtypes, drop_all_zero_vars
|
|
22
|
+
|
|
23
|
+
# Canonical allowed values for the ``instrument:`` field in mooring YAML files.
|
|
24
|
+
# Defined in parameters.py alongside the instrument→file_type mapping.
|
|
25
|
+
KNOWN_INSTRUMENT_TYPES: frozenset = P.KNOWN_INSTRUMENT_TYPES
|
|
26
|
+
|
|
27
|
+
STACK_VARS = [
|
|
28
|
+
"temperature",
|
|
29
|
+
"temperature_qc",
|
|
30
|
+
"salinity",
|
|
31
|
+
"salinity_qc",
|
|
32
|
+
"conductivity",
|
|
33
|
+
"pressure",
|
|
34
|
+
"pressure_qc",
|
|
35
|
+
"east_velocity",
|
|
36
|
+
"north_velocity",
|
|
37
|
+
"up_velocity",
|
|
38
|
+
"current_speed",
|
|
39
|
+
"current_direction",
|
|
40
|
+
"east_velocity_qc",
|
|
41
|
+
"north_velocity_qc",
|
|
42
|
+
"up_velocity_qc",
|
|
43
|
+
"pitch_qc",
|
|
44
|
+
"roll_qc",
|
|
45
|
+
"velocity_beam1",
|
|
46
|
+
"velocity_beam2",
|
|
47
|
+
"velocity_beam3",
|
|
48
|
+
"heading",
|
|
49
|
+
"pitch",
|
|
50
|
+
"roll",
|
|
51
|
+
"analog_input_1",
|
|
52
|
+
"analog_input_2",
|
|
53
|
+
"percent_good_qc",
|
|
54
|
+
"error_velocity_qc",
|
|
55
|
+
"seabed_qc",
|
|
56
|
+
"turbidity",
|
|
57
|
+
"turbidity_qc",
|
|
58
|
+
"dissolved_oxygen",
|
|
59
|
+
"dissolved_oxygen_qc",
|
|
60
|
+
"oxygen_saturation_pct",
|
|
61
|
+
"apparent_oxygen_utilization",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
# Variables passed through without QC masking at the stack step.
|
|
65
|
+
# Velocity and orientation are kept at full precision so downstream users
|
|
66
|
+
# (grid step, analysis) can apply their own masking via velocity_flag.
|
|
67
|
+
# QC flag arrays never need masking (there is no companion *_qc_qc* variable).
|
|
68
|
+
_STACK_RAW: frozenset = frozenset(
|
|
69
|
+
{
|
|
70
|
+
"east_velocity",
|
|
71
|
+
"north_velocity",
|
|
72
|
+
"up_velocity",
|
|
73
|
+
"current_speed",
|
|
74
|
+
"current_direction",
|
|
75
|
+
"velocity_beam1",
|
|
76
|
+
"velocity_beam2",
|
|
77
|
+
"velocity_beam3",
|
|
78
|
+
"heading",
|
|
79
|
+
"pitch",
|
|
80
|
+
"roll",
|
|
81
|
+
"east_velocity_qc",
|
|
82
|
+
"north_velocity_qc",
|
|
83
|
+
"up_velocity_qc",
|
|
84
|
+
"pitch_qc",
|
|
85
|
+
"roll_qc",
|
|
86
|
+
"percent_good_qc",
|
|
87
|
+
"error_velocity_qc",
|
|
88
|
+
"seabed_qc",
|
|
89
|
+
}
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _safe_serial(serial: Any) -> str:
|
|
94
|
+
"""Sanitise a serial number string for use in filenames.
|
|
95
|
+
|
|
96
|
+
If the raw YAML value contains a comma (e.g. ``16430, R01-024``), only
|
|
97
|
+
the first comma-separated token is used — the remainder is a beacon ID
|
|
98
|
+
or annotation and is not part of the filename.
|
|
99
|
+
"""
|
|
100
|
+
s = str(serial).split(",")[0]
|
|
101
|
+
return re.sub(r"[^\w\-]", "", s)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _dms_to_deg(s: str) -> float:
|
|
105
|
+
"""Parse 'DD MM.mmm N/S/E/W' or a plain float string to decimal degrees."""
|
|
106
|
+
s = s.strip()
|
|
107
|
+
try:
|
|
108
|
+
return float(s)
|
|
109
|
+
except ValueError:
|
|
110
|
+
pass
|
|
111
|
+
parts = s.upper().split()
|
|
112
|
+
hemi = parts[-1] if parts[-1] in ("N", "S", "E", "W") else None
|
|
113
|
+
nums = parts[:-1] if hemi else parts
|
|
114
|
+
val = sum(float(x) / 60.0**i for i, x in enumerate(nums))
|
|
115
|
+
if hemi in ("S", "W"):
|
|
116
|
+
val = -val
|
|
117
|
+
return val
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _parse_latlon(cfg: dict) -> tuple[float, float]:
|
|
121
|
+
"""Return (lat, lon) in decimal degrees from a mooring config dict."""
|
|
122
|
+
lat, lon, _ = _parse_latlon_with_source(cfg)
|
|
123
|
+
return lat, lon
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _parse_latlon_with_source(cfg: dict) -> tuple[float, float, str]:
|
|
127
|
+
"""Return (lat, lon, source_key) from a mooring config dict.
|
|
128
|
+
|
|
129
|
+
source_key is the YAML key pair used, e.g. 'seabed_latitude/seabed_longitude',
|
|
130
|
+
or 'unknown (defaulting to 0, 0)' if none found.
|
|
131
|
+
|
|
132
|
+
Zero values (lat == 0 and lon == 0) are treated as unfilled placeholders and
|
|
133
|
+
skipped so that a later key with a real location is used instead. This handles
|
|
134
|
+
YAML templates where fields like ``seabed_latitude: 00 00.000 N`` are present
|
|
135
|
+
but not yet populated.
|
|
136
|
+
"""
|
|
137
|
+
for lat_key, lon_key in [
|
|
138
|
+
("seabed_latitude", "seabed_longitude"),
|
|
139
|
+
("deployment_latitude", "deployment_longitude"),
|
|
140
|
+
("planned_latitude", "planned_longitude"),
|
|
141
|
+
("latitude", "longitude"),
|
|
142
|
+
]:
|
|
143
|
+
lat_s = cfg.get(lat_key)
|
|
144
|
+
lon_s = cfg.get(lon_key)
|
|
145
|
+
if lat_s is not None and lon_s is not None:
|
|
146
|
+
try:
|
|
147
|
+
lat_v = _dms_to_deg(str(lat_s))
|
|
148
|
+
lon_v = _dms_to_deg(str(lon_s))
|
|
149
|
+
except Exception: # noqa: BLE001 — _dms_to_deg parse failure; try next key
|
|
150
|
+
continue
|
|
151
|
+
if lat_v == 0.0 and lon_v == 0.0:
|
|
152
|
+
continue # treat (0, 0) as an unfilled placeholder
|
|
153
|
+
return lat_v, lon_v, f"{lat_key}/{lon_key}"
|
|
154
|
+
return 0.0, 0.0, "unknown (defaulting to 0, 0)"
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
_SIGMA_META = {
|
|
158
|
+
0: (
|
|
159
|
+
"sigma0",
|
|
160
|
+
"Potential density anomaly referenced to surface (sigma-0)",
|
|
161
|
+
"sea_water_sigma_t",
|
|
162
|
+
),
|
|
163
|
+
1000: (
|
|
164
|
+
"sigma1",
|
|
165
|
+
"Potential density anomaly referenced to 1000 dbar (sigma-1)",
|
|
166
|
+
"sea_water_sigma_1",
|
|
167
|
+
),
|
|
168
|
+
2000: (
|
|
169
|
+
"sigma2",
|
|
170
|
+
"Potential density anomaly referenced to 2000 dbar (sigma-2)",
|
|
171
|
+
"sea_water_sigma_2",
|
|
172
|
+
),
|
|
173
|
+
3000: (
|
|
174
|
+
"sigma3",
|
|
175
|
+
"Potential density anomaly referenced to 3000 dbar (sigma-3)",
|
|
176
|
+
"sea_water_sigma_3",
|
|
177
|
+
),
|
|
178
|
+
4000: (
|
|
179
|
+
"sigma4",
|
|
180
|
+
"Potential density anomaly referenced to 4000 dbar (sigma-4)",
|
|
181
|
+
"sea_water_sigma_4",
|
|
182
|
+
),
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _apply_qc_mask(src_v: np.ndarray, ds: "xr.Dataset", vname: str) -> np.ndarray:
|
|
187
|
+
"""Return src_v with flagged values replaced by NaN.
|
|
188
|
+
|
|
189
|
+
Flags kept as valid:
|
|
190
|
+
0 (no QC performed), 1 (good), 2 (probably good)
|
|
191
|
+
+ flag 8 (interpolated) for pressure only.
|
|
192
|
+
Flags masked (→ NaN):
|
|
193
|
+
3 (suspect), 4 (bad), 9 (missing value).
|
|
194
|
+
If no companion *_qc* variable exists the array is returned unchanged.
|
|
195
|
+
"""
|
|
196
|
+
qc_name = f"{vname}_qc"
|
|
197
|
+
if qc_name not in ds.data_vars:
|
|
198
|
+
return src_v
|
|
199
|
+
qc = ds[qc_name].values
|
|
200
|
+
if vname == "pressure":
|
|
201
|
+
keep = np.isin(qc, [0, 1, 2, 8])
|
|
202
|
+
else:
|
|
203
|
+
keep = np.isin(qc, [0, 1, 2])
|
|
204
|
+
out = src_v.copy()
|
|
205
|
+
out[~keep] = np.nan
|
|
206
|
+
return out
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _worst_flag(a: np.ndarray, b: np.ndarray) -> np.ndarray:
|
|
210
|
+
"""Element-wise worst QC flag (9 > 4 > 3 > 8 > 2 > 1 > 0).
|
|
211
|
+
|
|
212
|
+
NaN inputs are treated as flag 9 (missing value) so that levels with no
|
|
213
|
+
data are never silently promoted to flag 0 ("no QC performed").
|
|
214
|
+
"""
|
|
215
|
+
# rank[flag_value] gives priority; higher rank = worse flag
|
|
216
|
+
_rank = np.array([0, 1, 2, 4, 5, 6, 6, 6, 3, 7], dtype=np.int8)
|
|
217
|
+
a = np.where(np.isfinite(a), a, 9.0)
|
|
218
|
+
b = np.where(np.isfinite(b), b, 9.0)
|
|
219
|
+
ai = np.clip(np.round(a).astype(np.int8), 0, 9)
|
|
220
|
+
bi = np.clip(np.round(b).astype(np.int8), 0, 9)
|
|
221
|
+
take_b = _rank[bi] > _rank[ai]
|
|
222
|
+
out = ai.copy()
|
|
223
|
+
out[take_b] = bi[take_b]
|
|
224
|
+
return out
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _times_to_float(t: np.ndarray) -> np.ndarray:
|
|
228
|
+
return t.astype("datetime64[ns]").astype(np.float64)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _get_proc_dir(base_dir: Path, mooring_name: str) -> Path:
|
|
232
|
+
proc = base_dir / "proc"
|
|
233
|
+
if not proc.is_dir():
|
|
234
|
+
legacy = base_dir / "moor" / "proc"
|
|
235
|
+
proc = legacy if legacy.is_dir() else proc
|
|
236
|
+
return proc / mooring_name
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _best_nc(
|
|
240
|
+
proc_dir: Path, instr_type: str, mooring_name: str, serial: str
|
|
241
|
+
) -> Optional[Path]:
|
|
242
|
+
"""Return best available stage3 or stage2 file for one instrument; None if only stage1."""
|
|
243
|
+
base = proc_dir / instr_type / f"{mooring_name}_{serial}"
|
|
244
|
+
for suffix in ("_stage3.nc", "_stage2.nc"):
|
|
245
|
+
p = Path(str(base) + suffix)
|
|
246
|
+
if p.exists():
|
|
247
|
+
return p
|
|
248
|
+
return None
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _detect_interval_s(time_vals: np.ndarray) -> float:
|
|
252
|
+
if len(time_vals) < 2:
|
|
253
|
+
return 60.0
|
|
254
|
+
dt = np.diff(time_vals.astype("datetime64[s]").astype(np.float64))
|
|
255
|
+
return float(np.median(dt))
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
# Variables measured at the ADCP transducer head (point measurements, not per-bin).
|
|
259
|
+
# These must NOT appear in per-bin datasets — they belong only in the head entry.
|
|
260
|
+
_ADCP_HEAD_VARS: frozenset = frozenset(
|
|
261
|
+
{
|
|
262
|
+
"temperature",
|
|
263
|
+
"temperature_qc",
|
|
264
|
+
"heading",
|
|
265
|
+
"pitch",
|
|
266
|
+
"roll",
|
|
267
|
+
"battery_voltage",
|
|
268
|
+
"speed_of_sound",
|
|
269
|
+
"analog_input_1",
|
|
270
|
+
"analog_input_2",
|
|
271
|
+
}
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _make_adcp_head_ds(ds_parent: xr.Dataset) -> xr.Dataset:
|
|
276
|
+
"""Create a 1-D (time-only) view of the ADCP transducer head for stack insertion.
|
|
277
|
+
|
|
278
|
+
The ADCP transducer head is treated as a distinct instrument position in the mooring
|
|
279
|
+
stack. It carries instrument-diagnostic variables that describe the sensor, not the
|
|
280
|
+
water column:
|
|
281
|
+
|
|
282
|
+
- ``temperature`` (°C) — internal electronics temperature; useful for diagnosing
|
|
283
|
+
warm-up drift but not a reliable water temperature measurement.
|
|
284
|
+
- ``heading``, ``pitch``, ``roll`` (°) — instrument orientation.
|
|
285
|
+
- ``pressure`` (dbar) — pressure at the transducer face, i.e. the instrument depth.
|
|
286
|
+
|
|
287
|
+
Per-bin velocity and ``bin_pressure`` are deliberately excluded. Each velocity bin
|
|
288
|
+
gets its own stack entry at the pressure appropriate to that bin
|
|
289
|
+
(see ``_make_adcp_bin_ds``), rather than the transducer pressure.
|
|
290
|
+
|
|
291
|
+
Parameters
|
|
292
|
+
----------
|
|
293
|
+
ds_parent : xr.Dataset
|
|
294
|
+
Full ADCP stage-3 dataset, including 2-D ``(time, N_BINS)`` variables.
|
|
295
|
+
|
|
296
|
+
Returns
|
|
297
|
+
-------
|
|
298
|
+
xr.Dataset
|
|
299
|
+
Dataset containing only time-series (1-D) variables from ``_ADCP_HEAD_VARS``
|
|
300
|
+
plus the transducer ``pressure``. Suitable for merging into a point-instrument
|
|
301
|
+
stack.
|
|
302
|
+
|
|
303
|
+
"""
|
|
304
|
+
keep = [
|
|
305
|
+
v
|
|
306
|
+
for v in ds_parent.data_vars
|
|
307
|
+
if v in _ADCP_HEAD_VARS and ds_parent[v].dims == ("time",)
|
|
308
|
+
]
|
|
309
|
+
# Also keep the instrument-head pressure if present (1-D, not bin_pressure)
|
|
310
|
+
if "pressure" in ds_parent.data_vars and ds_parent["pressure"].dims == ("time",):
|
|
311
|
+
if "pressure" not in keep:
|
|
312
|
+
keep.append("pressure")
|
|
313
|
+
return ds_parent[keep]
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _make_adcp_bin_ds(ds_parent: xr.Dataset, bin_idx: int) -> xr.Dataset:
|
|
317
|
+
"""Create a 1-D (time-only) point-instrument view of one ADCP range bin.
|
|
318
|
+
|
|
319
|
+
Slices the parent dataset at *bin_idx* along ``P.ADCP_BIN_DIM`` and replaces the
|
|
320
|
+
transducer ``pressure`` with ``bin_pressure[:, bin_idx]`` so that this entry sits at
|
|
321
|
+
the correct depth in the stack. Computes derived scalar velocity quantities::
|
|
322
|
+
|
|
323
|
+
current_speed = sqrt(east_velocity² + north_velocity²) [m s⁻¹, always ≥ 0]
|
|
324
|
+
current_direction = atan2(east, north) mod 360 [degrees, 0 = N CW]
|
|
325
|
+
|
|
326
|
+
**Direction convention**: the direction *toward which* the water flows, clockwise
|
|
327
|
+
from True North. This is the oceanographic convention (opposite to meteorological
|
|
328
|
+
"wind from"). 0° = northward flow, 90° = eastward flow.
|
|
329
|
+
|
|
330
|
+
Instrument-head variables (``temperature``, ``heading``, ``pitch``, ``roll`` — see
|
|
331
|
+
``_ADCP_HEAD_VARS``) are dropped because they belong to the transducer location, not
|
|
332
|
+
to an individual velocity bin. Variables retaining non-time dimensions (e.g.
|
|
333
|
+
beam-dimension arrays ``amplitude``, ``correlation``, ``percent_good``) are also
|
|
334
|
+
dropped; they cannot be represented as point-instrument time series without
|
|
335
|
+
ambiguity about which beam to keep.
|
|
336
|
+
|
|
337
|
+
The result is compatible with ``_nearest_subsample`` and ``_linear_interp`` because
|
|
338
|
+
every remaining data variable has dimension ``("time",)`` or is scalar.
|
|
339
|
+
|
|
340
|
+
Parameters
|
|
341
|
+
----------
|
|
342
|
+
ds_parent : xr.Dataset
|
|
343
|
+
Full ADCP stage-3 dataset.
|
|
344
|
+
bin_idx : int
|
|
345
|
+
Zero-based index along the bin dimension (``P.ADCP_BIN_DIM``). Index 0 is
|
|
346
|
+
the bin nearest the transducer face.
|
|
347
|
+
|
|
348
|
+
Returns
|
|
349
|
+
-------
|
|
350
|
+
xr.Dataset
|
|
351
|
+
Single-bin time-series dataset with ``pressure`` set to ``bin_pressure``
|
|
352
|
+
at this bin and ``current_speed`` / ``current_direction`` added.
|
|
353
|
+
|
|
354
|
+
"""
|
|
355
|
+
bin_dim = P.ADCP_BIN_DIM
|
|
356
|
+
ds_bin = ds_parent.isel({bin_dim: bin_idx}, drop=True)
|
|
357
|
+
|
|
358
|
+
# Drop head-only variables — these belong in the separate head entry
|
|
359
|
+
head_to_drop = [v for v in _ADCP_HEAD_VARS if v in ds_bin.data_vars]
|
|
360
|
+
if head_to_drop:
|
|
361
|
+
ds_bin = ds_bin.drop_vars(head_to_drop)
|
|
362
|
+
|
|
363
|
+
if "bin_pressure" in ds_bin.data_vars:
|
|
364
|
+
ds_bin = ds_bin.assign(
|
|
365
|
+
pressure=xr.Variable(
|
|
366
|
+
"time",
|
|
367
|
+
ds_bin["bin_pressure"].values.astype(np.float32),
|
|
368
|
+
{"units": "dbar", "long_name": "Pressure at ADCP bin"},
|
|
369
|
+
)
|
|
370
|
+
).drop_vars("bin_pressure")
|
|
371
|
+
|
|
372
|
+
if "east_velocity" in ds_bin.data_vars and "north_velocity" in ds_bin.data_vars:
|
|
373
|
+
e = ds_bin["east_velocity"].values.astype(float)
|
|
374
|
+
n = ds_bin["north_velocity"].values.astype(float)
|
|
375
|
+
ds_bin = ds_bin.assign(
|
|
376
|
+
current_speed=xr.Variable(
|
|
377
|
+
"time",
|
|
378
|
+
np.hypot(e, n).astype(np.float32),
|
|
379
|
+
{"units": "m s-1", "long_name": "Current speed"},
|
|
380
|
+
),
|
|
381
|
+
current_direction=xr.Variable(
|
|
382
|
+
"time",
|
|
383
|
+
(np.degrees(np.arctan2(e, n)) % 360.0).astype(np.float32),
|
|
384
|
+
{
|
|
385
|
+
"units": "degrees",
|
|
386
|
+
"long_name": "Current direction (oceanographic, 0=N clockwise)",
|
|
387
|
+
},
|
|
388
|
+
),
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
to_drop = [v for v in ds_bin.data_vars if any(d != "time" for d in ds_bin[v].dims)]
|
|
392
|
+
if to_drop:
|
|
393
|
+
ds_bin = ds_bin.drop_vars(to_drop)
|
|
394
|
+
|
|
395
|
+
return ds_bin
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _nearest_subsample(
|
|
399
|
+
ds: xr.Dataset,
|
|
400
|
+
common_time: np.ndarray,
|
|
401
|
+
half_window_s: float,
|
|
402
|
+
) -> Dict[str, np.ndarray]:
|
|
403
|
+
"""Return nearest-neighbour values on common_time within ±half_window_s seconds.
|
|
404
|
+
|
|
405
|
+
Returns a dict {varname: array} with NaN where no sample falls within the window.
|
|
406
|
+
"""
|
|
407
|
+
src_t = _times_to_float(ds["time"].values)
|
|
408
|
+
tgt_t = _times_to_float(common_time)
|
|
409
|
+
n = len(common_time)
|
|
410
|
+
half_ns = half_window_s * 1e9 # ns
|
|
411
|
+
|
|
412
|
+
idx = np.searchsorted(src_t, tgt_t)
|
|
413
|
+
result: Dict[str, np.ndarray] = {}
|
|
414
|
+
for vname in STACK_VARS:
|
|
415
|
+
if vname not in ds.data_vars:
|
|
416
|
+
result[vname] = np.full(n, np.nan)
|
|
417
|
+
continue
|
|
418
|
+
src_v = ds[vname].values.astype(np.float64)
|
|
419
|
+
if vname not in _STACK_RAW:
|
|
420
|
+
src_v = _apply_qc_mask(src_v, ds, vname)
|
|
421
|
+
out = np.full(n, np.nan)
|
|
422
|
+
for i, (t_tgt, k) in enumerate(zip(tgt_t, idx)):
|
|
423
|
+
# Check candidates at k-1 and k
|
|
424
|
+
best_dt = np.inf
|
|
425
|
+
best_v = np.nan
|
|
426
|
+
for j in [k - 1, k]:
|
|
427
|
+
if 0 <= j < len(src_t):
|
|
428
|
+
dt = abs(src_t[j] - t_tgt)
|
|
429
|
+
if dt < best_dt and dt <= half_ns:
|
|
430
|
+
best_dt = dt
|
|
431
|
+
best_v = src_v[j]
|
|
432
|
+
out[i] = best_v
|
|
433
|
+
result[vname] = out
|
|
434
|
+
return result
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _linear_interp(
|
|
438
|
+
ds: xr.Dataset,
|
|
439
|
+
common_time: np.ndarray,
|
|
440
|
+
) -> Dict[str, np.ndarray]:
|
|
441
|
+
"""Linearly interpolate all stack variables onto common_time; NaN outside data range."""
|
|
442
|
+
src_t = _times_to_float(ds["time"].values)
|
|
443
|
+
tgt_t = _times_to_float(common_time)
|
|
444
|
+
result: Dict[str, np.ndarray] = {}
|
|
445
|
+
for vname in STACK_VARS:
|
|
446
|
+
if vname not in ds.data_vars:
|
|
447
|
+
result[vname] = np.full(len(common_time), np.nan)
|
|
448
|
+
continue
|
|
449
|
+
src_v = ds[vname].values.astype(np.float64)
|
|
450
|
+
if vname not in _STACK_RAW:
|
|
451
|
+
src_v = _apply_qc_mask(src_v, ds, vname)
|
|
452
|
+
valid = np.isfinite(src_v)
|
|
453
|
+
if valid.sum() < 2:
|
|
454
|
+
result[vname] = np.full(len(common_time), np.nan)
|
|
455
|
+
continue
|
|
456
|
+
if vname.endswith("_qc"):
|
|
457
|
+
# QC flag arrays must not be linearly interpolated — use nearest valid
|
|
458
|
+
src_t_v = src_t[valid]
|
|
459
|
+
src_v_v = src_v[valid]
|
|
460
|
+
nn_idx = np.clip(np.searchsorted(src_t_v, tgt_t), 0, len(src_t_v) - 1)
|
|
461
|
+
result[vname] = src_v_v[nn_idx]
|
|
462
|
+
else:
|
|
463
|
+
result[vname] = np.interp(
|
|
464
|
+
tgt_t, src_t[valid], src_v[valid], left=np.nan, right=np.nan
|
|
465
|
+
)
|
|
466
|
+
return result
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
class MooringStacker:
|
|
470
|
+
"""Step 1: stack all instruments onto a common time axis → ``_stack.nc``."""
|
|
471
|
+
|
|
472
|
+
def __init__(
|
|
473
|
+
self,
|
|
474
|
+
base_dir: Optional[str] = None,
|
|
475
|
+
*,
|
|
476
|
+
proc_dir: Optional[str] = None,
|
|
477
|
+
) -> None:
|
|
478
|
+
"""Resample all instruments from a mooring onto a common time axis.
|
|
479
|
+
|
|
480
|
+
Reads ``{mooring}_{serial}_stage3.nc`` (or ``_stage2.nc`` as fallback) for
|
|
481
|
+
every instrument listed in the mooring YAML, resamples each to a uniform
|
|
482
|
+
sampling interval, and writes a single multi-instrument file
|
|
483
|
+
``{mooring}_stack.nc`` in the mooring proc directory.
|
|
484
|
+
|
|
485
|
+
Parameters
|
|
486
|
+
----------
|
|
487
|
+
base_dir : str, optional
|
|
488
|
+
Legacy: cruise-level base directory containing a ``proc/`` subdirectory.
|
|
489
|
+
proc_dir : str, optional
|
|
490
|
+
Cruise-level processed output directory. Pipeline appends ``/{mooring}/``.
|
|
491
|
+
|
|
492
|
+
"""
|
|
493
|
+
if base_dir is not None:
|
|
494
|
+
self.base_dir: Optional[Path] = Path(base_dir)
|
|
495
|
+
self._proc_dir: Optional[Path] = None
|
|
496
|
+
self._legacy = True
|
|
497
|
+
else:
|
|
498
|
+
self.base_dir = None
|
|
499
|
+
self._proc_dir = Path(proc_dir) if proc_dir else None
|
|
500
|
+
self._legacy = False
|
|
501
|
+
|
|
502
|
+
def _resolve_proc_dir(self, mooring_name: str) -> Path:
|
|
503
|
+
"""Return the mooring-level proc directory."""
|
|
504
|
+
if not self._legacy and self._proc_dir is not None:
|
|
505
|
+
return self._proc_dir / mooring_name
|
|
506
|
+
return _get_proc_dir(self.base_dir, mooring_name)
|
|
507
|
+
|
|
508
|
+
def _rel(self, path: Path) -> str:
|
|
509
|
+
"""Return a short display path relative to base_dir or proc_dir."""
|
|
510
|
+
for root in (r for r in (self.base_dir, self._proc_dir) if r):
|
|
511
|
+
try:
|
|
512
|
+
return str(path.relative_to(root))
|
|
513
|
+
except ValueError:
|
|
514
|
+
continue
|
|
515
|
+
return path.name
|
|
516
|
+
|
|
517
|
+
def stack(
|
|
518
|
+
self,
|
|
519
|
+
mooring_name: str,
|
|
520
|
+
dt_seconds: int = 60,
|
|
521
|
+
force: bool = False,
|
|
522
|
+
) -> bool:
|
|
523
|
+
"""Stack all processed instruments for *mooring_name* onto a common time grid.
|
|
524
|
+
|
|
525
|
+
Reads ``_stage3.nc`` (falling back to ``_stage2.nc``) for every instrument
|
|
526
|
+
listed in the mooring YAML, resamples each to *dt_seconds* resolution, and
|
|
527
|
+
writes ``{mooring}_stack.nc`` under the mooring proc directory.
|
|
528
|
+
|
|
529
|
+
Returns True on success, False if no instruments could be loaded or an
|
|
530
|
+
unrecoverable error occurs.
|
|
531
|
+
"""
|
|
532
|
+
proc_dir = self._resolve_proc_dir(mooring_name)
|
|
533
|
+
try:
|
|
534
|
+
proc_dir_exists = proc_dir.exists()
|
|
535
|
+
except (TimeoutError, OSError) as exc:
|
|
536
|
+
print("ERROR: Cannot access data drive — is it connected?")
|
|
537
|
+
print(f" Path: {proc_dir}")
|
|
538
|
+
print(f" ({type(exc).__name__}: {exc})")
|
|
539
|
+
return False
|
|
540
|
+
|
|
541
|
+
if not proc_dir_exists:
|
|
542
|
+
print(f"ERROR: Processing directory not found: {proc_dir}")
|
|
543
|
+
return False
|
|
544
|
+
|
|
545
|
+
output_path = proc_dir / f"{mooring_name}_stack.nc"
|
|
546
|
+
if output_path.exists() and not force:
|
|
547
|
+
_status("skip", self._rel(output_path))
|
|
548
|
+
return True
|
|
549
|
+
|
|
550
|
+
config_file = proc_dir / f"{mooring_name}.mooring.yaml"
|
|
551
|
+
if not config_file.exists():
|
|
552
|
+
print(f"ERROR: Config not found: {config_file}")
|
|
553
|
+
return False
|
|
554
|
+
|
|
555
|
+
with open(config_file) as f:
|
|
556
|
+
mooring_config = yaml.safe_load(f)
|
|
557
|
+
|
|
558
|
+
deploy_time = np.datetime64(mooring_config["deployment_time"], "ns")
|
|
559
|
+
recover_time = np.datetime64(mooring_config["recovery_time"], "ns")
|
|
560
|
+
|
|
561
|
+
common_time = np.arange(
|
|
562
|
+
deploy_time.astype("datetime64[s]"),
|
|
563
|
+
recover_time.astype("datetime64[s]") + np.timedelta64(1, "s"),
|
|
564
|
+
np.timedelta64(dt_seconds, "s"),
|
|
565
|
+
).astype("datetime64[ns]")
|
|
566
|
+
n_time = len(common_time)
|
|
567
|
+
|
|
568
|
+
from .utilities import extract_inline_instruments
|
|
569
|
+
|
|
570
|
+
instrument_list = list(
|
|
571
|
+
mooring_config.get("clamp", mooring_config.get("instruments", []))
|
|
572
|
+
)
|
|
573
|
+
instrument_list += extract_inline_instruments(mooring_config.get("inline", []))
|
|
574
|
+
|
|
575
|
+
# Collect instruments with known hab and an available stage2/stage3 file
|
|
576
|
+
instruments = []
|
|
577
|
+
for entry in instrument_list:
|
|
578
|
+
if not isinstance(entry, dict):
|
|
579
|
+
continue
|
|
580
|
+
serial = _safe_serial(entry.get("serial", ""))
|
|
581
|
+
instr_type = entry.get("instrument", "unknown")
|
|
582
|
+
if instr_type not in KNOWN_INSTRUMENT_TYPES:
|
|
583
|
+
print(
|
|
584
|
+
f" WARNING: instrument '{instr_type}' (s/n {serial}) is not in "
|
|
585
|
+
f"KNOWN_INSTRUMENT_TYPES {sorted(KNOWN_INSTRUMENT_TYPES)}. "
|
|
586
|
+
"Aquadopp-specific plots and processing will be skipped. "
|
|
587
|
+
"Allowed values: microcat, aquadopp."
|
|
588
|
+
)
|
|
589
|
+
hab = entry.get("hab")
|
|
590
|
+
if hab is None:
|
|
591
|
+
continue
|
|
592
|
+
yaml_interval = entry.get("sample_interval_seconds")
|
|
593
|
+
nc_path = _best_nc(proc_dir, instr_type, mooring_name, serial)
|
|
594
|
+
if nc_path is None:
|
|
595
|
+
print(f" SKIP {instr_type} s/n {serial}: no stage2/stage3 file")
|
|
596
|
+
continue
|
|
597
|
+
instruments.append(
|
|
598
|
+
{
|
|
599
|
+
"serial": serial,
|
|
600
|
+
"instrument": instr_type,
|
|
601
|
+
"hab": float(hab),
|
|
602
|
+
"yaml_interval": yaml_interval,
|
|
603
|
+
"nc_path": nc_path,
|
|
604
|
+
}
|
|
605
|
+
)
|
|
606
|
+
|
|
607
|
+
if not instruments:
|
|
608
|
+
print("ERROR: No instruments found with stage2/stage3 files and hab values")
|
|
609
|
+
return False
|
|
610
|
+
|
|
611
|
+
# Sort deep-first (ascending hab: smallest hab = nearest bottom = deepest)
|
|
612
|
+
instruments.sort(key=lambda x: x["hab"])
|
|
613
|
+
|
|
614
|
+
# ── Expand ADCP instruments into per-bin pseudo-instruments ───────
|
|
615
|
+
# Each ADCP bin becomes a separate row in the stack, with its own
|
|
616
|
+
# bin_pressure, velocity, and QC time series sliced from the 2-D
|
|
617
|
+
# (time, N_BINS) arrays in the parent stage3 file.
|
|
618
|
+
# The parent file is loaded once and cached; _make_adcp_bin_ds slices it.
|
|
619
|
+
_adcp_parent_datasets: Dict[str, xr.Dataset] = {}
|
|
620
|
+
expanded: List[Dict] = []
|
|
621
|
+
for info in instruments:
|
|
622
|
+
if info["instrument"].lower() != "adcp":
|
|
623
|
+
expanded.append(info)
|
|
624
|
+
continue
|
|
625
|
+
nc_key = str(info["nc_path"])
|
|
626
|
+
if nc_key not in _adcp_parent_datasets:
|
|
627
|
+
try:
|
|
628
|
+
_adcp_parent_datasets[nc_key] = xr.open_dataset(
|
|
629
|
+
info["nc_path"], decode_timedelta=False
|
|
630
|
+
).load()
|
|
631
|
+
except Exception as e: # noqa: BLE001 — bad ADCP file must not abort stack
|
|
632
|
+
print(f" WARNING: Could not load ADCP {info['nc_path'].name}: {e}")
|
|
633
|
+
expanded.append(info)
|
|
634
|
+
continue
|
|
635
|
+
ds_adcp = _adcp_parent_datasets[nc_key]
|
|
636
|
+
bin_dim = P.ADCP_BIN_DIM
|
|
637
|
+
if bin_dim not in ds_adcp.dims:
|
|
638
|
+
print(
|
|
639
|
+
f" WARNING: ADCP s/n {info['serial']} has no {bin_dim} dim "
|
|
640
|
+
"— treating as point instrument"
|
|
641
|
+
)
|
|
642
|
+
expanded.append(info)
|
|
643
|
+
continue
|
|
644
|
+
n_bins = ds_adcp.sizes[bin_dim]
|
|
645
|
+
range_vals = (
|
|
646
|
+
ds_adcp["range"].values
|
|
647
|
+
if "range" in ds_adcp.coords
|
|
648
|
+
else np.arange(n_bins, dtype=float)
|
|
649
|
+
)
|
|
650
|
+
orientation = ds_adcp.attrs.get("orientation_yaml") or ds_adcp.attrs.get(
|
|
651
|
+
"orientation_instrument"
|
|
652
|
+
)
|
|
653
|
+
if not orientation:
|
|
654
|
+
orientation = "down"
|
|
655
|
+
print(
|
|
656
|
+
f" WARNING: ADCP {info['serial']} has no orientation attr — "
|
|
657
|
+
"assuming downward-looking. HAB offsets may be wrong for "
|
|
658
|
+
"upward-looking instruments."
|
|
659
|
+
)
|
|
660
|
+
looking_down = str(orientation).lower() == "down"
|
|
661
|
+
# Pre-compute seabed mask so always-submerged bins can be skipped.
|
|
662
|
+
# seabed_qc has dims (time, N_BINS); a bin is permanently below the
|
|
663
|
+
# seabed when every time step is flagged suspect or worse (>= 3).
|
|
664
|
+
_seabed_qc_vals = (
|
|
665
|
+
ds_adcp["seabed_qc"].values
|
|
666
|
+
if "seabed_qc" in ds_adcp.data_vars
|
|
667
|
+
else None
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
n_skipped = 0
|
|
671
|
+
valid_bins = []
|
|
672
|
+
for i in range(n_bins):
|
|
673
|
+
if _seabed_qc_vals is not None and np.all(_seabed_qc_vals[:, i] >= 3):
|
|
674
|
+
n_skipped += 1
|
|
675
|
+
else:
|
|
676
|
+
valid_bins.append(i)
|
|
677
|
+
|
|
678
|
+
print(
|
|
679
|
+
f" ADCP s/n {info['serial']}: expanding {len(valid_bins)} of {n_bins} bins "
|
|
680
|
+
f"({orientation}-looking) into stack + head entry"
|
|
681
|
+
+ (
|
|
682
|
+
f" [{n_skipped} bins always below seabed, skipped]"
|
|
683
|
+
if n_skipped
|
|
684
|
+
else ""
|
|
685
|
+
)
|
|
686
|
+
)
|
|
687
|
+
|
|
688
|
+
# Head entry — transducer location carries temperature and orientation.
|
|
689
|
+
# Uses the instrument HAB directly (range = 0 from the transducer).
|
|
690
|
+
expanded.append(
|
|
691
|
+
{
|
|
692
|
+
**info,
|
|
693
|
+
"serial": f"{info['serial']}_hd",
|
|
694
|
+
"hab": float(info["hab"]),
|
|
695
|
+
"_adcp_head": True,
|
|
696
|
+
"_adcp_nc_key": nc_key,
|
|
697
|
+
}
|
|
698
|
+
)
|
|
699
|
+
for i in valid_bins:
|
|
700
|
+
r = float(range_vals[i])
|
|
701
|
+
bin_hab = (
|
|
702
|
+
float(info["hab"]) - r if looking_down else float(info["hab"]) + r
|
|
703
|
+
)
|
|
704
|
+
expanded.append(
|
|
705
|
+
{
|
|
706
|
+
**info,
|
|
707
|
+
"serial": f"{info['serial']}_b{i:02d}",
|
|
708
|
+
"hab": bin_hab,
|
|
709
|
+
"_adcp_bin_idx": i,
|
|
710
|
+
"_adcp_nc_key": nc_key,
|
|
711
|
+
}
|
|
712
|
+
)
|
|
713
|
+
instruments = expanded
|
|
714
|
+
# Re-sort after expansion so ADCP bins interleave with other instruments by depth
|
|
715
|
+
instruments.sort(key=lambda x: x["hab"])
|
|
716
|
+
|
|
717
|
+
n_instr = len(instruments)
|
|
718
|
+
|
|
719
|
+
print(
|
|
720
|
+
f"Merging {n_instr} instruments onto {n_time}-point {dt_seconds}s time grid "
|
|
721
|
+
f"({mooring_name})"
|
|
722
|
+
)
|
|
723
|
+
|
|
724
|
+
stacked: Dict[str, np.ndarray] = {
|
|
725
|
+
v: np.full((n_instr, n_time), np.nan) for v in STACK_VARS
|
|
726
|
+
}
|
|
727
|
+
var_attrs: Dict[str, dict] = {v: {} for v in STACK_VARS}
|
|
728
|
+
serials: List[str] = []
|
|
729
|
+
habs: List[float] = []
|
|
730
|
+
instr_types: List[str] = []
|
|
731
|
+
stage_labels: List[str] = []
|
|
732
|
+
# Per-instrument scalar metadata: {varname: [value_for_instr0, value_for_instr1, ...]}
|
|
733
|
+
scalar_meta: Dict[str, list] = {} # populated during loop
|
|
734
|
+
scalar_attrs: Dict[str, dict] = {}
|
|
735
|
+
|
|
736
|
+
for i, info in enumerate(instruments):
|
|
737
|
+
serials.append(info["serial"])
|
|
738
|
+
habs.append(info["hab"])
|
|
739
|
+
instr_types.append(info["instrument"])
|
|
740
|
+
stage_labels.append(info["nc_path"].stem.split("_")[-1])
|
|
741
|
+
|
|
742
|
+
try:
|
|
743
|
+
if "_adcp_bin_idx" in info:
|
|
744
|
+
ds_parent = _adcp_parent_datasets[info["_adcp_nc_key"]]
|
|
745
|
+
ds = _make_adcp_bin_ds(ds_parent, info["_adcp_bin_idx"])
|
|
746
|
+
elif "_adcp_head" in info:
|
|
747
|
+
ds_parent = _adcp_parent_datasets[info["_adcp_nc_key"]]
|
|
748
|
+
ds = _make_adcp_head_ds(ds_parent)
|
|
749
|
+
else:
|
|
750
|
+
ds = xr.open_dataset(info["nc_path"], decode_timedelta=False).load()
|
|
751
|
+
ds.close()
|
|
752
|
+
except Exception as e: # noqa: BLE001 — one bad instrument must not abort the stack
|
|
753
|
+
print(f" WARNING: Could not load {info['nc_path'].name}: {e}")
|
|
754
|
+
# Ensure scalar_meta lists stay length-consistent
|
|
755
|
+
for lst in scalar_meta.values():
|
|
756
|
+
lst.append(None)
|
|
757
|
+
continue
|
|
758
|
+
|
|
759
|
+
interval_s = (
|
|
760
|
+
float(info["yaml_interval"])
|
|
761
|
+
if info["yaml_interval"]
|
|
762
|
+
else _detect_interval_s(ds["time"].values)
|
|
763
|
+
)
|
|
764
|
+
print(
|
|
765
|
+
f" [{i:2d}] {info['instrument']:10s} s/n {info['serial']:<8} "
|
|
766
|
+
f"hab={info['hab']:6.1f} m dt={interval_s:.0f}s "
|
|
767
|
+
f"({stage_labels[-1]})"
|
|
768
|
+
)
|
|
769
|
+
|
|
770
|
+
half_window = dt_seconds / 2.0
|
|
771
|
+
if interval_s <= dt_seconds:
|
|
772
|
+
values = _nearest_subsample(ds, common_time, half_window)
|
|
773
|
+
else:
|
|
774
|
+
values = _linear_interp(ds, common_time)
|
|
775
|
+
|
|
776
|
+
for vname in STACK_VARS:
|
|
777
|
+
arr = values[vname]
|
|
778
|
+
n = min(len(arr), n_time)
|
|
779
|
+
stacked[vname][i, :n] = arr[:n]
|
|
780
|
+
if not var_attrs[vname] and vname in ds.data_vars:
|
|
781
|
+
var_attrs[vname] = dict(ds[vname].attrs)
|
|
782
|
+
|
|
783
|
+
# Collect scalar (0-D) metadata variables — keep all instruments consistent
|
|
784
|
+
for vname, da in ds.data_vars.items():
|
|
785
|
+
if da.dims: # skip time-series variables
|
|
786
|
+
continue
|
|
787
|
+
if vname not in scalar_meta:
|
|
788
|
+
# Back-fill with None for instruments processed before this variable appeared
|
|
789
|
+
scalar_meta[vname] = [None] * i
|
|
790
|
+
scalar_attrs[vname] = dict(da.attrs)
|
|
791
|
+
scalar_meta[vname].append(da.values.item())
|
|
792
|
+
|
|
793
|
+
# Collect magnetic_declination from global attrs (set by stage3 BEAM→ENU).
|
|
794
|
+
# Must come BEFORE the fill-None loop so the length is already i+1 when
|
|
795
|
+
# the loop checks.
|
|
796
|
+
if "magnetic_declination" not in scalar_meta:
|
|
797
|
+
scalar_meta["magnetic_declination"] = [np.nan] * i
|
|
798
|
+
scalar_attrs["magnetic_declination"] = {
|
|
799
|
+
"units": "degrees_east",
|
|
800
|
+
"long_name": "Magnetic declination (IGRF)",
|
|
801
|
+
}
|
|
802
|
+
decl_val = ds.attrs.get("magnetic_declination")
|
|
803
|
+
scalar_meta["magnetic_declination"].append(
|
|
804
|
+
float(decl_val) if decl_val is not None else np.nan
|
|
805
|
+
)
|
|
806
|
+
|
|
807
|
+
# Fill None for variables not present in this instrument
|
|
808
|
+
for vname in scalar_meta:
|
|
809
|
+
if len(scalar_meta[vname]) < i + 1:
|
|
810
|
+
scalar_meta[vname].append(None)
|
|
811
|
+
|
|
812
|
+
# Release cached ADCP parent datasets
|
|
813
|
+
for _ds_adcp in _adcp_parent_datasets.values():
|
|
814
|
+
_ds_adcp.close()
|
|
815
|
+
|
|
816
|
+
# Build output dataset; skip physics variables that are entirely NaN
|
|
817
|
+
data_vars: Dict = {}
|
|
818
|
+
for vname in STACK_VARS:
|
|
819
|
+
if not np.all(np.isnan(stacked[vname])):
|
|
820
|
+
data_vars[vname] = xr.Variable(
|
|
821
|
+
("N_LEVELS", "time"), stacked[vname], attrs=var_attrs[vname]
|
|
822
|
+
)
|
|
823
|
+
|
|
824
|
+
# Compute potential density from stacked T, S, P
|
|
825
|
+
ref_p = int(mooring_config.get("density_reference", P.DENSITY_REFERENCE))
|
|
826
|
+
if (
|
|
827
|
+
"temperature" in data_vars
|
|
828
|
+
and "salinity" in data_vars
|
|
829
|
+
and "pressure" in data_vars
|
|
830
|
+
):
|
|
831
|
+
try:
|
|
832
|
+
import gsw
|
|
833
|
+
|
|
834
|
+
lat, lon = _parse_latlon(mooring_config)
|
|
835
|
+
T_arr = stacked["temperature"]
|
|
836
|
+
SP_arr = stacked["salinity"]
|
|
837
|
+
P_arr = stacked["pressure"]
|
|
838
|
+
SA = gsw.SA_from_SP(SP_arr, P_arr, lon, lat)
|
|
839
|
+
CT = gsw.CT_from_t(SA, T_arr, P_arr)
|
|
840
|
+
_sigma_fn = {
|
|
841
|
+
0: gsw.sigma0,
|
|
842
|
+
1000: gsw.sigma1,
|
|
843
|
+
2000: gsw.sigma2,
|
|
844
|
+
3000: gsw.sigma3,
|
|
845
|
+
4000: gsw.sigma4,
|
|
846
|
+
}
|
|
847
|
+
if ref_p in _sigma_fn:
|
|
848
|
+
sigma_vals = _sigma_fn[ref_p](SA, CT)
|
|
849
|
+
else:
|
|
850
|
+
sigma_vals = gsw.pot_rho_t_exact(SA, T_arr, P_arr, ref_p) - 1000.0
|
|
851
|
+
if not np.all(np.isnan(sigma_vals)):
|
|
852
|
+
vname, long_name, std_name = _SIGMA_META.get(
|
|
853
|
+
ref_p,
|
|
854
|
+
(
|
|
855
|
+
f"sigma_{ref_p}",
|
|
856
|
+
f"Potential density anomaly referenced to {ref_p} dbar",
|
|
857
|
+
"",
|
|
858
|
+
),
|
|
859
|
+
)
|
|
860
|
+
data_vars[vname] = xr.Variable(
|
|
861
|
+
("N_LEVELS", "time"),
|
|
862
|
+
sigma_vals,
|
|
863
|
+
{
|
|
864
|
+
"units": "kg m-3",
|
|
865
|
+
"long_name": long_name,
|
|
866
|
+
"standard_name": std_name,
|
|
867
|
+
"reference_pressure_dbar": ref_p,
|
|
868
|
+
},
|
|
869
|
+
)
|
|
870
|
+
except Exception as exc: # noqa: BLE001 — gsw failure must not abort stack
|
|
871
|
+
print(f" WARNING: could not compute potential density: {exc}")
|
|
872
|
+
|
|
873
|
+
# velocity_flag: element-wise worst QC flag across east/north/up velocity.
|
|
874
|
+
# Velocity is stored unmasked in the stack; this combined flag is what the
|
|
875
|
+
# grid step (and users) should apply before using velocity values.
|
|
876
|
+
_vel_qc_keys = [
|
|
877
|
+
v
|
|
878
|
+
for v in ("east_velocity_qc", "north_velocity_qc", "up_velocity_qc")
|
|
879
|
+
if v in data_vars
|
|
880
|
+
]
|
|
881
|
+
if _vel_qc_keys:
|
|
882
|
+
n_lev_v, n_t_v = (
|
|
883
|
+
stacked["east_velocity_qc"].shape
|
|
884
|
+
if "east_velocity_qc" in stacked
|
|
885
|
+
else (0, 0)
|
|
886
|
+
)
|
|
887
|
+
if n_lev_v > 0:
|
|
888
|
+
vel_flag = np.ones((n_lev_v, n_t_v), dtype=np.float64) # 1 = good
|
|
889
|
+
for _qk in _vel_qc_keys:
|
|
890
|
+
vel_flag = _worst_flag(vel_flag, stacked[_qk]).astype(np.float64)
|
|
891
|
+
data_vars["velocity_flag"] = xr.Variable(
|
|
892
|
+
("N_LEVELS", "time"),
|
|
893
|
+
vel_flag,
|
|
894
|
+
{
|
|
895
|
+
"long_name": "Combined velocity QC flag (worst of east/north/up)",
|
|
896
|
+
"comment": (
|
|
897
|
+
"OceanSITES flag: 1=good, 2=prob_good, 3=suspect, 4=bad, "
|
|
898
|
+
"9=missing. Apply to east/north/up_velocity before use."
|
|
899
|
+
),
|
|
900
|
+
"flag_values": "0 1 2 3 4 9",
|
|
901
|
+
"flag_meanings": (
|
|
902
|
+
"no_qc_performed good_data probably_good_data "
|
|
903
|
+
"probably_bad_data bad_data missing_value"
|
|
904
|
+
),
|
|
905
|
+
},
|
|
906
|
+
)
|
|
907
|
+
|
|
908
|
+
# Tilt estimated from pressure difference between two instrument levels.
|
|
909
|
+
# For each level i, find the nearest level j above it that is ≥10 m away
|
|
910
|
+
# in hab AND has at least some finite pressure data. Instruments close
|
|
911
|
+
# together (e.g. a microcat strapped to an Aquadopp frame) are skipped.
|
|
912
|
+
# rope_length = hab[j] - hab[i] (m, from YAML)
|
|
913
|
+
# ΔP = pressure[i,:] - pressure[j,:] (dbar ≈ m, >0 when upright)
|
|
914
|
+
# tilt = arccos(ΔP / rope_length) (degrees from vertical; 0 = upright)
|
|
915
|
+
_MIN_HAB_SEP = 10.0 # minimum hab separation (m) to use as reference
|
|
916
|
+
if "pressure" in data_vars and len(habs) > 1:
|
|
917
|
+
try:
|
|
918
|
+
p_arr = stacked["pressure"] # (N_LEVELS, time) numpy array
|
|
919
|
+
n_lev = len(habs)
|
|
920
|
+
tilt_p = np.full_like(p_arr, np.nan)
|
|
921
|
+
ref_hab_arr = np.full(n_lev, np.nan) # hab of the reference level
|
|
922
|
+
ref_serial_arr = np.array([""] * n_lev, dtype=object)
|
|
923
|
+
# pre-compute which levels have any valid pressure
|
|
924
|
+
_has_p = np.array(
|
|
925
|
+
[np.any(np.isfinite(p_arr[k, :])) for k in range(n_lev)]
|
|
926
|
+
)
|
|
927
|
+
_is_adcp = [t.lower() == "adcp" for t in instr_types]
|
|
928
|
+
for i in range(n_lev):
|
|
929
|
+
if not _has_p[i]:
|
|
930
|
+
continue
|
|
931
|
+
if _is_adcp[i]:
|
|
932
|
+
continue # ADCP transducer pressure is not suitable for tilt
|
|
933
|
+
# Find nearest level above i that is ≥_MIN_HAB_SEP away with pressure,
|
|
934
|
+
# excluding ADCP levels as reference (their bin pressures shift with tilt).
|
|
935
|
+
ref_j = None
|
|
936
|
+
for j in range(i + 1, n_lev):
|
|
937
|
+
if float(habs[j]) - float(habs[i]) < _MIN_HAB_SEP:
|
|
938
|
+
continue
|
|
939
|
+
if not _has_p[j]:
|
|
940
|
+
continue
|
|
941
|
+
if _is_adcp[j]:
|
|
942
|
+
continue
|
|
943
|
+
ref_j = j
|
|
944
|
+
break # first valid j = nearest ≥10 m above
|
|
945
|
+
if ref_j is None:
|
|
946
|
+
continue
|
|
947
|
+
rope = float(habs[ref_j]) - float(habs[i])
|
|
948
|
+
ref_hab_arr[i] = float(habs[ref_j])
|
|
949
|
+
ref_serial_arr[i] = str(serials[ref_j])
|
|
950
|
+
delta_p = p_arr[i, :] - p_arr[ref_j, :]
|
|
951
|
+
ratio = np.clip(delta_p / rope, 0.0, 1.0)
|
|
952
|
+
tilt_p[i, :] = np.degrees(np.arccos(ratio))
|
|
953
|
+
tilt_p[i, ~np.isfinite(delta_p)] = np.nan
|
|
954
|
+
print(
|
|
955
|
+
f" tilt_from_pressure[{i}] s/n {serials[i]} ({habs[i]:.0f} m): "
|
|
956
|
+
f"ref s/n {serials[ref_j]} ({habs[ref_j]:.0f} m, "
|
|
957
|
+
f"rope={rope:.0f} m)"
|
|
958
|
+
)
|
|
959
|
+
data_vars["tilt_from_pressure"] = xr.Variable(
|
|
960
|
+
("N_LEVELS", "time"),
|
|
961
|
+
tilt_p,
|
|
962
|
+
{
|
|
963
|
+
"units": "degrees",
|
|
964
|
+
"long_name": "Tilt estimated from pressure difference",
|
|
965
|
+
"comment": (
|
|
966
|
+
"arccos(ΔP / rope_length) where ΔP = pressure[i] - pressure[ref] "
|
|
967
|
+
"and rope_length = hab[ref] - hab[i] from mooring YAML; "
|
|
968
|
+
"ref is the nearest instrument ≥10 m above i with valid pressure."
|
|
969
|
+
),
|
|
970
|
+
},
|
|
971
|
+
)
|
|
972
|
+
data_vars["tilt_pressure_ref_hab"] = xr.Variable(
|
|
973
|
+
("N_LEVELS",),
|
|
974
|
+
ref_hab_arr,
|
|
975
|
+
{
|
|
976
|
+
"units": "m",
|
|
977
|
+
"long_name": "Height above bottom of the pressure reference instrument used for tilt",
|
|
978
|
+
},
|
|
979
|
+
)
|
|
980
|
+
data_vars["tilt_pressure_ref_serial"] = xr.Variable(
|
|
981
|
+
("N_LEVELS",),
|
|
982
|
+
ref_serial_arr,
|
|
983
|
+
{
|
|
984
|
+
"long_name": "Serial number of the pressure reference instrument used for tilt"
|
|
985
|
+
},
|
|
986
|
+
)
|
|
987
|
+
except Exception as exc: # noqa: BLE001 — optional derived var; must not abort stack
|
|
988
|
+
print(f" WARNING: could not compute tilt_from_pressure: {exc}")
|
|
989
|
+
|
|
990
|
+
# Coordinate names — exclude these from scalar metadata to avoid name conflicts
|
|
991
|
+
_coord_names = {"serial", "hab", "instrument_type", "instrument", "time"}
|
|
992
|
+
|
|
993
|
+
# Add scalar metadata — collapse to 0-D when all non-NaN values are identical,
|
|
994
|
+
# otherwise store as (N_LEVELS,) with one entry per instrument.
|
|
995
|
+
for vname, values_list in scalar_meta.items():
|
|
996
|
+
if vname in _coord_names:
|
|
997
|
+
continue
|
|
998
|
+
try:
|
|
999
|
+
arr = np.array(
|
|
1000
|
+
[v if v is not None else np.nan for v in values_list],
|
|
1001
|
+
dtype=np.float64,
|
|
1002
|
+
)
|
|
1003
|
+
unique_finite = np.unique(arr[np.isfinite(arr)])
|
|
1004
|
+
if len(unique_finite) <= 1:
|
|
1005
|
+
val = float(unique_finite[0]) if len(unique_finite) == 1 else np.nan
|
|
1006
|
+
data_vars[vname] = xr.Variable((), val, attrs=scalar_attrs[vname])
|
|
1007
|
+
else:
|
|
1008
|
+
data_vars[vname] = xr.Variable(
|
|
1009
|
+
("N_LEVELS",), arr, attrs=scalar_attrs[vname]
|
|
1010
|
+
)
|
|
1011
|
+
except (ValueError, TypeError):
|
|
1012
|
+
arr = np.array([str(v) if v is not None else "" for v in values_list])
|
|
1013
|
+
unique_vals = np.unique(arr)
|
|
1014
|
+
if len(unique_vals) == 1:
|
|
1015
|
+
data_vars[vname] = xr.Variable(
|
|
1016
|
+
(), unique_vals[0], attrs=scalar_attrs[vname]
|
|
1017
|
+
)
|
|
1018
|
+
else:
|
|
1019
|
+
data_vars[vname] = xr.Variable(
|
|
1020
|
+
("N_LEVELS",), arr, attrs=scalar_attrs[vname]
|
|
1021
|
+
)
|
|
1022
|
+
|
|
1023
|
+
ds_out = xr.Dataset(
|
|
1024
|
+
data_vars,
|
|
1025
|
+
coords={
|
|
1026
|
+
"time": xr.Variable(
|
|
1027
|
+
"time",
|
|
1028
|
+
common_time,
|
|
1029
|
+
{"long_name": "time", "axis": "T", "standard_name": "time"},
|
|
1030
|
+
),
|
|
1031
|
+
"serial": xr.Variable(
|
|
1032
|
+
"N_LEVELS",
|
|
1033
|
+
np.array(serials),
|
|
1034
|
+
{"long_name": "instrument serial number"},
|
|
1035
|
+
),
|
|
1036
|
+
"hab": xr.Variable(
|
|
1037
|
+
"N_LEVELS",
|
|
1038
|
+
np.array(habs),
|
|
1039
|
+
{"units": "m", "long_name": "height above bottom"},
|
|
1040
|
+
),
|
|
1041
|
+
"instrument_type": xr.Variable(
|
|
1042
|
+
"N_LEVELS",
|
|
1043
|
+
np.array(instr_types),
|
|
1044
|
+
{"long_name": "instrument type"},
|
|
1045
|
+
),
|
|
1046
|
+
},
|
|
1047
|
+
)
|
|
1048
|
+
_lat_str = (
|
|
1049
|
+
mooring_config.get("seabed_latitude")
|
|
1050
|
+
or mooring_config.get("deployment_latitude")
|
|
1051
|
+
or mooring_config.get("planned_latitude")
|
|
1052
|
+
or mooring_config.get("latitude")
|
|
1053
|
+
or ""
|
|
1054
|
+
)
|
|
1055
|
+
_lon_str = (
|
|
1056
|
+
mooring_config.get("seabed_longitude")
|
|
1057
|
+
or mooring_config.get("deployment_longitude")
|
|
1058
|
+
or mooring_config.get("planned_longitude")
|
|
1059
|
+
or mooring_config.get("longitude")
|
|
1060
|
+
or ""
|
|
1061
|
+
)
|
|
1062
|
+
ds_out.attrs.update(
|
|
1063
|
+
{
|
|
1064
|
+
"mooring_name": mooring_name,
|
|
1065
|
+
"waterdepth": str(mooring_config.get("waterdepth", "")),
|
|
1066
|
+
"latitude": str(_lat_str),
|
|
1067
|
+
"longitude": str(_lon_str),
|
|
1068
|
+
"deployment_time": str(deploy_time),
|
|
1069
|
+
"recovery_time": str(recover_time),
|
|
1070
|
+
"dt_seconds": dt_seconds,
|
|
1071
|
+
"Conventions": "CF-1.13",
|
|
1072
|
+
"history": (
|
|
1073
|
+
f"Step 1 stack: {n_instr} instruments onto {dt_seconds}s grid; "
|
|
1074
|
+
f"fast instruments (dt<={dt_seconds}s) subsampled (nearest-neighbour in time), "
|
|
1075
|
+
f"slow instruments interpolated (linear in time)"
|
|
1076
|
+
),
|
|
1077
|
+
}
|
|
1078
|
+
)
|
|
1079
|
+
|
|
1080
|
+
if output_path.exists():
|
|
1081
|
+
output_path.unlink()
|
|
1082
|
+
# Beam velocities are superseded by ENU components in the stacked file.
|
|
1083
|
+
beam_vel_vars = [v for v in ds_out.data_vars if v.startswith("velocity_beam")]
|
|
1084
|
+
if beam_vel_vars:
|
|
1085
|
+
ds_out = ds_out.drop_vars(beam_vel_vars)
|
|
1086
|
+
ds_out = drop_all_zero_vars(ds_out, ["amplitude_beam", "analog_input_"])
|
|
1087
|
+
# OceanSITES convention: time is the unlimited (first) dimension.
|
|
1088
|
+
ds_out = ds_out.transpose("time", "N_LEVELS")
|
|
1089
|
+
ds_out = cast_output_dtypes(ds_out)
|
|
1090
|
+
_enc = {
|
|
1091
|
+
v: {"zlib": True, "complevel": 5}
|
|
1092
|
+
for v in ds_out.data_vars
|
|
1093
|
+
if ds_out[v].dtype.kind not in ("O", "U", "S")
|
|
1094
|
+
}
|
|
1095
|
+
ds_out.to_netcdf(output_path, encoding=_enc)
|
|
1096
|
+
_status("file", self._rel(output_path))
|
|
1097
|
+
return True
|
|
1098
|
+
|
|
1099
|
+
|
|
1100
|
+
class MooringGridder:
|
|
1101
|
+
"""Step 2: vertically interpolate stacked instruments onto a pressure grid → ``_grid.nc``."""
|
|
1102
|
+
|
|
1103
|
+
def __init__(
|
|
1104
|
+
self,
|
|
1105
|
+
base_dir: Optional[str] = None,
|
|
1106
|
+
*,
|
|
1107
|
+
proc_dir: Optional[str] = None,
|
|
1108
|
+
) -> None:
|
|
1109
|
+
"""Interpolate a mooring stack onto a regular pressure grid.
|
|
1110
|
+
|
|
1111
|
+
Reads ``{mooring}_stack.nc`` (produced by :class:`MooringStacker`) and
|
|
1112
|
+
interpolates all scalar variables onto a uniform pressure axis defined by
|
|
1113
|
+
``p_start``, ``p_end``, and ``dp`` (in dbar). Writes
|
|
1114
|
+
``{mooring}_grid.nc`` in the mooring proc directory.
|
|
1115
|
+
|
|
1116
|
+
Parameters
|
|
1117
|
+
----------
|
|
1118
|
+
base_dir : str, optional
|
|
1119
|
+
Legacy: cruise-level base directory containing a ``proc/`` subdirectory.
|
|
1120
|
+
proc_dir : str, optional
|
|
1121
|
+
Cruise-level processed output directory. Pipeline appends ``/{mooring}/``.
|
|
1122
|
+
|
|
1123
|
+
"""
|
|
1124
|
+
if base_dir is not None:
|
|
1125
|
+
self.base_dir: Optional[Path] = Path(base_dir)
|
|
1126
|
+
self._proc_dir: Optional[Path] = None
|
|
1127
|
+
self._legacy = True
|
|
1128
|
+
else:
|
|
1129
|
+
self.base_dir = None
|
|
1130
|
+
self._proc_dir = Path(proc_dir) if proc_dir else None
|
|
1131
|
+
self._legacy = False
|
|
1132
|
+
|
|
1133
|
+
def _resolve_proc_dir(self, mooring_name: str) -> Path:
|
|
1134
|
+
"""Return the mooring-level proc directory."""
|
|
1135
|
+
if not self._legacy and self._proc_dir is not None:
|
|
1136
|
+
return self._proc_dir / mooring_name
|
|
1137
|
+
return _get_proc_dir(self.base_dir, mooring_name)
|
|
1138
|
+
|
|
1139
|
+
def _rel(self, path: Path) -> str:
|
|
1140
|
+
"""Return a short display path relative to base_dir or proc_dir."""
|
|
1141
|
+
for root in (r for r in (self.base_dir, self._proc_dir) if r):
|
|
1142
|
+
try:
|
|
1143
|
+
return str(path.relative_to(root))
|
|
1144
|
+
except ValueError:
|
|
1145
|
+
continue
|
|
1146
|
+
return path.name
|
|
1147
|
+
|
|
1148
|
+
def grid(
|
|
1149
|
+
self,
|
|
1150
|
+
mooring_name: str,
|
|
1151
|
+
p_start: float = 200.0,
|
|
1152
|
+
p_end: float = 1000.0,
|
|
1153
|
+
dp: float = 20.0,
|
|
1154
|
+
force: bool = False,
|
|
1155
|
+
) -> bool:
|
|
1156
|
+
"""Interpolate the stacked mooring dataset onto a regular pressure grid.
|
|
1157
|
+
|
|
1158
|
+
Reads ``{mooring}_stack.nc``, interpolates all variables onto a pressure
|
|
1159
|
+
axis from *p_start* to *p_end* in steps of *dp* dbar, and writes
|
|
1160
|
+
``{mooring}_grid.nc``.
|
|
1161
|
+
|
|
1162
|
+
Returns True on success, False on error.
|
|
1163
|
+
"""
|
|
1164
|
+
proc_dir = self._resolve_proc_dir(mooring_name)
|
|
1165
|
+
merge_path = proc_dir / f"{mooring_name}_stack.nc"
|
|
1166
|
+
output_path = proc_dir / f"{mooring_name}_grid.nc"
|
|
1167
|
+
|
|
1168
|
+
try:
|
|
1169
|
+
stack_found = merge_path.exists()
|
|
1170
|
+
output_found = output_path.exists()
|
|
1171
|
+
except (TimeoutError, OSError) as exc:
|
|
1172
|
+
print("ERROR: Cannot access data drive — is it connected?")
|
|
1173
|
+
print(f" Path: {merge_path.parent}")
|
|
1174
|
+
print(f" ({type(exc).__name__}: {exc})")
|
|
1175
|
+
return False
|
|
1176
|
+
|
|
1177
|
+
if not stack_found:
|
|
1178
|
+
print(f"ERROR: Stack file not found: {merge_path}")
|
|
1179
|
+
print(" Run 'oceanarray stack' first.")
|
|
1180
|
+
return False
|
|
1181
|
+
|
|
1182
|
+
if output_found and not force:
|
|
1183
|
+
_status("skip", self._rel(output_path))
|
|
1184
|
+
return True
|
|
1185
|
+
|
|
1186
|
+
p_grid = np.arange(p_start, p_end + dp * 0.5, dp)
|
|
1187
|
+
n_p = len(p_grid)
|
|
1188
|
+
|
|
1189
|
+
try:
|
|
1190
|
+
ds = xr.open_dataset(merge_path).load()
|
|
1191
|
+
except (TimeoutError, OSError) as exc:
|
|
1192
|
+
print("ERROR: Cannot read stack file — is the data drive connected?")
|
|
1193
|
+
print(f" ({type(exc).__name__}: {exc})")
|
|
1194
|
+
return False
|
|
1195
|
+
n_time = ds.sizes["time"]
|
|
1196
|
+
n_instr = ds.sizes["N_LEVELS"]
|
|
1197
|
+
|
|
1198
|
+
print(
|
|
1199
|
+
f"Gridding {n_instr} instruments → {n_p} pressure levels "
|
|
1200
|
+
f"({p_start:.0f}:{dp:.0f}:{p_end:.0f} dbar) × {n_time} time steps"
|
|
1201
|
+
)
|
|
1202
|
+
|
|
1203
|
+
if "pressure" not in ds.data_vars:
|
|
1204
|
+
print("ERROR: 'pressure' not found in stack file — cannot grid vertically")
|
|
1205
|
+
ds.close()
|
|
1206
|
+
return False
|
|
1207
|
+
|
|
1208
|
+
pressure = ds["pressure"].values.astype(np.float64) # (time, N_LEVELS)
|
|
1209
|
+
# Variables that are meaningful at the stacked per-instrument level but
|
|
1210
|
+
# should not be interpolated onto the pressure grid. Instrument-frame
|
|
1211
|
+
# and beam-frame velocities are excluded because the XYZ→ENU rotation
|
|
1212
|
+
# has already been applied; gridding the pre-rotation components would
|
|
1213
|
+
# be misleading. Diagnostic quantities (amplitude, correlation,
|
|
1214
|
+
# battery) are per-sensor and do not have a physical meaning on a
|
|
1215
|
+
# spatially-interpolated grid.
|
|
1216
|
+
_GRID_EXCLUDE: frozenset = frozenset(
|
|
1217
|
+
{
|
|
1218
|
+
"velocity_x",
|
|
1219
|
+
"velocity_y",
|
|
1220
|
+
"velocity_z",
|
|
1221
|
+
"velocity_beam1",
|
|
1222
|
+
"velocity_beam2",
|
|
1223
|
+
"velocity_beam3",
|
|
1224
|
+
"amplitude",
|
|
1225
|
+
"correlation",
|
|
1226
|
+
"battery_voltage",
|
|
1227
|
+
"velocity_flag", # flag array, not a gridded physics variable
|
|
1228
|
+
"tilt_from_pressure", # per-instrument diagnostic, not gridded
|
|
1229
|
+
"tilt_pressure_ref_hab",
|
|
1230
|
+
"heading", # instrument-frame orientation — not meaningful on a pressure grid
|
|
1231
|
+
"pitch",
|
|
1232
|
+
"roll",
|
|
1233
|
+
}
|
|
1234
|
+
)
|
|
1235
|
+
grid_vars = [
|
|
1236
|
+
v
|
|
1237
|
+
for v in ds.data_vars
|
|
1238
|
+
if v != "pressure"
|
|
1239
|
+
and ds[v].dims == ("time", "N_LEVELS")
|
|
1240
|
+
and not v.endswith("_qc")
|
|
1241
|
+
and v not in _GRID_EXCLUDE
|
|
1242
|
+
]
|
|
1243
|
+
|
|
1244
|
+
stacked: Dict[str, np.ndarray] = {
|
|
1245
|
+
v: np.full((n_p, n_time), np.nan) for v in grid_vars
|
|
1246
|
+
}
|
|
1247
|
+
var_data = {v: ds[v].values.astype(np.float64) for v in grid_vars}
|
|
1248
|
+
|
|
1249
|
+
# QC masking before vertical interpolation.
|
|
1250
|
+
# T/S/P: NaN any non-finite values (already masked at stack time).
|
|
1251
|
+
# Velocity: use velocity_flag from the stack to NaN suspect/bad samples.
|
|
1252
|
+
# Flag 3 (suspect), 4 (bad), 9 (missing) → NaN before interpolation.
|
|
1253
|
+
_GRIDDER_TSQC = {"temperature", "conductivity", "salinity"}
|
|
1254
|
+
for _v in grid_vars:
|
|
1255
|
+
if _v in _GRIDDER_TSQC:
|
|
1256
|
+
var_data[_v][~np.isfinite(var_data[_v])] = np.nan
|
|
1257
|
+
|
|
1258
|
+
_vel_vars = {
|
|
1259
|
+
"east_velocity",
|
|
1260
|
+
"north_velocity",
|
|
1261
|
+
"up_velocity",
|
|
1262
|
+
"current_speed",
|
|
1263
|
+
"current_direction",
|
|
1264
|
+
}
|
|
1265
|
+
if "velocity_flag" in ds.data_vars:
|
|
1266
|
+
_vflag = ds["velocity_flag"].values.astype(np.float64)
|
|
1267
|
+
_bad_vel = np.isin(np.round(_vflag).astype(np.int8), [3, 4, 9])
|
|
1268
|
+
for _v in _vel_vars:
|
|
1269
|
+
if _v in var_data:
|
|
1270
|
+
var_data[_v][_bad_vel] = np.nan
|
|
1271
|
+
else:
|
|
1272
|
+
for _v in _vel_vars:
|
|
1273
|
+
if _v in var_data:
|
|
1274
|
+
var_data[_v][~np.isfinite(var_data[_v])] = np.nan
|
|
1275
|
+
|
|
1276
|
+
# ADCP standalone QC variables — mask velocity before gridding.
|
|
1277
|
+
# seabed_qc removes bins at/below the seafloor; percent_good_qc and
|
|
1278
|
+
# error_velocity_qc remove poor-quality pings. All three use flag >= 3
|
|
1279
|
+
# (suspect or worse) as the NaN threshold, consistent with how the
|
|
1280
|
+
# rose diagram and stack report mask these variables.
|
|
1281
|
+
for _adcp_qc in ("seabed_qc", "percent_good_qc", "error_velocity_qc"):
|
|
1282
|
+
if _adcp_qc not in ds.data_vars:
|
|
1283
|
+
continue
|
|
1284
|
+
_qc = ds[_adcp_qc].values # (time, N_LEVELS) after stack transpose
|
|
1285
|
+
_bad = np.isin(np.round(_qc).astype(np.int8), [3, 4, 9])
|
|
1286
|
+
for _v in _vel_vars:
|
|
1287
|
+
if _v in var_data:
|
|
1288
|
+
var_data[_v][_bad] = np.nan
|
|
1289
|
+
|
|
1290
|
+
for t in range(n_time):
|
|
1291
|
+
p_col = pressure[t, :]
|
|
1292
|
+
p_valid_mask = np.isfinite(p_col)
|
|
1293
|
+
if p_valid_mask.sum() < 2:
|
|
1294
|
+
continue
|
|
1295
|
+
|
|
1296
|
+
for vname in grid_vars:
|
|
1297
|
+
v_col = var_data[vname][t, :]
|
|
1298
|
+
both_valid = p_valid_mask & np.isfinite(v_col)
|
|
1299
|
+
if both_valid.sum() < 2:
|
|
1300
|
+
continue
|
|
1301
|
+
p_v = p_col[both_valid]
|
|
1302
|
+
v_v = v_col[both_valid]
|
|
1303
|
+
sort_idx = np.argsort(p_v)
|
|
1304
|
+
stacked[vname][:, t] = np.interp(
|
|
1305
|
+
p_grid,
|
|
1306
|
+
p_v[sort_idx],
|
|
1307
|
+
v_v[sort_idx],
|
|
1308
|
+
left=np.nan,
|
|
1309
|
+
right=np.nan,
|
|
1310
|
+
)
|
|
1311
|
+
|
|
1312
|
+
# Build output dataset; skip variables that are entirely NaN
|
|
1313
|
+
data_vars: Dict = {}
|
|
1314
|
+
for vname in grid_vars:
|
|
1315
|
+
if not np.all(np.isnan(stacked[vname])):
|
|
1316
|
+
a = dict(ds[vname].attrs)
|
|
1317
|
+
a["vertical_interpolation"] = (
|
|
1318
|
+
f"linear in pressure; no extrapolation outside {p_start:.0f}–{p_end:.0f} dbar"
|
|
1319
|
+
)
|
|
1320
|
+
data_vars[vname] = xr.Variable(
|
|
1321
|
+
("time", "pressure"), stacked[vname].T, attrs=a
|
|
1322
|
+
)
|
|
1323
|
+
|
|
1324
|
+
ds_out = xr.Dataset(
|
|
1325
|
+
data_vars,
|
|
1326
|
+
coords={
|
|
1327
|
+
"time": ds["time"],
|
|
1328
|
+
"pressure": xr.Variable(
|
|
1329
|
+
"pressure",
|
|
1330
|
+
p_grid,
|
|
1331
|
+
{
|
|
1332
|
+
"units": "dbar",
|
|
1333
|
+
"long_name": "sea water pressure",
|
|
1334
|
+
"standard_name": "sea_water_pressure",
|
|
1335
|
+
"axis": "Z",
|
|
1336
|
+
"positive": "down",
|
|
1337
|
+
},
|
|
1338
|
+
),
|
|
1339
|
+
},
|
|
1340
|
+
)
|
|
1341
|
+
|
|
1342
|
+
prior_history = ds.attrs.get("history", "")
|
|
1343
|
+
ds_out.attrs.update(
|
|
1344
|
+
{
|
|
1345
|
+
**{k: v for k, v in ds.attrs.items() if k not in ("history",)},
|
|
1346
|
+
"p_start_dbar": p_start,
|
|
1347
|
+
"p_end_dbar": p_end,
|
|
1348
|
+
"dp_dbar": dp,
|
|
1349
|
+
"history": (
|
|
1350
|
+
prior_history
|
|
1351
|
+
+ f"; Step 2 grid: linear interpolation in pressure onto {dp:.0f} dbar pressure grid "
|
|
1352
|
+
f"({p_start:.0f}–{p_end:.0f} dbar); no extrapolation"
|
|
1353
|
+
),
|
|
1354
|
+
}
|
|
1355
|
+
)
|
|
1356
|
+
|
|
1357
|
+
ds.close()
|
|
1358
|
+
if output_path.exists():
|
|
1359
|
+
output_path.unlink()
|
|
1360
|
+
ds_out = cast_output_dtypes(ds_out)
|
|
1361
|
+
_enc = {
|
|
1362
|
+
v: {"zlib": True, "complevel": 5}
|
|
1363
|
+
for v in ds_out.data_vars
|
|
1364
|
+
if ds_out[v].dtype.kind not in ("O", "U", "S")
|
|
1365
|
+
}
|
|
1366
|
+
ds_out.to_netcdf(output_path, encoding=_enc)
|
|
1367
|
+
_status("file", self._rel(output_path))
|
|
1368
|
+
return True
|