microstructure-tpw 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- microstructure_tpw-0.3.0.dist-info/METADATA +234 -0
- microstructure_tpw-0.3.0.dist-info/RECORD +119 -0
- microstructure_tpw-0.3.0.dist-info/WHEEL +5 -0
- microstructure_tpw-0.3.0.dist-info/entry_points.txt +7 -0
- microstructure_tpw-0.3.0.dist-info/licenses/LICENSE +674 -0
- microstructure_tpw-0.3.0.dist-info/top_level.txt +1 -0
- odas_tpw/__init__.py +18 -0
- odas_tpw/chi/__init__.py +48 -0
- odas_tpw/chi/batchelor.py +189 -0
- odas_tpw/chi/chi.py +857 -0
- odas_tpw/chi/fp07.py +649 -0
- odas_tpw/chi/l2_chi.py +153 -0
- odas_tpw/chi/l3_chi.py +388 -0
- odas_tpw/chi/l4_chi.py +391 -0
- odas_tpw/config_base.py +423 -0
- odas_tpw/perturb/__init__.py +18 -0
- odas_tpw/perturb/adcp.py +274 -0
- odas_tpw/perturb/atomic_io.py +47 -0
- odas_tpw/perturb/binning.py +761 -0
- odas_tpw/perturb/cli.py +591 -0
- odas_tpw/perturb/combo.py +346 -0
- odas_tpw/perturb/config.py +1046 -0
- odas_tpw/perturb/ctd.py +424 -0
- odas_tpw/perturb/diag/__init__.py +12 -0
- odas_tpw/perturb/diag/_common.py +140 -0
- odas_tpw/perturb/diag/chi.py +42 -0
- odas_tpw/perturb/diag/cli.py +67 -0
- odas_tpw/perturb/diag/data.py +662 -0
- odas_tpw/perturb/diag/epsilon.py +45 -0
- odas_tpw/perturb/diag/inspector.py +342 -0
- odas_tpw/perturb/diag/mixing.py +94 -0
- odas_tpw/perturb/diag/render.py +543 -0
- odas_tpw/perturb/discover.py +40 -0
- odas_tpw/perturb/fp07_cal.py +354 -0
- odas_tpw/perturb/gps.py +458 -0
- odas_tpw/perturb/hotel.py +572 -0
- odas_tpw/perturb/logging_setup.py +181 -0
- odas_tpw/perturb/merge.py +320 -0
- odas_tpw/perturb/mk_sections.py +310 -0
- odas_tpw/perturb/netcdf_schema.py +520 -0
- odas_tpw/perturb/pipeline.py +2654 -0
- odas_tpw/perturb/plot/__init__.py +15 -0
- odas_tpw/perturb/plot/cli.py +107 -0
- odas_tpw/perturb/plot/diagnostics.py +195 -0
- odas_tpw/perturb/plot/eps_chi.py +524 -0
- odas_tpw/perturb/plot/figure.py +679 -0
- odas_tpw/perturb/plot/gamma_scaling.py +1097 -0
- odas_tpw/perturb/plot/grid.py +112 -0
- odas_tpw/perturb/plot/layout.py +361 -0
- odas_tpw/perturb/plot/overview.py +547 -0
- odas_tpw/perturb/plot/presets/eps-chi.yaml +16 -0
- odas_tpw/perturb/plot/presets/profiles.yaml +32 -0
- odas_tpw/perturb/plot/presets/scalar.yaml +21 -0
- odas_tpw/perturb/plot/profiles.py +652 -0
- odas_tpw/perturb/plot/scalar.py +391 -0
- odas_tpw/perturb/plot/sections.py +506 -0
- odas_tpw/perturb/plot/xaxis.py +396 -0
- odas_tpw/perturb/qc_gate.py +177 -0
- odas_tpw/perturb/qc_rules.py +373 -0
- odas_tpw/perturb/resolve.py +306 -0
- odas_tpw/perturb/seawater.py +76 -0
- odas_tpw/perturb/speed.py +14 -0
- odas_tpw/perturb/trim.py +313 -0
- odas_tpw/processing/__init__.py +62 -0
- odas_tpw/processing/bottom.py +215 -0
- odas_tpw/processing/chi_combine.py +300 -0
- odas_tpw/processing/ct_align.py +172 -0
- odas_tpw/processing/epsilon_combine.py +210 -0
- odas_tpw/processing/mixing.py +630 -0
- odas_tpw/processing/thorpe.py +458 -0
- odas_tpw/processing/top_trim.py +274 -0
- odas_tpw/pyturb/__init__.py +3 -0
- odas_tpw/pyturb/__main__.py +5 -0
- odas_tpw/pyturb/_compat.py +204 -0
- odas_tpw/pyturb/_profind.py +124 -0
- odas_tpw/pyturb/bin.py +113 -0
- odas_tpw/pyturb/cli.py +175 -0
- odas_tpw/pyturb/eps.py +353 -0
- odas_tpw/pyturb/merge.py +101 -0
- odas_tpw/pyturb/p2nc.py +76 -0
- odas_tpw/rsi/__init__.py +31 -0
- odas_tpw/rsi/adapter.py +314 -0
- odas_tpw/rsi/bench.py +904 -0
- odas_tpw/rsi/binning.py +193 -0
- odas_tpw/rsi/channels.py +376 -0
- odas_tpw/rsi/chi_io.py +980 -0
- odas_tpw/rsi/cli.py +1411 -0
- odas_tpw/rsi/combine.py +120 -0
- odas_tpw/rsi/config.py +126 -0
- odas_tpw/rsi/config_patch.py +790 -0
- odas_tpw/rsi/convert.py +651 -0
- odas_tpw/rsi/deconvolve.py +119 -0
- odas_tpw/rsi/diss_look.py +343 -0
- odas_tpw/rsi/dissipation.py +669 -0
- odas_tpw/rsi/helpers.py +527 -0
- odas_tpw/rsi/mixing_look.py +509 -0
- odas_tpw/rsi/p_file.py +771 -0
- odas_tpw/rsi/pipeline.py +924 -0
- odas_tpw/rsi/profile.py +624 -0
- odas_tpw/rsi/quick_look.py +758 -0
- odas_tpw/rsi/sensor_inventory.py +684 -0
- odas_tpw/rsi/shear_noise.py +128 -0
- odas_tpw/rsi/speed.py +221 -0
- odas_tpw/rsi/vehicle.py +74 -0
- odas_tpw/rsi/viewer_base.py +1071 -0
- odas_tpw/rsi/window.py +459 -0
- odas_tpw/scor160/__init__.py +36 -0
- odas_tpw/scor160/cli.py +242 -0
- odas_tpw/scor160/compare.py +484 -0
- odas_tpw/scor160/despike.py +227 -0
- odas_tpw/scor160/goodman.py +288 -0
- odas_tpw/scor160/io.py +690 -0
- odas_tpw/scor160/l2.py +311 -0
- odas_tpw/scor160/l3.py +279 -0
- odas_tpw/scor160/l4.py +850 -0
- odas_tpw/scor160/nasmyth.py +208 -0
- odas_tpw/scor160/ocean.py +287 -0
- odas_tpw/scor160/profile.py +231 -0
- odas_tpw/scor160/spectral.py +482 -0
|
@@ -0,0 +1,684 @@
|
|
|
1
|
+
# Jul-2026, Claude and Pat Welch, pat@mousebrains.com
|
|
2
|
+
"""Inventory microstructure sensors across a tree of Rockland ``.p`` files.
|
|
3
|
+
|
|
4
|
+
Point this at the top of a directory tree; it finds every ``.p`` file below it
|
|
5
|
+
and, for each sensor serial number, summarizes:
|
|
6
|
+
|
|
7
|
+
* the date range over which the sensor was used,
|
|
8
|
+
* the number of files it appears in,
|
|
9
|
+
* the platform(s) it was mounted on (vehicle type + instrument SN), and
|
|
10
|
+
* whether its calibration parameters changed across those uses.
|
|
11
|
+
|
|
12
|
+
Two sensor kinds are understood today and the registry is extensible:
|
|
13
|
+
|
|
14
|
+
* **shear** probes — config channel ``type`` ``shear`` / ``xmp_shear``
|
|
15
|
+
(tracks adc_fs, adc_bits, diff_gain, sens, cal_date), and
|
|
16
|
+
* **fp07** fast thermistors — config channel ``type`` ``therm``
|
|
17
|
+
(tracks adc_fs, adc_bits, a, b, g, e_b, beta_1, beta_2, t_0, cal_date).
|
|
18
|
+
|
|
19
|
+
Only the 128-byte binary header and the embedded INI configuration string of
|
|
20
|
+
each file are read (never the data records), so the scan is fast even over
|
|
21
|
+
large trees. The header supplies the timestamp; the config supplies the sensor
|
|
22
|
+
channels and their per-sensor parameters.
|
|
23
|
+
|
|
24
|
+
Sensors are grouped by serial number: a probe that moves between ports
|
|
25
|
+
(sh1↔sh2) or platforms is still one entry. A caveat follows from that — if a
|
|
26
|
+
config leaves the serial number blank or uses a placeholder shared by physically
|
|
27
|
+
distinct sensors (e.g. FP07 ``sn = T`` on both T1 and T2), those sensors are
|
|
28
|
+
merged under one SN and their differing calibrations show up as "CHANGED".
|
|
29
|
+
|
|
30
|
+
Usage
|
|
31
|
+
-----
|
|
32
|
+
python -m odas_tpw.rsi.sensor_inventory /path/to/tree [more paths ...]
|
|
33
|
+
python -m odas_tpw.rsi.sensor_inventory VMP/ --shear --csv probes.csv
|
|
34
|
+
python -m odas_tpw.rsi.sensor_inventory VMP/ --all --verbose
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import argparse
|
|
40
|
+
import csv
|
|
41
|
+
import glob as globmod
|
|
42
|
+
import math
|
|
43
|
+
import re
|
|
44
|
+
import struct
|
|
45
|
+
import sys
|
|
46
|
+
import warnings
|
|
47
|
+
from collections import defaultdict
|
|
48
|
+
from dataclasses import dataclass, field
|
|
49
|
+
from datetime import UTC, datetime, timedelta, timezone
|
|
50
|
+
from pathlib import Path
|
|
51
|
+
from typing import TextIO
|
|
52
|
+
|
|
53
|
+
from odas_tpw.rsi.p_file import (
|
|
54
|
+
HEADER_BYTES,
|
|
55
|
+
_detect_endian,
|
|
56
|
+
_parse_header,
|
|
57
|
+
parse_config,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class SensorKind:
|
|
63
|
+
"""A recognizable microstructure sensor family.
|
|
64
|
+
|
|
65
|
+
Attributes
|
|
66
|
+
----------
|
|
67
|
+
key : str
|
|
68
|
+
Short identifier / CLI selector (e.g. ``"shear"``).
|
|
69
|
+
label : str
|
|
70
|
+
Human-readable singular label (e.g. ``"Shear probe"``).
|
|
71
|
+
types : frozenset[str]
|
|
72
|
+
Config channel ``type`` values (lower-case) that denote this sensor.
|
|
73
|
+
params : tuple[str, ...]
|
|
74
|
+
Per-sensor calibration parameters tracked for change, in display order.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
key: str
|
|
78
|
+
label: str
|
|
79
|
+
types: frozenset[str]
|
|
80
|
+
params: tuple[str, ...]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# Registry of known sensor kinds. Add new kinds here; --all covers every entry.
|
|
84
|
+
SENSOR_KINDS: dict[str, SensorKind] = {
|
|
85
|
+
"shear": SensorKind(
|
|
86
|
+
key="shear",
|
|
87
|
+
label="Shear probe",
|
|
88
|
+
types=frozenset({"shear", "xmp_shear"}),
|
|
89
|
+
params=("adc_fs", "adc_bits", "diff_gain", "sens", "cal_date"),
|
|
90
|
+
),
|
|
91
|
+
"fp07": SensorKind(
|
|
92
|
+
key="fp07",
|
|
93
|
+
label="FP07 thermistor",
|
|
94
|
+
types=frozenset({"therm"}),
|
|
95
|
+
params=("adc_fs", "adc_bits", "a", "b", "g", "e_b", "beta_1", "beta_2", "t_0", "cal_date"),
|
|
96
|
+
),
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
# Pre-emphasized derivative channels (T1_dT1, P_dP, ...) share a base channel's
|
|
100
|
+
# ``type`` but carry only diff_gain, not the sensor's SN / calibration. They are
|
|
101
|
+
# the same physical sensor, so exclude them from the inventory. Mirrors the
|
|
102
|
+
# X_dX pattern matched in p_file.PFile._apply_deconvolution.
|
|
103
|
+
_PREEMPH_RE = re.compile(r"^(\w+)_d\1$")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def resolve_kinds(shear: bool = False, fp07: bool = False, want_all: bool = False) -> list[str]:
|
|
107
|
+
"""Map the --shear/--fp07/--all flags to an ordered list of kind keys.
|
|
108
|
+
|
|
109
|
+
With nothing selected (or --all), every registered kind is returned so the
|
|
110
|
+
tool inventories everything by default.
|
|
111
|
+
"""
|
|
112
|
+
if want_all or not (shear or fp07):
|
|
113
|
+
return list(SENSOR_KINDS)
|
|
114
|
+
kinds = []
|
|
115
|
+
if shear:
|
|
116
|
+
kinds.append("shear")
|
|
117
|
+
if fp07:
|
|
118
|
+
kinds.append("fp07")
|
|
119
|
+
return kinds
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ---------------------------------------------------------------------------
|
|
123
|
+
# Lightweight header + config reader (no data records)
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _read_header_and_config(path: Path) -> tuple[dict, dict]:
|
|
128
|
+
"""Return (parsed header, parsed config) reading only record 0 of *path*."""
|
|
129
|
+
with open(path, "rb") as f:
|
|
130
|
+
raw_hdr = f.read(HEADER_BYTES)
|
|
131
|
+
if len(raw_hdr) < HEADER_BYTES:
|
|
132
|
+
raise ValueError(f"{path.name}: file too small for header")
|
|
133
|
+
with warnings.catch_warnings():
|
|
134
|
+
# Over a large tree, MR/glider files whose clock had not yet been set
|
|
135
|
+
# at power-up carry a zeroed header (endian flag 0), which makes
|
|
136
|
+
# _detect_endian warn on potentially thousands of files. The warning
|
|
137
|
+
# is irrelevant to this tool -- serial numbers come from the ASCII
|
|
138
|
+
# config block, which is endian-independent -- so silence it to keep
|
|
139
|
+
# the run readable.
|
|
140
|
+
warnings.simplefilter("ignore")
|
|
141
|
+
endian = _detect_endian(raw_hdr, path)
|
|
142
|
+
header = _parse_header(raw_hdr, endian)
|
|
143
|
+
|
|
144
|
+
header_size = int(header["header_size"])
|
|
145
|
+
config_size = int(header["config_size"])
|
|
146
|
+
if header_size < HEADER_BYTES:
|
|
147
|
+
raise ValueError(f"{path.name}: invalid header_size={header_size}")
|
|
148
|
+
if config_size < 0:
|
|
149
|
+
raise ValueError(f"{path.name}: invalid config_size={config_size}")
|
|
150
|
+
|
|
151
|
+
f.seek(header_size)
|
|
152
|
+
config_str = f.read(config_size).decode("ascii", errors="replace")
|
|
153
|
+
return header, parse_config(config_str)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _start_time_utc(header: dict, config: dict) -> datetime | None:
|
|
157
|
+
"""Recording start time in UTC, or None if the header carries no valid clock.
|
|
158
|
+
|
|
159
|
+
Mirrors PFile._read: the header timestamp marks the END of record 0, so the
|
|
160
|
+
ODAS-consistent data start time subtracts the record duration (``recsize``,
|
|
161
|
+
default 1.0 s). Absolute alignment is immaterial for a date-range summary,
|
|
162
|
+
but staying consistent with the rest of the codebase avoids surprises.
|
|
163
|
+
|
|
164
|
+
An MR/glider that powers up before the host sets its clock writes a zeroed
|
|
165
|
+
(or garbage) date — year 0, month 36123, etc. Those still carry a valid
|
|
166
|
+
ASCII config with real probe serial numbers, so rather than discard the
|
|
167
|
+
file we return None here and let the caller inventory it without a date.
|
|
168
|
+
"""
|
|
169
|
+
h = header
|
|
170
|
+
tz_min = h["timezone_min"] - 2**16 if h["timezone_min"] >= 2**15 else h["timezone_min"]
|
|
171
|
+
root = config.get("root", {}) if isinstance(config, dict) else {}
|
|
172
|
+
raw = root.get("recsize", root.get("recordduration"))
|
|
173
|
+
try:
|
|
174
|
+
recsize = float(raw) if raw is not None else 1.0
|
|
175
|
+
except (TypeError, ValueError):
|
|
176
|
+
recsize = 1.0
|
|
177
|
+
if not math.isfinite(recsize): # a corrupt 'recsize = 1e999' -> inf -> timedelta OverflowError
|
|
178
|
+
recsize = 1.0
|
|
179
|
+
# Everything that can raise on a garbage header must stay inside the guard so
|
|
180
|
+
# a single bad file returns None (undated) rather than aborting the batch:
|
|
181
|
+
# out-of-range date fields and tz -> ValueError; a datetime at the floor of
|
|
182
|
+
# the range minus recsize, or a huge recsize -> OverflowError.
|
|
183
|
+
try:
|
|
184
|
+
st = datetime(
|
|
185
|
+
h["year"],
|
|
186
|
+
h["month"],
|
|
187
|
+
h["day"],
|
|
188
|
+
h["hour"],
|
|
189
|
+
h["minute"],
|
|
190
|
+
h["second"],
|
|
191
|
+
h["millisecond"] * 1000,
|
|
192
|
+
tzinfo=timezone(timedelta(minutes=tz_min)),
|
|
193
|
+
)
|
|
194
|
+
st -= timedelta(seconds=recsize)
|
|
195
|
+
return st.astimezone(UTC)
|
|
196
|
+
except (ValueError, OverflowError):
|
|
197
|
+
return None
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
@dataclass
|
|
201
|
+
class SensorUse:
|
|
202
|
+
"""One sensor channel as configured in one .p file."""
|
|
203
|
+
|
|
204
|
+
kind: str # SensorKind.key, e.g. "shear"
|
|
205
|
+
path: Path
|
|
206
|
+
start_time: datetime | None # None when the file's clock was unset
|
|
207
|
+
vehicle: str
|
|
208
|
+
platform_sn: str
|
|
209
|
+
channel: str # e.g. "sh1", "T1"
|
|
210
|
+
sensor_sn: str # e.g. "M2732"
|
|
211
|
+
params: dict[str, str] # kind.params -> raw string value
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _platform_from_instrument(inst: dict) -> tuple[str, str]:
|
|
215
|
+
"""(vehicle, platform_sn) from an instrument_info block, tolerant of gaps.
|
|
216
|
+
|
|
217
|
+
Strip BEFORE falling back, so a whitespace-only field (truthy but empty)
|
|
218
|
+
still degrades to model/"?" instead of collapsing to "" — otherwise the
|
|
219
|
+
("", sn) platform key would split from the ("?", sn) of a genuinely-missing
|
|
220
|
+
field. A microrider-on-glider leaves vehicle set ("slocum_glider"); a file
|
|
221
|
+
whose host had not populated instrument_info yet yields ("?", "?").
|
|
222
|
+
"""
|
|
223
|
+
vehicle = inst.get("vehicle", "").strip() or inst.get("model", "").strip() or "?"
|
|
224
|
+
platform_sn = inst.get("sn", "").strip() or "?"
|
|
225
|
+
return vehicle, platform_sn
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def scan_file(path: Path, kinds: list[str]) -> list[SensorUse]:
|
|
229
|
+
"""Extract every sensor use of the requested *kinds* from a single .p file.
|
|
230
|
+
|
|
231
|
+
Returns one :class:`SensorUse` per matching (non pre-emphasis) channel.
|
|
232
|
+
Raises on unreadable/corrupt headers so the caller can report the file.
|
|
233
|
+
"""
|
|
234
|
+
header, config = _read_header_and_config(path)
|
|
235
|
+
st = _start_time_utc(header, config)
|
|
236
|
+
|
|
237
|
+
vehicle, platform_sn = _platform_from_instrument(config.get("instrument_info", {}))
|
|
238
|
+
|
|
239
|
+
# Channel type -> owning kind, for the requested kinds only.
|
|
240
|
+
type_to_kind: dict[str, SensorKind] = {}
|
|
241
|
+
for key in kinds:
|
|
242
|
+
for ctype in SENSOR_KINDS[key].types:
|
|
243
|
+
type_to_kind[ctype] = SENSOR_KINDS[key]
|
|
244
|
+
|
|
245
|
+
uses: list[SensorUse] = []
|
|
246
|
+
for ch in config.get("channels", []):
|
|
247
|
+
name = (ch.get("name") or "").strip()
|
|
248
|
+
if _PREEMPH_RE.match(name):
|
|
249
|
+
continue
|
|
250
|
+
kind = type_to_kind.get(ch.get("type", "").strip().lower())
|
|
251
|
+
if kind is None:
|
|
252
|
+
continue
|
|
253
|
+
sensor_sn = (ch.get("sn") or "").strip() or "(no SN)"
|
|
254
|
+
uses.append(
|
|
255
|
+
SensorUse(
|
|
256
|
+
kind=kind.key,
|
|
257
|
+
path=path,
|
|
258
|
+
start_time=st,
|
|
259
|
+
vehicle=vehicle,
|
|
260
|
+
platform_sn=platform_sn,
|
|
261
|
+
channel=name or "?",
|
|
262
|
+
sensor_sn=sensor_sn,
|
|
263
|
+
params={p: (ch.get(p) or "").strip() for p in kind.params},
|
|
264
|
+
)
|
|
265
|
+
)
|
|
266
|
+
return uses
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
# ---------------------------------------------------------------------------
|
|
270
|
+
# Aggregation
|
|
271
|
+
# ---------------------------------------------------------------------------
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _norm(key: str, value: str):
|
|
275
|
+
"""Normalize a parameter value for change detection.
|
|
276
|
+
|
|
277
|
+
Numeric params compare by value so cosmetic formatting differences
|
|
278
|
+
("0.933" vs "0.9330") are not mistaken for a real change; non-numeric or
|
|
279
|
+
unparseable values (e.g. cal_date) fall back to the stripped string.
|
|
280
|
+
|
|
281
|
+
Non-finite floats from a corrupt config (nan/inf) fall back to the exact
|
|
282
|
+
string too — otherwise two identical "nan" values would compare unequal
|
|
283
|
+
(a false CHANGED) and two different overflowing values ("1e400"/"1e999",
|
|
284
|
+
both inf) would merge (a false constant).
|
|
285
|
+
"""
|
|
286
|
+
if value == "":
|
|
287
|
+
return ""
|
|
288
|
+
try:
|
|
289
|
+
f = float(value)
|
|
290
|
+
except ValueError:
|
|
291
|
+
return value
|
|
292
|
+
if not math.isfinite(f):
|
|
293
|
+
return value
|
|
294
|
+
# int() only after the finite check, so int(float("inf")) can't raise
|
|
295
|
+
# OverflowError here (this runs in build_inventory, which has no guard).
|
|
296
|
+
return int(f) if key == "adc_bits" else f
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _new_value_slot() -> dict:
|
|
300
|
+
"""Per-distinct-value accumulator: display text, files, and date range."""
|
|
301
|
+
return {"display": "", "files": set(), "first": None, "last": None}
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _new_param_map() -> defaultdict:
|
|
305
|
+
"""param key -> (normalized value -> value slot), created lazily."""
|
|
306
|
+
return defaultdict(lambda: defaultdict(_new_value_slot))
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
@dataclass
|
|
310
|
+
class SensorAgg:
|
|
311
|
+
"""Accumulated record for one sensor serial number."""
|
|
312
|
+
|
|
313
|
+
files: set[Path] = field(default_factory=set)
|
|
314
|
+
first: datetime | None = None
|
|
315
|
+
last: datetime | None = None
|
|
316
|
+
# platform (vehicle, sn) -> set of channel names it was wired to
|
|
317
|
+
platforms: dict[tuple[str, str], set[str]] = field(default_factory=lambda: defaultdict(set))
|
|
318
|
+
# param key -> normalized value -> {display, files, first, last}
|
|
319
|
+
params: defaultdict = field(default_factory=_new_param_map)
|
|
320
|
+
|
|
321
|
+
def add(self, use: SensorUse) -> None:
|
|
322
|
+
self.files.add(use.path)
|
|
323
|
+
st = use.start_time
|
|
324
|
+
if st is not None:
|
|
325
|
+
if self.first is None or st < self.first:
|
|
326
|
+
self.first = st
|
|
327
|
+
if self.last is None or st > self.last:
|
|
328
|
+
self.last = st
|
|
329
|
+
self.platforms[(use.vehicle, use.platform_sn)].add(use.channel)
|
|
330
|
+
for key, raw in use.params.items():
|
|
331
|
+
slot = self.params[key][_norm(key, raw)]
|
|
332
|
+
if not slot["display"]: # first-seen wins → deterministic (files sorted)
|
|
333
|
+
slot["display"] = raw if raw != "" else "(blank)"
|
|
334
|
+
slot["files"].add(use.path)
|
|
335
|
+
if st is not None:
|
|
336
|
+
if slot["first"] is None or st < slot["first"]:
|
|
337
|
+
slot["first"] = st
|
|
338
|
+
if slot["last"] is None or st > slot["last"]:
|
|
339
|
+
slot["last"] = st
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def build_inventory(uses: list[SensorUse]) -> dict[str, dict[str, SensorAgg]]:
|
|
343
|
+
"""Group uses into ``{kind_key: {serial_number: SensorAgg}}``."""
|
|
344
|
+
inventory: dict[str, dict[str, SensorAgg]] = defaultdict(lambda: defaultdict(SensorAgg))
|
|
345
|
+
for use in uses:
|
|
346
|
+
inventory[use.kind][use.sensor_sn].add(use)
|
|
347
|
+
return inventory
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def collect_uses(
|
|
351
|
+
files: list[Path], kinds: list[str]
|
|
352
|
+
) -> tuple[list[SensorUse], list[tuple[Path, str]], int]:
|
|
353
|
+
"""Scan *files*, returning (uses, errors, n_files_with_no_matching_sensor)."""
|
|
354
|
+
uses: list[SensorUse] = []
|
|
355
|
+
errors: list[tuple[Path, str]] = []
|
|
356
|
+
n_no_sensor = 0
|
|
357
|
+
for fp in files:
|
|
358
|
+
try:
|
|
359
|
+
found = scan_file(fp, kinds)
|
|
360
|
+
except (OSError, ValueError, KeyError, OverflowError, struct.error) as exc:
|
|
361
|
+
errors.append((fp, str(exc)))
|
|
362
|
+
continue
|
|
363
|
+
if not found:
|
|
364
|
+
n_no_sensor += 1
|
|
365
|
+
uses.extend(found)
|
|
366
|
+
return uses, errors, n_no_sensor
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
# ---------------------------------------------------------------------------
|
|
370
|
+
# File discovery
|
|
371
|
+
# ---------------------------------------------------------------------------
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _collect_from(p: Path, found: set[Path]) -> None:
|
|
375
|
+
"""Add every .p/.P file at or under *p* (a directory or a single file)."""
|
|
376
|
+
if p.is_dir():
|
|
377
|
+
for pat in ("*.p", "*.P"):
|
|
378
|
+
found.update(q for q in p.rglob(pat) if q.is_file())
|
|
379
|
+
elif p.is_file():
|
|
380
|
+
found.add(p)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def iter_pfiles(paths: list[Path]) -> list[Path]:
|
|
384
|
+
"""All ``.p``/``.P`` files under the given files/directories, de-duplicated.
|
|
385
|
+
|
|
386
|
+
Each argument may be a directory (scanned recursively), a single file, or a
|
|
387
|
+
glob pattern. A path that does not exist is passed to ``glob`` so that a
|
|
388
|
+
quoted or shell-unexpanded pattern (e.g. ``'VMP/*.p'``) works the same way it
|
|
389
|
+
does for the other rsi-tpw subcommands.
|
|
390
|
+
"""
|
|
391
|
+
found: set[Path] = set()
|
|
392
|
+
for p in paths:
|
|
393
|
+
if p.exists():
|
|
394
|
+
_collect_from(p, found)
|
|
395
|
+
continue
|
|
396
|
+
matches = globmod.glob(str(p), recursive=True)
|
|
397
|
+
if not matches:
|
|
398
|
+
print(f"warning: {p} matched no files or directories; skipping", file=sys.stderr)
|
|
399
|
+
continue
|
|
400
|
+
for m in matches:
|
|
401
|
+
_collect_from(Path(m), found)
|
|
402
|
+
return sorted(found)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
# ---------------------------------------------------------------------------
|
|
406
|
+
# Reporting
|
|
407
|
+
# ---------------------------------------------------------------------------
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _fmt_range(first: datetime | None, last: datetime | None) -> str:
|
|
411
|
+
if first is None:
|
|
412
|
+
return "(no valid timestamp)"
|
|
413
|
+
a = first.strftime("%Y-%m-%d %H:%M")
|
|
414
|
+
if last is None or last == first:
|
|
415
|
+
return f"{a} UTC"
|
|
416
|
+
b = last.strftime("%H:%M" if first.date() == last.date() else "%Y-%m-%d %H:%M")
|
|
417
|
+
return f"{a} → {b} UTC"
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _fmt_platform(vehicle: str, platform_sn: str, channels: set[str]) -> str:
|
|
421
|
+
"""Human platform label, tolerant of the empty instrument_info a file writes
|
|
422
|
+
before its host populates it (both vehicle and SN come through as "?")."""
|
|
423
|
+
chans = ", ".join(sorted(channels))
|
|
424
|
+
if vehicle == "?" and platform_sn == "?":
|
|
425
|
+
where = "unknown platform"
|
|
426
|
+
elif platform_sn == "?":
|
|
427
|
+
where = f"{vehicle} (SN unknown)"
|
|
428
|
+
elif vehicle == "?":
|
|
429
|
+
where = f"SN {platform_sn}"
|
|
430
|
+
else:
|
|
431
|
+
where = f"{vehicle} SN {platform_sn}"
|
|
432
|
+
return f"{where} (as {chans})"
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
# Acquisition/ADC settings dropped from the one-line --compact view: they are
|
|
436
|
+
# instrument config shared across probes, not part of a probe's own calibration.
|
|
437
|
+
_COMPACT_ADC_PARAMS = frozenset({"adc_fs", "adc_bits"})
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _compact_value(agg: SensorAgg, key: str) -> str:
|
|
441
|
+
"""The value of *key* for a compact line: the single value, or the distinct
|
|
442
|
+
values joined oldest-first by ``→`` when the parameter changed across uses."""
|
|
443
|
+
values = agg.params.get(key, {})
|
|
444
|
+
if not values:
|
|
445
|
+
return "(none)"
|
|
446
|
+
slots = sorted(values.values(), key=lambda s: s["first"] or datetime.max.replace(tzinfo=UTC))
|
|
447
|
+
displays = [s["display"] for s in slots]
|
|
448
|
+
return displays[0] if len(displays) == 1 else "→".join(displays)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _compact_line(kind: SensorKind, sn: str, agg: SensorAgg) -> str:
|
|
452
|
+
"""One-line probe summary: ``<sn> #<files> <param>: <value> ... used: <range>``.
|
|
453
|
+
|
|
454
|
+
Omits the ADC settings, the platform, and the ``(constant)`` markers of the
|
|
455
|
+
full report; a changed parameter shows its values joined by ``→``.
|
|
456
|
+
"""
|
|
457
|
+
parts = [sn, f"#{len(agg.files)}"]
|
|
458
|
+
parts += [
|
|
459
|
+
f"{key}: {_compact_value(agg, key)}"
|
|
460
|
+
for key in kind.params
|
|
461
|
+
if key not in _COMPACT_ADC_PARAMS
|
|
462
|
+
]
|
|
463
|
+
parts.append(f"used: {_fmt_range(agg.first, agg.last).removesuffix(' UTC')}")
|
|
464
|
+
return " ".join(parts)
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def print_report(
|
|
468
|
+
inventory: dict[str, dict[str, SensorAgg]],
|
|
469
|
+
kinds: list[str],
|
|
470
|
+
verbose: bool = False,
|
|
471
|
+
compact: bool = False,
|
|
472
|
+
stream: TextIO | None = None,
|
|
473
|
+
) -> None:
|
|
474
|
+
out = stream if stream is not None else sys.stdout
|
|
475
|
+
for kind_key in kinds:
|
|
476
|
+
probes = inventory.get(kind_key)
|
|
477
|
+
if not probes:
|
|
478
|
+
continue
|
|
479
|
+
kind = SENSOR_KINDS[kind_key]
|
|
480
|
+
heading = f"{kind.label}s ({len(probes)})"
|
|
481
|
+
print(heading, file=out)
|
|
482
|
+
print("=" * len(heading), file=out)
|
|
483
|
+
print(file=out)
|
|
484
|
+
|
|
485
|
+
if compact:
|
|
486
|
+
for sn in sorted(probes):
|
|
487
|
+
print(_compact_line(kind, sn, probes[sn]), file=out)
|
|
488
|
+
print(file=out)
|
|
489
|
+
continue
|
|
490
|
+
|
|
491
|
+
width = max(len(p) for p in kind.params) + 1
|
|
492
|
+
for sn in sorted(probes):
|
|
493
|
+
agg = probes[sn]
|
|
494
|
+
print(f"{kind.label} {sn}", file=out)
|
|
495
|
+
print(f" files: {len(agg.files)}", file=out)
|
|
496
|
+
print(f" date range: {_fmt_range(agg.first, agg.last)}", file=out)
|
|
497
|
+
|
|
498
|
+
plats = [
|
|
499
|
+
_fmt_platform(veh, psn, channels)
|
|
500
|
+
for (veh, psn), channels in sorted(agg.platforms.items())
|
|
501
|
+
]
|
|
502
|
+
print(f" platforms: {'; '.join(plats)}", file=out)
|
|
503
|
+
|
|
504
|
+
for key in kind.params:
|
|
505
|
+
values = agg.params.get(key, {})
|
|
506
|
+
if len(values) <= 1:
|
|
507
|
+
disp = next(iter(values.values()))["display"] if values else "(none)"
|
|
508
|
+
print(f" {key + ':':{width}s} {disp} (constant)", file=out)
|
|
509
|
+
else:
|
|
510
|
+
print(f" {key + ':':{width}s} CHANGED", file=out)
|
|
511
|
+
# Oldest-first so the sequence of changes reads naturally.
|
|
512
|
+
for slot in sorted(
|
|
513
|
+
values.values(),
|
|
514
|
+
key=lambda s: s["first"] or datetime.max.replace(tzinfo=UTC),
|
|
515
|
+
):
|
|
516
|
+
n = len(slot["files"])
|
|
517
|
+
unit = "file" if n == 1 else "files"
|
|
518
|
+
print(
|
|
519
|
+
f" {slot['display']:<12s} {n:3d} {unit:5s} "
|
|
520
|
+
f"{_fmt_range(slot['first'], slot['last'])}",
|
|
521
|
+
file=out,
|
|
522
|
+
)
|
|
523
|
+
if verbose:
|
|
524
|
+
for fp in sorted(slot["files"]):
|
|
525
|
+
print(f" {fp}", file=out)
|
|
526
|
+
print(file=out)
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def write_csv(uses: list[SensorUse], out: Path, kinds: list[str]) -> None:
|
|
530
|
+
"""One row per sensor channel per file — the raw data behind the summary.
|
|
531
|
+
|
|
532
|
+
Wide format: the union of the selected kinds' parameter columns, blank where
|
|
533
|
+
a parameter does not apply to that sensor kind.
|
|
534
|
+
"""
|
|
535
|
+
param_cols: list[str] = []
|
|
536
|
+
for key in kinds:
|
|
537
|
+
for p in SENSOR_KINDS[key].params:
|
|
538
|
+
if p not in param_cols:
|
|
539
|
+
param_cols.append(p)
|
|
540
|
+
cols = ["sensor", "sn", "file", "start_time_utc", "vehicle", "platform_sn", "channel"]
|
|
541
|
+
cols += param_cols
|
|
542
|
+
|
|
543
|
+
undated = datetime.min.replace(tzinfo=UTC) # sort undated uses first, deterministically
|
|
544
|
+
with open(out, "w", newline="") as f:
|
|
545
|
+
w = csv.writer(f)
|
|
546
|
+
w.writerow(cols)
|
|
547
|
+
for u in sorted(
|
|
548
|
+
uses, key=lambda u: (u.kind, u.sensor_sn, u.start_time or undated, str(u.path))
|
|
549
|
+
):
|
|
550
|
+
w.writerow(
|
|
551
|
+
[
|
|
552
|
+
u.kind,
|
|
553
|
+
u.sensor_sn,
|
|
554
|
+
str(u.path),
|
|
555
|
+
u.start_time.strftime("%Y-%m-%dT%H:%M:%SZ") if u.start_time else "",
|
|
556
|
+
u.vehicle,
|
|
557
|
+
u.platform_sn,
|
|
558
|
+
u.channel,
|
|
559
|
+
*(u.params.get(c, "") for c in param_cols),
|
|
560
|
+
]
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def run(
|
|
565
|
+
paths: list[Path],
|
|
566
|
+
kinds: list[str],
|
|
567
|
+
csv_out: Path | None = None,
|
|
568
|
+
verbose: bool = False,
|
|
569
|
+
compact: bool = False,
|
|
570
|
+
stream: TextIO | None = None,
|
|
571
|
+
) -> int:
|
|
572
|
+
"""Scan *paths* for the requested sensor *kinds* and print a summary.
|
|
573
|
+
|
|
574
|
+
Returns a process exit code: 0 = ok, 1 = no .p files found / unwritable CSV
|
|
575
|
+
target / every file failed to parse.
|
|
576
|
+
"""
|
|
577
|
+
out = stream if stream is not None else sys.stdout
|
|
578
|
+
|
|
579
|
+
# Fail fast on an unwritable CSV target BEFORE the (potentially long) scan,
|
|
580
|
+
# rather than crashing with a raw traceback after all the work is done.
|
|
581
|
+
if csv_out is not None and (csv_out.is_dir() or not csv_out.parent.is_dir()):
|
|
582
|
+
print(f"Error: cannot write CSV to {csv_out}", file=sys.stderr)
|
|
583
|
+
return 1
|
|
584
|
+
|
|
585
|
+
files = iter_pfiles(paths)
|
|
586
|
+
if not files:
|
|
587
|
+
print("No .p files found.", file=sys.stderr)
|
|
588
|
+
return 1
|
|
589
|
+
|
|
590
|
+
uses, errors, n_no_sensor = collect_uses(files, kinds)
|
|
591
|
+
inventory = build_inventory(uses)
|
|
592
|
+
|
|
593
|
+
n_parsed = len(files) - len(errors)
|
|
594
|
+
kind_labels = ", ".join(SENSOR_KINDS[k].label.lower() + "s" for k in kinds)
|
|
595
|
+
print(
|
|
596
|
+
f"Sensor inventory ({kind_labels}) — {len(files)} .p file(s): "
|
|
597
|
+
f"{n_parsed} parsed, {len(errors)} error(s)",
|
|
598
|
+
file=out,
|
|
599
|
+
)
|
|
600
|
+
if n_no_sensor:
|
|
601
|
+
print(f" ({n_no_sensor} file(s) had no matching sensor channels)", file=out)
|
|
602
|
+
n_undated = len({u.path for u in uses if u.start_time is None})
|
|
603
|
+
if n_undated:
|
|
604
|
+
print(
|
|
605
|
+
f" ({n_undated} file(s) had no valid clock — e.g. MR/glider startup — "
|
|
606
|
+
"inventoried without a date)",
|
|
607
|
+
file=out,
|
|
608
|
+
)
|
|
609
|
+
print(file=out)
|
|
610
|
+
|
|
611
|
+
print_report(inventory, kinds, verbose=verbose, compact=compact, stream=out)
|
|
612
|
+
|
|
613
|
+
if errors:
|
|
614
|
+
print("Errors:", file=out)
|
|
615
|
+
for fp, msg in errors:
|
|
616
|
+
print(f" {fp}: {msg}", file=out)
|
|
617
|
+
print(file=out)
|
|
618
|
+
|
|
619
|
+
if csv_out is not None:
|
|
620
|
+
try: # backstop for a race/permission change since the pre-scan check
|
|
621
|
+
write_csv(uses, csv_out, kinds)
|
|
622
|
+
except OSError as exc:
|
|
623
|
+
print(f"Error: could not write CSV to {csv_out}: {exc}", file=sys.stderr)
|
|
624
|
+
return 1
|
|
625
|
+
print(f"Wrote {len(uses)} rows to {csv_out}", file=out)
|
|
626
|
+
|
|
627
|
+
# Non-zero only when the scan wholly failed (files present, all errored),
|
|
628
|
+
# so a script can distinguish "nothing worked" from "found no sensors".
|
|
629
|
+
return 1 if errors and not uses else 0
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
# ---------------------------------------------------------------------------
|
|
633
|
+
# CLI (python -m odas_tpw.rsi.sensor_inventory)
|
|
634
|
+
# ---------------------------------------------------------------------------
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def build_arg_parser(prog: str = "sensor_inventory") -> argparse.ArgumentParser:
|
|
638
|
+
"""argparse parser shared by ``__main__`` and the ``rsi-tpw sensors`` command."""
|
|
639
|
+
ap = argparse.ArgumentParser(
|
|
640
|
+
prog=prog,
|
|
641
|
+
description="Inventory microstructure sensors across a tree of Rockland .p files.",
|
|
642
|
+
)
|
|
643
|
+
ap.add_argument(
|
|
644
|
+
"paths",
|
|
645
|
+
nargs="+",
|
|
646
|
+
type=Path,
|
|
647
|
+
help="Directories (scanned recursively), .p files, or glob patterns",
|
|
648
|
+
)
|
|
649
|
+
ap.add_argument("--shear", action="store_true", help="Inventory shear probes")
|
|
650
|
+
ap.add_argument("--fp07", action="store_true", help="Inventory FP07 thermistors")
|
|
651
|
+
ap.add_argument(
|
|
652
|
+
"--all",
|
|
653
|
+
dest="want_all",
|
|
654
|
+
action="store_true",
|
|
655
|
+
help="Inventory every sensor kind (shear + fp07; the default if none is given)",
|
|
656
|
+
)
|
|
657
|
+
ap.add_argument(
|
|
658
|
+
"--csv",
|
|
659
|
+
type=Path,
|
|
660
|
+
metavar="PATH",
|
|
661
|
+
help="Write a per-(file,channel) CSV table (overwritten if it exists)",
|
|
662
|
+
)
|
|
663
|
+
ap.add_argument(
|
|
664
|
+
"-v",
|
|
665
|
+
"--verbose",
|
|
666
|
+
action="store_true",
|
|
667
|
+
help="List the individual files behind each changed parameter value",
|
|
668
|
+
)
|
|
669
|
+
ap.add_argument(
|
|
670
|
+
"--compact",
|
|
671
|
+
action="store_true",
|
|
672
|
+
help="One line per probe: SN, file count, calibration, and date range",
|
|
673
|
+
)
|
|
674
|
+
return ap
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def main(argv: list[str] | None = None) -> int:
|
|
678
|
+
args = build_arg_parser().parse_args(argv)
|
|
679
|
+
kinds = resolve_kinds(args.shear, args.fp07, args.want_all)
|
|
680
|
+
return run(args.paths, kinds, csv_out=args.csv, verbose=args.verbose, compact=args.compact)
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
if __name__ == "__main__":
|
|
684
|
+
raise SystemExit(main())
|