amphi-ncdata 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- amphi_ncdata-0.3.1/MANIFEST.in +1 -0
- amphi_ncdata-0.3.1/PKG-INFO +637 -0
- amphi_ncdata-0.3.1/README.md +619 -0
- amphi_ncdata-0.3.1/pyproject.toml +38 -0
- amphi_ncdata-0.3.1/setup.cfg +4 -0
- amphi_ncdata-0.3.1/setup.py +70 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/__init__.py +17 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/_sampler.c +42218 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/_sampler.pyx +653 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/_version.py +1 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/amphi.py +335 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/constants.py +62 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/data.py +2052 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/discovery.py +221 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/errors.py +14 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/multi.py +572 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/reader.py +116 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/tides.py +833 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata/utils.py +98 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata.egg-info/PKG-INFO +637 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata.egg-info/SOURCES.txt +33 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata.egg-info/dependency_links.txt +1 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata.egg-info/requires.txt +6 -0
- amphi_ncdata-0.3.1/src/amphi_ncdata.egg-info/top_level.txt +1 -0
- amphi_ncdata-0.3.1/tests/test_amphi.py +126 -0
- amphi_ncdata-0.3.1/tests/test_at_batch_out.py +199 -0
- amphi_ncdata-0.3.1/tests/test_discovery.py +81 -0
- amphi_ncdata-0.3.1/tests/test_kernel_parity.py +284 -0
- amphi_ncdata-0.3.1/tests/test_longitude_wrap.py +184 -0
- amphi_ncdata-0.3.1/tests/test_multi.py +312 -0
- amphi_ncdata-0.3.1/tests/test_ncdata.py +229 -0
- amphi_ncdata-0.3.1/tests/test_oow.py +98 -0
- amphi_ncdata-0.3.1/tests/test_resolution_api.py +76 -0
- amphi_ncdata-0.3.1/tests/test_tides.py +361 -0
- amphi_ncdata-0.3.1/tests/test_time_units.py +31 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
include src/amphi_ncdata/_sampler.pyx
|
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: amphi-ncdata
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Python client library for loading and sampling Amphitrite's NetCDF datasets.
|
|
5
|
+
Author: Amphitrite S.A.
|
|
6
|
+
License: Proprietary
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
9
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
Requires-Dist: netCDF4>=1.6.5
|
|
13
|
+
Requires-Dist: numpy>=1.24
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
16
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
17
|
+
Dynamic: requires-python
|
|
18
|
+
|
|
19
|
+
# amphi-ncdata
|
|
20
|
+
|
|
21
|
+
`amphi-ncdata` is a Python client library for loading and sampling Amphi NetCDF datasets.
|
|
22
|
+
|
|
23
|
+
It exposes these main entry points:
|
|
24
|
+
|
|
25
|
+
- `AmphiNcData`: the easiest high-level API for clients using Amphi-style dataset folders.
|
|
26
|
+
- `MultiAmphiNcData`: an `AmphiNcData`-style env spanning **several bounding boxes**, routing each query to the owning box in O(1).
|
|
27
|
+
- `NcData`: a lower-level API for loading arbitrary NetCDF files or basepaths.
|
|
28
|
+
|
|
29
|
+
This README is for clients using an already built wheel. Repository maintainers should use `README_DEV.md`.
|
|
30
|
+
|
|
31
|
+
All timestamps in the public API are Unix timestamps in seconds.
|
|
32
|
+
|
|
33
|
+
## Install From The Wheel
|
|
34
|
+
|
|
35
|
+
Ask for the wheel that matches:
|
|
36
|
+
|
|
37
|
+
- your operating system
|
|
38
|
+
- your Python version
|
|
39
|
+
- your machine architecture
|
|
40
|
+
|
|
41
|
+
Install it with:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python3 -m pip install /path/to/amphi_ncdata-0.1.0-<platform>.whl
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Quick import check:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
python3 -c "import amphi_ncdata; print(amphi_ncdata.__version__)"
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
If installation succeeds but import fails with a missing shared-library error, ask us for a wheel built for your environment or install the required NetCDF runtime libraries for your platform.
|
|
54
|
+
|
|
55
|
+
## Quick Start
|
|
56
|
+
|
|
57
|
+
The fastest way to use the library is with `AmphiNcData`.
|
|
58
|
+
|
|
59
|
+
Example script:
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from amphi_ncdata import AmphiNcData
|
|
63
|
+
|
|
64
|
+
env = AmphiNcData(
|
|
65
|
+
{
|
|
66
|
+
"currentsBasepaths": ["/data/data_api/FORECAST-CURRENT/"],
|
|
67
|
+
"windBasepaths": ["/data/data_api/FORECAST/wind/"],
|
|
68
|
+
"waveBasepaths": ["/data/data_api/FORECAST/wave/"],
|
|
69
|
+
"climWindBasepaths": ["/data/data_api/HINDCAST-3Y/"],
|
|
70
|
+
"climWaveBasepaths": ["/data/data_api/HINDCAST-3Y-WAVE/"],
|
|
71
|
+
},
|
|
72
|
+
start=1781478000,
|
|
73
|
+
end=1781650800,
|
|
74
|
+
south=35.0,
|
|
75
|
+
west=-73.0,
|
|
76
|
+
north=43.0,
|
|
77
|
+
east=-30.0,
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
lat, lon = 40.0, -50.0
|
|
81
|
+
timestamp = 1781524800
|
|
82
|
+
|
|
83
|
+
currents = env.currents()
|
|
84
|
+
wind = env.wind()
|
|
85
|
+
waves = env.waves()
|
|
86
|
+
|
|
87
|
+
currents.set_interpolation_method("linear")
|
|
88
|
+
wind.set_interpolation_method("linear")
|
|
89
|
+
waves.set_interpolation_method("linear")
|
|
90
|
+
|
|
91
|
+
u_current, v_current = currents.at(lat, lon, timestamp)
|
|
92
|
+
u_wind, v_wind = wind.at(lat, lon, timestamp)
|
|
93
|
+
wave_height, wave_period, wave_direction = waves.at(lat, lon, timestamp)
|
|
94
|
+
|
|
95
|
+
print("Currents:", u_current, v_current)
|
|
96
|
+
print("Wind:", u_wind, v_wind)
|
|
97
|
+
print("Waves:", wave_height, wave_period, wave_direction)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
A ready-to-copy version also lives in `examples/client_quickstart.py`.
|
|
101
|
+
|
|
102
|
+
## Main Concepts
|
|
103
|
+
|
|
104
|
+
- `AmphiNcData` is the convenience layer for Amphi folder layouts, masks, and tides.
|
|
105
|
+
- `NcData` is the generic layer when you want to choose the files or variables yourself.
|
|
106
|
+
- `load()` registers which variables you want available.
|
|
107
|
+
- `set_interpolation_method()` sets the dataset interpolation mode once.
|
|
108
|
+
- `set_out_of_window_behaviour(behaviour, value, dim=...)` sets what a query outside the loaded window returns, **per dimension** (`dim="time"` / `"space"` / `"both"`): `"nearest"`, `"nan"`, `"value"`, or `"fallback"` (another `NcData`). Defaults: time→`nearest`, space→`nan`.
|
|
109
|
+
- `sample()` returns one variable.
|
|
110
|
+
- `at()` returns all loaded variables for the dataset.
|
|
111
|
+
- `at()` / `sample()` are fast for sequential, point-by-point access (e.g. routing); `sample_batch()` / `at_batch()` are best when you already have many independent points (roughly a few dozen or more) to sample at once. Pass `out=` (a buffer from `empty_batch()`) to sample in place with no per-call allocation.
|
|
112
|
+
- `env.wind()` / `env.waves()` **auto-fall-back to climatology past the forecast horizon** (when climatology basepaths are set); `env.climWind()` / `env.climWave()` are the underlying climatology datasets.
|
|
113
|
+
|
|
114
|
+
## Using `AmphiNcData`
|
|
115
|
+
|
|
116
|
+
`AmphiNcData` is the recommended API for most client code.
|
|
117
|
+
|
|
118
|
+
### Using an `input.json`-style config
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
from amphi_ncdata import AmphiNcData
|
|
122
|
+
import json
|
|
123
|
+
|
|
124
|
+
with open("input.json", "r", encoding="utf-8") as f:
|
|
125
|
+
cfg = json.load(f)
|
|
126
|
+
|
|
127
|
+
env = AmphiNcData(
|
|
128
|
+
cfg,
|
|
129
|
+
start=1781478000,
|
|
130
|
+
end=1781650800,
|
|
131
|
+
south=35.0,
|
|
132
|
+
west=-73.0,
|
|
133
|
+
north=43.0,
|
|
134
|
+
east=-30.0,
|
|
135
|
+
)
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### Using a flat config
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
env = AmphiNcData(
|
|
142
|
+
{
|
|
143
|
+
"currentsBasepaths": ["/data/data_api/FORECAST-CURRENT/"],
|
|
144
|
+
"tidesBasepaths": ["/data/data_api/FORECAST-TIDE/"],
|
|
145
|
+
"windBasepaths": ["/data/data_api/FORECAST/wind/"],
|
|
146
|
+
"waveBasepaths": ["/data/data_api/FORECAST/wave/"],
|
|
147
|
+
"climWindBasepaths": ["/data/data_api/HINDCAST-3Y/"],
|
|
148
|
+
"climWaveBasepaths": ["/data/data_api/HINDCAST-3Y-WAVE/"],
|
|
149
|
+
"bathymetryPath": "/data/data_save/ETOPO/bathy.nc",
|
|
150
|
+
"tidesMaskPath": "/data/data_api/FORECAST-TIDE/masks_tide.nc",
|
|
151
|
+
"nogoPath": "/data/data_user/nogo.nc",
|
|
152
|
+
"coastalZonePath": "/data/data_save/coastal_zone_mask.nc",
|
|
153
|
+
},
|
|
154
|
+
start=1781478000,
|
|
155
|
+
end=1781650800,
|
|
156
|
+
south=35.0,
|
|
157
|
+
west=-73.0,
|
|
158
|
+
north=43.0,
|
|
159
|
+
east=-30.0,
|
|
160
|
+
)
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### High-level helpers
|
|
164
|
+
|
|
165
|
+
```python
|
|
166
|
+
currents = env.currents()
|
|
167
|
+
wind = env.wind()
|
|
168
|
+
waves = env.waves()
|
|
169
|
+
tidal = env.tidal_currents()
|
|
170
|
+
bathy = env.bathymetry()
|
|
171
|
+
nogo = env.nogo()
|
|
172
|
+
coastal = env.coastal_zone()
|
|
173
|
+
tides_mask = env.tides_mask()
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Return shapes:
|
|
177
|
+
|
|
178
|
+
- `env.currents().at(...)` -> `(ucos, vcos)`
|
|
179
|
+
- `env.tidal_currents().at(...)` -> `(utotal, vtotal)`
|
|
180
|
+
- `env.wind().at(...)` -> `(u10, v10)`
|
|
181
|
+
- `env.waves().at(...)` -> `(swh, mwp, mwd)`
|
|
182
|
+
- `env.bathymetry().at(...)` -> `bathymetry`
|
|
183
|
+
- `env.nogo().at(...)` -> `mask`
|
|
184
|
+
- `env.coastal_zone().at(...)` -> `mask`
|
|
185
|
+
- `env.tides_mask().at(...)` -> `mask_TIDE`
|
|
186
|
+
|
|
187
|
+
### Tidal currents: the fused `tidal_currents()`
|
|
188
|
+
|
|
189
|
+
Tides only matter near coasts (the global tide mask is ~9% active), so
|
|
190
|
+
`env.tidal_currents()` does **not** load the whole bounding box. With a
|
|
191
|
+
`tidesMaskPath` configured it loads **eager 1-degree masked tiles**: every
|
|
192
|
+
1-degree tile that the mask marks active is pulled into RAM up front, in the raw
|
|
193
|
+
NetCDF dtype, so the routing hot path never touches NetCDF I/O. Open-ocean and
|
|
194
|
+
land tiles allocate nothing.
|
|
195
|
+
|
|
196
|
+
`tidal_currents()` is **fused**: `at()` returns the hourly tidal current where
|
|
197
|
+
the mask is active (coastal) and the open-ocean current where it is not — so a
|
|
198
|
+
single call gives the right water velocity everywhere, no manual branch:
|
|
199
|
+
|
|
200
|
+
```python
|
|
201
|
+
from datetime import datetime, timezone
|
|
202
|
+
from amphi_ncdata import AmphiNcData
|
|
203
|
+
|
|
204
|
+
start = int(datetime(2026, 6, 23, tzinfo=timezone.utc).timestamp())
|
|
205
|
+
env = AmphiNcData(
|
|
206
|
+
{
|
|
207
|
+
"tidesBasepaths": ["/data/data_api/FORECAST-TIDE/"],
|
|
208
|
+
# The mask is what turns tides into eager 1-degree masked tiles:
|
|
209
|
+
"tidesMaskPath": "/data/data_api/FORECAST-TIDE/masks_tide.nc",
|
|
210
|
+
# "tidesTileSizeDegrees": 1.0, # optional; 1.0 is the default
|
|
211
|
+
},
|
|
212
|
+
start=start, end=start + 24 * 3600,
|
|
213
|
+
south=48.0, west=-6.0, north=52.5, east=4.5, # English Channel
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
tides = env.tidal_currents() # eager 1° masked tiles + a daily open-ocean step
|
|
217
|
+
u, v = tides.at(lat, lon, timestamp) # tide near the coast, current offshore — one call
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
**How the open ocean is served.** Where the tide product is not tidal (mask 0),
|
|
221
|
+
its values are constant over the day and equal the surface currents (verified:
|
|
222
|
+
`p99 = 0`, mean diff `3.7e-5` m/s over thousands of open-ocean points). So
|
|
223
|
+
`tidal_currents()` loads **one** daily tide step over the full bbox as the
|
|
224
|
+
open-ocean field — self-contained (no separate currents product needed), same
|
|
225
|
+
variable names, and roughly half the RAM of loading the multi-step currents
|
|
226
|
+
product. Coastal cells are overridden by the hourly tide tiles.
|
|
227
|
+
|
|
228
|
+
You normally never call the gate yourself, but it is exposed for inspection and
|
|
229
|
+
for `fused=False`:
|
|
230
|
+
|
|
231
|
+
```python
|
|
232
|
+
tides.mask(lat, lon) # compiled gate (~0.2 us): is this point tidal?
|
|
233
|
+
tides.mask_batch(lats, lons) # vectorised
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
`mask()` is the nearest high-res `mask_TIDE` cell (active = non-zero); it is
|
|
237
|
+
spatial, so `timestamp` is accepted for symmetry but ignored. A masked-in cell
|
|
238
|
+
can still be land / fill (NaN) — `mask()` selects the data *source*, not its
|
|
239
|
+
presence.
|
|
240
|
+
|
|
241
|
+
Knobs: `tidesTileSizeDegrees` (default `1.0`) sets the tile size;
|
|
242
|
+
`TidalCurrents(..., eager=False)` keeps tiles lazy with an LRU cache (smaller
|
|
243
|
+
resident footprint, per-call I/O); `TidalCurrents(..., fused=False)` makes `at()`
|
|
244
|
+
tide-only again (NaN in the open ocean) if you want to gate manually.
|
|
245
|
+
|
|
246
|
+
#### What if you omit the tide paths?
|
|
247
|
+
|
|
248
|
+
| configuration | `env.tidal_currents()` behaviour |
|
|
249
|
+
|---|---|
|
|
250
|
+
| `tidesBasepaths` **and** `tidesMaskPath` set | **Eager 1° masked tiles + fused open ocean** (the recommended setup). `at()` is tide-where-coastal, current-where-not; `mask()` reflects the high-res tide mask. |
|
|
251
|
+
| `tidesBasepaths` set, **`tidesMaskPath` omitted** | Falls back to the **full-bbox tidal view** (lazy, no tiling, no fusion). `at()` samples tides everywhere in the box, and `mask()` returns `True` for any in-box point (no mask to gate on). Memory is the full bbox × all tide steps — fine for a small box, large for a wide one. |
|
|
252
|
+
| **`tidesBasepaths` omitted** | `env.tidal_currents()` raises `AmphiDiscoveryError` ("No valid NetCDF files were found …"). Only call it when tides are configured; `currents`, `wind`, and `waves` are independent and still work — `env.currents()` stays the currents-only, memory-light path. |
|
|
253
|
+
|
|
254
|
+
### Out-of-window behaviour (per dimension, with climatology fallback)
|
|
255
|
+
|
|
256
|
+
A query can fall outside the loaded window in **time** (past/before the forecast)
|
|
257
|
+
or in **space** (outside the bounding box). Each is handled independently:
|
|
258
|
+
|
|
259
|
+
```python
|
|
260
|
+
wind.set_out_of_window_behaviour(behaviour, value=0.0, dim="both")
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
- `dim`: `"time"`, `"space"`, or `"both"` (default).
|
|
264
|
+
- `behaviour`:
|
|
265
|
+
- `"nearest"` — clamp to the nearest timestep / edge cell.
|
|
266
|
+
- `"nan"` — return NaN.
|
|
267
|
+
- `"value"` — return the constant `value` (a float).
|
|
268
|
+
- `"fallback"` (alias `"data"`) — sample `value` instead, which must be another
|
|
269
|
+
`NcData` that covers the query (e.g. a climatology). The fallback dataset
|
|
270
|
+
carries its own out-of-window behaviour.
|
|
271
|
+
- **Defaults: time → `nearest`, space → `nan`.**
|
|
272
|
+
|
|
273
|
+
Precedence when out of window: a `fallback` dimension delegates the whole sample
|
|
274
|
+
to its dataset (time is checked before space); otherwise a `nan`/`value` on space
|
|
275
|
+
applies before time; `nearest` simply clamps that dimension. This is independent
|
|
276
|
+
of NaN coming from the data itself (land / fill), which is always NaN.
|
|
277
|
+
(`set_out_of_window_behaviour("nan")` / `("value", -999)` still set both
|
|
278
|
+
dimensions, for backward compatibility.)
|
|
279
|
+
|
|
280
|
+
#### Climatology fallback (automatic for wind / waves)
|
|
281
|
+
|
|
282
|
+
`env.wind()` and `env.waves()` **automatically fall back to climatology past the
|
|
283
|
+
forecast horizon** when climatology basepaths are configured — equivalent to:
|
|
284
|
+
|
|
285
|
+
```python
|
|
286
|
+
wind.set_out_of_window_behaviour("fallback", env.climWind(), dim="time")
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
So a long voyage just keeps sampling, with no manual gap handling:
|
|
290
|
+
|
|
291
|
+
```python
|
|
292
|
+
wind = env.wind() # forecast + climatology time-fallback
|
|
293
|
+
u, v = wind.at(40.0, -50.0, ts) # forecast in-window, climatology past the horizon
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
The climatology (`env.climWind()` -> `(u10, v10)`, `env.climWave()` ->
|
|
297
|
+
`(swh, mwp, mwd)`) is still its own dataset and is configured as the safety net:
|
|
298
|
+
`nearest` in both time and space and `linear` interpolation, so it always returns
|
|
299
|
+
a value. Pass no climatology basepaths, or call
|
|
300
|
+
`wind.set_out_of_window_behaviour("nan", dim="time")`, to opt out (e.g. to detect
|
|
301
|
+
the gap yourself).
|
|
302
|
+
|
|
303
|
+
### `load_defaults()`
|
|
304
|
+
|
|
305
|
+
```python
|
|
306
|
+
env.load_defaults(
|
|
307
|
+
include_tides=True,
|
|
308
|
+
include_bathymetry=True,
|
|
309
|
+
include_nogo=True,
|
|
310
|
+
include_coastal_zone=True,
|
|
311
|
+
)
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
This preloads:
|
|
315
|
+
|
|
316
|
+
- currents
|
|
317
|
+
- wind
|
|
318
|
+
- waves
|
|
319
|
+
- optionally tidal currents and tides mask (tides eager-load their active 1° tiles up
|
|
320
|
+
front — fast for a route corridor, heavier for a continental box; see the gate section)
|
|
321
|
+
- optionally bathymetry
|
|
322
|
+
- optionally no-go mask
|
|
323
|
+
- optionally coastal zone mask
|
|
324
|
+
|
|
325
|
+
### Overriding default variable names
|
|
326
|
+
|
|
327
|
+
If your files use different names:
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
custom_currents = env.currents(u_var="u_current", v_var="v_current")
|
|
331
|
+
custom_wind = env.wind(u_var="u_wind", v_var="v_wind")
|
|
332
|
+
custom_waves = env.waves(height_var="hs", period_var="tp", direction_var="dir")
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
## Multiple bounding boxes: `MultiAmphiNcData`
|
|
336
|
+
|
|
337
|
+
Use `MultiAmphiNcData` when one env should cover **several** bounding boxes /
|
|
338
|
+
time windows at once — e.g. tiling a large region, or layering a smaller box on
|
|
339
|
+
top of a bigger one. All boxes share the same `config`; each carries its own
|
|
340
|
+
`start, end, south, west, north, east`. Every accessor mirrors `AmphiNcData`
|
|
341
|
+
(`currents()`, `wind()`, `waves()`, `tidal_currents()`, `bathymetry()`, …) but
|
|
342
|
+
returns a routed dataset whose `at`/`at_batch`/`sample`/`sample_batch` dispatch
|
|
343
|
+
each point to the box that owns it.
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
from amphi_ncdata import MultiAmphiNcData # or AmphiNcData.from_bounding_boxes(...)
|
|
347
|
+
|
|
348
|
+
env = MultiAmphiNcData(
|
|
349
|
+
{"currentsBasepaths": ["/data/data_api/FORECAST-CURRENT/"]}, # shared by all boxes
|
|
350
|
+
boxes=[
|
|
351
|
+
dict(start=1781478000, end=1781650800, south=35.0, west=-73.0, north=43.0, east=-30.0),
|
|
352
|
+
dict(start=1781478000, end=1781650800, south=43.0, west=-30.0, north=60.0, east=10.0),
|
|
353
|
+
],
|
|
354
|
+
# out_of_box_value=float("nan"), # what a point in no box returns (default NaN)
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
currents = env.currents()
|
|
358
|
+
currents.set_interpolation_method("linear")
|
|
359
|
+
|
|
360
|
+
u, v = currents.at(40.0, -50.0, 1781524800) # routed to the owning box in C, O(1)
|
|
361
|
+
uv = currents.at_batch(lats, lons, timestamps) # routed + sampled in C, one call
|
|
362
|
+
|
|
363
|
+
buf = currents.empty_batch(len(lats)) # reusable (n, 2) float32 buffer
|
|
364
|
+
currents.at_batch(lats, lons, timestamps, out=buf) # fill in place, returns buf
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
How dispatch works:
|
|
368
|
+
|
|
369
|
+
- A dense `(lat, lon, hour)` lookup grid (1° × 1° × 1 h) stores, per cell, the
|
|
370
|
+
index of the winning box. A query computes three integer indices and reads one
|
|
371
|
+
cell — the heavy bilinear + time interpolation still runs in each box's
|
|
372
|
+
compiled sampler.
|
|
373
|
+
- **Both `.at()` and `.at_batch()` run compiled**, matching a plain dataset:
|
|
374
|
+
`at`/`sample` route each point in C, and `at_batch`/`sample_batch` route **and**
|
|
375
|
+
sample every point in C (no Python bucketing). They accept the same `out=`
|
|
376
|
+
buffer and `empty_batch()` helper (see *Reusing an output buffer with `out=`*
|
|
377
|
+
under **Using `NcData`** below), so batching is fast from small sizes up.
|
|
378
|
+
- **Overlap → first listed box wins.** When boxes overlap in both space and
|
|
379
|
+
time, the earliest box in `boxes` that covers the point is chosen (so you can
|
|
380
|
+
layer a high-res box on top of a coarse one by listing it first).
|
|
381
|
+
- A point in **no** box returns `out_of_box_value` (default `NaN`).
|
|
382
|
+
- If every box shares the same `[start, end]`, the time axis collapses and the
|
|
383
|
+
grid stays tiny (~130 KB); otherwise it grows with the total time span
|
|
384
|
+
(~22 MB for a 7-day window).
|
|
385
|
+
- **Boundary note:** at 1° resolution, a box with non-integer degree edges
|
|
386
|
+
shares its edge cell with neighbours; a point in that cell but just outside its
|
|
387
|
+
assigned box falls to that box's own out-of-window behaviour. This is exact for
|
|
388
|
+
integer-degree boxes (as in the examples above).
|
|
389
|
+
|
|
390
|
+
`env.tidal_currents()`, `env.wind()`, etc. work the same way and keep their
|
|
391
|
+
per-box behaviour (tide tiling, climatology fallback). `MultiAmphiNcData` is a
|
|
392
|
+
context manager and forwards `close()` to every box's env.
|
|
393
|
+
|
|
394
|
+
## Using `NcData`
|
|
395
|
+
|
|
396
|
+
Use `NcData` when you want more manual control.
|
|
397
|
+
|
|
398
|
+
### Load from basepaths
|
|
399
|
+
|
|
400
|
+
Use this when the library should discover the right NetCDF files for you.
|
|
401
|
+
|
|
402
|
+
```python
|
|
403
|
+
from amphi_ncdata import NcData
|
|
404
|
+
|
|
405
|
+
currents = NcData.from_basepaths(
|
|
406
|
+
basepaths=[
|
|
407
|
+
"/data/data_api/FORECAST-CURRENT/",
|
|
408
|
+
"/mnt/store/data_api/HIRES_v3/FORECAST/",
|
|
409
|
+
],
|
|
410
|
+
start=1781478000,
|
|
411
|
+
end=1781650800,
|
|
412
|
+
south=35.0,
|
|
413
|
+
west=-73.0,
|
|
414
|
+
north=43.0,
|
|
415
|
+
east=-30.0,
|
|
416
|
+
allow_last_available=True,
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
currents.load("ucos")
|
|
420
|
+
currents.load("vcos")
|
|
421
|
+
currents.set_interpolation_method("linear")
|
|
422
|
+
|
|
423
|
+
print(currents.at(40.0, -50.0, 1781524800))
|
|
424
|
+
```
|
|
425
|
+
|
|
426
|
+
### Load from explicit filepaths
|
|
427
|
+
|
|
428
|
+
Use this when you already know the exact files you want.
|
|
429
|
+
|
|
430
|
+
```python
|
|
431
|
+
from amphi_ncdata import NcData
|
|
432
|
+
|
|
433
|
+
waves = NcData.from_filepaths(
|
|
434
|
+
filepaths=[
|
|
435
|
+
"/tmp/20260614.nc",
|
|
436
|
+
"/tmp/20260615.nc",
|
|
437
|
+
],
|
|
438
|
+
start=1781391600,
|
|
439
|
+
end=1781478000,
|
|
440
|
+
south=35.0,
|
|
441
|
+
west=-73.0,
|
|
442
|
+
north=43.0,
|
|
443
|
+
east=-30.0,
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
waves.load("swh")
|
|
447
|
+
waves.load("mwp")
|
|
448
|
+
waves.load("mwd")
|
|
449
|
+
waves.set_interpolation_method("linear")
|
|
450
|
+
|
|
451
|
+
print(waves.at(40.0, -50.0, 1781434800))
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
### Sampling
|
|
455
|
+
|
|
456
|
+
```python
|
|
457
|
+
currents.set_interpolation_method("linear")
|
|
458
|
+
value = currents.sample("ucos", 40.0, -50.0, 1781524800)
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
### Batch sampling
|
|
462
|
+
|
|
463
|
+
For large point sets, batch APIs avoid Python per-sample overhead:
|
|
464
|
+
|
|
465
|
+
```python
|
|
466
|
+
lats = [39.5, 40.0, 40.5]
|
|
467
|
+
lons = [-51.0, -50.0, -49.0]
|
|
468
|
+
timestamps = [1781521200, 1781524800, 1781528400]
|
|
469
|
+
|
|
470
|
+
u_values = currents.sample_batch("ucos", lats, lons, timestamps)
|
|
471
|
+
uv_values = currents.at_batch(lats, lons, timestamps)
|
|
472
|
+
```
|
|
473
|
+
|
|
474
|
+
Notes:
|
|
475
|
+
|
|
476
|
+
- `sample_batch()` returns one NumPy array
|
|
477
|
+
- `at_batch()` returns one NumPy array shaped like `(sample_count, variable_count)` when multiple variables are loaded
|
|
478
|
+
- use batch APIs when you already hold many independent points; for sequential, point-by-point access (e.g. routing), `at()` / `sample()` are the right call
|
|
479
|
+
|
|
480
|
+
#### Reusing an output buffer with `out=`
|
|
481
|
+
|
|
482
|
+
`at_batch()` and `sample_batch()` accept an optional `out=` buffer: results are
|
|
483
|
+
written into it in place and the same array is returned — no per-call allocation.
|
|
484
|
+
Build the correctly-shaped, correctly-typed buffer once with `empty_batch()` and
|
|
485
|
+
reuse it across calls (e.g. re-sampling the same points on every step of a loop):
|
|
486
|
+
|
|
487
|
+
```python
|
|
488
|
+
import numpy as np
|
|
489
|
+
|
|
490
|
+
lats = [39.5, 40.0, 40.5]
|
|
491
|
+
lons = [-51.0, -50.0, -49.0]
|
|
492
|
+
timestamps = [1781521200, 1781524800, 1781528400]
|
|
493
|
+
|
|
494
|
+
buf = currents.empty_batch(len(lats)) # (n, 2) float32 for a 2-variable dataset
|
|
495
|
+
currents.at_batch(lats, lons, timestamps, out=buf) # returns buf, filled in place
|
|
496
|
+
|
|
497
|
+
sbuf = np.empty(len(lats), dtype=np.float32) # (n,) float32 for a single variable
|
|
498
|
+
currents.sample_batch("ucos", lats, lons, timestamps, out=sbuf)
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
`out` must be a C-contiguous `float32` array of shape `(n, variable_count)` for
|
|
502
|
+
`at_batch()` — or `(n,)` when the dataset has a single loaded variable, and always
|
|
503
|
+
`(n,)` for `sample_batch()`. `empty_batch(n)` returns exactly the shape `at_batch()`
|
|
504
|
+
needs. Passing `out=` implies a NumPy-array return (the `as_numpy` flag is ignored).
|
|
505
|
+
|
|
506
|
+
**When to batch:** the batch APIs run the same compiled sampler as `at()` /
|
|
507
|
+
`sample()`, but in one call. They carry a small fixed per-call cost, so for just a
|
|
508
|
+
few points a plain `at()` loop is still faster; batching pays off from roughly a
|
|
509
|
+
few dozen points upward and scales to millions.
|
|
510
|
+
|
|
511
|
+
Supported interpolation values for `set_interpolation_method()`:
|
|
512
|
+
|
|
513
|
+
- `"nearest"`
|
|
514
|
+
- `"linear"`
|
|
515
|
+
|
|
516
|
+
### Lazy loading and cache
|
|
517
|
+
|
|
518
|
+
```python
|
|
519
|
+
cache = {"size": 128 * 1024 * 1024, "nelems": 20000, "preemption": 0.8}
|
|
520
|
+
|
|
521
|
+
tidal = NcData.from_basepaths(
|
|
522
|
+
basepaths=["/data/data_api/FORECAST-TIDE/"],
|
|
523
|
+
start=1781478000,
|
|
524
|
+
end=1781650800,
|
|
525
|
+
south=35.0,
|
|
526
|
+
west=-73.0,
|
|
527
|
+
north=43.0,
|
|
528
|
+
east=-30.0,
|
|
529
|
+
allow_last_available=True,
|
|
530
|
+
)
|
|
531
|
+
|
|
532
|
+
tidal.load("utotal", lazy=True, cache=cache)
|
|
533
|
+
tidal.load("vtotal", lazy=True, cache=cache)
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
Loaded variables keep their **raw on-disk dtype** in RAM (e.g. int16 for currents/tides/waves);
|
|
537
|
+
`scale_factor`, `add_offset`, and `_FillValue` are applied inside the Cython sampler at lookup
|
|
538
|
+
time. This halves resident memory for packed products versus decoding to float32 up front, with
|
|
539
|
+
no loss of precision. (The `decode=` argument to `load()` is retained for API compatibility but
|
|
540
|
+
is now a no-op — storage is always the raw dtype.)
|
|
541
|
+
|
|
542
|
+
### Inspect loaded state
|
|
543
|
+
|
|
544
|
+
```python
|
|
545
|
+
print(currents.resolved_files)
|
|
546
|
+
print(currents.loaded_variables)
|
|
547
|
+
print(currents.time_range)
|
|
548
|
+
print(currents.bounds)
|
|
549
|
+
print(currents.contains_timestamp(1781524800))
|
|
550
|
+
```
|
|
551
|
+
|
|
552
|
+
### Context manager
|
|
553
|
+
|
|
554
|
+
```python
|
|
555
|
+
from amphi_ncdata import NcData
|
|
556
|
+
|
|
557
|
+
with NcData.from_basepaths(
|
|
558
|
+
basepaths=["/data/data_api/FORECAST-CURRENT/"],
|
|
559
|
+
start=1781478000,
|
|
560
|
+
end=1781650800,
|
|
561
|
+
south=35.0,
|
|
562
|
+
west=-73.0,
|
|
563
|
+
north=43.0,
|
|
564
|
+
east=-30.0,
|
|
565
|
+
) as data:
|
|
566
|
+
data.load("ucos")
|
|
567
|
+
data.set_interpolation_method("linear")
|
|
568
|
+
print(data.sample("ucos", 40.0, -50.0, 1781524800))
|
|
569
|
+
```
|
|
570
|
+
|
|
571
|
+
### Out-of-window behaviour
|
|
572
|
+
|
|
573
|
+
A query is "out of window" when its timestamp is outside the loaded time range,
|
|
574
|
+
or its `lat`/`lon` is outside the loaded bounding box. Choose what `at()` /
|
|
575
|
+
`sample()` (and the batch variants) return in that case:
|
|
576
|
+
|
|
577
|
+
```python
|
|
578
|
+
wind.set_out_of_window_behaviour("nearest") # default: clamp to the nearest
|
|
579
|
+
# available timestep / edge cell
|
|
580
|
+
wind.set_out_of_window_behaviour("nan") # return NaN (handy to detect gaps)
|
|
581
|
+
wind.set_out_of_window_behaviour("value", 0.0) # return a fixed fill value
|
|
582
|
+
```
|
|
583
|
+
|
|
584
|
+
This is independent of NaN coming from the data itself (land / fill gaps), which
|
|
585
|
+
is always returned as NaN regardless of the mode. The setting applies equally to
|
|
586
|
+
out-of-range time and out-of-box space.
|
|
587
|
+
|
|
588
|
+
## Timestamps And Coordinates
|
|
589
|
+
|
|
590
|
+
Accepted timestamp format:
|
|
591
|
+
|
|
592
|
+
- Unix timestamp integer in seconds
|
|
593
|
+
|
|
594
|
+
Sampling coordinate format:
|
|
595
|
+
|
|
596
|
+
- `lat` and `lon` are passed as separate numeric arguments
|
|
597
|
+
|
|
598
|
+
## Errors
|
|
599
|
+
|
|
600
|
+
The package raises:
|
|
601
|
+
|
|
602
|
+
- `AmphiDiscoveryError`: file discovery, open, or setup problems
|
|
603
|
+
- `AmphiVariableError`: variable loading or sampling problems
|
|
604
|
+
- `AmphiNativeError`: lower-level native errors
|
|
605
|
+
|
|
606
|
+
Example:
|
|
607
|
+
|
|
608
|
+
```python
|
|
609
|
+
from amphi_ncdata import NcData, AmphiDiscoveryError
|
|
610
|
+
|
|
611
|
+
try:
|
|
612
|
+
NcData.from_basepaths(
|
|
613
|
+
["/tmp/does-not-exist"],
|
|
614
|
+
start=0,
|
|
615
|
+
end=1,
|
|
616
|
+
south=0,
|
|
617
|
+
west=0,
|
|
618
|
+
north=1,
|
|
619
|
+
east=1,
|
|
620
|
+
)
|
|
621
|
+
except AmphiDiscoveryError as exc:
|
|
622
|
+
print(exc)
|
|
623
|
+
```
|
|
624
|
+
|
|
625
|
+
## Performance
|
|
626
|
+
|
|
627
|
+
- `at()` and `sample()` are backed by a compiled sampler, so single-point lookups are fast. This is the intended path for routing algorithms (Dijkstra / A* / isochrone / DP) that sample one point at a time, where each step depends on the previous one.
|
|
628
|
+
- Eager-loaded variables (the default) stay in memory, so sampling never touches disk. Use `lazy=True` only for variables too large to keep resident; lazy sampling reads from disk and is much slower per point.
|
|
629
|
+
- `at_batch()` / `sample_batch()` run the same compiled sampler over many points in one call. They carry a small fixed per-call cost, so use them once you have roughly a few dozen points or more (below that, an `at()` loop wins). Pass `out=` (from `empty_batch()`) to avoid per-call allocation when sampling repeatedly.
|
|
630
|
+
|
|
631
|
+
## Practical Notes
|
|
632
|
+
|
|
633
|
+
- A wheel is specific to an OS, Python version, and architecture.
|
|
634
|
+
- If you change Python versions, you may need a new wheel.
|
|
635
|
+
- `AmphiNcData(...)` accepts missing path keys, but a dataset helper such as `env.wind()` or `env.waves()` still raises immediately if that dataset cannot be resolved.
|
|
636
|
+
- `AmphiNcData` is the best starting point unless you specifically need file-level control.
|
|
637
|
+
- This package focuses on point sampling, not bulk NumPy-style array workflows.
|