forcingkit 0.16.0__tar.gz → 0.17.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.16.0 → forcingkit-0.17.0}/PKG-INFO +2 -2
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/conf.py +1 -1
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/fetchers.rst +8 -2
- {forcingkit-0.16.0 → forcingkit-0.17.0}/pyproject.toml +2 -2
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/ndbc.py +17 -5
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/noaa.py +37 -9
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_ndbc.py +26 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_noaa_datum.py +39 -2
- {forcingkit-0.16.0 → forcingkit-0.17.0}/uv.lock +5 -5
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.agents/forcingkit.md +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.env.template +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.gitignore +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.markdownlint.yaml +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/.python-version +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/AGENTS.md +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/CONTRIBUTING.md +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/Dockerfile +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/LICENSE +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/README.md +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/LICENSE +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/README.md +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/pyproject.toml +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/tests/conftest.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/tests/test_client.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/tests/test_schema.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/client/uv.lock +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docker-compose.yml +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/Makefile +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/make.bat +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/_static/custom.css +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/client.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/hycom.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/index.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/necofs.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/rivers.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/sources.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/docs/source/waves.rst +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/main.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/__init__.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/routers/viewer.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/service/run_server.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/render.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/static/app.js +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/static/favicon.ico +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/static/favicon.svg +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/static/index.html +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/static/logo.svg +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/static/preview3d.js +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/static/styles.css +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/__init__.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_main.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.16.0 → forcingkit-0.17.0}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.17.0
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -225,7 +225,7 @@ Requires-Dist: fastapi>=0.135.1
|
|
|
225
225
|
Requires-Dist: h5netcdf>=1.8.1
|
|
226
226
|
Requires-Dist: h5py>=3.16.0
|
|
227
227
|
Requires-Dist: httpx>=0.28.1
|
|
228
|
-
Requires-Dist: insitukit<0.
|
|
228
|
+
Requires-Dist: insitukit<0.6,>=0.5
|
|
229
229
|
Requires-Dist: jinja2>=3.0.0
|
|
230
230
|
Requires-Dist: matplotlib>=3.10.9
|
|
231
231
|
Requires-Dist: pandas>=2.2
|
|
@@ -149,13 +149,19 @@ These serve validation rather than forcing:
|
|
|
149
149
|
fallback to MSL where a station has no NAVD88 datum. Requests go through
|
|
150
150
|
`insitukit <https://github.com/lhzn-io/insitukit>`__, which keeps them at least 1 s apart,
|
|
151
151
|
splits windows longer than the API's 31-day limit, and retries server errors and throttling;
|
|
152
|
-
``end_time`` is inclusive, as in the API.
|
|
152
|
+
``end_time`` is inclusive, as in the API. Each reading carries CO-OPS's quality level (``q``:
|
|
153
|
+
``p`` preliminary, ``v`` verified) and ``metadata.quality`` sums up the window. Since 0.17.0 a
|
|
154
|
+
verified window is cached for good and one still holding preliminary readings for a day, because
|
|
155
|
+
CO-OPS replaces preliminary readings once it has reviewed them. Current predictions
|
|
156
|
+
(``/api/v1/currents``) and the
|
|
153
157
|
current-station list (``/api/v1/currents/stations``) use the same client.
|
|
154
158
|
- **NDBC** buoy meteorology and ADCP profiles (``/api/v1/ndbc``), from the `NOAA National Data
|
|
155
159
|
Buoy Center <https://www.ndbc.noaa.gov/>`__, through insitukit's NDBC client: each month from
|
|
156
160
|
the monthly, yearly or real-time file, each file downloaded once, and each column's own
|
|
157
161
|
missing-value sentinel read as missing (since 0.16.0; earlier releases also dropped real
|
|
158
|
-
readings equal to another column's sentinel, such as a 999 hPa low or a 99-degree wind).
|
|
162
|
+
readings equal to another column's sentinel, such as a 999 hPa low or a 99-degree wind). A
|
|
163
|
+
window is cached only when none of it came from the real-time files, whose readings have not
|
|
164
|
+
yet been through NDBC's quality control (since 0.17.0).
|
|
159
165
|
- **UConn ERDDAP** water-column profiles (``/api/v1/telemetry/station`` and ``/bbox``), from the
|
|
160
166
|
`University of Connecticut Department of Marine Sciences <https://marinesciences.uconn.edu/>`__
|
|
161
167
|
`ERDDAP server <http://merlin.dms.uconn.edu:8080/erddap/index.html>`__.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.17.0"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -27,7 +27,7 @@ dependencies = [
|
|
|
27
27
|
"fastapi>=0.135.1",
|
|
28
28
|
"httpx>=0.28.1",
|
|
29
29
|
# NOAA CO-OPS (fetchers/noaa.py), the UConn ERDDAP profiles (fetchers/erddap.py) and NDBC (fetchers/ndbc.py).
|
|
30
|
-
"insitukit>=0.
|
|
30
|
+
"insitukit>=0.5,<0.6",
|
|
31
31
|
"pandas>=2.2",
|
|
32
32
|
"pydantic>=2.12.5",
|
|
33
33
|
"jinja2>=3.0.0",
|
|
@@ -71,6 +71,11 @@ def fetch_product(
|
|
|
71
71
|
return df[df.index <= _utc_naive(end)], sources
|
|
72
72
|
|
|
73
73
|
|
|
74
|
+
def _real_time(sources: list[str]) -> bool:
|
|
75
|
+
"""Whether any source is one of NDBC's real-time files (not yet quality controlled)."""
|
|
76
|
+
return any("/data/realtime2/" in s for s in sources)
|
|
77
|
+
|
|
78
|
+
|
|
74
79
|
def _list(values) -> list:
|
|
75
80
|
return [
|
|
76
81
|
None if (v is None or (isinstance(v, float) and math.isnan(v))) else float(v)
|
|
@@ -92,8 +97,11 @@ def fetch_ndbc(
|
|
|
92
97
|
stdmet: times plus the columns of `STDMET_COLUMNS`. adcp: times, the bin depth and the
|
|
93
98
|
current as eastward and northward components (m/s); NDBC reports speed in cm/s and the
|
|
94
99
|
direction the current flows toward, in degrees true. Missing values are null. A window that
|
|
95
|
-
ended more than two days ago is cached as JSON under `cache_dir
|
|
96
|
-
|
|
100
|
+
ended more than two days ago is cached as JSON under `cache_dir`, unless any of it came from
|
|
101
|
+
NDBC's real-time files: those readings have not been through NDBC's quality control yet, and
|
|
102
|
+
the monthly and yearly files that replace them have, so such a window is fetched again until
|
|
103
|
+
they cover it. `get` (a URL to text or None) and `position` exist so tests can run without the
|
|
104
|
+
network.
|
|
97
105
|
"""
|
|
98
106
|
start, end = _utc_naive(start_time), _utc_naive(end_time)
|
|
99
107
|
unknown = set(products) - set(PRODUCTS)
|
|
@@ -107,7 +115,9 @@ def fetch_ndbc(
|
|
|
107
115
|
cache_path = os.path.join(cache_dir, "ndbc", f"{key}.json")
|
|
108
116
|
if os.path.exists(cache_path):
|
|
109
117
|
with open(cache_path) as f:
|
|
110
|
-
|
|
118
|
+
cached = json.load(f)
|
|
119
|
+
if not _real_time(cached.get("sources", [])):
|
|
120
|
+
return cached
|
|
111
121
|
|
|
112
122
|
lat, lon = position if position is not None else station_position(station, get)
|
|
113
123
|
out: dict = {
|
|
@@ -146,8 +156,10 @@ def fetch_ndbc(
|
|
|
146
156
|
},
|
|
147
157
|
}
|
|
148
158
|
|
|
149
|
-
if
|
|
150
|
-
|
|
159
|
+
if (
|
|
160
|
+
cache_path
|
|
161
|
+
and end < pd.Timestamp.utcnow().tz_localize(None) - pd.Timedelta(days=2)
|
|
162
|
+
and not _real_time(out["sources"])
|
|
151
163
|
):
|
|
152
164
|
with open(cache_path, "w") as f:
|
|
153
165
|
json.dump(out, f)
|
|
@@ -20,6 +20,9 @@ logger = logging.getLogger(__name__)
|
|
|
20
20
|
|
|
21
21
|
# Water-level datums to request, in order of preference.
|
|
22
22
|
TIDE_DATUMS = ("NAVD", "MSL")
|
|
23
|
+
# How long a cached water-level window that still holds preliminary readings is served before
|
|
24
|
+
# CO-OPS is asked again (a verified window is kept for good).
|
|
25
|
+
PRELIMINARY_TTL_S = 24 * 3600
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
def _utc(stamp: str) -> datetime:
|
|
@@ -56,9 +59,17 @@ def fetch_noaa_tide_data(
|
|
|
56
59
|
in metres, on NAVD88 where the station has it and MSL otherwise. Supports caching to avoid
|
|
57
60
|
redundant API calls.
|
|
58
61
|
|
|
59
|
-
Returns the shape of a CO-OPS response: ``{"metadata": {"id", "name", "lat", "lon", "datum"
|
|
60
|
-
"data": [{"t": "YYYY-MM-DD HH:MM", "v": "0.123"}, ...]}``, with ``lat``,
|
|
61
|
-
strings (``v`` empty for a missing reading).
|
|
62
|
+
Returns the shape of a CO-OPS response: ``{"metadata": {"id", "name", "lat", "lon", "datum",
|
|
63
|
+
"quality"}, "data": [{"t": "YYYY-MM-DD HH:MM", "v": "0.123", "q": "v"}, ...]}``, with ``lat``,
|
|
64
|
+
``lon`` and ``v`` as strings (``v`` empty for a missing reading). ``q`` is CO-OPS's quality
|
|
65
|
+
level for the reading, ``p`` (preliminary) or ``v`` (verified); ``metadata.quality`` is
|
|
66
|
+
``verified`` when every reading is, else ``preliminary``. Raises RuntimeError when the window
|
|
67
|
+
has no data.
|
|
68
|
+
|
|
69
|
+
A verified window is cached for good. CO-OPS replaces preliminary readings once it has
|
|
70
|
+
reviewed them, usually within a few months, so a window holding any is served from the cache
|
|
71
|
+
for ``PRELIMINARY_TTL_S`` (a day) and then fetched again, until it is verified. A cached file
|
|
72
|
+
without ``metadata.quality`` (written before 0.17.0) is fetched again.
|
|
62
73
|
|
|
63
74
|
Args:
|
|
64
75
|
station_id: NOAA station ID (e.g., '8516945' for Kings Point)
|
|
@@ -73,9 +84,18 @@ def fetch_noaa_tide_data(
|
|
|
73
84
|
cache_path = os.path.join(cache_dir, f"{cache_key}.json")
|
|
74
85
|
|
|
75
86
|
if not cache_bust and os.path.exists(cache_path):
|
|
76
|
-
logger.info(f"Cache hit for NOAA station {station_id}: {cache_path}")
|
|
77
87
|
with open(cache_path, "r") as f:
|
|
78
|
-
|
|
88
|
+
cached = json.load(f)
|
|
89
|
+
quality = (cached.get("metadata") or {}).get("quality")
|
|
90
|
+
age_s = datetime.now().timestamp() - os.path.getmtime(cache_path)
|
|
91
|
+
if quality == "verified" or (
|
|
92
|
+
quality == "preliminary" and age_s < PRELIMINARY_TTL_S
|
|
93
|
+
):
|
|
94
|
+
logger.info(f"Cache hit for NOAA station {station_id}: {cache_path}")
|
|
95
|
+
return cached
|
|
96
|
+
logger.info(
|
|
97
|
+
f"Cached NOAA window stale or unverified; fetching again: {cache_path}"
|
|
98
|
+
)
|
|
79
99
|
|
|
80
100
|
logger.info(
|
|
81
101
|
f"Fetching NOAA tide data for station {station_id} ({start_time} to {end_time})..."
|
|
@@ -87,7 +107,9 @@ def fetch_noaa_tide_data(
|
|
|
87
107
|
# 8465705, publishes only tidal datums). The datum used is recorded in the metadata.
|
|
88
108
|
for datum in TIDE_DATUMS:
|
|
89
109
|
try:
|
|
90
|
-
df = coops.fetch(
|
|
110
|
+
df = coops.fetch(
|
|
111
|
+
"water_level", station_id, start, end, datum=datum, quality=True
|
|
112
|
+
)
|
|
91
113
|
break
|
|
92
114
|
except coops.DatumUnavailable:
|
|
93
115
|
if datum == TIDE_DATUMS[-1]:
|
|
@@ -101,6 +123,7 @@ def fetch_noaa_tide_data(
|
|
|
101
123
|
f"({start_time} to {end_time})"
|
|
102
124
|
)
|
|
103
125
|
info = coops.metadata(station_id, expand="")
|
|
126
|
+
verified = bool((df["quality"] == "v").all())
|
|
104
127
|
data = {
|
|
105
128
|
"metadata": {
|
|
106
129
|
"id": station_id,
|
|
@@ -108,15 +131,20 @@ def fetch_noaa_tide_data(
|
|
|
108
131
|
"lat": str(info["lat"]),
|
|
109
132
|
"lon": str(info["lng"]),
|
|
110
133
|
"datum": datum,
|
|
134
|
+
"quality": "verified" if verified else "preliminary",
|
|
111
135
|
},
|
|
112
136
|
"data": [
|
|
113
137
|
# Written as CO-OPS writes it: 3 decimals at most, no trailing zeros.
|
|
114
|
-
{
|
|
115
|
-
|
|
138
|
+
{
|
|
139
|
+
"t": _t(t),
|
|
140
|
+
"v": "" if pd.isna(v) else str(round(float(v), 3)),
|
|
141
|
+
"q": q or "",
|
|
142
|
+
}
|
|
143
|
+
for t, v, q in zip(df["time"], df["water_level"], df["quality"])
|
|
116
144
|
],
|
|
117
145
|
}
|
|
118
146
|
|
|
119
|
-
#
|
|
147
|
+
# Kept for good when verified; a preliminary window expires (see PRELIMINARY_TTL_S).
|
|
120
148
|
with open(cache_path, "w") as f:
|
|
121
149
|
json.dump(data, f, indent=4)
|
|
122
150
|
|
|
@@ -95,3 +95,29 @@ def test_unknown_product_is_refused():
|
|
|
95
95
|
ndbc.fetch_ndbc(
|
|
96
96
|
"44091", "2026-04-02", "2026-04-03", ("waves",), position=(0, 0)
|
|
97
97
|
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _counting(files):
|
|
101
|
+
asked = []
|
|
102
|
+
inner = _fixture_get(files)
|
|
103
|
+
|
|
104
|
+
def get(url):
|
|
105
|
+
asked.append(url)
|
|
106
|
+
return inner(url)
|
|
107
|
+
|
|
108
|
+
return get, asked
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_only_quality_controlled_windows_are_cached(tmp_path):
|
|
112
|
+
window = ("44091", "2026-04-02T00:00:00Z", "2026-04-02T23:59:00Z", ("stdmet",))
|
|
113
|
+
# From the real-time file (not yet quality controlled): served, not cached.
|
|
114
|
+
get, asked = _counting({"realtime2": STDMET})
|
|
115
|
+
ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
|
|
116
|
+
assert not list((tmp_path / "ndbc").glob("*.json"))
|
|
117
|
+
# From the yearly file: cached, and served from the cache next time.
|
|
118
|
+
get, asked = _counting({"historical/stdmet": STDMET})
|
|
119
|
+
ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
|
|
120
|
+
assert len(list((tmp_path / "ndbc").glob("*.json"))) == 1
|
|
121
|
+
get, asked = _counting({})
|
|
122
|
+
out = ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
|
|
123
|
+
assert asked == [] and out["products"]["stdmet"]["WTMP"] == [6.0, 6.1]
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""CO-OPS water level: NAVD88 where a station has it, MSL where it does not."""
|
|
2
2
|
|
|
3
|
+
import os
|
|
4
|
+
|
|
3
5
|
import pytest
|
|
4
6
|
from insitukit import coops
|
|
5
7
|
|
|
@@ -56,14 +58,49 @@ def test_keeps_the_co_ops_shape_with_the_end_included(tmp_path, monkeypatch):
|
|
|
56
58
|
"lat": "41.2833",
|
|
57
59
|
"lon": "-72.9083",
|
|
58
60
|
"datum": "NAVD",
|
|
61
|
+
"quality": "preliminary",
|
|
59
62
|
}
|
|
60
63
|
# The reading at end_time is kept; a missing reading is an empty string, as CO-OPS sends it.
|
|
61
64
|
assert data["data"] == [
|
|
62
|
-
{"t": "2026-04-02 00:00", "v": "0.1"},
|
|
63
|
-
{"t": "2026-04-02 04:00", "v": ""},
|
|
65
|
+
{"t": "2026-04-02 00:00", "v": "0.1", "q": "p"},
|
|
66
|
+
{"t": "2026-04-02 04:00", "v": "", "q": "p"},
|
|
64
67
|
]
|
|
65
68
|
|
|
66
69
|
|
|
67
70
|
def test_an_empty_window_is_an_error(tmp_path, monkeypatch):
|
|
68
71
|
with pytest.raises(RuntimeError, match="No data"):
|
|
69
72
|
_fetch(tmp_path, monkeypatch, {"NAVD": None})
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
VERIFIED = {"data": [{**r, "q": "v"} for r in ROWS["data"]]}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _age(tmp_path, seconds):
|
|
79
|
+
for p in tmp_path.glob("*.json"):
|
|
80
|
+
t = p.stat().st_mtime - seconds
|
|
81
|
+
os.utime(p, (t, t))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_a_preliminary_window_expires_and_a_verified_one_is_kept(tmp_path, monkeypatch):
|
|
85
|
+
# Preliminary: cached for a day, then CO-OPS is asked again.
|
|
86
|
+
_, calls = _fetch(tmp_path, monkeypatch, {"NAVD": ROWS})
|
|
87
|
+
assert calls == ["NAVD"]
|
|
88
|
+
_, calls = _fetch(tmp_path, monkeypatch, {"NAVD": ROWS})
|
|
89
|
+
assert calls == []
|
|
90
|
+
_age(tmp_path, noaa.PRELIMINARY_TTL_S + 60)
|
|
91
|
+
# A day on, CO-OPS has verified the window: it replaces the cached copy, kept for good.
|
|
92
|
+
data, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
|
|
93
|
+
assert calls == ["NAVD"] and data["metadata"]["quality"] == "verified"
|
|
94
|
+
_age(tmp_path, 365 * 86400)
|
|
95
|
+
_, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
|
|
96
|
+
assert calls == []
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_a_cache_file_without_a_quality_level_is_fetched_again(tmp_path, monkeypatch):
|
|
100
|
+
# Written by forcingkit before 0.17.0 (raw CO-OPS JSON, or the 0.14 shape): no metadata.quality.
|
|
101
|
+
key = "8465705_2026-04-02T00-00-00_2026-04-02T04-00-00"
|
|
102
|
+
(tmp_path / f"{key}.json").write_text(
|
|
103
|
+
'{"metadata": {"id": "8465705", "datum": "NAVD"}, "data": [{"t": "2026-04-02 00:00", "v": "9.9"}]}'
|
|
104
|
+
)
|
|
105
|
+
data, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
|
|
106
|
+
assert calls == ["NAVD"] and data["data"][0]["v"] == "0.1"
|
|
@@ -1000,7 +1000,7 @@ wheels = [
|
|
|
1000
1000
|
|
|
1001
1001
|
[[package]]
|
|
1002
1002
|
name = "forcingkit"
|
|
1003
|
-
version = "0.
|
|
1003
|
+
version = "0.17.0"
|
|
1004
1004
|
source = { editable = "." }
|
|
1005
1005
|
dependencies = [
|
|
1006
1006
|
{ name = "bottleneck" },
|
|
@@ -1062,7 +1062,7 @@ requires-dist = [
|
|
|
1062
1062
|
{ name = "h5netcdf", specifier = ">=1.8.1" },
|
|
1063
1063
|
{ name = "h5py", specifier = ">=3.16.0" },
|
|
1064
1064
|
{ name = "httpx", specifier = ">=0.28.1" },
|
|
1065
|
-
{ name = "insitukit", specifier = ">=0.
|
|
1065
|
+
{ name = "insitukit", specifier = ">=0.5,<0.6" },
|
|
1066
1066
|
{ name = "jinja2", specifier = ">=3.0.0" },
|
|
1067
1067
|
{ name = "matplotlib", specifier = ">=3.10.9" },
|
|
1068
1068
|
{ name = "pandas", specifier = ">=2.2" },
|
|
@@ -1612,16 +1612,16 @@ wheels = [
|
|
|
1612
1612
|
|
|
1613
1613
|
[[package]]
|
|
1614
1614
|
name = "insitukit"
|
|
1615
|
-
version = "0.
|
|
1615
|
+
version = "0.5.0"
|
|
1616
1616
|
source = { registry = "https://pypi.org/simple" }
|
|
1617
1617
|
dependencies = [
|
|
1618
1618
|
{ name = "numpy" },
|
|
1619
1619
|
{ name = "pandas" },
|
|
1620
1620
|
{ name = "requests" },
|
|
1621
1621
|
]
|
|
1622
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
1622
|
+
sdist = { url = "https://files.pythonhosted.org/packages/7d/78/f978cdea497458927cec2cd2d49e9dda401780ce85063e156d889fe0b1f3/insitukit-0.5.0.tar.gz", hash = "sha256:77e4ec1fabc170ee49fc4750839da3cdd8fe900ccdc9bffb2e9b4951114bdd4e", size = 105398, upload-time = "2026-11-06T15:44:09.573Z" }
|
|
1623
1623
|
wheels = [
|
|
1624
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
1624
|
+
{ url = "https://files.pythonhosted.org/packages/8a/c8/9d0df8dded9c9b3962a66f2a6b7de0ddc20e43f7aad2c140c4886e9a65c4/insitukit-0.5.0-py3-none-any.whl", hash = "sha256:8dbc7c279e841eb10de5ca6b6239389a8d896c7f7a1e8066522b2ebd35efa18e", size = 27661, upload-time = "2026-11-06T15:44:08.173Z" },
|
|
1625
1625
|
]
|
|
1626
1626
|
|
|
1627
1627
|
[[package]]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|