forcingkit 0.15.1__tar.gz → 0.17.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.15.1 → forcingkit-0.17.0}/PKG-INFO +2 -2
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/conf.py +1 -1
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/fetchers.rst +11 -2
- {forcingkit-0.15.1 → forcingkit-0.17.0}/pyproject.toml +3 -3
- forcingkit-0.17.0/src/forcingkit/fetchers/ndbc.py +166 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/noaa.py +37 -9
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_ndbc.py +26 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_noaa_datum.py +39 -2
- {forcingkit-0.15.1 → forcingkit-0.17.0}/uv.lock +5 -5
- forcingkit-0.15.1/src/forcingkit/fetchers/ndbc.py +0 -231
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.agents/forcingkit.md +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.env.template +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.gitignore +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.markdownlint.yaml +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/.python-version +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/AGENTS.md +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/CONTRIBUTING.md +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/Dockerfile +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/LICENSE +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/README.md +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/LICENSE +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/README.md +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/pyproject.toml +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/tests/conftest.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/tests/test_client.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/tests/test_schema.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/client/uv.lock +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docker-compose.yml +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/Makefile +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/make.bat +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/_static/custom.css +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/client.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/hycom.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/index.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/necofs.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/rivers.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/sources.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/waves.rst +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/main.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/__init__.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/viewer.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/service/run_server.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/render.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/static/app.js +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/static/favicon.ico +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/static/favicon.svg +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/static/index.html +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/static/logo.svg +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/static/preview3d.js +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/static/styles.css +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/__init__.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_main.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.17.0
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -225,7 +225,7 @@ Requires-Dist: fastapi>=0.135.1
|
|
|
225
225
|
Requires-Dist: h5netcdf>=1.8.1
|
|
226
226
|
Requires-Dist: h5py>=3.16.0
|
|
227
227
|
Requires-Dist: httpx>=0.28.1
|
|
228
|
-
Requires-Dist: insitukit<0.
|
|
228
|
+
Requires-Dist: insitukit<0.6,>=0.5
|
|
229
229
|
Requires-Dist: jinja2>=3.0.0
|
|
230
230
|
Requires-Dist: matplotlib>=3.10.9
|
|
231
231
|
Requires-Dist: pandas>=2.2
|
|
@@ -149,10 +149,19 @@ These serve validation rather than forcing:
|
|
|
149
149
|
fallback to MSL where a station has no NAVD88 datum. Requests go through
|
|
150
150
|
`insitukit <https://github.com/lhzn-io/insitukit>`__, which keeps them at least 1 s apart,
|
|
151
151
|
splits windows longer than the API's 31-day limit, and retries server errors and throttling;
|
|
152
|
-
``end_time`` is inclusive, as in the API.
|
|
152
|
+
``end_time`` is inclusive, as in the API. Each reading carries CO-OPS's quality level (``q``:
|
|
153
|
+
``p`` preliminary, ``v`` verified) and ``metadata.quality`` sums up the window. Since 0.17.0 a
|
|
154
|
+
verified window is cached for good and one still holding preliminary readings for a day, because
|
|
155
|
+
CO-OPS replaces preliminary readings once it has reviewed them. Current predictions
|
|
156
|
+
(``/api/v1/currents``) and the
|
|
153
157
|
current-station list (``/api/v1/currents/stations``) use the same client.
|
|
154
158
|
- **NDBC** buoy meteorology and ADCP profiles (``/api/v1/ndbc``), from the `NOAA National Data
|
|
155
|
-
Buoy Center <https://www.ndbc.noaa.gov/>`__
|
|
159
|
+
Buoy Center <https://www.ndbc.noaa.gov/>`__, through insitukit's NDBC client: each month from
|
|
160
|
+
the monthly, yearly or real-time file, each file downloaded once, and each column's own
|
|
161
|
+
missing-value sentinel read as missing (since 0.16.0; earlier releases also dropped real
|
|
162
|
+
readings equal to another column's sentinel, such as a 999 hPa low or a 99-degree wind). A
|
|
163
|
+
window is cached only when none of it came from the real-time files, whose readings have not
|
|
164
|
+
yet been through NDBC's quality control (since 0.17.0).
|
|
156
165
|
- **UConn ERDDAP** water-column profiles (``/api/v1/telemetry/station`` and ``/bbox``), from the
|
|
157
166
|
`University of Connecticut Department of Marine Sciences <https://marinesciences.uconn.edu/>`__
|
|
158
167
|
`ERDDAP server <http://merlin.dms.uconn.edu:8080/erddap/index.html>`__.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.17.0"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -26,8 +26,8 @@ dependencies = [
|
|
|
26
26
|
"eccodeslib>=2.46.0",
|
|
27
27
|
"fastapi>=0.135.1",
|
|
28
28
|
"httpx>=0.28.1",
|
|
29
|
-
# NOAA CO-OPS (fetchers/noaa.py)
|
|
30
|
-
"insitukit>=0.
|
|
29
|
+
# NOAA CO-OPS (fetchers/noaa.py), the UConn ERDDAP profiles (fetchers/erddap.py) and NDBC (fetchers/ndbc.py).
|
|
30
|
+
"insitukit>=0.5,<0.6",
|
|
31
31
|
"pandas>=2.2",
|
|
32
32
|
"pydantic>=2.12.5",
|
|
33
33
|
"jinja2>=3.0.0",
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""NDBC buoy observations for model validation: standard meteorology (sea and air temperature,
|
|
2
|
+
pressure, wind, waves) and near-surface ADCP currents, through insitukit.
|
|
3
|
+
|
|
4
|
+
insitukit's NDBC client reads each month from the first file NDBC publishes it in: the per-month
|
|
5
|
+
file for months of the current year, then the yearly historical file, then the rolling 45-day
|
|
6
|
+
real-time file. It downloads each file once per call, paces and identifies its requests, and turns
|
|
7
|
+
NDBC's missing-value sentinels (99, 999, 9999, MM) into NaN. This module keeps forcingkit's
|
|
8
|
+
function names and the JSON shape of ``fetch_ndbc``. These observations validate a model; they
|
|
9
|
+
never force it.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import math
|
|
14
|
+
import os
|
|
15
|
+
from typing import Optional
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
import pandas as pd
|
|
19
|
+
from insitukit import ndbc as _ndbc
|
|
20
|
+
|
|
21
|
+
PRODUCTS = _ndbc.PRODUCTS
|
|
22
|
+
# stdmet columns carried through, with units.
|
|
23
|
+
STDMET_COLUMNS = {
|
|
24
|
+
"WTMP": "degC", # sea surface temperature
|
|
25
|
+
"ATMP": "degC", # air temperature
|
|
26
|
+
"PRES": "hPa", # sea level pressure
|
|
27
|
+
"WSPD": "m s-1", # wind speed
|
|
28
|
+
"WDIR": "degree from true north (direction the wind comes from)",
|
|
29
|
+
"GST": "m s-1",
|
|
30
|
+
"WVHT": "m",
|
|
31
|
+
"DPD": "s",
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
month_code = _ndbc.month_code
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def candidate_urls(station: str, product: str, year: int, month: int) -> list[str]:
|
|
38
|
+
"""URLs that may hold `product` for `station` in the given month, in the order to try."""
|
|
39
|
+
return _ndbc.urls(station, product, year, month)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_ndbc_text(text: str) -> pd.DataFrame:
|
|
43
|
+
"""Parse an NDBC whitespace-delimited file into a frame indexed by UTC time (naive),
|
|
44
|
+
sentinels and `MM` as NaN."""
|
|
45
|
+
df = _ndbc.parse(text)
|
|
46
|
+
return df.set_index(pd.DatetimeIndex(df.pop("time")).tz_localize(None))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _utc_naive(t) -> pd.Timestamp:
|
|
50
|
+
t = pd.Timestamp(t)
|
|
51
|
+
return t if t.tzinfo is None else t.tz_convert("UTC").tz_localize(None)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def station_position(station: str, get=None) -> tuple[float, float]:
|
|
55
|
+
"""(lat, lon) of an NDBC station from its station page."""
|
|
56
|
+
return _ndbc.position(station, get=get)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def fetch_product(
|
|
60
|
+
station: str, product: str, start: pd.Timestamp, end: pd.Timestamp, get=None
|
|
61
|
+
):
|
|
62
|
+
"""`product` for `station` over [start, end] (naive UTC, both included): (frame indexed by
|
|
63
|
+
naive UTC time, source URLs). Each month is taken from the first source that has it."""
|
|
64
|
+
t0 = _utc_naive(start).tz_localize("UTC")
|
|
65
|
+
t1 = _utc_naive(end).tz_localize("UTC") + pd.Timedelta(minutes=1)
|
|
66
|
+
df = _ndbc.fetch(station, t0, t1, product, get=get)
|
|
67
|
+
sources = list(df.attrs.get("sources", []))
|
|
68
|
+
if df.empty:
|
|
69
|
+
return pd.DataFrame(), sources
|
|
70
|
+
df = df.set_index(pd.DatetimeIndex(df.pop("time")).tz_localize(None))
|
|
71
|
+
return df[df.index <= _utc_naive(end)], sources
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _real_time(sources: list[str]) -> bool:
|
|
75
|
+
"""Whether any source is one of NDBC's real-time files (not yet quality controlled)."""
|
|
76
|
+
return any("/data/realtime2/" in s for s in sources)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _list(values) -> list:
|
|
80
|
+
return [
|
|
81
|
+
None if (v is None or (isinstance(v, float) and math.isnan(v))) else float(v)
|
|
82
|
+
for v in values
|
|
83
|
+
]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def fetch_ndbc(
|
|
87
|
+
station: str,
|
|
88
|
+
start_time: str,
|
|
89
|
+
end_time: str,
|
|
90
|
+
products: tuple[str, ...] = PRODUCTS,
|
|
91
|
+
cache_dir: Optional[str] = None,
|
|
92
|
+
get=None,
|
|
93
|
+
position: Optional[tuple[float, float]] = None,
|
|
94
|
+
) -> dict:
|
|
95
|
+
"""Observations at an NDBC station over [start_time, end_time] (UTC) as a JSON-ready dict.
|
|
96
|
+
|
|
97
|
+
stdmet: times plus the columns of `STDMET_COLUMNS`. adcp: times, the bin depth and the
|
|
98
|
+
current as eastward and northward components (m/s); NDBC reports speed in cm/s and the
|
|
99
|
+
direction the current flows toward, in degrees true. Missing values are null. A window that
|
|
100
|
+
ended more than two days ago is cached as JSON under `cache_dir`, unless any of it came from
|
|
101
|
+
NDBC's real-time files: those readings have not been through NDBC's quality control yet, and
|
|
102
|
+
the monthly and yearly files that replace them have, so such a window is fetched again until
|
|
103
|
+
they cover it. `get` (a URL to text or None) and `position` exist so tests can run without the
|
|
104
|
+
network.
|
|
105
|
+
"""
|
|
106
|
+
start, end = _utc_naive(start_time), _utc_naive(end_time)
|
|
107
|
+
unknown = set(products) - set(PRODUCTS)
|
|
108
|
+
if unknown:
|
|
109
|
+
raise ValueError(f"unknown NDBC products {sorted(unknown)}")
|
|
110
|
+
|
|
111
|
+
cache_path = None
|
|
112
|
+
if cache_dir:
|
|
113
|
+
os.makedirs(os.path.join(cache_dir, "ndbc"), exist_ok=True)
|
|
114
|
+
key = f"{station.lower()}_{start:%Y%m%dT%H%M}_{end:%Y%m%dT%H%M}_{'-'.join(sorted(products))}"
|
|
115
|
+
cache_path = os.path.join(cache_dir, "ndbc", f"{key}.json")
|
|
116
|
+
if os.path.exists(cache_path):
|
|
117
|
+
with open(cache_path) as f:
|
|
118
|
+
cached = json.load(f)
|
|
119
|
+
if not _real_time(cached.get("sources", [])):
|
|
120
|
+
return cached
|
|
121
|
+
|
|
122
|
+
lat, lon = position if position is not None else station_position(station, get)
|
|
123
|
+
out: dict = {
|
|
124
|
+
"station_id": station.upper(),
|
|
125
|
+
"lat": lat,
|
|
126
|
+
"lon": lon,
|
|
127
|
+
"start_time": f"{start:%Y-%m-%dT%H:%M:%SZ}",
|
|
128
|
+
"end_time": f"{end:%Y-%m-%dT%H:%M:%SZ}",
|
|
129
|
+
"products": {},
|
|
130
|
+
"sources": [],
|
|
131
|
+
}
|
|
132
|
+
for product in products:
|
|
133
|
+
df, sources = fetch_product(station, product, start, end, get=get)
|
|
134
|
+
out["sources"] += sources
|
|
135
|
+
times = [f"{t:%Y-%m-%dT%H:%M:%SZ}" for t in df.index]
|
|
136
|
+
if product == "stdmet":
|
|
137
|
+
out["products"]["stdmet"] = {
|
|
138
|
+
"times": times,
|
|
139
|
+
"units": {c: u for c, u in STDMET_COLUMNS.items() if c in df},
|
|
140
|
+
**{c: _list(df[c]) for c in STDMET_COLUMNS if c in df},
|
|
141
|
+
}
|
|
142
|
+
else:
|
|
143
|
+
speed = df["SPD01"] / 100.0 if "SPD01" in df else pd.Series(dtype=float)
|
|
144
|
+
toward = (
|
|
145
|
+
np.deg2rad(df["DIR01"]) if "DIR01" in df else pd.Series(dtype=float)
|
|
146
|
+
)
|
|
147
|
+
out["products"]["adcp"] = {
|
|
148
|
+
"times": times,
|
|
149
|
+
"depth_m": _list(df["DEP01"]) if "DEP01" in df else [],
|
|
150
|
+
"u": _list(speed * np.sin(toward)),
|
|
151
|
+
"v": _list(speed * np.cos(toward)),
|
|
152
|
+
"units": {
|
|
153
|
+
"u": "m s-1 eastward",
|
|
154
|
+
"v": "m s-1 northward",
|
|
155
|
+
"depth_m": "m",
|
|
156
|
+
},
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (
|
|
160
|
+
cache_path
|
|
161
|
+
and end < pd.Timestamp.utcnow().tz_localize(None) - pd.Timedelta(days=2)
|
|
162
|
+
and not _real_time(out["sources"])
|
|
163
|
+
):
|
|
164
|
+
with open(cache_path, "w") as f:
|
|
165
|
+
json.dump(out, f)
|
|
166
|
+
return out
|
|
@@ -20,6 +20,9 @@ logger = logging.getLogger(__name__)
|
|
|
20
20
|
|
|
21
21
|
# Water-level datums to request, in order of preference.
|
|
22
22
|
TIDE_DATUMS = ("NAVD", "MSL")
|
|
23
|
+
# How long a cached water-level window that still holds preliminary readings is served before
|
|
24
|
+
# CO-OPS is asked again (a verified window is kept for good).
|
|
25
|
+
PRELIMINARY_TTL_S = 24 * 3600
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
def _utc(stamp: str) -> datetime:
|
|
@@ -56,9 +59,17 @@ def fetch_noaa_tide_data(
|
|
|
56
59
|
in metres, on NAVD88 where the station has it and MSL otherwise. Supports caching to avoid
|
|
57
60
|
redundant API calls.
|
|
58
61
|
|
|
59
|
-
Returns the shape of a CO-OPS response: ``{"metadata": {"id", "name", "lat", "lon", "datum"
|
|
60
|
-
"data": [{"t": "YYYY-MM-DD HH:MM", "v": "0.123"}, ...]}``, with ``lat``,
|
|
61
|
-
strings (``v`` empty for a missing reading).
|
|
62
|
+
Returns the shape of a CO-OPS response: ``{"metadata": {"id", "name", "lat", "lon", "datum",
|
|
63
|
+
"quality"}, "data": [{"t": "YYYY-MM-DD HH:MM", "v": "0.123", "q": "v"}, ...]}``, with ``lat``,
|
|
64
|
+
``lon`` and ``v`` as strings (``v`` empty for a missing reading). ``q`` is CO-OPS's quality
|
|
65
|
+
level for the reading, ``p`` (preliminary) or ``v`` (verified); ``metadata.quality`` is
|
|
66
|
+
``verified`` when every reading is, else ``preliminary``. Raises RuntimeError when the window
|
|
67
|
+
has no data.
|
|
68
|
+
|
|
69
|
+
A verified window is cached for good. CO-OPS replaces preliminary readings once it has
|
|
70
|
+
reviewed them, usually within a few months, so a window holding any is served from the cache
|
|
71
|
+
for ``PRELIMINARY_TTL_S`` (a day) and then fetched again, until it is verified. A cached file
|
|
72
|
+
without ``metadata.quality`` (written before 0.17.0) is fetched again.
|
|
62
73
|
|
|
63
74
|
Args:
|
|
64
75
|
station_id: NOAA station ID (e.g., '8516945' for Kings Point)
|
|
@@ -73,9 +84,18 @@ def fetch_noaa_tide_data(
|
|
|
73
84
|
cache_path = os.path.join(cache_dir, f"{cache_key}.json")
|
|
74
85
|
|
|
75
86
|
if not cache_bust and os.path.exists(cache_path):
|
|
76
|
-
logger.info(f"Cache hit for NOAA station {station_id}: {cache_path}")
|
|
77
87
|
with open(cache_path, "r") as f:
|
|
78
|
-
|
|
88
|
+
cached = json.load(f)
|
|
89
|
+
quality = (cached.get("metadata") or {}).get("quality")
|
|
90
|
+
age_s = datetime.now().timestamp() - os.path.getmtime(cache_path)
|
|
91
|
+
if quality == "verified" or (
|
|
92
|
+
quality == "preliminary" and age_s < PRELIMINARY_TTL_S
|
|
93
|
+
):
|
|
94
|
+
logger.info(f"Cache hit for NOAA station {station_id}: {cache_path}")
|
|
95
|
+
return cached
|
|
96
|
+
logger.info(
|
|
97
|
+
f"Cached NOAA window stale or unverified; fetching again: {cache_path}"
|
|
98
|
+
)
|
|
79
99
|
|
|
80
100
|
logger.info(
|
|
81
101
|
f"Fetching NOAA tide data for station {station_id} ({start_time} to {end_time})..."
|
|
@@ -87,7 +107,9 @@ def fetch_noaa_tide_data(
|
|
|
87
107
|
# 8465705, publishes only tidal datums). The datum used is recorded in the metadata.
|
|
88
108
|
for datum in TIDE_DATUMS:
|
|
89
109
|
try:
|
|
90
|
-
df = coops.fetch(
|
|
110
|
+
df = coops.fetch(
|
|
111
|
+
"water_level", station_id, start, end, datum=datum, quality=True
|
|
112
|
+
)
|
|
91
113
|
break
|
|
92
114
|
except coops.DatumUnavailable:
|
|
93
115
|
if datum == TIDE_DATUMS[-1]:
|
|
@@ -101,6 +123,7 @@ def fetch_noaa_tide_data(
|
|
|
101
123
|
f"({start_time} to {end_time})"
|
|
102
124
|
)
|
|
103
125
|
info = coops.metadata(station_id, expand="")
|
|
126
|
+
verified = bool((df["quality"] == "v").all())
|
|
104
127
|
data = {
|
|
105
128
|
"metadata": {
|
|
106
129
|
"id": station_id,
|
|
@@ -108,15 +131,20 @@ def fetch_noaa_tide_data(
|
|
|
108
131
|
"lat": str(info["lat"]),
|
|
109
132
|
"lon": str(info["lng"]),
|
|
110
133
|
"datum": datum,
|
|
134
|
+
"quality": "verified" if verified else "preliminary",
|
|
111
135
|
},
|
|
112
136
|
"data": [
|
|
113
137
|
# Written as CO-OPS writes it: 3 decimals at most, no trailing zeros.
|
|
114
|
-
{
|
|
115
|
-
|
|
138
|
+
{
|
|
139
|
+
"t": _t(t),
|
|
140
|
+
"v": "" if pd.isna(v) else str(round(float(v), 3)),
|
|
141
|
+
"q": q or "",
|
|
142
|
+
}
|
|
143
|
+
for t, v, q in zip(df["time"], df["water_level"], df["quality"])
|
|
116
144
|
],
|
|
117
145
|
}
|
|
118
146
|
|
|
119
|
-
#
|
|
147
|
+
# Kept for good when verified; a preliminary window expires (see PRELIMINARY_TTL_S).
|
|
120
148
|
with open(cache_path, "w") as f:
|
|
121
149
|
json.dump(data, f, indent=4)
|
|
122
150
|
|
|
@@ -95,3 +95,29 @@ def test_unknown_product_is_refused():
|
|
|
95
95
|
ndbc.fetch_ndbc(
|
|
96
96
|
"44091", "2026-04-02", "2026-04-03", ("waves",), position=(0, 0)
|
|
97
97
|
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _counting(files):
|
|
101
|
+
asked = []
|
|
102
|
+
inner = _fixture_get(files)
|
|
103
|
+
|
|
104
|
+
def get(url):
|
|
105
|
+
asked.append(url)
|
|
106
|
+
return inner(url)
|
|
107
|
+
|
|
108
|
+
return get, asked
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_only_quality_controlled_windows_are_cached(tmp_path):
|
|
112
|
+
window = ("44091", "2026-04-02T00:00:00Z", "2026-04-02T23:59:00Z", ("stdmet",))
|
|
113
|
+
# From the real-time file (not yet quality controlled): served, not cached.
|
|
114
|
+
get, asked = _counting({"realtime2": STDMET})
|
|
115
|
+
ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
|
|
116
|
+
assert not list((tmp_path / "ndbc").glob("*.json"))
|
|
117
|
+
# From the yearly file: cached, and served from the cache next time.
|
|
118
|
+
get, asked = _counting({"historical/stdmet": STDMET})
|
|
119
|
+
ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
|
|
120
|
+
assert len(list((tmp_path / "ndbc").glob("*.json"))) == 1
|
|
121
|
+
get, asked = _counting({})
|
|
122
|
+
out = ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
|
|
123
|
+
assert asked == [] and out["products"]["stdmet"]["WTMP"] == [6.0, 6.1]
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""CO-OPS water level: NAVD88 where a station has it, MSL where it does not."""
|
|
2
2
|
|
|
3
|
+
import os
|
|
4
|
+
|
|
3
5
|
import pytest
|
|
4
6
|
from insitukit import coops
|
|
5
7
|
|
|
@@ -56,14 +58,49 @@ def test_keeps_the_co_ops_shape_with_the_end_included(tmp_path, monkeypatch):
|
|
|
56
58
|
"lat": "41.2833",
|
|
57
59
|
"lon": "-72.9083",
|
|
58
60
|
"datum": "NAVD",
|
|
61
|
+
"quality": "preliminary",
|
|
59
62
|
}
|
|
60
63
|
# The reading at end_time is kept; a missing reading is an empty string, as CO-OPS sends it.
|
|
61
64
|
assert data["data"] == [
|
|
62
|
-
{"t": "2026-04-02 00:00", "v": "0.1"},
|
|
63
|
-
{"t": "2026-04-02 04:00", "v": ""},
|
|
65
|
+
{"t": "2026-04-02 00:00", "v": "0.1", "q": "p"},
|
|
66
|
+
{"t": "2026-04-02 04:00", "v": "", "q": "p"},
|
|
64
67
|
]
|
|
65
68
|
|
|
66
69
|
|
|
67
70
|
def test_an_empty_window_is_an_error(tmp_path, monkeypatch):
|
|
68
71
|
with pytest.raises(RuntimeError, match="No data"):
|
|
69
72
|
_fetch(tmp_path, monkeypatch, {"NAVD": None})
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
VERIFIED = {"data": [{**r, "q": "v"} for r in ROWS["data"]]}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _age(tmp_path, seconds):
|
|
79
|
+
for p in tmp_path.glob("*.json"):
|
|
80
|
+
t = p.stat().st_mtime - seconds
|
|
81
|
+
os.utime(p, (t, t))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_a_preliminary_window_expires_and_a_verified_one_is_kept(tmp_path, monkeypatch):
|
|
85
|
+
# Preliminary: cached for a day, then CO-OPS is asked again.
|
|
86
|
+
_, calls = _fetch(tmp_path, monkeypatch, {"NAVD": ROWS})
|
|
87
|
+
assert calls == ["NAVD"]
|
|
88
|
+
_, calls = _fetch(tmp_path, monkeypatch, {"NAVD": ROWS})
|
|
89
|
+
assert calls == []
|
|
90
|
+
_age(tmp_path, noaa.PRELIMINARY_TTL_S + 60)
|
|
91
|
+
# A day on, CO-OPS has verified the window: it replaces the cached copy, kept for good.
|
|
92
|
+
data, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
|
|
93
|
+
assert calls == ["NAVD"] and data["metadata"]["quality"] == "verified"
|
|
94
|
+
_age(tmp_path, 365 * 86400)
|
|
95
|
+
_, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
|
|
96
|
+
assert calls == []
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_a_cache_file_without_a_quality_level_is_fetched_again(tmp_path, monkeypatch):
|
|
100
|
+
# Written by forcingkit before 0.17.0 (raw CO-OPS JSON, or the 0.14 shape): no metadata.quality.
|
|
101
|
+
key = "8465705_2026-04-02T00-00-00_2026-04-02T04-00-00"
|
|
102
|
+
(tmp_path / f"{key}.json").write_text(
|
|
103
|
+
'{"metadata": {"id": "8465705", "datum": "NAVD"}, "data": [{"t": "2026-04-02 00:00", "v": "9.9"}]}'
|
|
104
|
+
)
|
|
105
|
+
data, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
|
|
106
|
+
assert calls == ["NAVD"] and data["data"][0]["v"] == "0.1"
|
|
@@ -1000,7 +1000,7 @@ wheels = [
|
|
|
1000
1000
|
|
|
1001
1001
|
[[package]]
|
|
1002
1002
|
name = "forcingkit"
|
|
1003
|
-
version = "0.
|
|
1003
|
+
version = "0.17.0"
|
|
1004
1004
|
source = { editable = "." }
|
|
1005
1005
|
dependencies = [
|
|
1006
1006
|
{ name = "bottleneck" },
|
|
@@ -1062,7 +1062,7 @@ requires-dist = [
|
|
|
1062
1062
|
{ name = "h5netcdf", specifier = ">=1.8.1" },
|
|
1063
1063
|
{ name = "h5py", specifier = ">=3.16.0" },
|
|
1064
1064
|
{ name = "httpx", specifier = ">=0.28.1" },
|
|
1065
|
-
{ name = "insitukit", specifier = ">=0.
|
|
1065
|
+
{ name = "insitukit", specifier = ">=0.5,<0.6" },
|
|
1066
1066
|
{ name = "jinja2", specifier = ">=3.0.0" },
|
|
1067
1067
|
{ name = "matplotlib", specifier = ">=3.10.9" },
|
|
1068
1068
|
{ name = "pandas", specifier = ">=2.2" },
|
|
@@ -1612,16 +1612,16 @@ wheels = [
|
|
|
1612
1612
|
|
|
1613
1613
|
[[package]]
|
|
1614
1614
|
name = "insitukit"
|
|
1615
|
-
version = "0.
|
|
1615
|
+
version = "0.5.0"
|
|
1616
1616
|
source = { registry = "https://pypi.org/simple" }
|
|
1617
1617
|
dependencies = [
|
|
1618
1618
|
{ name = "numpy" },
|
|
1619
1619
|
{ name = "pandas" },
|
|
1620
1620
|
{ name = "requests" },
|
|
1621
1621
|
]
|
|
1622
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
1622
|
+
sdist = { url = "https://files.pythonhosted.org/packages/7d/78/f978cdea497458927cec2cd2d49e9dda401780ce85063e156d889fe0b1f3/insitukit-0.5.0.tar.gz", hash = "sha256:77e4ec1fabc170ee49fc4750839da3cdd8fe900ccdc9bffb2e9b4951114bdd4e", size = 105398, upload-time = "2026-11-06T15:44:09.573Z" }
|
|
1623
1623
|
wheels = [
|
|
1624
|
-
{ url = "https://files.pythonhosted.org/packages/8a/
|
|
1624
|
+
{ url = "https://files.pythonhosted.org/packages/8a/c8/9d0df8dded9c9b3962a66f2a6b7de0ddc20e43f7aad2c140c4886e9a65c4/insitukit-0.5.0-py3-none-any.whl", hash = "sha256:8dbc7c279e841eb10de5ca6b6239389a8d896c7f7a1e8066522b2ebd35efa18e", size = 27661, upload-time = "2026-11-06T15:44:08.173Z" },
|
|
1625
1625
|
]
|
|
1626
1626
|
|
|
1627
1627
|
[[package]]
|
|
@@ -1,231 +0,0 @@
|
|
|
1
|
-
"""NDBC buoy observations for model validation: standard meteorology (sea and air temperature,
|
|
2
|
-
pressure, wind, waves) and near-surface ADCP currents.
|
|
3
|
-
|
|
4
|
-
Files come from the NDBC data server in the order they are published: the per-month file for
|
|
5
|
-
months of the current year (`data/<product>/<Mon>/<id><m><yyyy>.txt.gz`), then the yearly
|
|
6
|
-
historical file (`data/historical/<product>/<id>h<yyyy>.txt.gz`), then the rolling 45-day
|
|
7
|
-
real-time file (`data/realtime2/<id>.txt` or `.adcp`). Missing-value sentinels (99, 999, 9999,
|
|
8
|
-
MM) become NaN. These observations validate a model; they never force it.
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
import gzip
|
|
12
|
-
import json
|
|
13
|
-
import logging
|
|
14
|
-
import math
|
|
15
|
-
import os
|
|
16
|
-
import re
|
|
17
|
-
from typing import Optional
|
|
18
|
-
|
|
19
|
-
import numpy as np
|
|
20
|
-
import pandas as pd
|
|
21
|
-
import requests
|
|
22
|
-
|
|
23
|
-
logger = logging.getLogger(__name__)
|
|
24
|
-
|
|
25
|
-
NDBC = "https://www.ndbc.noaa.gov"
|
|
26
|
-
PRODUCTS = ("stdmet", "adcp")
|
|
27
|
-
REALTIME_SUFFIX = {"stdmet": "txt", "adcp": "adcp"}
|
|
28
|
-
MONTH_ABBR = [
|
|
29
|
-
"Jan",
|
|
30
|
-
"Feb",
|
|
31
|
-
"Mar",
|
|
32
|
-
"Apr",
|
|
33
|
-
"May",
|
|
34
|
-
"Jun",
|
|
35
|
-
"Jul",
|
|
36
|
-
"Aug",
|
|
37
|
-
"Sep",
|
|
38
|
-
"Oct",
|
|
39
|
-
"Nov",
|
|
40
|
-
"Dec",
|
|
41
|
-
]
|
|
42
|
-
# Column sentinels NDBC uses for "not measured".
|
|
43
|
-
SENTINELS = {99.0, 999.0, 9999.0, 99.00}
|
|
44
|
-
# stdmet columns carried through, with units.
|
|
45
|
-
STDMET_COLUMNS = {
|
|
46
|
-
"WTMP": "degC", # sea surface temperature
|
|
47
|
-
"ATMP": "degC", # air temperature
|
|
48
|
-
"PRES": "hPa", # sea level pressure
|
|
49
|
-
"WSPD": "m s-1", # wind speed
|
|
50
|
-
"WDIR": "degree from true north (direction the wind comes from)",
|
|
51
|
-
"GST": "m s-1",
|
|
52
|
-
"WVHT": "m",
|
|
53
|
-
"DPD": "s",
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
def month_code(month: int) -> str:
|
|
58
|
-
"""NDBC's month code in per-month file names: 1 to 9, then a, b, c for October to December."""
|
|
59
|
-
return str(month) if month < 10 else "abc"[month - 10]
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
def candidate_urls(station: str, product: str, year: int, month: int) -> list[str]:
|
|
63
|
-
"""URLs that may hold `product` for `station` in the given month, in the order to try."""
|
|
64
|
-
station = station.lower()
|
|
65
|
-
return [
|
|
66
|
-
f"{NDBC}/data/{product}/{MONTH_ABBR[month - 1]}/{station}{month_code(month)}{year}.txt.gz",
|
|
67
|
-
f"{NDBC}/data/historical/{product}/{station}h{year}.txt.gz",
|
|
68
|
-
f"{NDBC}/data/realtime2/{station.upper()}.{REALTIME_SUFFIX[product]}",
|
|
69
|
-
]
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def parse_ndbc_text(text: str) -> pd.DataFrame:
|
|
73
|
-
"""Parse an NDBC whitespace-delimited file (two `#` header rows) into a frame indexed by UTC
|
|
74
|
-
time, sentinels and `MM` as NaN. Handles both four- and two-digit years, with or without a
|
|
75
|
-
minute column."""
|
|
76
|
-
lines = [ln for ln in text.splitlines() if ln.strip()]
|
|
77
|
-
if not lines or not lines[0].startswith("#"):
|
|
78
|
-
raise ValueError("not an NDBC data file: no header row")
|
|
79
|
-
names = lines[0].lstrip("#").split()
|
|
80
|
-
rows = [ln.split() for ln in lines if not ln.startswith("#")]
|
|
81
|
-
df = pd.DataFrame(rows, columns=names[: len(rows[0])] if rows else names)
|
|
82
|
-
year_col = names[0]
|
|
83
|
-
years = df[year_col].astype(int)
|
|
84
|
-
years = years.where(years >= 100, years + 1900)
|
|
85
|
-
minutes = df["mm"].astype(int) if "mm" in df else 0
|
|
86
|
-
df.index = pd.to_datetime(
|
|
87
|
-
dict(
|
|
88
|
-
year=years,
|
|
89
|
-
month=df["MM"].astype(int),
|
|
90
|
-
day=df["DD"].astype(int),
|
|
91
|
-
hour=df["hh"].astype(int),
|
|
92
|
-
minute=minutes,
|
|
93
|
-
)
|
|
94
|
-
)
|
|
95
|
-
df = df.drop(columns=[c for c in (year_col, "MM", "DD", "hh", "mm") if c in df])
|
|
96
|
-
df = df.replace("MM", np.nan).apply(pd.to_numeric, errors="coerce")
|
|
97
|
-
return df.mask(df.isin(SENTINELS))
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
def _get(url: str, timeout: float = 60.0) -> Optional[str]:
|
|
101
|
-
r = requests.get(url, timeout=timeout)
|
|
102
|
-
if r.status_code == 404:
|
|
103
|
-
return None
|
|
104
|
-
r.raise_for_status()
|
|
105
|
-
if url.endswith(".gz"):
|
|
106
|
-
return gzip.decompress(r.content).decode("utf-8", errors="replace")
|
|
107
|
-
return r.text
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
def _utc_naive(t) -> pd.Timestamp:
|
|
111
|
-
t = pd.Timestamp(t)
|
|
112
|
-
return t if t.tzinfo is None else t.tz_convert("UTC").tz_localize(None)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def station_position(station: str, get=_get) -> tuple[float, float]:
|
|
116
|
-
"""(lat, lon) of an NDBC station from its station page."""
|
|
117
|
-
html = get(f"{NDBC}/station_page.php?station={station.lower()}") or ""
|
|
118
|
-
m = re.search(r"(\d+\.\d+)\s*([NS])\s+(\d+\.\d+)\s*([EW])", html)
|
|
119
|
-
if not m:
|
|
120
|
-
raise ValueError(f"No position found for NDBC station {station}")
|
|
121
|
-
lat = float(m.group(1)) * (1 if m.group(2) == "N" else -1)
|
|
122
|
-
lon = float(m.group(3)) * (1 if m.group(4) == "E" else -1)
|
|
123
|
-
return lat, lon
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
def fetch_product(
|
|
127
|
-
station: str, product: str, start: pd.Timestamp, end: pd.Timestamp, get=_get
|
|
128
|
-
):
|
|
129
|
-
"""`product` for `station` over [start, end]: (frame, source URLs). Each month is taken from
|
|
130
|
-
the first source that has it."""
|
|
131
|
-
frames, sources = [], []
|
|
132
|
-
for month_start in pd.date_range(start.normalize().replace(day=1), end, freq="MS"):
|
|
133
|
-
for url in candidate_urls(
|
|
134
|
-
station, product, month_start.year, month_start.month
|
|
135
|
-
):
|
|
136
|
-
text = get(url)
|
|
137
|
-
if text is None:
|
|
138
|
-
continue
|
|
139
|
-
df = parse_ndbc_text(text)
|
|
140
|
-
df = df[(df.index >= start) & (df.index <= end)]
|
|
141
|
-
if len(df):
|
|
142
|
-
frames.append(df)
|
|
143
|
-
sources.append(url)
|
|
144
|
-
break
|
|
145
|
-
if not frames:
|
|
146
|
-
return pd.DataFrame(), sources
|
|
147
|
-
df = pd.concat(frames)
|
|
148
|
-
return df[~df.index.duplicated(keep="first")].sort_index(), sources
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def _list(values) -> list:
|
|
152
|
-
return [
|
|
153
|
-
None if (v is None or (isinstance(v, float) and math.isnan(v))) else float(v)
|
|
154
|
-
for v in values
|
|
155
|
-
]
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
def fetch_ndbc(
|
|
159
|
-
station: str,
|
|
160
|
-
start_time: str,
|
|
161
|
-
end_time: str,
|
|
162
|
-
products: tuple[str, ...] = PRODUCTS,
|
|
163
|
-
cache_dir: Optional[str] = None,
|
|
164
|
-
get=_get,
|
|
165
|
-
position: Optional[tuple[float, float]] = None,
|
|
166
|
-
) -> dict:
|
|
167
|
-
"""Observations at an NDBC station over [start_time, end_time] (UTC) as a JSON-ready dict.
|
|
168
|
-
|
|
169
|
-
stdmet: times plus the columns of `STDMET_COLUMNS`. adcp: times, the bin depth and the
|
|
170
|
-
current as eastward and northward components (m/s); NDBC reports speed in cm/s and the
|
|
171
|
-
direction the current flows toward, in degrees true. Missing values are null. A window that
|
|
172
|
-
ended more than two days ago is cached as JSON under `cache_dir`. `get` (a URL to text or
|
|
173
|
-
None) and `position` exist so tests can run without the network.
|
|
174
|
-
"""
|
|
175
|
-
start, end = _utc_naive(start_time), _utc_naive(end_time)
|
|
176
|
-
unknown = set(products) - set(PRODUCTS)
|
|
177
|
-
if unknown:
|
|
178
|
-
raise ValueError(f"unknown NDBC products {sorted(unknown)}")
|
|
179
|
-
|
|
180
|
-
cache_path = None
|
|
181
|
-
if cache_dir:
|
|
182
|
-
os.makedirs(os.path.join(cache_dir, "ndbc"), exist_ok=True)
|
|
183
|
-
key = f"{station.lower()}_{start:%Y%m%dT%H%M}_{end:%Y%m%dT%H%M}_{'-'.join(sorted(products))}"
|
|
184
|
-
cache_path = os.path.join(cache_dir, "ndbc", f"{key}.json")
|
|
185
|
-
if os.path.exists(cache_path):
|
|
186
|
-
with open(cache_path) as f:
|
|
187
|
-
return json.load(f)
|
|
188
|
-
|
|
189
|
-
lat, lon = position if position is not None else station_position(station, get)
|
|
190
|
-
out: dict = {
|
|
191
|
-
"station_id": station.upper(),
|
|
192
|
-
"lat": lat,
|
|
193
|
-
"lon": lon,
|
|
194
|
-
"start_time": f"{start:%Y-%m-%dT%H:%M:%SZ}",
|
|
195
|
-
"end_time": f"{end:%Y-%m-%dT%H:%M:%SZ}",
|
|
196
|
-
"products": {},
|
|
197
|
-
"sources": [],
|
|
198
|
-
}
|
|
199
|
-
for product in products:
|
|
200
|
-
df, sources = fetch_product(station, product, start, end, get=get)
|
|
201
|
-
out["sources"] += sources
|
|
202
|
-
times = [f"{t:%Y-%m-%dT%H:%M:%SZ}" for t in df.index]
|
|
203
|
-
if product == "stdmet":
|
|
204
|
-
out["products"]["stdmet"] = {
|
|
205
|
-
"times": times,
|
|
206
|
-
"units": {c: u for c, u in STDMET_COLUMNS.items() if c in df},
|
|
207
|
-
**{c: _list(df[c]) for c in STDMET_COLUMNS if c in df},
|
|
208
|
-
}
|
|
209
|
-
else:
|
|
210
|
-
speed = df["SPD01"] / 100.0 if "SPD01" in df else pd.Series(dtype=float)
|
|
211
|
-
toward = (
|
|
212
|
-
np.deg2rad(df["DIR01"]) if "DIR01" in df else pd.Series(dtype=float)
|
|
213
|
-
)
|
|
214
|
-
out["products"]["adcp"] = {
|
|
215
|
-
"times": times,
|
|
216
|
-
"depth_m": _list(df["DEP01"]) if "DEP01" in df else [],
|
|
217
|
-
"u": _list(speed * np.sin(toward)),
|
|
218
|
-
"v": _list(speed * np.cos(toward)),
|
|
219
|
-
"units": {
|
|
220
|
-
"u": "m s-1 eastward",
|
|
221
|
-
"v": "m s-1 northward",
|
|
222
|
-
"depth_m": "m",
|
|
223
|
-
},
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
if cache_path and end < pd.Timestamp.utcnow().tz_localize(None) - pd.Timedelta(
|
|
227
|
-
days=2
|
|
228
|
-
):
|
|
229
|
-
with open(cache_path, "w") as f:
|
|
230
|
-
json.dump(out, f)
|
|
231
|
-
return out
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|