forcingkit 0.19.0__tar.gz → 0.20.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.19.0 → forcingkit-0.20.1}/PKG-INFO +1 -1
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/requirements-docs.txt +1 -1
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/conf.py +7 -4
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/fetchers.rst +15 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/pyproject.toml +1 -1
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hfradar.py +329 -11
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/settings.py +7 -0
- forcingkit-0.20.1/tests/unit/test_hfradar.py +559 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/uv.lock +1 -1
- forcingkit-0.19.0/docs/source/_static/custom.css +0 -24
- forcingkit-0.19.0/tests/unit/test_hfradar.py +0 -237
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.agents/forcingkit.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.env.template +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.gitignore +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.markdownlint.yaml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/.python-version +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/AGENTS.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/CONTRIBUTING.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/Dockerfile +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/LICENSE +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/README.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/LICENSE +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/README.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/pyproject.toml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/tests/conftest.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/tests/test_client.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/tests/test_schema.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/client/uv.lock +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docker-compose.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/Makefile +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/make.bat +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/client.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/hycom.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/index.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/necofs.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/rivers.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/sources.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/waves.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/main.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/viewer.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/service/run_server.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrdps.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/ndbc.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/render.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/static/app.js +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/static/favicon.ico +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/static/favicon.svg +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/static/index.html +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/static/logo.svg +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/static/preview3d.js +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/static/styles.css +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrdps.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_main.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.20.1
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -9,13 +9,12 @@ sys.path.insert(0, os.path.abspath("../../src"))
|
|
|
9
9
|
project = "forcingkit"
|
|
10
10
|
copyright = "2026, Long Horizon Observatory"
|
|
11
11
|
author = "Daniel Fry"
|
|
12
|
-
release = "0.
|
|
12
|
+
release = "0.20.1"
|
|
13
13
|
|
|
14
14
|
extensions = [
|
|
15
15
|
"sphinx.ext.autodoc",
|
|
16
16
|
"sphinx.ext.napoleon",
|
|
17
17
|
"sphinx.ext.viewcode",
|
|
18
|
-
"sphinx_rtd_theme",
|
|
19
18
|
"myst_parser",
|
|
20
19
|
]
|
|
21
20
|
|
|
@@ -24,8 +23,12 @@ exclude_patterns: list[str] = []
|
|
|
24
23
|
|
|
25
24
|
language = "en"
|
|
26
25
|
|
|
27
|
-
html_theme = "
|
|
26
|
+
html_theme = "lhzn"
|
|
27
|
+
html_theme_options = {
|
|
28
|
+
"source_repository": "https://github.com/lhzn-io/forcingkit",
|
|
29
|
+
"source_branch": "main",
|
|
30
|
+
"source_directory": "docs/source/",
|
|
31
|
+
}
|
|
28
32
|
html_static_path = ["_static"]
|
|
29
|
-
html_css_files = ["custom.css"]
|
|
30
33
|
# CNAME for the custom domain forcingkit.docs.lhzn.io, copied to the site root.
|
|
31
34
|
html_extra_path = ["_extra"]
|
|
@@ -189,9 +189,24 @@ These serve validation rather than forcing:
|
|
|
189
189
|
with ``has`` and ``put`` (a create-only bucket, for example). ``load`` reads a box and window back
|
|
190
190
|
in m/s from any archived box that contains it, and the other days from the server while it holds
|
|
191
191
|
them. The ``hfradar`` directory is an archive, not a cache: do not purge it with the Zarr stores.
|
|
192
|
+
``FORCINGKIT_HFRADAR_ARCHIVE`` puts it elsewhere (since 0.20.0).
|
|
192
193
|
Coverage is uneven: in western Long Island Sound about 2 % of the 1 km cells in the box
|
|
193
194
|
40.85-41.3 N, 73.8-71.8 W have a solution in a covered hour, and whole weeks have none.
|
|
194
195
|
|
|
196
|
+
Older days come from `NOAA NCEI <https://www.ncei.noaa.gov/>`__, which archives the same hourly
|
|
197
|
+
totals (the same variables, stored the same way, on the same grids) as one file per hour in monthly
|
|
198
|
+
accessions, from about two months back to the network's start. ``archive_ncei`` finds a window's
|
|
199
|
+
accessions through NCEI's metadata search, reads each hour's file (about 0.8 MB at 1 km for the whole
|
|
200
|
+
coast), cuts the box out and writes the same daily files, so ``load`` reads days from either source
|
|
201
|
+
alike; ``source_dataset`` names the accession. Days already kept are skipped, and a day is written
|
|
202
|
+
with the hours NCEI holds (since 0.20.0). The month list and folders found are kept in
|
|
203
|
+
``ncei_accessions.json`` beside the archive, so a rerun skips NCEI's slow search (since 0.20.1).
|
|
204
|
+
From the command line::
|
|
205
|
+
|
|
206
|
+
python -m forcingkit.fetchers.hfradar ncei --box -73.8 40.85 -71.8 41.3 \
|
|
207
|
+
--start 2025-01-01 --end 2026-09-01 [--grids 1km 2km] [--root DIR] [--workers 4]
|
|
208
|
+
python -m forcingkit.fetchers.hfradar ndbc --box -73.8 40.85 -71.8 41.3 --start all [--root DIR]
|
|
209
|
+
|
|
195
210
|
NOAA OFS archive locations
|
|
196
211
|
--------------------------
|
|
197
212
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.20.1"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -24,17 +24,32 @@ kept, with no screening. Unlike the Zarr stores beside it, this directory is an
|
|
|
24
24
|
those days again once they leave its window, so do not purge it. `load` reads a box and window back, decoded to m/s,
|
|
25
25
|
from any archived box that contains it, and fetches days the archive lacks from the server while it still holds them.
|
|
26
26
|
|
|
27
|
+
Older days come from NOAA NCEI, which archives the same hourly totals (the same variables, stored the same way, on
|
|
28
|
+
the same grids) as one file per hour in monthly accessions. `archive_ncei` finds a window's accessions through NCEI's
|
|
29
|
+
metadata search, reads each hour's file, cuts the box out and writes the same daily files, so `load` reads both
|
|
30
|
+
alike; their ``source_dataset`` names the accession. NCEI's accessions lag the present by a month or two; NDBC's
|
|
31
|
+
window covers the rest. The archive's root is ``FORCINGKIT_HFRADAR_ARCHIVE`` when set (see `settings.hfradar_archive`).
|
|
32
|
+
|
|
33
|
+
From the command line::
|
|
34
|
+
|
|
35
|
+
python -m forcingkit.fetchers.hfradar ncei --box W S E N --start 2025-01-01 --end 2026-09-01 [--root DIR]
|
|
36
|
+
python -m forcingkit.fetchers.hfradar ndbc --box W S E N [--start all] [--root DIR]
|
|
37
|
+
|
|
27
38
|
Credit: the radar sites' operators (named in HFRNet's site list; for example the University of Connecticut's LISICOS
|
|
28
39
|
for western Long Island Sound) and the NOAA IOOS HF Radar National Network; see ``source_credit`` in each file.
|
|
29
40
|
"""
|
|
30
41
|
|
|
31
42
|
from __future__ import annotations
|
|
32
43
|
|
|
44
|
+
import argparse
|
|
33
45
|
import io
|
|
46
|
+
import json
|
|
34
47
|
import logging
|
|
35
48
|
import os
|
|
36
49
|
import re
|
|
50
|
+
import time as _time
|
|
37
51
|
from collections.abc import Sequence
|
|
52
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
38
53
|
from datetime import datetime, timedelta, timezone
|
|
39
54
|
from pathlib import Path
|
|
40
55
|
from typing import Protocol
|
|
@@ -94,6 +109,22 @@ CREDIT = (
|
|
|
94
109
|
"HF radar sites operated by their institutions (for western Long Island Sound, the University of Connecticut's "
|
|
95
110
|
"LISICOS); hourly total vectors from the NOAA IOOS HF Radar National Network (HFRNet), served by NOAA NDBC."
|
|
96
111
|
)
|
|
112
|
+
NCEI_CREDIT = CREDIT.replace("served by NOAA NDBC.", "archived by NOAA NCEI.")
|
|
113
|
+
NCEI = "https://www.ncei.noaa.gov"
|
|
114
|
+
NCEI_SEARCH = f"{NCEI}/metadata/granule/geoportal/opensearch"
|
|
115
|
+
NCEI_ARCHIVE = f"{NCEI}/data/oceans/archive/"
|
|
116
|
+
# The metadata series holding the hourly totals (the Radial series beside them hold each site's radials).
|
|
117
|
+
NCEI_SERIES = ("IOOS-HFRadarRTVector", "NDBC-HFRadarRTVector")
|
|
118
|
+
# Beside the archive: {"months": {"YYYY-MM": accession}, "accessions": {accession: folder}}, so the metadata
|
|
119
|
+
# search (slow: minutes) and each folder lookup run once.
|
|
120
|
+
NCEI_INDEX = "ncei_accessions.json"
|
|
121
|
+
_NCEI_TITLE = re.compile(r"during ([A-Z][a-z]+) (\d{4}) \(NCEI Accession (\d+)\)")
|
|
122
|
+
# Hourly file names: since 2026 ``rtv-usegc-<grid>-uwls_v1r0_hfr_s<YYYYmmddHHMM>..._c<created>.nc`` (a reissue
|
|
123
|
+
# has a later ``c``); before, ``<YYYYmmddHHMM>_hfr_usegc_<grid>_rtv_uwls_NDBC.nc``.
|
|
124
|
+
_NCEI_FILE = re.compile(
|
|
125
|
+
r'href="(rtv-usegc-([126]km)-[^"/]*_s(\d{12})\d*_e\d+_c(\d+)\.nc)"'
|
|
126
|
+
)
|
|
127
|
+
_NCEI_FILE_OLD = re.compile(r'href="((\d{12})_hfr_usegc_([126]km)_rtv_[^"/]*\.nc)"')
|
|
97
128
|
NAME = re.compile(
|
|
98
129
|
r"usegc_[126]km/\d{2}\.\d{3}[NS]_\d{2}\.\d{3}[NS]_\d{3}\.\d{3}[EW]_\d{3}\.\d{3}[EW]/\d{4}/\d{2}/"
|
|
99
130
|
r"hfradar_usegc_[126]km_\d{8}\.nc"
|
|
@@ -118,9 +149,7 @@ class LocalStore:
|
|
|
118
149
|
"""A directory; files are created once and never replaced."""
|
|
119
150
|
|
|
120
151
|
def __init__(self, root: str | os.PathLike | None = None):
|
|
121
|
-
self.root = Path(
|
|
122
|
-
root if root is not None else settings.cache_dir("hfradar")
|
|
123
|
-
).expanduser()
|
|
152
|
+
self.root = Path(root if root is not None else archive_root()).expanduser()
|
|
124
153
|
|
|
125
154
|
def has(self, name: str) -> bool:
|
|
126
155
|
return (self.root / name).exists()
|
|
@@ -138,6 +167,11 @@ class LocalStore:
|
|
|
138
167
|
return True
|
|
139
168
|
|
|
140
169
|
|
|
170
|
+
def archive_root() -> str:
|
|
171
|
+
"""The archive's default root: FORCINGKIT_HFRADAR_ARCHIVE, else the cache's ``hfradar`` directory."""
|
|
172
|
+
return settings.hfradar_archive() or settings.cache_dir("hfradar")
|
|
173
|
+
|
|
174
|
+
|
|
141
175
|
def dataset_url(grid: str) -> str:
|
|
142
176
|
return f"{SERVER}/hfradar_usegc_{grid}"
|
|
143
177
|
|
|
@@ -257,9 +291,17 @@ def _stored(var: xr.DataArray) -> tuple[np.ndarray, dict]:
|
|
|
257
291
|
|
|
258
292
|
|
|
259
293
|
def _normalize(
|
|
260
|
-
sub: xr.Dataset,
|
|
294
|
+
sub: xr.Dataset,
|
|
295
|
+
grid: str,
|
|
296
|
+
bbox: BBox,
|
|
297
|
+
source_attrs: dict,
|
|
298
|
+
fetched: datetime,
|
|
299
|
+
source: str | None = None,
|
|
300
|
+
credit: str = CREDIT,
|
|
261
301
|
) -> xr.Dataset:
|
|
262
|
-
"""The server's subset as a clean Dataset of stored values, with its encodings and credit.
|
|
302
|
+
"""The server's subset as a clean Dataset of stored values, with its encodings and credit. ``source`` names
|
|
303
|
+
where it came from (default: NDBC's dataset)."""
|
|
304
|
+
source = source or dataset_url(grid)
|
|
263
305
|
out = xr.Dataset(
|
|
264
306
|
coords={
|
|
265
307
|
"time": sub["time"].values,
|
|
@@ -294,10 +336,10 @@ def _normalize(
|
|
|
294
336
|
w, s, e, n = bbox
|
|
295
337
|
attrs.update(
|
|
296
338
|
{
|
|
297
|
-
"source_dataset":
|
|
298
|
-
"source_credit":
|
|
339
|
+
"source_dataset": source,
|
|
340
|
+
"source_credit": credit,
|
|
299
341
|
"subset": f"latitude {s} to {n}, longitude {w} to {e}; values as stored on the server",
|
|
300
|
-
"history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {
|
|
342
|
+
"history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {source} by forcingkit",
|
|
301
343
|
}
|
|
302
344
|
)
|
|
303
345
|
out.attrs = attrs
|
|
@@ -451,9 +493,7 @@ def load(
|
|
|
451
493
|
"""The box's hours in [start, end), decoded (u and v in m/s, NaN where there is no solution): archived days from
|
|
452
494
|
any archived box that contains this one, and, when ``remote``, the other days from the server while it still
|
|
453
495
|
holds them (not archived by this call). Empty in time when neither has any."""
|
|
454
|
-
base = Path(
|
|
455
|
-
root if root is not None else settings.cache_dir("hfradar")
|
|
456
|
-
).expanduser()
|
|
496
|
+
base = Path(root if root is not None else archive_root()).expanduser()
|
|
457
497
|
t0, t1 = _utc_naive(start), _utc_naive(end)
|
|
458
498
|
boxes = _containing_boxes(base, grid, bbox)
|
|
459
499
|
parts: list[xr.Dataset] = []
|
|
@@ -498,3 +538,281 @@ def load(
|
|
|
498
538
|
out = out.sel(time=slice(t0, t1 - pd.Timedelta(seconds=1)))
|
|
499
539
|
out.attrs = {"source_dataset": dataset_url(grid), "source_credit": CREDIT}
|
|
500
540
|
return out
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
# ---- NCEI's monthly archive ----
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _client(session: requests.Session | None) -> requests.Session:
|
|
547
|
+
if session is None:
|
|
548
|
+
session = requests.Session()
|
|
549
|
+
session.headers["User-Agent"] = USER_AGENT
|
|
550
|
+
return session
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _get(
|
|
554
|
+
session: requests.Session, url: str, tries: int = 4, **kw
|
|
555
|
+
) -> requests.Response:
|
|
556
|
+
"""GET with a few retries and backoff (NCEI answers a burst with the odd 5xx or reset)."""
|
|
557
|
+
for k in range(tries):
|
|
558
|
+
try:
|
|
559
|
+
r = session.get(url, timeout=120, **kw)
|
|
560
|
+
if r.status_code < 500:
|
|
561
|
+
r.raise_for_status()
|
|
562
|
+
return r
|
|
563
|
+
err: Exception = requests.HTTPError(f"{r.status_code} for {url}")
|
|
564
|
+
except (requests.ConnectionError, requests.Timeout) as exc:
|
|
565
|
+
err = exc
|
|
566
|
+
if k < tries - 1:
|
|
567
|
+
_time.sleep(2**k)
|
|
568
|
+
raise err
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def ncei_months(
|
|
572
|
+
start, end, session: requests.Session | None = None
|
|
573
|
+
) -> dict[pd.Timestamp, str]:
|
|
574
|
+
"""NCEI's accessions of hourly totals for the months that overlap [start, end): {first of month: accession}.
|
|
575
|
+
A month listed twice keeps the higher (later) accession number."""
|
|
576
|
+
session = _client(session)
|
|
577
|
+
first = _utc_naive(start).to_period("M").to_timestamp()
|
|
578
|
+
last = (_utc_naive(end) - pd.Timedelta(seconds=1)).to_period("M").to_timestamp()
|
|
579
|
+
found: dict[pd.Timestamp, str] = {}
|
|
580
|
+
for series in NCEI_SERIES:
|
|
581
|
+
at = 1
|
|
582
|
+
while True:
|
|
583
|
+
page = _get(
|
|
584
|
+
session,
|
|
585
|
+
NCEI_SEARCH,
|
|
586
|
+
params={"q": series, "start": at, "num": 200, "f": "json"},
|
|
587
|
+
).json()
|
|
588
|
+
for item in page.get("results", []):
|
|
589
|
+
if not str(item.get("id", "")).startswith(series + "."):
|
|
590
|
+
continue
|
|
591
|
+
m = _NCEI_TITLE.search(item.get("title", ""))
|
|
592
|
+
if not m:
|
|
593
|
+
continue
|
|
594
|
+
month = pd.Timestamp(f"{m.group(1)} 1 {m.group(2)}")
|
|
595
|
+
if first <= month <= last and int(m.group(3)) > int(
|
|
596
|
+
found.get(month, "0")
|
|
597
|
+
):
|
|
598
|
+
found[month] = m.group(3)
|
|
599
|
+
nxt = page.get("nextStart")
|
|
600
|
+
if not nxt or nxt <= at or nxt > page.get("total", 0):
|
|
601
|
+
break
|
|
602
|
+
at = nxt
|
|
603
|
+
return dict(sorted(found.items()))
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def ncei_accession_dir(
|
|
607
|
+
accession: str, session: requests.Session | None = None, known: dict | None = None
|
|
608
|
+
) -> str:
|
|
609
|
+
"""The folder of an accession's US East and Gulf Coast totals. NCEI files accessions under numbered ``arc``
|
|
610
|
+
folders; the newest are searched first. ``known`` (accession -> folder) is consulted and updated."""
|
|
611
|
+
if known is not None and accession in known:
|
|
612
|
+
return known[accession]
|
|
613
|
+
session = _client(session)
|
|
614
|
+
arcs = sorted(
|
|
615
|
+
set(re.findall(r'href="(arc\d+)/"', _get(session, NCEI_ARCHIVE).text)),
|
|
616
|
+
reverse=True,
|
|
617
|
+
)
|
|
618
|
+
for arc in arcs:
|
|
619
|
+
if f'href="{accession}/"' in _get(session, f"{NCEI_ARCHIVE}{arc}/").text:
|
|
620
|
+
url = f"{NCEI_ARCHIVE}{arc}/{accession}/1.1/data/0-data/USEGC/"
|
|
621
|
+
if known is not None:
|
|
622
|
+
known[accession] = url
|
|
623
|
+
return url
|
|
624
|
+
raise RuntimeError(f"NCEI accession {accession} not found under {NCEI_ARCHIVE}")
|
|
625
|
+
|
|
626
|
+
|
|
627
|
+
def ncei_hours(
|
|
628
|
+
folder: str, grid: str, session: requests.Session | None = None
|
|
629
|
+
) -> dict[pd.Timestamp, str]:
|
|
630
|
+
"""The hourly files of one grid in an accession's folder: {hour: URL}; a reissued hour keeps the latest file."""
|
|
631
|
+
best: dict[pd.Timestamp, tuple[str, str]] = {}
|
|
632
|
+
text = _get(_client(session), folder).text
|
|
633
|
+
found = _NCEI_FILE.findall(text) + [
|
|
634
|
+
(name, g, start, "") for name, start, g in _NCEI_FILE_OLD.findall(text)
|
|
635
|
+
]
|
|
636
|
+
for name, g, start, created in found:
|
|
637
|
+
if g != grid:
|
|
638
|
+
continue
|
|
639
|
+
hour = pd.Timestamp(datetime.strptime(start, "%Y%m%d%H%M"))
|
|
640
|
+
if hour not in best or created > best[hour][1]:
|
|
641
|
+
best[hour] = (name, created)
|
|
642
|
+
return {h: folder + n for h, (n, _) in sorted(best.items())}
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def _ncei_hour(
|
|
646
|
+
session: requests.Session, url: str, bbox: BBox
|
|
647
|
+
) -> tuple[xr.Dataset, dict]:
|
|
648
|
+
"""One hourly file, the box cut out, values as stored."""
|
|
649
|
+
data = _get(session, url).content
|
|
650
|
+
with xr.open_dataset(
|
|
651
|
+
io.BytesIO(data), engine="h5netcdf", mask_and_scale=False
|
|
652
|
+
) as ds:
|
|
653
|
+
ys, xs = box_slices(ds["lat"].values, ds["lon"].values, bbox)
|
|
654
|
+
names = [n for n in VARIABLES + SCALARS if n in ds]
|
|
655
|
+
return ds[names].isel(lat=ys, lon=xs).load(), dict(ds.attrs)
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def _read_index(store: Store) -> tuple[dict, Path | None]:
|
|
659
|
+
"""The index beside a directory store, and its path (an empty index and None for other stores). 0.20.0 kept
|
|
660
|
+
the accessions only, as a flat map."""
|
|
661
|
+
root = getattr(store, "root", None)
|
|
662
|
+
if root is None:
|
|
663
|
+
return {"months": {}, "accessions": {}}, None
|
|
664
|
+
path = Path(root) / NCEI_INDEX
|
|
665
|
+
try:
|
|
666
|
+
data = json.loads(path.read_text())
|
|
667
|
+
except (OSError, ValueError):
|
|
668
|
+
data = {}
|
|
669
|
+
if "accessions" not in data:
|
|
670
|
+
data = {"months": {}, "accessions": data}
|
|
671
|
+
data.setdefault("months", {})
|
|
672
|
+
return data, path
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def _write_index(path: Path | None, index: dict) -> None:
|
|
676
|
+
if path is not None:
|
|
677
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
678
|
+
path.write_text(json.dumps(index, indent=1, sort_keys=True))
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def archive_ncei(
|
|
682
|
+
bbox: BBox,
|
|
683
|
+
start,
|
|
684
|
+
end,
|
|
685
|
+
grids: Sequence[str] = GRIDS,
|
|
686
|
+
store: Store | None = None,
|
|
687
|
+
session: requests.Session | None = None,
|
|
688
|
+
workers: int = 4,
|
|
689
|
+
) -> dict[str, list[str]]:
|
|
690
|
+
"""Archive the box's days in [start, end) from NCEI's monthly accessions, per grid, skipping days already kept
|
|
691
|
+
(from either source); return the names written per grid. A day is written with the hours NCEI holds for it.
|
|
692
|
+
Days in months NCEI has not published are left for NDBC. A day that fails is skipped and the rest carry on; the
|
|
693
|
+
call then raises, naming the days missed."""
|
|
694
|
+
session = _client(session)
|
|
695
|
+
store = store if store is not None else LocalStore()
|
|
696
|
+
t0, t1 = _utc_naive(start).normalize(), _utc_naive(end)
|
|
697
|
+
days = pd.date_range(t0, (t1 - pd.Timedelta(seconds=1)).normalize(), freq="D")
|
|
698
|
+
index, index_path = _read_index(store)
|
|
699
|
+
want = pd.period_range(t0, t1 - pd.Timedelta(seconds=1), freq="M")
|
|
700
|
+
if all(f"{m}" in index["months"] for m in want):
|
|
701
|
+
months = {m.to_timestamp(): index["months"][f"{m}"] for m in want}
|
|
702
|
+
else: # a month not listed yet: search (months NCEI has not published stay unlisted)
|
|
703
|
+
months = ncei_months(t0, t1, session)
|
|
704
|
+
index["months"].update({f"{m:%Y-%m}": a for m, a in months.items()})
|
|
705
|
+
_write_index(index_path, index)
|
|
706
|
+
known = index["accessions"]
|
|
707
|
+
written: dict[str, list[str]] = {g: [] for g in grids}
|
|
708
|
+
failed: list[str] = []
|
|
709
|
+
for grid in grids:
|
|
710
|
+
for month, accession in months.items():
|
|
711
|
+
todo = [
|
|
712
|
+
d
|
|
713
|
+
for d in days
|
|
714
|
+
if d.to_period("M").to_timestamp() == month
|
|
715
|
+
and not store.has(day_name(grid, bbox, d))
|
|
716
|
+
]
|
|
717
|
+
if not todo:
|
|
718
|
+
continue
|
|
719
|
+
try:
|
|
720
|
+
folder = ncei_accession_dir(accession, session, known)
|
|
721
|
+
hours = ncei_hours(folder, grid, session)
|
|
722
|
+
except Exception as exc:
|
|
723
|
+
logger.warning(
|
|
724
|
+
"NCEI %s %s (%s): %s", grid, f"{month:%Y-%m}", accession, exc
|
|
725
|
+
)
|
|
726
|
+
failed.append(f"{grid} {month:%Y-%m}")
|
|
727
|
+
continue
|
|
728
|
+
_write_index(index_path, index)
|
|
729
|
+
for day in todo:
|
|
730
|
+
urls = [u for h, u in hours.items() if h.normalize() == day]
|
|
731
|
+
if not urls:
|
|
732
|
+
logger.info(
|
|
733
|
+
"NCEI %s %s: no hours in accession %s",
|
|
734
|
+
grid,
|
|
735
|
+
day.date(),
|
|
736
|
+
accession,
|
|
737
|
+
)
|
|
738
|
+
continue
|
|
739
|
+
try:
|
|
740
|
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
741
|
+
got = list(
|
|
742
|
+
pool.map(lambda u: _ncei_hour(session, u, bbox), urls)
|
|
743
|
+
)
|
|
744
|
+
except Exception as exc: # carry on; a later run fills the gap
|
|
745
|
+
logger.warning("NCEI %s %s: %s", grid, day.date(), exc)
|
|
746
|
+
failed.append(f"{grid} {day:%Y-%m-%d}")
|
|
747
|
+
continue
|
|
748
|
+
sub = xr.concat(
|
|
749
|
+
[g for g, _ in got],
|
|
750
|
+
dim="time",
|
|
751
|
+
data_vars="minimal",
|
|
752
|
+
coords="minimal",
|
|
753
|
+
compat="override",
|
|
754
|
+
join="exact",
|
|
755
|
+
).sortby("time")
|
|
756
|
+
ds = _normalize(
|
|
757
|
+
sub,
|
|
758
|
+
grid,
|
|
759
|
+
bbox,
|
|
760
|
+
got[0][1],
|
|
761
|
+
datetime.now(timezone.utc),
|
|
762
|
+
source=f"{NCEI}/archive/accession/{accession} ({folder})",
|
|
763
|
+
credit=NCEI_CREDIT,
|
|
764
|
+
)
|
|
765
|
+
name = day_name(grid, bbox, day)
|
|
766
|
+
if store.put(name, day_file(ds, day)):
|
|
767
|
+
written[grid].append(name)
|
|
768
|
+
logger.info(
|
|
769
|
+
"%s written (%d h, NCEI %s)", name, len(urls), accession
|
|
770
|
+
)
|
|
771
|
+
if failed:
|
|
772
|
+
raise RuntimeError(
|
|
773
|
+
f"HF radar days not read from NCEI: {', '.join(failed)} (written: {written})"
|
|
774
|
+
)
|
|
775
|
+
return written
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def main(argv: Sequence[str] | None = None) -> None:
|
|
779
|
+
"""Archive a box from NDBC (the last ~3 months) or NCEI (older months)."""
|
|
780
|
+
p = argparse.ArgumentParser(
|
|
781
|
+
prog="python -m forcingkit.fetchers.hfradar", description=main.__doc__
|
|
782
|
+
)
|
|
783
|
+
p.add_argument("source", choices=["ndbc", "ncei"])
|
|
784
|
+
p.add_argument(
|
|
785
|
+
"--box", nargs=4, type=float, required=True, metavar=("W", "S", "E", "N")
|
|
786
|
+
)
|
|
787
|
+
p.add_argument(
|
|
788
|
+
"--start", help="first day (ndbc: or 'all'; default the last 10 days)"
|
|
789
|
+
)
|
|
790
|
+
p.add_argument("--end", help="ncei: the day after the last; ndbc: the last day")
|
|
791
|
+
p.add_argument("--grids", nargs="+", default=list(GRIDS), choices=list(GRIDS))
|
|
792
|
+
p.add_argument(
|
|
793
|
+
"--root",
|
|
794
|
+
help="the archive's root (default: FORCINGKIT_HFRADAR_ARCHIVE, else the cache)",
|
|
795
|
+
)
|
|
796
|
+
p.add_argument(
|
|
797
|
+
"--workers", type=int, default=4, help="ncei: hourly files read at once"
|
|
798
|
+
)
|
|
799
|
+
p.add_argument("-v", "--verbose", action="store_true")
|
|
800
|
+
a = p.parse_args(argv)
|
|
801
|
+
logging.basicConfig(
|
|
802
|
+
level=logging.INFO if a.verbose else logging.WARNING,
|
|
803
|
+
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
804
|
+
)
|
|
805
|
+
store = LocalStore(a.root)
|
|
806
|
+
if a.source == "ncei":
|
|
807
|
+
if not (a.start and a.end):
|
|
808
|
+
p.error("ncei needs --start and --end")
|
|
809
|
+
out = archive_ncei(
|
|
810
|
+
a.box, a.start, a.end, a.grids, store=store, workers=a.workers
|
|
811
|
+
)
|
|
812
|
+
else:
|
|
813
|
+
out = archive(a.box, a.start, a.end, a.grids, store=store)
|
|
814
|
+
print(json.dumps({g: len(n) for g, n in out.items()}))
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
if __name__ == "__main__":
|
|
818
|
+
main()
|
|
@@ -82,6 +82,13 @@ def nwps_archive() -> str | None:
|
|
|
82
82
|
return env("FORCINGKIT_NWPS_ARCHIVE") or None
|
|
83
83
|
|
|
84
84
|
|
|
85
|
+
def hfradar_archive() -> str | None:
|
|
86
|
+
"""Where the HF radar archive lives (FORCINGKIT_HFRADAR_ARCHIVE), when it should not sit in the cache: the
|
|
87
|
+
archive keeps days the servers no longer hold, so a cache purge must never reach it. Unset or empty means
|
|
88
|
+
``cache_dir("hfradar")``."""
|
|
89
|
+
return env("FORCINGKIT_HFRADAR_ARCHIVE") or None
|
|
90
|
+
|
|
91
|
+
|
|
85
92
|
def erddap_timeouts() -> tuple[float, float]:
|
|
86
93
|
"""(connect, read) timeouts in seconds for ERDDAP profile reads
|
|
87
94
|
(FORCINGKIT_ERDDAP_CONNECT_TIMEOUT, default 10; FORCINGKIT_ERDDAP_READ_TIMEOUT, default 60)."""
|