forcingkit 0.19.0__tar.gz → 0.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.19.0 → forcingkit-0.20.0}/PKG-INFO +1 -1
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/requirements-docs.txt +1 -1
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/conf.py +7 -4
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/fetchers.rst +13 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/pyproject.toml +1 -1
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hfradar.py +305 -11
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/settings.py +7 -0
- forcingkit-0.20.0/tests/unit/test_hfradar.py +505 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/uv.lock +1 -1
- forcingkit-0.19.0/docs/source/_static/custom.css +0 -24
- forcingkit-0.19.0/tests/unit/test_hfradar.py +0 -237
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.agents/forcingkit.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.env.template +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.gitignore +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.markdownlint.yaml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/.python-version +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/AGENTS.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/CONTRIBUTING.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/Dockerfile +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/LICENSE +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/README.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/LICENSE +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/README.md +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/pyproject.toml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/tests/conftest.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/tests/test_client.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/tests/test_schema.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/client/uv.lock +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docker-compose.yml +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/Makefile +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/make.bat +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/client.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/hycom.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/index.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/necofs.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/rivers.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/sources.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/waves.rst +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/main.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/viewer.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/service/run_server.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrdps.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/ndbc.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/render.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/static/app.js +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/static/favicon.ico +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/static/favicon.svg +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/static/index.html +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/static/logo.svg +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/static/preview3d.js +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/static/styles.css +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/__init__.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrdps.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_main.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.20.0
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -9,13 +9,12 @@ sys.path.insert(0, os.path.abspath("../../src"))
|
|
|
9
9
|
project = "forcingkit"
|
|
10
10
|
copyright = "2026, Long Horizon Observatory"
|
|
11
11
|
author = "Daniel Fry"
|
|
12
|
-
release = "0.
|
|
12
|
+
release = "0.20.0"
|
|
13
13
|
|
|
14
14
|
extensions = [
|
|
15
15
|
"sphinx.ext.autodoc",
|
|
16
16
|
"sphinx.ext.napoleon",
|
|
17
17
|
"sphinx.ext.viewcode",
|
|
18
|
-
"sphinx_rtd_theme",
|
|
19
18
|
"myst_parser",
|
|
20
19
|
]
|
|
21
20
|
|
|
@@ -24,8 +23,12 @@ exclude_patterns: list[str] = []
|
|
|
24
23
|
|
|
25
24
|
language = "en"
|
|
26
25
|
|
|
27
|
-
html_theme = "
|
|
26
|
+
html_theme = "lhzn"
|
|
27
|
+
html_theme_options = {
|
|
28
|
+
"source_repository": "https://github.com/lhzn-io/forcingkit",
|
|
29
|
+
"source_branch": "main",
|
|
30
|
+
"source_directory": "docs/source/",
|
|
31
|
+
}
|
|
28
32
|
html_static_path = ["_static"]
|
|
29
|
-
html_css_files = ["custom.css"]
|
|
30
33
|
# CNAME for the custom domain forcingkit.docs.lhzn.io, copied to the site root.
|
|
31
34
|
html_extra_path = ["_extra"]
|
|
@@ -189,6 +189,19 @@ These serve validation rather than forcing:
|
|
|
189
189
|
with ``has`` and ``put`` (a create-only bucket, for example). ``load`` reads a box and window back
|
|
190
190
|
in m/s from any archived box that contains it, and the other days from the server while it holds
|
|
191
191
|
them. The ``hfradar`` directory is an archive, not a cache: do not purge it with the Zarr stores.
|
|
192
|
+
``FORCINGKIT_HFRADAR_ARCHIVE`` puts it elsewhere (since 0.20.0).
|
|
193
|
+
|
|
194
|
+
Older days come from `NOAA NCEI <https://www.ncei.noaa.gov/>`__, which archives the same hourly
|
|
195
|
+
totals (the same variables, stored the same way, on the same grids) as one file per hour in monthly
|
|
196
|
+
accessions, from about two months back to the network's start. ``archive_ncei`` finds a window's
|
|
197
|
+
accessions through NCEI's metadata search, reads each hour's file (about 0.8 MB at 1 km for the whole
|
|
198
|
+
coast), cuts the box out and writes the same daily files, so ``load`` reads days from either source
|
|
199
|
+
alike; ``source_dataset`` names the accession. Days already kept are skipped, and a day is written
|
|
200
|
+
with the hours NCEI holds (since 0.20.0). From the command line::
|
|
201
|
+
|
|
202
|
+
python -m forcingkit.fetchers.hfradar ncei --box -73.8 40.85 -71.8 41.3 \
|
|
203
|
+
--start 2025-01-01 --end 2026-09-01 [--grids 1km 2km] [--root DIR] [--workers 4]
|
|
204
|
+
python -m forcingkit.fetchers.hfradar ndbc --box -73.8 40.85 -71.8 41.3 --start all [--root DIR]
|
|
192
205
|
Coverage is uneven: in western Long Island Sound about 2 % of the 1 km cells in the box
|
|
193
206
|
40.85-41.3 N, 73.8-71.8 W have a solution in a covered hour, and whole weeks have none.
|
|
194
207
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.20.0"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -24,17 +24,32 @@ kept, with no screening. Unlike the Zarr stores beside it, this directory is an
|
|
|
24
24
|
those days again once they leave its window, so do not purge it. `load` reads a box and window back, decoded to m/s,
|
|
25
25
|
from any archived box that contains it, and fetches days the archive lacks from the server while it still holds them.
|
|
26
26
|
|
|
27
|
+
Older days come from NOAA NCEI, which archives the same hourly totals (the same variables, stored the same way, on
|
|
28
|
+
the same grids) as one file per hour in monthly accessions. `archive_ncei` finds a window's accessions through NCEI's
|
|
29
|
+
metadata search, reads each hour's file, cuts the box out and writes the same daily files, so `load` reads both
|
|
30
|
+
alike; their ``source_dataset`` names the accession. NCEI's accessions lag the present by a month or two; NDBC's
|
|
31
|
+
window covers the rest. The archive's root is ``FORCINGKIT_HFRADAR_ARCHIVE`` when set (see `settings.hfradar_archive`).
|
|
32
|
+
|
|
33
|
+
From the command line::
|
|
34
|
+
|
|
35
|
+
python -m forcingkit.fetchers.hfradar ncei --box W S E N --start 2025-01-01 --end 2026-09-01 [--root DIR]
|
|
36
|
+
python -m forcingkit.fetchers.hfradar ndbc --box W S E N [--start all] [--root DIR]
|
|
37
|
+
|
|
27
38
|
Credit: the radar sites' operators (named in HFRNet's site list; for example the University of Connecticut's LISICOS
|
|
28
39
|
for western Long Island Sound) and the NOAA IOOS HF Radar National Network; see ``source_credit`` in each file.
|
|
29
40
|
"""
|
|
30
41
|
|
|
31
42
|
from __future__ import annotations
|
|
32
43
|
|
|
44
|
+
import argparse
|
|
33
45
|
import io
|
|
46
|
+
import json
|
|
34
47
|
import logging
|
|
35
48
|
import os
|
|
36
49
|
import re
|
|
50
|
+
import time as _time
|
|
37
51
|
from collections.abc import Sequence
|
|
52
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
38
53
|
from datetime import datetime, timedelta, timezone
|
|
39
54
|
from pathlib import Path
|
|
40
55
|
from typing import Protocol
|
|
@@ -94,6 +109,17 @@ CREDIT = (
|
|
|
94
109
|
"HF radar sites operated by their institutions (for western Long Island Sound, the University of Connecticut's "
|
|
95
110
|
"LISICOS); hourly total vectors from the NOAA IOOS HF Radar National Network (HFRNet), served by NOAA NDBC."
|
|
96
111
|
)
|
|
112
|
+
NCEI_CREDIT = CREDIT.replace("served by NOAA NDBC.", "archived by NOAA NCEI.")
|
|
113
|
+
NCEI = "https://www.ncei.noaa.gov"
|
|
114
|
+
NCEI_SEARCH = f"{NCEI}/metadata/granule/geoportal/opensearch"
|
|
115
|
+
NCEI_ARCHIVE = f"{NCEI}/data/oceans/archive/"
|
|
116
|
+
# The metadata series holding the hourly totals (the Radial series beside them hold each site's radials).
|
|
117
|
+
NCEI_SERIES = ("IOOS-HFRadarRTVector", "NDBC-HFRadarRTVector")
|
|
118
|
+
NCEI_INDEX = "ncei_accessions.json" # accession -> its folder, beside the archive, so each is looked up once
|
|
119
|
+
_NCEI_TITLE = re.compile(r"during ([A-Z][a-z]+) (\d{4}) \(NCEI Accession (\d+)\)")
|
|
120
|
+
_NCEI_FILE = re.compile(
|
|
121
|
+
r'href="(rtv-usegc-([126]km)-[^"/]*_s(\d{12})\d*_e\d+_c(\d+)\.nc)"'
|
|
122
|
+
)
|
|
97
123
|
NAME = re.compile(
|
|
98
124
|
r"usegc_[126]km/\d{2}\.\d{3}[NS]_\d{2}\.\d{3}[NS]_\d{3}\.\d{3}[EW]_\d{3}\.\d{3}[EW]/\d{4}/\d{2}/"
|
|
99
125
|
r"hfradar_usegc_[126]km_\d{8}\.nc"
|
|
@@ -118,9 +144,7 @@ class LocalStore:
|
|
|
118
144
|
"""A directory; files are created once and never replaced."""
|
|
119
145
|
|
|
120
146
|
def __init__(self, root: str | os.PathLike | None = None):
|
|
121
|
-
self.root = Path(
|
|
122
|
-
root if root is not None else settings.cache_dir("hfradar")
|
|
123
|
-
).expanduser()
|
|
147
|
+
self.root = Path(root if root is not None else archive_root()).expanduser()
|
|
124
148
|
|
|
125
149
|
def has(self, name: str) -> bool:
|
|
126
150
|
return (self.root / name).exists()
|
|
@@ -138,6 +162,11 @@ class LocalStore:
|
|
|
138
162
|
return True
|
|
139
163
|
|
|
140
164
|
|
|
165
|
+
def archive_root() -> str:
|
|
166
|
+
"""The archive's default root: FORCINGKIT_HFRADAR_ARCHIVE, else the cache's ``hfradar`` directory."""
|
|
167
|
+
return settings.hfradar_archive() or settings.cache_dir("hfradar")
|
|
168
|
+
|
|
169
|
+
|
|
141
170
|
def dataset_url(grid: str) -> str:
|
|
142
171
|
return f"{SERVER}/hfradar_usegc_{grid}"
|
|
143
172
|
|
|
@@ -257,9 +286,17 @@ def _stored(var: xr.DataArray) -> tuple[np.ndarray, dict]:
|
|
|
257
286
|
|
|
258
287
|
|
|
259
288
|
def _normalize(
|
|
260
|
-
sub: xr.Dataset,
|
|
289
|
+
sub: xr.Dataset,
|
|
290
|
+
grid: str,
|
|
291
|
+
bbox: BBox,
|
|
292
|
+
source_attrs: dict,
|
|
293
|
+
fetched: datetime,
|
|
294
|
+
source: str | None = None,
|
|
295
|
+
credit: str = CREDIT,
|
|
261
296
|
) -> xr.Dataset:
|
|
262
|
-
"""The server's subset as a clean Dataset of stored values, with its encodings and credit.
|
|
297
|
+
"""The server's subset as a clean Dataset of stored values, with its encodings and credit. ``source`` names
|
|
298
|
+
where it came from (default: NDBC's dataset)."""
|
|
299
|
+
source = source or dataset_url(grid)
|
|
263
300
|
out = xr.Dataset(
|
|
264
301
|
coords={
|
|
265
302
|
"time": sub["time"].values,
|
|
@@ -294,10 +331,10 @@ def _normalize(
|
|
|
294
331
|
w, s, e, n = bbox
|
|
295
332
|
attrs.update(
|
|
296
333
|
{
|
|
297
|
-
"source_dataset":
|
|
298
|
-
"source_credit":
|
|
334
|
+
"source_dataset": source,
|
|
335
|
+
"source_credit": credit,
|
|
299
336
|
"subset": f"latitude {s} to {n}, longitude {w} to {e}; values as stored on the server",
|
|
300
|
-
"history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {
|
|
337
|
+
"history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {source} by forcingkit",
|
|
301
338
|
}
|
|
302
339
|
)
|
|
303
340
|
out.attrs = attrs
|
|
@@ -451,9 +488,7 @@ def load(
|
|
|
451
488
|
"""The box's hours in [start, end), decoded (u and v in m/s, NaN where there is no solution): archived days from
|
|
452
489
|
any archived box that contains this one, and, when ``remote``, the other days from the server while it still
|
|
453
490
|
holds them (not archived by this call). Empty in time when neither has any."""
|
|
454
|
-
base = Path(
|
|
455
|
-
root if root is not None else settings.cache_dir("hfradar")
|
|
456
|
-
).expanduser()
|
|
491
|
+
base = Path(root if root is not None else archive_root()).expanduser()
|
|
457
492
|
t0, t1 = _utc_naive(start), _utc_naive(end)
|
|
458
493
|
boxes = _containing_boxes(base, grid, bbox)
|
|
459
494
|
parts: list[xr.Dataset] = []
|
|
@@ -498,3 +533,262 @@ def load(
|
|
|
498
533
|
out = out.sel(time=slice(t0, t1 - pd.Timedelta(seconds=1)))
|
|
499
534
|
out.attrs = {"source_dataset": dataset_url(grid), "source_credit": CREDIT}
|
|
500
535
|
return out
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
# ---- NCEI's monthly archive ----
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def _client(session: requests.Session | None) -> requests.Session:
|
|
542
|
+
if session is None:
|
|
543
|
+
session = requests.Session()
|
|
544
|
+
session.headers["User-Agent"] = USER_AGENT
|
|
545
|
+
return session
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _get(
|
|
549
|
+
session: requests.Session, url: str, tries: int = 4, **kw
|
|
550
|
+
) -> requests.Response:
|
|
551
|
+
"""GET with a few retries and backoff (NCEI answers a burst with the odd 5xx or reset)."""
|
|
552
|
+
for k in range(tries):
|
|
553
|
+
try:
|
|
554
|
+
r = session.get(url, timeout=120, **kw)
|
|
555
|
+
if r.status_code < 500:
|
|
556
|
+
r.raise_for_status()
|
|
557
|
+
return r
|
|
558
|
+
err: Exception = requests.HTTPError(f"{r.status_code} for {url}")
|
|
559
|
+
except (requests.ConnectionError, requests.Timeout) as exc:
|
|
560
|
+
err = exc
|
|
561
|
+
if k < tries - 1:
|
|
562
|
+
_time.sleep(2**k)
|
|
563
|
+
raise err
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def ncei_months(
|
|
567
|
+
start, end, session: requests.Session | None = None
|
|
568
|
+
) -> dict[pd.Timestamp, str]:
|
|
569
|
+
"""NCEI's accessions of hourly totals for the months that overlap [start, end): {first of month: accession}.
|
|
570
|
+
A month listed twice keeps the higher (later) accession number."""
|
|
571
|
+
session = _client(session)
|
|
572
|
+
first = _utc_naive(start).to_period("M").to_timestamp()
|
|
573
|
+
last = (_utc_naive(end) - pd.Timedelta(seconds=1)).to_period("M").to_timestamp()
|
|
574
|
+
found: dict[pd.Timestamp, str] = {}
|
|
575
|
+
for series in NCEI_SERIES:
|
|
576
|
+
at = 1
|
|
577
|
+
while True:
|
|
578
|
+
page = _get(
|
|
579
|
+
session,
|
|
580
|
+
NCEI_SEARCH,
|
|
581
|
+
params={"q": series, "start": at, "num": 200, "f": "json"},
|
|
582
|
+
).json()
|
|
583
|
+
for item in page.get("results", []):
|
|
584
|
+
if not str(item.get("id", "")).startswith(series + "."):
|
|
585
|
+
continue
|
|
586
|
+
m = _NCEI_TITLE.search(item.get("title", ""))
|
|
587
|
+
if not m:
|
|
588
|
+
continue
|
|
589
|
+
month = pd.Timestamp(f"{m.group(1)} 1 {m.group(2)}")
|
|
590
|
+
if first <= month <= last and int(m.group(3)) > int(
|
|
591
|
+
found.get(month, "0")
|
|
592
|
+
):
|
|
593
|
+
found[month] = m.group(3)
|
|
594
|
+
nxt = page.get("nextStart")
|
|
595
|
+
if not nxt or nxt <= at or nxt > page.get("total", 0):
|
|
596
|
+
break
|
|
597
|
+
at = nxt
|
|
598
|
+
return dict(sorted(found.items()))
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def ncei_accession_dir(
|
|
602
|
+
accession: str, session: requests.Session | None = None, known: dict | None = None
|
|
603
|
+
) -> str:
|
|
604
|
+
"""The folder of an accession's US East and Gulf Coast totals. NCEI files accessions under numbered ``arc``
|
|
605
|
+
folders; the newest are searched first. ``known`` (accession -> folder) is consulted and updated."""
|
|
606
|
+
if known is not None and accession in known:
|
|
607
|
+
return known[accession]
|
|
608
|
+
session = _client(session)
|
|
609
|
+
arcs = sorted(
|
|
610
|
+
set(re.findall(r'href="(arc\d+)/"', _get(session, NCEI_ARCHIVE).text)),
|
|
611
|
+
reverse=True,
|
|
612
|
+
)
|
|
613
|
+
for arc in arcs:
|
|
614
|
+
if f'href="{accession}/"' in _get(session, f"{NCEI_ARCHIVE}{arc}/").text:
|
|
615
|
+
url = f"{NCEI_ARCHIVE}{arc}/{accession}/1.1/data/0-data/USEGC/"
|
|
616
|
+
if known is not None:
|
|
617
|
+
known[accession] = url
|
|
618
|
+
return url
|
|
619
|
+
raise RuntimeError(f"NCEI accession {accession} not found under {NCEI_ARCHIVE}")
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def ncei_hours(
|
|
623
|
+
folder: str, grid: str, session: requests.Session | None = None
|
|
624
|
+
) -> dict[pd.Timestamp, str]:
|
|
625
|
+
"""The hourly files of one grid in an accession's folder: {hour: URL}; a reissued hour keeps the latest file."""
|
|
626
|
+
best: dict[pd.Timestamp, tuple[str, str]] = {}
|
|
627
|
+
for name, g, start, created in _NCEI_FILE.findall(
|
|
628
|
+
_get(_client(session), folder).text
|
|
629
|
+
):
|
|
630
|
+
if g != grid:
|
|
631
|
+
continue
|
|
632
|
+
hour = pd.Timestamp(datetime.strptime(start, "%Y%m%d%H%M"))
|
|
633
|
+
if hour not in best or created > best[hour][1]:
|
|
634
|
+
best[hour] = (name, created)
|
|
635
|
+
return {h: folder + n for h, (n, _) in sorted(best.items())}
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def _ncei_hour(
|
|
639
|
+
session: requests.Session, url: str, bbox: BBox
|
|
640
|
+
) -> tuple[xr.Dataset, dict]:
|
|
641
|
+
"""One hourly file, the box cut out, values as stored."""
|
|
642
|
+
data = _get(session, url).content
|
|
643
|
+
with xr.open_dataset(
|
|
644
|
+
io.BytesIO(data), engine="h5netcdf", mask_and_scale=False
|
|
645
|
+
) as ds:
|
|
646
|
+
ys, xs = box_slices(ds["lat"].values, ds["lon"].values, bbox)
|
|
647
|
+
names = [n for n in VARIABLES + SCALARS if n in ds]
|
|
648
|
+
return ds[names].isel(lat=ys, lon=xs).load(), dict(ds.attrs)
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _read_index(store: Store) -> tuple[dict, Path | None]:
|
|
652
|
+
root = getattr(store, "root", None)
|
|
653
|
+
if root is None:
|
|
654
|
+
return {}, None
|
|
655
|
+
path = Path(root) / NCEI_INDEX
|
|
656
|
+
try:
|
|
657
|
+
return json.loads(path.read_text()), path
|
|
658
|
+
except (OSError, ValueError):
|
|
659
|
+
return {}, path
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def archive_ncei(
|
|
663
|
+
bbox: BBox,
|
|
664
|
+
start,
|
|
665
|
+
end,
|
|
666
|
+
grids: Sequence[str] = GRIDS,
|
|
667
|
+
store: Store | None = None,
|
|
668
|
+
session: requests.Session | None = None,
|
|
669
|
+
workers: int = 4,
|
|
670
|
+
) -> dict[str, list[str]]:
|
|
671
|
+
"""Archive the box's days in [start, end) from NCEI's monthly accessions, per grid, skipping days already kept
|
|
672
|
+
(from either source); return the names written per grid. A day is written with the hours NCEI holds for it.
|
|
673
|
+
Days in months NCEI has not published are left for NDBC. A day that fails is skipped and the rest carry on; the
|
|
674
|
+
call then raises, naming the days missed."""
|
|
675
|
+
session = _client(session)
|
|
676
|
+
store = store if store is not None else LocalStore()
|
|
677
|
+
t0, t1 = _utc_naive(start).normalize(), _utc_naive(end)
|
|
678
|
+
days = pd.date_range(t0, (t1 - pd.Timedelta(seconds=1)).normalize(), freq="D")
|
|
679
|
+
months = ncei_months(t0, t1, session)
|
|
680
|
+
known, index_path = _read_index(store)
|
|
681
|
+
written: dict[str, list[str]] = {g: [] for g in grids}
|
|
682
|
+
failed: list[str] = []
|
|
683
|
+
for grid in grids:
|
|
684
|
+
for month, accession in months.items():
|
|
685
|
+
todo = [
|
|
686
|
+
d
|
|
687
|
+
for d in days
|
|
688
|
+
if d.to_period("M").to_timestamp() == month
|
|
689
|
+
and not store.has(day_name(grid, bbox, d))
|
|
690
|
+
]
|
|
691
|
+
if not todo:
|
|
692
|
+
continue
|
|
693
|
+
try:
|
|
694
|
+
folder = ncei_accession_dir(accession, session, known)
|
|
695
|
+
hours = ncei_hours(folder, grid, session)
|
|
696
|
+
except Exception as exc:
|
|
697
|
+
logger.warning(
|
|
698
|
+
"NCEI %s %s (%s): %s", grid, f"{month:%Y-%m}", accession, exc
|
|
699
|
+
)
|
|
700
|
+
failed.append(f"{grid} {month:%Y-%m}")
|
|
701
|
+
continue
|
|
702
|
+
if index_path is not None:
|
|
703
|
+
index_path.parent.mkdir(parents=True, exist_ok=True)
|
|
704
|
+
index_path.write_text(json.dumps(known, indent=1, sort_keys=True))
|
|
705
|
+
for day in todo:
|
|
706
|
+
urls = [u for h, u in hours.items() if h.normalize() == day]
|
|
707
|
+
if not urls:
|
|
708
|
+
logger.info(
|
|
709
|
+
"NCEI %s %s: no hours in accession %s",
|
|
710
|
+
grid,
|
|
711
|
+
day.date(),
|
|
712
|
+
accession,
|
|
713
|
+
)
|
|
714
|
+
continue
|
|
715
|
+
try:
|
|
716
|
+
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
717
|
+
got = list(
|
|
718
|
+
pool.map(lambda u: _ncei_hour(session, u, bbox), urls)
|
|
719
|
+
)
|
|
720
|
+
except Exception as exc: # carry on; a later run fills the gap
|
|
721
|
+
logger.warning("NCEI %s %s: %s", grid, day.date(), exc)
|
|
722
|
+
failed.append(f"{grid} {day:%Y-%m-%d}")
|
|
723
|
+
continue
|
|
724
|
+
sub = xr.concat(
|
|
725
|
+
[g for g, _ in got],
|
|
726
|
+
dim="time",
|
|
727
|
+
data_vars="minimal",
|
|
728
|
+
coords="minimal",
|
|
729
|
+
compat="override",
|
|
730
|
+
join="exact",
|
|
731
|
+
).sortby("time")
|
|
732
|
+
ds = _normalize(
|
|
733
|
+
sub,
|
|
734
|
+
grid,
|
|
735
|
+
bbox,
|
|
736
|
+
got[0][1],
|
|
737
|
+
datetime.now(timezone.utc),
|
|
738
|
+
source=f"{NCEI}/archive/accession/{accession} ({folder})",
|
|
739
|
+
credit=NCEI_CREDIT,
|
|
740
|
+
)
|
|
741
|
+
name = day_name(grid, bbox, day)
|
|
742
|
+
if store.put(name, day_file(ds, day)):
|
|
743
|
+
written[grid].append(name)
|
|
744
|
+
logger.info(
|
|
745
|
+
"%s written (%d h, NCEI %s)", name, len(urls), accession
|
|
746
|
+
)
|
|
747
|
+
if failed:
|
|
748
|
+
raise RuntimeError(
|
|
749
|
+
f"HF radar days not read from NCEI: {', '.join(failed)} (written: {written})"
|
|
750
|
+
)
|
|
751
|
+
return written
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def main(argv: Sequence[str] | None = None) -> None:
|
|
755
|
+
"""Archive a box from NDBC (the last ~3 months) or NCEI (older months)."""
|
|
756
|
+
p = argparse.ArgumentParser(
|
|
757
|
+
prog="python -m forcingkit.fetchers.hfradar", description=main.__doc__
|
|
758
|
+
)
|
|
759
|
+
p.add_argument("source", choices=["ndbc", "ncei"])
|
|
760
|
+
p.add_argument(
|
|
761
|
+
"--box", nargs=4, type=float, required=True, metavar=("W", "S", "E", "N")
|
|
762
|
+
)
|
|
763
|
+
p.add_argument(
|
|
764
|
+
"--start", help="first day (ndbc: or 'all'; default the last 10 days)"
|
|
765
|
+
)
|
|
766
|
+
p.add_argument("--end", help="ncei: the day after the last; ndbc: the last day")
|
|
767
|
+
p.add_argument("--grids", nargs="+", default=list(GRIDS), choices=list(GRIDS))
|
|
768
|
+
p.add_argument(
|
|
769
|
+
"--root",
|
|
770
|
+
help="the archive's root (default: FORCINGKIT_HFRADAR_ARCHIVE, else the cache)",
|
|
771
|
+
)
|
|
772
|
+
p.add_argument(
|
|
773
|
+
"--workers", type=int, default=4, help="ncei: hourly files read at once"
|
|
774
|
+
)
|
|
775
|
+
p.add_argument("-v", "--verbose", action="store_true")
|
|
776
|
+
a = p.parse_args(argv)
|
|
777
|
+
logging.basicConfig(
|
|
778
|
+
level=logging.INFO if a.verbose else logging.WARNING,
|
|
779
|
+
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
780
|
+
)
|
|
781
|
+
store = LocalStore(a.root)
|
|
782
|
+
if a.source == "ncei":
|
|
783
|
+
if not (a.start and a.end):
|
|
784
|
+
p.error("ncei needs --start and --end")
|
|
785
|
+
out = archive_ncei(
|
|
786
|
+
a.box, a.start, a.end, a.grids, store=store, workers=a.workers
|
|
787
|
+
)
|
|
788
|
+
else:
|
|
789
|
+
out = archive(a.box, a.start, a.end, a.grids, store=store)
|
|
790
|
+
print(json.dumps({g: len(n) for g, n in out.items()}))
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
if __name__ == "__main__":
|
|
794
|
+
main()
|
|
@@ -82,6 +82,13 @@ def nwps_archive() -> str | None:
|
|
|
82
82
|
return env("FORCINGKIT_NWPS_ARCHIVE") or None
|
|
83
83
|
|
|
84
84
|
|
|
85
|
+
def hfradar_archive() -> str | None:
|
|
86
|
+
"""Where the HF radar archive lives (FORCINGKIT_HFRADAR_ARCHIVE), when it should not sit in the cache: the
|
|
87
|
+
archive keeps days the servers no longer hold, so a cache purge must never reach it. Unset or empty means
|
|
88
|
+
``cache_dir("hfradar")``."""
|
|
89
|
+
return env("FORCINGKIT_HFRADAR_ARCHIVE") or None
|
|
90
|
+
|
|
91
|
+
|
|
85
92
|
def erddap_timeouts() -> tuple[float, float]:
|
|
86
93
|
"""(connect, read) timeouts in seconds for ERDDAP profile reads
|
|
87
94
|
(FORCINGKIT_ERDDAP_CONNECT_TIMEOUT, default 10; FORCINGKIT_ERDDAP_READ_TIMEOUT, default 60)."""
|