forcingkit 0.19.0__tar.gz → 0.20.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. {forcingkit-0.19.0 → forcingkit-0.20.1}/PKG-INFO +1 -1
  2. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/requirements-docs.txt +1 -1
  3. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/conf.py +7 -4
  4. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/fetchers.rst +15 -0
  5. {forcingkit-0.19.0 → forcingkit-0.20.1}/pyproject.toml +1 -1
  6. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hfradar.py +329 -11
  7. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/settings.py +7 -0
  8. forcingkit-0.20.1/tests/unit/test_hfradar.py +559 -0
  9. {forcingkit-0.19.0 → forcingkit-0.20.1}/uv.lock +1 -1
  10. forcingkit-0.19.0/docs/source/_static/custom.css +0 -24
  11. forcingkit-0.19.0/tests/unit/test_hfradar.py +0 -237
  12. {forcingkit-0.19.0 → forcingkit-0.20.1}/.agents/forcingkit.md +0 -0
  13. {forcingkit-0.19.0 → forcingkit-0.20.1}/.claude/CLAUDE.md +0 -0
  14. {forcingkit-0.19.0 → forcingkit-0.20.1}/.env.template +0 -0
  15. {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/copilot-instructions.md +0 -0
  16. {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/workflows/docs.yml +0 -0
  17. {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/workflows/publish-client.yml +0 -0
  18. {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/workflows/publish.yml +0 -0
  19. {forcingkit-0.19.0 → forcingkit-0.20.1}/.github/workflows/tests.yml +0 -0
  20. {forcingkit-0.19.0 → forcingkit-0.20.1}/.gitignore +0 -0
  21. {forcingkit-0.19.0 → forcingkit-0.20.1}/.markdownlint.yaml +0 -0
  22. {forcingkit-0.19.0 → forcingkit-0.20.1}/.pre-commit-config.yaml +0 -0
  23. {forcingkit-0.19.0 → forcingkit-0.20.1}/.python-version +0 -0
  24. {forcingkit-0.19.0 → forcingkit-0.20.1}/AGENTS.md +0 -0
  25. {forcingkit-0.19.0 → forcingkit-0.20.1}/CONTRIBUTING.md +0 -0
  26. {forcingkit-0.19.0 → forcingkit-0.20.1}/Dockerfile +0 -0
  27. {forcingkit-0.19.0 → forcingkit-0.20.1}/LICENSE +0 -0
  28. {forcingkit-0.19.0 → forcingkit-0.20.1}/README.md +0 -0
  29. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/LICENSE +0 -0
  30. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/README.md +0 -0
  31. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/pyproject.toml +0 -0
  32. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/__init__.py +0 -0
  33. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/client.py +0 -0
  34. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/schema.py +0 -0
  35. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/store.py +0 -0
  36. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/src/forcingkit_client/trajectory.py +0 -0
  37. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/tests/conftest.py +0 -0
  38. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/tests/test_client.py +0 -0
  39. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/tests/test_schema.py +0 -0
  40. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/tests/test_trajectory.py +0 -0
  41. {forcingkit-0.19.0 → forcingkit-0.20.1}/client/uv.lock +0 -0
  42. {forcingkit-0.19.0 → forcingkit-0.20.1}/docker-compose.yml +0 -0
  43. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/Makefile +0 -0
  44. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/make.bat +0 -0
  45. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/_extra/CNAME +0 -0
  46. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  47. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/atmospheric_forcing.rst +0 -0
  48. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/client.rst +0 -0
  49. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/hycom.rst +0 -0
  50. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/index.rst +0 -0
  51. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/layered_mesh.rst +0 -0
  52. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/mesh_frames.rst +0 -0
  53. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/necofs.rst +0 -0
  54. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/nyofs.rst +0 -0
  55. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/rivers.rst +0 -0
  56. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/roadmap.rst +0 -0
  57. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/sources.rst +0 -0
  58. {forcingkit-0.19.0 → forcingkit-0.20.1}/docs/source/waves.rst +0 -0
  59. {forcingkit-0.19.0 → forcingkit-0.20.1}/main.py +0 -0
  60. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/__init__.py +0 -0
  61. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/__init__.py +0 -0
  62. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/jobs.py +0 -0
  63. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/main.py +0 -0
  64. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  65. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  66. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/removed.py +0 -0
  67. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/stores.py +0 -0
  68. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/viewer.py +0 -0
  69. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/forcingkit_serve/stac.py +0 -0
  70. {forcingkit-0.19.0 → forcingkit-0.20.1}/service/run_server.py +0 -0
  71. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/__init__.py +0 -0
  72. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  73. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/dispatcher.py +0 -0
  74. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/coastwatch.py +0 -0
  75. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/dbofs.py +0 -0
  76. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/erddap.py +0 -0
  77. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrdps.py +0 -0
  78. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr.py +0 -0
  79. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  80. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  81. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  82. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hycom.py +0 -0
  83. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hydrography.py +0 -0
  84. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/ndbc.py +0 -0
  85. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/necofs.py +0 -0
  86. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/noaa.py +0 -0
  87. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/nwps.py +0 -0
  88. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/nyofs.py +0 -0
  89. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/usgs.py +0 -0
  90. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  91. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/layered_mesh.py +0 -0
  92. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/mesh_frames.py +0 -0
  93. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis.csv +0 -0
  94. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  95. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  96. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  97. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/render.py +0 -0
  98. {forcingkit-0.19.0 → forcingkit-0.20.1}/src/forcingkit/zarr_stream.py +0 -0
  99. {forcingkit-0.19.0 → forcingkit-0.20.1}/static/app.js +0 -0
  100. {forcingkit-0.19.0 → forcingkit-0.20.1}/static/favicon.ico +0 -0
  101. {forcingkit-0.19.0 → forcingkit-0.20.1}/static/favicon.svg +0 -0
  102. {forcingkit-0.19.0 → forcingkit-0.20.1}/static/index.html +0 -0
  103. {forcingkit-0.19.0 → forcingkit-0.20.1}/static/logo.svg +0 -0
  104. {forcingkit-0.19.0 → forcingkit-0.20.1}/static/preview3d.js +0 -0
  105. {forcingkit-0.19.0 → forcingkit-0.20.1}/static/styles.css +0 -0
  106. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/__init__.py +0 -0
  107. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_auth.py +0 -0
  108. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_erddap_fetch.py +0 -0
  109. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_mesh_frames_necofs.py +0 -0
  110. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_necofs_volume_live.py +0 -0
  111. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_nwps_nomads.py +0 -0
  112. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  113. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_obc_offshore_nj.py +0 -0
  114. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_ofs_archive_paths.py +0 -0
  115. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/integration/test_s3_roms_fetchers.py +0 -0
  116. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/scripts/inspect_dem.py +0 -0
  117. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/scripts/inspect_grib.py +0 -0
  118. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/scripts/inspect_zarr.py +0 -0
  119. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_coastwatch.py +0 -0
  120. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_dbofs.py +0 -0
  121. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_dispatcher.py +0 -0
  122. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_erddap.py +0 -0
  123. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrdps.py +0 -0
  124. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_atm.py +0 -0
  125. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_idx.py +0 -0
  126. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_levels.py +0 -0
  127. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_points.py +0 -0
  128. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_hycom.py +0 -0
  129. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_jobs.py +0 -0
  130. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_layered_mesh.py +0 -0
  131. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_main.py +0 -0
  132. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_mesh_frames.py +0 -0
  133. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_ndbc.py +0 -0
  134. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_necofs_hindcast.py +0 -0
  135. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_necofs_mesh_cache.py +0 -0
  136. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_necofs_parent.py +0 -0
  137. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_necofs_volume.py +0 -0
  138. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_noaa_currents.py +0 -0
  139. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_noaa_datum.py +0 -0
  140. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_nwps.py +0 -0
  141. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_nyofs.py +0 -0
  142. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_obc_donor_report.py +0 -0
  143. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_obc_pipeline.py +0 -0
  144. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_removed_routes.py +0 -0
  145. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_settings.py +0 -0
  146. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_usgs.py +0 -0
  147. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_usgs_rivers.py +0 -0
  148. {forcingkit-0.19.0 → forcingkit-0.20.1}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.19.0
3
+ Version: 0.20.1
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -1,3 +1,3 @@
1
1
  sphinx>=7.0.0
2
- sphinx-rtd-theme>=2.0.0
2
+ lhzn-sphinx-theme>=0.1.1,<0.2
3
3
  myst-parser>=2.0.0
@@ -9,13 +9,12 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.19.0"
12
+ release = "0.20.1"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
16
16
  "sphinx.ext.napoleon",
17
17
  "sphinx.ext.viewcode",
18
- "sphinx_rtd_theme",
19
18
  "myst_parser",
20
19
  ]
21
20
 
@@ -24,8 +23,12 @@ exclude_patterns: list[str] = []
24
23
 
25
24
  language = "en"
26
25
 
27
- html_theme = "sphinx_rtd_theme"
26
+ html_theme = "lhzn"
27
+ html_theme_options = {
28
+ "source_repository": "https://github.com/lhzn-io/forcingkit",
29
+ "source_branch": "main",
30
+ "source_directory": "docs/source/",
31
+ }
28
32
  html_static_path = ["_static"]
29
- html_css_files = ["custom.css"]
30
33
  # CNAME for the custom domain forcingkit.docs.lhzn.io, copied to the site root.
31
34
  html_extra_path = ["_extra"]
@@ -189,9 +189,24 @@ These serve validation rather than forcing:
189
189
  with ``has`` and ``put`` (a create-only bucket, for example). ``load`` reads a box and window back
190
190
  in m/s from any archived box that contains it, and the other days from the server while it holds
191
191
  them. The ``hfradar`` directory is an archive, not a cache: do not purge it with the Zarr stores.
192
+ ``FORCINGKIT_HFRADAR_ARCHIVE`` puts it elsewhere (since 0.20.0).
192
193
  Coverage is uneven: in western Long Island Sound about 2 % of the 1 km cells in the box
193
194
  40.85-41.3 N, 73.8-71.8 W have a solution in a covered hour, and whole weeks have none.
194
195
 
196
+ Older days come from `NOAA NCEI <https://www.ncei.noaa.gov/>`__, which archives the same hourly
197
+ totals (the same variables, stored the same way, on the same grids) as one file per hour in monthly
198
+ accessions, from about two months back to the network's start. ``archive_ncei`` finds a window's
199
+ accessions through NCEI's metadata search, reads each hour's file (about 0.8 MB at 1 km for the whole
200
+ coast), cuts the box out and writes the same daily files, so ``load`` reads days from either source
201
+ alike; ``source_dataset`` names the accession. Days already kept are skipped, and a day is written
202
+ with the hours NCEI holds (since 0.20.0). The month list and folders found are kept in
203
+ ``ncei_accessions.json`` beside the archive, so a rerun skips NCEI's slow search (since 0.20.1).
204
+ From the command line::
205
+
206
+ python -m forcingkit.fetchers.hfradar ncei --box -73.8 40.85 -71.8 41.3 \
207
+ --start 2025-01-01 --end 2026-09-01 [--grids 1km 2km] [--root DIR] [--workers 4]
208
+ python -m forcingkit.fetchers.hfradar ndbc --box -73.8 40.85 -71.8 41.3 --start all [--root DIR]
209
+
195
210
  NOAA OFS archive locations
196
211
  --------------------------
197
212
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.19.0"
7
+ version = "0.20.1"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -24,17 +24,32 @@ kept, with no screening. Unlike the Zarr stores beside it, this directory is an
24
24
  those days again once they leave its window, so do not purge it. `load` reads a box and window back, decoded to m/s,
25
25
  from any archived box that contains it, and fetches days the archive lacks from the server while it still holds them.
26
26
 
27
+ Older days come from NOAA NCEI, which archives the same hourly totals (the same variables, stored the same way, on
28
+ the same grids) as one file per hour in monthly accessions. `archive_ncei` finds a window's accessions through NCEI's
29
+ metadata search, reads each hour's file, cuts the box out and writes the same daily files, so `load` reads both
30
+ alike; their ``source_dataset`` names the accession. NCEI's accessions lag the present by a month or two; NDBC's
31
+ window covers the rest. The archive's root is ``FORCINGKIT_HFRADAR_ARCHIVE`` when set (see `settings.hfradar_archive`).
32
+
33
+ From the command line::
34
+
35
+ python -m forcingkit.fetchers.hfradar ncei --box W S E N --start 2025-01-01 --end 2026-09-01 [--root DIR]
36
+ python -m forcingkit.fetchers.hfradar ndbc --box W S E N [--start all] [--root DIR]
37
+
27
38
  Credit: the radar sites' operators (named in HFRNet's site list; for example the University of Connecticut's LISICOS
28
39
  for western Long Island Sound) and the NOAA IOOS HF Radar National Network; see ``source_credit`` in each file.
29
40
  """
30
41
 
31
42
  from __future__ import annotations
32
43
 
44
+ import argparse
33
45
  import io
46
+ import json
34
47
  import logging
35
48
  import os
36
49
  import re
50
+ import time as _time
37
51
  from collections.abc import Sequence
52
+ from concurrent.futures import ThreadPoolExecutor
38
53
  from datetime import datetime, timedelta, timezone
39
54
  from pathlib import Path
40
55
  from typing import Protocol
@@ -94,6 +109,22 @@ CREDIT = (
94
109
  "HF radar sites operated by their institutions (for western Long Island Sound, the University of Connecticut's "
95
110
  "LISICOS); hourly total vectors from the NOAA IOOS HF Radar National Network (HFRNet), served by NOAA NDBC."
96
111
  )
112
+ NCEI_CREDIT = CREDIT.replace("served by NOAA NDBC.", "archived by NOAA NCEI.")
113
+ NCEI = "https://www.ncei.noaa.gov"
114
+ NCEI_SEARCH = f"{NCEI}/metadata/granule/geoportal/opensearch"
115
+ NCEI_ARCHIVE = f"{NCEI}/data/oceans/archive/"
116
+ # The metadata series holding the hourly totals (the Radial series beside them hold each site's radials).
117
+ NCEI_SERIES = ("IOOS-HFRadarRTVector", "NDBC-HFRadarRTVector")
118
+ # Beside the archive: {"months": {"YYYY-MM": accession}, "accessions": {accession: folder}}, so the metadata
119
+ # search (slow: minutes) and each folder lookup run once.
120
+ NCEI_INDEX = "ncei_accessions.json"
121
+ _NCEI_TITLE = re.compile(r"during ([A-Z][a-z]+) (\d{4}) \(NCEI Accession (\d+)\)")
122
+ # Hourly file names: since 2026 ``rtv-usegc-<grid>-uwls_v1r0_hfr_s<YYYYmmddHHMM>..._c<created>.nc`` (a reissue
123
+ # has a later ``c``); before, ``<YYYYmmddHHMM>_hfr_usegc_<grid>_rtv_uwls_NDBC.nc``.
124
+ _NCEI_FILE = re.compile(
125
+ r'href="(rtv-usegc-([126]km)-[^"/]*_s(\d{12})\d*_e\d+_c(\d+)\.nc)"'
126
+ )
127
+ _NCEI_FILE_OLD = re.compile(r'href="((\d{12})_hfr_usegc_([126]km)_rtv_[^"/]*\.nc)"')
97
128
  NAME = re.compile(
98
129
  r"usegc_[126]km/\d{2}\.\d{3}[NS]_\d{2}\.\d{3}[NS]_\d{3}\.\d{3}[EW]_\d{3}\.\d{3}[EW]/\d{4}/\d{2}/"
99
130
  r"hfradar_usegc_[126]km_\d{8}\.nc"
@@ -118,9 +149,7 @@ class LocalStore:
118
149
  """A directory; files are created once and never replaced."""
119
150
 
120
151
  def __init__(self, root: str | os.PathLike | None = None):
121
- self.root = Path(
122
- root if root is not None else settings.cache_dir("hfradar")
123
- ).expanduser()
152
+ self.root = Path(root if root is not None else archive_root()).expanduser()
124
153
 
125
154
  def has(self, name: str) -> bool:
126
155
  return (self.root / name).exists()
@@ -138,6 +167,11 @@ class LocalStore:
138
167
  return True
139
168
 
140
169
 
170
+ def archive_root() -> str:
171
+ """The archive's default root: FORCINGKIT_HFRADAR_ARCHIVE, else the cache's ``hfradar`` directory."""
172
+ return settings.hfradar_archive() or settings.cache_dir("hfradar")
173
+
174
+
141
175
  def dataset_url(grid: str) -> str:
142
176
  return f"{SERVER}/hfradar_usegc_{grid}"
143
177
 
@@ -257,9 +291,17 @@ def _stored(var: xr.DataArray) -> tuple[np.ndarray, dict]:
257
291
 
258
292
 
259
293
  def _normalize(
260
- sub: xr.Dataset, grid: str, bbox: BBox, source_attrs: dict, fetched: datetime
294
+ sub: xr.Dataset,
295
+ grid: str,
296
+ bbox: BBox,
297
+ source_attrs: dict,
298
+ fetched: datetime,
299
+ source: str | None = None,
300
+ credit: str = CREDIT,
261
301
  ) -> xr.Dataset:
262
- """The server's subset as a clean Dataset of stored values, with its encodings and credit."""
302
+ """The server's subset as a clean Dataset of stored values, with its encodings and credit. ``source`` names
303
+ where it came from (default: NDBC's dataset)."""
304
+ source = source or dataset_url(grid)
263
305
  out = xr.Dataset(
264
306
  coords={
265
307
  "time": sub["time"].values,
@@ -294,10 +336,10 @@ def _normalize(
294
336
  w, s, e, n = bbox
295
337
  attrs.update(
296
338
  {
297
- "source_dataset": dataset_url(grid),
298
- "source_credit": CREDIT,
339
+ "source_dataset": source,
340
+ "source_credit": credit,
299
341
  "subset": f"latitude {s} to {n}, longitude {w} to {e}; values as stored on the server",
300
- "history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {dataset_url(grid)} by forcingkit",
342
+ "history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {source} by forcingkit",
301
343
  }
302
344
  )
303
345
  out.attrs = attrs
@@ -451,9 +493,7 @@ def load(
451
493
  """The box's hours in [start, end), decoded (u and v in m/s, NaN where there is no solution): archived days from
452
494
  any archived box that contains this one, and, when ``remote``, the other days from the server while it still
453
495
  holds them (not archived by this call). Empty in time when neither has any."""
454
- base = Path(
455
- root if root is not None else settings.cache_dir("hfradar")
456
- ).expanduser()
496
+ base = Path(root if root is not None else archive_root()).expanduser()
457
497
  t0, t1 = _utc_naive(start), _utc_naive(end)
458
498
  boxes = _containing_boxes(base, grid, bbox)
459
499
  parts: list[xr.Dataset] = []
@@ -498,3 +538,281 @@ def load(
498
538
  out = out.sel(time=slice(t0, t1 - pd.Timedelta(seconds=1)))
499
539
  out.attrs = {"source_dataset": dataset_url(grid), "source_credit": CREDIT}
500
540
  return out
541
+
542
+
543
+ # ---- NCEI's monthly archive ----
544
+
545
+
546
+ def _client(session: requests.Session | None) -> requests.Session:
547
+ if session is None:
548
+ session = requests.Session()
549
+ session.headers["User-Agent"] = USER_AGENT
550
+ return session
551
+
552
+
553
+ def _get(
554
+ session: requests.Session, url: str, tries: int = 4, **kw
555
+ ) -> requests.Response:
556
+ """GET with a few retries and backoff (NCEI answers a burst with the odd 5xx or reset)."""
557
+ for k in range(tries):
558
+ try:
559
+ r = session.get(url, timeout=120, **kw)
560
+ if r.status_code < 500:
561
+ r.raise_for_status()
562
+ return r
563
+ err: Exception = requests.HTTPError(f"{r.status_code} for {url}")
564
+ except (requests.ConnectionError, requests.Timeout) as exc:
565
+ err = exc
566
+ if k < tries - 1:
567
+ _time.sleep(2**k)
568
+ raise err
569
+
570
+
571
+ def ncei_months(
572
+ start, end, session: requests.Session | None = None
573
+ ) -> dict[pd.Timestamp, str]:
574
+ """NCEI's accessions of hourly totals for the months that overlap [start, end): {first of month: accession}.
575
+ A month listed twice keeps the higher (later) accession number."""
576
+ session = _client(session)
577
+ first = _utc_naive(start).to_period("M").to_timestamp()
578
+ last = (_utc_naive(end) - pd.Timedelta(seconds=1)).to_period("M").to_timestamp()
579
+ found: dict[pd.Timestamp, str] = {}
580
+ for series in NCEI_SERIES:
581
+ at = 1
582
+ while True:
583
+ page = _get(
584
+ session,
585
+ NCEI_SEARCH,
586
+ params={"q": series, "start": at, "num": 200, "f": "json"},
587
+ ).json()
588
+ for item in page.get("results", []):
589
+ if not str(item.get("id", "")).startswith(series + "."):
590
+ continue
591
+ m = _NCEI_TITLE.search(item.get("title", ""))
592
+ if not m:
593
+ continue
594
+ month = pd.Timestamp(f"{m.group(1)} 1 {m.group(2)}")
595
+ if first <= month <= last and int(m.group(3)) > int(
596
+ found.get(month, "0")
597
+ ):
598
+ found[month] = m.group(3)
599
+ nxt = page.get("nextStart")
600
+ if not nxt or nxt <= at or nxt > page.get("total", 0):
601
+ break
602
+ at = nxt
603
+ return dict(sorted(found.items()))
604
+
605
+
606
+ def ncei_accession_dir(
607
+ accession: str, session: requests.Session | None = None, known: dict | None = None
608
+ ) -> str:
609
+ """The folder of an accession's US East and Gulf Coast totals. NCEI files accessions under numbered ``arc``
610
+ folders; the newest are searched first. ``known`` (accession -> folder) is consulted and updated."""
611
+ if known is not None and accession in known:
612
+ return known[accession]
613
+ session = _client(session)
614
+ arcs = sorted(
615
+ set(re.findall(r'href="(arc\d+)/"', _get(session, NCEI_ARCHIVE).text)),
616
+ reverse=True,
617
+ )
618
+ for arc in arcs:
619
+ if f'href="{accession}/"' in _get(session, f"{NCEI_ARCHIVE}{arc}/").text:
620
+ url = f"{NCEI_ARCHIVE}{arc}/{accession}/1.1/data/0-data/USEGC/"
621
+ if known is not None:
622
+ known[accession] = url
623
+ return url
624
+ raise RuntimeError(f"NCEI accession {accession} not found under {NCEI_ARCHIVE}")
625
+
626
+
627
+ def ncei_hours(
628
+ folder: str, grid: str, session: requests.Session | None = None
629
+ ) -> dict[pd.Timestamp, str]:
630
+ """The hourly files of one grid in an accession's folder: {hour: URL}; a reissued hour keeps the latest file."""
631
+ best: dict[pd.Timestamp, tuple[str, str]] = {}
632
+ text = _get(_client(session), folder).text
633
+ found = _NCEI_FILE.findall(text) + [
634
+ (name, g, start, "") for name, start, g in _NCEI_FILE_OLD.findall(text)
635
+ ]
636
+ for name, g, start, created in found:
637
+ if g != grid:
638
+ continue
639
+ hour = pd.Timestamp(datetime.strptime(start, "%Y%m%d%H%M"))
640
+ if hour not in best or created > best[hour][1]:
641
+ best[hour] = (name, created)
642
+ return {h: folder + n for h, (n, _) in sorted(best.items())}
643
+
644
+
645
+ def _ncei_hour(
646
+ session: requests.Session, url: str, bbox: BBox
647
+ ) -> tuple[xr.Dataset, dict]:
648
+ """One hourly file, the box cut out, values as stored."""
649
+ data = _get(session, url).content
650
+ with xr.open_dataset(
651
+ io.BytesIO(data), engine="h5netcdf", mask_and_scale=False
652
+ ) as ds:
653
+ ys, xs = box_slices(ds["lat"].values, ds["lon"].values, bbox)
654
+ names = [n for n in VARIABLES + SCALARS if n in ds]
655
+ return ds[names].isel(lat=ys, lon=xs).load(), dict(ds.attrs)
656
+
657
+
658
+ def _read_index(store: Store) -> tuple[dict, Path | None]:
659
+ """The index beside a directory store, and its path (an empty index and None for other stores). 0.20.0 kept
660
+ the accessions only, as a flat map."""
661
+ root = getattr(store, "root", None)
662
+ if root is None:
663
+ return {"months": {}, "accessions": {}}, None
664
+ path = Path(root) / NCEI_INDEX
665
+ try:
666
+ data = json.loads(path.read_text())
667
+ except (OSError, ValueError):
668
+ data = {}
669
+ if "accessions" not in data:
670
+ data = {"months": {}, "accessions": data}
671
+ data.setdefault("months", {})
672
+ return data, path
673
+
674
+
675
+ def _write_index(path: Path | None, index: dict) -> None:
676
+ if path is not None:
677
+ path.parent.mkdir(parents=True, exist_ok=True)
678
+ path.write_text(json.dumps(index, indent=1, sort_keys=True))
679
+
680
+
681
+ def archive_ncei(
682
+ bbox: BBox,
683
+ start,
684
+ end,
685
+ grids: Sequence[str] = GRIDS,
686
+ store: Store | None = None,
687
+ session: requests.Session | None = None,
688
+ workers: int = 4,
689
+ ) -> dict[str, list[str]]:
690
+ """Archive the box's days in [start, end) from NCEI's monthly accessions, per grid, skipping days already kept
691
+ (from either source); return the names written per grid. A day is written with the hours NCEI holds for it.
692
+ Days in months NCEI has not published are left for NDBC. A day that fails is skipped and the rest carry on; the
693
+ call then raises, naming the days missed."""
694
+ session = _client(session)
695
+ store = store if store is not None else LocalStore()
696
+ t0, t1 = _utc_naive(start).normalize(), _utc_naive(end)
697
+ days = pd.date_range(t0, (t1 - pd.Timedelta(seconds=1)).normalize(), freq="D")
698
+ index, index_path = _read_index(store)
699
+ want = pd.period_range(t0, t1 - pd.Timedelta(seconds=1), freq="M")
700
+ if all(f"{m}" in index["months"] for m in want):
701
+ months = {m.to_timestamp(): index["months"][f"{m}"] for m in want}
702
+ else: # a month not listed yet: search (months NCEI has not published stay unlisted)
703
+ months = ncei_months(t0, t1, session)
704
+ index["months"].update({f"{m:%Y-%m}": a for m, a in months.items()})
705
+ _write_index(index_path, index)
706
+ known = index["accessions"]
707
+ written: dict[str, list[str]] = {g: [] for g in grids}
708
+ failed: list[str] = []
709
+ for grid in grids:
710
+ for month, accession in months.items():
711
+ todo = [
712
+ d
713
+ for d in days
714
+ if d.to_period("M").to_timestamp() == month
715
+ and not store.has(day_name(grid, bbox, d))
716
+ ]
717
+ if not todo:
718
+ continue
719
+ try:
720
+ folder = ncei_accession_dir(accession, session, known)
721
+ hours = ncei_hours(folder, grid, session)
722
+ except Exception as exc:
723
+ logger.warning(
724
+ "NCEI %s %s (%s): %s", grid, f"{month:%Y-%m}", accession, exc
725
+ )
726
+ failed.append(f"{grid} {month:%Y-%m}")
727
+ continue
728
+ _write_index(index_path, index)
729
+ for day in todo:
730
+ urls = [u for h, u in hours.items() if h.normalize() == day]
731
+ if not urls:
732
+ logger.info(
733
+ "NCEI %s %s: no hours in accession %s",
734
+ grid,
735
+ day.date(),
736
+ accession,
737
+ )
738
+ continue
739
+ try:
740
+ with ThreadPoolExecutor(max_workers=workers) as pool:
741
+ got = list(
742
+ pool.map(lambda u: _ncei_hour(session, u, bbox), urls)
743
+ )
744
+ except Exception as exc: # carry on; a later run fills the gap
745
+ logger.warning("NCEI %s %s: %s", grid, day.date(), exc)
746
+ failed.append(f"{grid} {day:%Y-%m-%d}")
747
+ continue
748
+ sub = xr.concat(
749
+ [g for g, _ in got],
750
+ dim="time",
751
+ data_vars="minimal",
752
+ coords="minimal",
753
+ compat="override",
754
+ join="exact",
755
+ ).sortby("time")
756
+ ds = _normalize(
757
+ sub,
758
+ grid,
759
+ bbox,
760
+ got[0][1],
761
+ datetime.now(timezone.utc),
762
+ source=f"{NCEI}/archive/accession/{accession} ({folder})",
763
+ credit=NCEI_CREDIT,
764
+ )
765
+ name = day_name(grid, bbox, day)
766
+ if store.put(name, day_file(ds, day)):
767
+ written[grid].append(name)
768
+ logger.info(
769
+ "%s written (%d h, NCEI %s)", name, len(urls), accession
770
+ )
771
+ if failed:
772
+ raise RuntimeError(
773
+ f"HF radar days not read from NCEI: {', '.join(failed)} (written: {written})"
774
+ )
775
+ return written
776
+
777
+
778
+ def main(argv: Sequence[str] | None = None) -> None:
779
+ """Archive a box from NDBC (the last ~3 months) or NCEI (older months)."""
780
+ p = argparse.ArgumentParser(
781
+ prog="python -m forcingkit.fetchers.hfradar", description=main.__doc__
782
+ )
783
+ p.add_argument("source", choices=["ndbc", "ncei"])
784
+ p.add_argument(
785
+ "--box", nargs=4, type=float, required=True, metavar=("W", "S", "E", "N")
786
+ )
787
+ p.add_argument(
788
+ "--start", help="first day (ndbc: or 'all'; default the last 10 days)"
789
+ )
790
+ p.add_argument("--end", help="ncei: the day after the last; ndbc: the last day")
791
+ p.add_argument("--grids", nargs="+", default=list(GRIDS), choices=list(GRIDS))
792
+ p.add_argument(
793
+ "--root",
794
+ help="the archive's root (default: FORCINGKIT_HFRADAR_ARCHIVE, else the cache)",
795
+ )
796
+ p.add_argument(
797
+ "--workers", type=int, default=4, help="ncei: hourly files read at once"
798
+ )
799
+ p.add_argument("-v", "--verbose", action="store_true")
800
+ a = p.parse_args(argv)
801
+ logging.basicConfig(
802
+ level=logging.INFO if a.verbose else logging.WARNING,
803
+ format="%(asctime)s %(levelname)s %(name)s: %(message)s",
804
+ )
805
+ store = LocalStore(a.root)
806
+ if a.source == "ncei":
807
+ if not (a.start and a.end):
808
+ p.error("ncei needs --start and --end")
809
+ out = archive_ncei(
810
+ a.box, a.start, a.end, a.grids, store=store, workers=a.workers
811
+ )
812
+ else:
813
+ out = archive(a.box, a.start, a.end, a.grids, store=store)
814
+ print(json.dumps({g: len(n) for g, n in out.items()}))
815
+
816
+
817
+ if __name__ == "__main__":
818
+ main()
@@ -82,6 +82,13 @@ def nwps_archive() -> str | None:
82
82
  return env("FORCINGKIT_NWPS_ARCHIVE") or None
83
83
 
84
84
 
85
+ def hfradar_archive() -> str | None:
86
+ """Where the HF radar archive lives (FORCINGKIT_HFRADAR_ARCHIVE), when it should not sit in the cache: the
87
+ archive keeps days the servers no longer hold, so a cache purge must never reach it. Unset or empty means
88
+ ``cache_dir("hfradar")``."""
89
+ return env("FORCINGKIT_HFRADAR_ARCHIVE") or None
90
+
91
+
85
92
  def erddap_timeouts() -> tuple[float, float]:
86
93
  """(connect, read) timeouts in seconds for ERDDAP profile reads
87
94
  (FORCINGKIT_ERDDAP_CONNECT_TIMEOUT, default 10; FORCINGKIT_ERDDAP_READ_TIMEOUT, default 60)."""