forcingkit 0.19.0__tar.gz → 0.20.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. {forcingkit-0.19.0 → forcingkit-0.20.0}/PKG-INFO +1 -1
  2. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/requirements-docs.txt +1 -1
  3. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/conf.py +7 -4
  4. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/fetchers.rst +13 -0
  5. {forcingkit-0.19.0 → forcingkit-0.20.0}/pyproject.toml +1 -1
  6. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hfradar.py +305 -11
  7. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/settings.py +7 -0
  8. forcingkit-0.20.0/tests/unit/test_hfradar.py +505 -0
  9. {forcingkit-0.19.0 → forcingkit-0.20.0}/uv.lock +1 -1
  10. forcingkit-0.19.0/docs/source/_static/custom.css +0 -24
  11. forcingkit-0.19.0/tests/unit/test_hfradar.py +0 -237
  12. {forcingkit-0.19.0 → forcingkit-0.20.0}/.agents/forcingkit.md +0 -0
  13. {forcingkit-0.19.0 → forcingkit-0.20.0}/.claude/CLAUDE.md +0 -0
  14. {forcingkit-0.19.0 → forcingkit-0.20.0}/.env.template +0 -0
  15. {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/copilot-instructions.md +0 -0
  16. {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/workflows/docs.yml +0 -0
  17. {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/workflows/publish-client.yml +0 -0
  18. {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/workflows/publish.yml +0 -0
  19. {forcingkit-0.19.0 → forcingkit-0.20.0}/.github/workflows/tests.yml +0 -0
  20. {forcingkit-0.19.0 → forcingkit-0.20.0}/.gitignore +0 -0
  21. {forcingkit-0.19.0 → forcingkit-0.20.0}/.markdownlint.yaml +0 -0
  22. {forcingkit-0.19.0 → forcingkit-0.20.0}/.pre-commit-config.yaml +0 -0
  23. {forcingkit-0.19.0 → forcingkit-0.20.0}/.python-version +0 -0
  24. {forcingkit-0.19.0 → forcingkit-0.20.0}/AGENTS.md +0 -0
  25. {forcingkit-0.19.0 → forcingkit-0.20.0}/CONTRIBUTING.md +0 -0
  26. {forcingkit-0.19.0 → forcingkit-0.20.0}/Dockerfile +0 -0
  27. {forcingkit-0.19.0 → forcingkit-0.20.0}/LICENSE +0 -0
  28. {forcingkit-0.19.0 → forcingkit-0.20.0}/README.md +0 -0
  29. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/LICENSE +0 -0
  30. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/README.md +0 -0
  31. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/pyproject.toml +0 -0
  32. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/__init__.py +0 -0
  33. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/client.py +0 -0
  34. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/schema.py +0 -0
  35. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/store.py +0 -0
  36. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/src/forcingkit_client/trajectory.py +0 -0
  37. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/tests/conftest.py +0 -0
  38. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/tests/test_client.py +0 -0
  39. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/tests/test_schema.py +0 -0
  40. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/tests/test_trajectory.py +0 -0
  41. {forcingkit-0.19.0 → forcingkit-0.20.0}/client/uv.lock +0 -0
  42. {forcingkit-0.19.0 → forcingkit-0.20.0}/docker-compose.yml +0 -0
  43. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/Makefile +0 -0
  44. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/make.bat +0 -0
  45. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/_extra/CNAME +0 -0
  46. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  47. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/atmospheric_forcing.rst +0 -0
  48. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/client.rst +0 -0
  49. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/hycom.rst +0 -0
  50. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/index.rst +0 -0
  51. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/layered_mesh.rst +0 -0
  52. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/mesh_frames.rst +0 -0
  53. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/necofs.rst +0 -0
  54. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/nyofs.rst +0 -0
  55. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/rivers.rst +0 -0
  56. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/roadmap.rst +0 -0
  57. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/sources.rst +0 -0
  58. {forcingkit-0.19.0 → forcingkit-0.20.0}/docs/source/waves.rst +0 -0
  59. {forcingkit-0.19.0 → forcingkit-0.20.0}/main.py +0 -0
  60. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/__init__.py +0 -0
  61. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/__init__.py +0 -0
  62. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/jobs.py +0 -0
  63. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/main.py +0 -0
  64. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  65. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  66. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/removed.py +0 -0
  67. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/stores.py +0 -0
  68. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/routers/viewer.py +0 -0
  69. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/forcingkit_serve/stac.py +0 -0
  70. {forcingkit-0.19.0 → forcingkit-0.20.0}/service/run_server.py +0 -0
  71. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/__init__.py +0 -0
  72. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  73. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/dispatcher.py +0 -0
  74. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
  75. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/dbofs.py +0 -0
  76. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/erddap.py +0 -0
  77. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrdps.py +0 -0
  78. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrrr.py +0 -0
  79. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  80. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  81. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  82. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hycom.py +0 -0
  83. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/hydrography.py +0 -0
  84. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/ndbc.py +0 -0
  85. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/necofs.py +0 -0
  86. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/noaa.py +0 -0
  87. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/nwps.py +0 -0
  88. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/nyofs.py +0 -0
  89. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/usgs.py +0 -0
  90. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  91. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/layered_mesh.py +0 -0
  92. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/mesh_frames.py +0 -0
  93. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
  94. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  95. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  96. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  97. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/render.py +0 -0
  98. {forcingkit-0.19.0 → forcingkit-0.20.0}/src/forcingkit/zarr_stream.py +0 -0
  99. {forcingkit-0.19.0 → forcingkit-0.20.0}/static/app.js +0 -0
  100. {forcingkit-0.19.0 → forcingkit-0.20.0}/static/favicon.ico +0 -0
  101. {forcingkit-0.19.0 → forcingkit-0.20.0}/static/favicon.svg +0 -0
  102. {forcingkit-0.19.0 → forcingkit-0.20.0}/static/index.html +0 -0
  103. {forcingkit-0.19.0 → forcingkit-0.20.0}/static/logo.svg +0 -0
  104. {forcingkit-0.19.0 → forcingkit-0.20.0}/static/preview3d.js +0 -0
  105. {forcingkit-0.19.0 → forcingkit-0.20.0}/static/styles.css +0 -0
  106. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/__init__.py +0 -0
  107. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_auth.py +0 -0
  108. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_erddap_fetch.py +0 -0
  109. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
  110. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_necofs_volume_live.py +0 -0
  111. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_nwps_nomads.py +0 -0
  112. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  113. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_obc_offshore_nj.py +0 -0
  114. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_ofs_archive_paths.py +0 -0
  115. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
  116. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/scripts/inspect_dem.py +0 -0
  117. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/scripts/inspect_grib.py +0 -0
  118. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/scripts/inspect_zarr.py +0 -0
  119. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_coastwatch.py +0 -0
  120. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_dbofs.py +0 -0
  121. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_dispatcher.py +0 -0
  122. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_erddap.py +0 -0
  123. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrdps.py +0 -0
  124. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrrr_atm.py +0 -0
  125. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrrr_idx.py +0 -0
  126. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrrr_levels.py +0 -0
  127. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hrrr_points.py +0 -0
  128. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_hycom.py +0 -0
  129. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_jobs.py +0 -0
  130. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_layered_mesh.py +0 -0
  131. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_main.py +0 -0
  132. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_mesh_frames.py +0 -0
  133. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_ndbc.py +0 -0
  134. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_necofs_hindcast.py +0 -0
  135. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
  136. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_necofs_parent.py +0 -0
  137. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_necofs_volume.py +0 -0
  138. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_noaa_currents.py +0 -0
  139. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_noaa_datum.py +0 -0
  140. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_nwps.py +0 -0
  141. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_nyofs.py +0 -0
  142. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_obc_donor_report.py +0 -0
  143. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_obc_pipeline.py +0 -0
  144. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_removed_routes.py +0 -0
  145. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_settings.py +0 -0
  146. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_usgs.py +0 -0
  147. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_usgs_rivers.py +0 -0
  148. {forcingkit-0.19.0 → forcingkit-0.20.0}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.19.0
3
+ Version: 0.20.0
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -1,3 +1,3 @@
1
1
  sphinx>=7.0.0
2
- sphinx-rtd-theme>=2.0.0
2
+ lhzn-sphinx-theme>=0.1.1,<0.2
3
3
  myst-parser>=2.0.0
@@ -9,13 +9,12 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.19.0"
12
+ release = "0.20.0"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
16
16
  "sphinx.ext.napoleon",
17
17
  "sphinx.ext.viewcode",
18
- "sphinx_rtd_theme",
19
18
  "myst_parser",
20
19
  ]
21
20
 
@@ -24,8 +23,12 @@ exclude_patterns: list[str] = []
24
23
 
25
24
  language = "en"
26
25
 
27
- html_theme = "sphinx_rtd_theme"
26
+ html_theme = "lhzn"
27
+ html_theme_options = {
28
+ "source_repository": "https://github.com/lhzn-io/forcingkit",
29
+ "source_branch": "main",
30
+ "source_directory": "docs/source/",
31
+ }
28
32
  html_static_path = ["_static"]
29
- html_css_files = ["custom.css"]
30
33
  # CNAME for the custom domain forcingkit.docs.lhzn.io, copied to the site root.
31
34
  html_extra_path = ["_extra"]
@@ -189,6 +189,19 @@ These serve validation rather than forcing:
189
189
  with ``has`` and ``put`` (a create-only bucket, for example). ``load`` reads a box and window back
190
190
  in m/s from any archived box that contains it, and the other days from the server while it holds
191
191
  them. The ``hfradar`` directory is an archive, not a cache: do not purge it with the Zarr stores.
192
+ ``FORCINGKIT_HFRADAR_ARCHIVE`` puts it elsewhere (since 0.20.0).
193
+
194
+ Older days come from `NOAA NCEI <https://www.ncei.noaa.gov/>`__, which archives the same hourly
195
+ totals (the same variables, stored the same way, on the same grids) as one file per hour in monthly
196
+ accessions, from about two months back to the network's start. ``archive_ncei`` finds a window's
197
+ accessions through NCEI's metadata search, reads each hour's file (about 0.8 MB at 1 km for the whole
198
+ coast), cuts the box out and writes the same daily files, so ``load`` reads days from either source
199
+ alike; ``source_dataset`` names the accession. Days already kept are skipped, and a day is written
200
+ with the hours NCEI holds (since 0.20.0). From the command line::
201
+
202
+ python -m forcingkit.fetchers.hfradar ncei --box -73.8 40.85 -71.8 41.3 \
203
+ --start 2025-01-01 --end 2026-09-01 [--grids 1km 2km] [--root DIR] [--workers 4]
204
+ python -m forcingkit.fetchers.hfradar ndbc --box -73.8 40.85 -71.8 41.3 --start all [--root DIR]
192
205
  Coverage is uneven: in western Long Island Sound about 2 % of the 1 km cells in the box
193
206
  40.85-41.3 N, 73.8-71.8 W have a solution in a covered hour, and whole weeks have none.
194
207
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.19.0"
7
+ version = "0.20.0"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -24,17 +24,32 @@ kept, with no screening. Unlike the Zarr stores beside it, this directory is an
24
24
  those days again once they leave its window, so do not purge it. `load` reads a box and window back, decoded to m/s,
25
25
  from any archived box that contains it, and fetches days the archive lacks from the server while it still holds them.
26
26
 
27
+ Older days come from NOAA NCEI, which archives the same hourly totals (the same variables, stored the same way, on
28
+ the same grids) as one file per hour in monthly accessions. `archive_ncei` finds a window's accessions through NCEI's
29
+ metadata search, reads each hour's file, cuts the box out and writes the same daily files, so `load` reads both
30
+ alike; their ``source_dataset`` names the accession. NCEI's accessions lag the present by a month or two; NDBC's
31
+ window covers the rest. The archive's root is ``FORCINGKIT_HFRADAR_ARCHIVE`` when set (see `settings.hfradar_archive`).
32
+
33
+ From the command line::
34
+
35
+ python -m forcingkit.fetchers.hfradar ncei --box W S E N --start 2025-01-01 --end 2026-09-01 [--root DIR]
36
+ python -m forcingkit.fetchers.hfradar ndbc --box W S E N [--start all] [--root DIR]
37
+
27
38
  Credit: the radar sites' operators (named in HFRNet's site list; for example the University of Connecticut's LISICOS
28
39
  for western Long Island Sound) and the NOAA IOOS HF Radar National Network; see ``source_credit`` in each file.
29
40
  """
30
41
 
31
42
  from __future__ import annotations
32
43
 
44
+ import argparse
33
45
  import io
46
+ import json
34
47
  import logging
35
48
  import os
36
49
  import re
50
+ import time as _time
37
51
  from collections.abc import Sequence
52
+ from concurrent.futures import ThreadPoolExecutor
38
53
  from datetime import datetime, timedelta, timezone
39
54
  from pathlib import Path
40
55
  from typing import Protocol
@@ -94,6 +109,17 @@ CREDIT = (
94
109
  "HF radar sites operated by their institutions (for western Long Island Sound, the University of Connecticut's "
95
110
  "LISICOS); hourly total vectors from the NOAA IOOS HF Radar National Network (HFRNet), served by NOAA NDBC."
96
111
  )
112
+ NCEI_CREDIT = CREDIT.replace("served by NOAA NDBC.", "archived by NOAA NCEI.")
113
+ NCEI = "https://www.ncei.noaa.gov"
114
+ NCEI_SEARCH = f"{NCEI}/metadata/granule/geoportal/opensearch"
115
+ NCEI_ARCHIVE = f"{NCEI}/data/oceans/archive/"
116
+ # The metadata series holding the hourly totals (the Radial series beside them hold each site's radials).
117
+ NCEI_SERIES = ("IOOS-HFRadarRTVector", "NDBC-HFRadarRTVector")
118
+ NCEI_INDEX = "ncei_accessions.json" # accession -> its folder, beside the archive, so each is looked up once
119
+ _NCEI_TITLE = re.compile(r"during ([A-Z][a-z]+) (\d{4}) \(NCEI Accession (\d+)\)")
120
+ _NCEI_FILE = re.compile(
121
+ r'href="(rtv-usegc-([126]km)-[^"/]*_s(\d{12})\d*_e\d+_c(\d+)\.nc)"'
122
+ )
97
123
  NAME = re.compile(
98
124
  r"usegc_[126]km/\d{2}\.\d{3}[NS]_\d{2}\.\d{3}[NS]_\d{3}\.\d{3}[EW]_\d{3}\.\d{3}[EW]/\d{4}/\d{2}/"
99
125
  r"hfradar_usegc_[126]km_\d{8}\.nc"
@@ -118,9 +144,7 @@ class LocalStore:
118
144
  """A directory; files are created once and never replaced."""
119
145
 
120
146
  def __init__(self, root: str | os.PathLike | None = None):
121
- self.root = Path(
122
- root if root is not None else settings.cache_dir("hfradar")
123
- ).expanduser()
147
+ self.root = Path(root if root is not None else archive_root()).expanduser()
124
148
 
125
149
  def has(self, name: str) -> bool:
126
150
  return (self.root / name).exists()
@@ -138,6 +162,11 @@ class LocalStore:
138
162
  return True
139
163
 
140
164
 
165
+ def archive_root() -> str:
166
+ """The archive's default root: FORCINGKIT_HFRADAR_ARCHIVE, else the cache's ``hfradar`` directory."""
167
+ return settings.hfradar_archive() or settings.cache_dir("hfradar")
168
+
169
+
141
170
  def dataset_url(grid: str) -> str:
142
171
  return f"{SERVER}/hfradar_usegc_{grid}"
143
172
 
@@ -257,9 +286,17 @@ def _stored(var: xr.DataArray) -> tuple[np.ndarray, dict]:
257
286
 
258
287
 
259
288
  def _normalize(
260
- sub: xr.Dataset, grid: str, bbox: BBox, source_attrs: dict, fetched: datetime
289
+ sub: xr.Dataset,
290
+ grid: str,
291
+ bbox: BBox,
292
+ source_attrs: dict,
293
+ fetched: datetime,
294
+ source: str | None = None,
295
+ credit: str = CREDIT,
261
296
  ) -> xr.Dataset:
262
- """The server's subset as a clean Dataset of stored values, with its encodings and credit."""
297
+ """The server's subset as a clean Dataset of stored values, with its encodings and credit. ``source`` names
298
+ where it came from (default: NDBC's dataset)."""
299
+ source = source or dataset_url(grid)
263
300
  out = xr.Dataset(
264
301
  coords={
265
302
  "time": sub["time"].values,
@@ -294,10 +331,10 @@ def _normalize(
294
331
  w, s, e, n = bbox
295
332
  attrs.update(
296
333
  {
297
- "source_dataset": dataset_url(grid),
298
- "source_credit": CREDIT,
334
+ "source_dataset": source,
335
+ "source_credit": credit,
299
336
  "subset": f"latitude {s} to {n}, longitude {w} to {e}; values as stored on the server",
300
- "history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {dataset_url(grid)} by forcingkit",
337
+ "history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {source} by forcingkit",
301
338
  }
302
339
  )
303
340
  out.attrs = attrs
@@ -451,9 +488,7 @@ def load(
451
488
  """The box's hours in [start, end), decoded (u and v in m/s, NaN where there is no solution): archived days from
452
489
  any archived box that contains this one, and, when ``remote``, the other days from the server while it still
453
490
  holds them (not archived by this call). Empty in time when neither has any."""
454
- base = Path(
455
- root if root is not None else settings.cache_dir("hfradar")
456
- ).expanduser()
491
+ base = Path(root if root is not None else archive_root()).expanduser()
457
492
  t0, t1 = _utc_naive(start), _utc_naive(end)
458
493
  boxes = _containing_boxes(base, grid, bbox)
459
494
  parts: list[xr.Dataset] = []
@@ -498,3 +533,262 @@ def load(
498
533
  out = out.sel(time=slice(t0, t1 - pd.Timedelta(seconds=1)))
499
534
  out.attrs = {"source_dataset": dataset_url(grid), "source_credit": CREDIT}
500
535
  return out
536
+
537
+
538
+ # ---- NCEI's monthly archive ----
539
+
540
+
541
+ def _client(session: requests.Session | None) -> requests.Session:
542
+ if session is None:
543
+ session = requests.Session()
544
+ session.headers["User-Agent"] = USER_AGENT
545
+ return session
546
+
547
+
548
+ def _get(
549
+ session: requests.Session, url: str, tries: int = 4, **kw
550
+ ) -> requests.Response:
551
+ """GET with a few retries and backoff (NCEI answers a burst with the odd 5xx or reset)."""
552
+ for k in range(tries):
553
+ try:
554
+ r = session.get(url, timeout=120, **kw)
555
+ if r.status_code < 500:
556
+ r.raise_for_status()
557
+ return r
558
+ err: Exception = requests.HTTPError(f"{r.status_code} for {url}")
559
+ except (requests.ConnectionError, requests.Timeout) as exc:
560
+ err = exc
561
+ if k < tries - 1:
562
+ _time.sleep(2**k)
563
+ raise err
564
+
565
+
566
+ def ncei_months(
567
+ start, end, session: requests.Session | None = None
568
+ ) -> dict[pd.Timestamp, str]:
569
+ """NCEI's accessions of hourly totals for the months that overlap [start, end): {first of month: accession}.
570
+ A month listed twice keeps the higher (later) accession number."""
571
+ session = _client(session)
572
+ first = _utc_naive(start).to_period("M").to_timestamp()
573
+ last = (_utc_naive(end) - pd.Timedelta(seconds=1)).to_period("M").to_timestamp()
574
+ found: dict[pd.Timestamp, str] = {}
575
+ for series in NCEI_SERIES:
576
+ at = 1
577
+ while True:
578
+ page = _get(
579
+ session,
580
+ NCEI_SEARCH,
581
+ params={"q": series, "start": at, "num": 200, "f": "json"},
582
+ ).json()
583
+ for item in page.get("results", []):
584
+ if not str(item.get("id", "")).startswith(series + "."):
585
+ continue
586
+ m = _NCEI_TITLE.search(item.get("title", ""))
587
+ if not m:
588
+ continue
589
+ month = pd.Timestamp(f"{m.group(1)} 1 {m.group(2)}")
590
+ if first <= month <= last and int(m.group(3)) > int(
591
+ found.get(month, "0")
592
+ ):
593
+ found[month] = m.group(3)
594
+ nxt = page.get("nextStart")
595
+ if not nxt or nxt <= at or nxt > page.get("total", 0):
596
+ break
597
+ at = nxt
598
+ return dict(sorted(found.items()))
599
+
600
+
601
+ def ncei_accession_dir(
602
+ accession: str, session: requests.Session | None = None, known: dict | None = None
603
+ ) -> str:
604
+ """The folder of an accession's US East and Gulf Coast totals. NCEI files accessions under numbered ``arc``
605
+ folders; the newest are searched first. ``known`` (accession -> folder) is consulted and updated."""
606
+ if known is not None and accession in known:
607
+ return known[accession]
608
+ session = _client(session)
609
+ arcs = sorted(
610
+ set(re.findall(r'href="(arc\d+)/"', _get(session, NCEI_ARCHIVE).text)),
611
+ reverse=True,
612
+ )
613
+ for arc in arcs:
614
+ if f'href="{accession}/"' in _get(session, f"{NCEI_ARCHIVE}{arc}/").text:
615
+ url = f"{NCEI_ARCHIVE}{arc}/{accession}/1.1/data/0-data/USEGC/"
616
+ if known is not None:
617
+ known[accession] = url
618
+ return url
619
+ raise RuntimeError(f"NCEI accession {accession} not found under {NCEI_ARCHIVE}")
620
+
621
+
622
+ def ncei_hours(
623
+ folder: str, grid: str, session: requests.Session | None = None
624
+ ) -> dict[pd.Timestamp, str]:
625
+ """The hourly files of one grid in an accession's folder: {hour: URL}; a reissued hour keeps the latest file."""
626
+ best: dict[pd.Timestamp, tuple[str, str]] = {}
627
+ for name, g, start, created in _NCEI_FILE.findall(
628
+ _get(_client(session), folder).text
629
+ ):
630
+ if g != grid:
631
+ continue
632
+ hour = pd.Timestamp(datetime.strptime(start, "%Y%m%d%H%M"))
633
+ if hour not in best or created > best[hour][1]:
634
+ best[hour] = (name, created)
635
+ return {h: folder + n for h, (n, _) in sorted(best.items())}
636
+
637
+
638
+ def _ncei_hour(
639
+ session: requests.Session, url: str, bbox: BBox
640
+ ) -> tuple[xr.Dataset, dict]:
641
+ """One hourly file, the box cut out, values as stored."""
642
+ data = _get(session, url).content
643
+ with xr.open_dataset(
644
+ io.BytesIO(data), engine="h5netcdf", mask_and_scale=False
645
+ ) as ds:
646
+ ys, xs = box_slices(ds["lat"].values, ds["lon"].values, bbox)
647
+ names = [n for n in VARIABLES + SCALARS if n in ds]
648
+ return ds[names].isel(lat=ys, lon=xs).load(), dict(ds.attrs)
649
+
650
+
651
+ def _read_index(store: Store) -> tuple[dict, Path | None]:
652
+ root = getattr(store, "root", None)
653
+ if root is None:
654
+ return {}, None
655
+ path = Path(root) / NCEI_INDEX
656
+ try:
657
+ return json.loads(path.read_text()), path
658
+ except (OSError, ValueError):
659
+ return {}, path
660
+
661
+
662
+ def archive_ncei(
663
+ bbox: BBox,
664
+ start,
665
+ end,
666
+ grids: Sequence[str] = GRIDS,
667
+ store: Store | None = None,
668
+ session: requests.Session | None = None,
669
+ workers: int = 4,
670
+ ) -> dict[str, list[str]]:
671
+ """Archive the box's days in [start, end) from NCEI's monthly accessions, per grid, skipping days already kept
672
+ (from either source); return the names written per grid. A day is written with the hours NCEI holds for it.
673
+ Days in months NCEI has not published are left for NDBC. A day that fails is skipped and the rest carry on; the
674
+ call then raises, naming the days missed."""
675
+ session = _client(session)
676
+ store = store if store is not None else LocalStore()
677
+ t0, t1 = _utc_naive(start).normalize(), _utc_naive(end)
678
+ days = pd.date_range(t0, (t1 - pd.Timedelta(seconds=1)).normalize(), freq="D")
679
+ months = ncei_months(t0, t1, session)
680
+ known, index_path = _read_index(store)
681
+ written: dict[str, list[str]] = {g: [] for g in grids}
682
+ failed: list[str] = []
683
+ for grid in grids:
684
+ for month, accession in months.items():
685
+ todo = [
686
+ d
687
+ for d in days
688
+ if d.to_period("M").to_timestamp() == month
689
+ and not store.has(day_name(grid, bbox, d))
690
+ ]
691
+ if not todo:
692
+ continue
693
+ try:
694
+ folder = ncei_accession_dir(accession, session, known)
695
+ hours = ncei_hours(folder, grid, session)
696
+ except Exception as exc:
697
+ logger.warning(
698
+ "NCEI %s %s (%s): %s", grid, f"{month:%Y-%m}", accession, exc
699
+ )
700
+ failed.append(f"{grid} {month:%Y-%m}")
701
+ continue
702
+ if index_path is not None:
703
+ index_path.parent.mkdir(parents=True, exist_ok=True)
704
+ index_path.write_text(json.dumps(known, indent=1, sort_keys=True))
705
+ for day in todo:
706
+ urls = [u for h, u in hours.items() if h.normalize() == day]
707
+ if not urls:
708
+ logger.info(
709
+ "NCEI %s %s: no hours in accession %s",
710
+ grid,
711
+ day.date(),
712
+ accession,
713
+ )
714
+ continue
715
+ try:
716
+ with ThreadPoolExecutor(max_workers=workers) as pool:
717
+ got = list(
718
+ pool.map(lambda u: _ncei_hour(session, u, bbox), urls)
719
+ )
720
+ except Exception as exc: # carry on; a later run fills the gap
721
+ logger.warning("NCEI %s %s: %s", grid, day.date(), exc)
722
+ failed.append(f"{grid} {day:%Y-%m-%d}")
723
+ continue
724
+ sub = xr.concat(
725
+ [g for g, _ in got],
726
+ dim="time",
727
+ data_vars="minimal",
728
+ coords="minimal",
729
+ compat="override",
730
+ join="exact",
731
+ ).sortby("time")
732
+ ds = _normalize(
733
+ sub,
734
+ grid,
735
+ bbox,
736
+ got[0][1],
737
+ datetime.now(timezone.utc),
738
+ source=f"{NCEI}/archive/accession/{accession} ({folder})",
739
+ credit=NCEI_CREDIT,
740
+ )
741
+ name = day_name(grid, bbox, day)
742
+ if store.put(name, day_file(ds, day)):
743
+ written[grid].append(name)
744
+ logger.info(
745
+ "%s written (%d h, NCEI %s)", name, len(urls), accession
746
+ )
747
+ if failed:
748
+ raise RuntimeError(
749
+ f"HF radar days not read from NCEI: {', '.join(failed)} (written: {written})"
750
+ )
751
+ return written
752
+
753
+
754
+ def main(argv: Sequence[str] | None = None) -> None:
755
+ """Archive a box from NDBC (the last ~3 months) or NCEI (older months)."""
756
+ p = argparse.ArgumentParser(
757
+ prog="python -m forcingkit.fetchers.hfradar", description=main.__doc__
758
+ )
759
+ p.add_argument("source", choices=["ndbc", "ncei"])
760
+ p.add_argument(
761
+ "--box", nargs=4, type=float, required=True, metavar=("W", "S", "E", "N")
762
+ )
763
+ p.add_argument(
764
+ "--start", help="first day (ndbc: or 'all'; default the last 10 days)"
765
+ )
766
+ p.add_argument("--end", help="ncei: the day after the last; ndbc: the last day")
767
+ p.add_argument("--grids", nargs="+", default=list(GRIDS), choices=list(GRIDS))
768
+ p.add_argument(
769
+ "--root",
770
+ help="the archive's root (default: FORCINGKIT_HFRADAR_ARCHIVE, else the cache)",
771
+ )
772
+ p.add_argument(
773
+ "--workers", type=int, default=4, help="ncei: hourly files read at once"
774
+ )
775
+ p.add_argument("-v", "--verbose", action="store_true")
776
+ a = p.parse_args(argv)
777
+ logging.basicConfig(
778
+ level=logging.INFO if a.verbose else logging.WARNING,
779
+ format="%(asctime)s %(levelname)s %(name)s: %(message)s",
780
+ )
781
+ store = LocalStore(a.root)
782
+ if a.source == "ncei":
783
+ if not (a.start and a.end):
784
+ p.error("ncei needs --start and --end")
785
+ out = archive_ncei(
786
+ a.box, a.start, a.end, a.grids, store=store, workers=a.workers
787
+ )
788
+ else:
789
+ out = archive(a.box, a.start, a.end, a.grids, store=store)
790
+ print(json.dumps({g: len(n) for g, n in out.items()}))
791
+
792
+
793
+ if __name__ == "__main__":
794
+ main()
@@ -82,6 +82,13 @@ def nwps_archive() -> str | None:
82
82
  return env("FORCINGKIT_NWPS_ARCHIVE") or None
83
83
 
84
84
 
85
+ def hfradar_archive() -> str | None:
86
+ """Where the HF radar archive lives (FORCINGKIT_HFRADAR_ARCHIVE), when it should not sit in the cache: the
87
+ archive keeps days the servers no longer hold, so a cache purge must never reach it. Unset or empty means
88
+ ``cache_dir("hfradar")``."""
89
+ return env("FORCINGKIT_HFRADAR_ARCHIVE") or None
90
+
91
+
85
92
  def erddap_timeouts() -> tuple[float, float]:
86
93
  """(connect, read) timeouts in seconds for ERDDAP profile reads
87
94
  (FORCINGKIT_ERDDAP_CONNECT_TIMEOUT, default 10; FORCINGKIT_ERDDAP_READ_TIMEOUT, default 60)."""