forcingkit 0.18.1__tar.gz → 0.19.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (147) hide show
  1. {forcingkit-0.18.1 → forcingkit-0.19.0}/.agents/forcingkit.md +3 -1
  2. {forcingkit-0.18.1 → forcingkit-0.19.0}/PKG-INFO +1 -1
  3. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/conf.py +1 -1
  4. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/fetchers.rst +15 -0
  5. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/sources.rst +9 -0
  6. {forcingkit-0.18.1 → forcingkit-0.19.0}/pyproject.toml +1 -1
  7. forcingkit-0.19.0/src/forcingkit/fetchers/hfradar.py +500 -0
  8. {forcingkit-0.18.1 → forcingkit-0.19.0}/static/index.html +1 -1
  9. {forcingkit-0.18.1 → forcingkit-0.19.0}/static/preview3d.js +3 -3
  10. forcingkit-0.19.0/tests/unit/test_hfradar.py +237 -0
  11. {forcingkit-0.18.1 → forcingkit-0.19.0}/uv.lock +1 -1
  12. {forcingkit-0.18.1 → forcingkit-0.19.0}/.claude/CLAUDE.md +0 -0
  13. {forcingkit-0.18.1 → forcingkit-0.19.0}/.env.template +0 -0
  14. {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/copilot-instructions.md +0 -0
  15. {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/workflows/docs.yml +0 -0
  16. {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/workflows/publish-client.yml +0 -0
  17. {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/workflows/publish.yml +0 -0
  18. {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/workflows/tests.yml +0 -0
  19. {forcingkit-0.18.1 → forcingkit-0.19.0}/.gitignore +0 -0
  20. {forcingkit-0.18.1 → forcingkit-0.19.0}/.markdownlint.yaml +0 -0
  21. {forcingkit-0.18.1 → forcingkit-0.19.0}/.pre-commit-config.yaml +0 -0
  22. {forcingkit-0.18.1 → forcingkit-0.19.0}/.python-version +0 -0
  23. {forcingkit-0.18.1 → forcingkit-0.19.0}/AGENTS.md +0 -0
  24. {forcingkit-0.18.1 → forcingkit-0.19.0}/CONTRIBUTING.md +0 -0
  25. {forcingkit-0.18.1 → forcingkit-0.19.0}/Dockerfile +0 -0
  26. {forcingkit-0.18.1 → forcingkit-0.19.0}/LICENSE +0 -0
  27. {forcingkit-0.18.1 → forcingkit-0.19.0}/README.md +0 -0
  28. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/LICENSE +0 -0
  29. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/README.md +0 -0
  30. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/pyproject.toml +0 -0
  31. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/__init__.py +0 -0
  32. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/client.py +0 -0
  33. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/schema.py +0 -0
  34. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/store.py +0 -0
  35. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/trajectory.py +0 -0
  36. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/tests/conftest.py +0 -0
  37. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/tests/test_client.py +0 -0
  38. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/tests/test_schema.py +0 -0
  39. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/tests/test_trajectory.py +0 -0
  40. {forcingkit-0.18.1 → forcingkit-0.19.0}/client/uv.lock +0 -0
  41. {forcingkit-0.18.1 → forcingkit-0.19.0}/docker-compose.yml +0 -0
  42. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/Makefile +0 -0
  43. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/make.bat +0 -0
  44. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/requirements-docs.txt +0 -0
  45. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/_extra/CNAME +0 -0
  46. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/_static/custom.css +0 -0
  47. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  48. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/atmospheric_forcing.rst +0 -0
  49. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/client.rst +0 -0
  50. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/hycom.rst +0 -0
  51. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/index.rst +0 -0
  52. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/layered_mesh.rst +0 -0
  53. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/mesh_frames.rst +0 -0
  54. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/necofs.rst +0 -0
  55. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/nyofs.rst +0 -0
  56. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/rivers.rst +0 -0
  57. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/roadmap.rst +0 -0
  58. {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/waves.rst +0 -0
  59. {forcingkit-0.18.1 → forcingkit-0.19.0}/main.py +0 -0
  60. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/__init__.py +0 -0
  61. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/__init__.py +0 -0
  62. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/jobs.py +0 -0
  63. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/main.py +0 -0
  64. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  65. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  66. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/removed.py +0 -0
  67. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/stores.py +0 -0
  68. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/viewer.py +0 -0
  69. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/stac.py +0 -0
  70. {forcingkit-0.18.1 → forcingkit-0.19.0}/service/run_server.py +0 -0
  71. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/__init__.py +0 -0
  72. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  73. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/dispatcher.py +0 -0
  74. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
  75. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/dbofs.py +0 -0
  76. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/erddap.py +0 -0
  77. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrdps.py +0 -0
  78. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrrr.py +0 -0
  79. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  80. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  81. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  82. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hycom.py +0 -0
  83. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hydrography.py +0 -0
  84. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/ndbc.py +0 -0
  85. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/necofs.py +0 -0
  86. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/noaa.py +0 -0
  87. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/nwps.py +0 -0
  88. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/nyofs.py +0 -0
  89. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/usgs.py +0 -0
  90. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  91. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/layered_mesh.py +0 -0
  92. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/mesh_frames.py +0 -0
  93. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
  94. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  95. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  96. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  97. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/render.py +0 -0
  98. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/settings.py +0 -0
  99. {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/zarr_stream.py +0 -0
  100. {forcingkit-0.18.1 → forcingkit-0.19.0}/static/app.js +0 -0
  101. {forcingkit-0.18.1 → forcingkit-0.19.0}/static/favicon.ico +0 -0
  102. {forcingkit-0.18.1 → forcingkit-0.19.0}/static/favicon.svg +0 -0
  103. {forcingkit-0.18.1 → forcingkit-0.19.0}/static/logo.svg +0 -0
  104. {forcingkit-0.18.1 → forcingkit-0.19.0}/static/styles.css +0 -0
  105. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/__init__.py +0 -0
  106. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_auth.py +0 -0
  107. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_erddap_fetch.py +0 -0
  108. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
  109. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_necofs_volume_live.py +0 -0
  110. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_nwps_nomads.py +0 -0
  111. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  112. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_obc_offshore_nj.py +0 -0
  113. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_ofs_archive_paths.py +0 -0
  114. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
  115. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/scripts/inspect_dem.py +0 -0
  116. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/scripts/inspect_grib.py +0 -0
  117. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/scripts/inspect_zarr.py +0 -0
  118. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_coastwatch.py +0 -0
  119. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_dbofs.py +0 -0
  120. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_dispatcher.py +0 -0
  121. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_erddap.py +0 -0
  122. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrdps.py +0 -0
  123. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrrr_atm.py +0 -0
  124. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrrr_idx.py +0 -0
  125. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrrr_levels.py +0 -0
  126. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrrr_points.py +0 -0
  127. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hycom.py +0 -0
  128. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_jobs.py +0 -0
  129. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_layered_mesh.py +0 -0
  130. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_main.py +0 -0
  131. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_mesh_frames.py +0 -0
  132. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_ndbc.py +0 -0
  133. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_necofs_hindcast.py +0 -0
  134. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
  135. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_necofs_parent.py +0 -0
  136. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_necofs_volume.py +0 -0
  137. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_noaa_currents.py +0 -0
  138. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_noaa_datum.py +0 -0
  139. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_nwps.py +0 -0
  140. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_nyofs.py +0 -0
  141. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_obc_donor_report.py +0 -0
  142. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_obc_pipeline.py +0 -0
  143. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_removed_routes.py +0 -0
  144. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_settings.py +0 -0
  145. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_usgs.py +0 -0
  146. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_usgs_rivers.py +0 -0
  147. {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_zarr_stream.py +0 -0
@@ -95,4 +95,6 @@ uv run python -c "from forcingkit.fetchers import nyofs; print(nyofs.get_metadat
95
95
  - **Cache keys**: Cache Zarr keys are deterministic hashes of bbox
96
96
  - time window. Changing fetcher output schema (variable names, dims)
97
97
  will miss existing cache entries — purge
98
- `~/.cache/forcingkit/` when making breaking schema changes.
98
+ `~/.cache/forcingkit/` when making breaking schema changes, except
99
+ `hfradar/`: that is the HF radar archive (`fetchers/hfradar.py`), days the
100
+ server no longer holds, not a cache.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.18.1
3
+ Version: 0.19.0
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.18.1"
12
+ release = "0.19.0"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -177,6 +177,21 @@ These serve validation rather than forcing:
177
177
  are read at once, through `insitukit <https://github.com/lhzn-io/insitukit>`__'s ERDDAP client
178
178
  (one attempt each, starting at least 0.2 s apart).
179
179
 
180
+ - **HF radar** surface currents (library only: ``forcingkit.fetchers.hfradar``), the hourly total
181
+ vectors of the `NOAA IOOS HF Radar National Network <https://ioos.noaa.gov/project/hf-radar/>`__
182
+ (HFRNet) for the US East and Gulf Coast at 1 km and 2 km, from NDBC's `THREDDS server
183
+ <https://dods.ndbc.noaa.gov/thredds/catalog/hfradar/catalog.html>`__ over OPeNDAP (since 0.19.0).
184
+ They represent roughly the upper 0.5 m, for scoring a model's surface layer. ``fetch`` returns a box
185
+ and window as stored on the server (16-bit integers with scale 0.01, so nothing is rounded twice).
186
+ The server keeps only about 92 days, so ``archive`` keeps a box one NetCDF-4 file per UTC day under
187
+ ``FORCINGKIT_CACHE_DIR/hfradar/usegc_<grid>/<box>/<YYYY>/<MM>/``, written once the server has
188
+ published past the day's end and six hours have passed, and never rewritten; it takes any store
189
+ with ``has`` and ``put`` (a create-only bucket, for example). ``load`` reads a box and window back
190
+ in m/s from any archived box that contains it, and the other days from the server while it holds
191
+ them. The ``hfradar`` directory is an archive, not a cache: do not purge it with the Zarr stores.
192
+ Coverage is uneven: in western Long Island Sound about 2 % of the 1 km cells in the box
193
+ 40.85-41.3 N, 73.8-71.8 W have a solution in a covered hour, and whole weeks have none.
194
+
180
195
  NOAA OFS archive locations
181
196
  --------------------------
182
197
 
@@ -71,6 +71,13 @@ Providers
71
71
  - Acknowledge NOAA NESDIS CoastWatch and the datasets named in the store's ``products``
72
72
  attribute. MUR: JPL MUR MEaSUREs Project (2015) and Chin et al. (2017). OLCI and DINEOF
73
73
  products require "Contains modified Copernicus Sentinel data [year]".
74
+ * - HF radar surface currents
75
+ - The radar sites' operators (for western Long Island Sound, the `University of Connecticut
76
+ <https://lisicos.uconn.edu/>`__ LISICOS program), combined by the `NOAA IOOS HF Radar
77
+ National Network <https://ioos.noaa.gov/project/hf-radar/>`__ (HFRNet) and served by `NOAA
78
+ NDBC <https://www.ndbc.noaa.gov/>`__
79
+ - Acknowledge the site operators and HFRNet (Terrill et al., 2006), as in each file's
80
+ ``source_credit`` attribute.
74
81
  * - Water-column profiles
75
82
  - `University of Connecticut Department of Marine Sciences
76
83
  <https://marinesciences.uconn.edu/>`__ (`ERDDAP
@@ -140,6 +147,8 @@ References
140
147
  split-explicit, free-surface, topography-following-coordinate oceanic model. *Ocean
141
148
  Modelling*, 9(4), 347-404.
142
149
  `doi:10.1016/j.ocemod.2004.08.002 <https://doi.org/10.1016/j.ocemod.2004.08.002>`__
150
+ - Terrill, E., M. Otero, L. Hazard, et al. (2006). Data management and real-time distribution in
151
+ the HF-radar national network. *Proceedings of the MTS/IEEE OCEANS 2006 Conference*, Boston, MA.
143
152
  - Tsujino, H., S. Urakawa, H. Nakano, et al. (2018). JRA-55 based surface dataset for driving
144
153
  ocean-sea-ice models (JRA55-do). *Ocean Modelling*, 130, 79-139.
145
154
  `doi:10.1016/j.ocemod.2018.07.002 <https://doi.org/10.1016/j.ocemod.2018.07.002>`__
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.18.1"
7
+ version = "0.19.0"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -0,0 +1,500 @@
1
+ """Hourly HF radar surface currents from the NOAA IOOS HF Radar National Network, and an archive that outlasts the
2
+ server's rolling window.
3
+
4
+ Shore-based HF radars measure radial surface currents; the HF Radar National Network (HFRNet) combines them into
5
+ hourly total vectors on fixed 1 km, 2 km and 6 km grids. NOAA's National Data Buoy Center serves the US East and Gulf
6
+ Coast totals over OPeNDAP (``hfradar_usegc_1km``, ``_2km``), but keeps only about 92 days. The values are surface
7
+ currents representative of roughly the upper 0.5 m: an observation for scoring a model's surface layer, not a
8
+ forcing.
9
+
10
+ `fetch` reads a box and a time window from the server as stored: 16-bit integers with their scale factor
11
+ (0.01) and fill value, so nothing is rounded twice. DAP2 has no signed byte, so a signed byte arrives unsigned with
12
+ ``_Unsigned = "false"``; it is turned back into int8 here. The variables are ``u`` and ``v`` (eastward and northward
13
+ surface current, m/s), ``dopx``, ``dopy`` and ``hdop`` (dilution of precision, for screening the geometry),
14
+ ``number_of_sites`` and ``number_of_radials``, with the grid mapping and the processing settings.
15
+
16
+ `archive` keeps a box one file per UTC day and grid (NetCDF-4), under ``settings.cache_dir("hfradar")`` unless
17
+ another store is given::
18
+
19
+ usegc_<grid>/<box>/<YYYY>/<MM>/hfradar_usegc_<grid>_<YYYYMMDD>.nc
20
+
21
+ where ``<box>`` names the box, for example ``40.850N_41.300N_073.800W_071.800W``. A day is written once the server
22
+ has published past its end and six hours have gone by, and never rewritten; every hour the server holds for the day is
23
+ kept, with no screening. Unlike the Zarr stores beside it, this directory is an archive: the server cannot supply
24
+ those days again once they leave its window, so do not purge it. `load` reads a box and window back, decoded to m/s,
25
+ from any archived box that contains it, and fetches days the archive lacks from the server while it still holds them.
26
+
27
+ Credit: the radar sites' operators (named in HFRNet's site list; for example the University of Connecticut's LISICOS
28
+ for western Long Island Sound) and the NOAA IOOS HF Radar National Network; see ``source_credit`` in each file.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import io
34
+ import logging
35
+ import os
36
+ import re
37
+ from collections.abc import Sequence
38
+ from datetime import datetime, timedelta, timezone
39
+ from pathlib import Path
40
+ from typing import Protocol
41
+
42
+ import numpy as np
43
+ import pandas as pd
44
+ import requests
45
+ import xarray as xr
46
+
47
+ from forcingkit import settings
48
+
49
+ logger = logging.getLogger(__name__)
50
+
51
+ SERVER = "https://dods.ndbc.noaa.gov/thredds/dodsC"
52
+ USER_AGENT = "forcingkit (+https://github.com/lhzn-io/forcingkit)"
53
+ GRIDS = ("1km", "2km")
54
+ VARIABLES = ["u", "v", "dopx", "dopy", "hdop", "number_of_sites", "number_of_radials"]
55
+ # The grid mapping and the processing settings. (The server's radial_metadata names the radial files of its latest
56
+ # hour only, so it is left out.)
57
+ SCALARS = ["wgs84", "processing_parameters"]
58
+ LAG = timedelta(
59
+ hours=6
60
+ ) # after a day ends, before it is written (the server posts an hour about 2 h late)
61
+ LOOKBACK_DAYS = 10 # what a routine `archive` run looks back over
62
+ CHUNK_DAYS = 8 # days per request
63
+ # Global attributes carried over from the server's dataset (not the ones describing its latest hour).
64
+ KEEP_ATTRS = [
65
+ "Conventions",
66
+ "title",
67
+ "summary",
68
+ "institution",
69
+ "program",
70
+ "project",
71
+ "naming_authority",
72
+ "license",
73
+ "acknowledgment",
74
+ "acknowledgement",
75
+ "references",
76
+ "creator_name",
77
+ "creator_url",
78
+ "creator_email",
79
+ "publisher_name",
80
+ "publisher_url",
81
+ "publisher_email",
82
+ "contributor_name",
83
+ "contributor_role",
84
+ "processing_level",
85
+ "instrument",
86
+ "platform",
87
+ "product_version",
88
+ "metadata_link",
89
+ "keywords",
90
+ "keywords_vocabulary",
91
+ "standard_name_vocabulary",
92
+ ]
93
+ CREDIT = (
94
+ "HF radar sites operated by their institutions (for western Long Island Sound, the University of Connecticut's "
95
+ "LISICOS); hourly total vectors from the NOAA IOOS HF Radar National Network (HFRNet), served by NOAA NDBC."
96
+ )
97
+ NAME = re.compile(
98
+ r"usegc_[126]km/\d{2}\.\d{3}[NS]_\d{2}\.\d{3}[NS]_\d{3}\.\d{3}[EW]_\d{3}\.\d{3}[EW]/\d{4}/\d{2}/"
99
+ r"hfradar_usegc_[126]km_\d{8}\.nc"
100
+ )
101
+ _BOX = re.compile(
102
+ r"(\d{2}\.\d{3})([NS])_(\d{2}\.\d{3})([NS])_(\d{3}\.\d{3})([EW])_(\d{3}\.\d{3})([EW])"
103
+ )
104
+
105
+ BBox = Sequence[float] # [min_lon, min_lat, max_lon, max_lat]
106
+
107
+
108
+ class Store(Protocol):
109
+ """Where archived days go. `has` may answer False when it cannot tell (a create-only bucket); `put` then refuses
110
+ an existing file and returns False."""
111
+
112
+ def has(self, name: str) -> bool: ...
113
+
114
+ def put(self, name: str, data: bytes) -> bool: ...
115
+
116
+
117
+ class LocalStore:
118
+ """A directory; files are created once and never replaced."""
119
+
120
+ def __init__(self, root: str | os.PathLike | None = None):
121
+ self.root = Path(
122
+ root if root is not None else settings.cache_dir("hfradar")
123
+ ).expanduser()
124
+
125
+ def has(self, name: str) -> bool:
126
+ return (self.root / name).exists()
127
+
128
+ def put(self, name: str, data: bytes) -> bool:
129
+ if not NAME.fullmatch(name):
130
+ raise ValueError(f"refusing to write {name}")
131
+ path = self.root / name
132
+ if path.exists():
133
+ return False
134
+ path.parent.mkdir(parents=True, exist_ok=True)
135
+ tmp = path.with_suffix(".part")
136
+ tmp.write_bytes(data)
137
+ tmp.rename(path)
138
+ return True
139
+
140
+
141
+ def dataset_url(grid: str) -> str:
142
+ return f"{SERVER}/hfradar_usegc_{grid}"
143
+
144
+
145
+ def box_key(bbox: BBox) -> str:
146
+ """The box's directory name, to three decimals: ``40.850N_41.300N_073.800W_071.800W`` (south, north, west, east)."""
147
+ w, s, e, n = (float(x) for x in bbox)
148
+
149
+ def lat(v: float) -> str:
150
+ return f"{abs(v):06.3f}{'N' if v >= 0 else 'S'}"
151
+
152
+ def lon(v: float) -> str:
153
+ return f"{abs(v):07.3f}{'E' if v >= 0 else 'W'}"
154
+
155
+ return f"{lat(s)}_{lat(n)}_{lon(w)}_{lon(e)}"
156
+
157
+
158
+ def parse_box_key(key: str) -> tuple[float, float, float, float] | None:
159
+ """The bbox ([min_lon, min_lat, max_lon, max_lat]) a directory name stands for, or None."""
160
+ m = _BOX.fullmatch(key)
161
+ if not m:
162
+ return None
163
+ s, sh, n, nh, w, wh, e, eh = m.groups()
164
+ sign = {"N": 1, "S": -1, "E": 1, "W": -1}
165
+ return (
166
+ float(w) * sign[wh],
167
+ float(s) * sign[sh],
168
+ float(e) * sign[eh],
169
+ float(n) * sign[nh],
170
+ )
171
+
172
+
173
+ def day_name(grid: str, bbox: BBox, day: pd.Timestamp) -> str:
174
+ return (
175
+ f"usegc_{grid}/{box_key(bbox)}/{day:%Y/%m}/hfradar_usegc_{grid}_{day:%Y%m%d}.nc"
176
+ )
177
+
178
+
179
+ def open_remote(
180
+ grid: str = "1km", session: requests.Session | None = None
181
+ ) -> xr.Dataset:
182
+ """The server's dataset, lazily, with the stored integers left as they are."""
183
+ if session is None:
184
+ session = requests.Session()
185
+ session.headers["User-Agent"] = USER_AGENT
186
+ return xr.open_dataset(
187
+ dataset_url(grid).replace("https://", "dap2://"),
188
+ engine="pydap",
189
+ session=session,
190
+ mask_and_scale=False,
191
+ )
192
+
193
+
194
+ def box_slices(lat: np.ndarray, lon: np.ndarray, bbox: BBox) -> tuple[slice, slice]:
195
+ """Index slices of the grid cells whose centres fall in the box."""
196
+ w, s, e, n = bbox
197
+ iy = np.flatnonzero((lat >= s) & (lat <= n))
198
+ ix = np.flatnonzero((lon >= w) & (lon <= e))
199
+ if not len(iy) or not len(ix):
200
+ raise ValueError("the box misses the grid")
201
+ return slice(int(iy[0]), int(iy[-1]) + 1), slice(int(ix[0]), int(ix[-1]) + 1)
202
+
203
+
204
+ def _utc_naive(t) -> pd.Timestamp:
205
+ ts = pd.Timestamp(t)
206
+ return ts.tz_convert("UTC").tz_localize(None) if ts.tzinfo else ts
207
+
208
+
209
+ def ready_days(
210
+ times: pd.DatetimeIndex,
211
+ now: datetime,
212
+ start: pd.Timestamp | None = None,
213
+ end: pd.Timestamp | None = None,
214
+ ) -> list[pd.Timestamp]:
215
+ """The UTC days with hours on the server that may be archived: the server has published past the day's end and
216
+ the lag has passed. Bounded by ``start`` and ``end`` (days, inclusive) when given."""
217
+ if not len(times):
218
+ return []
219
+ last = times.max()
220
+ now_ts = _utc_naive(now)
221
+ days = []
222
+ for day in pd.DatetimeIndex(times.normalize().unique()).sort_values():
223
+ stop = day + pd.Timedelta(days=1)
224
+ if last < stop or now_ts < stop + LAG:
225
+ continue
226
+ if (start is not None and day < start) or (end is not None and day > end):
227
+ continue
228
+ days.append(day)
229
+ return days
230
+
231
+
232
+ def runs(days: list[pd.Timestamp], most: int = CHUNK_DAYS) -> list[list[pd.Timestamp]]:
233
+ """Consecutive days grouped into requests of at most ``most`` days."""
234
+ out: list[list[pd.Timestamp]] = []
235
+ for day in days:
236
+ if out and day - out[-1][-1] == pd.Timedelta(days=1) and len(out[-1]) < most:
237
+ out[-1].append(day)
238
+ else:
239
+ out.append([day])
240
+ return out
241
+
242
+
243
+ def _stored(var: xr.DataArray) -> tuple[np.ndarray, dict]:
244
+ """A variable's stored values and its attributes, minus the server's private ones, with signed bytes restored."""
245
+ values = np.asarray(var.values)
246
+ if (
247
+ values.dtype == np.uint8
248
+ and str(var.attrs.get("_Unsigned", "")).lower() == "false"
249
+ ):
250
+ values = values.view(np.int8)
251
+ attrs = {
252
+ k: v
253
+ for k, v in var.attrs.items()
254
+ if (not k.startswith("_") or k == "_FillValue") and k != "coordinates"
255
+ }
256
+ return values, attrs
257
+
258
+
259
+ def _normalize(
260
+ sub: xr.Dataset, grid: str, bbox: BBox, source_attrs: dict, fetched: datetime
261
+ ) -> xr.Dataset:
262
+ """The server's subset as a clean Dataset of stored values, with its encodings and credit."""
263
+ out = xr.Dataset(
264
+ coords={
265
+ "time": sub["time"].values,
266
+ "lat": sub["lat"].values,
267
+ "lon": sub["lon"].values,
268
+ }
269
+ )
270
+ for name in VARIABLES:
271
+ if name in sub:
272
+ values, attrs = _stored(sub[name])
273
+ fill = attrs.pop("_FillValue", None)
274
+ out[name] = (sub[name].dims, values, attrs)
275
+ out[name].encoding.update(
276
+ {"zlib": True, "complevel": 4, "shuffle": True, "dtype": values.dtype}
277
+ )
278
+ if fill is not None:
279
+ out[name].encoding["_FillValue"] = values.dtype.type(fill)
280
+ for name in SCALARS:
281
+ if name in sub:
282
+ values, attrs = _stored(sub[name])
283
+ out[name] = ((), values, attrs)
284
+ for c in ("lat", "lon"):
285
+ out[c].attrs = {k: v for k, v in sub[c].attrs.items() if not k.startswith("_")}
286
+ out["time"].attrs = {
287
+ "standard_name": "time",
288
+ "long_name": "time at the centre of the hour",
289
+ }
290
+ out["time"].encoding.update(
291
+ {"units": "seconds since 1970-01-01", "calendar": "gregorian", "dtype": "int32"}
292
+ )
293
+ attrs = {k: source_attrs[k] for k in KEEP_ATTRS if k in source_attrs}
294
+ w, s, e, n = bbox
295
+ attrs.update(
296
+ {
297
+ "source_dataset": dataset_url(grid),
298
+ "source_credit": CREDIT,
299
+ "subset": f"latitude {s} to {n}, longitude {w} to {e}; values as stored on the server",
300
+ "history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {dataset_url(grid)} by forcingkit",
301
+ }
302
+ )
303
+ out.attrs = attrs
304
+ return out
305
+
306
+
307
+ def fetch(
308
+ bbox: BBox,
309
+ start,
310
+ end,
311
+ grid: str = "1km",
312
+ src: xr.Dataset | None = None,
313
+ session: requests.Session | None = None,
314
+ ) -> xr.Dataset:
315
+ """The box's hours in [start, end) from the server, as stored (see the module notes). Pass an open `src` to reuse
316
+ one connection across calls. Empty in time when the server holds none of the window."""
317
+ own = src is None
318
+ src = src if src is not None else open_remote(grid, session)
319
+ try:
320
+ times = pd.DatetimeIndex(src["time"].values)
321
+ t = np.flatnonzero((times >= _utc_naive(start)) & (times < _utc_naive(end)))
322
+ ys, xs = box_slices(src["lat"].values, src["lon"].values, bbox)
323
+ names = [n for n in VARIABLES + SCALARS if n in src]
324
+ tsel = slice(int(t[0]), int(t[-1]) + 1) if len(t) else slice(0, 0)
325
+ sub = src[names].isel(time=tsel, lat=ys, lon=xs).load()
326
+ return _normalize(sub, grid, bbox, dict(src.attrs), datetime.now(timezone.utc))
327
+ finally:
328
+ if own:
329
+ src.close()
330
+
331
+
332
+ def day_file(ds: xr.Dataset, day: pd.Timestamp) -> bytes:
333
+ """One UTC day of a `fetch` result as NetCDF-4 bytes (the archive's file)."""
334
+ one = ds.sel(
335
+ time=slice(day, day + pd.Timedelta(days=1) - pd.Timedelta(seconds=1))
336
+ ).copy()
337
+ for name in VARIABLES: # sel keeps the encodings, but be explicit
338
+ if name in ds:
339
+ one[name].encoding = dict(ds[name].encoding)
340
+ one["time"].encoding = dict(ds["time"].encoding)
341
+ one.attrs = dict(
342
+ ds.attrs, subset_day=f"{day:%Y-%m-%d} UTC", subset_hours=int(one.sizes["time"])
343
+ )
344
+ buf = io.BytesIO()
345
+ one.to_netcdf(buf, engine="h5netcdf")
346
+ return buf.getvalue()
347
+
348
+
349
+ def archive(
350
+ bbox: BBox,
351
+ start: str | None = None,
352
+ end: str | None = None,
353
+ grids: Sequence[str] = GRIDS,
354
+ store: Store | None = None,
355
+ lookback_days: int = LOOKBACK_DAYS,
356
+ now: datetime | None = None,
357
+ session: requests.Session | None = None,
358
+ ) -> dict[str, list[str]]:
359
+ """Archive every ready day of the box not yet kept, per grid; return the names written per grid.
360
+
361
+ With neither ``start`` nor ``end``, the last ``lookback_days`` days; ``start="all"`` takes every day the server
362
+ still holds. The store defaults to ``settings.cache_dir("hfradar")``; ``session`` lets a caller send its own
363
+ User-Agent. A request that fails is skipped and the rest carry on; the call then raises, naming the days
364
+ missed."""
365
+ now = now or datetime.now(timezone.utc)
366
+ store = store if store is not None else LocalStore()
367
+ written: dict[str, list[str]] = {}
368
+ failed: list[str] = []
369
+ for grid in grids:
370
+ src = open_remote(grid, session)
371
+ try:
372
+ times = pd.DatetimeIndex(src["time"].values)
373
+ if start == "all":
374
+ first = None
375
+ elif start:
376
+ first = pd.Timestamp(start)
377
+ else:
378
+ first = _utc_naive(now).normalize() - pd.Timedelta(days=lookback_days)
379
+ last = pd.Timestamp(end) if end else None
380
+ todo = [
381
+ d
382
+ for d in ready_days(times, now, first, last)
383
+ if not store.has(day_name(grid, bbox, d))
384
+ ]
385
+ written[grid] = []
386
+ for group in runs(todo):
387
+ try:
388
+ ds = fetch(
389
+ bbox, group[0], group[-1] + pd.Timedelta(days=1), grid, src=src
390
+ )
391
+ except Exception as exc: # carry on; a later run fills the gap while the server holds it
392
+ logger.warning(
393
+ "HF radar %s %s to %s: %s",
394
+ grid,
395
+ group[0].date(),
396
+ group[-1].date(),
397
+ exc,
398
+ )
399
+ failed.append(f"{grid} {group[0]:%Y-%m-%d}..{group[-1]:%Y-%m-%d}")
400
+ continue
401
+ for day in group:
402
+ name = day_name(grid, bbox, day)
403
+ if store.put(name, day_file(ds, day)):
404
+ written[grid].append(name)
405
+ logger.info("%s written", name)
406
+ else:
407
+ logger.info("%s already kept", name)
408
+ finally:
409
+ src.close()
410
+ if failed:
411
+ raise RuntimeError(
412
+ f"HF radar days not fetched: {', '.join(failed)} (written: {written})"
413
+ )
414
+ return written
415
+
416
+
417
+ def _decoded(ds: xr.Dataset) -> xr.Dataset:
418
+ """Stored values decoded as a reader of the archive sees them (scale applied, fills as NaN)."""
419
+ buf = io.BytesIO()
420
+ ds.to_netcdf(buf, engine="h5netcdf")
421
+ buf.seek(0)
422
+ with xr.open_dataset(buf, engine="h5netcdf") as out:
423
+ return out.load()
424
+
425
+
426
+ def _containing_boxes(root: Path, grid: str, bbox: BBox) -> list[Path]:
427
+ """Archived box directories that contain the box, smallest first."""
428
+ found = []
429
+ for d in (root / f"usegc_{grid}").glob("*"):
430
+ b = parse_box_key(d.name)
431
+ if (
432
+ b
433
+ and b[0] <= bbox[0]
434
+ and b[1] <= bbox[1]
435
+ and b[2] >= bbox[2]
436
+ and b[3] >= bbox[3]
437
+ ):
438
+ found.append(((b[2] - b[0]) * (b[3] - b[1]), d))
439
+ return [d for _, d in sorted(found)]
440
+
441
+
442
+ def load(
443
+ bbox: BBox,
444
+ start,
445
+ end,
446
+ grid: str = "1km",
447
+ root: str | os.PathLike | None = None,
448
+ remote: bool = True,
449
+ session: requests.Session | None = None,
450
+ ) -> xr.Dataset:
451
+ """The box's hours in [start, end), decoded (u and v in m/s, NaN where there is no solution): archived days from
452
+ any archived box that contains this one, and, when ``remote``, the other days from the server while it still
453
+ holds them (not archived by this call). Empty in time when neither has any."""
454
+ base = Path(
455
+ root if root is not None else settings.cache_dir("hfradar")
456
+ ).expanduser()
457
+ t0, t1 = _utc_naive(start), _utc_naive(end)
458
+ boxes = _containing_boxes(base, grid, bbox)
459
+ parts: list[xr.Dataset] = []
460
+ missing: list[pd.Timestamp] = []
461
+ for day in pd.date_range(
462
+ t0.normalize(), (t1 - pd.Timedelta(seconds=1)).normalize(), freq="D"
463
+ ):
464
+ path = next(
465
+ (
466
+ p
467
+ for b in boxes
468
+ if (
469
+ p := b / f"{day:%Y/%m}" / f"hfradar_usegc_{grid}_{day:%Y%m%d}.nc"
470
+ ).exists()
471
+ ),
472
+ None,
473
+ )
474
+ if path is None:
475
+ missing.append(day)
476
+ continue
477
+ with xr.open_dataset(path) as ds:
478
+ ys, xs = box_slices(ds["lat"].values, ds["lon"].values, bbox)
479
+ parts.append(
480
+ ds[[v for v in VARIABLES if v in ds]].isel(lat=ys, lon=xs).load()
481
+ )
482
+ if remote and missing:
483
+ src = open_remote(grid, session)
484
+ try:
485
+ for group in runs(missing):
486
+ got = fetch(
487
+ bbox, group[0], group[-1] + pd.Timedelta(days=1), grid, src=src
488
+ )
489
+ if got.sizes["time"]:
490
+ parts.append(_decoded(got[[v for v in VARIABLES if v in got]]))
491
+ finally:
492
+ src.close()
493
+ if not parts:
494
+ return xr.Dataset()
495
+ out = xr.concat(
496
+ parts, dim="time", data_vars="all", coords="minimal", join="exact"
497
+ ).sortby("time")
498
+ out = out.sel(time=slice(t0, t1 - pd.Timedelta(seconds=1)))
499
+ out.attrs = {"source_dataset": dataset_url(grid), "source_credit": CREDIT}
500
+ return out
@@ -140,7 +140,7 @@
140
140
  { "imports": {
141
141
  "three": "https://cdn.jsdelivr.net/npm/three@0.170.0/build/three.module.js",
142
142
  "camera-controls": "https://cdn.jsdelivr.net/npm/camera-controls@3.1.2/dist/camera-controls.module.min.js",
143
- "vantagekit/three": "https://cdn.jsdelivr.net/gh/lhzn-io/vantagekit@v0.3.2/vantagekit-three.js"
143
+ "vantagekit/three": "https://cdn.jsdelivr.net/gh/lhzn-io/vantagekit@v0.8.1/vantagekit-three.js"
144
144
  } }
145
145
  </script>
146
146
  <script type="module" src="preview3d.js"></script>
@@ -138,12 +138,12 @@ function initThree(container) {
138
138
  camera = new THREE.PerspectiveCamera(55, container.clientWidth / container.clientHeight, 0.1, 5000);
139
139
  camera.position.set(0, 150, 250);
140
140
 
141
- // The scene is about 200 units across whatever the dataset's size, so metres per unit is set
141
+ // The scene is about 200 units across whatever the dataset's size, so meters per unit is set
142
142
  // per dataset (buildVectorField). The keyboard acts only while the canvas has focus, so the
143
143
  // page's own keys and scrolling keep working.
144
144
  nav = createVantage({
145
145
  THREE, CameraControls, camera, dom: canvas,
146
- metresPerUnit: 1, exaggeration: DEPTH_EXAG * DEPTH_BASE_MULT,
146
+ metersPerUnit: 1, exaggeration: DEPTH_EXAG * DEPTH_BASE_MULT,
147
147
  keys: 'focus', minDistance: 0.5, smoothTime: 0.2,
148
148
  });
149
149
 
@@ -207,7 +207,7 @@ function buildVectorField(data, fitCamera) {
207
207
  const latSpan = Math.max(latMax - latMin, 0.001) * degToM;
208
208
  const maxSpan = Math.max(lonSpan, latSpan);
209
209
  const sceneScale = 200 / maxSpan;
210
- nav.metresPerUnit = 1 / sceneScale; // movement speeds and readouts in real metres
210
+ nav.metersPerUnit = 1 / sceneScale; // movement speeds and readouts in real meters
211
211
 
212
212
  // Compute max magnitude
213
213
  let maxMag = 0;