forcingkit 0.17.1__tar.gz → 0.18.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. {forcingkit-0.17.1 → forcingkit-0.18.1}/PKG-INFO +2 -1
  2. {forcingkit-0.17.1 → forcingkit-0.18.1}/README.md +1 -0
  3. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/atmospheric_forcing.rst +52 -0
  4. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/conf.py +1 -1
  5. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/sources.rst +7 -0
  6. {forcingkit-0.17.1 → forcingkit-0.18.1}/pyproject.toml +1 -1
  7. forcingkit-0.18.1/src/forcingkit/fetchers/hrdps.py +402 -0
  8. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/hrrr_points.py +2 -1
  9. forcingkit-0.18.1/tests/unit/test_hrdps.py +250 -0
  10. {forcingkit-0.17.1 → forcingkit-0.18.1}/uv.lock +1 -1
  11. {forcingkit-0.17.1 → forcingkit-0.18.1}/.agents/forcingkit.md +0 -0
  12. {forcingkit-0.17.1 → forcingkit-0.18.1}/.claude/CLAUDE.md +0 -0
  13. {forcingkit-0.17.1 → forcingkit-0.18.1}/.env.template +0 -0
  14. {forcingkit-0.17.1 → forcingkit-0.18.1}/.github/copilot-instructions.md +0 -0
  15. {forcingkit-0.17.1 → forcingkit-0.18.1}/.github/workflows/docs.yml +0 -0
  16. {forcingkit-0.17.1 → forcingkit-0.18.1}/.github/workflows/publish-client.yml +0 -0
  17. {forcingkit-0.17.1 → forcingkit-0.18.1}/.github/workflows/publish.yml +0 -0
  18. {forcingkit-0.17.1 → forcingkit-0.18.1}/.github/workflows/tests.yml +0 -0
  19. {forcingkit-0.17.1 → forcingkit-0.18.1}/.gitignore +0 -0
  20. {forcingkit-0.17.1 → forcingkit-0.18.1}/.markdownlint.yaml +0 -0
  21. {forcingkit-0.17.1 → forcingkit-0.18.1}/.pre-commit-config.yaml +0 -0
  22. {forcingkit-0.17.1 → forcingkit-0.18.1}/.python-version +0 -0
  23. {forcingkit-0.17.1 → forcingkit-0.18.1}/AGENTS.md +0 -0
  24. {forcingkit-0.17.1 → forcingkit-0.18.1}/CONTRIBUTING.md +0 -0
  25. {forcingkit-0.17.1 → forcingkit-0.18.1}/Dockerfile +0 -0
  26. {forcingkit-0.17.1 → forcingkit-0.18.1}/LICENSE +0 -0
  27. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/LICENSE +0 -0
  28. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/README.md +0 -0
  29. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/pyproject.toml +0 -0
  30. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/src/forcingkit_client/__init__.py +0 -0
  31. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/src/forcingkit_client/client.py +0 -0
  32. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/src/forcingkit_client/schema.py +0 -0
  33. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/src/forcingkit_client/store.py +0 -0
  34. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/src/forcingkit_client/trajectory.py +0 -0
  35. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/tests/conftest.py +0 -0
  36. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/tests/test_client.py +0 -0
  37. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/tests/test_schema.py +0 -0
  38. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/tests/test_trajectory.py +0 -0
  39. {forcingkit-0.17.1 → forcingkit-0.18.1}/client/uv.lock +0 -0
  40. {forcingkit-0.17.1 → forcingkit-0.18.1}/docker-compose.yml +0 -0
  41. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/Makefile +0 -0
  42. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/make.bat +0 -0
  43. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/requirements-docs.txt +0 -0
  44. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/_extra/CNAME +0 -0
  45. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/_static/custom.css +0 -0
  46. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  47. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/client.rst +0 -0
  48. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/fetchers.rst +0 -0
  49. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/hycom.rst +0 -0
  50. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/index.rst +0 -0
  51. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/layered_mesh.rst +0 -0
  52. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/mesh_frames.rst +0 -0
  53. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/necofs.rst +0 -0
  54. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/nyofs.rst +0 -0
  55. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/rivers.rst +0 -0
  56. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/roadmap.rst +0 -0
  57. {forcingkit-0.17.1 → forcingkit-0.18.1}/docs/source/waves.rst +0 -0
  58. {forcingkit-0.17.1 → forcingkit-0.18.1}/main.py +0 -0
  59. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/__init__.py +0 -0
  60. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/__init__.py +0 -0
  61. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/jobs.py +0 -0
  62. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/main.py +0 -0
  63. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  64. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  65. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/routers/removed.py +0 -0
  66. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/routers/stores.py +0 -0
  67. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/routers/viewer.py +0 -0
  68. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/forcingkit_serve/stac.py +0 -0
  69. {forcingkit-0.17.1 → forcingkit-0.18.1}/service/run_server.py +0 -0
  70. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/__init__.py +0 -0
  71. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  72. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/dispatcher.py +0 -0
  73. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/coastwatch.py +0 -0
  74. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/dbofs.py +0 -0
  75. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/erddap.py +0 -0
  76. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/hrrr.py +0 -0
  77. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  78. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  79. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/hycom.py +0 -0
  80. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/hydrography.py +0 -0
  81. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/ndbc.py +0 -0
  82. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/necofs.py +0 -0
  83. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/noaa.py +0 -0
  84. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/nwps.py +0 -0
  85. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/nyofs.py +0 -0
  86. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/usgs.py +0 -0
  87. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  88. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/layered_mesh.py +0 -0
  89. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/mesh_frames.py +0 -0
  90. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/ramps/currents_lis.csv +0 -0
  91. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  92. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  93. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  94. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/render.py +0 -0
  95. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/settings.py +0 -0
  96. {forcingkit-0.17.1 → forcingkit-0.18.1}/src/forcingkit/zarr_stream.py +0 -0
  97. {forcingkit-0.17.1 → forcingkit-0.18.1}/static/app.js +0 -0
  98. {forcingkit-0.17.1 → forcingkit-0.18.1}/static/favicon.ico +0 -0
  99. {forcingkit-0.17.1 → forcingkit-0.18.1}/static/favicon.svg +0 -0
  100. {forcingkit-0.17.1 → forcingkit-0.18.1}/static/index.html +0 -0
  101. {forcingkit-0.17.1 → forcingkit-0.18.1}/static/logo.svg +0 -0
  102. {forcingkit-0.17.1 → forcingkit-0.18.1}/static/preview3d.js +0 -0
  103. {forcingkit-0.17.1 → forcingkit-0.18.1}/static/styles.css +0 -0
  104. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/__init__.py +0 -0
  105. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_auth.py +0 -0
  106. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_erddap_fetch.py +0 -0
  107. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_mesh_frames_necofs.py +0 -0
  108. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_necofs_volume_live.py +0 -0
  109. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_nwps_nomads.py +0 -0
  110. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  111. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_obc_offshore_nj.py +0 -0
  112. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_ofs_archive_paths.py +0 -0
  113. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/integration/test_s3_roms_fetchers.py +0 -0
  114. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/scripts/inspect_dem.py +0 -0
  115. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/scripts/inspect_grib.py +0 -0
  116. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/scripts/inspect_zarr.py +0 -0
  117. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_coastwatch.py +0 -0
  118. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_dbofs.py +0 -0
  119. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_dispatcher.py +0 -0
  120. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_erddap.py +0 -0
  121. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_hrrr_atm.py +0 -0
  122. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_hrrr_idx.py +0 -0
  123. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_hrrr_levels.py +0 -0
  124. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_hrrr_points.py +0 -0
  125. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_hycom.py +0 -0
  126. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_jobs.py +0 -0
  127. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_layered_mesh.py +0 -0
  128. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_main.py +0 -0
  129. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_mesh_frames.py +0 -0
  130. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_ndbc.py +0 -0
  131. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_necofs_hindcast.py +0 -0
  132. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_necofs_mesh_cache.py +0 -0
  133. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_necofs_parent.py +0 -0
  134. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_necofs_volume.py +0 -0
  135. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_noaa_currents.py +0 -0
  136. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_noaa_datum.py +0 -0
  137. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_nwps.py +0 -0
  138. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_nyofs.py +0 -0
  139. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_obc_donor_report.py +0 -0
  140. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_obc_pipeline.py +0 -0
  141. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_removed_routes.py +0 -0
  142. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_settings.py +0 -0
  143. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_usgs.py +0 -0
  144. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_usgs_rivers.py +0 -0
  145. {forcingkit-0.17.1 → forcingkit-0.18.1}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.17.1
3
+ Version: 0.18.1
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -281,6 +281,7 @@ The primary goal of forcingkit is to provide clean, standardized ocean and atmos
281
281
 
282
282
  - **Oceanic Forcing**: NYOFS, NECOFS, DBOFS, HYCOM (global, 1994 to present).
283
283
  - **Atmospheric Forcing**: HRRR (3 km, hourly, 2014-07-30 to present), regridded to 0.03 degrees.
284
+ - **Forecasts at Points**: HRRR (to 48 h) and ECCC's HRDPS (2.5 km, 10 m wind and gust to 48 h, from the MSC Datamart's 30-day window). HRDPS data: Data Source: Environment and Climate Change Canada ([license](https://eccc-msc.github.io/open-data/licence/readme_en/)).
284
285
  - **Surface Waves**: NWPS (one forecast office's grid per request, for example OKX CG1 at about 1.8 km, hourly, not regridded). NOMADS keeps about five days; older windows need an NWPS archive (`FORCINGKIT_NWPS_ARCHIVE`).
285
286
 
286
287
  ## Roadmap
@@ -34,6 +34,7 @@ The primary goal of forcingkit is to provide clean, standardized ocean and atmos
34
34
 
35
35
  - **Oceanic Forcing**: NYOFS, NECOFS, DBOFS, HYCOM (global, 1994 to present).
36
36
  - **Atmospheric Forcing**: HRRR (3 km, hourly, 2014-07-30 to present), regridded to 0.03 degrees.
37
+ - **Forecasts at Points**: HRRR (to 48 h) and ECCC's HRDPS (2.5 km, 10 m wind and gust to 48 h, from the MSC Datamart's 30-day window). HRDPS data: Data Source: Environment and Climate Change Canada ([license](https://eccc-msc.github.io/open-data/licence/readme_en/)).
37
38
  - **Surface Waves**: NWPS (one forecast office's grid per request, for example OKX CG1 at about 1.8 km, hourly, not regridded). NOMADS keeps about five days; older windows need an NWPS archive (`FORCINGKIT_NWPS_ARCHIVE`).
38
39
 
39
40
  ## Roadmap
@@ -47,6 +47,13 @@ Summary
47
47
  - Operational from 2026-10-06 (12 UTC cycle)
48
48
  - Hourly cycles to 18 h; to 84 h every 6 h
49
49
  - Not integrated; replaces NAM, SREF, HREF and HiresW, while HRRR stays in operations
50
+ * - **HRDPS** (`ECCC MSC <https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__)
51
+ - Forecast, GEM limited-area, 2.5 km
52
+ - 0.0225 degrees (about 2.5 km), rotated latitude-longitude over Canada and the northern US
53
+ - 1 h
54
+ - Runs on the MSC Datamart for the last 30 days; no longer public archive found
55
+ - 00, 06, 12 and 18 UTC runs to 48 h, complete about 3 h after the run
56
+ - **forcingkit** ``fetchers.hrdps.fetch_points`` (points only)
50
57
  * - **NAM CONUS Nest** (NOAA NCEP)
51
58
  - Forecast, 3 km
52
59
  - 3 km, CONUS
@@ -168,6 +175,51 @@ points took 28 s on a warm connection, so one cycle's leads 1 to 18 take about t
168
175
 
169
176
  df = fetch_points(["2026-10-14T08:00"], [1, 6, 18], {"buoy": (-73.58, 40.96)})
170
177
 
178
+ HRDPS forecasts at points
179
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
180
+
181
+ ``forcingkit.fetchers.hrdps.fetch_points(cycles, leads, points, fields)`` returns forecasts from
182
+ Environment and Climate Change Canada's `High Resolution Deterministic Prediction System
183
+ <https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__ (HRDPS) in the same
184
+ bitemporal rows as HRRR at points: 10 m wind rotated to east and north (``u10``, ``v10``) and the
185
+ instantaneous 10 m gust (``gust``), in m/s. HRDPS is a second, independent short-range model at a
186
+ similar resolution (2.5 km), run by a different centre with different physics and data
187
+ assimilation, so its errors at a station can be measured alongside HRRR's.
188
+
189
+ *Source.* The `MSC Datamart <https://eccc-msc.github.io/open-data/msc-datamart/readme_en/>`__
190
+ (``https://dd.weather.gc.ca``) publishes each 00, 06, 12 and 18 UTC run to 48 h as one GRIB2 file
191
+ per variable and lead (about 2 MB each, with no byte-range index), so a run's three fields at all
192
+ 49 leads are about 300 MB. The Datamart keeps 30 days; no longer public archive of the GRIB2 files
193
+ was found, so a record of HRDPS errors has to be accumulated from the day collection starts.
194
+ ``newest_cycle()`` returns the newest run whose 48 h files are all posted (about 3 h after the run).
195
+ ``base_url=hrdps.HPFX_URL`` reads the same files from MSC's HPFX server, which has more bandwidth
196
+ but no 24/7 support.
197
+
198
+ *Grid and winds.* A rotated latitude-longitude grid of 2540 x 1290 cells at 0.0225 degrees; Long
199
+ Island Sound is about 350 cells inside its southern edge. Winds are relative to the rotated grid and
200
+ are rotated to east and north by the angle between each cell's grid axis and east (about 32 degrees
201
+ near Long Island Sound). For the 2026-11-09 12 UTC run, lead 1, the rotated winds' direction
202
+ differed from MSC's own 10 m wind direction field by a median of 0.07 degrees (99th percentile 0.44
203
+ degrees) over every cell with wind above 2 m/s. Points take the same inverse-distance-squared mean of
204
+ up to four water cells within 6 km as HRRR, using HRDPS's land-sea mask.
205
+
206
+ A full run (49 leads, three fields) took about 50 s at 40 points, with a peak of 490 MB: each lead
207
+ keeps only the cells its points use, and at most two leads are read at once. Times that are not runs
208
+ and leads past 48 h are skipped, as is a lead with any field not yet posted.
209
+
210
+ .. code-block:: python
211
+
212
+ from forcingkit.fetchers import hrdps
213
+
214
+ run = hrdps.newest_cycle()
215
+ df = hrdps.fetch_points([run], range(49), {"buoy": (-73.73, 40.88)})
216
+
217
+ *License.* ECCC's `Data Server End-use Licence
218
+ <https://eccc-msc.github.io/open-data/licence/readme_en/>`__ allows use and redistribution,
219
+ including commercial use, with the attribution "Data Source: Environment and Climate Change
220
+ Canada" and, where possible, a link to the license. It does not allow any suggestion of official
221
+ status or endorsement.
222
+
171
223
  ERA5 (fallback)
172
224
  ~~~~~~~~~~~~~~~
173
225
 
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.17.1"
12
+ release = "0.18.1"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -23,6 +23,13 @@ Providers
23
23
  Dissemination <https://www.noaa.gov/information-technology/open-data-dissemination>`__
24
24
  - Dowell et al. (2022); James et al. (2022). NOAA asks for attribution and that modified
25
25
  products not be presented as NOAA's.
26
+ * - HRDPS
27
+ - `Environment and Climate Change Canada <https://www.canada.ca/en/environment-climate-change.html>`__,
28
+ Meteorological Service of Canada (`MSC open data
29
+ <https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__)
30
+ - "Data Source: Environment and Climate Change Canada", with a link to the `Data Server
31
+ End-use Licence <https://eccc-msc.github.io/open-data/licence/readme_en/>`__ where possible.
32
+ The license grants no right to suggest official status or endorsement.
26
33
  * - NECOFS
27
34
  - `UMass Dartmouth SMAST <https://www.umassd.edu/smast/>`__ and `Woods Hole Oceanographic
28
35
  Institution <https://www.whoi.edu/>`__; operation supported by `NERACOOS
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.17.1"
7
+ version = "0.18.1"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -0,0 +1,402 @@
1
+ """HRDPS forecasts at points, kept with both times: when each forecast was issued and the time it is for.
2
+
3
+ The High Resolution Deterministic Prediction System (HRDPS) is the 2.5 km limited-area model of Environment and
4
+ Climate Change Canada's Meteorological Service of Canada (MSC). Its continental domain reaches south past the
5
+ US Northeast coast: Long Island Sound lies about 350 cells inside the southern edge and the Gulf of Maine about 460
6
+ cells inside the eastern edge. It runs at 00, 06, 12 and 18 UTC to 48 h, with hourly output from lead 0.
7
+
8
+ Source. The MSC Datamart (``https://dd.weather.gc.ca``) publishes each run as one GRIB2 file per variable, level and
9
+ lead, with no byte-range index, under a directory for the run's date:
10
+
11
+ {base}/{YYYYMMDD}/WXO-DD/model_hrdps/continental/2.5km/{HH}/{hhh}/
12
+ {YYYYMMDD}T{HH}Z_MSC_HRDPS_{VAR}_{LEVEL}_RLatLon0.0225_PT{hhh}H.grib2
13
+
14
+ ``{base}/today/model_hrdps/...`` is an alias for the current date. The Datamart keeps 30 days of dated directories
15
+ (MSC's stated retention, which held on 2026-11-09: 2026-10-11 to 2026-11-09 were present). The HPFX server
16
+ (``https://hpfx.collab.science.gc.ca``) mirrors the same layout and retention with no 24/7 support; pass it as
17
+ ``base_url`` if the Datamart is slow or cannot be reached, with ``client=hpfx_client()``: on 2026-11-10 HPFX did not
18
+ answer a TLS 1.3 handshake that offers ALPN, which httpx always does (no reply within 30 s, every attempt), while it
19
+ answered TLS 1.3 without ALPN and TLS 1.2 with it at once, so ``hpfx_client`` caps TLS at 1.2. No longer public archive of the GRIB2 files was found, so a record of HRDPS
20
+ scores has to be accumulated from the day collection starts. A run's 48 h files were complete about 3 h after the
21
+ run time (the 2026-11-09 12 UTC run's PT048H files were posted at 15:08 UTC).
22
+
23
+ Fields. 10 m wind is ``UGRD`` and ``VGRD`` at ``AGL-10m``; ``GUST`` at ``AGL-10m`` is the instantaneous 10 m gust
24
+ (GRIB2 parameter 0/2/22, eccodes ``i10fg``). The land-sea mask is ``LAND`` at ``Sfc`` (a 0 to 1 fraction), published
25
+ at lead 0 only. Each file is about 2 MB, so the three wind fields of one 49-lead run are about 300 MB.
26
+
27
+ Grid. A rotated latitude-longitude grid of 2540 x 1290 cells at 0.0225 degrees, with the southern pole of rotation
28
+ at 36.08852 S, 245.305142 E and no rotation angle (from the GRIB2 headers). Winds are relative to the rotated grid
29
+ (``uvRelativeToGrid = 1``); here they are rotated to east and north by the angle between each cell's rotated
30
+ longitude axis and geographic east, which across the domain runs from -54 to +31 degrees. Checked against MSC's
31
+ own 10 m wind direction (``WDIR``, degrees true) for the 2026-11-09 12 UTC run, lead 1, at every cell with a wind
32
+ over 2 m/s: the median difference was 0.07 degrees and the 99th percentile 0.44 degrees (unrotated, 24 degrees).
33
+
34
+ The value at each point is an inverse-distance-squared mean of up to four HRDPS water cells (land fraction under
35
+ 0.5) within ``max_km``, as for HRRR in `hrrr_points`. Values stay in the units HRDPS publishes (m/s).
36
+
37
+ Licence. ECCC's Data Server End-use Licence (https://eccc-msc.github.io/open-data/licence/readme_en/) allows use,
38
+ including commercial use and redistribution, with attribution: "Data Source: Environment and Climate Change
39
+ Canada", with a link to the licence where possible. It grants no right to suggest official status or endorsement.
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import concurrent.futures
45
+ import contextlib
46
+ import logging
47
+ import os
48
+ import ssl
49
+ import tempfile
50
+ import time
51
+ from typing import Callable, Iterable
52
+
53
+ import httpx
54
+ import numpy as np
55
+ import pandas as pd
56
+
57
+ from .hrrr_atmosphere import _read_message
58
+ from .hrrr_points import PointWeights
59
+
60
+ logger = logging.getLogger(__name__)
61
+
62
+ BASE_URL = "https://dd.weather.gc.ca"
63
+ HPFX_URL = "https://hpfx.collab.science.gc.ca"
64
+ ATTRIBUTION = "Data Source: Environment and Climate Change Canada"
65
+ LICENCE_URL = "https://eccc-msc.github.io/open-data/licence/readme_en/"
66
+
67
+ CYCLE_HOURS = (0, 6, 12, 18)
68
+ MAX_LEAD = 48
69
+ RETENTION_DAYS = 30
70
+
71
+ # Datamart variable and level of each field.
72
+ FIELDS = {
73
+ "u10": ("UGRD", "AGL-10m"),
74
+ "v10": ("VGRD", "AGL-10m"),
75
+ "gust": ("GUST", "AGL-10m"),
76
+ }
77
+ DEFAULT_FIELDS = tuple(FIELDS)
78
+ UNITS = {"u10": "m s-1", "v10": "m s-1", "gust": "m s-1"}
79
+ LAND = ("LAND", "Sfc") # published at lead 0 only
80
+
81
+
82
+ def _utc_naive(t) -> pd.Timestamp:
83
+ t = pd.Timestamp(t)
84
+ return t.tz_convert("UTC").tz_localize(None) if t.tzinfo is not None else t
85
+
86
+
87
+ def file_url(cycle, lead: int, var: str, level: str, base_url: str = BASE_URL) -> str:
88
+ """The Datamart URL of one variable at one level and lead of a run."""
89
+ cycle = _utc_naive(cycle)
90
+ day, hh = cycle.strftime("%Y%m%d"), cycle.strftime("%H")
91
+ name = f"{day}T{hh}Z_MSC_HRDPS_{var}_{level}_RLatLon0.0225_PT{lead:03d}H.grib2"
92
+ return (
93
+ f"{base_url}/{day}/WXO-DD/model_hrdps/continental/2.5km/{hh}/{lead:03d}/{name}"
94
+ )
95
+
96
+
97
+ def field_url(cycle, lead: int, field: str, base_url: str = BASE_URL) -> str:
98
+ var, level = FIELDS[field]
99
+ return file_url(cycle, lead, var, level, base_url)
100
+
101
+
102
+ def is_cycle(cycle) -> bool:
103
+ cycle = _utc_naive(cycle)
104
+ return cycle == cycle.floor("h") and cycle.hour in CYCLE_HOURS
105
+
106
+
107
+ def hpfx_client(timeout: float = 120.0) -> httpx.Client:
108
+ """An httpx client for the HPFX server: TLS 1.2 at most (see the module notes). Pass it as ``client`` with
109
+ ``base_url=HPFX_URL``; the caller closes it."""
110
+ ctx = httpx.create_ssl_context()
111
+ ctx.maximum_version = ssl.TLSVersion.TLSv1_2
112
+ return httpx.Client(timeout=timeout, follow_redirects=True, verify=ctx)
113
+
114
+
115
+ def _exists(client: httpx.Client, url: str) -> bool:
116
+ r = client.head(url)
117
+ if r.status_code == 404:
118
+ return False
119
+ r.raise_for_status()
120
+ return True
121
+
122
+
123
+ def newest_cycle(
124
+ now=None,
125
+ fields: Iterable[str] = DEFAULT_FIELDS,
126
+ lead: int = MAX_LEAD,
127
+ lookback: int = 8,
128
+ base_url: str = BASE_URL,
129
+ exists: Callable[[str], bool] | None = None,
130
+ client: httpx.Client | None = None,
131
+ ) -> pd.Timestamp | None:
132
+ """The newest run at or before `now` whose `lead` file is published for every field, looking back `lookback`
133
+ runs (two days by default); None if none is. A run's last lead is posted last, so the default asks for a
134
+ complete run. `client`, if given, makes the requests and is left open (``hpfx_client()`` for HPFX)."""
135
+ now = _utc_naive(pd.Timestamp.now(tz="UTC") if now is None else now)
136
+ fields = tuple(fields)
137
+ cycle = now.floor("6h")
138
+ own = None
139
+ if exists is None:
140
+ if client is None:
141
+ client = own = httpx.Client(timeout=30.0, follow_redirects=True)
142
+ exists = lambda url: _exists(client, url) # noqa: E731
143
+ try:
144
+ for _ in range(lookback):
145
+ if all(exists(field_url(cycle, lead, f, base_url)) for f in fields):
146
+ return cycle
147
+ cycle -= pd.Timedelta(hours=6)
148
+ finally:
149
+ if own is not None:
150
+ own.close()
151
+ return None
152
+
153
+
154
+ # Rotated latitude-longitude grids (GRIB2 template 3.1), with the southern pole of rotation at (sp_lat, sp_lon) and
155
+ # no further rotation angle. The rotated system is the geographic one turned by sp_lon about the polar axis and
156
+ # then by 90 + sp_lat degrees about the new y axis.
157
+
158
+
159
+ def _cart(lon, lat):
160
+ lon, lat = np.deg2rad(lon), np.deg2rad(lat)
161
+ return np.cos(lat) * np.cos(lon), np.cos(lat) * np.sin(lon), np.sin(lat)
162
+
163
+
164
+ def geographic_to_rotated(lon, lat, sp_lat: float, sp_lon: float):
165
+ """Geographic (lon, lat) in degrees to rotated (rlon, rlat), rlon in -180..180."""
166
+ th = np.deg2rad(90.0 + sp_lat)
167
+ x, y, z = _cart(np.asarray(lon, dtype=float) - sp_lon, lat)
168
+ xr = np.cos(th) * x + np.sin(th) * z
169
+ zr = -np.sin(th) * x + np.cos(th) * z
170
+ return np.rad2deg(np.arctan2(y, xr)), np.rad2deg(np.arcsin(np.clip(zr, -1.0, 1.0)))
171
+
172
+
173
+ def rotated_to_geographic(rlon, rlat, sp_lat: float, sp_lon: float):
174
+ """Rotated (rlon, rlat) in degrees to geographic (lon, lat), lon in -180..180."""
175
+ th = np.deg2rad(90.0 + sp_lat)
176
+ xr, y, zr = _cart(rlon, rlat)
177
+ x = np.cos(th) * xr - np.sin(th) * zr
178
+ z = np.sin(th) * xr + np.cos(th) * zr
179
+ lon = np.rad2deg(np.arctan2(y, x)) + sp_lon
180
+ return ((lon + 180.0) % 360.0) - 180.0, np.rad2deg(np.arcsin(np.clip(z, -1.0, 1.0)))
181
+
182
+
183
+ def grid_east_angle(lon, lat, sp_lat: float, sp_lon: float, eps: float = 1e-3):
184
+ """Angle in radians, counterclockwise from geographic east, of the rotated grid's x axis at each (lon, lat)."""
185
+ rlon, rlat = geographic_to_rotated(lon, lat, sp_lat, sp_lon)
186
+ lon1, lat1 = rotated_to_geographic(rlon, rlat, sp_lat, sp_lon)
187
+ lon2, lat2 = rotated_to_geographic(rlon + eps, rlat, sp_lat, sp_lon)
188
+ dlon = ((lon2 - lon1 + 180.0) % 360.0) - 180.0
189
+ return np.arctan2(lat2 - lat1, dlon * np.cos(np.deg2rad(lat1)))
190
+
191
+
192
+ def earth_relative_winds(u, v, angle):
193
+ """Rotate grid-relative (u, v) to east and north, given the grid x axis `angle` from east (radians)."""
194
+ c, s = np.cos(angle), np.sin(angle)
195
+ return c * u - s * v, s * u + c * v
196
+
197
+
198
+ def _pole(attrs) -> tuple[float, float]:
199
+ if float(attrs.get("GRIB_angleOfRotationInDegrees", 0.0)) != 0.0:
200
+ raise NotImplementedError(
201
+ "HRDPS: rotated grid with a nonzero angle of rotation"
202
+ )
203
+ return (
204
+ float(attrs["GRIB_latitudeOfSouthernPoleInDegrees"]),
205
+ float(attrs["GRIB_longitudeOfSouthernPoleInDegrees"]),
206
+ )
207
+
208
+
209
+ RETRIES = 4
210
+
211
+
212
+ def _download(client: httpx.Client, url: str, path: str) -> bool:
213
+ """Write `url` to `path`; False if the Datamart has no such file. A dropped connection or a 5xx reply is
214
+ retried up to `RETRIES` times, 2, 4, 8 and 16 s apart (the Datamart closed connections during a full run)."""
215
+ for attempt in range(RETRIES + 1):
216
+ try:
217
+ with client.stream("GET", url) as r:
218
+ if r.status_code == 404:
219
+ return False
220
+ r.raise_for_status()
221
+ with open(path, "wb") as f:
222
+ for chunk in r.iter_bytes():
223
+ f.write(chunk)
224
+ return True
225
+ except (httpx.TransportError, httpx.HTTPStatusError) as e:
226
+ server_side = (
227
+ not isinstance(e, httpx.HTTPStatusError)
228
+ or e.response.status_code >= 500
229
+ )
230
+ if not server_side or attempt == RETRIES:
231
+ raise
232
+ logger.info("HRDPS %s: %s; retrying", url.rsplit("/", 1)[-1], e)
233
+ time.sleep(2.0 ** (attempt + 1))
234
+ return False # not reached
235
+
236
+
237
+ def _read_url(client, url: str, path: str):
238
+ if not _download(client, url, path):
239
+ return None
240
+ try:
241
+ return _read_message(path)
242
+ finally:
243
+ os.remove(path)
244
+
245
+
246
+ _land_cache: dict = {}
247
+
248
+
249
+ def _fetch_land(cycle, workdir, client, base_url):
250
+ """The land-sea mask from a run's lead 0, with the grid's coordinates; kept for the session (it does not change
251
+ between runs)."""
252
+ if "land" not in _land_cache:
253
+ path = os.path.join(workdir, f"land_{cycle:%Y%m%d%H}.grib2")
254
+ da = _read_url(client, file_url(cycle, 0, *LAND, base_url=base_url), path)
255
+ if da is None:
256
+ raise FileNotFoundError(
257
+ f"HRDPS land mask not published for the {cycle} run"
258
+ )
259
+ _land_cache["land"] = da
260
+ return _land_cache["land"]
261
+
262
+
263
+ class _Cells:
264
+ """The water cells the points draw on: their flat indices, each point's cells and weights as positions in that
265
+ list, and the angle of the grid's x axis at each cell (to rotate winds there only)."""
266
+
267
+ def __init__(self, land, points, k, max_km):
268
+ w = PointWeights(
269
+ land["longitude"].values,
270
+ land["latitude"].values,
271
+ land.values,
272
+ points,
273
+ k,
274
+ max_km,
275
+ label="HRDPS",
276
+ )
277
+ self.shape = land.shape
278
+ self.idx = np.unique(
279
+ np.concatenate([c for c in w.cells if len(c)] or [np.array([], dtype=int)])
280
+ )
281
+ pos = {int(c): n for n, c in enumerate(self.idx)}
282
+ self.points = {
283
+ name: (np.array([pos[int(c)] for c in cells], dtype=int), weights)
284
+ for name, cells, weights in zip(w.names, w.cells, w.weights)
285
+ }
286
+ lon = land["longitude"].values.ravel()[self.idx]
287
+ lat = land["latitude"].values.ravel()[self.idx]
288
+ self.angle = grid_east_angle(lon, lat, *_pole(land.attrs))
289
+
290
+ def at_points(self, values: np.ndarray) -> dict[str, float]:
291
+ """Each point's weighted mean of `values` (one per cell in `idx`)."""
292
+ return {
293
+ name: float(np.dot(values[c], w)) if len(c) else float("nan")
294
+ for name, (c, w) in self.points.items()
295
+ }
296
+
297
+
298
+ _cells_cache: dict = {}
299
+
300
+
301
+ def _cells(land, points, k, max_km) -> _Cells:
302
+ key = (land.shape, tuple(sorted(points.items())), k, max_km)
303
+ if key not in _cells_cache:
304
+ _cells_cache[key] = _Cells(land, points, k, max_km)
305
+ return _cells_cache[key]
306
+
307
+
308
+ def _fetch_lead(cycle, lead, fields, workdir, client, base_url, cells: _Cells):
309
+ """The requested fields of one run and lead at `cells` only (each decoded field, about 65 MB with its
310
+ coordinates, is dropped once read), with winds rotated to east and north there; None if any field is not
311
+ published."""
312
+ out = {}
313
+ grid_relative = False
314
+ for name in fields:
315
+ path = os.path.join(workdir, f"{name}_{cycle:%Y%m%d%H}_f{lead:03d}.grib2")
316
+ da = _read_url(client, field_url(cycle, lead, name, base_url), path)
317
+ if da is None:
318
+ logger.info("HRDPS %s f%03d: %s not published", cycle, lead, name)
319
+ return None
320
+ if da.shape != cells.shape:
321
+ raise ValueError(
322
+ f"HRDPS: field grid {da.shape} differs from land mask grid {cells.shape}"
323
+ )
324
+ if name == "u10":
325
+ grid_relative = int(da.attrs.get("GRIB_uvRelativeToGrid", 1)) == 1
326
+ out[name] = np.asarray(da.values, dtype=float).ravel()[cells.idx]
327
+ del da
328
+ if grid_relative and "u10" in out and "v10" in out:
329
+ out["u10"], out["v10"] = earth_relative_winds(
330
+ out["u10"], out["v10"], cells.angle
331
+ )
332
+ return out
333
+
334
+
335
+ def fetch_points(
336
+ cycles: Iterable,
337
+ leads: Iterable[int],
338
+ points: dict[str, tuple[float, float]],
339
+ fields: Iterable[str] = DEFAULT_FIELDS,
340
+ k: int = 4,
341
+ max_km: float = 6.0,
342
+ max_workers: int = 2,
343
+ base_url: str = BASE_URL,
344
+ client: httpx.Client | None = None,
345
+ ) -> pd.DataFrame:
346
+ """HRDPS forecasts at `points` ({name: (lon, lat)}) for every run and lead, as rows of
347
+ (issue_time, valid_time, lead_h, point, field, value). Times that are not HRDPS runs, and leads outside 0 to 48,
348
+ are skipped; so is a lead with any field not (yet) published. At most `max_workers` leads are read at once, and
349
+ each keeps only the cells its points use, so memory stays near one decoded field per worker. `client`, if given,
350
+ makes the requests and is left open (``hpfx_client()`` for HPFX)."""
351
+ fields = tuple(fields)
352
+ unknown = set(fields) - set(FIELDS)
353
+ if unknown:
354
+ raise ValueError(f"unknown HRDPS point fields: {sorted(unknown)}")
355
+ if ("u10" in fields) != ("v10" in fields):
356
+ raise ValueError(
357
+ "HRDPS u10 and v10 are grid-relative: request both to rotate them to east and north"
358
+ )
359
+ tasks = []
360
+ for c in cycles:
361
+ cycle = _utc_naive(c)
362
+ if not is_cycle(cycle):
363
+ logger.warning("HRDPS: %s is not a 00/06/12/18 UTC run; skipped", cycle)
364
+ continue
365
+ tasks += [(cycle, int(lead)) for lead in leads if 0 <= int(lead) <= MAX_LEAD]
366
+ rows: list[tuple] = []
367
+ if not tasks:
368
+ return pd.DataFrame(
369
+ rows,
370
+ columns=["issue_time", "valid_time", "lead_h", "point", "field", "value"],
371
+ )
372
+ with contextlib.ExitStack() as stack:
373
+ if client is None: # our own, closed on the way out; a caller's is left open
374
+ client = stack.enter_context(
375
+ httpx.Client(timeout=120.0, follow_redirects=True)
376
+ )
377
+ workdir = stack.enter_context(tempfile.TemporaryDirectory())
378
+ ex = stack.enter_context(
379
+ concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)
380
+ )
381
+ cells = _cells(
382
+ _fetch_land(tasks[0][0], workdir, client, base_url), points, k, max_km
383
+ )
384
+ # Submitted a batch at a time, so finished leads do not pile up ahead of the loop.
385
+ for b in range(0, len(tasks), max_workers):
386
+ batch = tasks[b : b + max_workers]
387
+ results = ex.map(
388
+ lambda t: _fetch_lead(
389
+ t[0], t[1], fields, workdir, client, base_url, cells
390
+ ),
391
+ batch,
392
+ )
393
+ for (cycle, lead), at_cells in zip(batch, results):
394
+ if at_cells is None:
395
+ continue
396
+ valid = cycle + pd.Timedelta(hours=lead)
397
+ for name in fields:
398
+ for p, v in cells.at_points(at_cells[name]).items():
399
+ rows.append((cycle, valid, lead, p, name, v))
400
+ return pd.DataFrame(
401
+ rows, columns=["issue_time", "valid_time", "lead_h", "point", "field", "value"]
402
+ )
@@ -77,6 +77,7 @@ class PointWeights:
77
77
  points: dict[str, tuple[float, float]],
78
78
  k: int = 4,
79
79
  max_km: float = 6.0,
80
+ label: str = "HRRR",
80
81
  ):
81
82
  lon = ((np.asarray(lon, dtype=float) + 180.0) % 360.0) - 180.0
82
83
  lat = np.asarray(lat, dtype=float)
@@ -95,7 +96,7 @@ class PointWeights:
95
96
  near = near[d[near] <= max_km]
96
97
  if not len(near):
97
98
  logger.warning(
98
- "HRRR points: no water cell within %.0f km of %s", max_km, name
99
+ "%s points: no water cell within %.0f km of %s", label, max_km, name
99
100
  )
100
101
  self.cells.append(np.array([], dtype=int))
101
102
  self.weights.append(np.array([]))