forcingkit 0.17.1__tar.gz → 0.18.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. {forcingkit-0.17.1 → forcingkit-0.18.0}/PKG-INFO +2 -1
  2. {forcingkit-0.17.1 → forcingkit-0.18.0}/README.md +1 -0
  3. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/atmospheric_forcing.rst +52 -0
  4. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/conf.py +1 -1
  5. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/sources.rst +7 -0
  6. {forcingkit-0.17.1 → forcingkit-0.18.0}/pyproject.toml +1 -1
  7. forcingkit-0.18.0/src/forcingkit/fetchers/hrdps.py +382 -0
  8. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hrrr_points.py +2 -1
  9. forcingkit-0.18.0/tests/unit/test_hrdps.py +182 -0
  10. {forcingkit-0.17.1 → forcingkit-0.18.0}/uv.lock +1 -1
  11. {forcingkit-0.17.1 → forcingkit-0.18.0}/.agents/forcingkit.md +0 -0
  12. {forcingkit-0.17.1 → forcingkit-0.18.0}/.claude/CLAUDE.md +0 -0
  13. {forcingkit-0.17.1 → forcingkit-0.18.0}/.env.template +0 -0
  14. {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/copilot-instructions.md +0 -0
  15. {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/workflows/docs.yml +0 -0
  16. {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/workflows/publish-client.yml +0 -0
  17. {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/workflows/publish.yml +0 -0
  18. {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/workflows/tests.yml +0 -0
  19. {forcingkit-0.17.1 → forcingkit-0.18.0}/.gitignore +0 -0
  20. {forcingkit-0.17.1 → forcingkit-0.18.0}/.markdownlint.yaml +0 -0
  21. {forcingkit-0.17.1 → forcingkit-0.18.0}/.pre-commit-config.yaml +0 -0
  22. {forcingkit-0.17.1 → forcingkit-0.18.0}/.python-version +0 -0
  23. {forcingkit-0.17.1 → forcingkit-0.18.0}/AGENTS.md +0 -0
  24. {forcingkit-0.17.1 → forcingkit-0.18.0}/CONTRIBUTING.md +0 -0
  25. {forcingkit-0.17.1 → forcingkit-0.18.0}/Dockerfile +0 -0
  26. {forcingkit-0.17.1 → forcingkit-0.18.0}/LICENSE +0 -0
  27. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/LICENSE +0 -0
  28. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/README.md +0 -0
  29. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/pyproject.toml +0 -0
  30. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/__init__.py +0 -0
  31. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/client.py +0 -0
  32. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/schema.py +0 -0
  33. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/store.py +0 -0
  34. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/trajectory.py +0 -0
  35. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/tests/conftest.py +0 -0
  36. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/tests/test_client.py +0 -0
  37. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/tests/test_schema.py +0 -0
  38. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/tests/test_trajectory.py +0 -0
  39. {forcingkit-0.17.1 → forcingkit-0.18.0}/client/uv.lock +0 -0
  40. {forcingkit-0.17.1 → forcingkit-0.18.0}/docker-compose.yml +0 -0
  41. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/Makefile +0 -0
  42. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/make.bat +0 -0
  43. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/requirements-docs.txt +0 -0
  44. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/_extra/CNAME +0 -0
  45. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/_static/custom.css +0 -0
  46. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  47. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/client.rst +0 -0
  48. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/fetchers.rst +0 -0
  49. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/hycom.rst +0 -0
  50. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/index.rst +0 -0
  51. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/layered_mesh.rst +0 -0
  52. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/mesh_frames.rst +0 -0
  53. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/necofs.rst +0 -0
  54. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/nyofs.rst +0 -0
  55. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/rivers.rst +0 -0
  56. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/roadmap.rst +0 -0
  57. {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/waves.rst +0 -0
  58. {forcingkit-0.17.1 → forcingkit-0.18.0}/main.py +0 -0
  59. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/__init__.py +0 -0
  60. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/__init__.py +0 -0
  61. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/jobs.py +0 -0
  62. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/main.py +0 -0
  63. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  64. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  65. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/removed.py +0 -0
  66. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/stores.py +0 -0
  67. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/viewer.py +0 -0
  68. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/stac.py +0 -0
  69. {forcingkit-0.17.1 → forcingkit-0.18.0}/service/run_server.py +0 -0
  70. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/__init__.py +0 -0
  71. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  72. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/dispatcher.py +0 -0
  73. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
  74. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/dbofs.py +0 -0
  75. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/erddap.py +0 -0
  76. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hrrr.py +0 -0
  77. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  78. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  79. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hycom.py +0 -0
  80. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hydrography.py +0 -0
  81. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/ndbc.py +0 -0
  82. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/necofs.py +0 -0
  83. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/noaa.py +0 -0
  84. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/nwps.py +0 -0
  85. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/nyofs.py +0 -0
  86. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/usgs.py +0 -0
  87. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  88. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/layered_mesh.py +0 -0
  89. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/mesh_frames.py +0 -0
  90. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
  91. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  92. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  93. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  94. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/render.py +0 -0
  95. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/settings.py +0 -0
  96. {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/zarr_stream.py +0 -0
  97. {forcingkit-0.17.1 → forcingkit-0.18.0}/static/app.js +0 -0
  98. {forcingkit-0.17.1 → forcingkit-0.18.0}/static/favicon.ico +0 -0
  99. {forcingkit-0.17.1 → forcingkit-0.18.0}/static/favicon.svg +0 -0
  100. {forcingkit-0.17.1 → forcingkit-0.18.0}/static/index.html +0 -0
  101. {forcingkit-0.17.1 → forcingkit-0.18.0}/static/logo.svg +0 -0
  102. {forcingkit-0.17.1 → forcingkit-0.18.0}/static/preview3d.js +0 -0
  103. {forcingkit-0.17.1 → forcingkit-0.18.0}/static/styles.css +0 -0
  104. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/__init__.py +0 -0
  105. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_auth.py +0 -0
  106. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_erddap_fetch.py +0 -0
  107. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
  108. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_necofs_volume_live.py +0 -0
  109. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_nwps_nomads.py +0 -0
  110. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  111. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_obc_offshore_nj.py +0 -0
  112. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_ofs_archive_paths.py +0 -0
  113. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
  114. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/scripts/inspect_dem.py +0 -0
  115. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/scripts/inspect_grib.py +0 -0
  116. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/scripts/inspect_zarr.py +0 -0
  117. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_coastwatch.py +0 -0
  118. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_dbofs.py +0 -0
  119. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_dispatcher.py +0 -0
  120. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_erddap.py +0 -0
  121. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hrrr_atm.py +0 -0
  122. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hrrr_idx.py +0 -0
  123. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hrrr_levels.py +0 -0
  124. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hrrr_points.py +0 -0
  125. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hycom.py +0 -0
  126. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_jobs.py +0 -0
  127. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_layered_mesh.py +0 -0
  128. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_main.py +0 -0
  129. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_mesh_frames.py +0 -0
  130. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_ndbc.py +0 -0
  131. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_necofs_hindcast.py +0 -0
  132. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
  133. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_necofs_parent.py +0 -0
  134. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_necofs_volume.py +0 -0
  135. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_noaa_currents.py +0 -0
  136. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_noaa_datum.py +0 -0
  137. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_nwps.py +0 -0
  138. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_nyofs.py +0 -0
  139. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_obc_donor_report.py +0 -0
  140. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_obc_pipeline.py +0 -0
  141. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_removed_routes.py +0 -0
  142. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_settings.py +0 -0
  143. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_usgs.py +0 -0
  144. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_usgs_rivers.py +0 -0
  145. {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.17.1
3
+ Version: 0.18.0
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -281,6 +281,7 @@ The primary goal of forcingkit is to provide clean, standardized ocean and atmos
281
281
 
282
282
  - **Oceanic Forcing**: NYOFS, NECOFS, DBOFS, HYCOM (global, 1994 to present).
283
283
  - **Atmospheric Forcing**: HRRR (3 km, hourly, 2014-07-30 to present), regridded to 0.03 degrees.
284
+ - **Forecasts at Points**: HRRR (to 48 h) and ECCC's HRDPS (2.5 km, 10 m wind and gust to 48 h, from the MSC Datamart's 30-day window). HRDPS data: Data Source: Environment and Climate Change Canada ([license](https://eccc-msc.github.io/open-data/licence/readme_en/)).
284
285
  - **Surface Waves**: NWPS (one forecast office's grid per request, for example OKX CG1 at about 1.8 km, hourly, not regridded). NOMADS keeps about five days; older windows need an NWPS archive (`FORCINGKIT_NWPS_ARCHIVE`).
285
286
 
286
287
  ## Roadmap
@@ -34,6 +34,7 @@ The primary goal of forcingkit is to provide clean, standardized ocean and atmos
34
34
 
35
35
  - **Oceanic Forcing**: NYOFS, NECOFS, DBOFS, HYCOM (global, 1994 to present).
36
36
  - **Atmospheric Forcing**: HRRR (3 km, hourly, 2014-07-30 to present), regridded to 0.03 degrees.
37
+ - **Forecasts at Points**: HRRR (to 48 h) and ECCC's HRDPS (2.5 km, 10 m wind and gust to 48 h, from the MSC Datamart's 30-day window). HRDPS data: Data Source: Environment and Climate Change Canada ([license](https://eccc-msc.github.io/open-data/licence/readme_en/)).
37
38
  - **Surface Waves**: NWPS (one forecast office's grid per request, for example OKX CG1 at about 1.8 km, hourly, not regridded). NOMADS keeps about five days; older windows need an NWPS archive (`FORCINGKIT_NWPS_ARCHIVE`).
38
39
 
39
40
  ## Roadmap
@@ -47,6 +47,13 @@ Summary
47
47
  - Operational from 2026-10-06 (12 UTC cycle)
48
48
  - Hourly cycles to 18 h; to 84 h every 6 h
49
49
  - Not integrated; replaces NAM, SREF, HREF and HiresW, while HRRR stays in operations
50
+ * - **HRDPS** (`ECCC MSC <https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__)
51
+ - Forecast, GEM limited-area, 2.5 km
52
+ - 0.0225 degrees (about 2.5 km), rotated latitude-longitude over Canada and the northern US
53
+ - 1 h
54
+ - Runs on the MSC Datamart for the last 30 days; no longer public archive found
55
+ - 00, 06, 12 and 18 UTC runs to 48 h, complete about 3 h after the run
56
+ - **forcingkit** ``fetchers.hrdps.fetch_points`` (points only)
50
57
  * - **NAM CONUS Nest** (NOAA NCEP)
51
58
  - Forecast, 3 km
52
59
  - 3 km, CONUS
@@ -168,6 +175,51 @@ points took 28 s on a warm connection, so one cycle's leads 1 to 18 take about t
168
175
 
169
176
  df = fetch_points(["2026-10-14T08:00"], [1, 6, 18], {"buoy": (-73.58, 40.96)})
170
177
 
178
+ HRDPS forecasts at points
179
+ ~~~~~~~~~~~~~~~~~~~~~~~~~
180
+
181
+ ``forcingkit.fetchers.hrdps.fetch_points(cycles, leads, points, fields)`` returns forecasts from
182
+ Environment and Climate Change Canada's `High Resolution Deterministic Prediction System
183
+ <https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__ (HRDPS) in the same
184
+ bitemporal rows as HRRR at points: 10 m wind rotated to east and north (``u10``, ``v10``) and the
185
+ instantaneous 10 m gust (``gust``), in m/s. HRDPS is a second, independent short-range model at a
186
+ similar resolution (2.5 km), run by a different centre with different physics and data
187
+ assimilation, so its errors at a station can be measured alongside HRRR's.
188
+
189
+ *Source.* The `MSC Datamart <https://eccc-msc.github.io/open-data/msc-datamart/readme_en/>`__
190
+ (``https://dd.weather.gc.ca``) publishes each 00, 06, 12 and 18 UTC run to 48 h as one GRIB2 file
191
+ per variable and lead (about 2 MB each, with no byte-range index), so a run's three fields at all
192
+ 49 leads are about 300 MB. The Datamart keeps 30 days; no longer public archive of the GRIB2 files
193
+ was found, so a record of HRDPS errors has to be accumulated from the day collection starts.
194
+ ``newest_cycle()`` returns the newest run whose 48 h files are all posted (about 3 h after the run).
195
+ ``base_url=hrdps.HPFX_URL`` reads the same files from MSC's HPFX server, which has more bandwidth
196
+ but no 24/7 support.
197
+
198
+ *Grid and winds.* A rotated latitude-longitude grid of 2540 x 1290 cells at 0.0225 degrees; Long
199
+ Island Sound is about 350 cells inside its southern edge. Winds are relative to the rotated grid and
200
+ are rotated to east and north by the angle between each cell's grid axis and east (about 32 degrees
201
+ near Long Island Sound). For the 2026-11-09 12 UTC run, lead 1, the rotated winds' direction
202
+ differed from MSC's own 10 m wind direction field by a median of 0.07 degrees (99th percentile 0.44
203
+ degrees) over every cell with wind above 2 m/s. Points take the same inverse-distance-squared mean of
204
+ up to four water cells within 6 km as HRRR, using HRDPS's land-sea mask.
205
+
206
+ A full run (49 leads, three fields) took about 50 s at 40 points, with a peak of 490 MB: each lead
207
+ keeps only the cells its points use, and at most two leads are read at once. Times that are not runs
208
+ and leads past 48 h are skipped, as is a lead with any field not yet posted.
209
+
210
+ .. code-block:: python
211
+
212
+ from forcingkit.fetchers import hrdps
213
+
214
+ run = hrdps.newest_cycle()
215
+ df = hrdps.fetch_points([run], range(49), {"buoy": (-73.73, 40.88)})
216
+
217
+ *License.* ECCC's `Data Server End-use Licence
218
+ <https://eccc-msc.github.io/open-data/licence/readme_en/>`__ allows use and redistribution,
219
+ including commercial use, with the attribution "Data Source: Environment and Climate Change
220
+ Canada" and, where possible, a link to the license. It does not allow any suggestion of official
221
+ status or endorsement.
222
+
171
223
  ERA5 (fallback)
172
224
  ~~~~~~~~~~~~~~~
173
225
 
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.17.1"
12
+ release = "0.18.0"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -23,6 +23,13 @@ Providers
23
23
  Dissemination <https://www.noaa.gov/information-technology/open-data-dissemination>`__
24
24
  - Dowell et al. (2022); James et al. (2022). NOAA asks for attribution and that modified
25
25
  products not be presented as NOAA's.
26
+ * - HRDPS
27
+ - `Environment and Climate Change Canada <https://www.canada.ca/en/environment-climate-change.html>`__,
28
+ Meteorological Service of Canada (`MSC open data
29
+ <https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__)
30
+ - "Data Source: Environment and Climate Change Canada", with a link to the `Data Server
31
+ End-use Licence <https://eccc-msc.github.io/open-data/licence/readme_en/>`__ where possible.
32
+ The license grants no right to suggest official status or endorsement.
26
33
  * - NECOFS
27
34
  - `UMass Dartmouth SMAST <https://www.umassd.edu/smast/>`__ and `Woods Hole Oceanographic
28
35
  Institution <https://www.whoi.edu/>`__; operation supported by `NERACOOS
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.17.1"
7
+ version = "0.18.0"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -0,0 +1,382 @@
1
+ """HRDPS forecasts at points, kept with both times: when each forecast was issued and the time it is for.
2
+
3
+ The High Resolution Deterministic Prediction System (HRDPS) is the 2.5 km limited-area model of Environment and
4
+ Climate Change Canada's Meteorological Service of Canada (MSC). Its continental domain reaches south past the
5
+ US Northeast coast: Long Island Sound lies about 350 cells inside the southern edge and the Gulf of Maine about 460
6
+ cells inside the eastern edge. It runs at 00, 06, 12 and 18 UTC to 48 h, with hourly output from lead 0.
7
+
8
+ Source. The MSC Datamart (``https://dd.weather.gc.ca``) publishes each run as one GRIB2 file per variable, level and
9
+ lead, with no byte-range index, under a directory for the run's date:
10
+
11
+ {base}/{YYYYMMDD}/WXO-DD/model_hrdps/continental/2.5km/{HH}/{hhh}/
12
+ {YYYYMMDD}T{HH}Z_MSC_HRDPS_{VAR}_{LEVEL}_RLatLon0.0225_PT{hhh}H.grib2
13
+
14
+ ``{base}/today/model_hrdps/...`` is an alias for the current date. The Datamart keeps 30 days of dated directories
15
+ (MSC's stated retention, which held on 2026-11-09: 2026-10-11 to 2026-11-09 were present). The HPFX server
16
+ (``https://hpfx.collab.science.gc.ca``) mirrors the same layout and retention with no 24/7 support; pass it as
17
+ ``base_url`` if the Datamart is slow. No longer public archive of the GRIB2 files was found, so a record of HRDPS
18
+ scores has to be accumulated from the day collection starts. A run's 48 h files were complete about 3 h after the
19
+ run time (the 2026-11-09 12 UTC run's PT048H files were posted at 15:08 UTC).
20
+
21
+ Fields. 10 m wind is ``UGRD`` and ``VGRD`` at ``AGL-10m``; ``GUST`` at ``AGL-10m`` is the instantaneous 10 m gust
22
+ (GRIB2 parameter 0/2/22, eccodes ``i10fg``). The land-sea mask is ``LAND`` at ``Sfc`` (a 0 to 1 fraction), published
23
+ at lead 0 only. Each file is about 2 MB, so the three wind fields of one 49-lead run are about 300 MB.
24
+
25
+ Grid. A rotated latitude-longitude grid of 2540 x 1290 cells at 0.0225 degrees, with the southern pole of rotation
26
+ at 36.08852 S, 245.305142 E and no rotation angle (from the GRIB2 headers). Winds are relative to the rotated grid
27
+ (``uvRelativeToGrid = 1``); here they are rotated to east and north by the angle between each cell's rotated
28
+ longitude axis and geographic east, which across the domain runs from -54 to +31 degrees. Checked against MSC's
29
+ own 10 m wind direction (``WDIR``, degrees true) for the 2026-11-09 12 UTC run, lead 1, at every cell with a wind
30
+ over 2 m/s: the median difference was 0.07 degrees and the 99th percentile 0.44 degrees (unrotated, 24 degrees).
31
+
32
+ The value at each point is an inverse-distance-squared mean of up to four HRDPS water cells (land fraction under
33
+ 0.5) within ``max_km``, as for HRRR in `hrrr_points`. Values stay in the units HRDPS publishes (m/s).
34
+
35
+ Licence. ECCC's Data Server End-use Licence (https://eccc-msc.github.io/open-data/licence/readme_en/) allows use,
36
+ including commercial use and redistribution, with attribution: "Data Source: Environment and Climate Change
37
+ Canada", with a link to the licence where possible. It grants no right to suggest official status or endorsement.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import concurrent.futures
43
+ import logging
44
+ import os
45
+ import tempfile
46
+ import time
47
+ from typing import Callable, Iterable
48
+
49
+ import httpx
50
+ import numpy as np
51
+ import pandas as pd
52
+
53
+ from .hrrr_atmosphere import _read_message
54
+ from .hrrr_points import PointWeights
55
+
56
+ logger = logging.getLogger(__name__)
57
+
58
+ BASE_URL = "https://dd.weather.gc.ca"
59
+ HPFX_URL = "https://hpfx.collab.science.gc.ca"
60
+ ATTRIBUTION = "Data Source: Environment and Climate Change Canada"
61
+ LICENCE_URL = "https://eccc-msc.github.io/open-data/licence/readme_en/"
62
+
63
+ CYCLE_HOURS = (0, 6, 12, 18)
64
+ MAX_LEAD = 48
65
+ RETENTION_DAYS = 30
66
+
67
+ # Datamart variable and level of each field.
68
+ FIELDS = {
69
+ "u10": ("UGRD", "AGL-10m"),
70
+ "v10": ("VGRD", "AGL-10m"),
71
+ "gust": ("GUST", "AGL-10m"),
72
+ }
73
+ DEFAULT_FIELDS = tuple(FIELDS)
74
+ UNITS = {"u10": "m s-1", "v10": "m s-1", "gust": "m s-1"}
75
+ LAND = ("LAND", "Sfc") # published at lead 0 only
76
+
77
+
78
+ def _utc_naive(t) -> pd.Timestamp:
79
+ t = pd.Timestamp(t)
80
+ return t.tz_convert("UTC").tz_localize(None) if t.tzinfo is not None else t
81
+
82
+
83
+ def file_url(cycle, lead: int, var: str, level: str, base_url: str = BASE_URL) -> str:
84
+ """The Datamart URL of one variable at one level and lead of a run."""
85
+ cycle = _utc_naive(cycle)
86
+ day, hh = cycle.strftime("%Y%m%d"), cycle.strftime("%H")
87
+ name = f"{day}T{hh}Z_MSC_HRDPS_{var}_{level}_RLatLon0.0225_PT{lead:03d}H.grib2"
88
+ return (
89
+ f"{base_url}/{day}/WXO-DD/model_hrdps/continental/2.5km/{hh}/{lead:03d}/{name}"
90
+ )
91
+
92
+
93
+ def field_url(cycle, lead: int, field: str, base_url: str = BASE_URL) -> str:
94
+ var, level = FIELDS[field]
95
+ return file_url(cycle, lead, var, level, base_url)
96
+
97
+
98
+ def is_cycle(cycle) -> bool:
99
+ cycle = _utc_naive(cycle)
100
+ return cycle == cycle.floor("h") and cycle.hour in CYCLE_HOURS
101
+
102
+
103
+ def _exists(client: httpx.Client, url: str) -> bool:
104
+ r = client.head(url)
105
+ if r.status_code == 404:
106
+ return False
107
+ r.raise_for_status()
108
+ return True
109
+
110
+
111
+ def newest_cycle(
112
+ now=None,
113
+ fields: Iterable[str] = DEFAULT_FIELDS,
114
+ lead: int = MAX_LEAD,
115
+ lookback: int = 8,
116
+ base_url: str = BASE_URL,
117
+ exists: Callable[[str], bool] | None = None,
118
+ ) -> pd.Timestamp | None:
119
+ """The newest run at or before `now` whose `lead` file is published for every field, looking back `lookback`
120
+ runs (two days by default); None if none is. A run's last lead is posted last, so the default asks for a
121
+ complete run."""
122
+ now = _utc_naive(pd.Timestamp.now(tz="UTC") if now is None else now)
123
+ fields = tuple(fields)
124
+ cycle = now.floor("6h")
125
+ client = None
126
+ if exists is None:
127
+ client = httpx.Client(timeout=30.0, follow_redirects=True)
128
+ exists = lambda url: _exists(client, url) # noqa: E731
129
+ try:
130
+ for _ in range(lookback):
131
+ if all(exists(field_url(cycle, lead, f, base_url)) for f in fields):
132
+ return cycle
133
+ cycle -= pd.Timedelta(hours=6)
134
+ finally:
135
+ if client is not None:
136
+ client.close()
137
+ return None
138
+
139
+
140
+ # Rotated latitude-longitude grids (GRIB2 template 3.1), with the southern pole of rotation at (sp_lat, sp_lon) and
141
+ # no further rotation angle. The rotated system is the geographic one turned by sp_lon about the polar axis and
142
+ # then by 90 + sp_lat degrees about the new y axis.
143
+
144
+
145
+ def _cart(lon, lat):
146
+ lon, lat = np.deg2rad(lon), np.deg2rad(lat)
147
+ return np.cos(lat) * np.cos(lon), np.cos(lat) * np.sin(lon), np.sin(lat)
148
+
149
+
150
+ def geographic_to_rotated(lon, lat, sp_lat: float, sp_lon: float):
151
+ """Geographic (lon, lat) in degrees to rotated (rlon, rlat), rlon in -180..180."""
152
+ th = np.deg2rad(90.0 + sp_lat)
153
+ x, y, z = _cart(np.asarray(lon, dtype=float) - sp_lon, lat)
154
+ xr = np.cos(th) * x + np.sin(th) * z
155
+ zr = -np.sin(th) * x + np.cos(th) * z
156
+ return np.rad2deg(np.arctan2(y, xr)), np.rad2deg(np.arcsin(np.clip(zr, -1.0, 1.0)))
157
+
158
+
159
+ def rotated_to_geographic(rlon, rlat, sp_lat: float, sp_lon: float):
160
+ """Rotated (rlon, rlat) in degrees to geographic (lon, lat), lon in -180..180."""
161
+ th = np.deg2rad(90.0 + sp_lat)
162
+ xr, y, zr = _cart(rlon, rlat)
163
+ x = np.cos(th) * xr - np.sin(th) * zr
164
+ z = np.sin(th) * xr + np.cos(th) * zr
165
+ lon = np.rad2deg(np.arctan2(y, x)) + sp_lon
166
+ return ((lon + 180.0) % 360.0) - 180.0, np.rad2deg(np.arcsin(np.clip(z, -1.0, 1.0)))
167
+
168
+
169
+ def grid_east_angle(lon, lat, sp_lat: float, sp_lon: float, eps: float = 1e-3):
170
+ """Angle in radians, counterclockwise from geographic east, of the rotated grid's x axis at each (lon, lat)."""
171
+ rlon, rlat = geographic_to_rotated(lon, lat, sp_lat, sp_lon)
172
+ lon1, lat1 = rotated_to_geographic(rlon, rlat, sp_lat, sp_lon)
173
+ lon2, lat2 = rotated_to_geographic(rlon + eps, rlat, sp_lat, sp_lon)
174
+ dlon = ((lon2 - lon1 + 180.0) % 360.0) - 180.0
175
+ return np.arctan2(lat2 - lat1, dlon * np.cos(np.deg2rad(lat1)))
176
+
177
+
178
+ def earth_relative_winds(u, v, angle):
179
+ """Rotate grid-relative (u, v) to east and north, given the grid x axis `angle` from east (radians)."""
180
+ c, s = np.cos(angle), np.sin(angle)
181
+ return c * u - s * v, s * u + c * v
182
+
183
+
184
+ def _pole(attrs) -> tuple[float, float]:
185
+ if float(attrs.get("GRIB_angleOfRotationInDegrees", 0.0)) != 0.0:
186
+ raise NotImplementedError(
187
+ "HRDPS: rotated grid with a nonzero angle of rotation"
188
+ )
189
+ return (
190
+ float(attrs["GRIB_latitudeOfSouthernPoleInDegrees"]),
191
+ float(attrs["GRIB_longitudeOfSouthernPoleInDegrees"]),
192
+ )
193
+
194
+
195
+ RETRIES = 4
196
+
197
+
198
+ def _download(client: httpx.Client, url: str, path: str) -> bool:
199
+ """Write `url` to `path`; False if the Datamart has no such file. A dropped connection or a 5xx reply is
200
+ retried up to `RETRIES` times, 2, 4, 8 and 16 s apart (the Datamart closed connections during a full run)."""
201
+ for attempt in range(RETRIES + 1):
202
+ try:
203
+ with client.stream("GET", url) as r:
204
+ if r.status_code == 404:
205
+ return False
206
+ r.raise_for_status()
207
+ with open(path, "wb") as f:
208
+ for chunk in r.iter_bytes():
209
+ f.write(chunk)
210
+ return True
211
+ except (httpx.TransportError, httpx.HTTPStatusError) as e:
212
+ server_side = (
213
+ not isinstance(e, httpx.HTTPStatusError)
214
+ or e.response.status_code >= 500
215
+ )
216
+ if not server_side or attempt == RETRIES:
217
+ raise
218
+ logger.info("HRDPS %s: %s; retrying", url.rsplit("/", 1)[-1], e)
219
+ time.sleep(2.0 ** (attempt + 1))
220
+ return False # not reached
221
+
222
+
223
+ def _read_url(client, url: str, path: str):
224
+ if not _download(client, url, path):
225
+ return None
226
+ try:
227
+ return _read_message(path)
228
+ finally:
229
+ os.remove(path)
230
+
231
+
232
+ _land_cache: dict = {}
233
+
234
+
235
+ def _fetch_land(cycle, workdir, client, base_url):
236
+ """The land-sea mask from a run's lead 0, with the grid's coordinates; kept for the session (it does not change
237
+ between runs)."""
238
+ if "land" not in _land_cache:
239
+ path = os.path.join(workdir, f"land_{cycle:%Y%m%d%H}.grib2")
240
+ da = _read_url(client, file_url(cycle, 0, *LAND, base_url=base_url), path)
241
+ if da is None:
242
+ raise FileNotFoundError(
243
+ f"HRDPS land mask not published for the {cycle} run"
244
+ )
245
+ _land_cache["land"] = da
246
+ return _land_cache["land"]
247
+
248
+
249
+ class _Cells:
250
+ """The water cells the points draw on: their flat indices, each point's cells and weights as positions in that
251
+ list, and the angle of the grid's x axis at each cell (to rotate winds there only)."""
252
+
253
+ def __init__(self, land, points, k, max_km):
254
+ w = PointWeights(
255
+ land["longitude"].values,
256
+ land["latitude"].values,
257
+ land.values,
258
+ points,
259
+ k,
260
+ max_km,
261
+ label="HRDPS",
262
+ )
263
+ self.shape = land.shape
264
+ self.idx = np.unique(
265
+ np.concatenate([c for c in w.cells if len(c)] or [np.array([], dtype=int)])
266
+ )
267
+ pos = {int(c): n for n, c in enumerate(self.idx)}
268
+ self.points = {
269
+ name: (np.array([pos[int(c)] for c in cells], dtype=int), weights)
270
+ for name, cells, weights in zip(w.names, w.cells, w.weights)
271
+ }
272
+ lon = land["longitude"].values.ravel()[self.idx]
273
+ lat = land["latitude"].values.ravel()[self.idx]
274
+ self.angle = grid_east_angle(lon, lat, *_pole(land.attrs))
275
+
276
+ def at_points(self, values: np.ndarray) -> dict[str, float]:
277
+ """Each point's weighted mean of `values` (one per cell in `idx`)."""
278
+ return {
279
+ name: float(np.dot(values[c], w)) if len(c) else float("nan")
280
+ for name, (c, w) in self.points.items()
281
+ }
282
+
283
+
284
+ _cells_cache: dict = {}
285
+
286
+
287
+ def _cells(land, points, k, max_km) -> _Cells:
288
+ key = (land.shape, tuple(sorted(points.items())), k, max_km)
289
+ if key not in _cells_cache:
290
+ _cells_cache[key] = _Cells(land, points, k, max_km)
291
+ return _cells_cache[key]
292
+
293
+
294
+ def _fetch_lead(cycle, lead, fields, workdir, client, base_url, cells: _Cells):
295
+ """The requested fields of one run and lead at `cells` only (each decoded field, about 65 MB with its
296
+ coordinates, is dropped once read), with winds rotated to east and north there; None if any field is not
297
+ published."""
298
+ out = {}
299
+ grid_relative = False
300
+ for name in fields:
301
+ path = os.path.join(workdir, f"{name}_{cycle:%Y%m%d%H}_f{lead:03d}.grib2")
302
+ da = _read_url(client, field_url(cycle, lead, name, base_url), path)
303
+ if da is None:
304
+ logger.info("HRDPS %s f%03d: %s not published", cycle, lead, name)
305
+ return None
306
+ if da.shape != cells.shape:
307
+ raise ValueError(
308
+ f"HRDPS: field grid {da.shape} differs from land mask grid {cells.shape}"
309
+ )
310
+ if name == "u10":
311
+ grid_relative = int(da.attrs.get("GRIB_uvRelativeToGrid", 1)) == 1
312
+ out[name] = np.asarray(da.values, dtype=float).ravel()[cells.idx]
313
+ del da
314
+ if grid_relative and "u10" in out and "v10" in out:
315
+ out["u10"], out["v10"] = earth_relative_winds(
316
+ out["u10"], out["v10"], cells.angle
317
+ )
318
+ return out
319
+
320
+
321
+ def fetch_points(
322
+ cycles: Iterable,
323
+ leads: Iterable[int],
324
+ points: dict[str, tuple[float, float]],
325
+ fields: Iterable[str] = DEFAULT_FIELDS,
326
+ k: int = 4,
327
+ max_km: float = 6.0,
328
+ max_workers: int = 2,
329
+ base_url: str = BASE_URL,
330
+ ) -> pd.DataFrame:
331
+ """HRDPS forecasts at `points` ({name: (lon, lat)}) for every run and lead, as rows of
332
+ (issue_time, valid_time, lead_h, point, field, value). Times that are not HRDPS runs, and leads outside 0 to 48,
333
+ are skipped; so is a lead with any field not (yet) published. At most `max_workers` leads are read at once, and
334
+ each keeps only the cells its points use, so memory stays near one decoded field per worker."""
335
+ fields = tuple(fields)
336
+ unknown = set(fields) - set(FIELDS)
337
+ if unknown:
338
+ raise ValueError(f"unknown HRDPS point fields: {sorted(unknown)}")
339
+ if ("u10" in fields) != ("v10" in fields):
340
+ raise ValueError(
341
+ "HRDPS u10 and v10 are grid-relative: request both to rotate them to east and north"
342
+ )
343
+ tasks = []
344
+ for c in cycles:
345
+ cycle = _utc_naive(c)
346
+ if not is_cycle(cycle):
347
+ logger.warning("HRDPS: %s is not a 00/06/12/18 UTC run; skipped", cycle)
348
+ continue
349
+ tasks += [(cycle, int(lead)) for lead in leads if 0 <= int(lead) <= MAX_LEAD]
350
+ rows: list[tuple] = []
351
+ if not tasks:
352
+ return pd.DataFrame(
353
+ rows,
354
+ columns=["issue_time", "valid_time", "lead_h", "point", "field", "value"],
355
+ )
356
+ with (
357
+ tempfile.TemporaryDirectory() as workdir,
358
+ httpx.Client(timeout=120.0, follow_redirects=True) as client,
359
+ concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex,
360
+ ):
361
+ cells = _cells(
362
+ _fetch_land(tasks[0][0], workdir, client, base_url), points, k, max_km
363
+ )
364
+ # Submitted a batch at a time, so finished leads do not pile up ahead of the loop.
365
+ for b in range(0, len(tasks), max_workers):
366
+ batch = tasks[b : b + max_workers]
367
+ results = ex.map(
368
+ lambda t: _fetch_lead(
369
+ t[0], t[1], fields, workdir, client, base_url, cells
370
+ ),
371
+ batch,
372
+ )
373
+ for (cycle, lead), at_cells in zip(batch, results):
374
+ if at_cells is None:
375
+ continue
376
+ valid = cycle + pd.Timedelta(hours=lead)
377
+ for name in fields:
378
+ for p, v in cells.at_points(at_cells[name]).items():
379
+ rows.append((cycle, valid, lead, p, name, v))
380
+ return pd.DataFrame(
381
+ rows, columns=["issue_time", "valid_time", "lead_h", "point", "field", "value"]
382
+ )
@@ -77,6 +77,7 @@ class PointWeights:
77
77
  points: dict[str, tuple[float, float]],
78
78
  k: int = 4,
79
79
  max_km: float = 6.0,
80
+ label: str = "HRRR",
80
81
  ):
81
82
  lon = ((np.asarray(lon, dtype=float) + 180.0) % 360.0) - 180.0
82
83
  lat = np.asarray(lat, dtype=float)
@@ -95,7 +96,7 @@ class PointWeights:
95
96
  near = near[d[near] <= max_km]
96
97
  if not len(near):
97
98
  logger.warning(
98
- "HRRR points: no water cell within %.0f km of %s", max_km, name
99
+ "%s points: no water cell within %.0f km of %s", label, max_km, name
99
100
  )
100
101
  self.cells.append(np.array([], dtype=int))
101
102
  self.weights.append(np.array([]))