forcingkit 0.17.1__tar.gz → 0.18.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.17.1 → forcingkit-0.18.0}/PKG-INFO +2 -1
- {forcingkit-0.17.1 → forcingkit-0.18.0}/README.md +1 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/atmospheric_forcing.rst +52 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/conf.py +1 -1
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/sources.rst +7 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/pyproject.toml +1 -1
- forcingkit-0.18.0/src/forcingkit/fetchers/hrdps.py +382 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hrrr_points.py +2 -1
- forcingkit-0.18.0/tests/unit/test_hrdps.py +182 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/uv.lock +1 -1
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.agents/forcingkit.md +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.env.template +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.gitignore +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.markdownlint.yaml +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/.python-version +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/AGENTS.md +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/CONTRIBUTING.md +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/Dockerfile +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/LICENSE +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/LICENSE +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/README.md +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/pyproject.toml +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/tests/conftest.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/tests/test_client.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/tests/test_schema.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/client/uv.lock +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docker-compose.yml +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/Makefile +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/make.bat +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/_static/custom.css +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/client.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/fetchers.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/hycom.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/index.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/necofs.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/rivers.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/docs/source/waves.rst +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/main.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/__init__.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/routers/viewer.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/service/run_server.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/ndbc.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/render.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/static/app.js +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/static/favicon.ico +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/static/favicon.svg +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/static/index.html +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/static/logo.svg +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/static/preview3d.js +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/static/styles.css +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/__init__.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_main.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.17.1 → forcingkit-0.18.0}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.18.0
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -281,6 +281,7 @@ The primary goal of forcingkit is to provide clean, standardized ocean and atmos
|
|
|
281
281
|
|
|
282
282
|
- **Oceanic Forcing**: NYOFS, NECOFS, DBOFS, HYCOM (global, 1994 to present).
|
|
283
283
|
- **Atmospheric Forcing**: HRRR (3 km, hourly, 2014-07-30 to present), regridded to 0.03 degrees.
|
|
284
|
+
- **Forecasts at Points**: HRRR (to 48 h) and ECCC's HRDPS (2.5 km, 10 m wind and gust to 48 h, from the MSC Datamart's 30-day window). HRDPS data: Data Source: Environment and Climate Change Canada ([license](https://eccc-msc.github.io/open-data/licence/readme_en/)).
|
|
284
285
|
- **Surface Waves**: NWPS (one forecast office's grid per request, for example OKX CG1 at about 1.8 km, hourly, not regridded). NOMADS keeps about five days; older windows need an NWPS archive (`FORCINGKIT_NWPS_ARCHIVE`).
|
|
285
286
|
|
|
286
287
|
## Roadmap
|
|
@@ -34,6 +34,7 @@ The primary goal of forcingkit is to provide clean, standardized ocean and atmos
|
|
|
34
34
|
|
|
35
35
|
- **Oceanic Forcing**: NYOFS, NECOFS, DBOFS, HYCOM (global, 1994 to present).
|
|
36
36
|
- **Atmospheric Forcing**: HRRR (3 km, hourly, 2014-07-30 to present), regridded to 0.03 degrees.
|
|
37
|
+
- **Forecasts at Points**: HRRR (to 48 h) and ECCC's HRDPS (2.5 km, 10 m wind and gust to 48 h, from the MSC Datamart's 30-day window). HRDPS data: Data Source: Environment and Climate Change Canada ([license](https://eccc-msc.github.io/open-data/licence/readme_en/)).
|
|
37
38
|
- **Surface Waves**: NWPS (one forecast office's grid per request, for example OKX CG1 at about 1.8 km, hourly, not regridded). NOMADS keeps about five days; older windows need an NWPS archive (`FORCINGKIT_NWPS_ARCHIVE`).
|
|
38
39
|
|
|
39
40
|
## Roadmap
|
|
@@ -47,6 +47,13 @@ Summary
|
|
|
47
47
|
- Operational from 2026-10-06 (12 UTC cycle)
|
|
48
48
|
- Hourly cycles to 18 h; to 84 h every 6 h
|
|
49
49
|
- Not integrated; replaces NAM, SREF, HREF and HiresW, while HRRR stays in operations
|
|
50
|
+
* - **HRDPS** (`ECCC MSC <https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__)
|
|
51
|
+
- Forecast, GEM limited-area, 2.5 km
|
|
52
|
+
- 0.0225 degrees (about 2.5 km), rotated latitude-longitude over Canada and the northern US
|
|
53
|
+
- 1 h
|
|
54
|
+
- Runs on the MSC Datamart for the last 30 days; no longer public archive found
|
|
55
|
+
- 00, 06, 12 and 18 UTC runs to 48 h, complete about 3 h after the run
|
|
56
|
+
- **forcingkit** ``fetchers.hrdps.fetch_points`` (points only)
|
|
50
57
|
* - **NAM CONUS Nest** (NOAA NCEP)
|
|
51
58
|
- Forecast, 3 km
|
|
52
59
|
- 3 km, CONUS
|
|
@@ -168,6 +175,51 @@ points took 28 s on a warm connection, so one cycle's leads 1 to 18 take about t
|
|
|
168
175
|
|
|
169
176
|
df = fetch_points(["2026-10-14T08:00"], [1, 6, 18], {"buoy": (-73.58, 40.96)})
|
|
170
177
|
|
|
178
|
+
HRDPS forecasts at points
|
|
179
|
+
~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
180
|
+
|
|
181
|
+
``forcingkit.fetchers.hrdps.fetch_points(cycles, leads, points, fields)`` returns forecasts from
|
|
182
|
+
Environment and Climate Change Canada's `High Resolution Deterministic Prediction System
|
|
183
|
+
<https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__ (HRDPS) in the same
|
|
184
|
+
bitemporal rows as HRRR at points: 10 m wind rotated to east and north (``u10``, ``v10``) and the
|
|
185
|
+
instantaneous 10 m gust (``gust``), in m/s. HRDPS is a second, independent short-range model at a
|
|
186
|
+
similar resolution (2.5 km), run by a different centre with different physics and data
|
|
187
|
+
assimilation, so its errors at a station can be measured alongside HRRR's.
|
|
188
|
+
|
|
189
|
+
*Source.* The `MSC Datamart <https://eccc-msc.github.io/open-data/msc-datamart/readme_en/>`__
|
|
190
|
+
(``https://dd.weather.gc.ca``) publishes each 00, 06, 12 and 18 UTC run to 48 h as one GRIB2 file
|
|
191
|
+
per variable and lead (about 2 MB each, with no byte-range index), so a run's three fields at all
|
|
192
|
+
49 leads are about 300 MB. The Datamart keeps 30 days; no longer public archive of the GRIB2 files
|
|
193
|
+
was found, so a record of HRDPS errors has to be accumulated from the day collection starts.
|
|
194
|
+
``newest_cycle()`` returns the newest run whose 48 h files are all posted (about 3 h after the run).
|
|
195
|
+
``base_url=hrdps.HPFX_URL`` reads the same files from MSC's HPFX server, which has more bandwidth
|
|
196
|
+
but no 24/7 support.
|
|
197
|
+
|
|
198
|
+
*Grid and winds.* A rotated latitude-longitude grid of 2540 x 1290 cells at 0.0225 degrees; Long
|
|
199
|
+
Island Sound is about 350 cells inside its southern edge. Winds are relative to the rotated grid and
|
|
200
|
+
are rotated to east and north by the angle between each cell's grid axis and east (about 32 degrees
|
|
201
|
+
near Long Island Sound). For the 2026-11-09 12 UTC run, lead 1, the rotated winds' direction
|
|
202
|
+
differed from MSC's own 10 m wind direction field by a median of 0.07 degrees (99th percentile 0.44
|
|
203
|
+
degrees) over every cell with wind above 2 m/s. Points take the same inverse-distance-squared mean of
|
|
204
|
+
up to four water cells within 6 km as HRRR, using HRDPS's land-sea mask.
|
|
205
|
+
|
|
206
|
+
A full run (49 leads, three fields) took about 50 s at 40 points, with a peak of 490 MB: each lead
|
|
207
|
+
keeps only the cells its points use, and at most two leads are read at once. Times that are not runs
|
|
208
|
+
and leads past 48 h are skipped, as is a lead with any field not yet posted.
|
|
209
|
+
|
|
210
|
+
.. code-block:: python
|
|
211
|
+
|
|
212
|
+
from forcingkit.fetchers import hrdps
|
|
213
|
+
|
|
214
|
+
run = hrdps.newest_cycle()
|
|
215
|
+
df = hrdps.fetch_points([run], range(49), {"buoy": (-73.73, 40.88)})
|
|
216
|
+
|
|
217
|
+
*License.* ECCC's `Data Server End-use Licence
|
|
218
|
+
<https://eccc-msc.github.io/open-data/licence/readme_en/>`__ allows use and redistribution,
|
|
219
|
+
including commercial use, with the attribution "Data Source: Environment and Climate Change
|
|
220
|
+
Canada" and, where possible, a link to the license. It does not allow any suggestion of official
|
|
221
|
+
status or endorsement.
|
|
222
|
+
|
|
171
223
|
ERA5 (fallback)
|
|
172
224
|
~~~~~~~~~~~~~~~
|
|
173
225
|
|
|
@@ -23,6 +23,13 @@ Providers
|
|
|
23
23
|
Dissemination <https://www.noaa.gov/information-technology/open-data-dissemination>`__
|
|
24
24
|
- Dowell et al. (2022); James et al. (2022). NOAA asks for attribution and that modified
|
|
25
25
|
products not be presented as NOAA's.
|
|
26
|
+
* - HRDPS
|
|
27
|
+
- `Environment and Climate Change Canada <https://www.canada.ca/en/environment-climate-change.html>`__,
|
|
28
|
+
Meteorological Service of Canada (`MSC open data
|
|
29
|
+
<https://eccc-msc.github.io/open-data/msc-data/nwp_hrdps/readme_hrdps_en/>`__)
|
|
30
|
+
- "Data Source: Environment and Climate Change Canada", with a link to the `Data Server
|
|
31
|
+
End-use Licence <https://eccc-msc.github.io/open-data/licence/readme_en/>`__ where possible.
|
|
32
|
+
The license grants no right to suggest official status or endorsement.
|
|
26
33
|
* - NECOFS
|
|
27
34
|
- `UMass Dartmouth SMAST <https://www.umassd.edu/smast/>`__ and `Woods Hole Oceanographic
|
|
28
35
|
Institution <https://www.whoi.edu/>`__; operation supported by `NERACOOS
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.18.0"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""HRDPS forecasts at points, kept with both times: when each forecast was issued and the time it is for.
|
|
2
|
+
|
|
3
|
+
The High Resolution Deterministic Prediction System (HRDPS) is the 2.5 km limited-area model of Environment and
|
|
4
|
+
Climate Change Canada's Meteorological Service of Canada (MSC). Its continental domain reaches south past the
|
|
5
|
+
US Northeast coast: Long Island Sound lies about 350 cells inside the southern edge and the Gulf of Maine about 460
|
|
6
|
+
cells inside the eastern edge. It runs at 00, 06, 12 and 18 UTC to 48 h, with hourly output from lead 0.
|
|
7
|
+
|
|
8
|
+
Source. The MSC Datamart (``https://dd.weather.gc.ca``) publishes each run as one GRIB2 file per variable, level and
|
|
9
|
+
lead, with no byte-range index, under a directory for the run's date:
|
|
10
|
+
|
|
11
|
+
{base}/{YYYYMMDD}/WXO-DD/model_hrdps/continental/2.5km/{HH}/{hhh}/
|
|
12
|
+
{YYYYMMDD}T{HH}Z_MSC_HRDPS_{VAR}_{LEVEL}_RLatLon0.0225_PT{hhh}H.grib2
|
|
13
|
+
|
|
14
|
+
``{base}/today/model_hrdps/...`` is an alias for the current date. The Datamart keeps 30 days of dated directories
|
|
15
|
+
(MSC's stated retention, which held on 2026-11-09: 2026-10-11 to 2026-11-09 were present). The HPFX server
|
|
16
|
+
(``https://hpfx.collab.science.gc.ca``) mirrors the same layout and retention with no 24/7 support; pass it as
|
|
17
|
+
``base_url`` if the Datamart is slow. No longer public archive of the GRIB2 files was found, so a record of HRDPS
|
|
18
|
+
scores has to be accumulated from the day collection starts. A run's 48 h files were complete about 3 h after the
|
|
19
|
+
run time (the 2026-11-09 12 UTC run's PT048H files were posted at 15:08 UTC).
|
|
20
|
+
|
|
21
|
+
Fields. 10 m wind is ``UGRD`` and ``VGRD`` at ``AGL-10m``; ``GUST`` at ``AGL-10m`` is the instantaneous 10 m gust
|
|
22
|
+
(GRIB2 parameter 0/2/22, eccodes ``i10fg``). The land-sea mask is ``LAND`` at ``Sfc`` (a 0 to 1 fraction), published
|
|
23
|
+
at lead 0 only. Each file is about 2 MB, so the three wind fields of one 49-lead run are about 300 MB.
|
|
24
|
+
|
|
25
|
+
Grid. A rotated latitude-longitude grid of 2540 x 1290 cells at 0.0225 degrees, with the southern pole of rotation
|
|
26
|
+
at 36.08852 S, 245.305142 E and no rotation angle (from the GRIB2 headers). Winds are relative to the rotated grid
|
|
27
|
+
(``uvRelativeToGrid = 1``); here they are rotated to east and north by the angle between each cell's rotated
|
|
28
|
+
longitude axis and geographic east, which across the domain runs from -54 to +31 degrees. Checked against MSC's
|
|
29
|
+
own 10 m wind direction (``WDIR``, degrees true) for the 2026-11-09 12 UTC run, lead 1, at every cell with a wind
|
|
30
|
+
over 2 m/s: the median difference was 0.07 degrees and the 99th percentile 0.44 degrees (unrotated, 24 degrees).
|
|
31
|
+
|
|
32
|
+
The value at each point is an inverse-distance-squared mean of up to four HRDPS water cells (land fraction under
|
|
33
|
+
0.5) within ``max_km``, as for HRRR in `hrrr_points`. Values stay in the units HRDPS publishes (m/s).
|
|
34
|
+
|
|
35
|
+
Licence. ECCC's Data Server End-use Licence (https://eccc-msc.github.io/open-data/licence/readme_en/) allows use,
|
|
36
|
+
including commercial use and redistribution, with attribution: "Data Source: Environment and Climate Change
|
|
37
|
+
Canada", with a link to the licence where possible. It grants no right to suggest official status or endorsement.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import concurrent.futures
|
|
43
|
+
import logging
|
|
44
|
+
import os
|
|
45
|
+
import tempfile
|
|
46
|
+
import time
|
|
47
|
+
from typing import Callable, Iterable
|
|
48
|
+
|
|
49
|
+
import httpx
|
|
50
|
+
import numpy as np
|
|
51
|
+
import pandas as pd
|
|
52
|
+
|
|
53
|
+
from .hrrr_atmosphere import _read_message
|
|
54
|
+
from .hrrr_points import PointWeights
|
|
55
|
+
|
|
56
|
+
logger = logging.getLogger(__name__)
|
|
57
|
+
|
|
58
|
+
BASE_URL = "https://dd.weather.gc.ca"
|
|
59
|
+
HPFX_URL = "https://hpfx.collab.science.gc.ca"
|
|
60
|
+
ATTRIBUTION = "Data Source: Environment and Climate Change Canada"
|
|
61
|
+
LICENCE_URL = "https://eccc-msc.github.io/open-data/licence/readme_en/"
|
|
62
|
+
|
|
63
|
+
CYCLE_HOURS = (0, 6, 12, 18)
|
|
64
|
+
MAX_LEAD = 48
|
|
65
|
+
RETENTION_DAYS = 30
|
|
66
|
+
|
|
67
|
+
# Datamart variable and level of each field.
|
|
68
|
+
FIELDS = {
|
|
69
|
+
"u10": ("UGRD", "AGL-10m"),
|
|
70
|
+
"v10": ("VGRD", "AGL-10m"),
|
|
71
|
+
"gust": ("GUST", "AGL-10m"),
|
|
72
|
+
}
|
|
73
|
+
DEFAULT_FIELDS = tuple(FIELDS)
|
|
74
|
+
UNITS = {"u10": "m s-1", "v10": "m s-1", "gust": "m s-1"}
|
|
75
|
+
LAND = ("LAND", "Sfc") # published at lead 0 only
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _utc_naive(t) -> pd.Timestamp:
|
|
79
|
+
t = pd.Timestamp(t)
|
|
80
|
+
return t.tz_convert("UTC").tz_localize(None) if t.tzinfo is not None else t
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def file_url(cycle, lead: int, var: str, level: str, base_url: str = BASE_URL) -> str:
|
|
84
|
+
"""The Datamart URL of one variable at one level and lead of a run."""
|
|
85
|
+
cycle = _utc_naive(cycle)
|
|
86
|
+
day, hh = cycle.strftime("%Y%m%d"), cycle.strftime("%H")
|
|
87
|
+
name = f"{day}T{hh}Z_MSC_HRDPS_{var}_{level}_RLatLon0.0225_PT{lead:03d}H.grib2"
|
|
88
|
+
return (
|
|
89
|
+
f"{base_url}/{day}/WXO-DD/model_hrdps/continental/2.5km/{hh}/{lead:03d}/{name}"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def field_url(cycle, lead: int, field: str, base_url: str = BASE_URL) -> str:
|
|
94
|
+
var, level = FIELDS[field]
|
|
95
|
+
return file_url(cycle, lead, var, level, base_url)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def is_cycle(cycle) -> bool:
|
|
99
|
+
cycle = _utc_naive(cycle)
|
|
100
|
+
return cycle == cycle.floor("h") and cycle.hour in CYCLE_HOURS
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _exists(client: httpx.Client, url: str) -> bool:
|
|
104
|
+
r = client.head(url)
|
|
105
|
+
if r.status_code == 404:
|
|
106
|
+
return False
|
|
107
|
+
r.raise_for_status()
|
|
108
|
+
return True
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def newest_cycle(
|
|
112
|
+
now=None,
|
|
113
|
+
fields: Iterable[str] = DEFAULT_FIELDS,
|
|
114
|
+
lead: int = MAX_LEAD,
|
|
115
|
+
lookback: int = 8,
|
|
116
|
+
base_url: str = BASE_URL,
|
|
117
|
+
exists: Callable[[str], bool] | None = None,
|
|
118
|
+
) -> pd.Timestamp | None:
|
|
119
|
+
"""The newest run at or before `now` whose `lead` file is published for every field, looking back `lookback`
|
|
120
|
+
runs (two days by default); None if none is. A run's last lead is posted last, so the default asks for a
|
|
121
|
+
complete run."""
|
|
122
|
+
now = _utc_naive(pd.Timestamp.now(tz="UTC") if now is None else now)
|
|
123
|
+
fields = tuple(fields)
|
|
124
|
+
cycle = now.floor("6h")
|
|
125
|
+
client = None
|
|
126
|
+
if exists is None:
|
|
127
|
+
client = httpx.Client(timeout=30.0, follow_redirects=True)
|
|
128
|
+
exists = lambda url: _exists(client, url) # noqa: E731
|
|
129
|
+
try:
|
|
130
|
+
for _ in range(lookback):
|
|
131
|
+
if all(exists(field_url(cycle, lead, f, base_url)) for f in fields):
|
|
132
|
+
return cycle
|
|
133
|
+
cycle -= pd.Timedelta(hours=6)
|
|
134
|
+
finally:
|
|
135
|
+
if client is not None:
|
|
136
|
+
client.close()
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# Rotated latitude-longitude grids (GRIB2 template 3.1), with the southern pole of rotation at (sp_lat, sp_lon) and
|
|
141
|
+
# no further rotation angle. The rotated system is the geographic one turned by sp_lon about the polar axis and
|
|
142
|
+
# then by 90 + sp_lat degrees about the new y axis.
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _cart(lon, lat):
|
|
146
|
+
lon, lat = np.deg2rad(lon), np.deg2rad(lat)
|
|
147
|
+
return np.cos(lat) * np.cos(lon), np.cos(lat) * np.sin(lon), np.sin(lat)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def geographic_to_rotated(lon, lat, sp_lat: float, sp_lon: float):
|
|
151
|
+
"""Geographic (lon, lat) in degrees to rotated (rlon, rlat), rlon in -180..180."""
|
|
152
|
+
th = np.deg2rad(90.0 + sp_lat)
|
|
153
|
+
x, y, z = _cart(np.asarray(lon, dtype=float) - sp_lon, lat)
|
|
154
|
+
xr = np.cos(th) * x + np.sin(th) * z
|
|
155
|
+
zr = -np.sin(th) * x + np.cos(th) * z
|
|
156
|
+
return np.rad2deg(np.arctan2(y, xr)), np.rad2deg(np.arcsin(np.clip(zr, -1.0, 1.0)))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def rotated_to_geographic(rlon, rlat, sp_lat: float, sp_lon: float):
|
|
160
|
+
"""Rotated (rlon, rlat) in degrees to geographic (lon, lat), lon in -180..180."""
|
|
161
|
+
th = np.deg2rad(90.0 + sp_lat)
|
|
162
|
+
xr, y, zr = _cart(rlon, rlat)
|
|
163
|
+
x = np.cos(th) * xr - np.sin(th) * zr
|
|
164
|
+
z = np.sin(th) * xr + np.cos(th) * zr
|
|
165
|
+
lon = np.rad2deg(np.arctan2(y, x)) + sp_lon
|
|
166
|
+
return ((lon + 180.0) % 360.0) - 180.0, np.rad2deg(np.arcsin(np.clip(z, -1.0, 1.0)))
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def grid_east_angle(lon, lat, sp_lat: float, sp_lon: float, eps: float = 1e-3):
|
|
170
|
+
"""Angle in radians, counterclockwise from geographic east, of the rotated grid's x axis at each (lon, lat)."""
|
|
171
|
+
rlon, rlat = geographic_to_rotated(lon, lat, sp_lat, sp_lon)
|
|
172
|
+
lon1, lat1 = rotated_to_geographic(rlon, rlat, sp_lat, sp_lon)
|
|
173
|
+
lon2, lat2 = rotated_to_geographic(rlon + eps, rlat, sp_lat, sp_lon)
|
|
174
|
+
dlon = ((lon2 - lon1 + 180.0) % 360.0) - 180.0
|
|
175
|
+
return np.arctan2(lat2 - lat1, dlon * np.cos(np.deg2rad(lat1)))
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def earth_relative_winds(u, v, angle):
|
|
179
|
+
"""Rotate grid-relative (u, v) to east and north, given the grid x axis `angle` from east (radians)."""
|
|
180
|
+
c, s = np.cos(angle), np.sin(angle)
|
|
181
|
+
return c * u - s * v, s * u + c * v
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _pole(attrs) -> tuple[float, float]:
|
|
185
|
+
if float(attrs.get("GRIB_angleOfRotationInDegrees", 0.0)) != 0.0:
|
|
186
|
+
raise NotImplementedError(
|
|
187
|
+
"HRDPS: rotated grid with a nonzero angle of rotation"
|
|
188
|
+
)
|
|
189
|
+
return (
|
|
190
|
+
float(attrs["GRIB_latitudeOfSouthernPoleInDegrees"]),
|
|
191
|
+
float(attrs["GRIB_longitudeOfSouthernPoleInDegrees"]),
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
RETRIES = 4
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _download(client: httpx.Client, url: str, path: str) -> bool:
|
|
199
|
+
"""Write `url` to `path`; False if the Datamart has no such file. A dropped connection or a 5xx reply is
|
|
200
|
+
retried up to `RETRIES` times, 2, 4, 8 and 16 s apart (the Datamart closed connections during a full run)."""
|
|
201
|
+
for attempt in range(RETRIES + 1):
|
|
202
|
+
try:
|
|
203
|
+
with client.stream("GET", url) as r:
|
|
204
|
+
if r.status_code == 404:
|
|
205
|
+
return False
|
|
206
|
+
r.raise_for_status()
|
|
207
|
+
with open(path, "wb") as f:
|
|
208
|
+
for chunk in r.iter_bytes():
|
|
209
|
+
f.write(chunk)
|
|
210
|
+
return True
|
|
211
|
+
except (httpx.TransportError, httpx.HTTPStatusError) as e:
|
|
212
|
+
server_side = (
|
|
213
|
+
not isinstance(e, httpx.HTTPStatusError)
|
|
214
|
+
or e.response.status_code >= 500
|
|
215
|
+
)
|
|
216
|
+
if not server_side or attempt == RETRIES:
|
|
217
|
+
raise
|
|
218
|
+
logger.info("HRDPS %s: %s; retrying", url.rsplit("/", 1)[-1], e)
|
|
219
|
+
time.sleep(2.0 ** (attempt + 1))
|
|
220
|
+
return False # not reached
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _read_url(client, url: str, path: str):
|
|
224
|
+
if not _download(client, url, path):
|
|
225
|
+
return None
|
|
226
|
+
try:
|
|
227
|
+
return _read_message(path)
|
|
228
|
+
finally:
|
|
229
|
+
os.remove(path)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
_land_cache: dict = {}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _fetch_land(cycle, workdir, client, base_url):
|
|
236
|
+
"""The land-sea mask from a run's lead 0, with the grid's coordinates; kept for the session (it does not change
|
|
237
|
+
between runs)."""
|
|
238
|
+
if "land" not in _land_cache:
|
|
239
|
+
path = os.path.join(workdir, f"land_{cycle:%Y%m%d%H}.grib2")
|
|
240
|
+
da = _read_url(client, file_url(cycle, 0, *LAND, base_url=base_url), path)
|
|
241
|
+
if da is None:
|
|
242
|
+
raise FileNotFoundError(
|
|
243
|
+
f"HRDPS land mask not published for the {cycle} run"
|
|
244
|
+
)
|
|
245
|
+
_land_cache["land"] = da
|
|
246
|
+
return _land_cache["land"]
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
class _Cells:
|
|
250
|
+
"""The water cells the points draw on: their flat indices, each point's cells and weights as positions in that
|
|
251
|
+
list, and the angle of the grid's x axis at each cell (to rotate winds there only)."""
|
|
252
|
+
|
|
253
|
+
def __init__(self, land, points, k, max_km):
|
|
254
|
+
w = PointWeights(
|
|
255
|
+
land["longitude"].values,
|
|
256
|
+
land["latitude"].values,
|
|
257
|
+
land.values,
|
|
258
|
+
points,
|
|
259
|
+
k,
|
|
260
|
+
max_km,
|
|
261
|
+
label="HRDPS",
|
|
262
|
+
)
|
|
263
|
+
self.shape = land.shape
|
|
264
|
+
self.idx = np.unique(
|
|
265
|
+
np.concatenate([c for c in w.cells if len(c)] or [np.array([], dtype=int)])
|
|
266
|
+
)
|
|
267
|
+
pos = {int(c): n for n, c in enumerate(self.idx)}
|
|
268
|
+
self.points = {
|
|
269
|
+
name: (np.array([pos[int(c)] for c in cells], dtype=int), weights)
|
|
270
|
+
for name, cells, weights in zip(w.names, w.cells, w.weights)
|
|
271
|
+
}
|
|
272
|
+
lon = land["longitude"].values.ravel()[self.idx]
|
|
273
|
+
lat = land["latitude"].values.ravel()[self.idx]
|
|
274
|
+
self.angle = grid_east_angle(lon, lat, *_pole(land.attrs))
|
|
275
|
+
|
|
276
|
+
def at_points(self, values: np.ndarray) -> dict[str, float]:
|
|
277
|
+
"""Each point's weighted mean of `values` (one per cell in `idx`)."""
|
|
278
|
+
return {
|
|
279
|
+
name: float(np.dot(values[c], w)) if len(c) else float("nan")
|
|
280
|
+
for name, (c, w) in self.points.items()
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
_cells_cache: dict = {}
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _cells(land, points, k, max_km) -> _Cells:
|
|
288
|
+
key = (land.shape, tuple(sorted(points.items())), k, max_km)
|
|
289
|
+
if key not in _cells_cache:
|
|
290
|
+
_cells_cache[key] = _Cells(land, points, k, max_km)
|
|
291
|
+
return _cells_cache[key]
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _fetch_lead(cycle, lead, fields, workdir, client, base_url, cells: _Cells):
|
|
295
|
+
"""The requested fields of one run and lead at `cells` only (each decoded field, about 65 MB with its
|
|
296
|
+
coordinates, is dropped once read), with winds rotated to east and north there; None if any field is not
|
|
297
|
+
published."""
|
|
298
|
+
out = {}
|
|
299
|
+
grid_relative = False
|
|
300
|
+
for name in fields:
|
|
301
|
+
path = os.path.join(workdir, f"{name}_{cycle:%Y%m%d%H}_f{lead:03d}.grib2")
|
|
302
|
+
da = _read_url(client, field_url(cycle, lead, name, base_url), path)
|
|
303
|
+
if da is None:
|
|
304
|
+
logger.info("HRDPS %s f%03d: %s not published", cycle, lead, name)
|
|
305
|
+
return None
|
|
306
|
+
if da.shape != cells.shape:
|
|
307
|
+
raise ValueError(
|
|
308
|
+
f"HRDPS: field grid {da.shape} differs from land mask grid {cells.shape}"
|
|
309
|
+
)
|
|
310
|
+
if name == "u10":
|
|
311
|
+
grid_relative = int(da.attrs.get("GRIB_uvRelativeToGrid", 1)) == 1
|
|
312
|
+
out[name] = np.asarray(da.values, dtype=float).ravel()[cells.idx]
|
|
313
|
+
del da
|
|
314
|
+
if grid_relative and "u10" in out and "v10" in out:
|
|
315
|
+
out["u10"], out["v10"] = earth_relative_winds(
|
|
316
|
+
out["u10"], out["v10"], cells.angle
|
|
317
|
+
)
|
|
318
|
+
return out
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def fetch_points(
|
|
322
|
+
cycles: Iterable,
|
|
323
|
+
leads: Iterable[int],
|
|
324
|
+
points: dict[str, tuple[float, float]],
|
|
325
|
+
fields: Iterable[str] = DEFAULT_FIELDS,
|
|
326
|
+
k: int = 4,
|
|
327
|
+
max_km: float = 6.0,
|
|
328
|
+
max_workers: int = 2,
|
|
329
|
+
base_url: str = BASE_URL,
|
|
330
|
+
) -> pd.DataFrame:
|
|
331
|
+
"""HRDPS forecasts at `points` ({name: (lon, lat)}) for every run and lead, as rows of
|
|
332
|
+
(issue_time, valid_time, lead_h, point, field, value). Times that are not HRDPS runs, and leads outside 0 to 48,
|
|
333
|
+
are skipped; so is a lead with any field not (yet) published. At most `max_workers` leads are read at once, and
|
|
334
|
+
each keeps only the cells its points use, so memory stays near one decoded field per worker."""
|
|
335
|
+
fields = tuple(fields)
|
|
336
|
+
unknown = set(fields) - set(FIELDS)
|
|
337
|
+
if unknown:
|
|
338
|
+
raise ValueError(f"unknown HRDPS point fields: {sorted(unknown)}")
|
|
339
|
+
if ("u10" in fields) != ("v10" in fields):
|
|
340
|
+
raise ValueError(
|
|
341
|
+
"HRDPS u10 and v10 are grid-relative: request both to rotate them to east and north"
|
|
342
|
+
)
|
|
343
|
+
tasks = []
|
|
344
|
+
for c in cycles:
|
|
345
|
+
cycle = _utc_naive(c)
|
|
346
|
+
if not is_cycle(cycle):
|
|
347
|
+
logger.warning("HRDPS: %s is not a 00/06/12/18 UTC run; skipped", cycle)
|
|
348
|
+
continue
|
|
349
|
+
tasks += [(cycle, int(lead)) for lead in leads if 0 <= int(lead) <= MAX_LEAD]
|
|
350
|
+
rows: list[tuple] = []
|
|
351
|
+
if not tasks:
|
|
352
|
+
return pd.DataFrame(
|
|
353
|
+
rows,
|
|
354
|
+
columns=["issue_time", "valid_time", "lead_h", "point", "field", "value"],
|
|
355
|
+
)
|
|
356
|
+
with (
|
|
357
|
+
tempfile.TemporaryDirectory() as workdir,
|
|
358
|
+
httpx.Client(timeout=120.0, follow_redirects=True) as client,
|
|
359
|
+
concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex,
|
|
360
|
+
):
|
|
361
|
+
cells = _cells(
|
|
362
|
+
_fetch_land(tasks[0][0], workdir, client, base_url), points, k, max_km
|
|
363
|
+
)
|
|
364
|
+
# Submitted a batch at a time, so finished leads do not pile up ahead of the loop.
|
|
365
|
+
for b in range(0, len(tasks), max_workers):
|
|
366
|
+
batch = tasks[b : b + max_workers]
|
|
367
|
+
results = ex.map(
|
|
368
|
+
lambda t: _fetch_lead(
|
|
369
|
+
t[0], t[1], fields, workdir, client, base_url, cells
|
|
370
|
+
),
|
|
371
|
+
batch,
|
|
372
|
+
)
|
|
373
|
+
for (cycle, lead), at_cells in zip(batch, results):
|
|
374
|
+
if at_cells is None:
|
|
375
|
+
continue
|
|
376
|
+
valid = cycle + pd.Timedelta(hours=lead)
|
|
377
|
+
for name in fields:
|
|
378
|
+
for p, v in cells.at_points(at_cells[name]).items():
|
|
379
|
+
rows.append((cycle, valid, lead, p, name, v))
|
|
380
|
+
return pd.DataFrame(
|
|
381
|
+
rows, columns=["issue_time", "valid_time", "lead_h", "point", "field", "value"]
|
|
382
|
+
)
|
|
@@ -77,6 +77,7 @@ class PointWeights:
|
|
|
77
77
|
points: dict[str, tuple[float, float]],
|
|
78
78
|
k: int = 4,
|
|
79
79
|
max_km: float = 6.0,
|
|
80
|
+
label: str = "HRRR",
|
|
80
81
|
):
|
|
81
82
|
lon = ((np.asarray(lon, dtype=float) + 180.0) % 360.0) - 180.0
|
|
82
83
|
lat = np.asarray(lat, dtype=float)
|
|
@@ -95,7 +96,7 @@ class PointWeights:
|
|
|
95
96
|
near = near[d[near] <= max_km]
|
|
96
97
|
if not len(near):
|
|
97
98
|
logger.warning(
|
|
98
|
-
"
|
|
99
|
+
"%s points: no water cell within %.0f km of %s", label, max_km, name
|
|
99
100
|
)
|
|
100
101
|
self.cells.append(np.array([], dtype=int))
|
|
101
102
|
self.weights.append(np.array([]))
|