forcingkit 0.18.1__tar.gz → 0.19.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.agents/forcingkit.md +3 -1
- {forcingkit-0.18.1 → forcingkit-0.19.0}/PKG-INFO +1 -1
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/conf.py +1 -1
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/fetchers.rst +15 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/sources.rst +9 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/pyproject.toml +1 -1
- forcingkit-0.19.0/src/forcingkit/fetchers/hfradar.py +500 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/static/index.html +1 -1
- {forcingkit-0.18.1 → forcingkit-0.19.0}/static/preview3d.js +3 -3
- forcingkit-0.19.0/tests/unit/test_hfradar.py +237 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/uv.lock +1 -1
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.env.template +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.gitignore +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.markdownlint.yaml +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/.python-version +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/AGENTS.md +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/CONTRIBUTING.md +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/Dockerfile +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/LICENSE +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/README.md +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/LICENSE +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/README.md +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/pyproject.toml +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/tests/conftest.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/tests/test_client.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/tests/test_schema.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/client/uv.lock +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docker-compose.yml +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/Makefile +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/make.bat +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/_static/custom.css +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/client.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/hycom.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/index.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/necofs.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/rivers.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/docs/source/waves.rst +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/main.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/__init__.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/routers/viewer.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/service/run_server.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrdps.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/ndbc.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/render.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/static/app.js +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/static/favicon.ico +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/static/favicon.svg +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/static/logo.svg +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/static/styles.css +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/__init__.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrdps.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_main.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.18.1 → forcingkit-0.19.0}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -95,4 +95,6 @@ uv run python -c "from forcingkit.fetchers import nyofs; print(nyofs.get_metadat
|
|
|
95
95
|
- **Cache keys**: Cache Zarr keys are deterministic hashes of bbox
|
|
96
96
|
- time window. Changing fetcher output schema (variable names, dims)
|
|
97
97
|
will miss existing cache entries — purge
|
|
98
|
-
`~/.cache/forcingkit/` when making breaking schema changes
|
|
98
|
+
`~/.cache/forcingkit/` when making breaking schema changes, except
|
|
99
|
+
`hfradar/`: that is the HF radar archive (`fetchers/hfradar.py`), days the
|
|
100
|
+
server no longer holds, not a cache.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.19.0
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -177,6 +177,21 @@ These serve validation rather than forcing:
|
|
|
177
177
|
are read at once, through `insitukit <https://github.com/lhzn-io/insitukit>`__'s ERDDAP client
|
|
178
178
|
(one attempt each, starting at least 0.2 s apart).
|
|
179
179
|
|
|
180
|
+
- **HF radar** surface currents (library only: ``forcingkit.fetchers.hfradar``), the hourly total
|
|
181
|
+
vectors of the `NOAA IOOS HF Radar National Network <https://ioos.noaa.gov/project/hf-radar/>`__
|
|
182
|
+
(HFRNet) for the US East and Gulf Coast at 1 km and 2 km, from NDBC's `THREDDS server
|
|
183
|
+
<https://dods.ndbc.noaa.gov/thredds/catalog/hfradar/catalog.html>`__ over OPeNDAP (since 0.19.0).
|
|
184
|
+
They represent roughly the upper 0.5 m, for scoring a model's surface layer. ``fetch`` returns a box
|
|
185
|
+
and window as stored on the server (16-bit integers with scale 0.01, so nothing is rounded twice).
|
|
186
|
+
The server keeps only about 92 days, so ``archive`` keeps a box one NetCDF-4 file per UTC day under
|
|
187
|
+
``FORCINGKIT_CACHE_DIR/hfradar/usegc_<grid>/<box>/<YYYY>/<MM>/``, written once the server has
|
|
188
|
+
published past the day's end and six hours have passed, and never rewritten; it takes any store
|
|
189
|
+
with ``has`` and ``put`` (a create-only bucket, for example). ``load`` reads a box and window back
|
|
190
|
+
in m/s from any archived box that contains it, and the other days from the server while it holds
|
|
191
|
+
them. The ``hfradar`` directory is an archive, not a cache: do not purge it with the Zarr stores.
|
|
192
|
+
Coverage is uneven: in western Long Island Sound about 2 % of the 1 km cells in the box
|
|
193
|
+
40.85-41.3 N, 73.8-71.8 W have a solution in a covered hour, and whole weeks have none.
|
|
194
|
+
|
|
180
195
|
NOAA OFS archive locations
|
|
181
196
|
--------------------------
|
|
182
197
|
|
|
@@ -71,6 +71,13 @@ Providers
|
|
|
71
71
|
- Acknowledge NOAA NESDIS CoastWatch and the datasets named in the store's ``products``
|
|
72
72
|
attribute. MUR: JPL MUR MEaSUREs Project (2015) and Chin et al. (2017). OLCI and DINEOF
|
|
73
73
|
products require "Contains modified Copernicus Sentinel data [year]".
|
|
74
|
+
* - HF radar surface currents
|
|
75
|
+
- The radar sites' operators (for western Long Island Sound, the `University of Connecticut
|
|
76
|
+
<https://lisicos.uconn.edu/>`__ LISICOS program), combined by the `NOAA IOOS HF Radar
|
|
77
|
+
National Network <https://ioos.noaa.gov/project/hf-radar/>`__ (HFRNet) and served by `NOAA
|
|
78
|
+
NDBC <https://www.ndbc.noaa.gov/>`__
|
|
79
|
+
- Acknowledge the site operators and HFRNet (Terrill et al., 2006), as in each file's
|
|
80
|
+
``source_credit`` attribute.
|
|
74
81
|
* - Water-column profiles
|
|
75
82
|
- `University of Connecticut Department of Marine Sciences
|
|
76
83
|
<https://marinesciences.uconn.edu/>`__ (`ERDDAP
|
|
@@ -140,6 +147,8 @@ References
|
|
|
140
147
|
split-explicit, free-surface, topography-following-coordinate oceanic model. *Ocean
|
|
141
148
|
Modelling*, 9(4), 347-404.
|
|
142
149
|
`doi:10.1016/j.ocemod.2004.08.002 <https://doi.org/10.1016/j.ocemod.2004.08.002>`__
|
|
150
|
+
- Terrill, E., M. Otero, L. Hazard, et al. (2006). Data management and real-time distribution in
|
|
151
|
+
the HF-radar national network. *Proceedings of the MTS/IEEE OCEANS 2006 Conference*, Boston, MA.
|
|
143
152
|
- Tsujino, H., S. Urakawa, H. Nakano, et al. (2018). JRA-55 based surface dataset for driving
|
|
144
153
|
ocean-sea-ice models (JRA55-do). *Ocean Modelling*, 130, 79-139.
|
|
145
154
|
`doi:10.1016/j.ocemod.2018.07.002 <https://doi.org/10.1016/j.ocemod.2018.07.002>`__
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.19.0"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
"""Hourly HF radar surface currents from the NOAA IOOS HF Radar National Network, and an archive that outlasts the
|
|
2
|
+
server's rolling window.
|
|
3
|
+
|
|
4
|
+
Shore-based HF radars measure radial surface currents; the HF Radar National Network (HFRNet) combines them into
|
|
5
|
+
hourly total vectors on fixed 1 km, 2 km and 6 km grids. NOAA's National Data Buoy Center serves the US East and Gulf
|
|
6
|
+
Coast totals over OPeNDAP (``hfradar_usegc_1km``, ``_2km``), but keeps only about 92 days. The values are surface
|
|
7
|
+
currents representative of roughly the upper 0.5 m: an observation for scoring a model's surface layer, not a
|
|
8
|
+
forcing.
|
|
9
|
+
|
|
10
|
+
`fetch` reads a box and a time window from the server as stored: 16-bit integers with their scale factor
|
|
11
|
+
(0.01) and fill value, so nothing is rounded twice. DAP2 has no signed byte, so a signed byte arrives unsigned with
|
|
12
|
+
``_Unsigned = "false"``; it is turned back into int8 here. The variables are ``u`` and ``v`` (eastward and northward
|
|
13
|
+
surface current, m/s), ``dopx``, ``dopy`` and ``hdop`` (dilution of precision, for screening the geometry),
|
|
14
|
+
``number_of_sites`` and ``number_of_radials``, with the grid mapping and the processing settings.
|
|
15
|
+
|
|
16
|
+
`archive` keeps a box one file per UTC day and grid (NetCDF-4), under ``settings.cache_dir("hfradar")`` unless
|
|
17
|
+
another store is given::
|
|
18
|
+
|
|
19
|
+
usegc_<grid>/<box>/<YYYY>/<MM>/hfradar_usegc_<grid>_<YYYYMMDD>.nc
|
|
20
|
+
|
|
21
|
+
where ``<box>`` names the box, for example ``40.850N_41.300N_073.800W_071.800W``. A day is written once the server
|
|
22
|
+
has published past its end and six hours have gone by, and never rewritten; every hour the server holds for the day is
|
|
23
|
+
kept, with no screening. Unlike the Zarr stores beside it, this directory is an archive: the server cannot supply
|
|
24
|
+
those days again once they leave its window, so do not purge it. `load` reads a box and window back, decoded to m/s,
|
|
25
|
+
from any archived box that contains it, and fetches days the archive lacks from the server while it still holds them.
|
|
26
|
+
|
|
27
|
+
Credit: the radar sites' operators (named in HFRNet's site list; for example the University of Connecticut's LISICOS
|
|
28
|
+
for western Long Island Sound) and the NOAA IOOS HF Radar National Network; see ``source_credit`` in each file.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import io
|
|
34
|
+
import logging
|
|
35
|
+
import os
|
|
36
|
+
import re
|
|
37
|
+
from collections.abc import Sequence
|
|
38
|
+
from datetime import datetime, timedelta, timezone
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
from typing import Protocol
|
|
41
|
+
|
|
42
|
+
import numpy as np
|
|
43
|
+
import pandas as pd
|
|
44
|
+
import requests
|
|
45
|
+
import xarray as xr
|
|
46
|
+
|
|
47
|
+
from forcingkit import settings
|
|
48
|
+
|
|
49
|
+
logger = logging.getLogger(__name__)
|
|
50
|
+
|
|
51
|
+
SERVER = "https://dods.ndbc.noaa.gov/thredds/dodsC"
|
|
52
|
+
USER_AGENT = "forcingkit (+https://github.com/lhzn-io/forcingkit)"
|
|
53
|
+
GRIDS = ("1km", "2km")
|
|
54
|
+
VARIABLES = ["u", "v", "dopx", "dopy", "hdop", "number_of_sites", "number_of_radials"]
|
|
55
|
+
# The grid mapping and the processing settings. (The server's radial_metadata names the radial files of its latest
|
|
56
|
+
# hour only, so it is left out.)
|
|
57
|
+
SCALARS = ["wgs84", "processing_parameters"]
|
|
58
|
+
LAG = timedelta(
|
|
59
|
+
hours=6
|
|
60
|
+
) # after a day ends, before it is written (the server posts an hour about 2 h late)
|
|
61
|
+
LOOKBACK_DAYS = 10 # what a routine `archive` run looks back over
|
|
62
|
+
CHUNK_DAYS = 8 # days per request
|
|
63
|
+
# Global attributes carried over from the server's dataset (not the ones describing its latest hour).
|
|
64
|
+
KEEP_ATTRS = [
|
|
65
|
+
"Conventions",
|
|
66
|
+
"title",
|
|
67
|
+
"summary",
|
|
68
|
+
"institution",
|
|
69
|
+
"program",
|
|
70
|
+
"project",
|
|
71
|
+
"naming_authority",
|
|
72
|
+
"license",
|
|
73
|
+
"acknowledgment",
|
|
74
|
+
"acknowledgement",
|
|
75
|
+
"references",
|
|
76
|
+
"creator_name",
|
|
77
|
+
"creator_url",
|
|
78
|
+
"creator_email",
|
|
79
|
+
"publisher_name",
|
|
80
|
+
"publisher_url",
|
|
81
|
+
"publisher_email",
|
|
82
|
+
"contributor_name",
|
|
83
|
+
"contributor_role",
|
|
84
|
+
"processing_level",
|
|
85
|
+
"instrument",
|
|
86
|
+
"platform",
|
|
87
|
+
"product_version",
|
|
88
|
+
"metadata_link",
|
|
89
|
+
"keywords",
|
|
90
|
+
"keywords_vocabulary",
|
|
91
|
+
"standard_name_vocabulary",
|
|
92
|
+
]
|
|
93
|
+
CREDIT = (
|
|
94
|
+
"HF radar sites operated by their institutions (for western Long Island Sound, the University of Connecticut's "
|
|
95
|
+
"LISICOS); hourly total vectors from the NOAA IOOS HF Radar National Network (HFRNet), served by NOAA NDBC."
|
|
96
|
+
)
|
|
97
|
+
NAME = re.compile(
|
|
98
|
+
r"usegc_[126]km/\d{2}\.\d{3}[NS]_\d{2}\.\d{3}[NS]_\d{3}\.\d{3}[EW]_\d{3}\.\d{3}[EW]/\d{4}/\d{2}/"
|
|
99
|
+
r"hfradar_usegc_[126]km_\d{8}\.nc"
|
|
100
|
+
)
|
|
101
|
+
_BOX = re.compile(
|
|
102
|
+
r"(\d{2}\.\d{3})([NS])_(\d{2}\.\d{3})([NS])_(\d{3}\.\d{3})([EW])_(\d{3}\.\d{3})([EW])"
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
BBox = Sequence[float] # [min_lon, min_lat, max_lon, max_lat]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class Store(Protocol):
|
|
109
|
+
"""Where archived days go. `has` may answer False when it cannot tell (a create-only bucket); `put` then refuses
|
|
110
|
+
an existing file and returns False."""
|
|
111
|
+
|
|
112
|
+
def has(self, name: str) -> bool: ...
|
|
113
|
+
|
|
114
|
+
def put(self, name: str, data: bytes) -> bool: ...
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
class LocalStore:
|
|
118
|
+
"""A directory; files are created once and never replaced."""
|
|
119
|
+
|
|
120
|
+
def __init__(self, root: str | os.PathLike | None = None):
|
|
121
|
+
self.root = Path(
|
|
122
|
+
root if root is not None else settings.cache_dir("hfradar")
|
|
123
|
+
).expanduser()
|
|
124
|
+
|
|
125
|
+
def has(self, name: str) -> bool:
|
|
126
|
+
return (self.root / name).exists()
|
|
127
|
+
|
|
128
|
+
def put(self, name: str, data: bytes) -> bool:
|
|
129
|
+
if not NAME.fullmatch(name):
|
|
130
|
+
raise ValueError(f"refusing to write {name}")
|
|
131
|
+
path = self.root / name
|
|
132
|
+
if path.exists():
|
|
133
|
+
return False
|
|
134
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
135
|
+
tmp = path.with_suffix(".part")
|
|
136
|
+
tmp.write_bytes(data)
|
|
137
|
+
tmp.rename(path)
|
|
138
|
+
return True
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def dataset_url(grid: str) -> str:
|
|
142
|
+
return f"{SERVER}/hfradar_usegc_{grid}"
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def box_key(bbox: BBox) -> str:
|
|
146
|
+
"""The box's directory name, to three decimals: ``40.850N_41.300N_073.800W_071.800W`` (south, north, west, east)."""
|
|
147
|
+
w, s, e, n = (float(x) for x in bbox)
|
|
148
|
+
|
|
149
|
+
def lat(v: float) -> str:
|
|
150
|
+
return f"{abs(v):06.3f}{'N' if v >= 0 else 'S'}"
|
|
151
|
+
|
|
152
|
+
def lon(v: float) -> str:
|
|
153
|
+
return f"{abs(v):07.3f}{'E' if v >= 0 else 'W'}"
|
|
154
|
+
|
|
155
|
+
return f"{lat(s)}_{lat(n)}_{lon(w)}_{lon(e)}"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def parse_box_key(key: str) -> tuple[float, float, float, float] | None:
|
|
159
|
+
"""The bbox ([min_lon, min_lat, max_lon, max_lat]) a directory name stands for, or None."""
|
|
160
|
+
m = _BOX.fullmatch(key)
|
|
161
|
+
if not m:
|
|
162
|
+
return None
|
|
163
|
+
s, sh, n, nh, w, wh, e, eh = m.groups()
|
|
164
|
+
sign = {"N": 1, "S": -1, "E": 1, "W": -1}
|
|
165
|
+
return (
|
|
166
|
+
float(w) * sign[wh],
|
|
167
|
+
float(s) * sign[sh],
|
|
168
|
+
float(e) * sign[eh],
|
|
169
|
+
float(n) * sign[nh],
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def day_name(grid: str, bbox: BBox, day: pd.Timestamp) -> str:
|
|
174
|
+
return (
|
|
175
|
+
f"usegc_{grid}/{box_key(bbox)}/{day:%Y/%m}/hfradar_usegc_{grid}_{day:%Y%m%d}.nc"
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def open_remote(
|
|
180
|
+
grid: str = "1km", session: requests.Session | None = None
|
|
181
|
+
) -> xr.Dataset:
|
|
182
|
+
"""The server's dataset, lazily, with the stored integers left as they are."""
|
|
183
|
+
if session is None:
|
|
184
|
+
session = requests.Session()
|
|
185
|
+
session.headers["User-Agent"] = USER_AGENT
|
|
186
|
+
return xr.open_dataset(
|
|
187
|
+
dataset_url(grid).replace("https://", "dap2://"),
|
|
188
|
+
engine="pydap",
|
|
189
|
+
session=session,
|
|
190
|
+
mask_and_scale=False,
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def box_slices(lat: np.ndarray, lon: np.ndarray, bbox: BBox) -> tuple[slice, slice]:
|
|
195
|
+
"""Index slices of the grid cells whose centres fall in the box."""
|
|
196
|
+
w, s, e, n = bbox
|
|
197
|
+
iy = np.flatnonzero((lat >= s) & (lat <= n))
|
|
198
|
+
ix = np.flatnonzero((lon >= w) & (lon <= e))
|
|
199
|
+
if not len(iy) or not len(ix):
|
|
200
|
+
raise ValueError("the box misses the grid")
|
|
201
|
+
return slice(int(iy[0]), int(iy[-1]) + 1), slice(int(ix[0]), int(ix[-1]) + 1)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _utc_naive(t) -> pd.Timestamp:
|
|
205
|
+
ts = pd.Timestamp(t)
|
|
206
|
+
return ts.tz_convert("UTC").tz_localize(None) if ts.tzinfo else ts
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def ready_days(
|
|
210
|
+
times: pd.DatetimeIndex,
|
|
211
|
+
now: datetime,
|
|
212
|
+
start: pd.Timestamp | None = None,
|
|
213
|
+
end: pd.Timestamp | None = None,
|
|
214
|
+
) -> list[pd.Timestamp]:
|
|
215
|
+
"""The UTC days with hours on the server that may be archived: the server has published past the day's end and
|
|
216
|
+
the lag has passed. Bounded by ``start`` and ``end`` (days, inclusive) when given."""
|
|
217
|
+
if not len(times):
|
|
218
|
+
return []
|
|
219
|
+
last = times.max()
|
|
220
|
+
now_ts = _utc_naive(now)
|
|
221
|
+
days = []
|
|
222
|
+
for day in pd.DatetimeIndex(times.normalize().unique()).sort_values():
|
|
223
|
+
stop = day + pd.Timedelta(days=1)
|
|
224
|
+
if last < stop or now_ts < stop + LAG:
|
|
225
|
+
continue
|
|
226
|
+
if (start is not None and day < start) or (end is not None and day > end):
|
|
227
|
+
continue
|
|
228
|
+
days.append(day)
|
|
229
|
+
return days
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def runs(days: list[pd.Timestamp], most: int = CHUNK_DAYS) -> list[list[pd.Timestamp]]:
|
|
233
|
+
"""Consecutive days grouped into requests of at most ``most`` days."""
|
|
234
|
+
out: list[list[pd.Timestamp]] = []
|
|
235
|
+
for day in days:
|
|
236
|
+
if out and day - out[-1][-1] == pd.Timedelta(days=1) and len(out[-1]) < most:
|
|
237
|
+
out[-1].append(day)
|
|
238
|
+
else:
|
|
239
|
+
out.append([day])
|
|
240
|
+
return out
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _stored(var: xr.DataArray) -> tuple[np.ndarray, dict]:
|
|
244
|
+
"""A variable's stored values and its attributes, minus the server's private ones, with signed bytes restored."""
|
|
245
|
+
values = np.asarray(var.values)
|
|
246
|
+
if (
|
|
247
|
+
values.dtype == np.uint8
|
|
248
|
+
and str(var.attrs.get("_Unsigned", "")).lower() == "false"
|
|
249
|
+
):
|
|
250
|
+
values = values.view(np.int8)
|
|
251
|
+
attrs = {
|
|
252
|
+
k: v
|
|
253
|
+
for k, v in var.attrs.items()
|
|
254
|
+
if (not k.startswith("_") or k == "_FillValue") and k != "coordinates"
|
|
255
|
+
}
|
|
256
|
+
return values, attrs
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _normalize(
|
|
260
|
+
sub: xr.Dataset, grid: str, bbox: BBox, source_attrs: dict, fetched: datetime
|
|
261
|
+
) -> xr.Dataset:
|
|
262
|
+
"""The server's subset as a clean Dataset of stored values, with its encodings and credit."""
|
|
263
|
+
out = xr.Dataset(
|
|
264
|
+
coords={
|
|
265
|
+
"time": sub["time"].values,
|
|
266
|
+
"lat": sub["lat"].values,
|
|
267
|
+
"lon": sub["lon"].values,
|
|
268
|
+
}
|
|
269
|
+
)
|
|
270
|
+
for name in VARIABLES:
|
|
271
|
+
if name in sub:
|
|
272
|
+
values, attrs = _stored(sub[name])
|
|
273
|
+
fill = attrs.pop("_FillValue", None)
|
|
274
|
+
out[name] = (sub[name].dims, values, attrs)
|
|
275
|
+
out[name].encoding.update(
|
|
276
|
+
{"zlib": True, "complevel": 4, "shuffle": True, "dtype": values.dtype}
|
|
277
|
+
)
|
|
278
|
+
if fill is not None:
|
|
279
|
+
out[name].encoding["_FillValue"] = values.dtype.type(fill)
|
|
280
|
+
for name in SCALARS:
|
|
281
|
+
if name in sub:
|
|
282
|
+
values, attrs = _stored(sub[name])
|
|
283
|
+
out[name] = ((), values, attrs)
|
|
284
|
+
for c in ("lat", "lon"):
|
|
285
|
+
out[c].attrs = {k: v for k, v in sub[c].attrs.items() if not k.startswith("_")}
|
|
286
|
+
out["time"].attrs = {
|
|
287
|
+
"standard_name": "time",
|
|
288
|
+
"long_name": "time at the centre of the hour",
|
|
289
|
+
}
|
|
290
|
+
out["time"].encoding.update(
|
|
291
|
+
{"units": "seconds since 1970-01-01", "calendar": "gregorian", "dtype": "int32"}
|
|
292
|
+
)
|
|
293
|
+
attrs = {k: source_attrs[k] for k in KEEP_ATTRS if k in source_attrs}
|
|
294
|
+
w, s, e, n = bbox
|
|
295
|
+
attrs.update(
|
|
296
|
+
{
|
|
297
|
+
"source_dataset": dataset_url(grid),
|
|
298
|
+
"source_credit": CREDIT,
|
|
299
|
+
"subset": f"latitude {s} to {n}, longitude {w} to {e}; values as stored on the server",
|
|
300
|
+
"history": f"{fetched:%Y-%m-%dT%H:%M:%SZ} subset from {dataset_url(grid)} by forcingkit",
|
|
301
|
+
}
|
|
302
|
+
)
|
|
303
|
+
out.attrs = attrs
|
|
304
|
+
return out
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def fetch(
|
|
308
|
+
bbox: BBox,
|
|
309
|
+
start,
|
|
310
|
+
end,
|
|
311
|
+
grid: str = "1km",
|
|
312
|
+
src: xr.Dataset | None = None,
|
|
313
|
+
session: requests.Session | None = None,
|
|
314
|
+
) -> xr.Dataset:
|
|
315
|
+
"""The box's hours in [start, end) from the server, as stored (see the module notes). Pass an open `src` to reuse
|
|
316
|
+
one connection across calls. Empty in time when the server holds none of the window."""
|
|
317
|
+
own = src is None
|
|
318
|
+
src = src if src is not None else open_remote(grid, session)
|
|
319
|
+
try:
|
|
320
|
+
times = pd.DatetimeIndex(src["time"].values)
|
|
321
|
+
t = np.flatnonzero((times >= _utc_naive(start)) & (times < _utc_naive(end)))
|
|
322
|
+
ys, xs = box_slices(src["lat"].values, src["lon"].values, bbox)
|
|
323
|
+
names = [n for n in VARIABLES + SCALARS if n in src]
|
|
324
|
+
tsel = slice(int(t[0]), int(t[-1]) + 1) if len(t) else slice(0, 0)
|
|
325
|
+
sub = src[names].isel(time=tsel, lat=ys, lon=xs).load()
|
|
326
|
+
return _normalize(sub, grid, bbox, dict(src.attrs), datetime.now(timezone.utc))
|
|
327
|
+
finally:
|
|
328
|
+
if own:
|
|
329
|
+
src.close()
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def day_file(ds: xr.Dataset, day: pd.Timestamp) -> bytes:
|
|
333
|
+
"""One UTC day of a `fetch` result as NetCDF-4 bytes (the archive's file)."""
|
|
334
|
+
one = ds.sel(
|
|
335
|
+
time=slice(day, day + pd.Timedelta(days=1) - pd.Timedelta(seconds=1))
|
|
336
|
+
).copy()
|
|
337
|
+
for name in VARIABLES: # sel keeps the encodings, but be explicit
|
|
338
|
+
if name in ds:
|
|
339
|
+
one[name].encoding = dict(ds[name].encoding)
|
|
340
|
+
one["time"].encoding = dict(ds["time"].encoding)
|
|
341
|
+
one.attrs = dict(
|
|
342
|
+
ds.attrs, subset_day=f"{day:%Y-%m-%d} UTC", subset_hours=int(one.sizes["time"])
|
|
343
|
+
)
|
|
344
|
+
buf = io.BytesIO()
|
|
345
|
+
one.to_netcdf(buf, engine="h5netcdf")
|
|
346
|
+
return buf.getvalue()
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def archive(
|
|
350
|
+
bbox: BBox,
|
|
351
|
+
start: str | None = None,
|
|
352
|
+
end: str | None = None,
|
|
353
|
+
grids: Sequence[str] = GRIDS,
|
|
354
|
+
store: Store | None = None,
|
|
355
|
+
lookback_days: int = LOOKBACK_DAYS,
|
|
356
|
+
now: datetime | None = None,
|
|
357
|
+
session: requests.Session | None = None,
|
|
358
|
+
) -> dict[str, list[str]]:
|
|
359
|
+
"""Archive every ready day of the box not yet kept, per grid; return the names written per grid.
|
|
360
|
+
|
|
361
|
+
With neither ``start`` nor ``end``, the last ``lookback_days`` days; ``start="all"`` takes every day the server
|
|
362
|
+
still holds. The store defaults to ``settings.cache_dir("hfradar")``; ``session`` lets a caller send its own
|
|
363
|
+
User-Agent. A request that fails is skipped and the rest carry on; the call then raises, naming the days
|
|
364
|
+
missed."""
|
|
365
|
+
now = now or datetime.now(timezone.utc)
|
|
366
|
+
store = store if store is not None else LocalStore()
|
|
367
|
+
written: dict[str, list[str]] = {}
|
|
368
|
+
failed: list[str] = []
|
|
369
|
+
for grid in grids:
|
|
370
|
+
src = open_remote(grid, session)
|
|
371
|
+
try:
|
|
372
|
+
times = pd.DatetimeIndex(src["time"].values)
|
|
373
|
+
if start == "all":
|
|
374
|
+
first = None
|
|
375
|
+
elif start:
|
|
376
|
+
first = pd.Timestamp(start)
|
|
377
|
+
else:
|
|
378
|
+
first = _utc_naive(now).normalize() - pd.Timedelta(days=lookback_days)
|
|
379
|
+
last = pd.Timestamp(end) if end else None
|
|
380
|
+
todo = [
|
|
381
|
+
d
|
|
382
|
+
for d in ready_days(times, now, first, last)
|
|
383
|
+
if not store.has(day_name(grid, bbox, d))
|
|
384
|
+
]
|
|
385
|
+
written[grid] = []
|
|
386
|
+
for group in runs(todo):
|
|
387
|
+
try:
|
|
388
|
+
ds = fetch(
|
|
389
|
+
bbox, group[0], group[-1] + pd.Timedelta(days=1), grid, src=src
|
|
390
|
+
)
|
|
391
|
+
except Exception as exc: # carry on; a later run fills the gap while the server holds it
|
|
392
|
+
logger.warning(
|
|
393
|
+
"HF radar %s %s to %s: %s",
|
|
394
|
+
grid,
|
|
395
|
+
group[0].date(),
|
|
396
|
+
group[-1].date(),
|
|
397
|
+
exc,
|
|
398
|
+
)
|
|
399
|
+
failed.append(f"{grid} {group[0]:%Y-%m-%d}..{group[-1]:%Y-%m-%d}")
|
|
400
|
+
continue
|
|
401
|
+
for day in group:
|
|
402
|
+
name = day_name(grid, bbox, day)
|
|
403
|
+
if store.put(name, day_file(ds, day)):
|
|
404
|
+
written[grid].append(name)
|
|
405
|
+
logger.info("%s written", name)
|
|
406
|
+
else:
|
|
407
|
+
logger.info("%s already kept", name)
|
|
408
|
+
finally:
|
|
409
|
+
src.close()
|
|
410
|
+
if failed:
|
|
411
|
+
raise RuntimeError(
|
|
412
|
+
f"HF radar days not fetched: {', '.join(failed)} (written: {written})"
|
|
413
|
+
)
|
|
414
|
+
return written
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _decoded(ds: xr.Dataset) -> xr.Dataset:
|
|
418
|
+
"""Stored values decoded as a reader of the archive sees them (scale applied, fills as NaN)."""
|
|
419
|
+
buf = io.BytesIO()
|
|
420
|
+
ds.to_netcdf(buf, engine="h5netcdf")
|
|
421
|
+
buf.seek(0)
|
|
422
|
+
with xr.open_dataset(buf, engine="h5netcdf") as out:
|
|
423
|
+
return out.load()
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _containing_boxes(root: Path, grid: str, bbox: BBox) -> list[Path]:
|
|
427
|
+
"""Archived box directories that contain the box, smallest first."""
|
|
428
|
+
found = []
|
|
429
|
+
for d in (root / f"usegc_{grid}").glob("*"):
|
|
430
|
+
b = parse_box_key(d.name)
|
|
431
|
+
if (
|
|
432
|
+
b
|
|
433
|
+
and b[0] <= bbox[0]
|
|
434
|
+
and b[1] <= bbox[1]
|
|
435
|
+
and b[2] >= bbox[2]
|
|
436
|
+
and b[3] >= bbox[3]
|
|
437
|
+
):
|
|
438
|
+
found.append(((b[2] - b[0]) * (b[3] - b[1]), d))
|
|
439
|
+
return [d for _, d in sorted(found)]
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def load(
|
|
443
|
+
bbox: BBox,
|
|
444
|
+
start,
|
|
445
|
+
end,
|
|
446
|
+
grid: str = "1km",
|
|
447
|
+
root: str | os.PathLike | None = None,
|
|
448
|
+
remote: bool = True,
|
|
449
|
+
session: requests.Session | None = None,
|
|
450
|
+
) -> xr.Dataset:
|
|
451
|
+
"""The box's hours in [start, end), decoded (u and v in m/s, NaN where there is no solution): archived days from
|
|
452
|
+
any archived box that contains this one, and, when ``remote``, the other days from the server while it still
|
|
453
|
+
holds them (not archived by this call). Empty in time when neither has any."""
|
|
454
|
+
base = Path(
|
|
455
|
+
root if root is not None else settings.cache_dir("hfradar")
|
|
456
|
+
).expanduser()
|
|
457
|
+
t0, t1 = _utc_naive(start), _utc_naive(end)
|
|
458
|
+
boxes = _containing_boxes(base, grid, bbox)
|
|
459
|
+
parts: list[xr.Dataset] = []
|
|
460
|
+
missing: list[pd.Timestamp] = []
|
|
461
|
+
for day in pd.date_range(
|
|
462
|
+
t0.normalize(), (t1 - pd.Timedelta(seconds=1)).normalize(), freq="D"
|
|
463
|
+
):
|
|
464
|
+
path = next(
|
|
465
|
+
(
|
|
466
|
+
p
|
|
467
|
+
for b in boxes
|
|
468
|
+
if (
|
|
469
|
+
p := b / f"{day:%Y/%m}" / f"hfradar_usegc_{grid}_{day:%Y%m%d}.nc"
|
|
470
|
+
).exists()
|
|
471
|
+
),
|
|
472
|
+
None,
|
|
473
|
+
)
|
|
474
|
+
if path is None:
|
|
475
|
+
missing.append(day)
|
|
476
|
+
continue
|
|
477
|
+
with xr.open_dataset(path) as ds:
|
|
478
|
+
ys, xs = box_slices(ds["lat"].values, ds["lon"].values, bbox)
|
|
479
|
+
parts.append(
|
|
480
|
+
ds[[v for v in VARIABLES if v in ds]].isel(lat=ys, lon=xs).load()
|
|
481
|
+
)
|
|
482
|
+
if remote and missing:
|
|
483
|
+
src = open_remote(grid, session)
|
|
484
|
+
try:
|
|
485
|
+
for group in runs(missing):
|
|
486
|
+
got = fetch(
|
|
487
|
+
bbox, group[0], group[-1] + pd.Timedelta(days=1), grid, src=src
|
|
488
|
+
)
|
|
489
|
+
if got.sizes["time"]:
|
|
490
|
+
parts.append(_decoded(got[[v for v in VARIABLES if v in got]]))
|
|
491
|
+
finally:
|
|
492
|
+
src.close()
|
|
493
|
+
if not parts:
|
|
494
|
+
return xr.Dataset()
|
|
495
|
+
out = xr.concat(
|
|
496
|
+
parts, dim="time", data_vars="all", coords="minimal", join="exact"
|
|
497
|
+
).sortby("time")
|
|
498
|
+
out = out.sel(time=slice(t0, t1 - pd.Timedelta(seconds=1)))
|
|
499
|
+
out.attrs = {"source_dataset": dataset_url(grid), "source_credit": CREDIT}
|
|
500
|
+
return out
|
|
@@ -140,7 +140,7 @@
|
|
|
140
140
|
{ "imports": {
|
|
141
141
|
"three": "https://cdn.jsdelivr.net/npm/three@0.170.0/build/three.module.js",
|
|
142
142
|
"camera-controls": "https://cdn.jsdelivr.net/npm/camera-controls@3.1.2/dist/camera-controls.module.min.js",
|
|
143
|
-
"vantagekit/three": "https://cdn.jsdelivr.net/gh/lhzn-io/vantagekit@v0.
|
|
143
|
+
"vantagekit/three": "https://cdn.jsdelivr.net/gh/lhzn-io/vantagekit@v0.8.1/vantagekit-three.js"
|
|
144
144
|
} }
|
|
145
145
|
</script>
|
|
146
146
|
<script type="module" src="preview3d.js"></script>
|
|
@@ -138,12 +138,12 @@ function initThree(container) {
|
|
|
138
138
|
camera = new THREE.PerspectiveCamera(55, container.clientWidth / container.clientHeight, 0.1, 5000);
|
|
139
139
|
camera.position.set(0, 150, 250);
|
|
140
140
|
|
|
141
|
-
// The scene is about 200 units across whatever the dataset's size, so
|
|
141
|
+
// The scene is about 200 units across whatever the dataset's size, so meters per unit is set
|
|
142
142
|
// per dataset (buildVectorField). The keyboard acts only while the canvas has focus, so the
|
|
143
143
|
// page's own keys and scrolling keep working.
|
|
144
144
|
nav = createVantage({
|
|
145
145
|
THREE, CameraControls, camera, dom: canvas,
|
|
146
|
-
|
|
146
|
+
metersPerUnit: 1, exaggeration: DEPTH_EXAG * DEPTH_BASE_MULT,
|
|
147
147
|
keys: 'focus', minDistance: 0.5, smoothTime: 0.2,
|
|
148
148
|
});
|
|
149
149
|
|
|
@@ -207,7 +207,7 @@ function buildVectorField(data, fitCamera) {
|
|
|
207
207
|
const latSpan = Math.max(latMax - latMin, 0.001) * degToM;
|
|
208
208
|
const maxSpan = Math.max(lonSpan, latSpan);
|
|
209
209
|
const sceneScale = 200 / maxSpan;
|
|
210
|
-
nav.
|
|
210
|
+
nav.metersPerUnit = 1 / sceneScale; // movement speeds and readouts in real meters
|
|
211
211
|
|
|
212
212
|
// Compute max magnitude
|
|
213
213
|
let maxMag = 0;
|