forcingkit 0.20.0__tar.gz → 0.20.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.20.0 → forcingkit-0.20.1}/PKG-INFO +1 -1
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/conf.py +1 -1
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/fetchers.rst +5 -3
- {forcingkit-0.20.0 → forcingkit-0.20.1}/pyproject.toml +1 -1
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hfradar.py +36 -12
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_hfradar.py +55 -1
- {forcingkit-0.20.0 → forcingkit-0.20.1}/uv.lock +1 -1
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.agents/forcingkit.md +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.env.template +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.gitignore +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.markdownlint.yaml +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/.python-version +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/AGENTS.md +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/CONTRIBUTING.md +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/Dockerfile +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/LICENSE +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/README.md +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/LICENSE +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/README.md +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/pyproject.toml +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/tests/conftest.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/tests/test_client.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/tests/test_schema.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/client/uv.lock +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docker-compose.yml +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/Makefile +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/make.bat +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/client.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/hycom.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/index.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/necofs.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/rivers.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/sources.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/docs/source/waves.rst +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/main.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/__init__.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/routers/viewer.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/service/run_server.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrdps.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/ndbc.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/render.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/static/app.js +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/static/favicon.ico +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/static/favicon.svg +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/static/index.html +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/static/logo.svg +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/static/preview3d.js +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/static/styles.css +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/__init__.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_hrdps.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_main.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.20.0 → forcingkit-0.20.1}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.20.
|
|
3
|
+
Version: 0.20.1
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -190,6 +190,8 @@ These serve validation rather than forcing:
|
|
|
190
190
|
in m/s from any archived box that contains it, and the other days from the server while it holds
|
|
191
191
|
them. The ``hfradar`` directory is an archive, not a cache: do not purge it with the Zarr stores.
|
|
192
192
|
``FORCINGKIT_HFRADAR_ARCHIVE`` puts it elsewhere (since 0.20.0).
|
|
193
|
+
Coverage is uneven: in western Long Island Sound about 2 % of the 1 km cells in the box
|
|
194
|
+
40.85-41.3 N, 73.8-71.8 W have a solution in a covered hour, and whole weeks have none.
|
|
193
195
|
|
|
194
196
|
Older days come from `NOAA NCEI <https://www.ncei.noaa.gov/>`__, which archives the same hourly
|
|
195
197
|
totals (the same variables, stored the same way, on the same grids) as one file per hour in monthly
|
|
@@ -197,13 +199,13 @@ These serve validation rather than forcing:
|
|
|
197
199
|
accessions through NCEI's metadata search, reads each hour's file (about 0.8 MB at 1 km for the whole
|
|
198
200
|
coast), cuts the box out and writes the same daily files, so ``load`` reads days from either source
|
|
199
201
|
alike; ``source_dataset`` names the accession. Days already kept are skipped, and a day is written
|
|
200
|
-
with the hours NCEI holds (since 0.20.0).
|
|
202
|
+
with the hours NCEI holds (since 0.20.0). The month list and folders found are kept in
|
|
203
|
+
``ncei_accessions.json`` beside the archive, so a rerun skips NCEI's slow search (since 0.20.1).
|
|
204
|
+
From the command line::
|
|
201
205
|
|
|
202
206
|
python -m forcingkit.fetchers.hfradar ncei --box -73.8 40.85 -71.8 41.3 \
|
|
203
207
|
--start 2025-01-01 --end 2026-09-01 [--grids 1km 2km] [--root DIR] [--workers 4]
|
|
204
208
|
python -m forcingkit.fetchers.hfradar ndbc --box -73.8 40.85 -71.8 41.3 --start all [--root DIR]
|
|
205
|
-
Coverage is uneven: in western Long Island Sound about 2 % of the 1 km cells in the box
|
|
206
|
-
40.85-41.3 N, 73.8-71.8 W have a solution in a covered hour, and whole weeks have none.
|
|
207
209
|
|
|
208
210
|
NOAA OFS archive locations
|
|
209
211
|
--------------------------
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.20.
|
|
7
|
+
version = "0.20.1"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -115,11 +115,16 @@ NCEI_SEARCH = f"{NCEI}/metadata/granule/geoportal/opensearch"
|
|
|
115
115
|
NCEI_ARCHIVE = f"{NCEI}/data/oceans/archive/"
|
|
116
116
|
# The metadata series holding the hourly totals (the Radial series beside them hold each site's radials).
|
|
117
117
|
NCEI_SERIES = ("IOOS-HFRadarRTVector", "NDBC-HFRadarRTVector")
|
|
118
|
-
|
|
118
|
+
# Beside the archive: {"months": {"YYYY-MM": accession}, "accessions": {accession: folder}}, so the metadata
|
|
119
|
+
# search (slow: minutes) and each folder lookup run once.
|
|
120
|
+
NCEI_INDEX = "ncei_accessions.json"
|
|
119
121
|
_NCEI_TITLE = re.compile(r"during ([A-Z][a-z]+) (\d{4}) \(NCEI Accession (\d+)\)")
|
|
122
|
+
# Hourly file names: since 2026 ``rtv-usegc-<grid>-uwls_v1r0_hfr_s<YYYYmmddHHMM>..._c<created>.nc`` (a reissue
|
|
123
|
+
# has a later ``c``); before, ``<YYYYmmddHHMM>_hfr_usegc_<grid>_rtv_uwls_NDBC.nc``.
|
|
120
124
|
_NCEI_FILE = re.compile(
|
|
121
125
|
r'href="(rtv-usegc-([126]km)-[^"/]*_s(\d{12})\d*_e\d+_c(\d+)\.nc)"'
|
|
122
126
|
)
|
|
127
|
+
_NCEI_FILE_OLD = re.compile(r'href="((\d{12})_hfr_usegc_([126]km)_rtv_[^"/]*\.nc)"')
|
|
123
128
|
NAME = re.compile(
|
|
124
129
|
r"usegc_[126]km/\d{2}\.\d{3}[NS]_\d{2}\.\d{3}[NS]_\d{3}\.\d{3}[EW]_\d{3}\.\d{3}[EW]/\d{4}/\d{2}/"
|
|
125
130
|
r"hfradar_usegc_[126]km_\d{8}\.nc"
|
|
@@ -624,9 +629,11 @@ def ncei_hours(
|
|
|
624
629
|
) -> dict[pd.Timestamp, str]:
|
|
625
630
|
"""The hourly files of one grid in an accession's folder: {hour: URL}; a reissued hour keeps the latest file."""
|
|
626
631
|
best: dict[pd.Timestamp, tuple[str, str]] = {}
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
632
|
+
text = _get(_client(session), folder).text
|
|
633
|
+
found = _NCEI_FILE.findall(text) + [
|
|
634
|
+
(name, g, start, "") for name, start, g in _NCEI_FILE_OLD.findall(text)
|
|
635
|
+
]
|
|
636
|
+
for name, g, start, created in found:
|
|
630
637
|
if g != grid:
|
|
631
638
|
continue
|
|
632
639
|
hour = pd.Timestamp(datetime.strptime(start, "%Y%m%d%H%M"))
|
|
@@ -649,14 +656,26 @@ def _ncei_hour(
|
|
|
649
656
|
|
|
650
657
|
|
|
651
658
|
def _read_index(store: Store) -> tuple[dict, Path | None]:
|
|
659
|
+
"""The index beside a directory store, and its path (an empty index and None for other stores). 0.20.0 kept
|
|
660
|
+
the accessions only, as a flat map."""
|
|
652
661
|
root = getattr(store, "root", None)
|
|
653
662
|
if root is None:
|
|
654
|
-
return {}, None
|
|
663
|
+
return {"months": {}, "accessions": {}}, None
|
|
655
664
|
path = Path(root) / NCEI_INDEX
|
|
656
665
|
try:
|
|
657
|
-
|
|
666
|
+
data = json.loads(path.read_text())
|
|
658
667
|
except (OSError, ValueError):
|
|
659
|
-
|
|
668
|
+
data = {}
|
|
669
|
+
if "accessions" not in data:
|
|
670
|
+
data = {"months": {}, "accessions": data}
|
|
671
|
+
data.setdefault("months", {})
|
|
672
|
+
return data, path
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def _write_index(path: Path | None, index: dict) -> None:
|
|
676
|
+
if path is not None:
|
|
677
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
678
|
+
path.write_text(json.dumps(index, indent=1, sort_keys=True))
|
|
660
679
|
|
|
661
680
|
|
|
662
681
|
def archive_ncei(
|
|
@@ -676,8 +695,15 @@ def archive_ncei(
|
|
|
676
695
|
store = store if store is not None else LocalStore()
|
|
677
696
|
t0, t1 = _utc_naive(start).normalize(), _utc_naive(end)
|
|
678
697
|
days = pd.date_range(t0, (t1 - pd.Timedelta(seconds=1)).normalize(), freq="D")
|
|
679
|
-
|
|
680
|
-
|
|
698
|
+
index, index_path = _read_index(store)
|
|
699
|
+
want = pd.period_range(t0, t1 - pd.Timedelta(seconds=1), freq="M")
|
|
700
|
+
if all(f"{m}" in index["months"] for m in want):
|
|
701
|
+
months = {m.to_timestamp(): index["months"][f"{m}"] for m in want}
|
|
702
|
+
else: # a month not listed yet: search (months NCEI has not published stay unlisted)
|
|
703
|
+
months = ncei_months(t0, t1, session)
|
|
704
|
+
index["months"].update({f"{m:%Y-%m}": a for m, a in months.items()})
|
|
705
|
+
_write_index(index_path, index)
|
|
706
|
+
known = index["accessions"]
|
|
681
707
|
written: dict[str, list[str]] = {g: [] for g in grids}
|
|
682
708
|
failed: list[str] = []
|
|
683
709
|
for grid in grids:
|
|
@@ -699,9 +725,7 @@ def archive_ncei(
|
|
|
699
725
|
)
|
|
700
726
|
failed.append(f"{grid} {month:%Y-%m}")
|
|
701
727
|
continue
|
|
702
|
-
|
|
703
|
-
index_path.parent.mkdir(parents=True, exist_ok=True)
|
|
704
|
-
index_path.write_text(json.dumps(known, indent=1, sort_keys=True))
|
|
728
|
+
_write_index(index_path, index)
|
|
705
729
|
for day in todo:
|
|
706
730
|
urls = [u for h, u in hours.items() if h.normalize() == day]
|
|
707
731
|
if not urls:
|
|
@@ -425,7 +425,10 @@ def test_an_ncei_day_stores_the_same_values_as_an_ndbc_day(tmp_path, ncei, remot
|
|
|
425
425
|
)
|
|
426
426
|
assert ACCESSION in x.attrs["source_dataset"] and "NCEI" in x.attrs["source_credit"]
|
|
427
427
|
assert "id_HFR" not in x.attrs
|
|
428
|
-
assert json.loads((a.root / hfradar.NCEI_INDEX).read_text()) == {
|
|
428
|
+
assert json.loads((a.root / hfradar.NCEI_INDEX).read_text()) == {
|
|
429
|
+
"months": {"2026-09": ACCESSION},
|
|
430
|
+
"accessions": {ACCESSION: FOLDER},
|
|
431
|
+
}
|
|
429
432
|
assert hfradar.archive_ncei(
|
|
430
433
|
BBOX, "2026-09-25", "2026-09-26", grids=("1km",), store=a
|
|
431
434
|
) == {"1km": []}
|
|
@@ -503,3 +506,54 @@ def test_cli_archives_from_ncei(tmp_path, ncei, capsys):
|
|
|
503
506
|
]
|
|
504
507
|
)
|
|
505
508
|
assert json.loads(capsys.readouterr().out) == {"1km": 2}
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def test_ncei_hours_read_the_older_file_names(tmp_path, ncei):
|
|
512
|
+
"""Accessions through 2025 name their files ``<YYYYmmddHHMM>_hfr_usegc_<grid>_rtv_uwls_NDBC.nc``."""
|
|
513
|
+
new = hfradar.ncei_hours(FOLDER, "1km")
|
|
514
|
+
old = []
|
|
515
|
+
for t, url in new.items():
|
|
516
|
+
name = f"{t:%Y%m%d%H%M}_hfr_usegc_1km_rtv_uwls_NDBC.nc"
|
|
517
|
+
ncei["files"][FOLDER + name] = ncei["files"][url]
|
|
518
|
+
old.append(f'<a href="{name}">')
|
|
519
|
+
ncei["listing"][:] = old + [
|
|
520
|
+
'<a href="202609250000_hfr_usegc_2km_rtv_uwls_NDBC.nc">'
|
|
521
|
+
]
|
|
522
|
+
hours = hfradar.ncei_hours(FOLDER, "1km")
|
|
523
|
+
assert list(hours) == list(new) and hours[pd.Timestamp("2026-09-25T05")].endswith(
|
|
524
|
+
"_rtv_uwls_NDBC.nc"
|
|
525
|
+
)
|
|
526
|
+
out = hfradar.archive_ncei(
|
|
527
|
+
BBOX,
|
|
528
|
+
"2026-09-25",
|
|
529
|
+
"2026-09-26",
|
|
530
|
+
grids=("1km",),
|
|
531
|
+
store=hfradar.LocalStore(tmp_path),
|
|
532
|
+
)
|
|
533
|
+
assert len(out["1km"]) == 1
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def test_a_rerun_over_listed_months_skips_the_search(tmp_path, ncei):
|
|
537
|
+
store = hfradar.LocalStore(tmp_path)
|
|
538
|
+
hfradar.archive_ncei(BBOX, "2026-09-25", "2026-09-26", grids=("1km",), store=store)
|
|
539
|
+
searches = ncei["gets"].count(hfradar.NCEI_SEARCH)
|
|
540
|
+
hfradar.archive_ncei(BBOX, "2026-09-26", "2026-09-27", grids=("1km",), store=store)
|
|
541
|
+
assert ncei["gets"].count(hfradar.NCEI_SEARCH) == searches
|
|
542
|
+
assert store.has(hfradar.day_name("1km", BBOX, pd.Timestamp("2026-09-26")))
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def test_the_0_20_0_index_is_read(tmp_path, ncei):
|
|
546
|
+
(tmp_path / hfradar.NCEI_INDEX).write_text(json.dumps({ACCESSION: FOLDER}))
|
|
547
|
+
hfradar.archive_ncei(
|
|
548
|
+
BBOX,
|
|
549
|
+
"2026-09-25",
|
|
550
|
+
"2026-09-26",
|
|
551
|
+
grids=("1km",),
|
|
552
|
+
store=hfradar.LocalStore(tmp_path),
|
|
553
|
+
)
|
|
554
|
+
assert (
|
|
555
|
+
f"{hfradar.NCEI_ARCHIVE}arc0002/" not in ncei["gets"]
|
|
556
|
+
) # the folder came from the old index
|
|
557
|
+
assert json.loads((tmp_path / hfradar.NCEI_INDEX).read_text())["accessions"] == {
|
|
558
|
+
ACCESSION: FOLDER
|
|
559
|
+
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|