forcingkit 0.22.1__tar.gz → 0.22.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.22.1 → forcingkit-0.22.2}/PKG-INFO +1 -1
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/cache_viewer.rst +3 -2
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/conf.py +1 -1
- {forcingkit-0.22.1 → forcingkit-0.22.2}/pyproject.toml +1 -1
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/routers/viewer.py +44 -8
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/app.js +34 -13
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_cache_inventory.py +35 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/uv.lock +1 -1
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.agents/forcingkit.md +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.env.template +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.gitignore +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.markdownlint.yaml +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/.python-version +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/AGENTS.md +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/CONTRIBUTING.md +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/Dockerfile +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/LICENSE +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/README.md +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/LICENSE +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/README.md +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/pyproject.toml +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/tests/conftest.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/tests/test_client.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/tests/test_schema.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/client/uv.lock +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docker-compose.yml +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/Makefile +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/make.bat +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/client.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/doppio.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/fetchers.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/hycom.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/index.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/necofs.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/rivers.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/shelf_models.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/sources.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/docs/source/waves.rst +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/main.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/__init__.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/routers/view.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/forcingkit_serve/view_store.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/service/run_server.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/doppio.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/hfradar.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrdps.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/ndbc.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/render.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/cells_map.js +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/coverage.js +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/favicon.ico +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/favicon.svg +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/index.html +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/logo.svg +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/preview.js +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/scene3d.js +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/styles.css +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/static/view_api.js +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/__init__.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/data/doppio_reanalysis_20220811_corner.nc +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_doppio.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_hfradar.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_hrdps.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_main.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_view_store.py +0 -0
- {forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.22.
|
|
3
|
+
Version: 0.22.2
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -3,8 +3,9 @@ Cache viewer
|
|
|
3
3
|
|
|
4
4
|
The service serves a page at ``/ui/`` that lists every store in ``FORCINGKIT_CACHE_DIR``: the
|
|
5
5
|
Zarr stores it built and the raw GRIB and NetCDF files it fetched. The page polls
|
|
6
|
-
``/api/v1/cache/inventory`` every
|
|
7
|
-
running.
|
|
6
|
+
``/api/v1/cache/inventory`` every five seconds while it is visible, so new stores appear while a
|
|
7
|
+
request is still running. The response carries an ETag, and a poll that sends it back in
|
|
8
|
+
``If-None-Match`` gets 304 with no body when nothing changed.
|
|
8
9
|
|
|
9
10
|
Coverage
|
|
10
11
|
--------
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.22.
|
|
7
|
+
version = "0.22.2"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -1,5 +1,9 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
1
4
|
from pathlib import Path
|
|
2
|
-
from fastapi import APIRouter, HTTPException, Response as FastAPIResponse
|
|
5
|
+
from fastapi import APIRouter, HTTPException, Request, Response as FastAPIResponse
|
|
6
|
+
from fastapi.encoders import jsonable_encoder
|
|
3
7
|
from pydantic import BaseModel
|
|
4
8
|
from typing import List
|
|
5
9
|
import logging
|
|
@@ -386,12 +390,43 @@ def _zarr_key(path: Path) -> tuple:
|
|
|
386
390
|
|
|
387
391
|
|
|
388
392
|
@router.get("/inventory", response_model=List[CacheItem])
|
|
389
|
-
async def get_cache_inventory(
|
|
390
|
-
"""Crawls the local data cache and returns an inventory of all processed forcing datasets.
|
|
393
|
+
async def get_cache_inventory(request: Request):
|
|
394
|
+
"""Crawls the local data cache and returns an inventory of all processed forcing datasets.
|
|
395
|
+
|
|
396
|
+
The response carries an ETag; a request whose ``If-None-Match`` matches gets 304 and no
|
|
397
|
+
body, so a page polling for changes downloads the list only when it changed."""
|
|
391
398
|
from starlette.concurrency import run_in_threadpool
|
|
392
399
|
|
|
393
|
-
|
|
394
|
-
|
|
400
|
+
items = await run_in_threadpool(_crawl_inventory)
|
|
401
|
+
body = json.dumps(
|
|
402
|
+
jsonable_encoder(items), separators=(",", ":"), ensure_ascii=False
|
|
403
|
+
).encode()
|
|
404
|
+
etag = '"' + hashlib.sha256(body).hexdigest()[:32] + '"'
|
|
405
|
+
headers = {"Cache-Control": "no-cache", "ETag": etag}
|
|
406
|
+
if request.headers.get("if-none-match") == etag:
|
|
407
|
+
return FastAPIResponse(status_code=304, headers=headers)
|
|
408
|
+
return FastAPIResponse(body, media_type="application/json", headers=headers)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _cache_entries(cache_dir: Path) -> tuple[list[Path], dict[str, list[Path]]]:
|
|
412
|
+
"""The cache's Zarr store directories and its raw donor files (by suffix), in one walk
|
|
413
|
+
that does not descend into a store: a store holds thousands of chunk files, and listing
|
|
414
|
+
them on every poll was most of the crawl's time."""
|
|
415
|
+
zarr_dirs: list[Path] = []
|
|
416
|
+
files: dict[str, list[Path]] = {".grib": [], ".grib2": [], ".nc": []}
|
|
417
|
+
for root, dirs, names in os.walk(cache_dir):
|
|
418
|
+
keep = []
|
|
419
|
+
for d in sorted(dirs):
|
|
420
|
+
if d.endswith(".zarr"):
|
|
421
|
+
zarr_dirs.append(Path(root) / d)
|
|
422
|
+
else:
|
|
423
|
+
keep.append(d)
|
|
424
|
+
dirs[:] = keep
|
|
425
|
+
for n in sorted(names):
|
|
426
|
+
suffix = os.path.splitext(n)[1]
|
|
427
|
+
if suffix in files:
|
|
428
|
+
files[suffix].append(Path(root) / n)
|
|
429
|
+
return zarr_dirs, files
|
|
395
430
|
|
|
396
431
|
|
|
397
432
|
def _crawl_inventory() -> list[CacheItem]:
|
|
@@ -400,10 +435,11 @@ def _crawl_inventory() -> list[CacheItem]:
|
|
|
400
435
|
return []
|
|
401
436
|
|
|
402
437
|
inventory = []
|
|
438
|
+
zarr_dirs, donor_files = _cache_entries(cache_dir)
|
|
403
439
|
|
|
404
440
|
# 1. Find Zarr datasets (directories)
|
|
405
441
|
seen: set[str] = set()
|
|
406
|
-
for zarr_path in
|
|
442
|
+
for zarr_path in zarr_dirs:
|
|
407
443
|
if zarr_path.is_dir():
|
|
408
444
|
seen.add(str(zarr_path))
|
|
409
445
|
try:
|
|
@@ -431,8 +467,8 @@ def _crawl_inventory() -> list[CacheItem]:
|
|
|
431
467
|
_INVENTORY_CACHE[str(zarr_path)] = (ckey, inventory[-1])
|
|
432
468
|
|
|
433
469
|
# 2. Find GRIB/NC datasets (files)
|
|
434
|
-
for
|
|
435
|
-
for file_path in
|
|
470
|
+
for suffix in (".grib", ".grib2", ".nc"):
|
|
471
|
+
for file_path in donor_files.get(suffix, []):
|
|
436
472
|
if file_path.is_file():
|
|
437
473
|
seen.add(str(file_path))
|
|
438
474
|
st = file_path.stat()
|
|
@@ -89,8 +89,13 @@ document.addEventListener("DOMContentLoaded", () => {
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
// - Initial load, deep link and polling -
|
|
92
|
+
// The list is polled for new stores: every POLL_MS after the previous poll returns, not while
|
|
93
|
+
// the tab is hidden, and with the last ETag, so an unchanged list costs no download.
|
|
94
|
+
const POLL_MS = 5000;
|
|
92
95
|
let inventoryFingerprint = "";
|
|
96
|
+
let inventoryEtag = null;
|
|
93
97
|
let pollPaused = false;
|
|
98
|
+
let pollTimer = null;
|
|
94
99
|
const fingerprint = (items) => items.map(i => `${i.id}:${i.modified_time}:${i.size_mb}`).join("|");
|
|
95
100
|
|
|
96
101
|
fetchInventory().then(() => {
|
|
@@ -98,23 +103,38 @@ document.addEventListener("DOMContentLoaded", () => {
|
|
|
98
103
|
const { id, want } = window.fkPreview.parseHash(window.location.hash);
|
|
99
104
|
if (id && currentInventory.some(item => item.id === id)) selectById(id, want);
|
|
100
105
|
}
|
|
101
|
-
|
|
106
|
+
schedulePoll();
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
function schedulePoll(ms = POLL_MS) {
|
|
110
|
+
clearTimeout(pollTimer);
|
|
111
|
+
pollTimer = setTimeout(pollInventory, ms);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
document.addEventListener("visibilitychange", () => {
|
|
115
|
+
if (!document.hidden) schedulePoll(0);
|
|
102
116
|
});
|
|
103
117
|
|
|
104
118
|
async function pollInventory() {
|
|
105
|
-
if (
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
119
|
+
if (document.hidden) return; // resumed by visibilitychange
|
|
120
|
+
if (!pollPaused) {
|
|
121
|
+
try {
|
|
122
|
+
const headers = inventoryEtag ? { "If-None-Match": inventoryEtag } : {};
|
|
123
|
+
const resp = await fetch("/api/v1/cache/inventory", { cache: "no-store", headers });
|
|
124
|
+
if (resp.ok) {
|
|
125
|
+
inventoryEtag = resp.headers.get("ETag");
|
|
126
|
+
const items = await resp.json();
|
|
127
|
+
const fp = fingerprint(items);
|
|
128
|
+
if (fp !== inventoryFingerprint) {
|
|
129
|
+
inventoryFingerprint = fp;
|
|
130
|
+
currentInventory = items;
|
|
131
|
+
renderInventory(items);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
// 304: unchanged. Other errors: a missed poll is retried at the next one.
|
|
135
|
+
} catch (_) { /* retried at the next poll */ }
|
|
117
136
|
}
|
|
137
|
+
schedulePoll();
|
|
118
138
|
}
|
|
119
139
|
|
|
120
140
|
if (purgeBtn) {
|
|
@@ -189,6 +209,7 @@ document.addEventListener("DOMContentLoaded", () => {
|
|
|
189
209
|
try {
|
|
190
210
|
const resp = await fetch("/api/v1/cache/inventory", { cache: "no-store" });
|
|
191
211
|
if (!resp.ok) throw new Error(`HTTP Error: ${resp.status}`);
|
|
212
|
+
inventoryEtag = resp.headers.get("ETag");
|
|
192
213
|
currentInventory = await resp.json();
|
|
193
214
|
inventoryFingerprint = fingerprint(currentInventory);
|
|
194
215
|
renderInventory(currentInventory);
|
|
@@ -88,3 +88,38 @@ async def test_inventory_route_returns_the_structured_fields(tmp_path, monkeypat
|
|
|
88
88
|
assert item["bbox"] == [-69.9, 39.7, -69.75, 39.82]
|
|
89
89
|
assert item["time_end"] == "2022-08-11T02:00:00Z"
|
|
90
90
|
assert (item["family"], item["kind"]) == ("Doppio", "ocean volume")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@pytest.mark.asyncio
|
|
94
|
+
async def test_inventory_etag_answers_304_until_the_cache_changes(
|
|
95
|
+
tmp_path, monkeypatch
|
|
96
|
+
):
|
|
97
|
+
"""A poll that sends back the ETag downloads nothing while the list is unchanged, and the
|
|
98
|
+
full list again once a store appears (here in a subfolder)."""
|
|
99
|
+
monkeypatch.setenv("FORCINGKIT_CACHE_DIR", str(tmp_path))
|
|
100
|
+
|
|
101
|
+
def store(path: Path) -> None:
|
|
102
|
+
xr.Dataset({"temp": (("time",), np.zeros(3))}, coords={"time": TIMES}).to_zarr(
|
|
103
|
+
path
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
store(tmp_path / "obc_000000000001.zarr")
|
|
107
|
+
from forcingkit_serve.main import app
|
|
108
|
+
|
|
109
|
+
async with AsyncClient(
|
|
110
|
+
transport=ASGITransport(app=app), base_url="http://test"
|
|
111
|
+
) as ac:
|
|
112
|
+
first = await ac.get("/api/v1/cache/inventory")
|
|
113
|
+
etag = first.headers["etag"]
|
|
114
|
+
assert [i["id"] for i in first.json()] == ["obc_000000000001"]
|
|
115
|
+
same = await ac.get("/api/v1/cache/inventory", headers={"If-None-Match": etag})
|
|
116
|
+
assert same.status_code == 304 and same.content == b""
|
|
117
|
+
store(tmp_path / "sub" / "obc_000000000002.zarr")
|
|
118
|
+
changed = await ac.get(
|
|
119
|
+
"/api/v1/cache/inventory", headers={"If-None-Match": etag}
|
|
120
|
+
)
|
|
121
|
+
assert changed.status_code == 200 and changed.headers["etag"] != etag
|
|
122
|
+
assert sorted(i["id"] for i in changed.json()) == [
|
|
123
|
+
"obc_000000000001",
|
|
124
|
+
"obc_000000000002",
|
|
125
|
+
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{forcingkit-0.22.1 → forcingkit-0.22.2}/tests/unit/data/doppio_reanalysis_20220811_corner.nc
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|