forcingkit 0.22.0__tar.gz → 0.22.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.22.0 → forcingkit-0.22.2}/PKG-INFO +1 -1
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/cache_viewer.rst +3 -2
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/conf.py +1 -1
- {forcingkit-0.22.0 → forcingkit-0.22.2}/pyproject.toml +1 -1
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/view.py +48 -7
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/viewer.py +44 -8
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/app.js +34 -13
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/preview.js +20 -6
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/scene3d.js +16 -12
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_cache_inventory.py +35 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_view_store.py +39 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/uv.lock +1 -1
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.agents/forcingkit.md +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.env.template +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.gitignore +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.markdownlint.yaml +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/.python-version +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/AGENTS.md +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/CONTRIBUTING.md +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/Dockerfile +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/LICENSE +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/README.md +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/LICENSE +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/README.md +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/pyproject.toml +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/tests/conftest.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/tests/test_client.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/tests/test_schema.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/client/uv.lock +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docker-compose.yml +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/Makefile +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/make.bat +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/client.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/doppio.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/fetchers.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/hycom.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/index.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/necofs.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/rivers.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/shelf_models.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/sources.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/waves.rst +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/main.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/__init__.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/view_store.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/service/run_server.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/doppio.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hfradar.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrdps.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/ndbc.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/render.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/cells_map.js +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/coverage.js +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/favicon.ico +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/favicon.svg +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/index.html +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/logo.svg +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/styles.css +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/static/view_api.js +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/__init__.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/data/doppio_reanalysis_20220811_corner.nc +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_doppio.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hfradar.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrdps.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_main.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.22.
|
|
3
|
+
Version: 0.22.2
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -3,8 +3,9 @@ Cache viewer
|
|
|
3
3
|
|
|
4
4
|
The service serves a page at ``/ui/`` that lists every store in ``FORCINGKIT_CACHE_DIR``: the
|
|
5
5
|
Zarr stores it built and the raw GRIB and NetCDF files it fetched. The page polls
|
|
6
|
-
``/api/v1/cache/inventory`` every
|
|
7
|
-
running.
|
|
6
|
+
``/api/v1/cache/inventory`` every five seconds while it is visible, so new stores appear while a
|
|
7
|
+
request is still running. The response carries an ETag, and a poll that sends it back in
|
|
8
|
+
``If-None-Match`` gets 304 with no body when nothing changed.
|
|
8
9
|
|
|
9
10
|
Coverage
|
|
10
11
|
--------
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.22.
|
|
7
|
+
version = "0.22.2"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -8,6 +8,9 @@ little-endian.
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import logging
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
11
14
|
from typing import Any, Callable
|
|
12
15
|
|
|
13
16
|
import numpy as np
|
|
@@ -15,6 +18,7 @@ from fastapi import APIRouter, HTTPException, Query
|
|
|
15
18
|
from fastapi.responses import JSONResponse, Response
|
|
16
19
|
from starlette.concurrency import run_in_threadpool
|
|
17
20
|
|
|
21
|
+
from forcingkit import settings
|
|
18
22
|
from forcingkit_serve import view_store
|
|
19
23
|
from forcingkit_serve.routers.viewer import _crawl_inventory
|
|
20
24
|
|
|
@@ -26,18 +30,55 @@ Detail = Query("2d", pattern="^(2d|3d)$", description="Cell budget: 2d or 3d")
|
|
|
26
30
|
BINARY = "application/octet-stream"
|
|
27
31
|
|
|
28
32
|
|
|
33
|
+
# Crawling the whole cache takes about 0.4 s on a cache of a few hundred stores, far more than
|
|
34
|
+
# drawing a cached record, so the store ids the crawl found are reused for a few seconds.
|
|
35
|
+
# Only ids a crawl listed resolve, and a store whose path is gone is looked up again.
|
|
36
|
+
RESOLVE_TTL_S = 10.0
|
|
37
|
+
_resolved: dict[str, Any] = {"dir": None, "at": 0.0, "stores": {}}
|
|
38
|
+
_resolve_lock = threading.Lock()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _fresh(cache_dir: str) -> bool:
|
|
42
|
+
return (
|
|
43
|
+
_resolved["dir"] == cache_dir
|
|
44
|
+
and time.monotonic() - _resolved["at"] < RESOLVE_TTL_S
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _locate(dataset_id: str) -> tuple[Path, Any] | None:
|
|
49
|
+
"""The store's path and change key (from the inventory), or None."""
|
|
50
|
+
cache_dir = str(settings.cache_dir())
|
|
51
|
+
if _fresh(cache_dir):
|
|
52
|
+
hit = _resolved["stores"].get(dataset_id)
|
|
53
|
+
if hit is not None and hit[0].exists():
|
|
54
|
+
return hit
|
|
55
|
+
with _resolve_lock:
|
|
56
|
+
# Another request may have crawled while this one waited.
|
|
57
|
+
hit = _resolved["stores"].get(dataset_id) if _fresh(cache_dir) else None
|
|
58
|
+
if hit is None or not hit[0].exists():
|
|
59
|
+
items = _crawl_inventory()
|
|
60
|
+
_resolved["stores"] = {
|
|
61
|
+
i.id: (Path(i.path), (i.modified_time, i.size_mb)) for i in items
|
|
62
|
+
}
|
|
63
|
+
_resolved["dir"], _resolved["at"] = cache_dir, time.monotonic()
|
|
64
|
+
return _resolved["stores"].get(dataset_id)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def forget_stores() -> None:
|
|
68
|
+
"""Drop the remembered store ids, so the next request crawls the cache."""
|
|
69
|
+
with _resolve_lock:
|
|
70
|
+
_resolved.update(dir=None, at=0.0, stores={})
|
|
71
|
+
|
|
72
|
+
|
|
29
73
|
def _view(dataset_id: str, detail: str) -> view_store.StoreView:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
if path is None:
|
|
74
|
+
found = _locate(dataset_id)
|
|
75
|
+
if found is None:
|
|
33
76
|
raise HTTPException(
|
|
34
77
|
status_code=404, detail=f"No store {dataset_id} in the cache."
|
|
35
78
|
)
|
|
36
|
-
|
|
79
|
+
path, key = found
|
|
37
80
|
try:
|
|
38
|
-
return view_store.get_view(
|
|
39
|
-
dataset_id, path, detail, (item.modified_time, item.size_mb)
|
|
40
|
-
)
|
|
81
|
+
return view_store.get_view(dataset_id, path, detail, key)
|
|
41
82
|
except ValueError as e:
|
|
42
83
|
raise HTTPException(status_code=422, detail=str(e))
|
|
43
84
|
|
|
@@ -1,5 +1,9 @@
|
|
|
1
|
+
import hashlib
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
1
4
|
from pathlib import Path
|
|
2
|
-
from fastapi import APIRouter, HTTPException, Response as FastAPIResponse
|
|
5
|
+
from fastapi import APIRouter, HTTPException, Request, Response as FastAPIResponse
|
|
6
|
+
from fastapi.encoders import jsonable_encoder
|
|
3
7
|
from pydantic import BaseModel
|
|
4
8
|
from typing import List
|
|
5
9
|
import logging
|
|
@@ -386,12 +390,43 @@ def _zarr_key(path: Path) -> tuple:
|
|
|
386
390
|
|
|
387
391
|
|
|
388
392
|
@router.get("/inventory", response_model=List[CacheItem])
|
|
389
|
-
async def get_cache_inventory(
|
|
390
|
-
"""Crawls the local data cache and returns an inventory of all processed forcing datasets.
|
|
393
|
+
async def get_cache_inventory(request: Request):
|
|
394
|
+
"""Crawls the local data cache and returns an inventory of all processed forcing datasets.
|
|
395
|
+
|
|
396
|
+
The response carries an ETag; a request whose ``If-None-Match`` matches gets 304 and no
|
|
397
|
+
body, so a page polling for changes downloads the list only when it changed."""
|
|
391
398
|
from starlette.concurrency import run_in_threadpool
|
|
392
399
|
|
|
393
|
-
|
|
394
|
-
|
|
400
|
+
items = await run_in_threadpool(_crawl_inventory)
|
|
401
|
+
body = json.dumps(
|
|
402
|
+
jsonable_encoder(items), separators=(",", ":"), ensure_ascii=False
|
|
403
|
+
).encode()
|
|
404
|
+
etag = '"' + hashlib.sha256(body).hexdigest()[:32] + '"'
|
|
405
|
+
headers = {"Cache-Control": "no-cache", "ETag": etag}
|
|
406
|
+
if request.headers.get("if-none-match") == etag:
|
|
407
|
+
return FastAPIResponse(status_code=304, headers=headers)
|
|
408
|
+
return FastAPIResponse(body, media_type="application/json", headers=headers)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _cache_entries(cache_dir: Path) -> tuple[list[Path], dict[str, list[Path]]]:
|
|
412
|
+
"""The cache's Zarr store directories and its raw donor files (by suffix), in one walk
|
|
413
|
+
that does not descend into a store: a store holds thousands of chunk files, and listing
|
|
414
|
+
them on every poll was most of the crawl's time."""
|
|
415
|
+
zarr_dirs: list[Path] = []
|
|
416
|
+
files: dict[str, list[Path]] = {".grib": [], ".grib2": [], ".nc": []}
|
|
417
|
+
for root, dirs, names in os.walk(cache_dir):
|
|
418
|
+
keep = []
|
|
419
|
+
for d in sorted(dirs):
|
|
420
|
+
if d.endswith(".zarr"):
|
|
421
|
+
zarr_dirs.append(Path(root) / d)
|
|
422
|
+
else:
|
|
423
|
+
keep.append(d)
|
|
424
|
+
dirs[:] = keep
|
|
425
|
+
for n in sorted(names):
|
|
426
|
+
suffix = os.path.splitext(n)[1]
|
|
427
|
+
if suffix in files:
|
|
428
|
+
files[suffix].append(Path(root) / n)
|
|
429
|
+
return zarr_dirs, files
|
|
395
430
|
|
|
396
431
|
|
|
397
432
|
def _crawl_inventory() -> list[CacheItem]:
|
|
@@ -400,10 +435,11 @@ def _crawl_inventory() -> list[CacheItem]:
|
|
|
400
435
|
return []
|
|
401
436
|
|
|
402
437
|
inventory = []
|
|
438
|
+
zarr_dirs, donor_files = _cache_entries(cache_dir)
|
|
403
439
|
|
|
404
440
|
# 1. Find Zarr datasets (directories)
|
|
405
441
|
seen: set[str] = set()
|
|
406
|
-
for zarr_path in
|
|
442
|
+
for zarr_path in zarr_dirs:
|
|
407
443
|
if zarr_path.is_dir():
|
|
408
444
|
seen.add(str(zarr_path))
|
|
409
445
|
try:
|
|
@@ -431,8 +467,8 @@ def _crawl_inventory() -> list[CacheItem]:
|
|
|
431
467
|
_INVENTORY_CACHE[str(zarr_path)] = (ckey, inventory[-1])
|
|
432
468
|
|
|
433
469
|
# 2. Find GRIB/NC datasets (files)
|
|
434
|
-
for
|
|
435
|
-
for file_path in
|
|
470
|
+
for suffix in (".grib", ".grib2", ".nc"):
|
|
471
|
+
for file_path in donor_files.get(suffix, []):
|
|
436
472
|
if file_path.is_file():
|
|
437
473
|
seen.add(str(file_path))
|
|
438
474
|
st = file_path.stat()
|
|
@@ -89,8 +89,13 @@ document.addEventListener("DOMContentLoaded", () => {
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
// - Initial load, deep link and polling -
|
|
92
|
+
// The list is polled for new stores: every POLL_MS after the previous poll returns, not while
|
|
93
|
+
// the tab is hidden, and with the last ETag, so an unchanged list costs no download.
|
|
94
|
+
const POLL_MS = 5000;
|
|
92
95
|
let inventoryFingerprint = "";
|
|
96
|
+
let inventoryEtag = null;
|
|
93
97
|
let pollPaused = false;
|
|
98
|
+
let pollTimer = null;
|
|
94
99
|
const fingerprint = (items) => items.map(i => `${i.id}:${i.modified_time}:${i.size_mb}`).join("|");
|
|
95
100
|
|
|
96
101
|
fetchInventory().then(() => {
|
|
@@ -98,23 +103,38 @@ document.addEventListener("DOMContentLoaded", () => {
|
|
|
98
103
|
const { id, want } = window.fkPreview.parseHash(window.location.hash);
|
|
99
104
|
if (id && currentInventory.some(item => item.id === id)) selectById(id, want);
|
|
100
105
|
}
|
|
101
|
-
|
|
106
|
+
schedulePoll();
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
function schedulePoll(ms = POLL_MS) {
|
|
110
|
+
clearTimeout(pollTimer);
|
|
111
|
+
pollTimer = setTimeout(pollInventory, ms);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
document.addEventListener("visibilitychange", () => {
|
|
115
|
+
if (!document.hidden) schedulePoll(0);
|
|
102
116
|
});
|
|
103
117
|
|
|
104
118
|
async function pollInventory() {
|
|
105
|
-
if (
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
119
|
+
if (document.hidden) return; // resumed by visibilitychange
|
|
120
|
+
if (!pollPaused) {
|
|
121
|
+
try {
|
|
122
|
+
const headers = inventoryEtag ? { "If-None-Match": inventoryEtag } : {};
|
|
123
|
+
const resp = await fetch("/api/v1/cache/inventory", { cache: "no-store", headers });
|
|
124
|
+
if (resp.ok) {
|
|
125
|
+
inventoryEtag = resp.headers.get("ETag");
|
|
126
|
+
const items = await resp.json();
|
|
127
|
+
const fp = fingerprint(items);
|
|
128
|
+
if (fp !== inventoryFingerprint) {
|
|
129
|
+
inventoryFingerprint = fp;
|
|
130
|
+
currentInventory = items;
|
|
131
|
+
renderInventory(items);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
// 304: unchanged. Other errors: a missed poll is retried at the next one.
|
|
135
|
+
} catch (_) { /* retried at the next poll */ }
|
|
117
136
|
}
|
|
137
|
+
schedulePoll();
|
|
118
138
|
}
|
|
119
139
|
|
|
120
140
|
if (purgeBtn) {
|
|
@@ -189,6 +209,7 @@ document.addEventListener("DOMContentLoaded", () => {
|
|
|
189
209
|
try {
|
|
190
210
|
const resp = await fetch("/api/v1/cache/inventory", { cache: "no-store" });
|
|
191
211
|
if (!resp.ok) throw new Error(`HTTP Error: ${resp.status}`);
|
|
212
|
+
inventoryEtag = resp.headers.get("ETag");
|
|
192
213
|
currentInventory = await resp.json();
|
|
193
214
|
inventoryFingerprint = fingerprint(currentInventory);
|
|
194
215
|
renderInventory(currentInventory);
|
|
@@ -221,6 +221,19 @@ async function setMode(mode, quiet) {
|
|
|
221
221
|
|
|
222
222
|
// ---- drawing --------------------------------------------------------------------------------
|
|
223
223
|
|
|
224
|
+
// Sliders fire many times a second. One draw runs at a time; changes that arrive meanwhile are
|
|
225
|
+
// folded into one more draw of the latest state when it finishes, so dragging never queues up
|
|
226
|
+
// records the pointer has already left.
|
|
227
|
+
let drawing = null, drawAgain = false;
|
|
228
|
+
|
|
229
|
+
function requestDraw() {
|
|
230
|
+
if (drawing) { drawAgain = true; return drawing; }
|
|
231
|
+
drawing = (async () => {
|
|
232
|
+
do { drawAgain = false; await draw(); } while (drawAgain);
|
|
233
|
+
})().finally(() => { drawing = null; });
|
|
234
|
+
return drawing;
|
|
235
|
+
}
|
|
236
|
+
|
|
224
237
|
async function draw() {
|
|
225
238
|
if (!S.m) return;
|
|
226
239
|
const token = S.token;
|
|
@@ -238,8 +251,8 @@ async function draw() {
|
|
|
238
251
|
const codes = await api.frame(S.item.id, S.field, f.layered ? S.layer : 0, S.t, "2d");
|
|
239
252
|
if (token !== S.token) return;
|
|
240
253
|
map.setCodes(codes);
|
|
241
|
-
//
|
|
242
|
-
if (nRecords() > 1) api.frame(S.item.id, S.field, f.layered ? S.layer : 0, (S.t + 1) % nRecords(), "2d").catch(() => {});
|
|
254
|
+
// While playing, fetch the next record while this one is on screen.
|
|
255
|
+
if (S.playing && nRecords() > 1) api.frame(S.item.id, S.field, f.layered ? S.layer : 0, (S.t + 1) % nRecords(), "2d").catch(() => {});
|
|
243
256
|
}
|
|
244
257
|
} catch (e) {
|
|
245
258
|
console.error("draw:", e);
|
|
@@ -437,7 +450,7 @@ async function playLoop() {
|
|
|
437
450
|
S.t = (S.t + 1) % nRecords();
|
|
438
451
|
$("timeRange").value = S.t;
|
|
439
452
|
updateTimeLabel();
|
|
440
|
-
await
|
|
453
|
+
await requestDraw();
|
|
441
454
|
if (S.section) loadSection();
|
|
442
455
|
await new Promise((r) => setTimeout(r, Math.max(0, FRAME_MS - (performance.now() - started))));
|
|
443
456
|
}
|
|
@@ -477,14 +490,15 @@ function wire() {
|
|
|
477
490
|
$("layerRange").addEventListener("input", (e) => {
|
|
478
491
|
S.layer = +e.target.value;
|
|
479
492
|
updateLayerControls();
|
|
480
|
-
|
|
493
|
+
requestDraw();
|
|
481
494
|
});
|
|
482
495
|
$("timeRange").addEventListener("input", (e) => {
|
|
483
496
|
S.t = +e.target.value;
|
|
484
497
|
updateTimeLabel();
|
|
485
|
-
|
|
486
|
-
if (S.sectionPts) loadSection();
|
|
498
|
+
requestDraw();
|
|
487
499
|
});
|
|
500
|
+
// A section samples every layer, so it follows the record once the slider is let go.
|
|
501
|
+
$("timeRange").addEventListener("change", () => { if (S.sectionPts) loadSection(); });
|
|
488
502
|
$("playBtn").addEventListener("click", () => {
|
|
489
503
|
if (S.playing) { setPlaying(false); return; }
|
|
490
504
|
setPlaying(true);
|
|
@@ -407,13 +407,19 @@ export class Scene3D {
|
|
|
407
407
|
if (!this.luts.has(field)) this.luts.set(field, lutTexture(api.ramp(f)));
|
|
408
408
|
const lut = this.luts.get(field);
|
|
409
409
|
const levels = f.layered ? this.levels : [];
|
|
410
|
-
|
|
410
|
+
// Everything a record needs is requested at once, not one after another.
|
|
411
|
+
const vec = this.m.vectors[0];
|
|
412
|
+
const wantTracers = this.showTracers && vec;
|
|
413
|
+
const [codes, levelData, tracerData] = await Promise.all([
|
|
411
414
|
api.frame(this.id, field, 0, t, "3d"),
|
|
412
|
-
Promise.all(levels.map(async (k) =>
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
415
|
+
Promise.all(levels.map(async (k) => {
|
|
416
|
+
const [c, z] = await Promise.all([
|
|
417
|
+
api.frame(this.id, field, k, t, "3d"),
|
|
418
|
+
this.nominal ? null : api.heights(this.id, k, t, "3d"),
|
|
419
|
+
]);
|
|
420
|
+
return { k, codes: c, z };
|
|
421
|
+
})),
|
|
422
|
+
wantTracers ? api.tracers(this.id, vec.name, t, "3d") : null,
|
|
417
423
|
]);
|
|
418
424
|
if (token !== this.token) return;
|
|
419
425
|
this._setCodes(this.objects.surface, codes, lut);
|
|
@@ -424,7 +430,7 @@ export class Scene3D {
|
|
|
424
430
|
if (!keep.has(k)) { this.scene.remove(obj); dispose(obj); this.objects.levels.delete(k); }
|
|
425
431
|
}
|
|
426
432
|
for (const { k, codes: c, z } of levelData) this._levelMap(k, c, z, lut);
|
|
427
|
-
|
|
433
|
+
this._tracers(tracerData);
|
|
428
434
|
this._curtain();
|
|
429
435
|
if (this._fitPending) {
|
|
430
436
|
this._fitPending = false;
|
|
@@ -464,11 +470,9 @@ export class Scene3D {
|
|
|
464
470
|
this._setCodes(obj, codes, lut);
|
|
465
471
|
}
|
|
466
472
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
if (!
|
|
470
|
-
const d = await api.tracers(this.id, vec.name, t, "3d");
|
|
471
|
-
if (token !== this.token) return;
|
|
473
|
+
// Tracers from one record's velocities (null: tracers off or no vector pair).
|
|
474
|
+
_tracers(d) {
|
|
475
|
+
if (!d) { this.tracers.object.visible = false; return; }
|
|
472
476
|
const tri = this.geom.tri, items = [], nl = Math.max(1, this.m.layers.length);
|
|
473
477
|
d.element.forEach((e, c) => {
|
|
474
478
|
const P = [0, 1, 2].map((j) => { const v = tri[3 * e + j]; return [this.mx[v], this.my[v]]; });
|
|
@@ -88,3 +88,38 @@ async def test_inventory_route_returns_the_structured_fields(tmp_path, monkeypat
|
|
|
88
88
|
assert item["bbox"] == [-69.9, 39.7, -69.75, 39.82]
|
|
89
89
|
assert item["time_end"] == "2022-08-11T02:00:00Z"
|
|
90
90
|
assert (item["family"], item["kind"]) == ("Doppio", "ocean volume")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@pytest.mark.asyncio
|
|
94
|
+
async def test_inventory_etag_answers_304_until_the_cache_changes(
|
|
95
|
+
tmp_path, monkeypatch
|
|
96
|
+
):
|
|
97
|
+
"""A poll that sends back the ETag downloads nothing while the list is unchanged, and the
|
|
98
|
+
full list again once a store appears (here in a subfolder)."""
|
|
99
|
+
monkeypatch.setenv("FORCINGKIT_CACHE_DIR", str(tmp_path))
|
|
100
|
+
|
|
101
|
+
def store(path: Path) -> None:
|
|
102
|
+
xr.Dataset({"temp": (("time",), np.zeros(3))}, coords={"time": TIMES}).to_zarr(
|
|
103
|
+
path
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
store(tmp_path / "obc_000000000001.zarr")
|
|
107
|
+
from forcingkit_serve.main import app
|
|
108
|
+
|
|
109
|
+
async with AsyncClient(
|
|
110
|
+
transport=ASGITransport(app=app), base_url="http://test"
|
|
111
|
+
) as ac:
|
|
112
|
+
first = await ac.get("/api/v1/cache/inventory")
|
|
113
|
+
etag = first.headers["etag"]
|
|
114
|
+
assert [i["id"] for i in first.json()] == ["obc_000000000001"]
|
|
115
|
+
same = await ac.get("/api/v1/cache/inventory", headers={"If-None-Match": etag})
|
|
116
|
+
assert same.status_code == 304 and same.content == b""
|
|
117
|
+
store(tmp_path / "sub" / "obc_000000000002.zarr")
|
|
118
|
+
changed = await ac.get(
|
|
119
|
+
"/api/v1/cache/inventory", headers={"If-None-Match": etag}
|
|
120
|
+
)
|
|
121
|
+
assert changed.status_code == 200 and changed.headers["etag"] != etag
|
|
122
|
+
assert sorted(i["id"] for i in changed.json()) == [
|
|
123
|
+
"obc_000000000001",
|
|
124
|
+
"obc_000000000002",
|
|
125
|
+
]
|
|
@@ -240,6 +240,45 @@ async def test_api_serves_a_store_in_a_subfolder(tmp_path, monkeypatch):
|
|
|
240
240
|
assert unknown.status_code == 404
|
|
241
241
|
|
|
242
242
|
|
|
243
|
+
@pytest.mark.asyncio
|
|
244
|
+
async def test_api_reuses_the_store_list_between_requests(tmp_path, monkeypatch):
|
|
245
|
+
"""Drawing a record must not crawl the whole cache each time; a removed store is looked up
|
|
246
|
+
again and then reported missing."""
|
|
247
|
+
import shutil
|
|
248
|
+
|
|
249
|
+
from httpx import ASGITransport, AsyncClient
|
|
250
|
+
|
|
251
|
+
from forcingkit_serve.main import app
|
|
252
|
+
from forcingkit_serve.routers import view as view_api
|
|
253
|
+
from forcingkit_serve.routers import viewer
|
|
254
|
+
|
|
255
|
+
path = fvcom_store(tmp_path)
|
|
256
|
+
monkeypatch.setenv("FORCINGKIT_CACHE_DIR", str(tmp_path))
|
|
257
|
+
view_store.clear_views()
|
|
258
|
+
view_api.forget_stores()
|
|
259
|
+
crawls = []
|
|
260
|
+
real = viewer._crawl_inventory
|
|
261
|
+
|
|
262
|
+
def counting():
|
|
263
|
+
crawls.append(1)
|
|
264
|
+
return real()
|
|
265
|
+
|
|
266
|
+
monkeypatch.setattr(view_api, "_crawl_inventory", counting)
|
|
267
|
+
sid = path.stem
|
|
268
|
+
async with AsyncClient(
|
|
269
|
+
transport=ASGITransport(app=app), base_url="http://test"
|
|
270
|
+
) as ac:
|
|
271
|
+
for t in range(4):
|
|
272
|
+
r = await ac.get(f"/api/v1/cache/view/{sid}/frame?field=temp&layer=0&t={t}")
|
|
273
|
+
assert r.status_code == 200
|
|
274
|
+
assert len(crawls) == 1
|
|
275
|
+
shutil.rmtree(path)
|
|
276
|
+
gone = await ac.get(f"/api/v1/cache/view/{sid}/manifest")
|
|
277
|
+
assert gone.status_code == 404
|
|
278
|
+
assert len(crawls) == 2
|
|
279
|
+
view_api.forget_stores()
|
|
280
|
+
|
|
281
|
+
|
|
243
282
|
def test_curvilinear_sigma_store_with_depths_gets_true_heights(tmp_path):
|
|
244
283
|
"""A Doppio-shaped volume: 2-D centers as coordinates, sigma per cell surface first, the water
|
|
245
284
|
depth and a land mask."""
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/data/doppio_reanalysis_20220811_corner.nc
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|