forcingkit 0.22.0__tar.gz → 0.22.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. {forcingkit-0.22.0 → forcingkit-0.22.2}/PKG-INFO +1 -1
  2. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/cache_viewer.rst +3 -2
  3. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/conf.py +1 -1
  4. {forcingkit-0.22.0 → forcingkit-0.22.2}/pyproject.toml +1 -1
  5. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/view.py +48 -7
  6. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/viewer.py +44 -8
  7. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/app.js +34 -13
  8. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/preview.js +20 -6
  9. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/scene3d.js +16 -12
  10. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_cache_inventory.py +35 -0
  11. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_view_store.py +39 -0
  12. {forcingkit-0.22.0 → forcingkit-0.22.2}/uv.lock +1 -1
  13. {forcingkit-0.22.0 → forcingkit-0.22.2}/.agents/forcingkit.md +0 -0
  14. {forcingkit-0.22.0 → forcingkit-0.22.2}/.claude/CLAUDE.md +0 -0
  15. {forcingkit-0.22.0 → forcingkit-0.22.2}/.env.template +0 -0
  16. {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/copilot-instructions.md +0 -0
  17. {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/workflows/docs.yml +0 -0
  18. {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/workflows/publish-client.yml +0 -0
  19. {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/workflows/publish.yml +0 -0
  20. {forcingkit-0.22.0 → forcingkit-0.22.2}/.github/workflows/tests.yml +0 -0
  21. {forcingkit-0.22.0 → forcingkit-0.22.2}/.gitignore +0 -0
  22. {forcingkit-0.22.0 → forcingkit-0.22.2}/.markdownlint.yaml +0 -0
  23. {forcingkit-0.22.0 → forcingkit-0.22.2}/.pre-commit-config.yaml +0 -0
  24. {forcingkit-0.22.0 → forcingkit-0.22.2}/.python-version +0 -0
  25. {forcingkit-0.22.0 → forcingkit-0.22.2}/AGENTS.md +0 -0
  26. {forcingkit-0.22.0 → forcingkit-0.22.2}/CONTRIBUTING.md +0 -0
  27. {forcingkit-0.22.0 → forcingkit-0.22.2}/Dockerfile +0 -0
  28. {forcingkit-0.22.0 → forcingkit-0.22.2}/LICENSE +0 -0
  29. {forcingkit-0.22.0 → forcingkit-0.22.2}/README.md +0 -0
  30. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/LICENSE +0 -0
  31. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/README.md +0 -0
  32. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/pyproject.toml +0 -0
  33. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/__init__.py +0 -0
  34. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/client.py +0 -0
  35. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/schema.py +0 -0
  36. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/store.py +0 -0
  37. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/src/forcingkit_client/trajectory.py +0 -0
  38. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/tests/conftest.py +0 -0
  39. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/tests/test_client.py +0 -0
  40. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/tests/test_schema.py +0 -0
  41. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/tests/test_trajectory.py +0 -0
  42. {forcingkit-0.22.0 → forcingkit-0.22.2}/client/uv.lock +0 -0
  43. {forcingkit-0.22.0 → forcingkit-0.22.2}/docker-compose.yml +0 -0
  44. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/Makefile +0 -0
  45. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/make.bat +0 -0
  46. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/requirements-docs.txt +0 -0
  47. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/_extra/CNAME +0 -0
  48. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  49. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/atmospheric_forcing.rst +0 -0
  50. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/client.rst +0 -0
  51. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/doppio.rst +0 -0
  52. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/fetchers.rst +0 -0
  53. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/hycom.rst +0 -0
  54. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/index.rst +0 -0
  55. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/layered_mesh.rst +0 -0
  56. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/mesh_frames.rst +0 -0
  57. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/necofs.rst +0 -0
  58. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/nyofs.rst +0 -0
  59. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/rivers.rst +0 -0
  60. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/roadmap.rst +0 -0
  61. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/shelf_models.rst +0 -0
  62. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/sources.rst +0 -0
  63. {forcingkit-0.22.0 → forcingkit-0.22.2}/docs/source/waves.rst +0 -0
  64. {forcingkit-0.22.0 → forcingkit-0.22.2}/main.py +0 -0
  65. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/__init__.py +0 -0
  66. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/__init__.py +0 -0
  67. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/jobs.py +0 -0
  68. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/main.py +0 -0
  69. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  70. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/removed.py +0 -0
  71. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/routers/stores.py +0 -0
  72. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/stac.py +0 -0
  73. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/forcingkit_serve/view_store.py +0 -0
  74. {forcingkit-0.22.0 → forcingkit-0.22.2}/service/run_server.py +0 -0
  75. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/__init__.py +0 -0
  76. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  77. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/dispatcher.py +0 -0
  78. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/coastwatch.py +0 -0
  79. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/dbofs.py +0 -0
  80. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/doppio.py +0 -0
  81. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/erddap.py +0 -0
  82. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hfradar.py +0 -0
  83. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrdps.py +0 -0
  84. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr.py +0 -0
  85. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  86. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  87. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  88. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hycom.py +0 -0
  89. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/hydrography.py +0 -0
  90. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/ndbc.py +0 -0
  91. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/necofs.py +0 -0
  92. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/noaa.py +0 -0
  93. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/nwps.py +0 -0
  94. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/nyofs.py +0 -0
  95. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/usgs.py +0 -0
  96. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  97. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/layered_mesh.py +0 -0
  98. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/mesh_frames.py +0 -0
  99. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis.csv +0 -0
  100. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  101. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  102. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  103. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/render.py +0 -0
  104. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/settings.py +0 -0
  105. {forcingkit-0.22.0 → forcingkit-0.22.2}/src/forcingkit/zarr_stream.py +0 -0
  106. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/cells_map.js +0 -0
  107. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/coverage.js +0 -0
  108. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/favicon.ico +0 -0
  109. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/favicon.svg +0 -0
  110. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/index.html +0 -0
  111. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/logo.svg +0 -0
  112. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/styles.css +0 -0
  113. {forcingkit-0.22.0 → forcingkit-0.22.2}/static/view_api.js +0 -0
  114. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/__init__.py +0 -0
  115. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_auth.py +0 -0
  116. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_erddap_fetch.py +0 -0
  117. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_mesh_frames_necofs.py +0 -0
  118. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_necofs_volume_live.py +0 -0
  119. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_nwps_nomads.py +0 -0
  120. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  121. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_obc_offshore_nj.py +0 -0
  122. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_ofs_archive_paths.py +0 -0
  123. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/integration/test_s3_roms_fetchers.py +0 -0
  124. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/scripts/inspect_dem.py +0 -0
  125. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/scripts/inspect_grib.py +0 -0
  126. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/scripts/inspect_zarr.py +0 -0
  127. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/data/doppio_reanalysis_20220811_corner.nc +0 -0
  128. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_coastwatch.py +0 -0
  129. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_dbofs.py +0 -0
  130. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_dispatcher.py +0 -0
  131. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_doppio.py +0 -0
  132. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_erddap.py +0 -0
  133. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hfradar.py +0 -0
  134. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrdps.py +0 -0
  135. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrrr_atm.py +0 -0
  136. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrrr_idx.py +0 -0
  137. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrrr_levels.py +0 -0
  138. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hrrr_points.py +0 -0
  139. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_hycom.py +0 -0
  140. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_jobs.py +0 -0
  141. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_layered_mesh.py +0 -0
  142. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_main.py +0 -0
  143. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_mesh_frames.py +0 -0
  144. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_ndbc.py +0 -0
  145. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_necofs_hindcast.py +0 -0
  146. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_necofs_mesh_cache.py +0 -0
  147. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_necofs_parent.py +0 -0
  148. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_necofs_volume.py +0 -0
  149. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_noaa_currents.py +0 -0
  150. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_noaa_datum.py +0 -0
  151. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_nwps.py +0 -0
  152. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_nyofs.py +0 -0
  153. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_obc_donor_report.py +0 -0
  154. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_obc_pipeline.py +0 -0
  155. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_removed_routes.py +0 -0
  156. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_settings.py +0 -0
  157. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_usgs.py +0 -0
  158. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_usgs_rivers.py +0 -0
  159. {forcingkit-0.22.0 → forcingkit-0.22.2}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.22.0
3
+ Version: 0.22.2
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -3,8 +3,9 @@ Cache viewer
3
3
 
4
4
  The service serves a page at ``/ui/`` that lists every store in ``FORCINGKIT_CACHE_DIR``: the
5
5
  Zarr stores it built and the raw GRIB and NetCDF files it fetched. The page polls
6
- ``/api/v1/cache/inventory`` every two seconds, so new stores appear while a request is still
7
- running.
6
+ ``/api/v1/cache/inventory`` every five seconds while it is visible, so new stores appear while a
7
+ request is still running. The response carries an ETag, and a poll that sends it back in
8
+ ``If-None-Match`` gets 304 with no body when nothing changed.
8
9
 
9
10
  Coverage
10
11
  --------
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.22.0"
12
+ release = "0.22.2"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.22.0"
7
+ version = "0.22.2"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -8,6 +8,9 @@ little-endian.
8
8
  from __future__ import annotations
9
9
 
10
10
  import logging
11
+ import threading
12
+ import time
13
+ from pathlib import Path
11
14
  from typing import Any, Callable
12
15
 
13
16
  import numpy as np
@@ -15,6 +18,7 @@ from fastapi import APIRouter, HTTPException, Query
15
18
  from fastapi.responses import JSONResponse, Response
16
19
  from starlette.concurrency import run_in_threadpool
17
20
 
21
+ from forcingkit import settings
18
22
  from forcingkit_serve import view_store
19
23
  from forcingkit_serve.routers.viewer import _crawl_inventory
20
24
 
@@ -26,18 +30,55 @@ Detail = Query("2d", pattern="^(2d|3d)$", description="Cell budget: 2d or 3d")
26
30
  BINARY = "application/octet-stream"
27
31
 
28
32
 
33
+ # Crawling the whole cache takes about 0.4 s on a cache of a few hundred stores, far more than
34
+ # drawing a cached record, so the store ids the crawl found are reused for a few seconds.
35
+ # Only ids a crawl listed resolve, and a store whose path is gone is looked up again.
36
+ RESOLVE_TTL_S = 10.0
37
+ _resolved: dict[str, Any] = {"dir": None, "at": 0.0, "stores": {}}
38
+ _resolve_lock = threading.Lock()
39
+
40
+
41
+ def _fresh(cache_dir: str) -> bool:
42
+ return (
43
+ _resolved["dir"] == cache_dir
44
+ and time.monotonic() - _resolved["at"] < RESOLVE_TTL_S
45
+ )
46
+
47
+
48
+ def _locate(dataset_id: str) -> tuple[Path, Any] | None:
49
+ """The store's path and change key (from the inventory), or None."""
50
+ cache_dir = str(settings.cache_dir())
51
+ if _fresh(cache_dir):
52
+ hit = _resolved["stores"].get(dataset_id)
53
+ if hit is not None and hit[0].exists():
54
+ return hit
55
+ with _resolve_lock:
56
+ # Another request may have crawled while this one waited.
57
+ hit = _resolved["stores"].get(dataset_id) if _fresh(cache_dir) else None
58
+ if hit is None or not hit[0].exists():
59
+ items = _crawl_inventory()
60
+ _resolved["stores"] = {
61
+ i.id: (Path(i.path), (i.modified_time, i.size_mb)) for i in items
62
+ }
63
+ _resolved["dir"], _resolved["at"] = cache_dir, time.monotonic()
64
+ return _resolved["stores"].get(dataset_id)
65
+
66
+
67
+ def forget_stores() -> None:
68
+ """Drop the remembered store ids, so the next request crawls the cache."""
69
+ with _resolve_lock:
70
+ _resolved.update(dir=None, at=0.0, stores={})
71
+
72
+
29
73
  def _view(dataset_id: str, detail: str) -> view_store.StoreView:
30
- items = _crawl_inventory()
31
- path = view_store.resolve_store(dataset_id, items)
32
- if path is None:
74
+ found = _locate(dataset_id)
75
+ if found is None:
33
76
  raise HTTPException(
34
77
  status_code=404, detail=f"No store {dataset_id} in the cache."
35
78
  )
36
- item = next(i for i in items if i.id == dataset_id)
79
+ path, key = found
37
80
  try:
38
- return view_store.get_view(
39
- dataset_id, path, detail, (item.modified_time, item.size_mb)
40
- )
81
+ return view_store.get_view(dataset_id, path, detail, key)
41
82
  except ValueError as e:
42
83
  raise HTTPException(status_code=422, detail=str(e))
43
84
 
@@ -1,5 +1,9 @@
1
+ import hashlib
2
+ import json
3
+ import os
1
4
  from pathlib import Path
2
- from fastapi import APIRouter, HTTPException, Response as FastAPIResponse
5
+ from fastapi import APIRouter, HTTPException, Request, Response as FastAPIResponse
6
+ from fastapi.encoders import jsonable_encoder
3
7
  from pydantic import BaseModel
4
8
  from typing import List
5
9
  import logging
@@ -386,12 +390,43 @@ def _zarr_key(path: Path) -> tuple:
386
390
 
387
391
 
388
392
  @router.get("/inventory", response_model=List[CacheItem])
389
- async def get_cache_inventory(response: FastAPIResponse):
390
- """Crawls the local data cache and returns an inventory of all processed forcing datasets."""
393
+ async def get_cache_inventory(request: Request):
394
+ """Crawls the local data cache and returns an inventory of all processed forcing datasets.
395
+
396
+ The response carries an ETag; a request whose ``If-None-Match`` matches gets 304 and no
397
+ body, so a page polling for changes downloads the list only when it changed."""
391
398
  from starlette.concurrency import run_in_threadpool
392
399
 
393
- response.headers["Cache-Control"] = "no-store"
394
- return await run_in_threadpool(_crawl_inventory)
400
+ items = await run_in_threadpool(_crawl_inventory)
401
+ body = json.dumps(
402
+ jsonable_encoder(items), separators=(",", ":"), ensure_ascii=False
403
+ ).encode()
404
+ etag = '"' + hashlib.sha256(body).hexdigest()[:32] + '"'
405
+ headers = {"Cache-Control": "no-cache", "ETag": etag}
406
+ if request.headers.get("if-none-match") == etag:
407
+ return FastAPIResponse(status_code=304, headers=headers)
408
+ return FastAPIResponse(body, media_type="application/json", headers=headers)
409
+
410
+
411
+ def _cache_entries(cache_dir: Path) -> tuple[list[Path], dict[str, list[Path]]]:
412
+ """The cache's Zarr store directories and its raw donor files (by suffix), in one walk
413
+ that does not descend into a store: a store holds thousands of chunk files, and listing
414
+ them on every poll was most of the crawl's time."""
415
+ zarr_dirs: list[Path] = []
416
+ files: dict[str, list[Path]] = {".grib": [], ".grib2": [], ".nc": []}
417
+ for root, dirs, names in os.walk(cache_dir):
418
+ keep = []
419
+ for d in sorted(dirs):
420
+ if d.endswith(".zarr"):
421
+ zarr_dirs.append(Path(root) / d)
422
+ else:
423
+ keep.append(d)
424
+ dirs[:] = keep
425
+ for n in sorted(names):
426
+ suffix = os.path.splitext(n)[1]
427
+ if suffix in files:
428
+ files[suffix].append(Path(root) / n)
429
+ return zarr_dirs, files
395
430
 
396
431
 
397
432
  def _crawl_inventory() -> list[CacheItem]:
@@ -400,10 +435,11 @@ def _crawl_inventory() -> list[CacheItem]:
400
435
  return []
401
436
 
402
437
  inventory = []
438
+ zarr_dirs, donor_files = _cache_entries(cache_dir)
403
439
 
404
440
  # 1. Find Zarr datasets (directories)
405
441
  seen: set[str] = set()
406
- for zarr_path in cache_dir.rglob("*.zarr"):
442
+ for zarr_path in zarr_dirs:
407
443
  if zarr_path.is_dir():
408
444
  seen.add(str(zarr_path))
409
445
  try:
@@ -431,8 +467,8 @@ def _crawl_inventory() -> list[CacheItem]:
431
467
  _INVENTORY_CACHE[str(zarr_path)] = (ckey, inventory[-1])
432
468
 
433
469
  # 2. Find GRIB/NC datasets (files)
434
- for ext_glob in ["*.grib", "*.grib2", "*.nc"]:
435
- for file_path in cache_dir.rglob(ext_glob):
470
+ for suffix in (".grib", ".grib2", ".nc"):
471
+ for file_path in donor_files.get(suffix, []):
436
472
  if file_path.is_file():
437
473
  seen.add(str(file_path))
438
474
  st = file_path.stat()
@@ -89,8 +89,13 @@ document.addEventListener("DOMContentLoaded", () => {
89
89
  }
90
90
 
91
91
  // - Initial load, deep link and polling -
92
+ // The list is polled for new stores: every POLL_MS after the previous poll returns, not while
93
+ // the tab is hidden, and with the last ETag, so an unchanged list costs no download.
94
+ const POLL_MS = 5000;
92
95
  let inventoryFingerprint = "";
96
+ let inventoryEtag = null;
93
97
  let pollPaused = false;
98
+ let pollTimer = null;
94
99
  const fingerprint = (items) => items.map(i => `${i.id}:${i.modified_time}:${i.size_mb}`).join("|");
95
100
 
96
101
  fetchInventory().then(() => {
@@ -98,23 +103,38 @@ document.addEventListener("DOMContentLoaded", () => {
98
103
  const { id, want } = window.fkPreview.parseHash(window.location.hash);
99
104
  if (id && currentInventory.some(item => item.id === id)) selectById(id, want);
100
105
  }
101
- setInterval(pollInventory, 2000);
106
+ schedulePoll();
107
+ });
108
+
109
+ function schedulePoll(ms = POLL_MS) {
110
+ clearTimeout(pollTimer);
111
+ pollTimer = setTimeout(pollInventory, ms);
112
+ }
113
+
114
+ document.addEventListener("visibilitychange", () => {
115
+ if (!document.hidden) schedulePoll(0);
102
116
  });
103
117
 
104
118
  async function pollInventory() {
105
- if (pollPaused) return;
106
- try {
107
- const resp = await fetch("/api/v1/cache/inventory", { cache: "no-store" });
108
- if (!resp.ok) return;
109
- const items = await resp.json();
110
- const fp = fingerprint(items);
111
- if (fp === inventoryFingerprint) return;
112
- inventoryFingerprint = fp;
113
- currentInventory = items;
114
- renderInventory(items);
115
- } catch (_) {
116
- // A missed poll is retried two seconds later.
119
+ if (document.hidden) return; // resumed by visibilitychange
120
+ if (!pollPaused) {
121
+ try {
122
+ const headers = inventoryEtag ? { "If-None-Match": inventoryEtag } : {};
123
+ const resp = await fetch("/api/v1/cache/inventory", { cache: "no-store", headers });
124
+ if (resp.ok) {
125
+ inventoryEtag = resp.headers.get("ETag");
126
+ const items = await resp.json();
127
+ const fp = fingerprint(items);
128
+ if (fp !== inventoryFingerprint) {
129
+ inventoryFingerprint = fp;
130
+ currentInventory = items;
131
+ renderInventory(items);
132
+ }
133
+ }
134
+ // 304: unchanged. Other errors: a missed poll is retried at the next one.
135
+ } catch (_) { /* retried at the next poll */ }
117
136
  }
137
+ schedulePoll();
118
138
  }
119
139
 
120
140
  if (purgeBtn) {
@@ -189,6 +209,7 @@ document.addEventListener("DOMContentLoaded", () => {
189
209
  try {
190
210
  const resp = await fetch("/api/v1/cache/inventory", { cache: "no-store" });
191
211
  if (!resp.ok) throw new Error(`HTTP Error: ${resp.status}`);
212
+ inventoryEtag = resp.headers.get("ETag");
192
213
  currentInventory = await resp.json();
193
214
  inventoryFingerprint = fingerprint(currentInventory);
194
215
  renderInventory(currentInventory);
@@ -221,6 +221,19 @@ async function setMode(mode, quiet) {
221
221
 
222
222
  // ---- drawing --------------------------------------------------------------------------------
223
223
 
224
+ // Sliders fire many times a second. One draw runs at a time; changes that arrive meanwhile are
225
+ // folded into one more draw of the latest state when it finishes, so dragging never queues up
226
+ // records the pointer has already left.
227
+ let drawing = null, drawAgain = false;
228
+
229
+ function requestDraw() {
230
+ if (drawing) { drawAgain = true; return drawing; }
231
+ drawing = (async () => {
232
+ do { drawAgain = false; await draw(); } while (drawAgain);
233
+ })().finally(() => { drawing = null; });
234
+ return drawing;
235
+ }
236
+
224
237
  async function draw() {
225
238
  if (!S.m) return;
226
239
  const token = S.token;
@@ -238,8 +251,8 @@ async function draw() {
238
251
  const codes = await api.frame(S.item.id, S.field, f.layered ? S.layer : 0, S.t, "2d");
239
252
  if (token !== S.token) return;
240
253
  map.setCodes(codes);
241
- // Prefetch the next record while this one is on screen.
242
- if (nRecords() > 1) api.frame(S.item.id, S.field, f.layered ? S.layer : 0, (S.t + 1) % nRecords(), "2d").catch(() => {});
254
+ // While playing, fetch the next record while this one is on screen.
255
+ if (S.playing && nRecords() > 1) api.frame(S.item.id, S.field, f.layered ? S.layer : 0, (S.t + 1) % nRecords(), "2d").catch(() => {});
243
256
  }
244
257
  } catch (e) {
245
258
  console.error("draw:", e);
@@ -437,7 +450,7 @@ async function playLoop() {
437
450
  S.t = (S.t + 1) % nRecords();
438
451
  $("timeRange").value = S.t;
439
452
  updateTimeLabel();
440
- await draw();
453
+ await requestDraw();
441
454
  if (S.section) loadSection();
442
455
  await new Promise((r) => setTimeout(r, Math.max(0, FRAME_MS - (performance.now() - started))));
443
456
  }
@@ -477,14 +490,15 @@ function wire() {
477
490
  $("layerRange").addEventListener("input", (e) => {
478
491
  S.layer = +e.target.value;
479
492
  updateLayerControls();
480
- draw();
493
+ requestDraw();
481
494
  });
482
495
  $("timeRange").addEventListener("input", (e) => {
483
496
  S.t = +e.target.value;
484
497
  updateTimeLabel();
485
- draw();
486
- if (S.sectionPts) loadSection();
498
+ requestDraw();
487
499
  });
500
+ // A section samples every layer, so it follows the record once the slider is let go.
501
+ $("timeRange").addEventListener("change", () => { if (S.sectionPts) loadSection(); });
488
502
  $("playBtn").addEventListener("click", () => {
489
503
  if (S.playing) { setPlaying(false); return; }
490
504
  setPlaying(true);
@@ -407,13 +407,19 @@ export class Scene3D {
407
407
  if (!this.luts.has(field)) this.luts.set(field, lutTexture(api.ramp(f)));
408
408
  const lut = this.luts.get(field);
409
409
  const levels = f.layered ? this.levels : [];
410
- const [codes, levelData] = await Promise.all([
410
+ // Everything a record needs is requested at once, not one after another.
411
+ const vec = this.m.vectors[0];
412
+ const wantTracers = this.showTracers && vec;
413
+ const [codes, levelData, tracerData] = await Promise.all([
411
414
  api.frame(this.id, field, 0, t, "3d"),
412
- Promise.all(levels.map(async (k) => ({
413
- k,
414
- codes: await api.frame(this.id, field, k, t, "3d"),
415
- z: this.nominal ? null : await api.heights(this.id, k, t, "3d"),
416
- }))),
415
+ Promise.all(levels.map(async (k) => {
416
+ const [c, z] = await Promise.all([
417
+ api.frame(this.id, field, k, t, "3d"),
418
+ this.nominal ? null : api.heights(this.id, k, t, "3d"),
419
+ ]);
420
+ return { k, codes: c, z };
421
+ })),
422
+ wantTracers ? api.tracers(this.id, vec.name, t, "3d") : null,
417
423
  ]);
418
424
  if (token !== this.token) return;
419
425
  this._setCodes(this.objects.surface, codes, lut);
@@ -424,7 +430,7 @@ export class Scene3D {
424
430
  if (!keep.has(k)) { this.scene.remove(obj); dispose(obj); this.objects.levels.delete(k); }
425
431
  }
426
432
  for (const { k, codes: c, z } of levelData) this._levelMap(k, c, z, lut);
427
- await this._tracers(t, token);
433
+ this._tracers(tracerData);
428
434
  this._curtain();
429
435
  if (this._fitPending) {
430
436
  this._fitPending = false;
@@ -464,11 +470,9 @@ export class Scene3D {
464
470
  this._setCodes(obj, codes, lut);
465
471
  }
466
472
 
467
- async _tracers(t, token) {
468
- const vec = this.m.vectors[0];
469
- if (!this.showTracers || !vec) { this.tracers.object.visible = false; return; }
470
- const d = await api.tracers(this.id, vec.name, t, "3d");
471
- if (token !== this.token) return;
473
+ // Tracers from one record's velocities (null: tracers off or no vector pair).
474
+ _tracers(d) {
475
+ if (!d) { this.tracers.object.visible = false; return; }
472
476
  const tri = this.geom.tri, items = [], nl = Math.max(1, this.m.layers.length);
473
477
  d.element.forEach((e, c) => {
474
478
  const P = [0, 1, 2].map((j) => { const v = tri[3 * e + j]; return [this.mx[v], this.my[v]]; });
@@ -88,3 +88,38 @@ async def test_inventory_route_returns_the_structured_fields(tmp_path, monkeypat
88
88
  assert item["bbox"] == [-69.9, 39.7, -69.75, 39.82]
89
89
  assert item["time_end"] == "2022-08-11T02:00:00Z"
90
90
  assert (item["family"], item["kind"]) == ("Doppio", "ocean volume")
91
+
92
+
93
+ @pytest.mark.asyncio
94
+ async def test_inventory_etag_answers_304_until_the_cache_changes(
95
+ tmp_path, monkeypatch
96
+ ):
97
+ """A poll that sends back the ETag downloads nothing while the list is unchanged, and the
98
+ full list again once a store appears (here in a subfolder)."""
99
+ monkeypatch.setenv("FORCINGKIT_CACHE_DIR", str(tmp_path))
100
+
101
+ def store(path: Path) -> None:
102
+ xr.Dataset({"temp": (("time",), np.zeros(3))}, coords={"time": TIMES}).to_zarr(
103
+ path
104
+ )
105
+
106
+ store(tmp_path / "obc_000000000001.zarr")
107
+ from forcingkit_serve.main import app
108
+
109
+ async with AsyncClient(
110
+ transport=ASGITransport(app=app), base_url="http://test"
111
+ ) as ac:
112
+ first = await ac.get("/api/v1/cache/inventory")
113
+ etag = first.headers["etag"]
114
+ assert [i["id"] for i in first.json()] == ["obc_000000000001"]
115
+ same = await ac.get("/api/v1/cache/inventory", headers={"If-None-Match": etag})
116
+ assert same.status_code == 304 and same.content == b""
117
+ store(tmp_path / "sub" / "obc_000000000002.zarr")
118
+ changed = await ac.get(
119
+ "/api/v1/cache/inventory", headers={"If-None-Match": etag}
120
+ )
121
+ assert changed.status_code == 200 and changed.headers["etag"] != etag
122
+ assert sorted(i["id"] for i in changed.json()) == [
123
+ "obc_000000000001",
124
+ "obc_000000000002",
125
+ ]
@@ -240,6 +240,45 @@ async def test_api_serves_a_store_in_a_subfolder(tmp_path, monkeypatch):
240
240
  assert unknown.status_code == 404
241
241
 
242
242
 
243
+ @pytest.mark.asyncio
244
+ async def test_api_reuses_the_store_list_between_requests(tmp_path, monkeypatch):
245
+ """Drawing a record must not crawl the whole cache each time; a removed store is looked up
246
+ again and then reported missing."""
247
+ import shutil
248
+
249
+ from httpx import ASGITransport, AsyncClient
250
+
251
+ from forcingkit_serve.main import app
252
+ from forcingkit_serve.routers import view as view_api
253
+ from forcingkit_serve.routers import viewer
254
+
255
+ path = fvcom_store(tmp_path)
256
+ monkeypatch.setenv("FORCINGKIT_CACHE_DIR", str(tmp_path))
257
+ view_store.clear_views()
258
+ view_api.forget_stores()
259
+ crawls = []
260
+ real = viewer._crawl_inventory
261
+
262
+ def counting():
263
+ crawls.append(1)
264
+ return real()
265
+
266
+ monkeypatch.setattr(view_api, "_crawl_inventory", counting)
267
+ sid = path.stem
268
+ async with AsyncClient(
269
+ transport=ASGITransport(app=app), base_url="http://test"
270
+ ) as ac:
271
+ for t in range(4):
272
+ r = await ac.get(f"/api/v1/cache/view/{sid}/frame?field=temp&layer=0&t={t}")
273
+ assert r.status_code == 200
274
+ assert len(crawls) == 1
275
+ shutil.rmtree(path)
276
+ gone = await ac.get(f"/api/v1/cache/view/{sid}/manifest")
277
+ assert gone.status_code == 404
278
+ assert len(crawls) == 2
279
+ view_api.forget_stores()
280
+
281
+
243
282
  def test_curvilinear_sigma_store_with_depths_gets_true_heights(tmp_path):
244
283
  """A Doppio-shaped volume: 2-D centers as coordinates, sigma per cell surface first, the water
245
284
  depth and a land mask."""
@@ -1000,7 +1000,7 @@ wheels = [
1000
1000
 
1001
1001
  [[package]]
1002
1002
  name = "forcingkit"
1003
- version = "0.22.0"
1003
+ version = "0.22.2"
1004
1004
  source = { editable = "." }
1005
1005
  dependencies = [
1006
1006
  { name = "bottleneck" },
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes