forcingkit 0.14.0__tar.gz → 0.15.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {forcingkit-0.14.0 → forcingkit-0.15.1}/PKG-INFO +2 -2
  2. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/conf.py +1 -1
  3. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/fetchers.rst +3 -1
  4. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/roadmap.rst +1 -1
  5. {forcingkit-0.14.0 → forcingkit-0.15.1}/pyproject.toml +3 -3
  6. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/jobs.py +9 -0
  7. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/main.py +16 -2
  8. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/routers/viewer.py +4 -0
  9. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/erddap.py +54 -95
  10. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_erddap.py +7 -7
  11. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_jobs.py +69 -0
  12. {forcingkit-0.14.0 → forcingkit-0.15.1}/uv.lock +5 -5
  13. {forcingkit-0.14.0 → forcingkit-0.15.1}/.agents/forcingkit.md +0 -0
  14. {forcingkit-0.14.0 → forcingkit-0.15.1}/.claude/CLAUDE.md +0 -0
  15. {forcingkit-0.14.0 → forcingkit-0.15.1}/.env.template +0 -0
  16. {forcingkit-0.14.0 → forcingkit-0.15.1}/.github/copilot-instructions.md +0 -0
  17. {forcingkit-0.14.0 → forcingkit-0.15.1}/.github/workflows/docs.yml +0 -0
  18. {forcingkit-0.14.0 → forcingkit-0.15.1}/.github/workflows/publish-client.yml +0 -0
  19. {forcingkit-0.14.0 → forcingkit-0.15.1}/.github/workflows/publish.yml +0 -0
  20. {forcingkit-0.14.0 → forcingkit-0.15.1}/.github/workflows/tests.yml +0 -0
  21. {forcingkit-0.14.0 → forcingkit-0.15.1}/.gitignore +0 -0
  22. {forcingkit-0.14.0 → forcingkit-0.15.1}/.markdownlint.yaml +0 -0
  23. {forcingkit-0.14.0 → forcingkit-0.15.1}/.pre-commit-config.yaml +0 -0
  24. {forcingkit-0.14.0 → forcingkit-0.15.1}/.python-version +0 -0
  25. {forcingkit-0.14.0 → forcingkit-0.15.1}/AGENTS.md +0 -0
  26. {forcingkit-0.14.0 → forcingkit-0.15.1}/CONTRIBUTING.md +0 -0
  27. {forcingkit-0.14.0 → forcingkit-0.15.1}/Dockerfile +0 -0
  28. {forcingkit-0.14.0 → forcingkit-0.15.1}/LICENSE +0 -0
  29. {forcingkit-0.14.0 → forcingkit-0.15.1}/README.md +0 -0
  30. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/LICENSE +0 -0
  31. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/README.md +0 -0
  32. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/pyproject.toml +0 -0
  33. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/src/forcingkit_client/__init__.py +0 -0
  34. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/src/forcingkit_client/client.py +0 -0
  35. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/src/forcingkit_client/schema.py +0 -0
  36. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/src/forcingkit_client/store.py +0 -0
  37. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/src/forcingkit_client/trajectory.py +0 -0
  38. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/tests/conftest.py +0 -0
  39. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/tests/test_client.py +0 -0
  40. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/tests/test_schema.py +0 -0
  41. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/tests/test_trajectory.py +0 -0
  42. {forcingkit-0.14.0 → forcingkit-0.15.1}/client/uv.lock +0 -0
  43. {forcingkit-0.14.0 → forcingkit-0.15.1}/docker-compose.yml +0 -0
  44. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/Makefile +0 -0
  45. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/make.bat +0 -0
  46. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/requirements-docs.txt +0 -0
  47. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/_extra/CNAME +0 -0
  48. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/_static/custom.css +0 -0
  49. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  50. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/atmospheric_forcing.rst +0 -0
  51. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/client.rst +0 -0
  52. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/hycom.rst +0 -0
  53. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/index.rst +0 -0
  54. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/layered_mesh.rst +0 -0
  55. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/mesh_frames.rst +0 -0
  56. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/necofs.rst +0 -0
  57. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/nyofs.rst +0 -0
  58. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/rivers.rst +0 -0
  59. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/sources.rst +0 -0
  60. {forcingkit-0.14.0 → forcingkit-0.15.1}/docs/source/waves.rst +0 -0
  61. {forcingkit-0.14.0 → forcingkit-0.15.1}/main.py +0 -0
  62. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/__init__.py +0 -0
  63. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/__init__.py +0 -0
  64. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  65. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  66. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/routers/removed.py +0 -0
  67. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/routers/stores.py +0 -0
  68. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/forcingkit_serve/stac.py +0 -0
  69. {forcingkit-0.14.0 → forcingkit-0.15.1}/service/run_server.py +0 -0
  70. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/__init__.py +0 -0
  71. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  72. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/dispatcher.py +0 -0
  73. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/coastwatch.py +0 -0
  74. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/dbofs.py +0 -0
  75. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/hrrr.py +0 -0
  76. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  77. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  78. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  79. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/hycom.py +0 -0
  80. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/hydrography.py +0 -0
  81. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/ndbc.py +0 -0
  82. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/necofs.py +0 -0
  83. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/noaa.py +0 -0
  84. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/nwps.py +0 -0
  85. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/nyofs.py +0 -0
  86. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/usgs.py +0 -0
  87. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  88. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/layered_mesh.py +0 -0
  89. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/mesh_frames.py +0 -0
  90. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/ramps/currents_lis.csv +0 -0
  91. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  92. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  93. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  94. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/render.py +0 -0
  95. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/settings.py +0 -0
  96. {forcingkit-0.14.0 → forcingkit-0.15.1}/src/forcingkit/zarr_stream.py +0 -0
  97. {forcingkit-0.14.0 → forcingkit-0.15.1}/static/app.js +0 -0
  98. {forcingkit-0.14.0 → forcingkit-0.15.1}/static/favicon.ico +0 -0
  99. {forcingkit-0.14.0 → forcingkit-0.15.1}/static/favicon.svg +0 -0
  100. {forcingkit-0.14.0 → forcingkit-0.15.1}/static/index.html +0 -0
  101. {forcingkit-0.14.0 → forcingkit-0.15.1}/static/logo.svg +0 -0
  102. {forcingkit-0.14.0 → forcingkit-0.15.1}/static/preview3d.js +0 -0
  103. {forcingkit-0.14.0 → forcingkit-0.15.1}/static/styles.css +0 -0
  104. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/__init__.py +0 -0
  105. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_auth.py +0 -0
  106. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_erddap_fetch.py +0 -0
  107. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_mesh_frames_necofs.py +0 -0
  108. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_necofs_volume_live.py +0 -0
  109. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_nwps_nomads.py +0 -0
  110. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  111. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_obc_offshore_nj.py +0 -0
  112. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_ofs_archive_paths.py +0 -0
  113. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/integration/test_s3_roms_fetchers.py +0 -0
  114. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/scripts/inspect_dem.py +0 -0
  115. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/scripts/inspect_grib.py +0 -0
  116. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/scripts/inspect_zarr.py +0 -0
  117. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_coastwatch.py +0 -0
  118. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_dbofs.py +0 -0
  119. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_dispatcher.py +0 -0
  120. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_hrrr_atm.py +0 -0
  121. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_hrrr_idx.py +0 -0
  122. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_hrrr_levels.py +0 -0
  123. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_hrrr_points.py +0 -0
  124. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_hycom.py +0 -0
  125. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_layered_mesh.py +0 -0
  126. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_main.py +0 -0
  127. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_mesh_frames.py +0 -0
  128. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_ndbc.py +0 -0
  129. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_necofs_hindcast.py +0 -0
  130. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_necofs_mesh_cache.py +0 -0
  131. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_necofs_parent.py +0 -0
  132. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_necofs_volume.py +0 -0
  133. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_noaa_currents.py +0 -0
  134. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_noaa_datum.py +0 -0
  135. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_nwps.py +0 -0
  136. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_nyofs.py +0 -0
  137. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_obc_donor_report.py +0 -0
  138. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_obc_pipeline.py +0 -0
  139. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_removed_routes.py +0 -0
  140. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_settings.py +0 -0
  141. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_usgs.py +0 -0
  142. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_usgs_rivers.py +0 -0
  143. {forcingkit-0.14.0 → forcingkit-0.15.1}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.14.0
3
+ Version: 0.15.1
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -225,7 +225,7 @@ Requires-Dist: fastapi>=0.135.1
225
225
  Requires-Dist: h5netcdf>=1.8.1
226
226
  Requires-Dist: h5py>=3.16.0
227
227
  Requires-Dist: httpx>=0.28.1
228
- Requires-Dist: insitukit<0.3,>=0.2
228
+ Requires-Dist: insitukit<0.4,>=0.3
229
229
  Requires-Dist: jinja2>=3.0.0
230
230
  Requires-Dist: matplotlib>=3.10.9
231
231
  Requires-Dist: pandas>=2.2
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.14.0"
12
+ release = "0.15.1"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -163,7 +163,9 @@ These serve validation rather than forcing:
163
163
  has a 10 s connect and 60 s read timeout (``FORCINGKIT_ERDDAP_CONNECT_TIMEOUT``,
164
164
  ``FORCINGKIT_ERDDAP_READ_TIMEOUT``). A host that fails to connect, or a dataset that times out
165
165
  or errors, is skipped for 300 s (``FORCINGKIT_ERDDAP_FAILURE_TTL``, 0 to turn off), and the
166
- response's ``skipped`` lists each dataset left out, with the reason.
166
+ response's ``skipped`` lists each dataset left out, with the reason. A station's three datasets
167
+ are read at once, through `insitukit <https://github.com/lhzn-io/insitukit>`__'s ERDDAP client
168
+ (one attempt each, starting at least 0.2 s apart).
167
169
 
168
170
  NOAA OFS archive locations
169
171
  --------------------------
@@ -126,7 +126,7 @@ Planned work
126
126
  ``noaa.py`` serves CO-OPS water levels and current predictions (``coops.py``; since
127
127
  0.14.0 a thin layer over ``insitukit.coops``, which may replace it outright),
128
128
  ``erddap.py`` serves the UConn profile stations (named for the protocol, not the source;
129
- a move to ``insitukit.erddap`` waits for its expiring failure cache and parallel reads),
129
+ since 0.15.0 its requests go through ``insitukit.erddap``),
130
130
  and ``hydrography.py`` finds heads of tide from OpenStreetMap (``osm.py`` or similar).
131
131
  Renaming them breaks imports from earlier releases, so it belongs in a release that says
132
132
  so, with the old names kept as deprecated aliases for one release.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.14.0"
7
+ version = "0.15.1"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -26,8 +26,8 @@ dependencies = [
26
26
  "eccodeslib>=2.46.0",
27
27
  "fastapi>=0.135.1",
28
28
  "httpx>=0.28.1",
29
- # NOAA CO-OPS water level and current predictions (fetchers/noaa.py).
30
- "insitukit>=0.2,<0.3",
29
+ # NOAA CO-OPS (fetchers/noaa.py) and the UConn ERDDAP profiles (fetchers/erddap.py).
30
+ "insitukit>=0.3,<0.4",
31
31
  "pandas>=2.2",
32
32
  "pydantic>=2.12.5",
33
33
  "jinja2>=3.0.0",
@@ -104,6 +104,15 @@ class JobRegistry:
104
104
  job = self.get(job_id)
105
105
  return job is not None and job.status in ACTIVE
106
106
 
107
+ def forget(self, job_id: Optional[str] = None) -> None:
108
+ """Drop the finished job under `job_id`, or every finished job, once its store has been
109
+ deleted. An active job is kept: its build is still writing the store."""
110
+ with self._lock:
111
+ for key in [job_id] if job_id is not None else list(self._jobs):
112
+ job = self._jobs.get(key)
113
+ if job is not None and job.status not in ACTIVE:
114
+ del self._jobs[key]
115
+
107
116
  def _run(self, job: Job, build: Callable[[], Dict[str, Any]]) -> None:
108
117
  job.status, job.started = "running", _now()
109
118
  try:
@@ -217,6 +217,9 @@ async def purge_cache() -> Dict[str, str]:
217
217
  try:
218
218
  shutil.rmtree(cache_dir)
219
219
  os.makedirs(cache_dir, exist_ok=True)
220
+ from forcingkit_serve.jobs import registry
221
+
222
+ registry.forget()
220
223
  return {"status": "success", "message": "Cache purged successfully."}
221
224
  except Exception as e:
222
225
  logger.error(f"Failed to purge cache: {e}")
@@ -760,10 +763,13 @@ def _submit(
760
763
  from forcingkit_serve.routers.stores import store_complete
761
764
 
762
765
  kind = JOB_KINDS[zarr_id.split("_", 1)[0]]
763
- if not cache_bust and not registry.is_active(zarr_id) and store_complete(zarr_path):
766
+ complete = store_complete(zarr_path)
767
+ if not cache_bust and not registry.is_active(zarr_id) and complete:
764
768
  job = registry.finished(zarr_id, kind, cached())
765
769
  else:
766
- job = registry.submit(zarr_id, kind, build, rebuild=cache_bust)
770
+ # A succeeded job whose store is no longer complete (deleted from the cache since) is
771
+ # stale: rebuild rather than return it.
772
+ job = registry.submit(zarr_id, kind, build, rebuild=cache_bust or not complete)
767
773
  return JSONResponse(status_code=202, content=_job_view(job, http))
768
774
 
769
775
 
@@ -827,6 +833,14 @@ def get_job(job_id: str, http: Request) -> Dict[str, Any]:
827
833
  if not STORE_ID.match(job_id):
828
834
  raise HTTPException(status_code=400, detail="invalid job id")
829
835
  job = registry.get(job_id)
836
+ if (
837
+ job is not None
838
+ and job.result is not None
839
+ and not store_complete(job.result["zarr_path"])
840
+ ):
841
+ # Its store was deleted after the build: the job no longer has a result to report.
842
+ registry.forget(job_id)
843
+ job = None
830
844
  if job is None:
831
845
  zarr_path = os.path.join(settings.cache_dir(), f"{job_id}.zarr")
832
846
  if not store_complete(zarr_path):
@@ -470,6 +470,10 @@ async def delete_dataset(dataset_id: str):
470
470
  old_sidecar = zarr_path.parent / f"{dataset_id}_metadata.json"
471
471
  if old_sidecar.exists():
472
472
  old_sidecar.unlink()
473
+ # A finished job for this store would otherwise still report it as built.
474
+ from forcingkit_serve.jobs import registry
475
+
476
+ registry.forget(dataset_id)
473
477
  logger.info(f"Deleted Zarr dataset: {zarr_path}")
474
478
  return {"status": "success", "message": f"Deleted {dataset_id}"}
475
479
 
@@ -1,15 +1,12 @@
1
- import io
2
1
  import logging
3
2
  import math
4
3
  import os
5
- import threading
6
- import time
7
4
  from concurrent.futures import ThreadPoolExecutor
5
+ from datetime import datetime, timezone
8
6
  from typing import Optional
9
- from urllib.parse import urlsplit
10
7
 
11
8
  import pandas as pd
12
- import requests
9
+ from insitukit.erddap import DatasetMissing, Erddap, FailureCache, Unavailable
13
10
 
14
11
  from forcingkit import settings
15
12
 
@@ -17,114 +14,74 @@ logger = logging.getLogger(__name__)
17
14
 
18
15
  BASE_ERDDAP_URL = "http://merlin.dms.uconn.edu:8080/erddap"
19
16
 
17
+ # Hosts and datasets that failed recently, shared by every request this process serves. A host is
18
+ # recorded when a connect fails (refused, unreachable or timed out), so the other datasets on it, in
19
+ # this request and the next few, are skipped instead of each waiting out the connect timeout. A
20
+ # dataset is recorded when its read times out or the server returns a 5xx. The lifetime is
21
+ # settings.erddap_failure_ttl(); 0 records nothing.
22
+ _failures = FailureCache()
20
23
 
21
- class ErddapUnavailable(Exception):
22
- """A dataset was not read, because its host or the dataset failed now or recently."""
23
24
 
24
-
25
- class _FailureCache:
26
- """Hosts and datasets that failed recently, skipped until their entry expires.
27
-
28
- A host is recorded when a connect fails (refused, unreachable or timed out), so the other
29
- datasets on it, in this request and the next few, are skipped instead of each waiting out the
30
- connect timeout. A dataset is recorded when its read times out or the server returns a 5xx.
31
- The lifetime is settings.erddap_failure_ttl(); 0 records nothing.
32
- """
33
-
34
- def __init__(self) -> None:
35
- self._entries: dict[str, tuple[float, str]] = {}
36
- self._lock = threading.Lock()
37
-
38
- def record(self, key: str, reason: str) -> None:
39
- ttl = settings.erddap_failure_ttl()
40
- if ttl <= 0:
41
- return
42
- with self._lock:
43
- self._entries[key] = (time.monotonic() + ttl, reason)
44
-
45
- def check(self, key: str) -> Optional[str]:
46
- """The recorded reason while `key` is still failed, else None."""
47
- with self._lock:
48
- entry = self._entries.get(key)
49
- if entry is None:
50
- return None
51
- expires, reason = entry
52
- if time.monotonic() >= expires:
53
- del self._entries[key]
54
- return None
55
- return reason
56
-
57
- def clear(self) -> None:
58
- with self._lock:
59
- self._entries.clear()
25
+ def clear_failure_cache() -> None:
26
+ """Forget every recorded host and dataset failure."""
27
+ _failures.clear()
60
28
 
61
29
 
62
- _failures = _FailureCache()
30
+ def _server() -> Erddap:
31
+ """An insitukit client for the UConn server, built per call so the settings are read each time:
32
+ one attempt per dataset, the configured timeouts, and the shared failure cache."""
33
+ connect, read = settings.erddap_timeouts()
34
+ return Erddap(
35
+ BASE_ERDDAP_URL,
36
+ budget=None,
37
+ min_interval=0.2,
38
+ retries=1,
39
+ connect_timeout=connect,
40
+ timeout=read,
41
+ failure_ttl=settings.erddap_failure_ttl(),
42
+ failures=_failures,
43
+ fmt="csvp",
44
+ )
63
45
 
64
46
 
65
- def clear_failure_cache() -> None:
66
- """Forget every recorded host and dataset failure."""
67
- _failures.clear()
47
+ def _utc(stamp: str) -> datetime:
48
+ ts = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
49
+ return (
50
+ ts.replace(tzinfo=timezone.utc)
51
+ if ts.tzinfo is None
52
+ else ts.astimezone(timezone.utc)
53
+ )
68
54
 
69
55
 
70
56
  def _read_dataset(
71
- dataset_name: str, start_time: str, end_time: str
57
+ server: Erddap, dataset_name: str, start_time: str, end_time: str
72
58
  ) -> Optional[pd.DataFrame]:
73
59
  """One dataset's temperature series from ERDDAP, indexed by UTC time.
74
60
 
75
- Returns None when ERDDAP has no rows for the window (HTTP 404). Raises ErddapUnavailable when
76
- the host or dataset is negatively cached, or when this read fails.
61
+ Returns None when ERDDAP has no rows for the window (any HTTP 404). Raises
62
+ insitukit.erddap.Unavailable when the host or dataset is negatively cached or this read fails,
63
+ and ValueError when the response cannot be parsed.
77
64
  """
78
- host = urlsplit(BASE_ERDDAP_URL).netloc
79
- host_key = f"host:{host}"
80
- dataset_key = f"dataset:{dataset_name}"
81
- for key in (host_key, dataset_key):
82
- reason = _failures.check(key)
83
- if reason is not None:
84
- raise ErddapUnavailable(f"skipped after a recent failure: {reason}")
85
-
86
- query_url = f"{BASE_ERDDAP_URL}/tabledap/{dataset_name}.csvp?time,sea_water_temperature&time%3E={start_time}&time%3C={end_time}"
87
65
  logger.info(
88
66
  f"Fetching ERDDAP data for {dataset_name} ({start_time} to {end_time})..."
89
67
  )
90
68
  try:
91
- response = requests.get(query_url, timeout=settings.erddap_timeouts())
92
- except requests.ConnectionError as e:
93
- # Includes ConnectTimeout: the host is unreachable, so skip it for a while.
94
- reason = f"{host} unreachable ({type(e).__name__})"
95
- _failures.record(host_key, reason)
96
- raise ErddapUnavailable(reason) from e
97
- except requests.Timeout as e:
98
- reason = f"{dataset_name} read timed out"
99
- _failures.record(dataset_key, reason)
100
- raise ErddapUnavailable(reason) from e
101
- except requests.RequestException as e:
102
- raise ErddapUnavailable(
103
- f"{dataset_name} request failed ({type(e).__name__})"
104
- ) from e
105
-
106
- if response.status_code == 404:
107
- # ERDDAP answers 404 when the query matches no rows.
69
+ df = server.fetch(
70
+ dataset_name,
71
+ _utc(start_time),
72
+ _utc(end_time),
73
+ ["sea_water_temperature"],
74
+ )
75
+ except DatasetMissing:
108
76
  logger.info(f"ERDDAP has no rows for {dataset_name} in the window")
109
77
  return None
110
- if response.status_code >= 500:
111
- reason = f"{dataset_name} returned HTTP {response.status_code}"
112
- _failures.record(dataset_key, reason)
113
- raise ErddapUnavailable(reason)
114
- if response.status_code != 200:
115
- raise ErddapUnavailable(f"{dataset_name} returned HTTP {response.status_code}")
116
-
117
- try:
118
- df = pd.read_csv(io.StringIO(response.text))
119
- if df.empty:
120
- return None
121
- # The headers are like "time (UTC)" and "sea_water_temperature (celsius)".
122
- df.columns = ["time", "sea_water_temperature"]
123
- df = df.set_index("time")
124
- df.index = pd.to_datetime(df.index, utc=True)
78
+ except Unavailable:
79
+ raise
125
80
  except Exception as e:
126
- raise ErddapUnavailable(f"{dataset_name} could not be parsed ({e})") from e
127
- return df
81
+ raise ValueError(f"{dataset_name} could not be parsed ({e})") from e
82
+ if df.empty:
83
+ return None
84
+ return df.set_index("time")
128
85
 
129
86
 
130
87
  def fetch_erddap_station_profiles(
@@ -169,6 +126,8 @@ def fetch_erddap_station_profiles(
169
126
  if station_upper == "ARTG":
170
127
  datasets["bottom"] = "ARTG_WQ_BTM_1"
171
128
 
129
+ server = _server()
130
+
172
131
  def load(
173
132
  level: str, dataset_name: str
174
133
  ) -> tuple[Optional[pd.DataFrame], Optional[dict]]:
@@ -181,8 +140,8 @@ def fetch_erddap_station_profiles(
181
140
  logger.info(f"Cache hit for {dataset_name}: {cache_path}")
182
141
  return pd.read_csv(cache_path, index_col=0, parse_dates=True), None
183
142
  try:
184
- df = _read_dataset(dataset_name, start_time, end_time)
185
- except ErddapUnavailable as e:
143
+ df = _read_dataset(server, dataset_name, start_time, end_time)
144
+ except (Unavailable, ValueError) as e:
186
145
  logger.error(f"Failed fetching or parsing ERDDAP for {dataset_name}: {e}")
187
146
  return None, {
188
147
  "station": station_upper,
@@ -46,18 +46,18 @@ def clean_failures(monkeypatch):
46
46
 
47
47
  @pytest.fixture
48
48
  def fake_get(monkeypatch):
49
- """Replace requests.get in the fetcher; `behaviour(url, timeout)` returns or raises."""
49
+ """Replace requests.Session.get (insitukit's client); `behaviour(url, timeout)` returns or raises."""
50
50
  lock = threading.Lock()
51
51
 
52
52
  def install(behaviour):
53
53
  calls: list[tuple[str, object]] = []
54
54
 
55
- def get(url, timeout=None):
55
+ def get(self, url, timeout=None):
56
56
  with lock:
57
57
  calls.append((url, timeout))
58
58
  return behaviour(url, timeout)
59
59
 
60
- monkeypatch.setattr(erddap.requests, "get", get)
60
+ monkeypatch.setattr(requests.Session, "get", get)
61
61
  return calls
62
62
 
63
63
  return install
@@ -177,7 +177,7 @@ def test_failures_are_negatively_cached(fake_get, tmp_path):
177
177
 
178
178
  def test_negative_cache_expires(fake_get, monkeypatch, tmp_path):
179
179
  clock = [1000.0]
180
- monkeypatch.setattr(erddap.time, "monotonic", lambda: clock[0])
180
+ monkeypatch.setattr(time, "monotonic", lambda: clock[0])
181
181
  monkeypatch.setenv("FORCINGKIT_ERDDAP_FAILURE_TTL", "60")
182
182
  calls = fake_get(lambda url, timeout: FakeResponse(status_code=503, text=""))
183
183
 
@@ -235,7 +235,7 @@ def test_cached_csv_is_served_while_the_host_is_down(fake_get, tmp_path):
235
235
 
236
236
  def test_bbox_with_no_known_station_is_empty_and_fetches_nothing(mocker, tmp_path):
237
237
  fetch = mocker.patch("forcingkit.fetchers.erddap.fetch_erddap_station_profiles")
238
- get = mocker.patch("forcingkit.fetchers.erddap.requests.get")
238
+ get = mocker.patch("requests.Session.get")
239
239
 
240
240
  assert (
241
241
  erddap.fetch_erddap_stations_in_bbox(OFFSHORE_NJ, START, END, str(tmp_path))
@@ -313,7 +313,7 @@ def _bbox_body(bbox, **extra):
313
313
 
314
314
  @pytest.mark.asyncio
315
315
  async def test_bbox_route_with_no_station_returns_empty(mocker):
316
- get = mocker.patch("forcingkit.fetchers.erddap.requests.get")
316
+ get = mocker.patch("requests.Session.get")
317
317
  async with AsyncClient(
318
318
  transport=ASGITransport(app=_app()), base_url="http://test"
319
319
  ) as ac:
@@ -363,7 +363,7 @@ async def test_bbox_route_reports_skipped_datasets(mocker, tmp_path):
363
363
  cache_dir=str(tmp_path), **kw
364
364
  ),
365
365
  )
366
- mocker.patch("forcingkit.fetchers.erddap.requests.get", side_effect=down)
366
+ mocker.patch("requests.Session.get", side_effect=down)
367
367
  async with AsyncClient(
368
368
  transport=ASGITransport(app=_app()), base_url="http://test"
369
369
  ) as ac:
@@ -137,6 +137,75 @@ def test_store_finished_before_restart_is_a_succeeded_job(client):
137
137
  assert body["status"] == "succeeded" and body["result"]["id"] == zarr_id
138
138
 
139
139
 
140
+ def test_registry_rebuilds_succeeded_job_on_request_and_forgets_finished_ones():
141
+ registry = jobs.JobRegistry(workers=1)
142
+ release = threading.Event()
143
+ first = registry.submit("obc_0123456789ab", "ocean", lambda: {"n": 1})
144
+ deadline = time.monotonic() + 5
145
+ while first.status != "succeeded" and time.monotonic() < deadline:
146
+ time.sleep(0.01)
147
+ assert first.status == "succeeded"
148
+ assert registry.submit("obc_0123456789ab", "ocean", lambda: {"n": 2}) is first
149
+
150
+ def slow_build():
151
+ release.wait(5)
152
+ return {"n": 3}
153
+
154
+ rebuilt = registry.submit("obc_0123456789ab", "ocean", slow_build, rebuild=True)
155
+ assert rebuilt is not first
156
+
157
+ # An active job survives forget; a finished one does not.
158
+ registry.forget("obc_0123456789ab")
159
+ registry.forget()
160
+ assert registry.get("obc_0123456789ab") is rebuilt
161
+ release.set()
162
+ while rebuilt.status != "succeeded" and time.monotonic() < deadline:
163
+ time.sleep(0.01)
164
+ registry.forget("obc_0123456789ab")
165
+ assert registry.get("obc_0123456789ab") is None
166
+
167
+
168
+ def _built_obc_job(client, monkeypatch, calls):
169
+ def fake_dispatch(**kwargs):
170
+ calls.append(kwargs)
171
+ _write_store(kwargs["zarr_path"])
172
+ return kwargs["zarr_path"]
173
+
174
+ monkeypatch.setattr(dispatcher, "dispatch_obc_request", fake_dispatch)
175
+ body = client.post("/api/v1/jobs/obc", json=OBC).json()
176
+ assert _wait(client, body["status_url"])["status"] == "succeeded"
177
+ return body
178
+
179
+
180
+ def test_succeeded_job_whose_store_vanished_is_rebuilt(client, monkeypatch):
181
+ import shutil
182
+
183
+ calls = []
184
+ body = _built_obc_job(client, monkeypatch, calls)
185
+ _, zarr_path, _ = main._obc_store(main.OBCRequest(**OBC))
186
+ shutil.rmtree(zarr_path) # removed behind the service's back, registry untouched
187
+
188
+ assert client.get(body["status_url"]).status_code == 404
189
+ again = client.post("/api/v1/jobs/obc", json=OBC)
190
+ assert again.status_code == 202
191
+ done = _wait(client, again.json()["status_url"])
192
+ assert done["status"] == "succeeded" and len(calls) == 2
193
+ assert done["result"]["id"] == body["job_id"]
194
+
195
+
196
+ def test_deleting_a_store_drops_its_job(client, monkeypatch):
197
+ calls = []
198
+ body = _built_obc_job(client, monkeypatch, calls)
199
+ deleted = client.delete(f"/api/v1/cache/dataset/{body['job_id']}")
200
+ assert deleted.status_code == 200
201
+ assert jobs.registry.get(body["job_id"]) is None
202
+
203
+ again = client.post("/api/v1/jobs/obc", json=OBC)
204
+ assert again.status_code == 202
205
+ assert _wait(client, again.json()["status_url"])["status"] == "succeeded"
206
+ assert len(calls) == 2
207
+
208
+
140
209
  def test_unknown_and_invalid_job_ids(client):
141
210
  assert client.get("/api/v1/jobs/obc_000000000000").status_code == 404
142
211
  assert client.get("/api/v1/jobs/not-a-store").status_code == 400
@@ -1000,7 +1000,7 @@ wheels = [
1000
1000
 
1001
1001
  [[package]]
1002
1002
  name = "forcingkit"
1003
- version = "0.14.0"
1003
+ version = "0.15.1"
1004
1004
  source = { editable = "." }
1005
1005
  dependencies = [
1006
1006
  { name = "bottleneck" },
@@ -1062,7 +1062,7 @@ requires-dist = [
1062
1062
  { name = "h5netcdf", specifier = ">=1.8.1" },
1063
1063
  { name = "h5py", specifier = ">=3.16.0" },
1064
1064
  { name = "httpx", specifier = ">=0.28.1" },
1065
- { name = "insitukit", specifier = ">=0.2,<0.3" },
1065
+ { name = "insitukit", specifier = ">=0.3,<0.4" },
1066
1066
  { name = "jinja2", specifier = ">=3.0.0" },
1067
1067
  { name = "matplotlib", specifier = ">=3.10.9" },
1068
1068
  { name = "pandas", specifier = ">=2.2" },
@@ -1612,16 +1612,16 @@ wheels = [
1612
1612
 
1613
1613
  [[package]]
1614
1614
  name = "insitukit"
1615
- version = "0.2.0"
1615
+ version = "0.3.0"
1616
1616
  source = { registry = "https://pypi.org/simple" }
1617
1617
  dependencies = [
1618
1618
  { name = "numpy" },
1619
1619
  { name = "pandas" },
1620
1620
  { name = "requests" },
1621
1621
  ]
1622
- sdist = { url = "https://files.pythonhosted.org/packages/fb/61/423e9c96684ca33c2b68c4bd397f0c04c56ceb7f46bebbb3d885f97a8f1f/insitukit-0.2.0.tar.gz", hash = "sha256:4512cf3e7d829de87da305c4f19a78cca711a64c4f04c7d47a7960f9bd168045", size = 91954, upload-time = "2026-11-05T22:59:29.034Z" }
1622
+ sdist = { url = "https://files.pythonhosted.org/packages/d6/2f/64ab83ca7705afcab426cfe64953dec222e6fc148bd47bf19118d9df9d70/insitukit-0.3.0.tar.gz", hash = "sha256:fa1ed9c1cf89995b6edf4a550f8835e327f352b0df69a756b3185f5dc7b67aa6", size = 99038, upload-time = "2026-11-06T00:48:46.489Z" }
1623
1623
  wheels = [
1624
- { url = "https://files.pythonhosted.org/packages/c1/c1/909423b4f2f428e7ca2f6e49228a0585637b921d7c569f4f9fbdb3896ac8/insitukit-0.2.0-py3-none-any.whl", hash = "sha256:e081b5996096b7e70ff37594e244dedff4d6345a4a8c4d72c0ec8036f3f09959", size = 19523, upload-time = "2026-11-05T22:59:27.469Z" },
1624
+ { url = "https://files.pythonhosted.org/packages/8a/00/5b0fa89c8a08b99d36533c4849e20f6061a65901693de02806ff3a22f936/insitukit-0.3.0-py3-none-any.whl", hash = "sha256:396df69830c4d921a40c7693c0f24a7e02a86617730acf010cdc4fdf2a6abb35", size = 22860, upload-time = "2026-11-06T00:48:45.069Z" },
1625
1625
  ]
1626
1626
 
1627
1627
  [[package]]
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes