forcingkit 0.15.0__tar.gz → 0.16.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. {forcingkit-0.15.0 → forcingkit-0.16.0}/PKG-INFO +2 -2
  2. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/conf.py +1 -1
  3. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/fetchers.rst +4 -1
  4. {forcingkit-0.15.0 → forcingkit-0.16.0}/pyproject.toml +3 -3
  5. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/jobs.py +9 -0
  6. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/main.py +16 -2
  7. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/viewer.py +4 -0
  8. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/ndbc.py +30 -107
  9. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_jobs.py +69 -0
  10. {forcingkit-0.15.0 → forcingkit-0.16.0}/uv.lock +5 -5
  11. {forcingkit-0.15.0 → forcingkit-0.16.0}/.agents/forcingkit.md +0 -0
  12. {forcingkit-0.15.0 → forcingkit-0.16.0}/.claude/CLAUDE.md +0 -0
  13. {forcingkit-0.15.0 → forcingkit-0.16.0}/.env.template +0 -0
  14. {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/copilot-instructions.md +0 -0
  15. {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/workflows/docs.yml +0 -0
  16. {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/workflows/publish-client.yml +0 -0
  17. {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/workflows/publish.yml +0 -0
  18. {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/workflows/tests.yml +0 -0
  19. {forcingkit-0.15.0 → forcingkit-0.16.0}/.gitignore +0 -0
  20. {forcingkit-0.15.0 → forcingkit-0.16.0}/.markdownlint.yaml +0 -0
  21. {forcingkit-0.15.0 → forcingkit-0.16.0}/.pre-commit-config.yaml +0 -0
  22. {forcingkit-0.15.0 → forcingkit-0.16.0}/.python-version +0 -0
  23. {forcingkit-0.15.0 → forcingkit-0.16.0}/AGENTS.md +0 -0
  24. {forcingkit-0.15.0 → forcingkit-0.16.0}/CONTRIBUTING.md +0 -0
  25. {forcingkit-0.15.0 → forcingkit-0.16.0}/Dockerfile +0 -0
  26. {forcingkit-0.15.0 → forcingkit-0.16.0}/LICENSE +0 -0
  27. {forcingkit-0.15.0 → forcingkit-0.16.0}/README.md +0 -0
  28. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/LICENSE +0 -0
  29. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/README.md +0 -0
  30. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/pyproject.toml +0 -0
  31. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/__init__.py +0 -0
  32. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/client.py +0 -0
  33. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/schema.py +0 -0
  34. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/store.py +0 -0
  35. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/trajectory.py +0 -0
  36. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/tests/conftest.py +0 -0
  37. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/tests/test_client.py +0 -0
  38. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/tests/test_schema.py +0 -0
  39. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/tests/test_trajectory.py +0 -0
  40. {forcingkit-0.15.0 → forcingkit-0.16.0}/client/uv.lock +0 -0
  41. {forcingkit-0.15.0 → forcingkit-0.16.0}/docker-compose.yml +0 -0
  42. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/Makefile +0 -0
  43. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/make.bat +0 -0
  44. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/requirements-docs.txt +0 -0
  45. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/_extra/CNAME +0 -0
  46. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/_static/custom.css +0 -0
  47. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  48. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/atmospheric_forcing.rst +0 -0
  49. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/client.rst +0 -0
  50. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/hycom.rst +0 -0
  51. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/index.rst +0 -0
  52. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/layered_mesh.rst +0 -0
  53. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/mesh_frames.rst +0 -0
  54. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/necofs.rst +0 -0
  55. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/nyofs.rst +0 -0
  56. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/rivers.rst +0 -0
  57. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/roadmap.rst +0 -0
  58. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/sources.rst +0 -0
  59. {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/waves.rst +0 -0
  60. {forcingkit-0.15.0 → forcingkit-0.16.0}/main.py +0 -0
  61. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/__init__.py +0 -0
  62. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/__init__.py +0 -0
  63. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  64. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  65. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/removed.py +0 -0
  66. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/stores.py +0 -0
  67. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/stac.py +0 -0
  68. {forcingkit-0.15.0 → forcingkit-0.16.0}/service/run_server.py +0 -0
  69. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/__init__.py +0 -0
  70. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  71. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/dispatcher.py +0 -0
  72. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
  73. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/dbofs.py +0 -0
  74. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/erddap.py +0 -0
  75. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hrrr.py +0 -0
  76. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  77. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  78. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  79. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hycom.py +0 -0
  80. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hydrography.py +0 -0
  81. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/necofs.py +0 -0
  82. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/noaa.py +0 -0
  83. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/nwps.py +0 -0
  84. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/nyofs.py +0 -0
  85. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/usgs.py +0 -0
  86. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  87. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/layered_mesh.py +0 -0
  88. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/mesh_frames.py +0 -0
  89. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
  90. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  91. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  92. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  93. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/render.py +0 -0
  94. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/settings.py +0 -0
  95. {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/zarr_stream.py +0 -0
  96. {forcingkit-0.15.0 → forcingkit-0.16.0}/static/app.js +0 -0
  97. {forcingkit-0.15.0 → forcingkit-0.16.0}/static/favicon.ico +0 -0
  98. {forcingkit-0.15.0 → forcingkit-0.16.0}/static/favicon.svg +0 -0
  99. {forcingkit-0.15.0 → forcingkit-0.16.0}/static/index.html +0 -0
  100. {forcingkit-0.15.0 → forcingkit-0.16.0}/static/logo.svg +0 -0
  101. {forcingkit-0.15.0 → forcingkit-0.16.0}/static/preview3d.js +0 -0
  102. {forcingkit-0.15.0 → forcingkit-0.16.0}/static/styles.css +0 -0
  103. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/__init__.py +0 -0
  104. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_auth.py +0 -0
  105. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_erddap_fetch.py +0 -0
  106. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
  107. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_necofs_volume_live.py +0 -0
  108. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_nwps_nomads.py +0 -0
  109. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  110. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_obc_offshore_nj.py +0 -0
  111. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_ofs_archive_paths.py +0 -0
  112. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
  113. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/scripts/inspect_dem.py +0 -0
  114. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/scripts/inspect_grib.py +0 -0
  115. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/scripts/inspect_zarr.py +0 -0
  116. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_coastwatch.py +0 -0
  117. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_dbofs.py +0 -0
  118. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_dispatcher.py +0 -0
  119. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_erddap.py +0 -0
  120. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hrrr_atm.py +0 -0
  121. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hrrr_idx.py +0 -0
  122. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hrrr_levels.py +0 -0
  123. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hrrr_points.py +0 -0
  124. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hycom.py +0 -0
  125. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_layered_mesh.py +0 -0
  126. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_main.py +0 -0
  127. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_mesh_frames.py +0 -0
  128. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_ndbc.py +0 -0
  129. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_necofs_hindcast.py +0 -0
  130. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
  131. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_necofs_parent.py +0 -0
  132. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_necofs_volume.py +0 -0
  133. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_noaa_currents.py +0 -0
  134. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_noaa_datum.py +0 -0
  135. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_nwps.py +0 -0
  136. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_nyofs.py +0 -0
  137. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_obc_donor_report.py +0 -0
  138. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_obc_pipeline.py +0 -0
  139. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_removed_routes.py +0 -0
  140. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_settings.py +0 -0
  141. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_usgs.py +0 -0
  142. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_usgs_rivers.py +0 -0
  143. {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.15.0
3
+ Version: 0.16.0
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -225,7 +225,7 @@ Requires-Dist: fastapi>=0.135.1
225
225
  Requires-Dist: h5netcdf>=1.8.1
226
226
  Requires-Dist: h5py>=3.16.0
227
227
  Requires-Dist: httpx>=0.28.1
228
- Requires-Dist: insitukit<0.4,>=0.3
228
+ Requires-Dist: insitukit<0.5,>=0.4
229
229
  Requires-Dist: jinja2>=3.0.0
230
230
  Requires-Dist: matplotlib>=3.10.9
231
231
  Requires-Dist: pandas>=2.2
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.15.0"
12
+ release = "0.16.0"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -152,7 +152,10 @@ These serve validation rather than forcing:
152
152
  ``end_time`` is inclusive, as in the API. Current predictions (``/api/v1/currents``) and the
153
153
  current-station list (``/api/v1/currents/stations``) use the same client.
154
154
  - **NDBC** buoy meteorology and ADCP profiles (``/api/v1/ndbc``), from the `NOAA National Data
155
- Buoy Center <https://www.ndbc.noaa.gov/>`__.
155
+ Buoy Center <https://www.ndbc.noaa.gov/>`__, through insitukit's NDBC client: each month from
156
+ the monthly, yearly or real-time file, each file downloaded once, and each column's own
157
+ missing-value sentinel read as missing (since 0.16.0; earlier releases also dropped real
158
+ readings equal to another column's sentinel, such as a 999 hPa low or a 99-degree wind).
156
159
  - **UConn ERDDAP** water-column profiles (``/api/v1/telemetry/station`` and ``/bbox``), from the
157
160
  `University of Connecticut Department of Marine Sciences <https://marinesciences.uconn.edu/>`__
158
161
  `ERDDAP server <http://merlin.dms.uconn.edu:8080/erddap/index.html>`__.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.15.0"
7
+ version = "0.16.0"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -26,8 +26,8 @@ dependencies = [
26
26
  "eccodeslib>=2.46.0",
27
27
  "fastapi>=0.135.1",
28
28
  "httpx>=0.28.1",
29
- # NOAA CO-OPS (fetchers/noaa.py) and the UConn ERDDAP profiles (fetchers/erddap.py).
30
- "insitukit>=0.3,<0.4",
29
+ # NOAA CO-OPS (fetchers/noaa.py), the UConn ERDDAP profiles (fetchers/erddap.py) and NDBC (fetchers/ndbc.py).
30
+ "insitukit>=0.4,<0.5",
31
31
  "pandas>=2.2",
32
32
  "pydantic>=2.12.5",
33
33
  "jinja2>=3.0.0",
@@ -104,6 +104,15 @@ class JobRegistry:
104
104
  job = self.get(job_id)
105
105
  return job is not None and job.status in ACTIVE
106
106
 
107
+ def forget(self, job_id: Optional[str] = None) -> None:
108
+ """Drop the finished job under `job_id`, or every finished job, once its store has been
109
+ deleted. An active job is kept: its build is still writing the store."""
110
+ with self._lock:
111
+ for key in [job_id] if job_id is not None else list(self._jobs):
112
+ job = self._jobs.get(key)
113
+ if job is not None and job.status not in ACTIVE:
114
+ del self._jobs[key]
115
+
107
116
  def _run(self, job: Job, build: Callable[[], Dict[str, Any]]) -> None:
108
117
  job.status, job.started = "running", _now()
109
118
  try:
@@ -217,6 +217,9 @@ async def purge_cache() -> Dict[str, str]:
217
217
  try:
218
218
  shutil.rmtree(cache_dir)
219
219
  os.makedirs(cache_dir, exist_ok=True)
220
+ from forcingkit_serve.jobs import registry
221
+
222
+ registry.forget()
220
223
  return {"status": "success", "message": "Cache purged successfully."}
221
224
  except Exception as e:
222
225
  logger.error(f"Failed to purge cache: {e}")
@@ -760,10 +763,13 @@ def _submit(
760
763
  from forcingkit_serve.routers.stores import store_complete
761
764
 
762
765
  kind = JOB_KINDS[zarr_id.split("_", 1)[0]]
763
- if not cache_bust and not registry.is_active(zarr_id) and store_complete(zarr_path):
766
+ complete = store_complete(zarr_path)
767
+ if not cache_bust and not registry.is_active(zarr_id) and complete:
764
768
  job = registry.finished(zarr_id, kind, cached())
765
769
  else:
766
- job = registry.submit(zarr_id, kind, build, rebuild=cache_bust)
770
+ # A succeeded job whose store is no longer complete (deleted from the cache since) is
771
+ # stale: rebuild rather than return it.
772
+ job = registry.submit(zarr_id, kind, build, rebuild=cache_bust or not complete)
767
773
  return JSONResponse(status_code=202, content=_job_view(job, http))
768
774
 
769
775
 
@@ -827,6 +833,14 @@ def get_job(job_id: str, http: Request) -> Dict[str, Any]:
827
833
  if not STORE_ID.match(job_id):
828
834
  raise HTTPException(status_code=400, detail="invalid job id")
829
835
  job = registry.get(job_id)
836
+ if (
837
+ job is not None
838
+ and job.result is not None
839
+ and not store_complete(job.result["zarr_path"])
840
+ ):
841
+ # Its store was deleted after the build: the job no longer has a result to report.
842
+ registry.forget(job_id)
843
+ job = None
830
844
  if job is None:
831
845
  zarr_path = os.path.join(settings.cache_dir(), f"{job_id}.zarr")
832
846
  if not store_complete(zarr_path):
@@ -470,6 +470,10 @@ async def delete_dataset(dataset_id: str):
470
470
  old_sidecar = zarr_path.parent / f"{dataset_id}_metadata.json"
471
471
  if old_sidecar.exists():
472
472
  old_sidecar.unlink()
473
+ # A finished job for this store would otherwise still report it as built.
474
+ from forcingkit_serve.jobs import registry
475
+
476
+ registry.forget(dataset_id)
473
477
  logger.info(f"Deleted Zarr dataset: {zarr_path}")
474
478
  return {"status": "success", "message": f"Deleted {dataset_id}"}
475
479
 
@@ -1,46 +1,24 @@
1
1
  """NDBC buoy observations for model validation: standard meteorology (sea and air temperature,
2
- pressure, wind, waves) and near-surface ADCP currents.
3
-
4
- Files come from the NDBC data server in the order they are published: the per-month file for
5
- months of the current year (`data/<product>/<Mon>/<id><m><yyyy>.txt.gz`), then the yearly
6
- historical file (`data/historical/<product>/<id>h<yyyy>.txt.gz`), then the rolling 45-day
7
- real-time file (`data/realtime2/<id>.txt` or `.adcp`). Missing-value sentinels (99, 999, 9999,
8
- MM) become NaN. These observations validate a model; they never force it.
2
+ pressure, wind, waves) and near-surface ADCP currents, through insitukit.
3
+
4
+ insitukit's NDBC client reads each month from the first file NDBC publishes it in: the per-month
5
+ file for months of the current year, then the yearly historical file, then the rolling 45-day
6
+ real-time file. It downloads each file once per call, paces and identifies its requests, and turns
7
+ NDBC's missing-value sentinels (99, 999, 9999, MM) into NaN. This module keeps forcingkit's
8
+ function names and the JSON shape of ``fetch_ndbc``. These observations validate a model; they
9
+ never force it.
9
10
  """
10
11
 
11
- import gzip
12
12
  import json
13
- import logging
14
13
  import math
15
14
  import os
16
- import re
17
15
  from typing import Optional
18
16
 
19
17
  import numpy as np
20
18
  import pandas as pd
21
- import requests
22
-
23
- logger = logging.getLogger(__name__)
24
-
25
- NDBC = "https://www.ndbc.noaa.gov"
26
- PRODUCTS = ("stdmet", "adcp")
27
- REALTIME_SUFFIX = {"stdmet": "txt", "adcp": "adcp"}
28
- MONTH_ABBR = [
29
- "Jan",
30
- "Feb",
31
- "Mar",
32
- "Apr",
33
- "May",
34
- "Jun",
35
- "Jul",
36
- "Aug",
37
- "Sep",
38
- "Oct",
39
- "Nov",
40
- "Dec",
41
- ]
42
- # Column sentinels NDBC uses for "not measured".
43
- SENTINELS = {99.0, 999.0, 9999.0, 99.00}
19
+ from insitukit import ndbc as _ndbc
20
+
21
+ PRODUCTS = _ndbc.PRODUCTS
44
22
  # stdmet columns carried through, with units.
45
23
  STDMET_COLUMNS = {
46
24
  "WTMP": "degC", # sea surface temperature
@@ -53,58 +31,19 @@ STDMET_COLUMNS = {
53
31
  "DPD": "s",
54
32
  }
55
33
 
56
-
57
- def month_code(month: int) -> str:
58
- """NDBC's month code in per-month file names: 1 to 9, then a, b, c for October to December."""
59
- return str(month) if month < 10 else "abc"[month - 10]
34
+ month_code = _ndbc.month_code
60
35
 
61
36
 
62
37
  def candidate_urls(station: str, product: str, year: int, month: int) -> list[str]:
63
38
  """URLs that may hold `product` for `station` in the given month, in the order to try."""
64
- station = station.lower()
65
- return [
66
- f"{NDBC}/data/{product}/{MONTH_ABBR[month - 1]}/{station}{month_code(month)}{year}.txt.gz",
67
- f"{NDBC}/data/historical/{product}/{station}h{year}.txt.gz",
68
- f"{NDBC}/data/realtime2/{station.upper()}.{REALTIME_SUFFIX[product]}",
69
- ]
39
+ return _ndbc.urls(station, product, year, month)
70
40
 
71
41
 
72
42
  def parse_ndbc_text(text: str) -> pd.DataFrame:
73
- """Parse an NDBC whitespace-delimited file (two `#` header rows) into a frame indexed by UTC
74
- time, sentinels and `MM` as NaN. Handles both four- and two-digit years, with or without a
75
- minute column."""
76
- lines = [ln for ln in text.splitlines() if ln.strip()]
77
- if not lines or not lines[0].startswith("#"):
78
- raise ValueError("not an NDBC data file: no header row")
79
- names = lines[0].lstrip("#").split()
80
- rows = [ln.split() for ln in lines if not ln.startswith("#")]
81
- df = pd.DataFrame(rows, columns=names[: len(rows[0])] if rows else names)
82
- year_col = names[0]
83
- years = df[year_col].astype(int)
84
- years = years.where(years >= 100, years + 1900)
85
- minutes = df["mm"].astype(int) if "mm" in df else 0
86
- df.index = pd.to_datetime(
87
- dict(
88
- year=years,
89
- month=df["MM"].astype(int),
90
- day=df["DD"].astype(int),
91
- hour=df["hh"].astype(int),
92
- minute=minutes,
93
- )
94
- )
95
- df = df.drop(columns=[c for c in (year_col, "MM", "DD", "hh", "mm") if c in df])
96
- df = df.replace("MM", np.nan).apply(pd.to_numeric, errors="coerce")
97
- return df.mask(df.isin(SENTINELS))
98
-
99
-
100
- def _get(url: str, timeout: float = 60.0) -> Optional[str]:
101
- r = requests.get(url, timeout=timeout)
102
- if r.status_code == 404:
103
- return None
104
- r.raise_for_status()
105
- if url.endswith(".gz"):
106
- return gzip.decompress(r.content).decode("utf-8", errors="replace")
107
- return r.text
43
+ """Parse an NDBC whitespace-delimited file into a frame indexed by UTC time (naive),
44
+ sentinels and `MM` as NaN."""
45
+ df = _ndbc.parse(text)
46
+ return df.set_index(pd.DatetimeIndex(df.pop("time")).tz_localize(None))
108
47
 
109
48
 
110
49
  def _utc_naive(t) -> pd.Timestamp:
@@ -112,40 +51,24 @@ def _utc_naive(t) -> pd.Timestamp:
112
51
  return t if t.tzinfo is None else t.tz_convert("UTC").tz_localize(None)
113
52
 
114
53
 
115
- def station_position(station: str, get=_get) -> tuple[float, float]:
54
+ def station_position(station: str, get=None) -> tuple[float, float]:
116
55
  """(lat, lon) of an NDBC station from its station page."""
117
- html = get(f"{NDBC}/station_page.php?station={station.lower()}") or ""
118
- m = re.search(r"(\d+\.\d+)\s*([NS])\s+(\d+\.\d+)\s*([EW])", html)
119
- if not m:
120
- raise ValueError(f"No position found for NDBC station {station}")
121
- lat = float(m.group(1)) * (1 if m.group(2) == "N" else -1)
122
- lon = float(m.group(3)) * (1 if m.group(4) == "E" else -1)
123
- return lat, lon
56
+ return _ndbc.position(station, get=get)
124
57
 
125
58
 
126
59
  def fetch_product(
127
- station: str, product: str, start: pd.Timestamp, end: pd.Timestamp, get=_get
60
+ station: str, product: str, start: pd.Timestamp, end: pd.Timestamp, get=None
128
61
  ):
129
- """`product` for `station` over [start, end]: (frame, source URLs). Each month is taken from
130
- the first source that has it."""
131
- frames, sources = [], []
132
- for month_start in pd.date_range(start.normalize().replace(day=1), end, freq="MS"):
133
- for url in candidate_urls(
134
- station, product, month_start.year, month_start.month
135
- ):
136
- text = get(url)
137
- if text is None:
138
- continue
139
- df = parse_ndbc_text(text)
140
- df = df[(df.index >= start) & (df.index <= end)]
141
- if len(df):
142
- frames.append(df)
143
- sources.append(url)
144
- break
145
- if not frames:
62
+ """`product` for `station` over [start, end] (naive UTC, both included): (frame indexed by
63
+ naive UTC time, source URLs). Each month is taken from the first source that has it."""
64
+ t0 = _utc_naive(start).tz_localize("UTC")
65
+ t1 = _utc_naive(end).tz_localize("UTC") + pd.Timedelta(minutes=1)
66
+ df = _ndbc.fetch(station, t0, t1, product, get=get)
67
+ sources = list(df.attrs.get("sources", []))
68
+ if df.empty:
146
69
  return pd.DataFrame(), sources
147
- df = pd.concat(frames)
148
- return df[~df.index.duplicated(keep="first")].sort_index(), sources
70
+ df = df.set_index(pd.DatetimeIndex(df.pop("time")).tz_localize(None))
71
+ return df[df.index <= _utc_naive(end)], sources
149
72
 
150
73
 
151
74
  def _list(values) -> list:
@@ -161,7 +84,7 @@ def fetch_ndbc(
161
84
  end_time: str,
162
85
  products: tuple[str, ...] = PRODUCTS,
163
86
  cache_dir: Optional[str] = None,
164
- get=_get,
87
+ get=None,
165
88
  position: Optional[tuple[float, float]] = None,
166
89
  ) -> dict:
167
90
  """Observations at an NDBC station over [start_time, end_time] (UTC) as a JSON-ready dict.
@@ -137,6 +137,75 @@ def test_store_finished_before_restart_is_a_succeeded_job(client):
137
137
  assert body["status"] == "succeeded" and body["result"]["id"] == zarr_id
138
138
 
139
139
 
140
+ def test_registry_rebuilds_succeeded_job_on_request_and_forgets_finished_ones():
141
+ registry = jobs.JobRegistry(workers=1)
142
+ release = threading.Event()
143
+ first = registry.submit("obc_0123456789ab", "ocean", lambda: {"n": 1})
144
+ deadline = time.monotonic() + 5
145
+ while first.status != "succeeded" and time.monotonic() < deadline:
146
+ time.sleep(0.01)
147
+ assert first.status == "succeeded"
148
+ assert registry.submit("obc_0123456789ab", "ocean", lambda: {"n": 2}) is first
149
+
150
+ def slow_build():
151
+ release.wait(5)
152
+ return {"n": 3}
153
+
154
+ rebuilt = registry.submit("obc_0123456789ab", "ocean", slow_build, rebuild=True)
155
+ assert rebuilt is not first
156
+
157
+ # An active job survives forget; a finished one does not.
158
+ registry.forget("obc_0123456789ab")
159
+ registry.forget()
160
+ assert registry.get("obc_0123456789ab") is rebuilt
161
+ release.set()
162
+ while rebuilt.status != "succeeded" and time.monotonic() < deadline:
163
+ time.sleep(0.01)
164
+ registry.forget("obc_0123456789ab")
165
+ assert registry.get("obc_0123456789ab") is None
166
+
167
+
168
+ def _built_obc_job(client, monkeypatch, calls):
169
+ def fake_dispatch(**kwargs):
170
+ calls.append(kwargs)
171
+ _write_store(kwargs["zarr_path"])
172
+ return kwargs["zarr_path"]
173
+
174
+ monkeypatch.setattr(dispatcher, "dispatch_obc_request", fake_dispatch)
175
+ body = client.post("/api/v1/jobs/obc", json=OBC).json()
176
+ assert _wait(client, body["status_url"])["status"] == "succeeded"
177
+ return body
178
+
179
+
180
+ def test_succeeded_job_whose_store_vanished_is_rebuilt(client, monkeypatch):
181
+ import shutil
182
+
183
+ calls = []
184
+ body = _built_obc_job(client, monkeypatch, calls)
185
+ _, zarr_path, _ = main._obc_store(main.OBCRequest(**OBC))
186
+ shutil.rmtree(zarr_path) # removed behind the service's back, registry untouched
187
+
188
+ assert client.get(body["status_url"]).status_code == 404
189
+ again = client.post("/api/v1/jobs/obc", json=OBC)
190
+ assert again.status_code == 202
191
+ done = _wait(client, again.json()["status_url"])
192
+ assert done["status"] == "succeeded" and len(calls) == 2
193
+ assert done["result"]["id"] == body["job_id"]
194
+
195
+
196
+ def test_deleting_a_store_drops_its_job(client, monkeypatch):
197
+ calls = []
198
+ body = _built_obc_job(client, monkeypatch, calls)
199
+ deleted = client.delete(f"/api/v1/cache/dataset/{body['job_id']}")
200
+ assert deleted.status_code == 200
201
+ assert jobs.registry.get(body["job_id"]) is None
202
+
203
+ again = client.post("/api/v1/jobs/obc", json=OBC)
204
+ assert again.status_code == 202
205
+ assert _wait(client, again.json()["status_url"])["status"] == "succeeded"
206
+ assert len(calls) == 2
207
+
208
+
140
209
  def test_unknown_and_invalid_job_ids(client):
141
210
  assert client.get("/api/v1/jobs/obc_000000000000").status_code == 404
142
211
  assert client.get("/api/v1/jobs/not-a-store").status_code == 400
@@ -1000,7 +1000,7 @@ wheels = [
1000
1000
 
1001
1001
  [[package]]
1002
1002
  name = "forcingkit"
1003
- version = "0.15.0"
1003
+ version = "0.16.0"
1004
1004
  source = { editable = "." }
1005
1005
  dependencies = [
1006
1006
  { name = "bottleneck" },
@@ -1062,7 +1062,7 @@ requires-dist = [
1062
1062
  { name = "h5netcdf", specifier = ">=1.8.1" },
1063
1063
  { name = "h5py", specifier = ">=3.16.0" },
1064
1064
  { name = "httpx", specifier = ">=0.28.1" },
1065
- { name = "insitukit", specifier = ">=0.3,<0.4" },
1065
+ { name = "insitukit", specifier = ">=0.4,<0.5" },
1066
1066
  { name = "jinja2", specifier = ">=3.0.0" },
1067
1067
  { name = "matplotlib", specifier = ">=3.10.9" },
1068
1068
  { name = "pandas", specifier = ">=2.2" },
@@ -1612,16 +1612,16 @@ wheels = [
1612
1612
 
1613
1613
  [[package]]
1614
1614
  name = "insitukit"
1615
- version = "0.3.0"
1615
+ version = "0.4.0"
1616
1616
  source = { registry = "https://pypi.org/simple" }
1617
1617
  dependencies = [
1618
1618
  { name = "numpy" },
1619
1619
  { name = "pandas" },
1620
1620
  { name = "requests" },
1621
1621
  ]
1622
- sdist = { url = "https://files.pythonhosted.org/packages/d6/2f/64ab83ca7705afcab426cfe64953dec222e6fc148bd47bf19118d9df9d70/insitukit-0.3.0.tar.gz", hash = "sha256:fa1ed9c1cf89995b6edf4a550f8835e327f352b0df69a756b3185f5dc7b67aa6", size = 99038, upload-time = "2026-11-06T00:48:46.489Z" }
1622
+ sdist = { url = "https://files.pythonhosted.org/packages/52/1c/347ad724e468073a23c7c6c8ebac61f847fcdfd56515b02edf5c8007a624/insitukit-0.4.0.tar.gz", hash = "sha256:cf98f653dd0235a2b7d9b015185a43e5974839eb037ad30d4855acd95166a8ec", size = 104916, upload-time = "2026-11-06T12:24:57.858Z" }
1623
1623
  wheels = [
1624
- { url = "https://files.pythonhosted.org/packages/8a/00/5b0fa89c8a08b99d36533c4849e20f6061a65901693de02806ff3a22f936/insitukit-0.3.0-py3-none-any.whl", hash = "sha256:396df69830c4d921a40c7693c0f24a7e02a86617730acf010cdc4fdf2a6abb35", size = 22860, upload-time = "2026-11-06T00:48:45.069Z" },
1624
+ { url = "https://files.pythonhosted.org/packages/28/55/9e8392e4ead1704ee9b279b7cdfe2b610554990172cad717f37a0d8a415a/insitukit-0.4.0-py3-none-any.whl", hash = "sha256:b1222ca7b90d335139b8d96445cd3c600e9678aef6a564e122f28b3986b9c4e3", size = 27409, upload-time = "2026-11-06T12:24:56.3Z" },
1625
1625
  ]
1626
1626
 
1627
1627
  [[package]]
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes