forcingkit 0.13.0__tar.gz → 0.15.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. {forcingkit-0.13.0 → forcingkit-0.15.0}/PKG-INFO +3 -1
  2. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/conf.py +1 -1
  3. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/fetchers.rst +8 -2
  4. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/roadmap.rst +4 -2
  5. {forcingkit-0.13.0 → forcingkit-0.15.0}/pyproject.toml +4 -1
  6. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/main.py +10 -0
  7. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/erddap.py +54 -95
  8. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/noaa.py +99 -99
  9. {forcingkit-0.13.0 → forcingkit-0.15.0}/static/index.html +5 -1
  10. {forcingkit-0.13.0 → forcingkit-0.15.0}/static/preview3d.js +21 -11
  11. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_erddap.py +7 -7
  12. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_noaa_currents.py +28 -39
  13. forcingkit-0.15.0/tests/unit/test_noaa_datum.py +69 -0
  14. {forcingkit-0.13.0 → forcingkit-0.15.0}/uv.lock +19 -1
  15. forcingkit-0.13.0/tests/unit/test_noaa_datum.py +0 -66
  16. {forcingkit-0.13.0 → forcingkit-0.15.0}/.agents/forcingkit.md +0 -0
  17. {forcingkit-0.13.0 → forcingkit-0.15.0}/.claude/CLAUDE.md +0 -0
  18. {forcingkit-0.13.0 → forcingkit-0.15.0}/.env.template +0 -0
  19. {forcingkit-0.13.0 → forcingkit-0.15.0}/.github/copilot-instructions.md +0 -0
  20. {forcingkit-0.13.0 → forcingkit-0.15.0}/.github/workflows/docs.yml +0 -0
  21. {forcingkit-0.13.0 → forcingkit-0.15.0}/.github/workflows/publish-client.yml +0 -0
  22. {forcingkit-0.13.0 → forcingkit-0.15.0}/.github/workflows/publish.yml +0 -0
  23. {forcingkit-0.13.0 → forcingkit-0.15.0}/.github/workflows/tests.yml +0 -0
  24. {forcingkit-0.13.0 → forcingkit-0.15.0}/.gitignore +0 -0
  25. {forcingkit-0.13.0 → forcingkit-0.15.0}/.markdownlint.yaml +0 -0
  26. {forcingkit-0.13.0 → forcingkit-0.15.0}/.pre-commit-config.yaml +0 -0
  27. {forcingkit-0.13.0 → forcingkit-0.15.0}/.python-version +0 -0
  28. {forcingkit-0.13.0 → forcingkit-0.15.0}/AGENTS.md +0 -0
  29. {forcingkit-0.13.0 → forcingkit-0.15.0}/CONTRIBUTING.md +0 -0
  30. {forcingkit-0.13.0 → forcingkit-0.15.0}/Dockerfile +0 -0
  31. {forcingkit-0.13.0 → forcingkit-0.15.0}/LICENSE +0 -0
  32. {forcingkit-0.13.0 → forcingkit-0.15.0}/README.md +0 -0
  33. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/LICENSE +0 -0
  34. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/README.md +0 -0
  35. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/pyproject.toml +0 -0
  36. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/src/forcingkit_client/__init__.py +0 -0
  37. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/src/forcingkit_client/client.py +0 -0
  38. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/src/forcingkit_client/schema.py +0 -0
  39. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/src/forcingkit_client/store.py +0 -0
  40. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/src/forcingkit_client/trajectory.py +0 -0
  41. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/tests/conftest.py +0 -0
  42. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/tests/test_client.py +0 -0
  43. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/tests/test_schema.py +0 -0
  44. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/tests/test_trajectory.py +0 -0
  45. {forcingkit-0.13.0 → forcingkit-0.15.0}/client/uv.lock +0 -0
  46. {forcingkit-0.13.0 → forcingkit-0.15.0}/docker-compose.yml +0 -0
  47. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/Makefile +0 -0
  48. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/make.bat +0 -0
  49. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/requirements-docs.txt +0 -0
  50. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/_extra/CNAME +0 -0
  51. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/_static/custom.css +0 -0
  52. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  53. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/atmospheric_forcing.rst +0 -0
  54. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/client.rst +0 -0
  55. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/hycom.rst +0 -0
  56. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/index.rst +0 -0
  57. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/layered_mesh.rst +0 -0
  58. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/mesh_frames.rst +0 -0
  59. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/necofs.rst +0 -0
  60. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/nyofs.rst +0 -0
  61. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/rivers.rst +0 -0
  62. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/sources.rst +0 -0
  63. {forcingkit-0.13.0 → forcingkit-0.15.0}/docs/source/waves.rst +0 -0
  64. {forcingkit-0.13.0 → forcingkit-0.15.0}/main.py +0 -0
  65. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/__init__.py +0 -0
  66. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/__init__.py +0 -0
  67. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/jobs.py +0 -0
  68. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  69. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  70. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/removed.py +0 -0
  71. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/stores.py +0 -0
  72. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/viewer.py +0 -0
  73. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/forcingkit_serve/stac.py +0 -0
  74. {forcingkit-0.13.0 → forcingkit-0.15.0}/service/run_server.py +0 -0
  75. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/__init__.py +0 -0
  76. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  77. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/dispatcher.py +0 -0
  78. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
  79. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/dbofs.py +0 -0
  80. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hrrr.py +0 -0
  81. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  82. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  83. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  84. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hycom.py +0 -0
  85. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hydrography.py +0 -0
  86. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/ndbc.py +0 -0
  87. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/necofs.py +0 -0
  88. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/nwps.py +0 -0
  89. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/nyofs.py +0 -0
  90. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/usgs.py +0 -0
  91. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  92. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/layered_mesh.py +0 -0
  93. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/mesh_frames.py +0 -0
  94. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
  95. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  96. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  97. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  98. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/render.py +0 -0
  99. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/settings.py +0 -0
  100. {forcingkit-0.13.0 → forcingkit-0.15.0}/src/forcingkit/zarr_stream.py +0 -0
  101. {forcingkit-0.13.0 → forcingkit-0.15.0}/static/app.js +0 -0
  102. {forcingkit-0.13.0 → forcingkit-0.15.0}/static/favicon.ico +0 -0
  103. {forcingkit-0.13.0 → forcingkit-0.15.0}/static/favicon.svg +0 -0
  104. {forcingkit-0.13.0 → forcingkit-0.15.0}/static/logo.svg +0 -0
  105. {forcingkit-0.13.0 → forcingkit-0.15.0}/static/styles.css +0 -0
  106. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/__init__.py +0 -0
  107. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_auth.py +0 -0
  108. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_erddap_fetch.py +0 -0
  109. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
  110. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_necofs_volume_live.py +0 -0
  111. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_nwps_nomads.py +0 -0
  112. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  113. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_obc_offshore_nj.py +0 -0
  114. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_ofs_archive_paths.py +0 -0
  115. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
  116. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/scripts/inspect_dem.py +0 -0
  117. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/scripts/inspect_grib.py +0 -0
  118. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/scripts/inspect_zarr.py +0 -0
  119. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_coastwatch.py +0 -0
  120. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_dbofs.py +0 -0
  121. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_dispatcher.py +0 -0
  122. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_hrrr_atm.py +0 -0
  123. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_hrrr_idx.py +0 -0
  124. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_hrrr_levels.py +0 -0
  125. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_hrrr_points.py +0 -0
  126. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_hycom.py +0 -0
  127. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_jobs.py +0 -0
  128. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_layered_mesh.py +0 -0
  129. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_main.py +0 -0
  130. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_mesh_frames.py +0 -0
  131. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_ndbc.py +0 -0
  132. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_necofs_hindcast.py +0 -0
  133. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
  134. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_necofs_parent.py +0 -0
  135. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_necofs_volume.py +0 -0
  136. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_nwps.py +0 -0
  137. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_nyofs.py +0 -0
  138. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_obc_donor_report.py +0 -0
  139. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_obc_pipeline.py +0 -0
  140. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_removed_routes.py +0 -0
  141. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_settings.py +0 -0
  142. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_usgs.py +0 -0
  143. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_usgs_rivers.py +0 -0
  144. {forcingkit-0.13.0 → forcingkit-0.15.0}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.13.0
3
+ Version: 0.15.0
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -225,8 +225,10 @@ Requires-Dist: fastapi>=0.135.1
225
225
  Requires-Dist: h5netcdf>=1.8.1
226
226
  Requires-Dist: h5py>=3.16.0
227
227
  Requires-Dist: httpx>=0.28.1
228
+ Requires-Dist: insitukit<0.4,>=0.3
228
229
  Requires-Dist: jinja2>=3.0.0
229
230
  Requires-Dist: matplotlib>=3.10.9
231
+ Requires-Dist: pandas>=2.2
230
232
  Requires-Dist: pillow>=10.0
231
233
  Requires-Dist: pydantic>=2.12.5
232
234
  Requires-Dist: pydap>=3.5.8
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.13.0"
12
+ release = "0.15.0"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -146,7 +146,11 @@ These serve validation rather than forcing:
146
146
 
147
147
  - **CO-OPS** water level (``/api/v1/tide``), from the `NOAA CO-OPS data API
148
148
  <https://api.tidesandcurrents.noaa.gov/api/prod/>`__, with a datum
149
- fallback to MSL where a station has no NAVD88 datum.
149
+ fallback to MSL where a station has no NAVD88 datum. Requests go through
150
+ `insitukit <https://github.com/lhzn-io/insitukit>`__, which keeps them at least 1 s apart,
151
+ splits windows longer than the API's 31-day limit, and retries server errors and throttling;
152
+ ``end_time`` is inclusive, as in the API. Current predictions (``/api/v1/currents``) and the
153
+ current-station list (``/api/v1/currents/stations``) use the same client.
150
154
  - **NDBC** buoy meteorology and ADCP profiles (``/api/v1/ndbc``), from the `NOAA National Data
151
155
  Buoy Center <https://www.ndbc.noaa.gov/>`__.
152
156
  - **UConn ERDDAP** water-column profiles (``/api/v1/telemetry/station`` and ``/bbox``), from the
@@ -159,7 +163,9 @@ These serve validation rather than forcing:
159
163
  has a 10 s connect and 60 s read timeout (``FORCINGKIT_ERDDAP_CONNECT_TIMEOUT``,
160
164
  ``FORCINGKIT_ERDDAP_READ_TIMEOUT``). A host that fails to connect, or a dataset that times out
161
165
  or errors, is skipped for 300 s (``FORCINGKIT_ERDDAP_FAILURE_TTL``, 0 to turn off), and the
162
- response's ``skipped`` lists each dataset left out, with the reason.
166
+ response's ``skipped`` lists each dataset left out, with the reason. A station's three datasets
167
+ are read at once, through `insitukit <https://github.com/lhzn-io/insitukit>`__'s ERDDAP client
168
+ (one attempt each, starting at least 0.2 s apart).
163
169
 
164
170
  NOAA OFS archive locations
165
171
  --------------------------
@@ -123,8 +123,10 @@ Planned work
123
123
  - Modules in ``forcingkit.fetchers`` are named for their source (``necofs``, ``hycom``,
124
124
  ``ndbc``, ``coastwatch``, ``usgs``), and a model-ready delivery for the source and what it
125
125
  delivers (``hrrr_atmosphere``, ``usgs_rivers``). Three predate the convention:
126
- ``noaa.py`` serves CO-OPS water levels and current predictions (``coops.py``),
127
- ``erddap.py`` serves the UConn profile stations (named for the protocol, not the source),
126
+ ``noaa.py`` serves CO-OPS water levels and current predictions (``coops.py``; since
127
+ 0.14.0 a thin layer over ``insitukit.coops``, which may replace it outright),
128
+ ``erddap.py`` serves the UConn profile stations (named for the protocol, not the source;
129
+ since 0.15.0 its requests go through ``insitukit.erddap``),
128
130
  and ``hydrography.py`` finds heads of tide from OpenStreetMap (``osm.py`` or similar).
129
131
  Renaming them breaks imports from earlier releases, so it belongs in a release that says
130
132
  so, with the old names kept as deprecated aliases for one release.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.13.0"
7
+ version = "0.15.0"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -26,6 +26,9 @@ dependencies = [
26
26
  "eccodeslib>=2.46.0",
27
27
  "fastapi>=0.135.1",
28
28
  "httpx>=0.28.1",
29
+ # NOAA CO-OPS (fetchers/noaa.py) and the UConn ERDDAP profiles (fetchers/erddap.py).
30
+ "insitukit>=0.3,<0.4",
31
+ "pandas>=2.2",
29
32
  "pydantic>=2.12.5",
30
33
  "jinja2>=3.0.0",
31
34
  "pydap>=3.5.8",
@@ -29,6 +29,16 @@ from forcingkit_serve.routers import ( # noqa: E402
29
29
  stores,
30
30
  )
31
31
  from forcingkit.fetchers.noaa import fetch_noaa_tide_data # noqa: E402
32
+ from importlib.metadata import version as _package_version # noqa: E402
33
+
34
+ import insitukit # noqa: E402
35
+
36
+ # Every CO-OPS request (through insitukit) names this service. Set here, not in the library, so a
37
+ # program that imports forcingkit keeps its own identity.
38
+ insitukit.identify(
39
+ f"forcingkit/{_package_version('forcingkit')} (+https://github.com/lhzn-io/forcingkit)",
40
+ application="forcingkit",
41
+ )
32
42
 
33
43
  # Configure Logging
34
44
  log_dir = Path("logs")
@@ -1,15 +1,12 @@
1
- import io
2
1
  import logging
3
2
  import math
4
3
  import os
5
- import threading
6
- import time
7
4
  from concurrent.futures import ThreadPoolExecutor
5
+ from datetime import datetime, timezone
8
6
  from typing import Optional
9
- from urllib.parse import urlsplit
10
7
 
11
8
  import pandas as pd
12
- import requests
9
+ from insitukit.erddap import DatasetMissing, Erddap, FailureCache, Unavailable
13
10
 
14
11
  from forcingkit import settings
15
12
 
@@ -17,114 +14,74 @@ logger = logging.getLogger(__name__)
17
14
 
18
15
  BASE_ERDDAP_URL = "http://merlin.dms.uconn.edu:8080/erddap"
19
16
 
17
+ # Hosts and datasets that failed recently, shared by every request this process serves. A host is
18
+ # recorded when a connect fails (refused, unreachable or timed out), so the other datasets on it, in
19
+ # this request and the next few, are skipped instead of each waiting out the connect timeout. A
20
+ # dataset is recorded when its read times out or the server returns a 5xx. The lifetime is
21
+ # settings.erddap_failure_ttl(); 0 records nothing.
22
+ _failures = FailureCache()
20
23
 
21
- class ErddapUnavailable(Exception):
22
- """A dataset was not read, because its host or the dataset failed now or recently."""
23
24
 
24
-
25
- class _FailureCache:
26
- """Hosts and datasets that failed recently, skipped until their entry expires.
27
-
28
- A host is recorded when a connect fails (refused, unreachable or timed out), so the other
29
- datasets on it, in this request and the next few, are skipped instead of each waiting out the
30
- connect timeout. A dataset is recorded when its read times out or the server returns a 5xx.
31
- The lifetime is settings.erddap_failure_ttl(); 0 records nothing.
32
- """
33
-
34
- def __init__(self) -> None:
35
- self._entries: dict[str, tuple[float, str]] = {}
36
- self._lock = threading.Lock()
37
-
38
- def record(self, key: str, reason: str) -> None:
39
- ttl = settings.erddap_failure_ttl()
40
- if ttl <= 0:
41
- return
42
- with self._lock:
43
- self._entries[key] = (time.monotonic() + ttl, reason)
44
-
45
- def check(self, key: str) -> Optional[str]:
46
- """The recorded reason while `key` is still failed, else None."""
47
- with self._lock:
48
- entry = self._entries.get(key)
49
- if entry is None:
50
- return None
51
- expires, reason = entry
52
- if time.monotonic() >= expires:
53
- del self._entries[key]
54
- return None
55
- return reason
56
-
57
- def clear(self) -> None:
58
- with self._lock:
59
- self._entries.clear()
25
+ def clear_failure_cache() -> None:
26
+ """Forget every recorded host and dataset failure."""
27
+ _failures.clear()
60
28
 
61
29
 
62
- _failures = _FailureCache()
30
+ def _server() -> Erddap:
31
+ """An insitukit client for the UConn server, built per call so the settings are read each time:
32
+ one attempt per dataset, the configured timeouts, and the shared failure cache."""
33
+ connect, read = settings.erddap_timeouts()
34
+ return Erddap(
35
+ BASE_ERDDAP_URL,
36
+ budget=None,
37
+ min_interval=0.2,
38
+ retries=1,
39
+ connect_timeout=connect,
40
+ timeout=read,
41
+ failure_ttl=settings.erddap_failure_ttl(),
42
+ failures=_failures,
43
+ fmt="csvp",
44
+ )
63
45
 
64
46
 
65
- def clear_failure_cache() -> None:
66
- """Forget every recorded host and dataset failure."""
67
- _failures.clear()
47
+ def _utc(stamp: str) -> datetime:
48
+ ts = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
49
+ return (
50
+ ts.replace(tzinfo=timezone.utc)
51
+ if ts.tzinfo is None
52
+ else ts.astimezone(timezone.utc)
53
+ )
68
54
 
69
55
 
70
56
  def _read_dataset(
71
- dataset_name: str, start_time: str, end_time: str
57
+ server: Erddap, dataset_name: str, start_time: str, end_time: str
72
58
  ) -> Optional[pd.DataFrame]:
73
59
  """One dataset's temperature series from ERDDAP, indexed by UTC time.
74
60
 
75
- Returns None when ERDDAP has no rows for the window (HTTP 404). Raises ErddapUnavailable when
76
- the host or dataset is negatively cached, or when this read fails.
61
+ Returns None when ERDDAP has no rows for the window (any HTTP 404). Raises
62
+ insitukit.erddap.Unavailable when the host or dataset is negatively cached or this read fails,
63
+ and ValueError when the response cannot be parsed.
77
64
  """
78
- host = urlsplit(BASE_ERDDAP_URL).netloc
79
- host_key = f"host:{host}"
80
- dataset_key = f"dataset:{dataset_name}"
81
- for key in (host_key, dataset_key):
82
- reason = _failures.check(key)
83
- if reason is not None:
84
- raise ErddapUnavailable(f"skipped after a recent failure: {reason}")
85
-
86
- query_url = f"{BASE_ERDDAP_URL}/tabledap/{dataset_name}.csvp?time,sea_water_temperature&time%3E={start_time}&time%3C={end_time}"
87
65
  logger.info(
88
66
  f"Fetching ERDDAP data for {dataset_name} ({start_time} to {end_time})..."
89
67
  )
90
68
  try:
91
- response = requests.get(query_url, timeout=settings.erddap_timeouts())
92
- except requests.ConnectionError as e:
93
- # Includes ConnectTimeout: the host is unreachable, so skip it for a while.
94
- reason = f"{host} unreachable ({type(e).__name__})"
95
- _failures.record(host_key, reason)
96
- raise ErddapUnavailable(reason) from e
97
- except requests.Timeout as e:
98
- reason = f"{dataset_name} read timed out"
99
- _failures.record(dataset_key, reason)
100
- raise ErddapUnavailable(reason) from e
101
- except requests.RequestException as e:
102
- raise ErddapUnavailable(
103
- f"{dataset_name} request failed ({type(e).__name__})"
104
- ) from e
105
-
106
- if response.status_code == 404:
107
- # ERDDAP answers 404 when the query matches no rows.
69
+ df = server.fetch(
70
+ dataset_name,
71
+ _utc(start_time),
72
+ _utc(end_time),
73
+ ["sea_water_temperature"],
74
+ )
75
+ except DatasetMissing:
108
76
  logger.info(f"ERDDAP has no rows for {dataset_name} in the window")
109
77
  return None
110
- if response.status_code >= 500:
111
- reason = f"{dataset_name} returned HTTP {response.status_code}"
112
- _failures.record(dataset_key, reason)
113
- raise ErddapUnavailable(reason)
114
- if response.status_code != 200:
115
- raise ErddapUnavailable(f"{dataset_name} returned HTTP {response.status_code}")
116
-
117
- try:
118
- df = pd.read_csv(io.StringIO(response.text))
119
- if df.empty:
120
- return None
121
- # The headers are like "time (UTC)" and "sea_water_temperature (celsius)".
122
- df.columns = ["time", "sea_water_temperature"]
123
- df = df.set_index("time")
124
- df.index = pd.to_datetime(df.index, utc=True)
78
+ except Unavailable:
79
+ raise
125
80
  except Exception as e:
126
- raise ErddapUnavailable(f"{dataset_name} could not be parsed ({e})") from e
127
- return df
81
+ raise ValueError(f"{dataset_name} could not be parsed ({e})") from e
82
+ if df.empty:
83
+ return None
84
+ return df.set_index("time")
128
85
 
129
86
 
130
87
  def fetch_erddap_station_profiles(
@@ -169,6 +126,8 @@ def fetch_erddap_station_profiles(
169
126
  if station_upper == "ARTG":
170
127
  datasets["bottom"] = "ARTG_WQ_BTM_1"
171
128
 
129
+ server = _server()
130
+
172
131
  def load(
173
132
  level: str, dataset_name: str
174
133
  ) -> tuple[Optional[pd.DataFrame], Optional[dict]]:
@@ -181,8 +140,8 @@ def fetch_erddap_station_profiles(
181
140
  logger.info(f"Cache hit for {dataset_name}: {cache_path}")
182
141
  return pd.read_csv(cache_path, index_col=0, parse_dates=True), None
183
142
  try:
184
- df = _read_dataset(dataset_name, start_time, end_time)
185
- except ErddapUnavailable as e:
143
+ df = _read_dataset(server, dataset_name, start_time, end_time)
144
+ except (Unavailable, ValueError) as e:
186
145
  logger.error(f"Failed fetching or parsing ERDDAP for {dataset_name}: {e}")
187
146
  return None, {
188
147
  "station": station_upper,
@@ -1,8 +1,18 @@
1
- import os
2
- import logging
3
- import requests
1
+ """NOAA CO-OPS water level and tidal current predictions, through insitukit.
2
+
3
+ insitukit's CO-OPS client paces requests (1 s apart, across threads), splits long windows to the API's
4
+ range limits, retries server errors and throttling, and names the caller (``insitukit.identify``, set by
5
+ the service at start-up). This module keeps forcingkit's call signatures, caching and return shapes.
6
+ """
7
+
4
8
  import json
5
- from datetime import datetime
9
+ import logging
10
+ import os
11
+ from datetime import datetime, timedelta, timezone
12
+
13
+ import pandas as pd
14
+ from insitukit import coops
15
+
6
16
  from forcingkit import settings
7
17
 
8
18
  logger = logging.getLogger(__name__)
@@ -12,6 +22,25 @@ logger = logging.getLogger(__name__)
12
22
  TIDE_DATUMS = ("NAVD", "MSL")
13
23
 
14
24
 
25
+ def _utc(stamp: str) -> datetime:
26
+ """An ISO 8601 time, read as UTC (a trailing Z or an offset is accepted)."""
27
+ ts = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
28
+ return (
29
+ ts.replace(tzinfo=timezone.utc)
30
+ if ts.tzinfo is None
31
+ else ts.astimezone(timezone.utc)
32
+ )
33
+
34
+
35
+ def _through(stamp: str) -> datetime:
36
+ """The exclusive end for an inclusive ``end_time``: CO-OPS's own end_date includes its minute."""
37
+ return _utc(stamp) + timedelta(minutes=1)
38
+
39
+
40
+ def _t(ts: pd.Timestamp) -> str:
41
+ return ts.strftime("%Y-%m-%d %H:%M")
42
+
43
+
15
44
  def fetch_noaa_tide_data(
16
45
  station_id: str,
17
46
  start_time: str,
@@ -23,13 +52,18 @@ def fetch_noaa_tide_data(
23
52
  cache_bust: bool = False,
24
53
  ) -> dict:
25
54
  """
26
- Fetches water level data from NOAA CO-OPS API for a specific station and time window.
27
- Supports caching to avoid redundant API calls.
55
+ Fetches water level data from NOAA CO-OPS for a station over [start_time, end_time] (UTC),
56
+ in metres, on NAVD88 where the station has it and MSL otherwise. Supports caching to avoid
57
+ redundant API calls.
58
+
59
+ Returns the shape of a CO-OPS response: ``{"metadata": {"id", "name", "lat", "lon", "datum"},
60
+ "data": [{"t": "YYYY-MM-DD HH:MM", "v": "0.123"}, ...]}``, with ``lat``, ``lon`` and ``v`` as
61
+ strings (``v`` empty for a missing reading). Raises RuntimeError when the window has no data.
28
62
 
29
63
  Args:
30
64
  station_id: NOAA station ID (e.g., '8516945' for Kings Point)
31
65
  start_time: ISO8601 string (YYYY-MM-DDTHH:MM:SSZ)
32
- end_time: ISO8601 string (YYYY-MM-DDTHH:MM:SSZ)
66
+ end_time: ISO8601 string (YYYY-MM-DDTHH:MM:SSZ), inclusive
33
67
  cache_dir: Local directory for caching JSON responses
34
68
  """
35
69
  os.makedirs(cache_dir, exist_ok=True)
@@ -46,52 +80,41 @@ def fetch_noaa_tide_data(
46
80
  logger.info(
47
81
  f"Fetching NOAA tide data for station {station_id} ({start_time} to {end_time})..."
48
82
  )
49
-
50
- # NOAA API expects yyyyMMdd HH:mm
51
- # Note: We assume the input strings are ISO8601 UTC
52
- s_dt = datetime.fromisoformat(start_time.replace("Z", ""))
53
- e_dt = datetime.fromisoformat(end_time.replace("Z", ""))
54
-
55
- begin_str = s_dt.strftime("%Y%m%d %H:%M")
56
- end_str = e_dt.strftime("%Y%m%d %H:%M")
57
-
58
- url = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter"
59
- params = {
60
- "begin_date": begin_str,
61
- "end_date": end_str,
62
- "station": station_id,
63
- "product": "water_level",
64
- "units": "metric",
65
- "time_zone": "gmt",
66
- "format": "json",
67
- "application": "forcingkit",
68
- }
83
+ start, end = _utc(start_time), _through(end_time)
69
84
 
70
85
  try:
71
86
  # NAVD88 where the station has it, so stations share a datum; otherwise MSL (New Haven,
72
87
  # 8465705, publishes only tidal datums). The datum used is recorded in the metadata.
73
88
  for datum in TIDE_DATUMS:
74
- response = requests.get(url, params={**params, "datum": datum}, timeout=30)
75
- data = response.json() if response.content else {}
76
- message = (
77
- data.get("error", {}).get("message", "")
78
- if isinstance(data, dict)
79
- else ""
80
- )
81
- if "datum" in message.lower() and datum != TIDE_DATUMS[-1]:
89
+ try:
90
+ df = coops.fetch("water_level", station_id, start, end, datum=datum)
91
+ break
92
+ except coops.DatumUnavailable:
93
+ if datum == TIDE_DATUMS[-1]:
94
+ raise
82
95
  logger.info(
83
96
  f"NOAA station {station_id} has no {datum} datum; trying the next"
84
97
  )
85
- continue
86
- response.raise_for_status()
87
- break
88
-
89
- if "error" in data:
90
- logger.warning(
91
- f"NOAA API returned error: {data['error'].get('message', 'Unknown error')}"
98
+ if df.empty:
99
+ raise RuntimeError(
100
+ f"NOAA API Error: No data was found for {station_id} "
101
+ f"({start_time} to {end_time})"
92
102
  )
93
- raise RuntimeError(f"NOAA API Error: {data['error'].get('message')}")
94
- data.setdefault("metadata", {})["datum"] = datum
103
+ info = coops.metadata(station_id, expand="")
104
+ data = {
105
+ "metadata": {
106
+ "id": station_id,
107
+ "name": info.get("name"),
108
+ "lat": str(info["lat"]),
109
+ "lon": str(info["lng"]),
110
+ "datum": datum,
111
+ },
112
+ "data": [
113
+ # Written as CO-OPS writes it: 3 decimals at most, no trailing zeros.
114
+ {"t": _t(t), "v": "" if pd.isna(v) else str(round(float(v), 3))}
115
+ for t, v in zip(df["time"], df["water_level"])
116
+ ],
117
+ }
95
118
 
96
119
  # Cache the successful response
97
120
  with open(cache_path, "w") as f:
@@ -104,25 +127,19 @@ def fetch_noaa_tide_data(
104
127
  raise e
105
128
 
106
129
 
107
- CURRENTS_URL = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter"
108
- STATION_URL = (
109
- "https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations/{station}.json"
110
- )
111
- STATIONS_URL = "https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations.json"
112
-
113
-
114
130
  def fetch_noaa_current_stations(
115
131
  bbox: list[float],
116
132
  cache_dir: str = os.path.join(settings.cache_dir(), "noaa"),
117
133
  cache_bust: bool = False,
118
- get=requests.get,
134
+ get=None,
119
135
  ) -> list[dict]:
120
136
  """Every harmonic CO-OPS current-prediction station and depth bin inside ``bbox``
121
137
  ([min_lon, min_lat, max_lon, max_lat]).
122
138
 
123
139
  Returns one entry per station-bin: ``{"id", "name", "lat", "lon", "bin", "depth_m"}``.
124
140
  Harmonic stations (type H) give a predicted series; subordinate (S) and weak or variable (W)
125
- entries are left out. The full station list is cached for a day.
141
+ entries are left out. The full station list is cached for a day. ``get`` is ignored (kept for
142
+ callers of earlier releases; requests go through insitukit).
126
143
  """
127
144
  os.makedirs(cache_dir, exist_ok=True)
128
145
  cache_path = os.path.join(cache_dir, "currentpredictions_stations.json")
@@ -133,13 +150,7 @@ def fetch_noaa_current_stations(
133
150
  with open(cache_path) as f:
134
151
  stations = json.load(f)
135
152
  else:
136
- response = get(
137
- STATIONS_URL,
138
- params={"type": "currentpredictions", "units": "metric"},
139
- timeout=60,
140
- )
141
- response.raise_for_status()
142
- stations = response.json().get("stations", [])
153
+ stations = coops.stations("currentpredictions", units="metric")
143
154
  with open(cache_path, "w") as f:
144
155
  json.dump(stations, f)
145
156
 
@@ -172,16 +183,18 @@ def fetch_noaa_current_predictions(
172
183
  interval_minutes: int = 30,
173
184
  cache_dir: str = os.path.join(settings.cache_dir(), "noaa"),
174
185
  cache_bust: bool = False,
175
- get=requests.get,
186
+ get=None,
176
187
  ) -> dict:
177
- """Harmonic current predictions for one CO-OPS station and depth bin.
188
+ """Harmonic current predictions for one CO-OPS station and depth bin over
189
+ [start_time, end_time] (UTC).
178
190
 
179
191
  Returns ``{"metadata": {...}, "data": [{"t": "YYYY-MM-DD HH:MM", "v": m/s}, ...]}``: the
180
192
  predicted velocity along the station's flood-ebb axis, positive on the flood, in m/s, every
181
193
  ``interval_minutes``. The metadata carries the station's position, the bin depth (metres
182
194
  below the surface), and the mean flood and ebb directions (degrees true, toward which the
183
195
  water flows). Only harmonic stations give a series; subordinate stations give slack and
184
- maximum times only and are refused.
196
+ maximum times only and are refused. ``get`` is ignored (kept for callers of earlier
197
+ releases; requests go through insitukit).
185
198
  """
186
199
  os.makedirs(cache_dir, exist_ok=True)
187
200
  key = f"currents_{station_id}_b{bin_number}_{start_time}_{end_time}_{interval_minutes}"
@@ -190,40 +203,26 @@ def fetch_noaa_current_predictions(
190
203
  with open(cache_path) as f:
191
204
  return json.load(f)
192
205
 
193
- s_dt = datetime.fromisoformat(start_time.replace("Z", ""))
194
- e_dt = datetime.fromisoformat(end_time.replace("Z", ""))
195
- params = {
196
- "product": "currents_predictions",
197
- "station": station_id,
198
- "bin": str(bin_number),
199
- "begin_date": s_dt.strftime("%Y%m%d %H:%M"),
200
- "end_date": e_dt.strftime("%Y%m%d %H:%M"),
201
- "interval": str(interval_minutes),
202
- "vel_type": "default",
203
- "units": "metric",
204
- "time_zone": "gmt",
205
- "format": "json",
206
- "application": "lhzn_forcingkit",
207
- }
208
- response = get(CURRENTS_URL, params=params, timeout=30)
209
- response.raise_for_status()
210
- payload = response.json()
211
- if "error" in payload:
212
- raise RuntimeError(f"NOAA API Error: {payload['error'].get('message')}")
213
- rows = payload.get("current_predictions", {}).get("cp", [])
214
- if not rows:
215
- raise RuntimeError(f"No current predictions for {station_id} bin {bin_number}")
216
- if any("Type" in r for r in rows):
217
- raise RuntimeError(
218
- f"{station_id} is a subordinate station (slack and maximum times only); "
219
- "a harmonic station is needed for a time series"
206
+ try:
207
+ df = coops.current_predictions(
208
+ station_id,
209
+ _utc(start_time),
210
+ _through(end_time),
211
+ bin=int(bin_number),
212
+ interval=int(interval_minutes),
213
+ vel_type="default",
220
214
  )
215
+ except coops.CoopsError as exc:
216
+ if "subordinate" in str(exc):
217
+ raise RuntimeError(
218
+ f"{station_id} is a subordinate station (slack and maximum times only); "
219
+ "a harmonic station is needed for a time series"
220
+ ) from exc
221
+ raise RuntimeError(f"NOAA API Error: {exc}") from exc
222
+ if df.empty:
223
+ raise RuntimeError(f"No current predictions for {station_id} bin {bin_number}")
221
224
 
222
- station = get(STATION_URL.format(station=station_id), params={}, timeout=30)
223
- station.raise_for_status()
224
- info = station.json().get("stations", [{}])[0]
225
-
226
- first = rows[0]
225
+ info = coops.metadata(station_id, expand="")
227
226
  data = {
228
227
  "metadata": {
229
228
  "id": station_id,
@@ -231,13 +230,14 @@ def fetch_noaa_current_predictions(
231
230
  "lat": float(info["lat"]),
232
231
  "lon": float(info["lng"]),
233
232
  "bin": int(bin_number),
234
- "depth_m": float(first["Depth"]),
235
- "flood_dir_deg": float(first["meanFloodDir"]),
236
- "ebb_dir_deg": float(first["meanEbbDir"]),
233
+ "depth_m": df.attrs["depth_m"],
234
+ "flood_dir_deg": df.attrs["flood_dir_deg"],
235
+ "ebb_dir_deg": df.attrs["ebb_dir_deg"],
237
236
  "units": "m/s along the flood axis, positive on the flood",
238
237
  },
239
238
  "data": [
240
- {"t": r["Time"], "v": float(r["Velocity_Major"]) / 100.0} for r in rows
239
+ {"t": _t(t), "v": float(v)}
240
+ for t, v in zip(df["time"], df["velocity_major"])
241
241
  ],
242
242
  }
243
243
  with open(cache_path, "w") as f:
@@ -137,7 +137,11 @@
137
137
 
138
138
  <script defer src="app.js"></script>
139
139
  <script type="importmap">
140
- { "imports": { "three": "https://cdn.jsdelivr.net/npm/three@0.170.0/build/three.module.js", "three/addons/": "https://cdn.jsdelivr.net/npm/three@0.170.0/examples/jsm/" } }
140
+ { "imports": {
141
+ "three": "https://cdn.jsdelivr.net/npm/three@0.170.0/build/three.module.js",
142
+ "camera-controls": "https://cdn.jsdelivr.net/npm/camera-controls@3.1.2/dist/camera-controls.module.min.js",
143
+ "vantagekit/three": "https://cdn.jsdelivr.net/gh/lhzn-io/vantagekit@v0.3.2/vantagekit-three.js"
144
+ } }
141
145
  </script>
142
146
  <script type="module" src="preview3d.js"></script>
143
147
  </body>