forcingkit 0.7.1__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. {forcingkit-0.7.1 → forcingkit-0.9.0}/PKG-INFO +2 -1
  2. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/conf.py +1 -1
  3. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/fetchers.rst +8 -0
  4. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/index.rst +1 -0
  5. forcingkit-0.9.0/docs/source/mesh_frames.rst +89 -0
  6. {forcingkit-0.7.1 → forcingkit-0.9.0}/pyproject.toml +2 -1
  7. {forcingkit-0.7.1 → forcingkit-0.9.0}/service/forcingkit_serve/main.py +28 -7
  8. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/dispatcher.py +10 -1
  9. forcingkit-0.9.0/src/forcingkit/fetchers/erddap.py +305 -0
  10. forcingkit-0.9.0/src/forcingkit/mesh_frames.py +228 -0
  11. forcingkit-0.9.0/src/forcingkit/ramps/currents_lis.csv +70 -0
  12. forcingkit-0.9.0/src/forcingkit/ramps/currents_lis_dark.csv +69 -0
  13. forcingkit-0.9.0/src/forcingkit/render.py +487 -0
  14. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/settings.py +14 -0
  15. forcingkit-0.9.0/tests/integration/test_mesh_frames_necofs.py +32 -0
  16. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_dispatcher.py +1 -0
  17. forcingkit-0.9.0/tests/unit/test_erddap.py +427 -0
  18. forcingkit-0.9.0/tests/unit/test_mesh_frames.py +383 -0
  19. {forcingkit-0.7.1 → forcingkit-0.9.0}/uv.lock +3 -1
  20. forcingkit-0.7.1/src/forcingkit/fetchers/erddap.py +0 -143
  21. {forcingkit-0.7.1 → forcingkit-0.9.0}/.agents/forcingkit.md +0 -0
  22. {forcingkit-0.7.1 → forcingkit-0.9.0}/.claude/CLAUDE.md +0 -0
  23. {forcingkit-0.7.1 → forcingkit-0.9.0}/.env.template +0 -0
  24. {forcingkit-0.7.1 → forcingkit-0.9.0}/.github/copilot-instructions.md +0 -0
  25. {forcingkit-0.7.1 → forcingkit-0.9.0}/.github/workflows/docs.yml +0 -0
  26. {forcingkit-0.7.1 → forcingkit-0.9.0}/.github/workflows/publish.yml +0 -0
  27. {forcingkit-0.7.1 → forcingkit-0.9.0}/.github/workflows/tests.yml +0 -0
  28. {forcingkit-0.7.1 → forcingkit-0.9.0}/.gitignore +0 -0
  29. {forcingkit-0.7.1 → forcingkit-0.9.0}/.markdownlint.yaml +0 -0
  30. {forcingkit-0.7.1 → forcingkit-0.9.0}/.pre-commit-config.yaml +0 -0
  31. {forcingkit-0.7.1 → forcingkit-0.9.0}/.python-version +0 -0
  32. {forcingkit-0.7.1 → forcingkit-0.9.0}/AGENTS.md +0 -0
  33. {forcingkit-0.7.1 → forcingkit-0.9.0}/CONTRIBUTING.md +0 -0
  34. {forcingkit-0.7.1 → forcingkit-0.9.0}/Dockerfile +0 -0
  35. {forcingkit-0.7.1 → forcingkit-0.9.0}/LICENSE +0 -0
  36. {forcingkit-0.7.1 → forcingkit-0.9.0}/README.md +0 -0
  37. {forcingkit-0.7.1 → forcingkit-0.9.0}/docker-compose.yml +0 -0
  38. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/Makefile +0 -0
  39. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/make.bat +0 -0
  40. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/requirements-docs.txt +0 -0
  41. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/_extra/CNAME +0 -0
  42. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/_static/custom.css +0 -0
  43. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  44. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/atmospheric_forcing.rst +0 -0
  45. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/hycom.rst +0 -0
  46. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/necofs.rst +0 -0
  47. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/nyofs.rst +0 -0
  48. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/rivers.rst +0 -0
  49. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/roadmap.rst +0 -0
  50. {forcingkit-0.7.1 → forcingkit-0.9.0}/docs/source/sources.rst +0 -0
  51. {forcingkit-0.7.1 → forcingkit-0.9.0}/main.py +0 -0
  52. {forcingkit-0.7.1 → forcingkit-0.9.0}/service/__init__.py +0 -0
  53. {forcingkit-0.7.1 → forcingkit-0.9.0}/service/forcingkit_serve/__init__.py +0 -0
  54. {forcingkit-0.7.1 → forcingkit-0.9.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  55. {forcingkit-0.7.1 → forcingkit-0.9.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  56. {forcingkit-0.7.1 → forcingkit-0.9.0}/service/forcingkit_serve/routers/removed.py +0 -0
  57. {forcingkit-0.7.1 → forcingkit-0.9.0}/service/forcingkit_serve/routers/viewer.py +0 -0
  58. {forcingkit-0.7.1 → forcingkit-0.9.0}/service/run_server.py +0 -0
  59. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/__init__.py +0 -0
  60. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  61. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
  62. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/dbofs.py +0 -0
  63. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/hrrr.py +0 -0
  64. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  65. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  66. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/hycom.py +0 -0
  67. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/hydrography.py +0 -0
  68. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/ndbc.py +0 -0
  69. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/necofs.py +0 -0
  70. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/noaa.py +0 -0
  71. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/nyofs.py +0 -0
  72. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/usgs.py +0 -0
  73. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  74. {forcingkit-0.7.1 → forcingkit-0.9.0}/src/forcingkit/zarr_stream.py +0 -0
  75. {forcingkit-0.7.1 → forcingkit-0.9.0}/static/app.js +0 -0
  76. {forcingkit-0.7.1 → forcingkit-0.9.0}/static/favicon.ico +0 -0
  77. {forcingkit-0.7.1 → forcingkit-0.9.0}/static/favicon.svg +0 -0
  78. {forcingkit-0.7.1 → forcingkit-0.9.0}/static/index.html +0 -0
  79. {forcingkit-0.7.1 → forcingkit-0.9.0}/static/logo.svg +0 -0
  80. {forcingkit-0.7.1 → forcingkit-0.9.0}/static/preview3d.js +0 -0
  81. {forcingkit-0.7.1 → forcingkit-0.9.0}/static/styles.css +0 -0
  82. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/__init__.py +0 -0
  83. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/integration/test_auth.py +0 -0
  84. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/integration/test_erddap_fetch.py +0 -0
  85. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  86. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/integration/test_obc_offshore_nj.py +0 -0
  87. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/integration/test_ofs_archive_paths.py +0 -0
  88. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
  89. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/scripts/inspect_dem.py +0 -0
  90. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/scripts/inspect_grib.py +0 -0
  91. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/scripts/inspect_zarr.py +0 -0
  92. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_coastwatch.py +0 -0
  93. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_dbofs.py +0 -0
  94. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_hrrr_atm.py +0 -0
  95. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_hrrr_idx.py +0 -0
  96. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_hrrr_points.py +0 -0
  97. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_hycom.py +0 -0
  98. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_main.py +0 -0
  99. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_ndbc.py +0 -0
  100. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_necofs_parent.py +0 -0
  101. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_noaa_currents.py +0 -0
  102. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_noaa_datum.py +0 -0
  103. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_nyofs.py +0 -0
  104. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_obc_donor_report.py +0 -0
  105. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_obc_pipeline.py +0 -0
  106. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_removed_routes.py +0 -0
  107. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_settings.py +0 -0
  108. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_usgs.py +0 -0
  109. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_usgs_rivers.py +0 -0
  110. {forcingkit-0.7.1 → forcingkit-0.9.0}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.7.1
3
+ Version: 0.9.0
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -226,6 +226,7 @@ Requires-Dist: h5py>=3.16.0
226
226
  Requires-Dist: httpx>=0.28.1
227
227
  Requires-Dist: jinja2>=3.0.0
228
228
  Requires-Dist: matplotlib>=3.10.9
229
+ Requires-Dist: pillow>=10.0
229
230
  Requires-Dist: pydantic>=2.12.5
230
231
  Requires-Dist: pydap>=3.5.8
231
232
  Requires-Dist: pyproj>=3.7.2
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.7.1"
12
+ release = "0.9.0"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -123,6 +123,14 @@ These serve validation rather than forcing:
123
123
  - **UConn ERDDAP** water-column profiles (``/api/v1/telemetry/station`` and ``/bbox``), from the
124
124
  `University of Connecticut Department of Marine Sciences <https://marinesciences.uconn.edu/>`__
125
125
  `ERDDAP server <http://merlin.dms.uconn.edu:8080/erddap/index.html>`__.
126
+ The station list holds four Long Island Sound moorings, so a box elsewhere contains none and
127
+ ``/bbox`` returns empty ``data``. Setting ``"allow_proxy_stations": true`` substitutes EXRX and
128
+ WLIS instead, for testing; every station carries ``within_bbox`` and ``distance_km`` (0 inside
129
+ the box), and the response's ``fallback`` is true when any lies outside it. Each dataset read
130
+ has a 10 s connect and 60 s read timeout (``FORCINGKIT_ERDDAP_CONNECT_TIMEOUT``,
131
+ ``FORCINGKIT_ERDDAP_READ_TIMEOUT``). A host that fails to connect, or a dataset that times out
132
+ or errors, is skipped for 300 s (``FORCINGKIT_ERDDAP_FAILURE_TTL``, 0 to turn off), and the
133
+ response's ``skipped`` lists each dataset left out, with the reason.
126
134
 
127
135
  NOAA OFS archive locations
128
136
  --------------------------
@@ -166,5 +166,6 @@ PyPI as ``forcingkit``.
166
166
  :maxdepth: 1
167
167
  :caption: Reference
168
168
 
169
+ mesh_frames
169
170
  sources
170
171
  roadmap
@@ -0,0 +1,89 @@
1
+ Mesh frames: native-mesh image frames
2
+ =====================================
3
+
4
+ ``forcingkit.mesh_frames`` reads model output on the model's own unstructured mesh (no
5
+ regridding) and ``forcingkit.render`` draws it as a sequence of PNG frames: one flat colour per
6
+ triangle, with an optional mesh-edge overlay. NECOFS FVCOM GOM7 surface currents are the first
7
+ registered model.
8
+
9
+ Data contract
10
+ -------------
11
+
12
+ ``MeshFrames`` holds ``times`` (UTC), ``node_lon`` and ``node_lat``, ``tri`` (``n_tri`` x 3,
13
+ zero-based node indices) and ``u``, ``v`` (``[t, n_tri]``, one value per element, NaN where an
14
+ element has no value). Any triangle-mesh model fits it; an adapter is one function registered with
15
+ ``@register_adapter("name")`` that takes ``(bbox, times)`` and returns a ``MeshFrames``. The
16
+ ``forcingkit.mesh_frames`` module docstring shows the template.
17
+
18
+ Projection
19
+ ----------
20
+
21
+ Pixels are plain longitude/latitude, defined exactly by the caller's box ``(W, S, E, N)`` and image
22
+ size ``(w, h)``: pixel column ``i`` spans longitudes ``W + [i, i+1) * (E - W) / w`` and row ``j``
23
+ spans latitudes ``N - [j, j+1) * (N - S) / h``. A pixel takes the colour of the element containing
24
+ its centre, so frames drawn with the same box and size line up pixel for pixel with any other
25
+ renderer that uses the same convention. Outside the mesh (and wherever an element is NaN) is
26
+ transparent, or the background colour when one is given.
27
+
28
+ The mesh is rasterised once per (mesh, box, size) into an element-index image and the edges once
29
+ into an overlay; both are cached, and each frame is a colour lookup per element and per pixel. On a
30
+ synthetic mesh of 52 000 elements, 45 frames at 1600 x 900 render in about 0.6 s plus 0.5 s for the
31
+ index image; writing the PNGs takes about 4.5 s.
32
+
33
+ Colour
34
+ ------
35
+
36
+ ``color`` is any vectorised function from speed (m/s, shape ``(n,)``) to uint8 RGB or RGBA
37
+ (``(n, 3)`` or ``(n, 4)``). Use a fixed scale so a colour means the same speed in every frame:
38
+
39
+ - ``default_color``: viridis, linear over 0 to 1.5 m/s;
40
+ - ``named_color(name, vmin, vmax, log=False, floor=None)``: a built-in ramp (``viridis``,
41
+ ``greys``) over a chosen range;
42
+ - ``lut_color(rows, log=False, floor=None)``: a lookup table of ``(value, r, g, b[, a])`` rows.
43
+ Values below the first row, or below ``floor``, take the first colour; between rows each channel
44
+ is interpolated linearly in the value, or in log(value) when ``log`` is true; values above the
45
+ last row take the last colour; NaN is transparent. ``load_lut_csv(path)`` reads such a table
46
+ from a ``value,r,g,b[,a]`` CSV;
47
+ - ``packaged_color("currents_lis")``: the surface-current table shipped with the package
48
+ (``forcingkit/ramps/currents_lis.csv``), 0.02 to 3.1 m/s on a log axis, pale blue through
49
+ yellow to dark green with CIELAB lightness falling monotonically. Its CSV header records the
50
+ knots and their source;
51
+ - ``packaged_color("currents_lis_dark")``: its twin for a dark basemap, on the same speeds, from
52
+ midnight blue through steel blue to pale seafoam with lightness rising monotonically, so the
53
+ two swap by theme alone.
54
+
55
+ Python
56
+ ------
57
+
58
+ .. code-block:: python
59
+
60
+ from forcingkit.mesh_frames import mesh_frames
61
+ from forcingkit.render import lut_color, render_frames, write_frames
62
+
63
+ bbox = (-73.0, 40.95, -72.6, 41.15) # central Long Island Sound
64
+ frames = mesh_frames("necofs", bbox, "2026-10-15T22:00Z", "2026-10-16T02:00Z", stride_h=1)
65
+ color = lut_color([(0.05, 230, 240, 250), (0.5, 120, 160, 200), (2.0, 20, 40, 90)], log=True)
66
+ images = render_frames(
67
+ frames, bbox, (1200, 800), color,
68
+ edges=((0, 0, 0, 40), 1), background=(242, 243, 240, 255),
69
+ )
70
+ write_frames(images, "frames/", times=frames.times)
71
+
72
+ NECOFS hours are read from the SMAST daily history file that holds each one (falling back to the
73
+ rolling forecast), so a window may cross several archive days.
74
+
75
+ Command line
76
+ ------------
77
+
78
+ .. code-block:: bash
79
+
80
+ uv run python -m forcingkit.render --model necofs \
81
+ --bbox -73.0 40.95 -72.6 41.15 \
82
+ --start 2026-10-15T22:00Z --end 2026-10-16T02:00Z --stride-h 1 \
83
+ --size 1200x800 --ramp viridis --vmax 1.5 \
84
+ --edges 0,0,0,40 --background 242,243,240 --out frames/
85
+
86
+ ``--ramp`` takes a built-in name, a packaged table (``currents_lis``, ``currents_lis_dark``) or a path to a
87
+ ``value,r,g,b[,a]`` CSV (with ``--log`` and ``--floor``). The output directory receives
88
+ ``frame_0000.png``, ``frame_0001.png``, ... and ``frames.json``, which lists the frame times, the box
89
+ and the size.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.7.1"
7
+ version = "0.9.0"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -43,6 +43,7 @@ dependencies = [
43
43
  "dask[array]>=2026.3.0",
44
44
  "cftime>=1.6.5",
45
45
  "matplotlib>=3.10.9",
46
+ "pillow>=10.0",
46
47
  ]
47
48
 
48
49
  [tool.hatch.build.targets.wheel]
@@ -206,8 +206,10 @@ async def purge_cache() -> Dict[str, str]:
206
206
  return {"status": "success", "message": "Cache directory does not exist."}
207
207
 
208
208
 
209
+ # Plain `def` for the tide and telemetry handlers too: their remote reads block, and FastAPI runs a
210
+ # plain handler in its thread pool rather than on the event loop.
209
211
  @app.post("/api/v1/tide")
210
- async def get_tide_data(request: TideRequest) -> Dict[str, Any]:
212
+ def get_tide_data(request: TideRequest) -> Dict[str, Any]:
211
213
  """Fetch and cache NOAA tide data for a given station and time window."""
212
214
  try:
213
215
  data = fetch_noaa_tide_data(
@@ -268,30 +270,40 @@ class TelemetryBBoxRequest(BaseModel):
268
270
  start_time: str # ISO8601 string
269
271
  end_time: str # ISO8601 string
270
272
  cache_bust: bool = False
273
+ # Substitute proxy stations, marked within_bbox false, when the box contains none.
274
+ allow_proxy_stations: bool = False
271
275
 
272
276
 
273
277
  @app.post("/api/v1/telemetry/station")
274
- async def get_station_telemetry(request: TelemetryRequest) -> Dict[str, Any]:
275
- """Fetch 3-depth telemetry profile data for structural nudging."""
278
+ def get_station_telemetry(request: TelemetryRequest) -> Dict[str, Any]:
279
+ """Fetch 3-depth telemetry profile data for structural nudging. `skipped` lists the datasets
280
+ left out because they, or their host, failed now or recently."""
276
281
  logger.info(
277
282
  f"Received telemetry request for station {request.station_id} from {request.start_time} to {request.end_time}"
278
283
  )
279
284
  try:
285
+ skipped: List[Dict[str, Any]] = []
280
286
  data = dispatch_station_profiles_request(
281
287
  station_id=request.station_id,
282
288
  start_time=request.start_time,
283
289
  end_time=request.end_time,
284
290
  cache_bust=request.cache_bust,
291
+ skipped=skipped,
285
292
  )
286
- return {"status": "success", "data": data}
293
+ return {"status": "success", "data": data, "skipped": skipped}
287
294
  except Exception as e:
288
295
  logger.error(f"Telemetry fetch failed: {e}")
289
296
  raise HTTPException(status_code=500, detail=str(e))
290
297
 
291
298
 
292
299
  @app.post("/api/v1/telemetry/bbox")
293
- async def get_bbox_telemetry(request: TelemetryBBoxRequest) -> Dict[str, Any]:
294
- """Fetch structured nudge telemetry for all stations in bounding box."""
300
+ def get_bbox_telemetry(request: TelemetryBBoxRequest) -> Dict[str, Any]:
301
+ """Fetch structured nudge telemetry for all stations in bounding box.
302
+
303
+ `data` is empty when the box contains no known station, unless `allow_proxy_stations` is
304
+ set. Each station carries `within_bbox` and `distance_km`; `fallback` is true when any
305
+ returned station lies outside the box. `skipped` lists the datasets left out because they,
306
+ or their host, failed now or recently."""
295
307
  logger.info(
296
308
  f"Received bbox telemetry request for {request.bbox} from {request.start_time} to {request.end_time}"
297
309
  )
@@ -303,13 +315,22 @@ async def get_bbox_telemetry(request: TelemetryBBoxRequest) -> Dict[str, Any]:
303
315
  request.bbox.max_lon,
304
316
  request.bbox.max_lat,
305
317
  ]
318
+ skipped: List[Dict[str, Any]] = []
306
319
  data = dispatch_bounding_box_profiles_request(
307
320
  bbox=bbox_list,
308
321
  start_time=request.start_time,
309
322
  end_time=request.end_time,
310
323
  cache_bust=request.cache_bust,
324
+ allow_proxy_stations=request.allow_proxy_stations,
325
+ skipped=skipped,
311
326
  )
312
- return {"status": "success", "data": data}
327
+ fallback = any(not st.get("within_bbox", True) for st in data.values())
328
+ return {
329
+ "status": "success",
330
+ "data": data,
331
+ "fallback": fallback,
332
+ "skipped": skipped,
333
+ }
313
334
  except Exception as e:
314
335
  logger.error(f"BBox telemetry fetch failed: {e}")
315
336
  raise HTTPException(status_code=500, detail=str(e))
@@ -25,10 +25,12 @@ def dispatch_station_profiles_request(
25
25
  "erddap",
26
26
  ),
27
27
  cache_bust: bool = False,
28
+ skipped: Optional[list] = None,
28
29
  ) -> dict:
29
30
  """
30
31
  Tiered modality dispatcher for internal nudging profiles.
31
- Currently routes WLIS directly to the ERDDAP fetcher.
32
+ Currently routes WLIS directly to the ERDDAP fetcher. Datasets left out (failed or
33
+ negatively cached) are appended to `skipped` when it is given.
32
34
  """
33
35
  logger.info(
34
36
  f"Dispatching station profile request for {station_id} ({start_time} to {end_time})"
@@ -44,6 +46,7 @@ def dispatch_station_profiles_request(
44
46
  end_time=end_time,
45
47
  cache_dir=cache_dir,
46
48
  cache_bust=cache_bust,
49
+ skipped=skipped,
47
50
  )
48
51
  return profiles
49
52
  except Exception as e:
@@ -60,9 +63,13 @@ def dispatch_bounding_box_profiles_request(
60
63
  "erddap",
61
64
  ),
62
65
  cache_bust: bool = False,
66
+ allow_proxy_stations: bool = False,
67
+ skipped: Optional[list] = None,
63
68
  ) -> dict:
64
69
  """
65
70
  Tiered modality dispatcher for internal nudging profiles across a domain.
71
+ A box with no known station returns {} unless allow_proxy_stations is set; see
72
+ fetch_erddap_stations_in_bbox.
66
73
  """
67
74
  logger.info(
68
75
  f"Dispatching bounded profile request for {bbox} ({start_time} to {end_time})"
@@ -77,6 +84,8 @@ def dispatch_bounding_box_profiles_request(
77
84
  end_time=end_time,
78
85
  cache_dir=cache_dir,
79
86
  cache_bust=cache_bust,
87
+ allow_proxy_stations=allow_proxy_stations,
88
+ skipped=skipped,
80
89
  )
81
90
  return profiles
82
91
  except Exception as e:
@@ -0,0 +1,305 @@
1
+ import io
2
+ import logging
3
+ import math
4
+ import os
5
+ import threading
6
+ import time
7
+ from concurrent.futures import ThreadPoolExecutor
8
+ from typing import Optional
9
+ from urllib.parse import urlsplit
10
+
11
+ import pandas as pd
12
+ import requests
13
+
14
+ from forcingkit import settings
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ BASE_ERDDAP_URL = "http://merlin.dms.uconn.edu:8080/erddap"
19
+
20
+
21
+ class ErddapUnavailable(Exception):
22
+ """A dataset was not read, because its host or the dataset failed now or recently."""
23
+
24
+
25
+ class _FailureCache:
26
+ """Hosts and datasets that failed recently, skipped until their entry expires.
27
+
28
+ A host is recorded when a connect fails (refused, unreachable or timed out), so the other
29
+ datasets on it, in this request and the next few, are skipped instead of each waiting out the
30
+ connect timeout. A dataset is recorded when its read times out or the server returns a 5xx.
31
+ The lifetime is settings.erddap_failure_ttl(); 0 records nothing.
32
+ """
33
+
34
+ def __init__(self) -> None:
35
+ self._entries: dict[str, tuple[float, str]] = {}
36
+ self._lock = threading.Lock()
37
+
38
+ def record(self, key: str, reason: str) -> None:
39
+ ttl = settings.erddap_failure_ttl()
40
+ if ttl <= 0:
41
+ return
42
+ with self._lock:
43
+ self._entries[key] = (time.monotonic() + ttl, reason)
44
+
45
+ def check(self, key: str) -> Optional[str]:
46
+ """The recorded reason while `key` is still failed, else None."""
47
+ with self._lock:
48
+ entry = self._entries.get(key)
49
+ if entry is None:
50
+ return None
51
+ expires, reason = entry
52
+ if time.monotonic() >= expires:
53
+ del self._entries[key]
54
+ return None
55
+ return reason
56
+
57
+ def clear(self) -> None:
58
+ with self._lock:
59
+ self._entries.clear()
60
+
61
+
62
+ _failures = _FailureCache()
63
+
64
+
65
+ def clear_failure_cache() -> None:
66
+ """Forget every recorded host and dataset failure."""
67
+ _failures.clear()
68
+
69
+
70
+ def _read_dataset(
71
+ dataset_name: str, start_time: str, end_time: str
72
+ ) -> Optional[pd.DataFrame]:
73
+ """One dataset's temperature series from ERDDAP, indexed by UTC time.
74
+
75
+ Returns None when ERDDAP has no rows for the window (HTTP 404). Raises ErddapUnavailable when
76
+ the host or dataset is negatively cached, or when this read fails.
77
+ """
78
+ host = urlsplit(BASE_ERDDAP_URL).netloc
79
+ host_key = f"host:{host}"
80
+ dataset_key = f"dataset:{dataset_name}"
81
+ for key in (host_key, dataset_key):
82
+ reason = _failures.check(key)
83
+ if reason is not None:
84
+ raise ErddapUnavailable(f"skipped after a recent failure: {reason}")
85
+
86
+ query_url = f"{BASE_ERDDAP_URL}/tabledap/{dataset_name}.csvp?time,sea_water_temperature&time%3E={start_time}&time%3C={end_time}"
87
+ logger.info(
88
+ f"Fetching ERDDAP data for {dataset_name} ({start_time} to {end_time})..."
89
+ )
90
+ try:
91
+ response = requests.get(query_url, timeout=settings.erddap_timeouts())
92
+ except requests.ConnectionError as e:
93
+ # Includes ConnectTimeout: the host is unreachable, so skip it for a while.
94
+ reason = f"{host} unreachable ({type(e).__name__})"
95
+ _failures.record(host_key, reason)
96
+ raise ErddapUnavailable(reason) from e
97
+ except requests.Timeout as e:
98
+ reason = f"{dataset_name} read timed out"
99
+ _failures.record(dataset_key, reason)
100
+ raise ErddapUnavailable(reason) from e
101
+ except requests.RequestException as e:
102
+ raise ErddapUnavailable(
103
+ f"{dataset_name} request failed ({type(e).__name__})"
104
+ ) from e
105
+
106
+ if response.status_code == 404:
107
+ # ERDDAP answers 404 when the query matches no rows.
108
+ logger.info(f"ERDDAP has no rows for {dataset_name} in the window")
109
+ return None
110
+ if response.status_code >= 500:
111
+ reason = f"{dataset_name} returned HTTP {response.status_code}"
112
+ _failures.record(dataset_key, reason)
113
+ raise ErddapUnavailable(reason)
114
+ if response.status_code != 200:
115
+ raise ErddapUnavailable(f"{dataset_name} returned HTTP {response.status_code}")
116
+
117
+ try:
118
+ df = pd.read_csv(io.StringIO(response.text))
119
+ if df.empty:
120
+ return None
121
+ # The headers are like "time (UTC)" and "sea_water_temperature (celsius)".
122
+ df.columns = ["time", "sea_water_temperature"]
123
+ df = df.set_index("time")
124
+ df.index = pd.to_datetime(df.index, utc=True)
125
+ except Exception as e:
126
+ raise ErddapUnavailable(f"{dataset_name} could not be parsed ({e})") from e
127
+ return df
128
+
129
+
130
+ def fetch_erddap_station_profiles(
131
+ station_id: str,
132
+ start_time: str,
133
+ end_time: str,
134
+ cache_dir: str = os.path.join(
135
+ settings.cache_dir(),
136
+ "erddap",
137
+ ),
138
+ cache_bust: bool = False,
139
+ skipped: Optional[list] = None,
140
+ ) -> dict:
141
+ """
142
+ Fetches 3-depth temperature profiles for a given station.
143
+ Targets Long Island Sound stations on the UConn ERDDAP server.
144
+
145
+ The three datasets are read concurrently, each with the timeouts of
146
+ settings.erddap_timeouts(). A dataset that fails, or whose host or name failed within
147
+ settings.erddap_failure_ttl() seconds, is left out of the result.
148
+
149
+ Args:
150
+ station_id: Station ID, e.g., "WLIS", "EXRX"
151
+ start_time: ISO8601 string (e.g., "YYYY-MM-DDTHH:MM:SSZ")
152
+ end_time: ISO8601 string
153
+ skipped: when given, a dict per dataset left out is appended to it, with keys
154
+ "station", "level", "dataset" and "reason".
155
+
156
+ Returns:
157
+ dictionary with keys: "surface", "mid", "bottom", each containing a dictionary of time (ISO) -> temperature (float)
158
+ """
159
+ os.makedirs(cache_dir, exist_ok=True)
160
+
161
+ station_upper = station_id.upper()
162
+ datasets = {
163
+ "surface": f"{station_upper}_WQ_SFC",
164
+ "mid": f"{station_upper}_WQ_MID",
165
+ "bottom": f"{station_upper}_WQ_BTM",
166
+ }
167
+
168
+ # Handle irregular dataset names in LISICOS
169
+ if station_upper == "ARTG":
170
+ datasets["bottom"] = "ARTG_WQ_BTM_1"
171
+
172
+ def load(
173
+ level: str, dataset_name: str
174
+ ) -> tuple[Optional[pd.DataFrame], Optional[dict]]:
175
+ cache_key = f"{dataset_name}_{start_time}_{end_time}".replace(":", "-").replace(
176
+ "Z", ""
177
+ )
178
+ cache_path = os.path.join(cache_dir, f"{cache_key}.csv")
179
+
180
+ if not cache_bust and os.path.exists(cache_path):
181
+ logger.info(f"Cache hit for {dataset_name}: {cache_path}")
182
+ return pd.read_csv(cache_path, index_col=0, parse_dates=True), None
183
+ try:
184
+ df = _read_dataset(dataset_name, start_time, end_time)
185
+ except ErddapUnavailable as e:
186
+ logger.error(f"Failed fetching or parsing ERDDAP for {dataset_name}: {e}")
187
+ return None, {
188
+ "station": station_upper,
189
+ "level": level,
190
+ "dataset": dataset_name,
191
+ "reason": str(e),
192
+ }
193
+ if df is not None and not df.empty:
194
+ df.to_csv(cache_path)
195
+ return df, None
196
+
197
+ with ThreadPoolExecutor(max_workers=len(datasets)) as pool:
198
+ loaded = list(pool.map(lambda item: (item[0], *load(*item)), datasets.items()))
199
+
200
+ profiles = {}
201
+ for level, df, skip in loaded:
202
+ if skip is not None and skipped is not None:
203
+ skipped.append(skip)
204
+ if df is not None and not df.empty:
205
+ df = df.dropna(subset=["sea_water_temperature"])
206
+ # We must convert timestamp back to isoformat strings to match json expectations if returned by API
207
+ time_dict = {
208
+ ts.isoformat(): temp for ts, temp in df["sea_water_temperature"].items()
209
+ }
210
+ profiles[level] = time_dict
211
+
212
+ return profiles
213
+
214
+
215
+ KNOWN_STATIONS = {
216
+ "WLIS": {"lat": 40.96, "lon": -73.58},
217
+ "EXRX": {"lat": 40.88, "lon": -73.73},
218
+ "CLIS": {"lat": 41.14, "lon": -72.66},
219
+ "ARTG": {"lat": 41.01, "lon": -73.29},
220
+ }
221
+
222
+ # Stations substituted for a box that contains none of KNOWN_STATIONS, only when a request sets
223
+ # allow_proxy_stations. Each substitute is returned with within_bbox False.
224
+ PROXY_STATIONS = ("EXRX", "WLIS")
225
+
226
+
227
+ def _distance_to_bbox_km(lat: float, lon: float, bbox: list[float]) -> float:
228
+ """Great-circle distance in km from (lat, lon) to the nearest point of
229
+ bbox [min_lon, min_lat, max_lon, max_lat]; 0 inside it."""
230
+ min_lon, min_lat, max_lon, max_lat = bbox
231
+ near_lat = min(max(lat, min_lat), max_lat)
232
+ near_lon = min(max(lon, min_lon), max_lon)
233
+ phi1, phi2 = math.radians(lat), math.radians(near_lat)
234
+ dphi = phi2 - phi1
235
+ dlmb = math.radians(near_lon - lon)
236
+ a = (
237
+ math.sin(dphi / 2) ** 2
238
+ + math.cos(phi1) * math.cos(phi2) * math.sin(dlmb / 2) ** 2
239
+ )
240
+ return 2 * 6371.0 * math.asin(math.sqrt(min(1.0, a)))
241
+
242
+
243
+ def fetch_erddap_stations_in_bbox(
244
+ bbox: list[float],
245
+ start_time: str,
246
+ end_time: str,
247
+ cache_dir: str = os.path.join(
248
+ settings.cache_dir(),
249
+ "erddap",
250
+ ),
251
+ cache_bust: bool = False,
252
+ allow_proxy_stations: bool = False,
253
+ skipped: Optional[list] = None,
254
+ ) -> dict:
255
+ """
256
+ Finds the known Long Island Sound stations within the bounding box and fetches their
257
+ profiles.
258
+ bbox: [min_lon, min_lat, max_lon, max_lat]
259
+
260
+ A box that contains no known station returns {} without fetching anything. With
261
+ allow_proxy_stations, it gets PROXY_STATIONS instead. Every returned station carries
262
+ within_bbox and distance_km (0 inside the box), so a substitute is never mistaken for a
263
+ station in range.
264
+ """
265
+ if len(bbox) == 4:
266
+ min_lon, min_lat, max_lon, max_lat = bbox
267
+ else:
268
+ return {}
269
+
270
+ in_bbox = [
271
+ st_id
272
+ for st_id, coords in KNOWN_STATIONS.items()
273
+ if min_lat <= coords["lat"] <= max_lat and min_lon <= coords["lon"] <= max_lon
274
+ ]
275
+
276
+ stations_to_fetch = in_bbox
277
+ if not in_bbox:
278
+ if not allow_proxy_stations:
279
+ logger.info(f"No known ERDDAP station in bbox {bbox}; returning none.")
280
+ return {}
281
+ logger.warning(
282
+ f"No stations found in bbox {bbox}. Substituting proxy stations "
283
+ f"{', '.join(PROXY_STATIONS)} (allow_proxy_stations), marked within_bbox false"
284
+ )
285
+ stations_to_fetch = list(PROXY_STATIONS)
286
+
287
+ results = {}
288
+ for st_id in stations_to_fetch:
289
+ logger.info(f"Fetching station {st_id} for bounding box request...")
290
+ profile = fetch_erddap_station_profiles(
291
+ st_id, start_time, end_time, cache_dir, cache_bust, skipped=skipped
292
+ )
293
+ if profile and any(v for v in profile.values()):
294
+ coords = KNOWN_STATIONS[st_id]
295
+ results[st_id] = {
296
+ "lat": coords["lat"],
297
+ "lon": coords["lon"],
298
+ "profiles": profile,
299
+ "within_bbox": st_id in in_bbox,
300
+ "distance_km": round(
301
+ _distance_to_bbox_km(coords["lat"], coords["lon"], bbox), 1
302
+ ),
303
+ }
304
+
305
+ return results