forcingkit 0.14.0__tar.gz → 0.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.14.0 → forcingkit-0.15.0}/PKG-INFO +2 -2
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/conf.py +1 -1
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/fetchers.rst +3 -1
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/roadmap.rst +1 -1
- {forcingkit-0.14.0 → forcingkit-0.15.0}/pyproject.toml +3 -3
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/erddap.py +54 -95
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_erddap.py +7 -7
- {forcingkit-0.14.0 → forcingkit-0.15.0}/uv.lock +5 -5
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.agents/forcingkit.md +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.env.template +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.gitignore +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.markdownlint.yaml +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/.python-version +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/AGENTS.md +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/CONTRIBUTING.md +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/Dockerfile +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/LICENSE +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/README.md +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/LICENSE +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/README.md +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/pyproject.toml +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/tests/conftest.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/tests/test_client.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/tests/test_schema.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/client/uv.lock +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docker-compose.yml +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/Makefile +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/make.bat +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/_static/custom.css +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/client.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/hycom.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/index.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/necofs.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/rivers.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/sources.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/docs/source/waves.rst +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/main.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/__init__.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/jobs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/main.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/routers/viewer.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/service/run_server.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/ndbc.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/render.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/static/app.js +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/static/favicon.ico +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/static/favicon.svg +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/static/index.html +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/static/logo.svg +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/static/preview3d.js +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/static/styles.css +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/__init__.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_jobs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_main.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.14.0 → forcingkit-0.15.0}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -225,7 +225,7 @@ Requires-Dist: fastapi>=0.135.1
|
|
|
225
225
|
Requires-Dist: h5netcdf>=1.8.1
|
|
226
226
|
Requires-Dist: h5py>=3.16.0
|
|
227
227
|
Requires-Dist: httpx>=0.28.1
|
|
228
|
-
Requires-Dist: insitukit<0.
|
|
228
|
+
Requires-Dist: insitukit<0.4,>=0.3
|
|
229
229
|
Requires-Dist: jinja2>=3.0.0
|
|
230
230
|
Requires-Dist: matplotlib>=3.10.9
|
|
231
231
|
Requires-Dist: pandas>=2.2
|
|
@@ -163,7 +163,9 @@ These serve validation rather than forcing:
|
|
|
163
163
|
has a 10 s connect and 60 s read timeout (``FORCINGKIT_ERDDAP_CONNECT_TIMEOUT``,
|
|
164
164
|
``FORCINGKIT_ERDDAP_READ_TIMEOUT``). A host that fails to connect, or a dataset that times out
|
|
165
165
|
or errors, is skipped for 300 s (``FORCINGKIT_ERDDAP_FAILURE_TTL``, 0 to turn off), and the
|
|
166
|
-
response's ``skipped`` lists each dataset left out, with the reason.
|
|
166
|
+
response's ``skipped`` lists each dataset left out, with the reason. A station's three datasets
|
|
167
|
+
are read at once, through `insitukit <https://github.com/lhzn-io/insitukit>`__'s ERDDAP client
|
|
168
|
+
(one attempt each, starting at least 0.2 s apart).
|
|
167
169
|
|
|
168
170
|
NOAA OFS archive locations
|
|
169
171
|
--------------------------
|
|
@@ -126,7 +126,7 @@ Planned work
|
|
|
126
126
|
``noaa.py`` serves CO-OPS water levels and current predictions (``coops.py``; since
|
|
127
127
|
0.14.0 a thin layer over ``insitukit.coops``, which may replace it outright),
|
|
128
128
|
``erddap.py`` serves the UConn profile stations (named for the protocol, not the source;
|
|
129
|
-
|
|
129
|
+
since 0.15.0 its requests go through ``insitukit.erddap``),
|
|
130
130
|
and ``hydrography.py`` finds heads of tide from OpenStreetMap (``osm.py`` or similar).
|
|
131
131
|
Renaming them breaks imports from earlier releases, so it belongs in a release that says
|
|
132
132
|
so, with the old names kept as deprecated aliases for one release.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.15.0"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -26,8 +26,8 @@ dependencies = [
|
|
|
26
26
|
"eccodeslib>=2.46.0",
|
|
27
27
|
"fastapi>=0.135.1",
|
|
28
28
|
"httpx>=0.28.1",
|
|
29
|
-
# NOAA CO-OPS
|
|
30
|
-
"insitukit>=0.
|
|
29
|
+
# NOAA CO-OPS (fetchers/noaa.py) and the UConn ERDDAP profiles (fetchers/erddap.py).
|
|
30
|
+
"insitukit>=0.3,<0.4",
|
|
31
31
|
"pandas>=2.2",
|
|
32
32
|
"pydantic>=2.12.5",
|
|
33
33
|
"jinja2>=3.0.0",
|
|
@@ -1,15 +1,12 @@
|
|
|
1
|
-
import io
|
|
2
1
|
import logging
|
|
3
2
|
import math
|
|
4
3
|
import os
|
|
5
|
-
import threading
|
|
6
|
-
import time
|
|
7
4
|
from concurrent.futures import ThreadPoolExecutor
|
|
5
|
+
from datetime import datetime, timezone
|
|
8
6
|
from typing import Optional
|
|
9
|
-
from urllib.parse import urlsplit
|
|
10
7
|
|
|
11
8
|
import pandas as pd
|
|
12
|
-
import
|
|
9
|
+
from insitukit.erddap import DatasetMissing, Erddap, FailureCache, Unavailable
|
|
13
10
|
|
|
14
11
|
from forcingkit import settings
|
|
15
12
|
|
|
@@ -17,114 +14,74 @@ logger = logging.getLogger(__name__)
|
|
|
17
14
|
|
|
18
15
|
BASE_ERDDAP_URL = "http://merlin.dms.uconn.edu:8080/erddap"
|
|
19
16
|
|
|
17
|
+
# Hosts and datasets that failed recently, shared by every request this process serves. A host is
|
|
18
|
+
# recorded when a connect fails (refused, unreachable or timed out), so the other datasets on it, in
|
|
19
|
+
# this request and the next few, are skipped instead of each waiting out the connect timeout. A
|
|
20
|
+
# dataset is recorded when its read times out or the server returns a 5xx. The lifetime is
|
|
21
|
+
# settings.erddap_failure_ttl(); 0 records nothing.
|
|
22
|
+
_failures = FailureCache()
|
|
20
23
|
|
|
21
|
-
class ErddapUnavailable(Exception):
|
|
22
|
-
"""A dataset was not read, because its host or the dataset failed now or recently."""
|
|
23
24
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
A host is recorded when a connect fails (refused, unreachable or timed out), so the other
|
|
29
|
-
datasets on it, in this request and the next few, are skipped instead of each waiting out the
|
|
30
|
-
connect timeout. A dataset is recorded when its read times out or the server returns a 5xx.
|
|
31
|
-
The lifetime is settings.erddap_failure_ttl(); 0 records nothing.
|
|
32
|
-
"""
|
|
33
|
-
|
|
34
|
-
def __init__(self) -> None:
|
|
35
|
-
self._entries: dict[str, tuple[float, str]] = {}
|
|
36
|
-
self._lock = threading.Lock()
|
|
37
|
-
|
|
38
|
-
def record(self, key: str, reason: str) -> None:
|
|
39
|
-
ttl = settings.erddap_failure_ttl()
|
|
40
|
-
if ttl <= 0:
|
|
41
|
-
return
|
|
42
|
-
with self._lock:
|
|
43
|
-
self._entries[key] = (time.monotonic() + ttl, reason)
|
|
44
|
-
|
|
45
|
-
def check(self, key: str) -> Optional[str]:
|
|
46
|
-
"""The recorded reason while `key` is still failed, else None."""
|
|
47
|
-
with self._lock:
|
|
48
|
-
entry = self._entries.get(key)
|
|
49
|
-
if entry is None:
|
|
50
|
-
return None
|
|
51
|
-
expires, reason = entry
|
|
52
|
-
if time.monotonic() >= expires:
|
|
53
|
-
del self._entries[key]
|
|
54
|
-
return None
|
|
55
|
-
return reason
|
|
56
|
-
|
|
57
|
-
def clear(self) -> None:
|
|
58
|
-
with self._lock:
|
|
59
|
-
self._entries.clear()
|
|
25
|
+
def clear_failure_cache() -> None:
|
|
26
|
+
"""Forget every recorded host and dataset failure."""
|
|
27
|
+
_failures.clear()
|
|
60
28
|
|
|
61
29
|
|
|
62
|
-
|
|
30
|
+
def _server() -> Erddap:
|
|
31
|
+
"""An insitukit client for the UConn server, built per call so the settings are read each time:
|
|
32
|
+
one attempt per dataset, the configured timeouts, and the shared failure cache."""
|
|
33
|
+
connect, read = settings.erddap_timeouts()
|
|
34
|
+
return Erddap(
|
|
35
|
+
BASE_ERDDAP_URL,
|
|
36
|
+
budget=None,
|
|
37
|
+
min_interval=0.2,
|
|
38
|
+
retries=1,
|
|
39
|
+
connect_timeout=connect,
|
|
40
|
+
timeout=read,
|
|
41
|
+
failure_ttl=settings.erddap_failure_ttl(),
|
|
42
|
+
failures=_failures,
|
|
43
|
+
fmt="csvp",
|
|
44
|
+
)
|
|
63
45
|
|
|
64
46
|
|
|
65
|
-
def
|
|
66
|
-
|
|
67
|
-
|
|
47
|
+
def _utc(stamp: str) -> datetime:
|
|
48
|
+
ts = datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
|
49
|
+
return (
|
|
50
|
+
ts.replace(tzinfo=timezone.utc)
|
|
51
|
+
if ts.tzinfo is None
|
|
52
|
+
else ts.astimezone(timezone.utc)
|
|
53
|
+
)
|
|
68
54
|
|
|
69
55
|
|
|
70
56
|
def _read_dataset(
|
|
71
|
-
dataset_name: str, start_time: str, end_time: str
|
|
57
|
+
server: Erddap, dataset_name: str, start_time: str, end_time: str
|
|
72
58
|
) -> Optional[pd.DataFrame]:
|
|
73
59
|
"""One dataset's temperature series from ERDDAP, indexed by UTC time.
|
|
74
60
|
|
|
75
|
-
Returns None when ERDDAP has no rows for the window (HTTP 404). Raises
|
|
76
|
-
the host or dataset is negatively cached
|
|
61
|
+
Returns None when ERDDAP has no rows for the window (any HTTP 404). Raises
|
|
62
|
+
insitukit.erddap.Unavailable when the host or dataset is negatively cached or this read fails,
|
|
63
|
+
and ValueError when the response cannot be parsed.
|
|
77
64
|
"""
|
|
78
|
-
host = urlsplit(BASE_ERDDAP_URL).netloc
|
|
79
|
-
host_key = f"host:{host}"
|
|
80
|
-
dataset_key = f"dataset:{dataset_name}"
|
|
81
|
-
for key in (host_key, dataset_key):
|
|
82
|
-
reason = _failures.check(key)
|
|
83
|
-
if reason is not None:
|
|
84
|
-
raise ErddapUnavailable(f"skipped after a recent failure: {reason}")
|
|
85
|
-
|
|
86
|
-
query_url = f"{BASE_ERDDAP_URL}/tabledap/{dataset_name}.csvp?time,sea_water_temperature&time%3E={start_time}&time%3C={end_time}"
|
|
87
65
|
logger.info(
|
|
88
66
|
f"Fetching ERDDAP data for {dataset_name} ({start_time} to {end_time})..."
|
|
89
67
|
)
|
|
90
68
|
try:
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
except
|
|
98
|
-
reason = f"{dataset_name} read timed out"
|
|
99
|
-
_failures.record(dataset_key, reason)
|
|
100
|
-
raise ErddapUnavailable(reason) from e
|
|
101
|
-
except requests.RequestException as e:
|
|
102
|
-
raise ErddapUnavailable(
|
|
103
|
-
f"{dataset_name} request failed ({type(e).__name__})"
|
|
104
|
-
) from e
|
|
105
|
-
|
|
106
|
-
if response.status_code == 404:
|
|
107
|
-
# ERDDAP answers 404 when the query matches no rows.
|
|
69
|
+
df = server.fetch(
|
|
70
|
+
dataset_name,
|
|
71
|
+
_utc(start_time),
|
|
72
|
+
_utc(end_time),
|
|
73
|
+
["sea_water_temperature"],
|
|
74
|
+
)
|
|
75
|
+
except DatasetMissing:
|
|
108
76
|
logger.info(f"ERDDAP has no rows for {dataset_name} in the window")
|
|
109
77
|
return None
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
_failures.record(dataset_key, reason)
|
|
113
|
-
raise ErddapUnavailable(reason)
|
|
114
|
-
if response.status_code != 200:
|
|
115
|
-
raise ErddapUnavailable(f"{dataset_name} returned HTTP {response.status_code}")
|
|
116
|
-
|
|
117
|
-
try:
|
|
118
|
-
df = pd.read_csv(io.StringIO(response.text))
|
|
119
|
-
if df.empty:
|
|
120
|
-
return None
|
|
121
|
-
# The headers are like "time (UTC)" and "sea_water_temperature (celsius)".
|
|
122
|
-
df.columns = ["time", "sea_water_temperature"]
|
|
123
|
-
df = df.set_index("time")
|
|
124
|
-
df.index = pd.to_datetime(df.index, utc=True)
|
|
78
|
+
except Unavailable:
|
|
79
|
+
raise
|
|
125
80
|
except Exception as e:
|
|
126
|
-
raise
|
|
127
|
-
|
|
81
|
+
raise ValueError(f"{dataset_name} could not be parsed ({e})") from e
|
|
82
|
+
if df.empty:
|
|
83
|
+
return None
|
|
84
|
+
return df.set_index("time")
|
|
128
85
|
|
|
129
86
|
|
|
130
87
|
def fetch_erddap_station_profiles(
|
|
@@ -169,6 +126,8 @@ def fetch_erddap_station_profiles(
|
|
|
169
126
|
if station_upper == "ARTG":
|
|
170
127
|
datasets["bottom"] = "ARTG_WQ_BTM_1"
|
|
171
128
|
|
|
129
|
+
server = _server()
|
|
130
|
+
|
|
172
131
|
def load(
|
|
173
132
|
level: str, dataset_name: str
|
|
174
133
|
) -> tuple[Optional[pd.DataFrame], Optional[dict]]:
|
|
@@ -181,8 +140,8 @@ def fetch_erddap_station_profiles(
|
|
|
181
140
|
logger.info(f"Cache hit for {dataset_name}: {cache_path}")
|
|
182
141
|
return pd.read_csv(cache_path, index_col=0, parse_dates=True), None
|
|
183
142
|
try:
|
|
184
|
-
df = _read_dataset(dataset_name, start_time, end_time)
|
|
185
|
-
except
|
|
143
|
+
df = _read_dataset(server, dataset_name, start_time, end_time)
|
|
144
|
+
except (Unavailable, ValueError) as e:
|
|
186
145
|
logger.error(f"Failed fetching or parsing ERDDAP for {dataset_name}: {e}")
|
|
187
146
|
return None, {
|
|
188
147
|
"station": station_upper,
|
|
@@ -46,18 +46,18 @@ def clean_failures(monkeypatch):
|
|
|
46
46
|
|
|
47
47
|
@pytest.fixture
|
|
48
48
|
def fake_get(monkeypatch):
|
|
49
|
-
"""Replace requests.get
|
|
49
|
+
"""Replace requests.Session.get (insitukit's client); `behaviour(url, timeout)` returns or raises."""
|
|
50
50
|
lock = threading.Lock()
|
|
51
51
|
|
|
52
52
|
def install(behaviour):
|
|
53
53
|
calls: list[tuple[str, object]] = []
|
|
54
54
|
|
|
55
|
-
def get(url, timeout=None):
|
|
55
|
+
def get(self, url, timeout=None):
|
|
56
56
|
with lock:
|
|
57
57
|
calls.append((url, timeout))
|
|
58
58
|
return behaviour(url, timeout)
|
|
59
59
|
|
|
60
|
-
monkeypatch.setattr(
|
|
60
|
+
monkeypatch.setattr(requests.Session, "get", get)
|
|
61
61
|
return calls
|
|
62
62
|
|
|
63
63
|
return install
|
|
@@ -177,7 +177,7 @@ def test_failures_are_negatively_cached(fake_get, tmp_path):
|
|
|
177
177
|
|
|
178
178
|
def test_negative_cache_expires(fake_get, monkeypatch, tmp_path):
|
|
179
179
|
clock = [1000.0]
|
|
180
|
-
monkeypatch.setattr(
|
|
180
|
+
monkeypatch.setattr(time, "monotonic", lambda: clock[0])
|
|
181
181
|
monkeypatch.setenv("FORCINGKIT_ERDDAP_FAILURE_TTL", "60")
|
|
182
182
|
calls = fake_get(lambda url, timeout: FakeResponse(status_code=503, text=""))
|
|
183
183
|
|
|
@@ -235,7 +235,7 @@ def test_cached_csv_is_served_while_the_host_is_down(fake_get, tmp_path):
|
|
|
235
235
|
|
|
236
236
|
def test_bbox_with_no_known_station_is_empty_and_fetches_nothing(mocker, tmp_path):
|
|
237
237
|
fetch = mocker.patch("forcingkit.fetchers.erddap.fetch_erddap_station_profiles")
|
|
238
|
-
get = mocker.patch("
|
|
238
|
+
get = mocker.patch("requests.Session.get")
|
|
239
239
|
|
|
240
240
|
assert (
|
|
241
241
|
erddap.fetch_erddap_stations_in_bbox(OFFSHORE_NJ, START, END, str(tmp_path))
|
|
@@ -313,7 +313,7 @@ def _bbox_body(bbox, **extra):
|
|
|
313
313
|
|
|
314
314
|
@pytest.mark.asyncio
|
|
315
315
|
async def test_bbox_route_with_no_station_returns_empty(mocker):
|
|
316
|
-
get = mocker.patch("
|
|
316
|
+
get = mocker.patch("requests.Session.get")
|
|
317
317
|
async with AsyncClient(
|
|
318
318
|
transport=ASGITransport(app=_app()), base_url="http://test"
|
|
319
319
|
) as ac:
|
|
@@ -363,7 +363,7 @@ async def test_bbox_route_reports_skipped_datasets(mocker, tmp_path):
|
|
|
363
363
|
cache_dir=str(tmp_path), **kw
|
|
364
364
|
),
|
|
365
365
|
)
|
|
366
|
-
mocker.patch("
|
|
366
|
+
mocker.patch("requests.Session.get", side_effect=down)
|
|
367
367
|
async with AsyncClient(
|
|
368
368
|
transport=ASGITransport(app=_app()), base_url="http://test"
|
|
369
369
|
) as ac:
|
|
@@ -1000,7 +1000,7 @@ wheels = [
|
|
|
1000
1000
|
|
|
1001
1001
|
[[package]]
|
|
1002
1002
|
name = "forcingkit"
|
|
1003
|
-
version = "0.
|
|
1003
|
+
version = "0.15.0"
|
|
1004
1004
|
source = { editable = "." }
|
|
1005
1005
|
dependencies = [
|
|
1006
1006
|
{ name = "bottleneck" },
|
|
@@ -1062,7 +1062,7 @@ requires-dist = [
|
|
|
1062
1062
|
{ name = "h5netcdf", specifier = ">=1.8.1" },
|
|
1063
1063
|
{ name = "h5py", specifier = ">=3.16.0" },
|
|
1064
1064
|
{ name = "httpx", specifier = ">=0.28.1" },
|
|
1065
|
-
{ name = "insitukit", specifier = ">=0.
|
|
1065
|
+
{ name = "insitukit", specifier = ">=0.3,<0.4" },
|
|
1066
1066
|
{ name = "jinja2", specifier = ">=3.0.0" },
|
|
1067
1067
|
{ name = "matplotlib", specifier = ">=3.10.9" },
|
|
1068
1068
|
{ name = "pandas", specifier = ">=2.2" },
|
|
@@ -1612,16 +1612,16 @@ wheels = [
|
|
|
1612
1612
|
|
|
1613
1613
|
[[package]]
|
|
1614
1614
|
name = "insitukit"
|
|
1615
|
-
version = "0.
|
|
1615
|
+
version = "0.3.0"
|
|
1616
1616
|
source = { registry = "https://pypi.org/simple" }
|
|
1617
1617
|
dependencies = [
|
|
1618
1618
|
{ name = "numpy" },
|
|
1619
1619
|
{ name = "pandas" },
|
|
1620
1620
|
{ name = "requests" },
|
|
1621
1621
|
]
|
|
1622
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
1622
|
+
sdist = { url = "https://files.pythonhosted.org/packages/d6/2f/64ab83ca7705afcab426cfe64953dec222e6fc148bd47bf19118d9df9d70/insitukit-0.3.0.tar.gz", hash = "sha256:fa1ed9c1cf89995b6edf4a550f8835e327f352b0df69a756b3185f5dc7b67aa6", size = 99038, upload-time = "2026-11-06T00:48:46.489Z" }
|
|
1623
1623
|
wheels = [
|
|
1624
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
1624
|
+
{ url = "https://files.pythonhosted.org/packages/8a/00/5b0fa89c8a08b99d36533c4849e20f6061a65901693de02806ff3a22f936/insitukit-0.3.0-py3-none-any.whl", hash = "sha256:396df69830c4d921a40c7693c0f24a7e02a86617730acf010cdc4fdf2a6abb35", size = 22860, upload-time = "2026-11-06T00:48:45.069Z" },
|
|
1625
1625
|
]
|
|
1626
1626
|
|
|
1627
1627
|
[[package]]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|