forcingkit 0.15.0__tar.gz → 0.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {forcingkit-0.15.0 → forcingkit-0.16.0}/PKG-INFO +2 -2
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/conf.py +1 -1
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/fetchers.rst +4 -1
- {forcingkit-0.15.0 → forcingkit-0.16.0}/pyproject.toml +3 -3
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/jobs.py +9 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/main.py +16 -2
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/viewer.py +4 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/ndbc.py +30 -107
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_jobs.py +69 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/uv.lock +5 -5
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.agents/forcingkit.md +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.claude/CLAUDE.md +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.env.template +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/copilot-instructions.md +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/workflows/docs.yml +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/workflows/publish-client.yml +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/workflows/publish.yml +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.github/workflows/tests.yml +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.gitignore +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.markdownlint.yaml +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.pre-commit-config.yaml +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/.python-version +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/AGENTS.md +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/CONTRIBUTING.md +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/Dockerfile +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/LICENSE +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/README.md +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/LICENSE +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/README.md +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/pyproject.toml +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/__init__.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/client.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/schema.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/store.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/src/forcingkit_client/trajectory.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/tests/conftest.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/tests/test_client.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/tests/test_schema.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/tests/test_trajectory.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/client/uv.lock +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docker-compose.yml +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/Makefile +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/make.bat +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/requirements-docs.txt +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/_extra/CNAME +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/_static/custom.css +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/atmospheric_forcing.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/client.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/hycom.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/index.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/layered_mesh.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/mesh_frames.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/necofs.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/nyofs.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/rivers.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/roadmap.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/sources.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/docs/source/waves.rst +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/main.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/__init__.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/__init__.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/removed.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/routers/stores.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/forcingkit_serve/stac.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/service/run_server.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/__init__.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/dispatcher.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/dbofs.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/erddap.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hrrr.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hycom.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/hydrography.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/necofs.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/noaa.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/nwps.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/nyofs.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/usgs.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/layered_mesh.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/mesh_frames.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/render.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/settings.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/src/forcingkit/zarr_stream.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/static/app.js +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/static/favicon.ico +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/static/favicon.svg +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/static/index.html +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/static/logo.svg +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/static/preview3d.js +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/static/styles.css +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/__init__.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_auth.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_erddap_fetch.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_necofs_volume_live.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_nwps_nomads.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_obc_offshore_nj.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_ofs_archive_paths.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/scripts/inspect_dem.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/scripts/inspect_grib.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/scripts/inspect_zarr.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_coastwatch.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_dbofs.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_dispatcher.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_erddap.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hrrr_atm.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hrrr_idx.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hrrr_levels.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hrrr_points.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_hycom.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_layered_mesh.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_main.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_mesh_frames.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_ndbc.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_necofs_hindcast.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_necofs_parent.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_necofs_volume.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_noaa_currents.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_noaa_datum.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_nwps.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_nyofs.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_obc_donor_report.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_obc_pipeline.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_removed_routes.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_settings.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_usgs.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_usgs_rivers.py +0 -0
- {forcingkit-0.15.0 → forcingkit-0.16.0}/tests/unit/test_zarr_stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: forcingkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.16.0
|
|
4
4
|
Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
|
|
5
5
|
Project-URL: Documentation, https://forcingkit.docs.lhzn.io
|
|
6
6
|
Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
|
|
@@ -225,7 +225,7 @@ Requires-Dist: fastapi>=0.135.1
|
|
|
225
225
|
Requires-Dist: h5netcdf>=1.8.1
|
|
226
226
|
Requires-Dist: h5py>=3.16.0
|
|
227
227
|
Requires-Dist: httpx>=0.28.1
|
|
228
|
-
Requires-Dist: insitukit<0.
|
|
228
|
+
Requires-Dist: insitukit<0.5,>=0.4
|
|
229
229
|
Requires-Dist: jinja2>=3.0.0
|
|
230
230
|
Requires-Dist: matplotlib>=3.10.9
|
|
231
231
|
Requires-Dist: pandas>=2.2
|
|
@@ -152,7 +152,10 @@ These serve validation rather than forcing:
|
|
|
152
152
|
``end_time`` is inclusive, as in the API. Current predictions (``/api/v1/currents``) and the
|
|
153
153
|
current-station list (``/api/v1/currents/stations``) use the same client.
|
|
154
154
|
- **NDBC** buoy meteorology and ADCP profiles (``/api/v1/ndbc``), from the `NOAA National Data
|
|
155
|
-
Buoy Center <https://www.ndbc.noaa.gov/>`__
|
|
155
|
+
Buoy Center <https://www.ndbc.noaa.gov/>`__, through insitukit's NDBC client: each month from
|
|
156
|
+
the monthly, yearly or real-time file, each file downloaded once, and each column's own
|
|
157
|
+
missing-value sentinel read as missing (since 0.16.0; earlier releases also dropped real
|
|
158
|
+
readings equal to another column's sentinel, such as a 999 hPa low or a 99-degree wind).
|
|
156
159
|
- **UConn ERDDAP** water-column profiles (``/api/v1/telemetry/station`` and ``/bbox``), from the
|
|
157
160
|
`University of Connecticut Department of Marine Sciences <https://marinesciences.uconn.edu/>`__
|
|
158
161
|
`ERDDAP server <http://merlin.dms.uconn.edu:8080/erddap/index.html>`__.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "forcingkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.16.0"
|
|
8
8
|
description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -26,8 +26,8 @@ dependencies = [
|
|
|
26
26
|
"eccodeslib>=2.46.0",
|
|
27
27
|
"fastapi>=0.135.1",
|
|
28
28
|
"httpx>=0.28.1",
|
|
29
|
-
# NOAA CO-OPS (fetchers/noaa.py)
|
|
30
|
-
"insitukit>=0.
|
|
29
|
+
# NOAA CO-OPS (fetchers/noaa.py), the UConn ERDDAP profiles (fetchers/erddap.py) and NDBC (fetchers/ndbc.py).
|
|
30
|
+
"insitukit>=0.4,<0.5",
|
|
31
31
|
"pandas>=2.2",
|
|
32
32
|
"pydantic>=2.12.5",
|
|
33
33
|
"jinja2>=3.0.0",
|
|
@@ -104,6 +104,15 @@ class JobRegistry:
|
|
|
104
104
|
job = self.get(job_id)
|
|
105
105
|
return job is not None and job.status in ACTIVE
|
|
106
106
|
|
|
107
|
+
def forget(self, job_id: Optional[str] = None) -> None:
|
|
108
|
+
"""Drop the finished job under `job_id`, or every finished job, once its store has been
|
|
109
|
+
deleted. An active job is kept: its build is still writing the store."""
|
|
110
|
+
with self._lock:
|
|
111
|
+
for key in [job_id] if job_id is not None else list(self._jobs):
|
|
112
|
+
job = self._jobs.get(key)
|
|
113
|
+
if job is not None and job.status not in ACTIVE:
|
|
114
|
+
del self._jobs[key]
|
|
115
|
+
|
|
107
116
|
def _run(self, job: Job, build: Callable[[], Dict[str, Any]]) -> None:
|
|
108
117
|
job.status, job.started = "running", _now()
|
|
109
118
|
try:
|
|
@@ -217,6 +217,9 @@ async def purge_cache() -> Dict[str, str]:
|
|
|
217
217
|
try:
|
|
218
218
|
shutil.rmtree(cache_dir)
|
|
219
219
|
os.makedirs(cache_dir, exist_ok=True)
|
|
220
|
+
from forcingkit_serve.jobs import registry
|
|
221
|
+
|
|
222
|
+
registry.forget()
|
|
220
223
|
return {"status": "success", "message": "Cache purged successfully."}
|
|
221
224
|
except Exception as e:
|
|
222
225
|
logger.error(f"Failed to purge cache: {e}")
|
|
@@ -760,10 +763,13 @@ def _submit(
|
|
|
760
763
|
from forcingkit_serve.routers.stores import store_complete
|
|
761
764
|
|
|
762
765
|
kind = JOB_KINDS[zarr_id.split("_", 1)[0]]
|
|
763
|
-
|
|
766
|
+
complete = store_complete(zarr_path)
|
|
767
|
+
if not cache_bust and not registry.is_active(zarr_id) and complete:
|
|
764
768
|
job = registry.finished(zarr_id, kind, cached())
|
|
765
769
|
else:
|
|
766
|
-
job
|
|
770
|
+
# A succeeded job whose store is no longer complete (deleted from the cache since) is
|
|
771
|
+
# stale: rebuild rather than return it.
|
|
772
|
+
job = registry.submit(zarr_id, kind, build, rebuild=cache_bust or not complete)
|
|
767
773
|
return JSONResponse(status_code=202, content=_job_view(job, http))
|
|
768
774
|
|
|
769
775
|
|
|
@@ -827,6 +833,14 @@ def get_job(job_id: str, http: Request) -> Dict[str, Any]:
|
|
|
827
833
|
if not STORE_ID.match(job_id):
|
|
828
834
|
raise HTTPException(status_code=400, detail="invalid job id")
|
|
829
835
|
job = registry.get(job_id)
|
|
836
|
+
if (
|
|
837
|
+
job is not None
|
|
838
|
+
and job.result is not None
|
|
839
|
+
and not store_complete(job.result["zarr_path"])
|
|
840
|
+
):
|
|
841
|
+
# Its store was deleted after the build: the job no longer has a result to report.
|
|
842
|
+
registry.forget(job_id)
|
|
843
|
+
job = None
|
|
830
844
|
if job is None:
|
|
831
845
|
zarr_path = os.path.join(settings.cache_dir(), f"{job_id}.zarr")
|
|
832
846
|
if not store_complete(zarr_path):
|
|
@@ -470,6 +470,10 @@ async def delete_dataset(dataset_id: str):
|
|
|
470
470
|
old_sidecar = zarr_path.parent / f"{dataset_id}_metadata.json"
|
|
471
471
|
if old_sidecar.exists():
|
|
472
472
|
old_sidecar.unlink()
|
|
473
|
+
# A finished job for this store would otherwise still report it as built.
|
|
474
|
+
from forcingkit_serve.jobs import registry
|
|
475
|
+
|
|
476
|
+
registry.forget(dataset_id)
|
|
473
477
|
logger.info(f"Deleted Zarr dataset: {zarr_path}")
|
|
474
478
|
return {"status": "success", "message": f"Deleted {dataset_id}"}
|
|
475
479
|
|
|
@@ -1,46 +1,24 @@
|
|
|
1
1
|
"""NDBC buoy observations for model validation: standard meteorology (sea and air temperature,
|
|
2
|
-
pressure, wind, waves) and near-surface ADCP currents.
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
months of the current year
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
2
|
+
pressure, wind, waves) and near-surface ADCP currents, through insitukit.
|
|
3
|
+
|
|
4
|
+
insitukit's NDBC client reads each month from the first file NDBC publishes it in: the per-month
|
|
5
|
+
file for months of the current year, then the yearly historical file, then the rolling 45-day
|
|
6
|
+
real-time file. It downloads each file once per call, paces and identifies its requests, and turns
|
|
7
|
+
NDBC's missing-value sentinels (99, 999, 9999, MM) into NaN. This module keeps forcingkit's
|
|
8
|
+
function names and the JSON shape of ``fetch_ndbc``. These observations validate a model; they
|
|
9
|
+
never force it.
|
|
9
10
|
"""
|
|
10
11
|
|
|
11
|
-
import gzip
|
|
12
12
|
import json
|
|
13
|
-
import logging
|
|
14
13
|
import math
|
|
15
14
|
import os
|
|
16
|
-
import re
|
|
17
15
|
from typing import Optional
|
|
18
16
|
|
|
19
17
|
import numpy as np
|
|
20
18
|
import pandas as pd
|
|
21
|
-
import
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
NDBC = "https://www.ndbc.noaa.gov"
|
|
26
|
-
PRODUCTS = ("stdmet", "adcp")
|
|
27
|
-
REALTIME_SUFFIX = {"stdmet": "txt", "adcp": "adcp"}
|
|
28
|
-
MONTH_ABBR = [
|
|
29
|
-
"Jan",
|
|
30
|
-
"Feb",
|
|
31
|
-
"Mar",
|
|
32
|
-
"Apr",
|
|
33
|
-
"May",
|
|
34
|
-
"Jun",
|
|
35
|
-
"Jul",
|
|
36
|
-
"Aug",
|
|
37
|
-
"Sep",
|
|
38
|
-
"Oct",
|
|
39
|
-
"Nov",
|
|
40
|
-
"Dec",
|
|
41
|
-
]
|
|
42
|
-
# Column sentinels NDBC uses for "not measured".
|
|
43
|
-
SENTINELS = {99.0, 999.0, 9999.0, 99.00}
|
|
19
|
+
from insitukit import ndbc as _ndbc
|
|
20
|
+
|
|
21
|
+
PRODUCTS = _ndbc.PRODUCTS
|
|
44
22
|
# stdmet columns carried through, with units.
|
|
45
23
|
STDMET_COLUMNS = {
|
|
46
24
|
"WTMP": "degC", # sea surface temperature
|
|
@@ -53,58 +31,19 @@ STDMET_COLUMNS = {
|
|
|
53
31
|
"DPD": "s",
|
|
54
32
|
}
|
|
55
33
|
|
|
56
|
-
|
|
57
|
-
def month_code(month: int) -> str:
|
|
58
|
-
"""NDBC's month code in per-month file names: 1 to 9, then a, b, c for October to December."""
|
|
59
|
-
return str(month) if month < 10 else "abc"[month - 10]
|
|
34
|
+
month_code = _ndbc.month_code
|
|
60
35
|
|
|
61
36
|
|
|
62
37
|
def candidate_urls(station: str, product: str, year: int, month: int) -> list[str]:
|
|
63
38
|
"""URLs that may hold `product` for `station` in the given month, in the order to try."""
|
|
64
|
-
station
|
|
65
|
-
return [
|
|
66
|
-
f"{NDBC}/data/{product}/{MONTH_ABBR[month - 1]}/{station}{month_code(month)}{year}.txt.gz",
|
|
67
|
-
f"{NDBC}/data/historical/{product}/{station}h{year}.txt.gz",
|
|
68
|
-
f"{NDBC}/data/realtime2/{station.upper()}.{REALTIME_SUFFIX[product]}",
|
|
69
|
-
]
|
|
39
|
+
return _ndbc.urls(station, product, year, month)
|
|
70
40
|
|
|
71
41
|
|
|
72
42
|
def parse_ndbc_text(text: str) -> pd.DataFrame:
|
|
73
|
-
"""Parse an NDBC whitespace-delimited file
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if not lines or not lines[0].startswith("#"):
|
|
78
|
-
raise ValueError("not an NDBC data file: no header row")
|
|
79
|
-
names = lines[0].lstrip("#").split()
|
|
80
|
-
rows = [ln.split() for ln in lines if not ln.startswith("#")]
|
|
81
|
-
df = pd.DataFrame(rows, columns=names[: len(rows[0])] if rows else names)
|
|
82
|
-
year_col = names[0]
|
|
83
|
-
years = df[year_col].astype(int)
|
|
84
|
-
years = years.where(years >= 100, years + 1900)
|
|
85
|
-
minutes = df["mm"].astype(int) if "mm" in df else 0
|
|
86
|
-
df.index = pd.to_datetime(
|
|
87
|
-
dict(
|
|
88
|
-
year=years,
|
|
89
|
-
month=df["MM"].astype(int),
|
|
90
|
-
day=df["DD"].astype(int),
|
|
91
|
-
hour=df["hh"].astype(int),
|
|
92
|
-
minute=minutes,
|
|
93
|
-
)
|
|
94
|
-
)
|
|
95
|
-
df = df.drop(columns=[c for c in (year_col, "MM", "DD", "hh", "mm") if c in df])
|
|
96
|
-
df = df.replace("MM", np.nan).apply(pd.to_numeric, errors="coerce")
|
|
97
|
-
return df.mask(df.isin(SENTINELS))
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
def _get(url: str, timeout: float = 60.0) -> Optional[str]:
|
|
101
|
-
r = requests.get(url, timeout=timeout)
|
|
102
|
-
if r.status_code == 404:
|
|
103
|
-
return None
|
|
104
|
-
r.raise_for_status()
|
|
105
|
-
if url.endswith(".gz"):
|
|
106
|
-
return gzip.decompress(r.content).decode("utf-8", errors="replace")
|
|
107
|
-
return r.text
|
|
43
|
+
"""Parse an NDBC whitespace-delimited file into a frame indexed by UTC time (naive),
|
|
44
|
+
sentinels and `MM` as NaN."""
|
|
45
|
+
df = _ndbc.parse(text)
|
|
46
|
+
return df.set_index(pd.DatetimeIndex(df.pop("time")).tz_localize(None))
|
|
108
47
|
|
|
109
48
|
|
|
110
49
|
def _utc_naive(t) -> pd.Timestamp:
|
|
@@ -112,40 +51,24 @@ def _utc_naive(t) -> pd.Timestamp:
|
|
|
112
51
|
return t if t.tzinfo is None else t.tz_convert("UTC").tz_localize(None)
|
|
113
52
|
|
|
114
53
|
|
|
115
|
-
def station_position(station: str, get=
|
|
54
|
+
def station_position(station: str, get=None) -> tuple[float, float]:
|
|
116
55
|
"""(lat, lon) of an NDBC station from its station page."""
|
|
117
|
-
|
|
118
|
-
m = re.search(r"(\d+\.\d+)\s*([NS])\s+(\d+\.\d+)\s*([EW])", html)
|
|
119
|
-
if not m:
|
|
120
|
-
raise ValueError(f"No position found for NDBC station {station}")
|
|
121
|
-
lat = float(m.group(1)) * (1 if m.group(2) == "N" else -1)
|
|
122
|
-
lon = float(m.group(3)) * (1 if m.group(4) == "E" else -1)
|
|
123
|
-
return lat, lon
|
|
56
|
+
return _ndbc.position(station, get=get)
|
|
124
57
|
|
|
125
58
|
|
|
126
59
|
def fetch_product(
|
|
127
|
-
station: str, product: str, start: pd.Timestamp, end: pd.Timestamp, get=
|
|
60
|
+
station: str, product: str, start: pd.Timestamp, end: pd.Timestamp, get=None
|
|
128
61
|
):
|
|
129
|
-
"""`product` for `station` over [start, end]
|
|
130
|
-
the first source that has it."""
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
text = get(url)
|
|
137
|
-
if text is None:
|
|
138
|
-
continue
|
|
139
|
-
df = parse_ndbc_text(text)
|
|
140
|
-
df = df[(df.index >= start) & (df.index <= end)]
|
|
141
|
-
if len(df):
|
|
142
|
-
frames.append(df)
|
|
143
|
-
sources.append(url)
|
|
144
|
-
break
|
|
145
|
-
if not frames:
|
|
62
|
+
"""`product` for `station` over [start, end] (naive UTC, both included): (frame indexed by
|
|
63
|
+
naive UTC time, source URLs). Each month is taken from the first source that has it."""
|
|
64
|
+
t0 = _utc_naive(start).tz_localize("UTC")
|
|
65
|
+
t1 = _utc_naive(end).tz_localize("UTC") + pd.Timedelta(minutes=1)
|
|
66
|
+
df = _ndbc.fetch(station, t0, t1, product, get=get)
|
|
67
|
+
sources = list(df.attrs.get("sources", []))
|
|
68
|
+
if df.empty:
|
|
146
69
|
return pd.DataFrame(), sources
|
|
147
|
-
df = pd.
|
|
148
|
-
return df[
|
|
70
|
+
df = df.set_index(pd.DatetimeIndex(df.pop("time")).tz_localize(None))
|
|
71
|
+
return df[df.index <= _utc_naive(end)], sources
|
|
149
72
|
|
|
150
73
|
|
|
151
74
|
def _list(values) -> list:
|
|
@@ -161,7 +84,7 @@ def fetch_ndbc(
|
|
|
161
84
|
end_time: str,
|
|
162
85
|
products: tuple[str, ...] = PRODUCTS,
|
|
163
86
|
cache_dir: Optional[str] = None,
|
|
164
|
-
get=
|
|
87
|
+
get=None,
|
|
165
88
|
position: Optional[tuple[float, float]] = None,
|
|
166
89
|
) -> dict:
|
|
167
90
|
"""Observations at an NDBC station over [start_time, end_time] (UTC) as a JSON-ready dict.
|
|
@@ -137,6 +137,75 @@ def test_store_finished_before_restart_is_a_succeeded_job(client):
|
|
|
137
137
|
assert body["status"] == "succeeded" and body["result"]["id"] == zarr_id
|
|
138
138
|
|
|
139
139
|
|
|
140
|
+
def test_registry_rebuilds_succeeded_job_on_request_and_forgets_finished_ones():
|
|
141
|
+
registry = jobs.JobRegistry(workers=1)
|
|
142
|
+
release = threading.Event()
|
|
143
|
+
first = registry.submit("obc_0123456789ab", "ocean", lambda: {"n": 1})
|
|
144
|
+
deadline = time.monotonic() + 5
|
|
145
|
+
while first.status != "succeeded" and time.monotonic() < deadline:
|
|
146
|
+
time.sleep(0.01)
|
|
147
|
+
assert first.status == "succeeded"
|
|
148
|
+
assert registry.submit("obc_0123456789ab", "ocean", lambda: {"n": 2}) is first
|
|
149
|
+
|
|
150
|
+
def slow_build():
|
|
151
|
+
release.wait(5)
|
|
152
|
+
return {"n": 3}
|
|
153
|
+
|
|
154
|
+
rebuilt = registry.submit("obc_0123456789ab", "ocean", slow_build, rebuild=True)
|
|
155
|
+
assert rebuilt is not first
|
|
156
|
+
|
|
157
|
+
# An active job survives forget; a finished one does not.
|
|
158
|
+
registry.forget("obc_0123456789ab")
|
|
159
|
+
registry.forget()
|
|
160
|
+
assert registry.get("obc_0123456789ab") is rebuilt
|
|
161
|
+
release.set()
|
|
162
|
+
while rebuilt.status != "succeeded" and time.monotonic() < deadline:
|
|
163
|
+
time.sleep(0.01)
|
|
164
|
+
registry.forget("obc_0123456789ab")
|
|
165
|
+
assert registry.get("obc_0123456789ab") is None
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _built_obc_job(client, monkeypatch, calls):
|
|
169
|
+
def fake_dispatch(**kwargs):
|
|
170
|
+
calls.append(kwargs)
|
|
171
|
+
_write_store(kwargs["zarr_path"])
|
|
172
|
+
return kwargs["zarr_path"]
|
|
173
|
+
|
|
174
|
+
monkeypatch.setattr(dispatcher, "dispatch_obc_request", fake_dispatch)
|
|
175
|
+
body = client.post("/api/v1/jobs/obc", json=OBC).json()
|
|
176
|
+
assert _wait(client, body["status_url"])["status"] == "succeeded"
|
|
177
|
+
return body
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def test_succeeded_job_whose_store_vanished_is_rebuilt(client, monkeypatch):
|
|
181
|
+
import shutil
|
|
182
|
+
|
|
183
|
+
calls = []
|
|
184
|
+
body = _built_obc_job(client, monkeypatch, calls)
|
|
185
|
+
_, zarr_path, _ = main._obc_store(main.OBCRequest(**OBC))
|
|
186
|
+
shutil.rmtree(zarr_path) # removed behind the service's back, registry untouched
|
|
187
|
+
|
|
188
|
+
assert client.get(body["status_url"]).status_code == 404
|
|
189
|
+
again = client.post("/api/v1/jobs/obc", json=OBC)
|
|
190
|
+
assert again.status_code == 202
|
|
191
|
+
done = _wait(client, again.json()["status_url"])
|
|
192
|
+
assert done["status"] == "succeeded" and len(calls) == 2
|
|
193
|
+
assert done["result"]["id"] == body["job_id"]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def test_deleting_a_store_drops_its_job(client, monkeypatch):
|
|
197
|
+
calls = []
|
|
198
|
+
body = _built_obc_job(client, monkeypatch, calls)
|
|
199
|
+
deleted = client.delete(f"/api/v1/cache/dataset/{body['job_id']}")
|
|
200
|
+
assert deleted.status_code == 200
|
|
201
|
+
assert jobs.registry.get(body["job_id"]) is None
|
|
202
|
+
|
|
203
|
+
again = client.post("/api/v1/jobs/obc", json=OBC)
|
|
204
|
+
assert again.status_code == 202
|
|
205
|
+
assert _wait(client, again.json()["status_url"])["status"] == "succeeded"
|
|
206
|
+
assert len(calls) == 2
|
|
207
|
+
|
|
208
|
+
|
|
140
209
|
def test_unknown_and_invalid_job_ids(client):
|
|
141
210
|
assert client.get("/api/v1/jobs/obc_000000000000").status_code == 404
|
|
142
211
|
assert client.get("/api/v1/jobs/not-a-store").status_code == 400
|
|
@@ -1000,7 +1000,7 @@ wheels = [
|
|
|
1000
1000
|
|
|
1001
1001
|
[[package]]
|
|
1002
1002
|
name = "forcingkit"
|
|
1003
|
-
version = "0.
|
|
1003
|
+
version = "0.16.0"
|
|
1004
1004
|
source = { editable = "." }
|
|
1005
1005
|
dependencies = [
|
|
1006
1006
|
{ name = "bottleneck" },
|
|
@@ -1062,7 +1062,7 @@ requires-dist = [
|
|
|
1062
1062
|
{ name = "h5netcdf", specifier = ">=1.8.1" },
|
|
1063
1063
|
{ name = "h5py", specifier = ">=3.16.0" },
|
|
1064
1064
|
{ name = "httpx", specifier = ">=0.28.1" },
|
|
1065
|
-
{ name = "insitukit", specifier = ">=0.
|
|
1065
|
+
{ name = "insitukit", specifier = ">=0.4,<0.5" },
|
|
1066
1066
|
{ name = "jinja2", specifier = ">=3.0.0" },
|
|
1067
1067
|
{ name = "matplotlib", specifier = ">=3.10.9" },
|
|
1068
1068
|
{ name = "pandas", specifier = ">=2.2" },
|
|
@@ -1612,16 +1612,16 @@ wheels = [
|
|
|
1612
1612
|
|
|
1613
1613
|
[[package]]
|
|
1614
1614
|
name = "insitukit"
|
|
1615
|
-
version = "0.
|
|
1615
|
+
version = "0.4.0"
|
|
1616
1616
|
source = { registry = "https://pypi.org/simple" }
|
|
1617
1617
|
dependencies = [
|
|
1618
1618
|
{ name = "numpy" },
|
|
1619
1619
|
{ name = "pandas" },
|
|
1620
1620
|
{ name = "requests" },
|
|
1621
1621
|
]
|
|
1622
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
1622
|
+
sdist = { url = "https://files.pythonhosted.org/packages/52/1c/347ad724e468073a23c7c6c8ebac61f847fcdfd56515b02edf5c8007a624/insitukit-0.4.0.tar.gz", hash = "sha256:cf98f653dd0235a2b7d9b015185a43e5974839eb037ad30d4855acd95166a8ec", size = 104916, upload-time = "2026-11-06T12:24:57.858Z" }
|
|
1623
1623
|
wheels = [
|
|
1624
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
1624
|
+
{ url = "https://files.pythonhosted.org/packages/28/55/9e8392e4ead1704ee9b279b7cdfe2b610554990172cad717f37a0d8a415a/insitukit-0.4.0-py3-none-any.whl", hash = "sha256:b1222ca7b90d335139b8d96445cd3c600e9678aef6a564e122f28b3986b9c4e3", size = 27409, upload-time = "2026-11-06T12:24:56.3Z" },
|
|
1625
1625
|
]
|
|
1626
1626
|
|
|
1627
1627
|
[[package]]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|