forcingkit 0.15.1__tar.gz → 0.17.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. {forcingkit-0.15.1 → forcingkit-0.17.0}/PKG-INFO +2 -2
  2. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/conf.py +1 -1
  3. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/fetchers.rst +11 -2
  4. {forcingkit-0.15.1 → forcingkit-0.17.0}/pyproject.toml +3 -3
  5. forcingkit-0.17.0/src/forcingkit/fetchers/ndbc.py +166 -0
  6. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/noaa.py +37 -9
  7. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_ndbc.py +26 -0
  8. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_noaa_datum.py +39 -2
  9. {forcingkit-0.15.1 → forcingkit-0.17.0}/uv.lock +5 -5
  10. forcingkit-0.15.1/src/forcingkit/fetchers/ndbc.py +0 -231
  11. {forcingkit-0.15.1 → forcingkit-0.17.0}/.agents/forcingkit.md +0 -0
  12. {forcingkit-0.15.1 → forcingkit-0.17.0}/.claude/CLAUDE.md +0 -0
  13. {forcingkit-0.15.1 → forcingkit-0.17.0}/.env.template +0 -0
  14. {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/copilot-instructions.md +0 -0
  15. {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/workflows/docs.yml +0 -0
  16. {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/workflows/publish-client.yml +0 -0
  17. {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/workflows/publish.yml +0 -0
  18. {forcingkit-0.15.1 → forcingkit-0.17.0}/.github/workflows/tests.yml +0 -0
  19. {forcingkit-0.15.1 → forcingkit-0.17.0}/.gitignore +0 -0
  20. {forcingkit-0.15.1 → forcingkit-0.17.0}/.markdownlint.yaml +0 -0
  21. {forcingkit-0.15.1 → forcingkit-0.17.0}/.pre-commit-config.yaml +0 -0
  22. {forcingkit-0.15.1 → forcingkit-0.17.0}/.python-version +0 -0
  23. {forcingkit-0.15.1 → forcingkit-0.17.0}/AGENTS.md +0 -0
  24. {forcingkit-0.15.1 → forcingkit-0.17.0}/CONTRIBUTING.md +0 -0
  25. {forcingkit-0.15.1 → forcingkit-0.17.0}/Dockerfile +0 -0
  26. {forcingkit-0.15.1 → forcingkit-0.17.0}/LICENSE +0 -0
  27. {forcingkit-0.15.1 → forcingkit-0.17.0}/README.md +0 -0
  28. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/LICENSE +0 -0
  29. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/README.md +0 -0
  30. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/pyproject.toml +0 -0
  31. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/__init__.py +0 -0
  32. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/client.py +0 -0
  33. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/schema.py +0 -0
  34. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/store.py +0 -0
  35. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/src/forcingkit_client/trajectory.py +0 -0
  36. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/tests/conftest.py +0 -0
  37. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/tests/test_client.py +0 -0
  38. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/tests/test_schema.py +0 -0
  39. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/tests/test_trajectory.py +0 -0
  40. {forcingkit-0.15.1 → forcingkit-0.17.0}/client/uv.lock +0 -0
  41. {forcingkit-0.15.1 → forcingkit-0.17.0}/docker-compose.yml +0 -0
  42. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/Makefile +0 -0
  43. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/make.bat +0 -0
  44. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/requirements-docs.txt +0 -0
  45. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/_extra/CNAME +0 -0
  46. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/_static/custom.css +0 -0
  47. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  48. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/atmospheric_forcing.rst +0 -0
  49. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/client.rst +0 -0
  50. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/hycom.rst +0 -0
  51. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/index.rst +0 -0
  52. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/layered_mesh.rst +0 -0
  53. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/mesh_frames.rst +0 -0
  54. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/necofs.rst +0 -0
  55. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/nyofs.rst +0 -0
  56. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/rivers.rst +0 -0
  57. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/roadmap.rst +0 -0
  58. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/sources.rst +0 -0
  59. {forcingkit-0.15.1 → forcingkit-0.17.0}/docs/source/waves.rst +0 -0
  60. {forcingkit-0.15.1 → forcingkit-0.17.0}/main.py +0 -0
  61. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/__init__.py +0 -0
  62. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/__init__.py +0 -0
  63. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/jobs.py +0 -0
  64. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/main.py +0 -0
  65. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  66. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  67. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/removed.py +0 -0
  68. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/stores.py +0 -0
  69. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/routers/viewer.py +0 -0
  70. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/forcingkit_serve/stac.py +0 -0
  71. {forcingkit-0.15.1 → forcingkit-0.17.0}/service/run_server.py +0 -0
  72. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/__init__.py +0 -0
  73. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/data/rivers/northeast_us.json +0 -0
  74. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/dispatcher.py +0 -0
  75. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/coastwatch.py +0 -0
  76. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/dbofs.py +0 -0
  77. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/erddap.py +0 -0
  78. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr.py +0 -0
  79. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  80. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_levels.py +0 -0
  81. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hrrr_points.py +0 -0
  82. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hycom.py +0 -0
  83. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/hydrography.py +0 -0
  84. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/necofs.py +0 -0
  85. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/nwps.py +0 -0
  86. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/nyofs.py +0 -0
  87. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/usgs.py +0 -0
  88. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/fetchers/usgs_rivers.py +0 -0
  89. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/layered_mesh.py +0 -0
  90. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/mesh_frames.py +0 -0
  91. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis.csv +0 -0
  92. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_dark.csv +0 -0
  93. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_deep.csv +0 -0
  94. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/ramps/currents_lis_haline.csv +0 -0
  95. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/render.py +0 -0
  96. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/settings.py +0 -0
  97. {forcingkit-0.15.1 → forcingkit-0.17.0}/src/forcingkit/zarr_stream.py +0 -0
  98. {forcingkit-0.15.1 → forcingkit-0.17.0}/static/app.js +0 -0
  99. {forcingkit-0.15.1 → forcingkit-0.17.0}/static/favicon.ico +0 -0
  100. {forcingkit-0.15.1 → forcingkit-0.17.0}/static/favicon.svg +0 -0
  101. {forcingkit-0.15.1 → forcingkit-0.17.0}/static/index.html +0 -0
  102. {forcingkit-0.15.1 → forcingkit-0.17.0}/static/logo.svg +0 -0
  103. {forcingkit-0.15.1 → forcingkit-0.17.0}/static/preview3d.js +0 -0
  104. {forcingkit-0.15.1 → forcingkit-0.17.0}/static/styles.css +0 -0
  105. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/__init__.py +0 -0
  106. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_auth.py +0 -0
  107. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_erddap_fetch.py +0 -0
  108. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_mesh_frames_necofs.py +0 -0
  109. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_necofs_volume_live.py +0 -0
  110. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_nwps_nomads.py +0 -0
  111. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  112. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_obc_offshore_nj.py +0 -0
  113. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_ofs_archive_paths.py +0 -0
  114. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
  115. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/scripts/inspect_dem.py +0 -0
  116. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/scripts/inspect_grib.py +0 -0
  117. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/scripts/inspect_zarr.py +0 -0
  118. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_coastwatch.py +0 -0
  119. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_dbofs.py +0 -0
  120. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_dispatcher.py +0 -0
  121. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_erddap.py +0 -0
  122. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hrrr_atm.py +0 -0
  123. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hrrr_idx.py +0 -0
  124. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hrrr_levels.py +0 -0
  125. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hrrr_points.py +0 -0
  126. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_hycom.py +0 -0
  127. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_jobs.py +0 -0
  128. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_layered_mesh.py +0 -0
  129. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_main.py +0 -0
  130. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_mesh_frames.py +0 -0
  131. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_necofs_hindcast.py +0 -0
  132. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_necofs_mesh_cache.py +0 -0
  133. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_necofs_parent.py +0 -0
  134. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_necofs_volume.py +0 -0
  135. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_noaa_currents.py +0 -0
  136. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_nwps.py +0 -0
  137. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_nyofs.py +0 -0
  138. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_obc_donor_report.py +0 -0
  139. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_obc_pipeline.py +0 -0
  140. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_removed_routes.py +0 -0
  141. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_settings.py +0 -0
  142. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_usgs.py +0 -0
  143. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_usgs_rivers.py +0 -0
  144. {forcingkit-0.15.1 → forcingkit-0.17.0}/tests/unit/test_zarr_stream.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.15.1
3
+ Version: 0.17.0
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -225,7 +225,7 @@ Requires-Dist: fastapi>=0.135.1
225
225
  Requires-Dist: h5netcdf>=1.8.1
226
226
  Requires-Dist: h5py>=3.16.0
227
227
  Requires-Dist: httpx>=0.28.1
228
- Requires-Dist: insitukit<0.4,>=0.3
228
+ Requires-Dist: insitukit<0.6,>=0.5
229
229
  Requires-Dist: jinja2>=3.0.0
230
230
  Requires-Dist: matplotlib>=3.10.9
231
231
  Requires-Dist: pandas>=2.2
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.15.1"
12
+ release = "0.17.0"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -149,10 +149,19 @@ These serve validation rather than forcing:
149
149
  fallback to MSL where a station has no NAVD88 datum. Requests go through
150
150
  `insitukit <https://github.com/lhzn-io/insitukit>`__, which keeps them at least 1 s apart,
151
151
  splits windows longer than the API's 31-day limit, and retries server errors and throttling;
152
- ``end_time`` is inclusive, as in the API. Current predictions (``/api/v1/currents``) and the
152
+ ``end_time`` is inclusive, as in the API. Each reading carries CO-OPS's quality level (``q``:
153
+ ``p`` preliminary, ``v`` verified) and ``metadata.quality`` sums up the window. Since 0.17.0 a
154
+ verified window is cached for good and one still holding preliminary readings for a day, because
155
+ CO-OPS replaces preliminary readings once it has reviewed them. Current predictions
156
+ (``/api/v1/currents``) and the
153
157
  current-station list (``/api/v1/currents/stations``) use the same client.
154
158
  - **NDBC** buoy meteorology and ADCP profiles (``/api/v1/ndbc``), from the `NOAA National Data
155
- Buoy Center <https://www.ndbc.noaa.gov/>`__.
159
+ Buoy Center <https://www.ndbc.noaa.gov/>`__, through insitukit's NDBC client: each month from
160
+ the monthly, yearly or real-time file, each file downloaded once, and each column's own
161
+ missing-value sentinel read as missing (since 0.16.0; earlier releases also dropped real
162
+ readings equal to another column's sentinel, such as a 999 hPa low or a 99-degree wind). A
163
+ window is cached only when none of it came from the real-time files, whose readings have not
164
+ yet been through NDBC's quality control (since 0.17.0).
156
165
  - **UConn ERDDAP** water-column profiles (``/api/v1/telemetry/station`` and ``/bbox``), from the
157
166
  `University of Connecticut Department of Marine Sciences <https://marinesciences.uconn.edu/>`__
158
167
  `ERDDAP server <http://merlin.dms.uconn.edu:8080/erddap/index.html>`__.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.15.1"
7
+ version = "0.17.0"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -26,8 +26,8 @@ dependencies = [
26
26
  "eccodeslib>=2.46.0",
27
27
  "fastapi>=0.135.1",
28
28
  "httpx>=0.28.1",
29
- # NOAA CO-OPS (fetchers/noaa.py) and the UConn ERDDAP profiles (fetchers/erddap.py).
30
- "insitukit>=0.3,<0.4",
29
+ # NOAA CO-OPS (fetchers/noaa.py), the UConn ERDDAP profiles (fetchers/erddap.py) and NDBC (fetchers/ndbc.py).
30
+ "insitukit>=0.5,<0.6",
31
31
  "pandas>=2.2",
32
32
  "pydantic>=2.12.5",
33
33
  "jinja2>=3.0.0",
@@ -0,0 +1,166 @@
1
+ """NDBC buoy observations for model validation: standard meteorology (sea and air temperature,
2
+ pressure, wind, waves) and near-surface ADCP currents, through insitukit.
3
+
4
+ insitukit's NDBC client reads each month from the first file NDBC publishes it in: the per-month
5
+ file for months of the current year, then the yearly historical file, then the rolling 45-day
6
+ real-time file. It downloads each file once per call, paces and identifies its requests, and turns
7
+ NDBC's missing-value sentinels (99, 999, 9999, MM) into NaN. This module keeps forcingkit's
8
+ function names and the JSON shape of ``fetch_ndbc``. These observations validate a model; they
9
+ never force it.
10
+ """
11
+
12
+ import json
13
+ import math
14
+ import os
15
+ from typing import Optional
16
+
17
+ import numpy as np
18
+ import pandas as pd
19
+ from insitukit import ndbc as _ndbc
20
+
21
+ PRODUCTS = _ndbc.PRODUCTS
22
+ # stdmet columns carried through, with units.
23
+ STDMET_COLUMNS = {
24
+ "WTMP": "degC", # sea surface temperature
25
+ "ATMP": "degC", # air temperature
26
+ "PRES": "hPa", # sea level pressure
27
+ "WSPD": "m s-1", # wind speed
28
+ "WDIR": "degree from true north (direction the wind comes from)",
29
+ "GST": "m s-1",
30
+ "WVHT": "m",
31
+ "DPD": "s",
32
+ }
33
+
34
+ month_code = _ndbc.month_code
35
+
36
+
37
+ def candidate_urls(station: str, product: str, year: int, month: int) -> list[str]:
38
+ """URLs that may hold `product` for `station` in the given month, in the order to try."""
39
+ return _ndbc.urls(station, product, year, month)
40
+
41
+
42
+ def parse_ndbc_text(text: str) -> pd.DataFrame:
43
+ """Parse an NDBC whitespace-delimited file into a frame indexed by UTC time (naive),
44
+ sentinels and `MM` as NaN."""
45
+ df = _ndbc.parse(text)
46
+ return df.set_index(pd.DatetimeIndex(df.pop("time")).tz_localize(None))
47
+
48
+
49
+ def _utc_naive(t) -> pd.Timestamp:
50
+ t = pd.Timestamp(t)
51
+ return t if t.tzinfo is None else t.tz_convert("UTC").tz_localize(None)
52
+
53
+
54
+ def station_position(station: str, get=None) -> tuple[float, float]:
55
+ """(lat, lon) of an NDBC station from its station page."""
56
+ return _ndbc.position(station, get=get)
57
+
58
+
59
+ def fetch_product(
60
+ station: str, product: str, start: pd.Timestamp, end: pd.Timestamp, get=None
61
+ ):
62
+ """`product` for `station` over [start, end] (naive UTC, both included): (frame indexed by
63
+ naive UTC time, source URLs). Each month is taken from the first source that has it."""
64
+ t0 = _utc_naive(start).tz_localize("UTC")
65
+ t1 = _utc_naive(end).tz_localize("UTC") + pd.Timedelta(minutes=1)
66
+ df = _ndbc.fetch(station, t0, t1, product, get=get)
67
+ sources = list(df.attrs.get("sources", []))
68
+ if df.empty:
69
+ return pd.DataFrame(), sources
70
+ df = df.set_index(pd.DatetimeIndex(df.pop("time")).tz_localize(None))
71
+ return df[df.index <= _utc_naive(end)], sources
72
+
73
+
74
+ def _real_time(sources: list[str]) -> bool:
75
+ """Whether any source is one of NDBC's real-time files (not yet quality controlled)."""
76
+ return any("/data/realtime2/" in s for s in sources)
77
+
78
+
79
+ def _list(values) -> list:
80
+ return [
81
+ None if (v is None or (isinstance(v, float) and math.isnan(v))) else float(v)
82
+ for v in values
83
+ ]
84
+
85
+
86
+ def fetch_ndbc(
87
+ station: str,
88
+ start_time: str,
89
+ end_time: str,
90
+ products: tuple[str, ...] = PRODUCTS,
91
+ cache_dir: Optional[str] = None,
92
+ get=None,
93
+ position: Optional[tuple[float, float]] = None,
94
+ ) -> dict:
95
+ """Observations at an NDBC station over [start_time, end_time] (UTC) as a JSON-ready dict.
96
+
97
+ stdmet: times plus the columns of `STDMET_COLUMNS`. adcp: times, the bin depth and the
98
+ current as eastward and northward components (m/s); NDBC reports speed in cm/s and the
99
+ direction the current flows toward, in degrees true. Missing values are null. A window that
100
+ ended more than two days ago is cached as JSON under `cache_dir`, unless any of it came from
101
+ NDBC's real-time files: those readings have not been through NDBC's quality control yet, and
102
+ the monthly and yearly files that replace them have, so such a window is fetched again until
103
+ they cover it. `get` (a URL to text or None) and `position` exist so tests can run without the
104
+ network.
105
+ """
106
+ start, end = _utc_naive(start_time), _utc_naive(end_time)
107
+ unknown = set(products) - set(PRODUCTS)
108
+ if unknown:
109
+ raise ValueError(f"unknown NDBC products {sorted(unknown)}")
110
+
111
+ cache_path = None
112
+ if cache_dir:
113
+ os.makedirs(os.path.join(cache_dir, "ndbc"), exist_ok=True)
114
+ key = f"{station.lower()}_{start:%Y%m%dT%H%M}_{end:%Y%m%dT%H%M}_{'-'.join(sorted(products))}"
115
+ cache_path = os.path.join(cache_dir, "ndbc", f"{key}.json")
116
+ if os.path.exists(cache_path):
117
+ with open(cache_path) as f:
118
+ cached = json.load(f)
119
+ if not _real_time(cached.get("sources", [])):
120
+ return cached
121
+
122
+ lat, lon = position if position is not None else station_position(station, get)
123
+ out: dict = {
124
+ "station_id": station.upper(),
125
+ "lat": lat,
126
+ "lon": lon,
127
+ "start_time": f"{start:%Y-%m-%dT%H:%M:%SZ}",
128
+ "end_time": f"{end:%Y-%m-%dT%H:%M:%SZ}",
129
+ "products": {},
130
+ "sources": [],
131
+ }
132
+ for product in products:
133
+ df, sources = fetch_product(station, product, start, end, get=get)
134
+ out["sources"] += sources
135
+ times = [f"{t:%Y-%m-%dT%H:%M:%SZ}" for t in df.index]
136
+ if product == "stdmet":
137
+ out["products"]["stdmet"] = {
138
+ "times": times,
139
+ "units": {c: u for c, u in STDMET_COLUMNS.items() if c in df},
140
+ **{c: _list(df[c]) for c in STDMET_COLUMNS if c in df},
141
+ }
142
+ else:
143
+ speed = df["SPD01"] / 100.0 if "SPD01" in df else pd.Series(dtype=float)
144
+ toward = (
145
+ np.deg2rad(df["DIR01"]) if "DIR01" in df else pd.Series(dtype=float)
146
+ )
147
+ out["products"]["adcp"] = {
148
+ "times": times,
149
+ "depth_m": _list(df["DEP01"]) if "DEP01" in df else [],
150
+ "u": _list(speed * np.sin(toward)),
151
+ "v": _list(speed * np.cos(toward)),
152
+ "units": {
153
+ "u": "m s-1 eastward",
154
+ "v": "m s-1 northward",
155
+ "depth_m": "m",
156
+ },
157
+ }
158
+
159
+ if (
160
+ cache_path
161
+ and end < pd.Timestamp.utcnow().tz_localize(None) - pd.Timedelta(days=2)
162
+ and not _real_time(out["sources"])
163
+ ):
164
+ with open(cache_path, "w") as f:
165
+ json.dump(out, f)
166
+ return out
@@ -20,6 +20,9 @@ logger = logging.getLogger(__name__)
20
20
 
21
21
  # Water-level datums to request, in order of preference.
22
22
  TIDE_DATUMS = ("NAVD", "MSL")
23
+ # How long a cached water-level window that still holds preliminary readings is served before
24
+ # CO-OPS is asked again (a verified window is kept for good).
25
+ PRELIMINARY_TTL_S = 24 * 3600
23
26
 
24
27
 
25
28
  def _utc(stamp: str) -> datetime:
@@ -56,9 +59,17 @@ def fetch_noaa_tide_data(
56
59
  in metres, on NAVD88 where the station has it and MSL otherwise. Supports caching to avoid
57
60
  redundant API calls.
58
61
 
59
- Returns the shape of a CO-OPS response: ``{"metadata": {"id", "name", "lat", "lon", "datum"},
60
- "data": [{"t": "YYYY-MM-DD HH:MM", "v": "0.123"}, ...]}``, with ``lat``, ``lon`` and ``v`` as
61
- strings (``v`` empty for a missing reading). Raises RuntimeError when the window has no data.
62
+ Returns the shape of a CO-OPS response: ``{"metadata": {"id", "name", "lat", "lon", "datum",
63
+ "quality"}, "data": [{"t": "YYYY-MM-DD HH:MM", "v": "0.123", "q": "v"}, ...]}``, with ``lat``,
64
+ ``lon`` and ``v`` as strings (``v`` empty for a missing reading). ``q`` is CO-OPS's quality
65
+ level for the reading, ``p`` (preliminary) or ``v`` (verified); ``metadata.quality`` is
66
+ ``verified`` when every reading is, else ``preliminary``. Raises RuntimeError when the window
67
+ has no data.
68
+
69
+ A verified window is cached for good. CO-OPS replaces preliminary readings once it has
70
+ reviewed them, usually within a few months, so a window holding any is served from the cache
71
+ for ``PRELIMINARY_TTL_S`` (a day) and then fetched again, until it is verified. A cached file
72
+ without ``metadata.quality`` (written before 0.17.0) is fetched again.
62
73
 
63
74
  Args:
64
75
  station_id: NOAA station ID (e.g., '8516945' for Kings Point)
@@ -73,9 +84,18 @@ def fetch_noaa_tide_data(
73
84
  cache_path = os.path.join(cache_dir, f"{cache_key}.json")
74
85
 
75
86
  if not cache_bust and os.path.exists(cache_path):
76
- logger.info(f"Cache hit for NOAA station {station_id}: {cache_path}")
77
87
  with open(cache_path, "r") as f:
78
- return json.load(f)
88
+ cached = json.load(f)
89
+ quality = (cached.get("metadata") or {}).get("quality")
90
+ age_s = datetime.now().timestamp() - os.path.getmtime(cache_path)
91
+ if quality == "verified" or (
92
+ quality == "preliminary" and age_s < PRELIMINARY_TTL_S
93
+ ):
94
+ logger.info(f"Cache hit for NOAA station {station_id}: {cache_path}")
95
+ return cached
96
+ logger.info(
97
+ f"Cached NOAA window stale or unverified; fetching again: {cache_path}"
98
+ )
79
99
 
80
100
  logger.info(
81
101
  f"Fetching NOAA tide data for station {station_id} ({start_time} to {end_time})..."
@@ -87,7 +107,9 @@ def fetch_noaa_tide_data(
87
107
  # 8465705, publishes only tidal datums). The datum used is recorded in the metadata.
88
108
  for datum in TIDE_DATUMS:
89
109
  try:
90
- df = coops.fetch("water_level", station_id, start, end, datum=datum)
110
+ df = coops.fetch(
111
+ "water_level", station_id, start, end, datum=datum, quality=True
112
+ )
91
113
  break
92
114
  except coops.DatumUnavailable:
93
115
  if datum == TIDE_DATUMS[-1]:
@@ -101,6 +123,7 @@ def fetch_noaa_tide_data(
101
123
  f"({start_time} to {end_time})"
102
124
  )
103
125
  info = coops.metadata(station_id, expand="")
126
+ verified = bool((df["quality"] == "v").all())
104
127
  data = {
105
128
  "metadata": {
106
129
  "id": station_id,
@@ -108,15 +131,20 @@ def fetch_noaa_tide_data(
108
131
  "lat": str(info["lat"]),
109
132
  "lon": str(info["lng"]),
110
133
  "datum": datum,
134
+ "quality": "verified" if verified else "preliminary",
111
135
  },
112
136
  "data": [
113
137
  # Written as CO-OPS writes it: 3 decimals at most, no trailing zeros.
114
- {"t": _t(t), "v": "" if pd.isna(v) else str(round(float(v), 3))}
115
- for t, v in zip(df["time"], df["water_level"])
138
+ {
139
+ "t": _t(t),
140
+ "v": "" if pd.isna(v) else str(round(float(v), 3)),
141
+ "q": q or "",
142
+ }
143
+ for t, v, q in zip(df["time"], df["water_level"], df["quality"])
116
144
  ],
117
145
  }
118
146
 
119
- # Cache the successful response
147
+ # Kept for good when verified; a preliminary window expires (see PRELIMINARY_TTL_S).
120
148
  with open(cache_path, "w") as f:
121
149
  json.dump(data, f, indent=4)
122
150
 
@@ -95,3 +95,29 @@ def test_unknown_product_is_refused():
95
95
  ndbc.fetch_ndbc(
96
96
  "44091", "2026-04-02", "2026-04-03", ("waves",), position=(0, 0)
97
97
  )
98
+
99
+
100
+ def _counting(files):
101
+ asked = []
102
+ inner = _fixture_get(files)
103
+
104
+ def get(url):
105
+ asked.append(url)
106
+ return inner(url)
107
+
108
+ return get, asked
109
+
110
+
111
+ def test_only_quality_controlled_windows_are_cached(tmp_path):
112
+ window = ("44091", "2026-04-02T00:00:00Z", "2026-04-02T23:59:00Z", ("stdmet",))
113
+ # From the real-time file (not yet quality controlled): served, not cached.
114
+ get, asked = _counting({"realtime2": STDMET})
115
+ ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
116
+ assert not list((tmp_path / "ndbc").glob("*.json"))
117
+ # From the yearly file: cached, and served from the cache next time.
118
+ get, asked = _counting({"historical/stdmet": STDMET})
119
+ ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
120
+ assert len(list((tmp_path / "ndbc").glob("*.json"))) == 1
121
+ get, asked = _counting({})
122
+ out = ndbc.fetch_ndbc(*window, cache_dir=str(tmp_path), get=get, position=(0, 0))
123
+ assert asked == [] and out["products"]["stdmet"]["WTMP"] == [6.0, 6.1]
@@ -1,5 +1,7 @@
1
1
  """CO-OPS water level: NAVD88 where a station has it, MSL where it does not."""
2
2
 
3
+ import os
4
+
3
5
  import pytest
4
6
  from insitukit import coops
5
7
 
@@ -56,14 +58,49 @@ def test_keeps_the_co_ops_shape_with_the_end_included(tmp_path, monkeypatch):
56
58
  "lat": "41.2833",
57
59
  "lon": "-72.9083",
58
60
  "datum": "NAVD",
61
+ "quality": "preliminary",
59
62
  }
60
63
  # The reading at end_time is kept; a missing reading is an empty string, as CO-OPS sends it.
61
64
  assert data["data"] == [
62
- {"t": "2026-04-02 00:00", "v": "0.1"},
63
- {"t": "2026-04-02 04:00", "v": ""},
65
+ {"t": "2026-04-02 00:00", "v": "0.1", "q": "p"},
66
+ {"t": "2026-04-02 04:00", "v": "", "q": "p"},
64
67
  ]
65
68
 
66
69
 
67
70
  def test_an_empty_window_is_an_error(tmp_path, monkeypatch):
68
71
  with pytest.raises(RuntimeError, match="No data"):
69
72
  _fetch(tmp_path, monkeypatch, {"NAVD": None})
73
+
74
+
75
+ VERIFIED = {"data": [{**r, "q": "v"} for r in ROWS["data"]]}
76
+
77
+
78
+ def _age(tmp_path, seconds):
79
+ for p in tmp_path.glob("*.json"):
80
+ t = p.stat().st_mtime - seconds
81
+ os.utime(p, (t, t))
82
+
83
+
84
+ def test_a_preliminary_window_expires_and_a_verified_one_is_kept(tmp_path, monkeypatch):
85
+ # Preliminary: cached for a day, then CO-OPS is asked again.
86
+ _, calls = _fetch(tmp_path, monkeypatch, {"NAVD": ROWS})
87
+ assert calls == ["NAVD"]
88
+ _, calls = _fetch(tmp_path, monkeypatch, {"NAVD": ROWS})
89
+ assert calls == []
90
+ _age(tmp_path, noaa.PRELIMINARY_TTL_S + 60)
91
+ # A day on, CO-OPS has verified the window: it replaces the cached copy, kept for good.
92
+ data, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
93
+ assert calls == ["NAVD"] and data["metadata"]["quality"] == "verified"
94
+ _age(tmp_path, 365 * 86400)
95
+ _, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
96
+ assert calls == []
97
+
98
+
99
+ def test_a_cache_file_without_a_quality_level_is_fetched_again(tmp_path, monkeypatch):
100
+ # Written by forcingkit before 0.17.0 (raw CO-OPS JSON, or the 0.14 shape): no metadata.quality.
101
+ key = "8465705_2026-04-02T00-00-00_2026-04-02T04-00-00"
102
+ (tmp_path / f"{key}.json").write_text(
103
+ '{"metadata": {"id": "8465705", "datum": "NAVD"}, "data": [{"t": "2026-04-02 00:00", "v": "9.9"}]}'
104
+ )
105
+ data, calls = _fetch(tmp_path, monkeypatch, {"NAVD": VERIFIED})
106
+ assert calls == ["NAVD"] and data["data"][0]["v"] == "0.1"
@@ -1000,7 +1000,7 @@ wheels = [
1000
1000
 
1001
1001
  [[package]]
1002
1002
  name = "forcingkit"
1003
- version = "0.15.1"
1003
+ version = "0.17.0"
1004
1004
  source = { editable = "." }
1005
1005
  dependencies = [
1006
1006
  { name = "bottleneck" },
@@ -1062,7 +1062,7 @@ requires-dist = [
1062
1062
  { name = "h5netcdf", specifier = ">=1.8.1" },
1063
1063
  { name = "h5py", specifier = ">=3.16.0" },
1064
1064
  { name = "httpx", specifier = ">=0.28.1" },
1065
- { name = "insitukit", specifier = ">=0.3,<0.4" },
1065
+ { name = "insitukit", specifier = ">=0.5,<0.6" },
1066
1066
  { name = "jinja2", specifier = ">=3.0.0" },
1067
1067
  { name = "matplotlib", specifier = ">=3.10.9" },
1068
1068
  { name = "pandas", specifier = ">=2.2" },
@@ -1612,16 +1612,16 @@ wheels = [
1612
1612
 
1613
1613
  [[package]]
1614
1614
  name = "insitukit"
1615
- version = "0.3.0"
1615
+ version = "0.5.0"
1616
1616
  source = { registry = "https://pypi.org/simple" }
1617
1617
  dependencies = [
1618
1618
  { name = "numpy" },
1619
1619
  { name = "pandas" },
1620
1620
  { name = "requests" },
1621
1621
  ]
1622
- sdist = { url = "https://files.pythonhosted.org/packages/d6/2f/64ab83ca7705afcab426cfe64953dec222e6fc148bd47bf19118d9df9d70/insitukit-0.3.0.tar.gz", hash = "sha256:fa1ed9c1cf89995b6edf4a550f8835e327f352b0df69a756b3185f5dc7b67aa6", size = 99038, upload-time = "2026-11-06T00:48:46.489Z" }
1622
+ sdist = { url = "https://files.pythonhosted.org/packages/7d/78/f978cdea497458927cec2cd2d49e9dda401780ce85063e156d889fe0b1f3/insitukit-0.5.0.tar.gz", hash = "sha256:77e4ec1fabc170ee49fc4750839da3cdd8fe900ccdc9bffb2e9b4951114bdd4e", size = 105398, upload-time = "2026-11-06T15:44:09.573Z" }
1623
1623
  wheels = [
1624
- { url = "https://files.pythonhosted.org/packages/8a/00/5b0fa89c8a08b99d36533c4849e20f6061a65901693de02806ff3a22f936/insitukit-0.3.0-py3-none-any.whl", hash = "sha256:396df69830c4d921a40c7693c0f24a7e02a86617730acf010cdc4fdf2a6abb35", size = 22860, upload-time = "2026-11-06T00:48:45.069Z" },
1624
+ { url = "https://files.pythonhosted.org/packages/8a/c8/9d0df8dded9c9b3962a66f2a6b7de0ddc20e43f7aad2c140c4886e9a65c4/insitukit-0.5.0-py3-none-any.whl", hash = "sha256:8dbc7c279e841eb10de5ca6b6239389a8d896c7f7a1e8066522b2ebd35efa18e", size = 27661, upload-time = "2026-11-06T15:44:08.173Z" },
1625
1625
  ]
1626
1626
 
1627
1627
  [[package]]
@@ -1,231 +0,0 @@
1
- """NDBC buoy observations for model validation: standard meteorology (sea and air temperature,
2
- pressure, wind, waves) and near-surface ADCP currents.
3
-
4
- Files come from the NDBC data server in the order they are published: the per-month file for
5
- months of the current year (`data/<product>/<Mon>/<id><m><yyyy>.txt.gz`), then the yearly
6
- historical file (`data/historical/<product>/<id>h<yyyy>.txt.gz`), then the rolling 45-day
7
- real-time file (`data/realtime2/<id>.txt` or `.adcp`). Missing-value sentinels (99, 999, 9999,
8
- MM) become NaN. These observations validate a model; they never force it.
9
- """
10
-
11
- import gzip
12
- import json
13
- import logging
14
- import math
15
- import os
16
- import re
17
- from typing import Optional
18
-
19
- import numpy as np
20
- import pandas as pd
21
- import requests
22
-
23
- logger = logging.getLogger(__name__)
24
-
25
- NDBC = "https://www.ndbc.noaa.gov"
26
- PRODUCTS = ("stdmet", "adcp")
27
- REALTIME_SUFFIX = {"stdmet": "txt", "adcp": "adcp"}
28
- MONTH_ABBR = [
29
- "Jan",
30
- "Feb",
31
- "Mar",
32
- "Apr",
33
- "May",
34
- "Jun",
35
- "Jul",
36
- "Aug",
37
- "Sep",
38
- "Oct",
39
- "Nov",
40
- "Dec",
41
- ]
42
- # Column sentinels NDBC uses for "not measured".
43
- SENTINELS = {99.0, 999.0, 9999.0, 99.00}
44
- # stdmet columns carried through, with units.
45
- STDMET_COLUMNS = {
46
- "WTMP": "degC", # sea surface temperature
47
- "ATMP": "degC", # air temperature
48
- "PRES": "hPa", # sea level pressure
49
- "WSPD": "m s-1", # wind speed
50
- "WDIR": "degree from true north (direction the wind comes from)",
51
- "GST": "m s-1",
52
- "WVHT": "m",
53
- "DPD": "s",
54
- }
55
-
56
-
57
- def month_code(month: int) -> str:
58
- """NDBC's month code in per-month file names: 1 to 9, then a, b, c for October to December."""
59
- return str(month) if month < 10 else "abc"[month - 10]
60
-
61
-
62
- def candidate_urls(station: str, product: str, year: int, month: int) -> list[str]:
63
- """URLs that may hold `product` for `station` in the given month, in the order to try."""
64
- station = station.lower()
65
- return [
66
- f"{NDBC}/data/{product}/{MONTH_ABBR[month - 1]}/{station}{month_code(month)}{year}.txt.gz",
67
- f"{NDBC}/data/historical/{product}/{station}h{year}.txt.gz",
68
- f"{NDBC}/data/realtime2/{station.upper()}.{REALTIME_SUFFIX[product]}",
69
- ]
70
-
71
-
72
- def parse_ndbc_text(text: str) -> pd.DataFrame:
73
- """Parse an NDBC whitespace-delimited file (two `#` header rows) into a frame indexed by UTC
74
- time, sentinels and `MM` as NaN. Handles both four- and two-digit years, with or without a
75
- minute column."""
76
- lines = [ln for ln in text.splitlines() if ln.strip()]
77
- if not lines or not lines[0].startswith("#"):
78
- raise ValueError("not an NDBC data file: no header row")
79
- names = lines[0].lstrip("#").split()
80
- rows = [ln.split() for ln in lines if not ln.startswith("#")]
81
- df = pd.DataFrame(rows, columns=names[: len(rows[0])] if rows else names)
82
- year_col = names[0]
83
- years = df[year_col].astype(int)
84
- years = years.where(years >= 100, years + 1900)
85
- minutes = df["mm"].astype(int) if "mm" in df else 0
86
- df.index = pd.to_datetime(
87
- dict(
88
- year=years,
89
- month=df["MM"].astype(int),
90
- day=df["DD"].astype(int),
91
- hour=df["hh"].astype(int),
92
- minute=minutes,
93
- )
94
- )
95
- df = df.drop(columns=[c for c in (year_col, "MM", "DD", "hh", "mm") if c in df])
96
- df = df.replace("MM", np.nan).apply(pd.to_numeric, errors="coerce")
97
- return df.mask(df.isin(SENTINELS))
98
-
99
-
100
- def _get(url: str, timeout: float = 60.0) -> Optional[str]:
101
- r = requests.get(url, timeout=timeout)
102
- if r.status_code == 404:
103
- return None
104
- r.raise_for_status()
105
- if url.endswith(".gz"):
106
- return gzip.decompress(r.content).decode("utf-8", errors="replace")
107
- return r.text
108
-
109
-
110
- def _utc_naive(t) -> pd.Timestamp:
111
- t = pd.Timestamp(t)
112
- return t if t.tzinfo is None else t.tz_convert("UTC").tz_localize(None)
113
-
114
-
115
- def station_position(station: str, get=_get) -> tuple[float, float]:
116
- """(lat, lon) of an NDBC station from its station page."""
117
- html = get(f"{NDBC}/station_page.php?station={station.lower()}") or ""
118
- m = re.search(r"(\d+\.\d+)\s*([NS])\s+(\d+\.\d+)\s*([EW])", html)
119
- if not m:
120
- raise ValueError(f"No position found for NDBC station {station}")
121
- lat = float(m.group(1)) * (1 if m.group(2) == "N" else -1)
122
- lon = float(m.group(3)) * (1 if m.group(4) == "E" else -1)
123
- return lat, lon
124
-
125
-
126
- def fetch_product(
127
- station: str, product: str, start: pd.Timestamp, end: pd.Timestamp, get=_get
128
- ):
129
- """`product` for `station` over [start, end]: (frame, source URLs). Each month is taken from
130
- the first source that has it."""
131
- frames, sources = [], []
132
- for month_start in pd.date_range(start.normalize().replace(day=1), end, freq="MS"):
133
- for url in candidate_urls(
134
- station, product, month_start.year, month_start.month
135
- ):
136
- text = get(url)
137
- if text is None:
138
- continue
139
- df = parse_ndbc_text(text)
140
- df = df[(df.index >= start) & (df.index <= end)]
141
- if len(df):
142
- frames.append(df)
143
- sources.append(url)
144
- break
145
- if not frames:
146
- return pd.DataFrame(), sources
147
- df = pd.concat(frames)
148
- return df[~df.index.duplicated(keep="first")].sort_index(), sources
149
-
150
-
151
- def _list(values) -> list:
152
- return [
153
- None if (v is None or (isinstance(v, float) and math.isnan(v))) else float(v)
154
- for v in values
155
- ]
156
-
157
-
158
- def fetch_ndbc(
159
- station: str,
160
- start_time: str,
161
- end_time: str,
162
- products: tuple[str, ...] = PRODUCTS,
163
- cache_dir: Optional[str] = None,
164
- get=_get,
165
- position: Optional[tuple[float, float]] = None,
166
- ) -> dict:
167
- """Observations at an NDBC station over [start_time, end_time] (UTC) as a JSON-ready dict.
168
-
169
- stdmet: times plus the columns of `STDMET_COLUMNS`. adcp: times, the bin depth and the
170
- current as eastward and northward components (m/s); NDBC reports speed in cm/s and the
171
- direction the current flows toward, in degrees true. Missing values are null. A window that
172
- ended more than two days ago is cached as JSON under `cache_dir`. `get` (a URL to text or
173
- None) and `position` exist so tests can run without the network.
174
- """
175
- start, end = _utc_naive(start_time), _utc_naive(end_time)
176
- unknown = set(products) - set(PRODUCTS)
177
- if unknown:
178
- raise ValueError(f"unknown NDBC products {sorted(unknown)}")
179
-
180
- cache_path = None
181
- if cache_dir:
182
- os.makedirs(os.path.join(cache_dir, "ndbc"), exist_ok=True)
183
- key = f"{station.lower()}_{start:%Y%m%dT%H%M}_{end:%Y%m%dT%H%M}_{'-'.join(sorted(products))}"
184
- cache_path = os.path.join(cache_dir, "ndbc", f"{key}.json")
185
- if os.path.exists(cache_path):
186
- with open(cache_path) as f:
187
- return json.load(f)
188
-
189
- lat, lon = position if position is not None else station_position(station, get)
190
- out: dict = {
191
- "station_id": station.upper(),
192
- "lat": lat,
193
- "lon": lon,
194
- "start_time": f"{start:%Y-%m-%dT%H:%M:%SZ}",
195
- "end_time": f"{end:%Y-%m-%dT%H:%M:%SZ}",
196
- "products": {},
197
- "sources": [],
198
- }
199
- for product in products:
200
- df, sources = fetch_product(station, product, start, end, get=get)
201
- out["sources"] += sources
202
- times = [f"{t:%Y-%m-%dT%H:%M:%SZ}" for t in df.index]
203
- if product == "stdmet":
204
- out["products"]["stdmet"] = {
205
- "times": times,
206
- "units": {c: u for c, u in STDMET_COLUMNS.items() if c in df},
207
- **{c: _list(df[c]) for c in STDMET_COLUMNS if c in df},
208
- }
209
- else:
210
- speed = df["SPD01"] / 100.0 if "SPD01" in df else pd.Series(dtype=float)
211
- toward = (
212
- np.deg2rad(df["DIR01"]) if "DIR01" in df else pd.Series(dtype=float)
213
- )
214
- out["products"]["adcp"] = {
215
- "times": times,
216
- "depth_m": _list(df["DEP01"]) if "DEP01" in df else [],
217
- "u": _list(speed * np.sin(toward)),
218
- "v": _list(speed * np.cos(toward)),
219
- "units": {
220
- "u": "m s-1 eastward",
221
- "v": "m s-1 northward",
222
- "depth_m": "m",
223
- },
224
- }
225
-
226
- if cache_path and end < pd.Timestamp.utcnow().tz_localize(None) - pd.Timedelta(
227
- days=2
228
- ):
229
- with open(cache_path, "w") as f:
230
- json.dump(out, f)
231
- return out
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes