forcingkit 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. {forcingkit-0.3.0 → forcingkit-0.4.0}/.agents/forcingkit.md +4 -4
  2. {forcingkit-0.3.0 → forcingkit-0.4.0}/.env.template +3 -4
  3. {forcingkit-0.3.0 → forcingkit-0.4.0}/AGENTS.md +4 -3
  4. {forcingkit-0.3.0 → forcingkit-0.4.0}/PKG-INFO +1 -1
  5. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/atmospheric_forcing.rst +10 -5
  6. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/conf.py +1 -1
  7. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/fetchers.rst +19 -10
  8. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/hycom.rst +76 -16
  9. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/index.rst +3 -2
  10. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/necofs.rst +2 -1
  11. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/nyofs.rst +4 -2
  12. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/roadmap.rst +18 -12
  13. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/sources.rst +2 -1
  14. {forcingkit-0.3.0 → forcingkit-0.4.0}/pyproject.toml +1 -1
  15. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/dispatcher.py +33 -3
  16. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/erddap.py +3 -2
  17. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/hycom.py +44 -11
  18. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/necofs.py +2 -1
  19. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/noaa.py +1 -1
  20. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/settings.py +2 -3
  21. forcingkit-0.3.0/tests/integration/test_obc_mab.py → forcingkit-0.4.0/tests/integration/test_obc_offshore_nj.py +4 -4
  22. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/scripts/inspect_dem.py +12 -5
  23. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_hycom.py +111 -6
  24. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_settings.py +4 -7
  25. {forcingkit-0.3.0 → forcingkit-0.4.0}/uv.lock +1 -1
  26. {forcingkit-0.3.0 → forcingkit-0.4.0}/.claude/CLAUDE.md +0 -0
  27. {forcingkit-0.3.0 → forcingkit-0.4.0}/.github/copilot-instructions.md +0 -0
  28. {forcingkit-0.3.0 → forcingkit-0.4.0}/.github/workflows/docs.yml +0 -0
  29. {forcingkit-0.3.0 → forcingkit-0.4.0}/.github/workflows/publish.yml +0 -0
  30. {forcingkit-0.3.0 → forcingkit-0.4.0}/.github/workflows/tests.yml +0 -0
  31. {forcingkit-0.3.0 → forcingkit-0.4.0}/.gitignore +0 -0
  32. {forcingkit-0.3.0 → forcingkit-0.4.0}/.markdownlint.yaml +0 -0
  33. {forcingkit-0.3.0 → forcingkit-0.4.0}/.pre-commit-config.yaml +0 -0
  34. {forcingkit-0.3.0 → forcingkit-0.4.0}/.python-version +0 -0
  35. {forcingkit-0.3.0 → forcingkit-0.4.0}/CONTRIBUTING.md +0 -0
  36. {forcingkit-0.3.0 → forcingkit-0.4.0}/Dockerfile +0 -0
  37. {forcingkit-0.3.0 → forcingkit-0.4.0}/LICENSE +0 -0
  38. {forcingkit-0.3.0 → forcingkit-0.4.0}/README.md +0 -0
  39. {forcingkit-0.3.0 → forcingkit-0.4.0}/docker-compose.yml +0 -0
  40. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/Makefile +0 -0
  41. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/make.bat +0 -0
  42. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/requirements-docs.txt +0 -0
  43. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/_extra/CNAME +0 -0
  44. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/_static/custom.css +0 -0
  45. {forcingkit-0.3.0 → forcingkit-0.4.0}/docs/source/_static/forcingkit-noreaster-wind.gif +0 -0
  46. {forcingkit-0.3.0 → forcingkit-0.4.0}/main.py +0 -0
  47. {forcingkit-0.3.0 → forcingkit-0.4.0}/service/__init__.py +0 -0
  48. {forcingkit-0.3.0 → forcingkit-0.4.0}/service/forcingkit_serve/__init__.py +0 -0
  49. {forcingkit-0.3.0 → forcingkit-0.4.0}/service/forcingkit_serve/main.py +0 -0
  50. {forcingkit-0.3.0 → forcingkit-0.4.0}/service/forcingkit_serve/routers/bathymetry.py +0 -0
  51. {forcingkit-0.3.0 → forcingkit-0.4.0}/service/forcingkit_serve/routers/plotly_api.py +0 -0
  52. {forcingkit-0.3.0 → forcingkit-0.4.0}/service/forcingkit_serve/routers/removed.py +0 -0
  53. {forcingkit-0.3.0 → forcingkit-0.4.0}/service/forcingkit_serve/routers/viewer.py +0 -0
  54. {forcingkit-0.3.0 → forcingkit-0.4.0}/service/run_server.py +0 -0
  55. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/__init__.py +0 -0
  56. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/dbofs.py +0 -0
  57. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/hrrr.py +0 -0
  58. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/hrrr_atmosphere.py +0 -0
  59. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/hydrography.py +0 -0
  60. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/ndbc.py +0 -0
  61. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/fetchers/nyofs.py +0 -0
  62. {forcingkit-0.3.0 → forcingkit-0.4.0}/src/forcingkit/zarr_stream.py +0 -0
  63. {forcingkit-0.3.0 → forcingkit-0.4.0}/static/app.js +0 -0
  64. {forcingkit-0.3.0 → forcingkit-0.4.0}/static/favicon.ico +0 -0
  65. {forcingkit-0.3.0 → forcingkit-0.4.0}/static/favicon.svg +0 -0
  66. {forcingkit-0.3.0 → forcingkit-0.4.0}/static/index.html +0 -0
  67. {forcingkit-0.3.0 → forcingkit-0.4.0}/static/logo.svg +0 -0
  68. {forcingkit-0.3.0 → forcingkit-0.4.0}/static/preview3d.js +0 -0
  69. {forcingkit-0.3.0 → forcingkit-0.4.0}/static/styles.css +0 -0
  70. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/__init__.py +0 -0
  71. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/integration/test_auth.py +0 -0
  72. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/integration/test_erddap_fetch.py +0 -0
  73. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/integration/test_nyofs_obc_fetch.py +0 -0
  74. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/integration/test_ofs_archive_paths.py +0 -0
  75. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/integration/test_s3_roms_fetchers.py +0 -0
  76. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/scripts/inspect_grib.py +0 -0
  77. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/scripts/inspect_zarr.py +0 -0
  78. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_dbofs.py +0 -0
  79. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_dispatcher.py +0 -0
  80. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_hrrr_atm.py +0 -0
  81. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_hrrr_idx.py +0 -0
  82. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_main.py +0 -0
  83. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_ndbc.py +0 -0
  84. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_necofs_parent.py +0 -0
  85. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_noaa_currents.py +0 -0
  86. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_noaa_datum.py +0 -0
  87. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_nyofs.py +0 -0
  88. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_obc_donor_report.py +0 -0
  89. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_obc_pipeline.py +0 -0
  90. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_removed_routes.py +0 -0
  91. {forcingkit-0.3.0 → forcingkit-0.4.0}/tests/unit/test_zarr_stream.py +0 -0
@@ -3,8 +3,8 @@
3
3
  ## Mission
4
4
 
5
5
  Python microservice providing real-time and historical forcing (the parent ocean and
6
- the HRRR atmosphere) and validation observations to the `coastal-sim` Julia physics
7
- engine.
6
+ the HRRR atmosphere) and validation observations to downstream ocean models, for
7
+ example ones built on Oceananigans.jl or NumericalEarth.
8
8
 
9
9
  ## Environment
10
10
 
@@ -41,7 +41,7 @@ HTTP Request → service/forcingkit_serve/main.py
41
41
  *Note: Because the architecture is modular, a future roadmap item includes the integration of TPXO (via `tpxo.py`) given usage rights.*
42
42
 
43
43
  Atmospheric forcing: HRRR from 2014-07-30 (`hrrr_atmosphere.py`, `/api/v1/atmosphere`).
44
- Earlier runs use ERA5 through NumericalEarth in coastal-sim, not this service.
44
+ Earlier runs use ERA5 through NumericalEarth in the model, not this service.
45
45
 
46
46
  ## Testing
47
47
 
@@ -89,7 +89,7 @@ uv run python -c "from forcingkit.fetchers import nyofs; print(nyofs.get_metadat
89
89
  - **Mocking strategy**: Dispatcher unit tests check tuple/dict return
90
90
  boundaries carefully. When modifying mocked fetchers, track
91
91
  keyword-argument vs positional argument boundaries.
92
- - **Grid normalization**: `coastal-sim` expects elevations positive
92
+ - **Grid normalization**: downstream models expect elevations positive
93
93
  up (LMSL/NAVD88). Normalize any inverted datasets in the fetcher
94
94
  tier before the dispatcher sees them.
95
95
  - **Cache keys**: Cache Zarr keys are deterministic hashes of bbox
@@ -1,13 +1,12 @@
1
1
  # forcingkit environment variables
2
2
  # Copy this file to .env and fill in your credentials.
3
- # This file is loaded by coastal-sim/docker-compose.yml via env_file.
3
+ # A docker-compose service can load it with env_file.
4
4
 
5
5
  # No credentials are required at present: HRRR, NOAA OFS, NECOFS, HYCOM, NDBC and
6
6
  # CO-OPS are all served without authentication.
7
7
 
8
- # Cache root for every store (default ~/.cache/forcingkit). The old names
9
- # ECODATA_CACHE_CACHE_DIR and COASTAL_SIM_DATA_CACHE_DIR are still read, with a
10
- # warning, until the next release.
8
+ # Cache root for every store (default ~/.cache/forcingkit). The old name
9
+ # ECODATA_CACHE_CACHE_DIR is still read, with a warning, until the next release.
11
10
  # FORCINGKIT_CACHE_DIR=~/.cache/forcingkit
12
11
 
13
12
  # Threads for parallel OPeNDAP reads (default 4; was ECODATA_CACHE_MAX_WORKERS).
@@ -22,9 +22,10 @@ guidance in this repository.
22
22
  ## Repository Focus
23
23
 
24
24
  forcingkit is a Python microservice that fetches, harmonizes, regrids, and serves
25
- real-time and historical forcing to the `coastal-sim` Julia physics engine: the parent
26
- ocean (`/api/v1/obc`, schema z-v2), the HRRR atmosphere (`/api/v1/atmosphere`), tides,
27
- and station telemetry and NDBC observations for validation.
25
+ real-time and historical forcing to downstream ocean models (for example ones built on
26
+ Oceananigans.jl or NumericalEarth): the parent ocean (`/api/v1/obc`, schema z-v3), the
27
+ HRRR atmosphere (`/api/v1/atmosphere`), tides, and station telemetry and NDBC
28
+ observations for validation.
28
29
 
29
30
  ## Core Rules
30
31
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: forcingkit
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
5
5
  Project-URL: Documentation, https://forcingkit.docs.lhzn.io
6
6
  Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues
@@ -2,7 +2,8 @@ Atmospheric Forcing Datasets
2
2
  ============================
3
3
 
4
4
  As of 2026-10-04. Which atmospheric datasets can force a coastal ocean model through this service
5
- or through `NumericalEarth <https://github.com/NumericalEarth/NumericalEarth.jl>`__, how they compare, and which to choose.
5
+ or through `NumericalEarth <https://github.com/NumericalEarth/NumericalEarth.jl>`__, how they
6
+ compare, and which to choose.
6
7
 
7
8
  An ocean model's surface fluxes need, at every hour of the run: 10 m wind (eastward and
8
9
  northward), 2 m air temperature and specific humidity, surface pressure, precipitation, and
@@ -151,9 +152,11 @@ ERA5 (fallback)
151
152
 
152
153
  *Provenance.* `ECMWF <https://www.ecmwf.int/>`__'s fifth-generation global reanalysis for the
153
154
  `Copernicus Climate Change Service <https://climate.copernicus.eu/>`__ (`Hersbach et al., 2020
154
- <https://doi.org/10.1002/qj.3803>`__): 0.25 degree grid (about 31 km), hourly, 1940 to the present. ERA5T, the initial release,
155
+ <https://doi.org/10.1002/qj.3803>`__): 0.25 degree grid (about 31 km), hourly, 1940 to the present.
156
+ ERA5T, the initial release,
155
157
  appears about five days behind real time and is overwritten by the final ERA5 two to three months
156
- later. Requires a free `Copernicus Climate Data Store <https://cds.climate.copernicus.eu/>`__ account (credentials in ``~/.cdsapirc``);
158
+ later. Requires a free `Copernicus Climate Data Store <https://cds.climate.copernicus.eu/>`__
159
+ account (credentials in ``~/.cdsapirc``);
157
160
  Copernicus licence, attribution required.
158
161
 
159
162
  *How it is used.* A model reads it through NumericalEarth's ``ERA5PrescribedAtmosphere`` and
@@ -167,7 +170,8 @@ standard against which forcing biases are judged.
167
170
 
168
171
  *Limits.* At 31 km a 15 km coastal domain spans one or two ERA5 cells, so the forcing is nearly
169
172
  uniform and smears the coast: in the first hours of 2026-04-02 over such a domain off New Jersey,
170
- the ERA5 box (which includes New Jersey land) was about 3 K warmer at 2 m and had about half HRRR's wind speed. Latency rules out
173
+ the ERA5 box (which includes New Jersey land) was about 3 K warmer at 2 m and had about half HRRR's
174
+ wind speed. Latency rules out
171
175
  anything closer than five days to the present.
172
176
 
173
177
  Choosing
@@ -220,6 +224,7 @@ Links
220
224
  - NOAA HRRR on AWS: https://registry.opendata.aws/noaa-hrrr-pds/
221
225
  - NOAA RRFS: https://gsl.noaa.gov/rrfs/ ; operational date and retirements:
222
226
  https://gribstream.com/blog/noaa-rrfs-refs-operational-august-2026
223
- - ERA5T latency: https://climate.copernicus.eu/key-update-climate-dataset-brings-data-five-days-behind-real-time
227
+ - ERA5T latency:
228
+ https://climate.copernicus.eu/key-update-climate-dataset-brings-data-five-days-behind-real-time
224
229
  - ECMWF open data: https://www.ecmwf.int/en/forecasts/datasets/open-data
225
230
  - JRA55-do: https://climate.mri-jma.go.jp/pub/ocean/JRA55-do/
@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
9
9
  project = "forcingkit"
10
10
  copyright = "2026, Long Horizon Observatory"
11
11
  author = "Daniel Fry"
12
- release = "0.3.0"
12
+ release = "0.4.0"
13
13
 
14
14
  extensions = [
15
15
  "sphinx.ext.autodoc",
@@ -19,11 +19,12 @@ Atmospheric forcing (``/api/v1/atmosphere``)
19
19
  - NOAA `High-Resolution Rapid Refresh <https://rapidrefresh.noaa.gov/hrrr/>`__ (developed by
20
20
  the `NOAA Global Systems Laboratory <https://gsl.noaa.gov/>`__, run operationally by
21
21
  `NCEP <https://www.nco.ncep.noaa.gov/>`__), 3 km, from the anonymous AWS S3 bucket
22
- `noaa-hrrr-bdp-pds <https://registry.opendata.aws/noaa-hrrr-pds/>`__; the archive starts 2014-07-30. Surface fields are interpolated to a
23
- regular 0.03 degree grid and streamed hour by hour (schema ``hrrr-atm-v1``). Earlier dates
24
- use `ERA5 <https://cds.climate.copernicus.eu/>`__ through
25
- `NumericalEarth <https://github.com/NumericalEarth/NumericalEarth.jl>`__ in the model, outside this service. See
26
- :doc:`atmospheric_forcing`.
22
+ `noaa-hrrr-bdp-pds <https://registry.opendata.aws/noaa-hrrr-pds/>`__; the archive starts
23
+ 2014-07-30. Surface fields are interpolated to a regular 0.03 degree grid and streamed hour
24
+ by hour (schema ``hrrr-atm-v1``). Earlier dates use `ERA5
25
+ <https://cds.climate.copernicus.eu/>`__ through `NumericalEarth
26
+ <https://github.com/NumericalEarth/NumericalEarth.jl>`__ in the model, outside this
27
+ service. See :doc:`atmospheric_forcing`.
27
28
 
28
29
  Parent ocean (``/api/v1/obc``)
29
30
  ------------------------------
@@ -37,7 +38,8 @@ Parent ocean (``/api/v1/obc``)
37
38
  * - **NECOFS**
38
39
  - `Northeast Coastal Ocean Forecast System <https://fvcom.smast.umassd.edu/?p=20>`__, `FVCOM
39
40
  <https://fvcom.smast.umassd.edu/>`__ GOM7 (`UMass Dartmouth SMAST
40
- <https://www.umassd.edu/smast/>`__ and `WHOI <https://www.whoi.edu/>`__), unstructured mesh with 45 sigma layers. Daily history files from 2025-01-01, then the rolling forecast.
41
+ <https://www.umassd.edu/smast/>`__ and `WHOI <https://www.whoi.edu/>`__), unstructured mesh
42
+ with 45 sigma layers. Daily history files from 2025-01-01, then the rolling forecast.
41
43
  The only donor that streams the ``z-v3`` parent store. See :doc:`necofs`.
42
44
  * - **NYOFS**
43
45
  - `NOAA New York/New Jersey Operational Forecast System
@@ -58,12 +60,16 @@ Parent ocean (``/api/v1/obc``)
58
60
  the present. The experiment follows the date: GLBv0.08 ``expt_53.X`` (reanalysis) to
59
61
  2015, a chain of GLBv0.08 analysis experiments to 2018-12-04, GLBy0.08 ``expt_93.0`` to
60
62
  2024-09-05, then ESPC-D-V02. A window that spans a switch is stitched; if any piece fails,
61
- the whole window fails. Last-resort fallback; legacy output layout.
63
+ the whole window fails. Tidal only from 2024-09-05, recorded in the store's ``tides``
64
+ attribute (``"none"``, ``"included"`` or ``"mixed"``). Last-resort fallback; legacy
65
+ output layout.
62
66
 
63
67
  `TPXO10 <https://www.tpxo.net/>`__ tidal harmonics are a roadmap item, not integrated; the
64
68
  `pyTMD <https://github.com/pyTMD/pyTMD>`__ dependency was removed on
65
69
  2026-10-05. The ``include_tides`` and ``tidal_model`` request fields
66
70
  are still accepted and form part of the cache key, but no tide is added to the parent store.
71
+ A parent that already contains the tide (a HYCOM store with ``tides = "included"``) should not
72
+ have tides added again downstream; see :doc:`hycom`.
67
73
 
68
74
  Donor selection
69
75
  ~~~~~~~~~~~~~~~
@@ -149,8 +155,9 @@ As checked on 2026-10-06, NYOFS and DBOFS output is available from three places:
149
155
  - From 2024-11-19, one directory per day, ``<ofs>/netcdf/{yyyy}/{mm}/{dd}/``; the first day
150
156
  holds only its last cycle. Before that, one flat directory per month,
151
157
  ``<ofs>/netcdf/{yyyymm}/``, with a mix of file names (NYOFS 2024-07 uses the older names,
152
- DBOFS 2024-07 the newer ones); several of those months are partial and some are absent (NYOFS lacks 2024-01 to 2024-03, 2024-05
153
- and 2024-06; DBOFS lacks 2024-02 and starts 2024-01 on the 29th).
158
+ DBOFS 2024-07 the newer ones); several of those months are partial and some are absent
159
+ (NYOFS lacks 2024-01 to 2024-03, 2024-05 and 2024-06; DBOFS lacks 2024-02 and starts
160
+ 2024-01 on the 29th).
154
161
 
155
162
  .. note::
156
163
  An earlier version of this page said that NCEI stopped archiving at the end of November 2023,
@@ -181,7 +188,9 @@ Common processing
181
188
  -----------------
182
189
 
183
190
  - **Hourly time axis.** Non-streaming donors are resampled to a strict hourly index by linear
184
- interpolation. Streaming donors deliver hourly records directly.
191
+ interpolation. Streaming donors deliver hourly records directly. HYCOM steps are 3 hours apart
192
+ with occasional gaps of up to 51 hours, which this bridges with a straight line; a store does
193
+ not yet record the longest gap it filled (:doc:`hycom`).
185
194
  - **Float32 fields, full-precision coordinates.** Data variables are cast to Float32 for the
186
195
  model; ``lat``, ``lon``, ``z`` and ``z_face`` keep full precision, since Float32 longitudes near
187
196
  -74 resolve only about 8e-6 degrees.
@@ -8,7 +8,9 @@ live against it on the same day.
8
8
  HYCOM is forcingkit's global parent ocean and the dispatcher's last-resort donor: it accepts any
9
9
  box and is tried after NYOFS, DBOFS and NECOFS (:doc:`fetchers`). It covers 1994 to the present
10
10
  by switching between several experiments, and delivers the legacy output layout rather than the
11
- ``z-v3`` parent store.
11
+ ``z-v3`` parent store. Its fields are already on a regular longitude/latitude grid at fixed
12
+ depths in metres, with temperature and salinity, so it is the donor closest to ``z-v3``
13
+ (:doc:`roadmap`).
12
14
 
13
15
  System overview
14
16
  ---------------
@@ -73,6 +75,7 @@ Experiments forcingkit reads
73
75
  Each experiment serves from the time in the third column until the next one starts. Where two
74
76
  overlap, the switch is at the newer one's first time step, with one exception: GLBy0.08
75
77
  ``expt_93.0`` is kept to its last step on 2024-09-05 although ESPC-D-V02 begins on 2024-08-10.
78
+ The GLB experiments carry no tides; ESPC-D-V02 does (see `Tides`_).
76
79
  Times before 1994-01-01 are not served.
77
80
 
78
81
  Grids
@@ -105,6 +108,48 @@ experiments carry all five in one dataset, 3-hourly, plus ``tau`` (analysis time
105
108
  field: ``u3z``, ``v3z``, ``t3z`` and ``s3z`` are 3-hourly; ``ssh`` is hourly. All use the time
106
109
  unit "hours since 2000-01-01 00:00:00".
107
110
 
111
+ Tides
112
+ ~~~~~
113
+
114
+ The record changes character at 2024-09-05. Sea surface height sampled on 2026-10-06 at
115
+ 40.40 N, 72.48 W (open water south of Long Island) shows:
116
+
117
+ - **ESPC-D-V02** (hourly ``ssh``): a semidiurnal tide of about 0.43 m amplitude with a period
118
+ near 12.4 hours.
119
+ - **GLBy0.08** ``expt_93.0`` (3-hourly): variations within about 5 cm and no tidal period. The
120
+ GLB experiments are run without tidal forcing.
121
+
122
+ So a window before 2024-09-05 is a non-tidal ocean and a window after it is a tidal one, and a
123
+ window across the date changes from one to the other. The fetcher records which in the
124
+ ``tides`` attribute of the output, which the dispatcher carries into the store:
125
+
126
+ .. list-table::
127
+ :header-rows: 1
128
+ :widths: 20 80
129
+
130
+ * - ``tides``
131
+ - Window
132
+ * - ``"none"``
133
+ - Entirely before 2024-09-05 (GLB experiments).
134
+ * - ``"included"``
135
+ - Entirely from 2024-09-05 (ESPC-D-V02).
136
+ * - ``"mixed"``
137
+ - Spans 2024-09-05: non-tidal before the switch, tidal after it. The fetcher also logs a
138
+ warning.
139
+
140
+ Two consequences:
141
+
142
+ - **Do not add tides twice.** A tidal parent (``tides = "included"``) already carries the tide
143
+ in ``zeta`` and in the currents. A downstream model that adds its own tidal boundary forcing
144
+ (harmonic constituents from TPXO, FES or similar) should do so only when ``tides`` is
145
+ ``"none"``; adding it to a tidal parent doubles the tide. A ``"mixed"`` window has no single
146
+ correct treatment and is best split at 2024-09-05 or avoided.
147
+ - **Tidal sampling.** ESPC-D-V02's ``zeta`` is kept at its native hourly steps (step 3 below),
148
+ and the dispatcher's hourly resample leaves those values unchanged. Velocities, temperature
149
+ and salinity are published only 3-hourly, so the tidal currents are sampled four times per
150
+ semidiurnal cycle and linearly interpolated to hourly in between, which flattens their peaks
151
+ by up to about 30 percent.
152
+
108
153
  Where the data lives
109
154
  --------------------
110
155
 
@@ -127,9 +172,13 @@ How forcingkit reads it
127
172
  dataset's own convention (-180..180 or 0..360, detected per dataset); a box that crosses the
128
173
  dataset's seam is read in two parts. Only the five fields above are downloaded.
129
174
  3. **ESPC-D-V02 merge.** The five per-field datasets are read separately and merged on the
130
- times they share, so hourly ``zeta`` is reduced to the 3-hourly steps.
175
+ union of their times, so ``zeta`` keeps its hourly steps and ``u``, ``v``, ``temp`` and
176
+ ``salt`` are NaN at the two hours between their 3-hourly steps. Times outside the span that
177
+ all five fields share are dropped, so no field begins or ends on a step it has no data for.
131
178
  4. **Clean-up.** Longitudes are returned on 0..360 for every experiment. Times are sorted,
132
- duplicates dropped, and decoded to datetimes.
179
+ duplicates dropped, decoded to datetimes and rounded to the second (ESPC-D-V02 ``ssh``
180
+ decodes a few hundred nanoseconds off the hour, which would otherwise keep its steps apart
181
+ from those of the other fields in step 3).
133
182
  5. **Stitch.** Pieces from different experiments are concatenated in time. A piece on a
134
183
  different grid is interpolated (bilinear) onto the grid of the first piece. The step at a
135
184
  switch is read from both experiments; the newer one's copy is kept.
@@ -139,9 +188,12 @@ How forcingkit reads it
139
188
  data in a dataset's time range is not retried.
140
189
 
141
190
  Output is a Dataset with ``u``, ``v``, ``temp``, ``salt`` (``time, depth, lat, lon``) and
142
- ``zeta`` (``time, lat, lon``). Land and cells below the sea floor are NaN. The dispatcher then
143
- resamples to hourly by linear interpolation, casts to Float32, and writes the store with
144
- ``schema = "legacy"``.
191
+ ``zeta`` (``time, lat, lon``), and the ``tides`` attribute (see `Tides`_). Land and cells below
192
+ the sea floor are NaN. The time axis is 3-hourly for GLB experiments and hourly for ESPC-D-V02,
193
+ where only ``zeta`` has data at every step. The dispatcher then resamples to hourly by linear
194
+ interpolation, each variable from the steps at which it has data (so the 3-hourly fields are
195
+ bridged between their own steps and hourly ``zeta`` is kept as is), casts to Float32, and
196
+ writes the store with ``schema = "legacy"``.
145
197
 
146
198
  Known issues
147
199
  ~~~~~~~~~~~~
@@ -161,8 +213,8 @@ Known issues
161
213
  - **GLBv0.08 ``expt_93.0`` steps back.** Its time axis goes from 2018-06-21 09:00 to 06:00,
162
214
  repeating 06:00. The fetcher selects by position and drops the duplicate.
163
215
  - **Dead path for ``expt_53.X``.** ``GLBy0.08/expt_53.X`` answers HTTP 200 with an empty
164
- dataset description; the reanalysis lives under ``GLBv0.08``. forcingkit read the GLBy0.08
165
- path before 2026-10-06, so HYCOM served nothing before 2018-12-04 until then.
216
+ dataset description; the reanalysis lives under ``GLBv0.08``, which is the path forcingkit
217
+ reads.
166
218
  - **GLBy0.08 ``expt_93.0`` starts at 12:00.** Its first step is 2018-12-04 12:00, not the
167
219
  midnight that the catalogue date suggests; the switch from GLBv0.08 ``expt_93.0`` is at
168
220
  12:00.
@@ -186,27 +238,35 @@ these returned all five fields on 10 x 5 cells and 40 depths:
186
238
  * - 2018-12-03 18:00, 12 h
187
239
  - GLBv0.08 ``expt_93.0``
188
240
  - 5 steps, in 12 s.
189
- * - 2026-10-01 00:00, 6 h
241
+ * - 2026-10-01 00:00, 24 h
190
242
  - ESPC-D-V02 (five datasets)
191
- - 3 steps, in 216 s; ``zeta`` reduced from hourly to 3-hourly.
243
+ - 25 hourly steps, ``zeta`` at all 25 and the other fields at 9, in 547 s (one read
244
+ retried); ``tides = "included"``. After the dispatcher's resample, ``zeta`` was
245
+ unchanged (a tidal range of about 0.8 m at one cell) and the surface ``u`` had no
246
+ gaps.
192
247
  * - 2018-12-04 06:00, 12 h
193
248
  - GLBv0.08 then GLBy0.08 ``expt_93.0``
194
249
  - 5 steps, in 367 s; 12:00 read from both, kept once.
195
250
  * - 2024-09-04 18:00, 12 h
196
251
  - GLBy0.08 ``expt_93.0`` then ESPC-D-V02
197
- - 5 steps, in 334 s. ESPC-D-V02 marks a few more cells as land or below the sea floor
198
- than GLBy0.08 in this box.
252
+ - 9 steps (3-hourly to 2024-09-05 00:00, then hourly), 13 after the dispatcher's
253
+ resample, in 623 s (one read retried); ``tides = "mixed"`` and the crossing warning
254
+ logged. An earlier attempt failed after two timeouts on ``v3z``. ESPC-D-V02 marks a
255
+ few more cells as land or below the sea floor than GLBy0.08 in this box.
199
256
 
200
- Before the per-dataset retry was added, five earlier attempts at the two stitched windows each
201
- stopped on one read timeout (see `Known issues`_).
257
+ The times are for windows of 6 to 12 hours; multi-day windows have not been timed. Most of the
258
+ time goes to per-request latency on the server rather than to data volume, so a longer window
259
+ need not take proportionally longer, but plan for minutes per fetch and see `Known issues`_.
202
260
 
203
261
  Tests
204
262
  -----
205
263
 
206
264
  - ``tests/unit/test_hycom.py``: experiment resolution and window splitting at every switch,
207
265
  the URLs for GLBv0.08 and the ESPC-D-V02 fields, stitching across 2018-12-04 (regridding and
208
- the duplicated switch step) and across 2024-09-05 (merged ESPC-D-V02 fields), the all or
209
- nothing rule, the retry after a failed read (and none for an empty time range), both
266
+ the duplicated switch step) and across 2024-09-05 (merged ESPC-D-V02 fields), hourly ``zeta``
267
+ in the ESPC-D-V02 merge and through the dispatcher's hourly resample, trimming to the span
268
+ all fields share, rounding decoded times to the second, the ``tides`` attribute and the warning for a window across 2024-09-05, the
269
+ all or nothing rule, the retry after a failed read (and none for an empty time range), both
210
270
  longitude conventions and the seam, and the non-monotonic time axis. No network.
211
271
 
212
272
  .. code-block:: bash
@@ -27,7 +27,8 @@ that change those details without notice.
27
27
 
28
28
  forcingkit is a FastAPI service that sits between those providers and model codes such as
29
29
  `Oceananigans.jl <https://github.com/CliMA/Oceananigans.jl>`__ (for example through
30
- `NumericalEarth <https://github.com/NumericalEarth/NumericalEarth.jl>`__). For a bounding box and a time window it:
30
+ `NumericalEarth <https://github.com/NumericalEarth/NumericalEarth.jl>`__). For a bounding box and a
31
+ time window it:
31
32
 
32
33
  1. picks the source (the "donor") best suited to the box, and falls back to the next one if the
33
34
  first cannot deliver;
@@ -107,7 +108,7 @@ Parent-ocean donors at a glance
107
108
  - HYCOM GLBv0.08, GLBy0.08 and ESPC-D-V02, regular
108
109
  - 1/12 degree (about 9 km)
109
110
  - Global; 1994 to the present
110
- - Legacy output; last-resort fallback
111
+ - Legacy output; last-resort fallback; tidal only from 2024-09-05
111
112
 
112
113
  Quick start
113
114
  -----------
@@ -20,7 +20,8 @@ Regional Association of Coastal Ocean Observing Systems <https://neracoos.org/>`
20
20
  It is built on `FVCOM <https://fvcom.smast.umassd.edu/>`__, the unstructured-grid, finite-volume,
21
21
  free-surface, primitive-equation coastal ocean model introduced by `Chen, Liu and Beardsley
22
22
  (2003) <https://doi.org/10.1175/1520-0426(2003)020%3C0159:AUGFVT%3E2.0.CO;2>`__ and developed by
23
- Changsheng Chen's group at SMAST with Robert C. Beardsley at WHOI. forcingkit reads its **GOM7** configuration, which
23
+ Changsheng Chen's group at SMAST with Robert C. Beardsley at WHOI. forcingkit reads its **GOM7**
24
+ configuration, which
24
25
  spans the Gulf of Maine, Georges Bank, southern New England, Long Island Sound, New York Harbor
25
26
  and the Mid-Atlantic Bight shelf.
26
27
 
@@ -321,7 +321,9 @@ Institutions and data services
321
321
  <https://opendap.co-ops.nos.noaa.gov/thredds/catalog/catalog.html>`__
322
322
  - `NOAA NCEI <https://www.ncei.noaa.gov/>`__: `NYOFS files
323
323
  <https://www.ncei.noaa.gov/thredds/catalog/model-nyofs-files/catalog.html>`__
324
- - `NOAA Open Data Dissemination <https://www.noaa.gov/information-technology/open-data-dissemination>`__:
324
+ - `NOAA Open Data Dissemination
325
+ <https://www.noaa.gov/information-technology/open-data-dissemination>`__:
325
326
  `OFS on AWS <https://github.com/NOAA-Big-Data-Program/nodd-data-docs/blob/main/OFS/README.md>`__
326
- - `Stevens Institute of Technology, Davidson Laboratory <https://www.stevens.edu/davidson-laboratory>`__:
327
+ - `Stevens Institute of Technology, Davidson Laboratory
328
+ <https://www.stevens.edu/davidson-laboratory>`__:
327
329
  `NYHOPS <https://hudson.dl.stevens-tech.edu/maritimeforecast/>`__
@@ -45,8 +45,9 @@ Current coverage
45
45
  - 2014-07-30 on
46
46
  - 3 km
47
47
 
48
- So today the only current-date parent ocean outside the US Northeast is HYCOM, at 1/12 degree and
49
- in the legacy output layout, and there is no atmosphere outside the contiguous US. Older years and other regions rely on ERA5 through NumericalEarth in the model.
48
+ So today the only parent ocean outside the US Northeast is HYCOM, at 1/12 degree and in the
49
+ legacy output layout, and there is no atmosphere outside the contiguous US: there, and before
50
+ 2014-07-30, the atmosphere comes from ERA5 through NumericalEarth in the model.
50
51
 
51
52
  Planned work
52
53
  ------------
@@ -57,13 +58,16 @@ Planned work
57
58
 
58
59
  * - Item
59
60
  - Detail
61
+ * - HYCOM as a ``z-v3`` parent
62
+ - The cheapest conversion, and the one that gives a ``z-v3`` parent everywhere: HYCOM is
63
+ already on a regular longitude/latitude grid at fixed depths in metres, with temperature
64
+ and salinity, so it needs regridding onto the ``z-v3`` grid and an hour-by-hour
65
+ ``iter_parent``, but no sigma-to-depth step. Tidal content changes at 2024-09-05
66
+ (:doc:`hycom`), so the store should record it.
60
67
  * - NYOFS and DBOFS as ``z-v3`` parents
61
68
  - Deliver both on true depths with geographic axes, like :doc:`necofs`. NYOFS carries no
62
69
  temperature or salinity, so those would come from another donor. The scope is in
63
70
  :doc:`nyofs`.
64
- * - A global parent on ``z-v3``
65
- - HYCOM reaches the present through ESPC-D-V02 (:doc:`hycom`), but at 1/12 degree and in the
66
- legacy output layout. Copernicus GLO12 or RTOFS (below) would add a second global source.
67
71
  * - HRRR forecast mode
68
72
  - One cycle's f01 to f48 for forecasts, alongside the chained one-hour forecasts used for
69
73
  hindcasts (:doc:`atmospheric_forcing`).
@@ -89,25 +93,27 @@ Global parent ocean
89
93
  <https://data.marine.copernicus.eu/product/GLOBAL_ANALYSISFORECAST_PHY_001_024/description>`__)
90
94
  - NEMO, 1/12 degree, 50 levels, hourly and daily fields, from 2020-11-01 to 10 days ahead,
91
95
  global. Free registration; cloud-native (ARCO Zarr) access.
92
- - The global fallback for current dates, and the backbone for any region outside the US.
96
+ - Hourly fields and a second, independent model to set against HYCOM; the backbone for
97
+ any region outside the US.
93
98
  * - **GLORYS12 reanalysis** (Mercator Ocean, `GLOBAL_MULTIYEAR_PHY_001_030
94
99
  <https://data.marine.copernicus.eu/product/GLOBAL_MULTIYEAR_PHY_001_030/description>`__)
95
100
  - NEMO with data assimilation, 1/12 degree, 50 levels, daily means from 1993-01-01 to
96
101
  2026-08 (Lellouche et al., 2021).
97
- - A consistent parent for any year since 1993, anywhere; daily, so tides must come from
98
- elsewhere.
102
+ - One continuous reanalysis from 1993, where HYCOM chains several experiments from 1994;
103
+ daily, so tides must come from elsewhere.
99
104
  * - **NOAA Global RTOFS** (`on AWS <https://registry.opendata.aws/noaa-rtofs/>`__)
100
105
  - HYCOM-based, 1/12 degree. Besides the global output, 6-hourly netCDF already on z levels
101
106
  for three US subdomains (``US_east``, ``US_west``, ``alaska``), from 2024-01-27.
102
- - A NOAA global fallback whose US cuts need no vertical regridding.
107
+ - US subsets already on z levels, so they need no vertical regridding.
103
108
 
104
109
  US coasts without a parent
105
110
  ~~~~~~~~~~~~~~~~~~~~~~~~~~
106
111
 
107
112
  The rest of the NOAA Operational Forecast Systems. Each is on AWS (``noaa-nos-ofs-pds``) per day
108
- from late 2024 (2024-10-01 for CBOFS, 2024-11-19 for most) and at NCEI before that, in the layouts forcingkit's NYOFS and DBOFS readers
109
- already handle. The ROMS systems can share the DBOFS reader and the FVCOM systems the NECOFS mesh
110
- interpolation, so each is mostly configuration and a domain.
113
+ from late 2024 (2024-10-01 for CBOFS, 2024-11-19 for most) and at NCEI before that, in the
114
+ layouts forcingkit's NYOFS and DBOFS readers already handle. The ROMS systems can share the DBOFS
115
+ reader and the FVCOM systems the NECOFS mesh interpolation, so each is mostly configuration and
116
+ a domain.
111
117
 
112
118
  .. list-table::
113
119
  :header-rows: 1
@@ -35,7 +35,8 @@ Providers
35
35
  - Blumberg and Mellor (1987) for POM (NYOFS); Shchepetkin and McWilliams (2005) for ROMS
36
36
  (DBOFS); NOAA attribution as for HRRR.
37
37
  * - HYCOM
38
- - The `HYCOM consortium <https://www.hycom.org/>`__ (US Navy, NOAA and academic partners)
38
+ - The `HYCOM consortium <https://www.hycom.org/>`__ (US Navy, NOAA and academic partners);
39
+ from 2024-09-05 the US Navy's ESPC-D-V02 analysis, served by the consortium
39
40
  - Bleck (2002) for the model; Chassignet et al. (2007) for the data-assimilative system.
40
41
  * - ERA5 (used through NumericalEarth, not served here)
41
42
  - `ECMWF <https://www.ecmwf.int/>`__ for the `Copernicus Climate Change Service
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "forcingkit"
7
- version = "0.3.0"
7
+ version = "0.4.0"
8
8
  description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -1,9 +1,12 @@
1
1
  import os
2
2
  import logging
3
- from typing import Optional
3
+ from typing import TYPE_CHECKING, Optional
4
4
  import numpy as np
5
5
  from forcingkit import settings
6
6
 
7
+ if TYPE_CHECKING:
8
+ import xarray as xr
9
+
7
10
 
8
11
  logger = logging.getLogger(__name__)
9
12
 
@@ -262,8 +265,7 @@ def dispatch_obc_request(
262
265
  ds["time"] = ds.indexes["time"].to_datetimeindex()
263
266
  except AttributeError:
264
267
  ds["time"] = pd.to_datetime(ds.indexes["time"].values)
265
- # Interpolate to strictly hourly
266
- ds = ds.resample(time="1h").interpolate("linear")
268
+ ds = _resample_hourly(ds)
267
269
 
268
270
  # REQ-1.3: Tidal boundary condition integration via GOT4.10c or EOT20 (pyTMD).
269
271
 
@@ -310,6 +312,34 @@ def dispatch_obc_request(
310
312
  _FULL_PRECISION_COORDS = ("lat", "lon", "z", "z_face")
311
313
 
312
314
 
315
+ def _resample_hourly(ds: "xr.Dataset") -> "xr.Dataset":
316
+ """Linearly interpolate every time-dependent variable onto an hourly axis.
317
+
318
+ Variables may arrive at different cadences on one time axis: HYCOM ESPC-D-V02 has hourly
319
+ surface elevation and 3-hourly currents, temperature and salinity, which are NaN at the
320
+ hours in between. Each variable is therefore interpolated from the times at which it has
321
+ any data, so a coarser field is bridged between its own steps and a finer one keeps its
322
+ native values. A single `resample().interpolate()` would treat those NaNs as data.
323
+ """
324
+ import pandas as pd
325
+
326
+ times = ds.indexes["time"]
327
+ hourly = pd.date_range(times.min().floor("h"), times.max().floor("h"), freq="h")
328
+ out = ds.drop_dims("time").assign_coords(time=hourly)
329
+ for name, da in ds.data_vars.items():
330
+ if "time" not in da.dims:
331
+ continue
332
+ others = [d for d in da.dims if d != "time"]
333
+ valid = (da.notnull().any(others) if others else da.notnull()).values
334
+ da = da.isel(time=valid)
335
+ out[name] = (
336
+ da.interp(time=hourly, assume_sorted=True)
337
+ if da.sizes["time"] > 1
338
+ else da.reindex(time=hourly)
339
+ )
340
+ return out[list(ds.data_vars)]
341
+
342
+
313
343
  def atmosphere_key(
314
344
  bbox: list[float],
315
345
  start_time: str,
@@ -20,7 +20,7 @@ def fetch_erddap_station_profiles(
20
20
  ) -> dict:
21
21
  """
22
22
  Fetches 3-depth temperature profiles for a given station.
23
- Targeting LIS stations via UConn ERDDAP.
23
+ Targets Long Island Sound stations on the UConn ERDDAP server.
24
24
 
25
25
  Args:
26
26
  station_id: Station ID, e.g., "WLIS", "EXRX"
@@ -105,7 +105,8 @@ def fetch_erddap_stations_in_bbox(
105
105
  cache_bust: bool = False,
106
106
  ) -> dict:
107
107
  """
108
- Finds all known LIS stations within the bounding box and fetches their profiles.
108
+ Finds the known Long Island Sound stations within the bounding box and fetches their
109
+ profiles.
109
110
  bbox: [max_lat, min_lon, min_lat, max_lon] or [min_lon, min_lat, max_lon, max_lat]
110
111
  """
111
112
  # Accept standard Julia order (min_lon, min_lat, max_lon, max_lat)
@@ -34,6 +34,8 @@ class _Segment:
34
34
  name: str
35
35
  # One aggregated dataset, or one dataset per variable to be merged.
36
36
  urls: tuple[str, ...]
37
+ # Whether the experiment's sea surface height and currents contain the tide.
38
+ tidal: bool = False
37
39
 
38
40
 
39
41
  def _glb(path: str, start: str) -> _Segment:
@@ -57,7 +59,11 @@ def _glb(path: str, start: str) -> _Segment:
57
59
  # - Time. All use "hours since 2000-01-01 00:00:00", 3-hourly, with occasional gaps of up to
58
60
  # 51 h that are passed through unfilled. GLBv0.08/expt_93.0 steps back once.
59
61
  # - ESPC-D-V02 serves u, v, temperature and salinity (3-hourly) and surface elevation (hourly)
60
- # as separate datasets, merged on the shared 3-hourly times.
62
+ # as separate datasets. They are merged on the union of their times, so surface elevation
63
+ # stays hourly and the other fields are NaN between their 3-hourly steps.
64
+ # - Tides. The GLB experiments are run without tidal forcing; ESPC-D-V02 includes the tide
65
+ # (about 0.43 m semidiurnal amplitude south of Long Island against about 5 cm of non-tidal
66
+ # variation in GLBy0.08/expt_93.0). The record therefore turns tidal on 2024-09-05.
61
67
  _SEGMENTS: tuple[_Segment, ...] = (
62
68
  _glb("GLBv0.08/expt_53.X", "1994-01-01"),
63
69
  _glb("GLBv0.08/expt_56.3", "2015-12-31"),
@@ -73,6 +79,7 @@ _SEGMENTS: tuple[_Segment, ...] = (
73
79
  tuple(
74
80
  f"{_TDS}/ESPC-D-V02/{var}" for var in ("u3z", "v3z", "t3z", "s3z", "ssh")
75
81
  ),
82
+ tidal=True,
76
83
  ),
77
84
  )
78
85
 
@@ -122,6 +129,16 @@ def _split_window(
122
129
  return pieces
123
130
 
124
131
 
132
+ def _tides(pieces: list[tuple[_Segment, pd.Timestamp, pd.Timestamp]]) -> str:
133
+ """Tidal content of a window: "included", "none", or "mixed" across a switch."""
134
+ tidal = {seg.tidal for seg, _, _ in pieces}
135
+ if tidal == {True}:
136
+ return "included"
137
+ if tidal == {False}:
138
+ return "none"
139
+ return "mixed"
140
+
141
+
125
142
  def _normalize_lons(lons: np.ndarray) -> np.ndarray:
126
143
  """Convert -180/180 to 0/360."""
127
144
  return np.where(lons < 0, lons + 360, lons)
@@ -149,6 +166,15 @@ def fetch_hycom_boundary_conditions(
149
166
  names = ", ".join(seg.name for seg, _, _ in pieces)
150
167
  logger.info(f"Hindcast spans HYCOM experiments; stitching {names}...")
151
168
 
169
+ tides = _tides(pieces)
170
+ if tides == "mixed":
171
+ switch = next(seg.start for seg, _, _ in pieces if seg.tidal)
172
+ logger.warning(
173
+ f"HYCOM window {start_dt} to {end_dt} crosses {switch}, where the record changes "
174
+ "from non-tidal (GLB experiments) to tidal (ESPC-D-V02); sea surface height and "
175
+ "currents gain the tide part way through."
176
+ )
177
+
152
178
  parts = []
153
179
  for seg, piece_start, piece_end in pieces:
154
180
  part = _fetch_segment(seg, piece_start, piece_end, bbox)
@@ -160,9 +186,9 @@ def fetch_hycom_boundary_conditions(
160
186
  return None
161
187
  parts.append(part)
162
188
 
163
- if len(parts) == 1:
164
- return parts[0]
165
- return _stitch(parts)
189
+ ds = parts[0] if len(parts) == 1 else _stitch(parts)
190
+ ds.attrs["tides"] = tides
191
+ return ds
166
192
 
167
193
 
168
194
  def _fetch_segment(
@@ -180,9 +206,13 @@ def _fetch_segment(
180
206
  parts.append(part)
181
207
  if len(parts) == 1:
182
208
  return parts[0]
183
- # The inner join keeps the 3-hourly times common to all variables (surface elevation is
184
- # hourly).
185
- return xr.merge(parts, join="inner")
209
+ # Surface elevation is hourly and the other fields 3-hourly. The outer join keeps every
210
+ # time, leaving the 3-hourly fields NaN in between (the dispatcher interpolates each field
211
+ # from its own steps). Times outside the span all fields share are dropped, so no field
212
+ # starts or ends with a step it has no data for.
213
+ first = max(part.indexes["time"][0] for part in parts)
214
+ last = min(part.indexes["time"][-1] for part in parts)
215
+ return xr.merge(parts, join="outer").sel(time=slice(first, last))
186
216
 
187
217
 
188
218
  def _stitch(parts: list[xr.Dataset]) -> xr.Dataset:
@@ -309,10 +339,13 @@ def _read_hycom_data(
309
339
  actual_rename = {k: v for k, v in _RENAME.items() if k in ds_subset.data_vars}
310
340
  ds_subset = ds_subset.rename(actual_rename)
311
341
 
312
- # Explicitly decode the raw float time coordinate to pandas DatetimeIndex lengths
342
+ # Explicitly decode the raw float time coordinate to pandas DatetimeIndex lengths.
343
+ # ESPC-D-V02 `ssh` decodes a few hundred nanoseconds off the hour (01:00 as
344
+ # 00:59:59.999999791, read 2026-10-06); rounding to the second keeps its steps equal to
345
+ # those of the 3-hourly fields it is merged with.
313
346
  if time_var in ds_subset.coords:
314
- ds_subset[time_var] = epoch + pd.to_timedelta(
315
- ds_subset[time_var].values, unit="h"
316
- )
347
+ ds_subset[time_var] = (
348
+ epoch + pd.to_timedelta(ds_subset[time_var].values, unit="h")
349
+ ).round("s")
317
350
 
318
351
  return ds_subset
@@ -130,7 +130,8 @@ class Barycentric:
130
130
  barycentric weights computed once.
131
131
 
132
132
  Equivalent to `LinearNDInterpolator(tri, values)(targets)`, which repeats the search for every
133
- call: per parent hour that is one search per layer per variable (181 for LIS), each over every
133
+ call: per parent hour that is one search per layer per variable (181 for 45 layers of four
134
+ fields plus sea surface height), each over every
134
135
  target point. Values may carry leading dimensions: (..., npoints) -> (..., *shape). Targets
135
136
  outside the triangulation are NaN.
136
137
  """
@@ -64,7 +64,7 @@ def fetch_noaa_tide_data(
64
64
  "units": "metric",
65
65
  "time_zone": "gmt",
66
66
  "format": "json",
67
- "application": "lhzn_coastal_sim",
67
+ "application": "forcingkit",
68
68
  }
69
69
 
70
70
  try:
@@ -12,10 +12,9 @@ from pathlib import Path
12
12
 
13
13
  logger = logging.getLogger("forcingkit")
14
14
 
15
- # New name -> old names still read, in order. The cache directory had two names: the service
16
- # routes read COASTAL_SIM_DATA_CACHE_DIR and the fetchers ECODATA_CACHE_CACHE_DIR.
15
+ # New name -> old names still read, in order.
17
16
  LEGACY_ENV: dict[str, tuple[str, ...]] = {
18
- "FORCINGKIT_CACHE_DIR": ("ECODATA_CACHE_CACHE_DIR", "COASTAL_SIM_DATA_CACHE_DIR"),
17
+ "FORCINGKIT_CACHE_DIR": ("ECODATA_CACHE_CACHE_DIR",),
19
18
  "FORCINGKIT_MAX_WORKERS": ("ECODATA_CACHE_MAX_WORKERS",),
20
19
  # The elevation service, renamed topobathysim -> topobathykit on 2026-10-05.
21
20
  "TOPOBATHYKIT_URL": ("TOPOBATHYSIM_URL",),
@@ -10,9 +10,9 @@ from forcingkit.dispatcher import dispatch_obc_request
10
10
 
11
11
 
12
12
  @pytest.mark.integration
13
- def test_dispatch_mab_obc_24h():
13
+ def test_dispatch_offshore_nj_obc():
14
14
  """
15
- Test a 24 hour OBC fetch over the Mid-Atlantic Bight to ensure
15
+ Test an OBC fetch off central New Jersey to ensure
16
16
  NECOFS does not crash (`import os` issue), and HYCOM gracefully
17
17
  passes over without issuing invalid empty slices (`tau[16809:1:16808]`).
18
18
  """
@@ -39,9 +39,9 @@ def test_dispatch_mab_obc_24h():
39
39
 
40
40
  assert len(ds.time) > 0, "No time steps fetched."
41
41
  print(
42
- f"\nSuccessfully fetched MAB OBC data for {hours} hours on {ds.attrs.get('Description', 'NECOFS')}!"
42
+ f"\nFetched {hours} h of OBC data off central New Jersey from {ds.attrs.get('source', 'unknown')}"
43
43
  )
44
44
 
45
45
 
46
46
  if __name__ == "__main__":
47
- test_dispatch_mab_obc_24h()
47
+ test_dispatch_offshore_nj_obc()
@@ -1,10 +1,17 @@
1
- import zarr
1
+ """Report NaN content and range of the `elevation` array in a DEM Zarr store.
2
+
3
+ Usage: uv run python tests/scripts/inspect_dem.py PATH_TO_DEM.zarr
4
+ """
5
+
6
+ import sys
7
+
2
8
  import numpy as np
9
+ import zarr
10
+
11
+ if len(sys.argv) != 2:
12
+ sys.exit(__doc__)
3
13
 
4
- ds = zarr.open(
5
- "/home/lhzn/Projects/lhzn-io/coastal-sim/config/topobathykit/policies/throgs_neck_dem.zarr",
6
- mode="r",
7
- )
14
+ ds = zarr.open(sys.argv[1], mode="r")
8
15
  elev = ds["elevation"][:] # type: ignore
9
16
  print("Elevation contains NaN:", np.isnan(elev).any()) # type: ignore
10
17
  print("NaN count:", np.isnan(elev).sum()) # type: ignore
@@ -1,9 +1,12 @@
1
+ import logging
2
+
1
3
  import xarray as xr
2
4
  import pandas as pd
3
5
  import numpy as np
4
6
  import pytest
5
7
  from unittest.mock import patch
6
8
 
9
+ from forcingkit.dispatcher import _resample_hourly
7
10
  from forcingkit.fetchers.hycom import (
8
11
  _SEGMENTS,
9
12
  _fetch_hycom_data,
@@ -125,8 +128,16 @@ def test_hycom_historical_stitch():
125
128
  assert not np.isnan(ds["u"].values).any()
126
129
 
127
130
 
131
+ def _three_hourly(ds, var):
132
+ """Whether `var` has data exactly at the 3-hourly steps of `ds` and NaN in between."""
133
+ on_step = ds.indexes["time"].hour % 3 == 0
134
+ has_data = ds[var].notnull().any([d for d in ds[var].dims if d != "time"]).values
135
+ return bool((has_data == on_step).all())
136
+
137
+
128
138
  def test_espc_merges_per_variable_datasets():
129
- """ESPC-D-V02 fetches each variable separately and keeps the 3-hourly common times."""
139
+ """ESPC-D-V02 fetches each variable separately; zeta keeps its hourly steps and the
140
+ 3-hourly fields are NaN between theirs."""
130
141
  with patch("forcingkit.fetchers.hycom._fetch_hycom_data") as mock_fetch:
131
142
  mock_fetch.side_effect = _espc("2025-03-01", 48)
132
143
  ds = fetch_hycom_boundary_conditions("2025-03-01", 48, BBOX)
@@ -136,22 +147,99 @@ def test_espc_merges_per_variable_datasets():
136
147
  ]
137
148
  assert set(ds.data_vars) == {"u", "v", "temp", "salt", "zeta"}
138
149
  assert ds.indexes["time"].equals(
139
- pd.date_range("2025-03-01", "2025-03-03", freq="3h")
150
+ pd.date_range("2025-03-01", "2025-03-03", freq="h")
140
151
  )
141
152
  assert not np.isnan(ds["zeta"].values).any()
153
+ for var in ("u", "v", "temp", "salt"):
154
+ assert _three_hourly(ds, var)
155
+ assert ds.attrs["tides"] == "included"
142
156
 
143
157
 
144
- def test_stitch_across_espc_switch():
145
- """A window across 2024-09-05 stitches GLBy0.08/expt_93.0 with merged ESPC-D-V02."""
146
- start_date = "2024-09-04"
158
+ def test_espc_trims_to_shared_span():
159
+ """Hourly zeta past the last 3-hourly step is dropped, so no field ends on a step it has
160
+ no data for."""
161
+ parts = _espc("2025-03-01", 48)
162
+ parts[-1] = _ds("2025-02-28 23:00", 51, freq="h", var="zeta", depth=False)
147
163
  with patch("forcingkit.fetchers.hycom._fetch_hycom_data") as mock_fetch:
164
+ mock_fetch.side_effect = parts
165
+ ds = fetch_hycom_boundary_conditions("2025-03-01", 48, BBOX)
166
+ assert ds.indexes["time"][0] == pd.Timestamp("2025-03-01")
167
+ assert ds.indexes["time"][-1] == pd.Timestamp("2025-03-03")
168
+
169
+
170
+ def test_glb_window_is_not_tidal(caplog):
171
+ with (
172
+ patch("forcingkit.fetchers.hycom._fetch_hycom_data") as mock_fetch,
173
+ caplog.at_level(logging.WARNING, logger="forcingkit.fetchers.hycom"),
174
+ ):
175
+ mock_fetch.side_effect = [_glb("2024-09-01", 9)]
176
+ ds = fetch_hycom_boundary_conditions("2024-09-01", 24, BBOX)
177
+ assert ds.attrs["tides"] == "none"
178
+ assert not caplog.records
179
+
180
+
181
+ def test_stitch_across_espc_switch(caplog):
182
+ """A window across 2024-09-05 stitches GLBy0.08/expt_93.0 with merged ESPC-D-V02, is
183
+ marked as mixed and logs a warning."""
184
+ start_date = "2024-09-04"
185
+ with (
186
+ patch("forcingkit.fetchers.hycom._fetch_hycom_data") as mock_fetch,
187
+ caplog.at_level(logging.WARNING, logger="forcingkit.fetchers.hycom"),
188
+ ):
148
189
  mock_fetch.side_effect = [_glb(start_date, 9), *_espc("2024-09-05", 24)]
149
190
  ds = fetch_hycom_boundary_conditions(start_date, 48, BBOX)
150
191
 
151
192
  assert mock_fetch.call_count == 6
152
193
  assert set(ds.data_vars) == {"u", "v", "temp", "salt", "zeta"}
153
- assert ds.indexes["time"].equals(pd.date_range(start_date, "2024-09-06", freq="3h"))
194
+ # 3-hourly before the switch, hourly after it.
195
+ expected = pd.date_range(start_date, "2024-09-05", freq="3h").append(
196
+ pd.date_range("2024-09-05 01:00", "2024-09-06", freq="h")
197
+ )
198
+ assert ds.indexes["time"].equals(expected)
154
199
  assert not np.isnan(ds["zeta"].values).any()
200
+ assert _three_hourly(ds, "u")
201
+ assert ds.attrs["tides"] == "mixed"
202
+ [record] = caplog.records
203
+ assert record.levelno == logging.WARNING
204
+ assert "2024-09-05" in record.getMessage() and "tidal" in record.getMessage()
205
+
206
+
207
+ def test_resample_keeps_hourly_zeta():
208
+ """The dispatcher's hourly resample keeps hourly zeta at its native values and
209
+ interpolates the 3-hourly fields between their own steps. Treating the gaps as data
210
+ would leave the 3-hourly fields NaN at two hours in three."""
211
+ hours = np.arange(25)
212
+ time = pd.date_range("2025-03-01", periods=hours.size, freq="h")
213
+ # An M2 tide, which 3-hourly sampling and linear interpolation would flatten.
214
+ tide = 0.43 * np.sin(2 * np.pi * hours / 12.42)
215
+ u = np.where(hours % 3 == 0, hours.astype(float), np.nan)
216
+ land = np.full(hours.size, np.nan)
217
+ ds = xr.Dataset(
218
+ {
219
+ "zeta": (("time", "lat"), np.stack([tide, land], axis=1)),
220
+ "u": (("time", "lat"), np.stack([u, land], axis=1)),
221
+ "h": (("lat",), [10.0, 0.0]),
222
+ },
223
+ coords={"time": time, "lat": [40.0, 40.1]},
224
+ attrs={"tides": "included"},
225
+ )
226
+ out = _resample_hourly(ds)
227
+
228
+ assert out.indexes["time"].equals(time)
229
+ np.testing.assert_allclose(out["zeta"].values[:, 0], tide)
230
+ np.testing.assert_allclose(out["u"].values[:, 0], hours)
231
+ assert np.isnan(out["u"].values[:, 1]).all()
232
+ assert out["h"].values.tolist() == [10.0, 0.0]
233
+ assert out.attrs["tides"] == "included"
234
+
235
+
236
+ def test_resample_regular_input_unchanged():
237
+ """A dataset with one cadence resamples as `resample().interpolate()` does."""
238
+ ds = _glb("2025-03-01", 9)
239
+ ds["u"][:] = np.arange(9.0)[:, None, None, None]
240
+ out = _resample_hourly(ds)
241
+ ref = ds.resample(time="1h").interpolate("linear")
242
+ xr.testing.assert_allclose(out, ref)
155
243
 
156
244
 
157
245
  def test_missing_piece_drops_window():
@@ -221,6 +309,23 @@ def test_fetch_non_monotonic_time():
221
309
  assert ds.indexes["time"].equals(expected)
222
310
 
223
311
 
312
+ def test_fetch_rounds_times_to_the_second():
313
+ """Float hours that decode a few hundred nanoseconds off the hour (as ESPC-D-V02 `ssh`
314
+ does) come back on whole seconds, so they align with the other fields in the merge."""
315
+ t0 = (pd.Timestamp("2025-01-01 12:00") - pd.Timestamp("2000-01-01")).total_seconds()
316
+ time = t0 / 3600 + np.arange(4.0) - np.array([0.0, 2e-13, 0.0, -2e-13]) * 3600
317
+ raw = _raw(np.arange(0.0, 360.0), time=time)
318
+ with patch("forcingkit.fetchers.hycom.xr.open_dataset", return_value=raw):
319
+ ds = _fetch_hycom_data(
320
+ pd.Timestamp("2025-01-01 12:00"),
321
+ pd.Timestamp("2025-01-01 15:00"),
322
+ BBOX,
323
+ f"{TDS}/ESPC-D-V02/ssh",
324
+ )
325
+ expected = pd.date_range("2025-01-01 12:00", periods=4, freq="h")
326
+ assert ds.indexes["time"].equals(expected)
327
+
328
+
224
329
  def _fetch_2025(open_dataset):
225
330
  """`_fetch_hycom_data` for 2025-01-01 12:00 to 18:00 with `xr.open_dataset` and
226
331
  `time.sleep` patched; returns (result, open_dataset mock, sleep mock)."""
@@ -7,7 +7,6 @@ from forcingkit import settings
7
7
  ALL_NAMES = [
8
8
  "FORCINGKIT_CACHE_DIR",
9
9
  "ECODATA_CACHE_CACHE_DIR",
10
- "COASTAL_SIM_DATA_CACHE_DIR",
11
10
  "FORCINGKIT_MAX_WORKERS",
12
11
  "TOPOBATHYKIT_URL",
13
12
  "TOPOBATHYSIM_URL",
@@ -38,13 +37,11 @@ def test_new_name_wins_over_legacy_names(clean_env, monkeypatch):
38
37
  assert settings.cache_dir() == "/new"
39
38
 
40
39
 
41
- @pytest.mark.parametrize(
42
- "old", ["ECODATA_CACHE_CACHE_DIR", "COASTAL_SIM_DATA_CACHE_DIR"]
43
- )
44
- def test_legacy_cache_names_still_read_with_a_warning(clean_env, monkeypatch, old):
45
- monkeypatch.setenv(old, "/old")
40
+ def test_legacy_cache_name_still_read_with_a_warning(clean_env, monkeypatch):
41
+ monkeypatch.setenv("ECODATA_CACHE_CACHE_DIR", "/old")
46
42
  with pytest.warns(
47
- FutureWarning, match=f"{old} is deprecated; use FORCINGKIT_CACHE_DIR"
43
+ FutureWarning,
44
+ match="ECODATA_CACHE_CACHE_DIR is deprecated; use FORCINGKIT_CACHE_DIR",
48
45
  ):
49
46
  assert settings.cache_dir("erddap") == os.path.join("/old", "erddap")
50
47
 
@@ -873,7 +873,7 @@ wheels = [
873
873
 
874
874
  [[package]]
875
875
  name = "forcingkit"
876
- version = "0.3.0"
876
+ version = "0.4.0"
877
877
  source = { editable = "." }
878
878
  dependencies = [
879
879
  { name = "bottleneck" },
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes