insitukit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- insitukit-0.1.0/.github/copilot-instructions.md +3 -0
- insitukit-0.1.0/.github/workflows/publish.yml +55 -0
- insitukit-0.1.0/.github/workflows/tests.yml +33 -0
- insitukit-0.1.0/.gitignore +24 -0
- insitukit-0.1.0/.pre-commit-config.yaml +17 -0
- insitukit-0.1.0/AGENTS.md +34 -0
- insitukit-0.1.0/LICENSE +21 -0
- insitukit-0.1.0/PKG-INFO +115 -0
- insitukit-0.1.0/README.md +72 -0
- insitukit-0.1.0/pyproject.toml +51 -0
- insitukit-0.1.0/src/insitukit/__init__.py +24 -0
- insitukit-0.1.0/src/insitukit/coops.py +198 -0
- insitukit-0.1.0/src/insitukit/erddap.py +107 -0
- insitukit-0.1.0/src/insitukit/py.typed +0 -0
- insitukit-0.1.0/src/insitukit/series.py +78 -0
- insitukit-0.1.0/src/insitukit/stats.py +77 -0
- insitukit-0.1.0/src/insitukit/store.py +98 -0
- insitukit-0.1.0/tests/test_insitukit.py +120 -0
- insitukit-0.1.0/uv.lock +1100 -0
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
# Pushing a tag vX.Y.Z builds the sdist and wheel and publishes them to PyPI through trusted publishing (OIDC): no API
|
|
4
|
+
# token is stored anywhere. PyPI must list this repository, this workflow file and the `pypi` environment as a
|
|
5
|
+
# trusted publisher for insitukit.
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
tags: ["v*"]
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- uses: astral-sh/setup-uv@v6
|
|
17
|
+
|
|
18
|
+
- name: Check the tag matches both version strings
|
|
19
|
+
run: |
|
|
20
|
+
version=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
|
|
21
|
+
init=$(sed -n 's/^__version__ = "\(.*\)"$/\1/p' src/insitukit/__init__.py)
|
|
22
|
+
if [ "v$version" != "$GITHUB_REF_NAME" ] || [ "$init" != "$version" ]; then
|
|
23
|
+
echo "Tag $GITHUB_REF_NAME, pyproject.toml $version and __version__ $init must agree" >&2
|
|
24
|
+
exit 1
|
|
25
|
+
fi
|
|
26
|
+
|
|
27
|
+
- name: Test
|
|
28
|
+
run: uv sync --dev --all-extras && uv run pytest
|
|
29
|
+
|
|
30
|
+
- name: Build
|
|
31
|
+
run: uv build
|
|
32
|
+
|
|
33
|
+
- name: Check the distributions
|
|
34
|
+
run: uvx twine check --strict dist/*
|
|
35
|
+
|
|
36
|
+
- uses: actions/upload-artifact@v4
|
|
37
|
+
with:
|
|
38
|
+
name: dist
|
|
39
|
+
path: dist/
|
|
40
|
+
|
|
41
|
+
publish:
|
|
42
|
+
needs: build
|
|
43
|
+
runs-on: ubuntu-latest
|
|
44
|
+
environment:
|
|
45
|
+
name: pypi
|
|
46
|
+
url: https://pypi.org/p/insitukit
|
|
47
|
+
permissions:
|
|
48
|
+
id-token: write
|
|
49
|
+
steps:
|
|
50
|
+
- uses: actions/download-artifact@v4
|
|
51
|
+
with:
|
|
52
|
+
name: dist
|
|
53
|
+
path: dist/
|
|
54
|
+
|
|
55
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
unit:
|
|
11
|
+
name: Unit tests
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
python: ["3.11", "3.12", "3.13"]
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
|
|
19
|
+
- name: Install uv
|
|
20
|
+
uses: astral-sh/setup-uv@v6
|
|
21
|
+
with:
|
|
22
|
+
enable-cache: true
|
|
23
|
+
python-version: ${{ matrix.python }}
|
|
24
|
+
|
|
25
|
+
- name: Sync dependencies
|
|
26
|
+
run: uv sync --dev --all-extras
|
|
27
|
+
|
|
28
|
+
# The pinned hooks from .pre-commit-config.yaml, so CI and a local commit apply the same ruff.
|
|
29
|
+
- name: Lint
|
|
30
|
+
run: uv run pre-commit run --all-files --show-diff-on-failure
|
|
31
|
+
|
|
32
|
+
- name: Unit tests
|
|
33
|
+
run: uv run pytest --durations=10
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# Python-generated files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info
|
|
8
|
+
|
|
9
|
+
# Virtual environments
|
|
10
|
+
.venv
|
|
11
|
+
.env
|
|
12
|
+
|
|
13
|
+
# Tooling
|
|
14
|
+
.pytest_cache/
|
|
15
|
+
.ruff_cache/
|
|
16
|
+
.mypy_cache/
|
|
17
|
+
.coverage
|
|
18
|
+
htmlcov/
|
|
19
|
+
.vscode/
|
|
20
|
+
|
|
21
|
+
# Planning ledger files stay out of public repositories
|
|
22
|
+
task.md
|
|
23
|
+
implementation_plan.md
|
|
24
|
+
walkthrough.md
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
3
|
+
rev: v5.0.0
|
|
4
|
+
hooks:
|
|
5
|
+
- id: trailing-whitespace
|
|
6
|
+
- id: end-of-file-fixer
|
|
7
|
+
- id: check-yaml
|
|
8
|
+
- id: check-added-large-files
|
|
9
|
+
- id: check-toml
|
|
10
|
+
- id: detect-private-key
|
|
11
|
+
|
|
12
|
+
# Pinned deliberately, as in forcingkit: CI runs these same hooks, and the rule selection is pinned alongside in
|
|
13
|
+
# pyproject.toml [tool.ruff.lint]. Linting only: no formatter, so the modules keep the layout they came with.
|
|
14
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
15
|
+
rev: v0.16.9
|
|
16
|
+
hooks:
|
|
17
|
+
- id: ruff-check
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# AGENTS.md
|
|
2
|
+
|
|
3
|
+
Context for AI developer agents working in `lhzn-io/insitukit`. Human entry point: `README.md`.
|
|
4
|
+
|
|
5
|
+
## Architecture
|
|
6
|
+
|
|
7
|
+
- `src/insitukit/`: `coops.py` (NOAA CO-OPS; an HTTP 200 with an error body or "No data was found" is an empty
|
|
8
|
+
result), `erddap.py` (ERDDAP tabledap with a per-instance budget and unreachable flag), `stats.py`, `series.py`
|
|
9
|
+
(the JSON series form), `store.py` (directory, Workers KV or GCS behind a name and size guard), and `identify` in
|
|
10
|
+
`__init__.py`.
|
|
11
|
+
- `tests/`: offline unit tests; HTTP calls are replaced.
|
|
12
|
+
- Dependencies stay small: numpy, pandas, requests. Anything heavier is an optional extra (`gcs`).
|
|
13
|
+
|
|
14
|
+
## Rules
|
|
15
|
+
|
|
16
|
+
- **Public repository and package.** Name only public data sources. No site-, partner- or project-specific code,
|
|
17
|
+
defaults or examples beyond public stations; those belong in the projects that use the library.
|
|
18
|
+
- **Polite clients.** Every request is paced and identified; never remove the pacing, budget or retry limits.
|
|
19
|
+
- **Stable formats.** `series` output is read by web pages; change it only in a way old readers still handle, and
|
|
20
|
+
note it in the release.
|
|
21
|
+
- Style: no emojis, no em dashes, plain technical language. Organisation name: Long Horizon Observatory.
|
|
22
|
+
- **Releases.** Bump `version` in `pyproject.toml` and `__version__` together; a `v<version>` tag publishes to PyPI
|
|
23
|
+
through `.github/workflows/publish.yml` (trusted publishing, no token).
|
|
24
|
+
- **Commits.** Author and commit as the human contributor (`git config` user). Record the agent harness and model as
|
|
25
|
+
a plain-text line at the end of the body, for example `Collaborated with Claude Code (Opus 5.5).` Never a
|
|
26
|
+
`Co-Authored-By:` trailer.
|
|
27
|
+
|
|
28
|
+
## Commands
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv sync --dev
|
|
32
|
+
uv run pytest
|
|
33
|
+
uv run pre-commit run --all-files
|
|
34
|
+
```
|
insitukit-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Long Horizon Observatory
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
insitukit-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: insitukit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Fetch, summarise and publish in situ ocean observations: NOAA CO-OPS and ERDDAP clients, record statistics, and a compact JSON time-series format.
|
|
5
|
+
Project-URL: Issues, https://github.com/lhzn-io/insitukit/issues
|
|
6
|
+
Project-URL: Source, https://github.com/lhzn-io/insitukit
|
|
7
|
+
Author-email: Long Horizon Observatory <dfry@lhzn.io>
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026 Long Horizon Observatory
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Classifier: Development Status :: 3 - Alpha
|
|
31
|
+
Classifier: Intended Audience :: Science/Research
|
|
32
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
33
|
+
Classifier: Programming Language :: Python :: 3
|
|
34
|
+
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
35
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
36
|
+
Requires-Python: >=3.11
|
|
37
|
+
Requires-Dist: numpy>=1.26
|
|
38
|
+
Requires-Dist: pandas>=2.2
|
|
39
|
+
Requires-Dist: requests>=2.32
|
|
40
|
+
Provides-Extra: gcs
|
|
41
|
+
Requires-Dist: google-cloud-storage>=2.18; extra == 'gcs'
|
|
42
|
+
Description-Content-Type: text/markdown
|
|
43
|
+
|
|
44
|
+
# insitukit
|
|
45
|
+
|
|
46
|
+
[](https://www.python.org/)
|
|
47
|
+
[](https://opensource.org/licenses/MIT)
|
|
48
|
+
|
|
49
|
+
Fetch, summarise and publish in situ ocean observations: NOAA CO-OPS and ERDDAP clients, record statistics, and a
|
|
50
|
+
compact JSON time-series format.
|
|
51
|
+
|
|
52
|
+
forcingkit prepares what drives a coastal model and topobathykit builds the ground it runs on; insitukit handles what
|
|
53
|
+
the instruments in the water actually measured. It reads tide gauges and buoys from NOAA CO-OPS and any ERDDAP server
|
|
54
|
+
with care for the provider: requests are paced, budgeted and identified, and an empty window is an empty result, not
|
|
55
|
+
an error. It reduces long records to the summaries people read: daily means only where a day is at least half
|
|
56
|
+
covered, exceedance curves, joint histograms, and marine heatwave climatologies after Hobday et al. (2016). And it
|
|
57
|
+
writes them as small JSON series behind a guard that refuses unexpected object names or sizes, so a web page can draw
|
|
58
|
+
years of data from a few files.
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install insitukit # CO-OPS, ERDDAP, statistics, the JSON form, local and Workers KV storage
|
|
64
|
+
pip install "insitukit[gcs]" # also Google Cloud Storage
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Use
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from datetime import datetime, timezone
|
|
71
|
+
|
|
72
|
+
import insitukit
|
|
73
|
+
from insitukit import coops, series, stats
|
|
74
|
+
from insitukit.erddap import Erddap
|
|
75
|
+
|
|
76
|
+
insitukit.identify("my-site/1.0 (+https://example.org; data@example.org)", application="my-site")
|
|
77
|
+
|
|
78
|
+
start, end = datetime(2026, 10, 1, tzinfo=timezone.utc), datetime(2026, 11, 1, tzinfo=timezone.utc)
|
|
79
|
+
|
|
80
|
+
# Observed water level and the station's own predictions; their difference is the surge.
|
|
81
|
+
level = coops.fetch("water_level", "8419870", start, end).set_index("time")["water_level"]
|
|
82
|
+
surge = level - coops.predictions("8419870", start, end).reindex(level.index)
|
|
83
|
+
|
|
84
|
+
# A buoy's temperature from an ERDDAP server, at most 20 requests this run.
|
|
85
|
+
server = Erddap("https://data.neracoos.org/erddap", budget=20)
|
|
86
|
+
temp = server.fetch("UNH_CML", start, end, ["temperature"]).set_index("time")["temperature"]
|
|
87
|
+
|
|
88
|
+
daily = stats.day_means(temp, per_day=24, digits=2, tz="America/New_York")
|
|
89
|
+
hourly = series.encode(surge.resample("h").mean(), unit="m", places=3) # {"t0": ..., "step": 3600, ...}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Modules
|
|
93
|
+
|
|
94
|
+
| Module | Contents |
|
|
95
|
+
| :--- | :--- |
|
|
96
|
+
| `insitukit.coops` | `fetch` (observations; long windows split to the API's limits), `predictions` (6-minute tides), `tide_events` (highs and lows), `metadata`. Metric units and UTC throughout. |
|
|
97
|
+
| `insitukit.erddap` | `Erddap(base_url, budget, min_interval)`: tabledap CSV into a DataFrame, with a per-run request budget and an unreachable flag. |
|
|
98
|
+
| `insitukit.stats` | `day_means` (coverage rule), `exceedance`, `joint` (two series paired by nearest time), `climatology` (Hobday et al. 2016: 11-day window, 31-day smoothing, mean, SD, 90th percentile). |
|
|
99
|
+
| `insitukit.series` | `encode` and `decode` for one series, `encode_frame` and `decode_frame` for several on one grid, `iso`. |
|
|
100
|
+
| `insitukit.store` | `Store(root, writable, max_bytes)`: JSON and bytes to a directory, `kv://<account>/<namespace>` (Cloudflare Workers KV, token in `CLOUDFLARE_API_TOKEN`) or `gs://bucket/prefix`, refusing names outside `writable` and objects over `max_bytes`. |
|
|
101
|
+
|
|
102
|
+
## Etiquette
|
|
103
|
+
|
|
104
|
+
CO-OPS requests are at least 1 s apart, and 403 and 429 answers are treated as throttling and retried with backoff.
|
|
105
|
+
ERDDAP requests are at least `min_interval` apart and stop at the run's `budget`. Both send the User-Agent given to
|
|
106
|
+
`identify`; include a way for the provider to reach you.
|
|
107
|
+
|
|
108
|
+
## Reference
|
|
109
|
+
|
|
110
|
+
Hobday, A. J., et al. (2016). A hierarchical approach to defining marine heatwaves. *Progress in Oceanography*, 141,
|
|
111
|
+
227-238. doi:10.1016/j.pocean.2015.12.014
|
|
112
|
+
|
|
113
|
+
## License
|
|
114
|
+
|
|
115
|
+
MIT. Copyright (c) 2026 Long Horizon Observatory.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# insitukit
|
|
2
|
+
|
|
3
|
+
[](https://www.python.org/)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
|
|
6
|
+
Fetch, summarise and publish in situ ocean observations: NOAA CO-OPS and ERDDAP clients, record statistics, and a
|
|
7
|
+
compact JSON time-series format.
|
|
8
|
+
|
|
9
|
+
forcingkit prepares what drives a coastal model and topobathykit builds the ground it runs on; insitukit handles what
|
|
10
|
+
the instruments in the water actually measured. It reads tide gauges and buoys from NOAA CO-OPS and any ERDDAP server
|
|
11
|
+
with care for the provider: requests are paced, budgeted and identified, and an empty window is an empty result, not
|
|
12
|
+
an error. It reduces long records to the summaries people read: daily means only where a day is at least half
|
|
13
|
+
covered, exceedance curves, joint histograms, and marine heatwave climatologies after Hobday et al. (2016). And it
|
|
14
|
+
writes them as small JSON series behind a guard that refuses unexpected object names or sizes, so a web page can draw
|
|
15
|
+
years of data from a few files.
|
|
16
|
+
|
|
17
|
+
## Install
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
pip install insitukit # CO-OPS, ERDDAP, statistics, the JSON form, local and Workers KV storage
|
|
21
|
+
pip install "insitukit[gcs]" # also Google Cloud Storage
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Use
|
|
25
|
+
|
|
26
|
+
```python
|
|
27
|
+
from datetime import datetime, timezone
|
|
28
|
+
|
|
29
|
+
import insitukit
|
|
30
|
+
from insitukit import coops, series, stats
|
|
31
|
+
from insitukit.erddap import Erddap
|
|
32
|
+
|
|
33
|
+
insitukit.identify("my-site/1.0 (+https://example.org; data@example.org)", application="my-site")
|
|
34
|
+
|
|
35
|
+
start, end = datetime(2026, 10, 1, tzinfo=timezone.utc), datetime(2026, 11, 1, tzinfo=timezone.utc)
|
|
36
|
+
|
|
37
|
+
# Observed water level and the station's own predictions; their difference is the surge.
|
|
38
|
+
level = coops.fetch("water_level", "8419870", start, end).set_index("time")["water_level"]
|
|
39
|
+
surge = level - coops.predictions("8419870", start, end).reindex(level.index)
|
|
40
|
+
|
|
41
|
+
# A buoy's temperature from an ERDDAP server, at most 20 requests this run.
|
|
42
|
+
server = Erddap("https://data.neracoos.org/erddap", budget=20)
|
|
43
|
+
temp = server.fetch("UNH_CML", start, end, ["temperature"]).set_index("time")["temperature"]
|
|
44
|
+
|
|
45
|
+
daily = stats.day_means(temp, per_day=24, digits=2, tz="America/New_York")
|
|
46
|
+
hourly = series.encode(surge.resample("h").mean(), unit="m", places=3) # {"t0": ..., "step": 3600, ...}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Modules
|
|
50
|
+
|
|
51
|
+
| Module | Contents |
|
|
52
|
+
| :--- | :--- |
|
|
53
|
+
| `insitukit.coops` | `fetch` (observations; long windows split to the API's limits), `predictions` (6-minute tides), `tide_events` (highs and lows), `metadata`. Metric units and UTC throughout. |
|
|
54
|
+
| `insitukit.erddap` | `Erddap(base_url, budget, min_interval)`: tabledap CSV into a DataFrame, with a per-run request budget and an unreachable flag. |
|
|
55
|
+
| `insitukit.stats` | `day_means` (coverage rule), `exceedance`, `joint` (two series paired by nearest time), `climatology` (Hobday et al. 2016: 11-day window, 31-day smoothing, mean, SD, 90th percentile). |
|
|
56
|
+
| `insitukit.series` | `encode` and `decode` for one series, `encode_frame` and `decode_frame` for several on one grid, `iso`. |
|
|
57
|
+
| `insitukit.store` | `Store(root, writable, max_bytes)`: JSON and bytes to a directory, `kv://<account>/<namespace>` (Cloudflare Workers KV, token in `CLOUDFLARE_API_TOKEN`) or `gs://bucket/prefix`, refusing names outside `writable` and objects over `max_bytes`. |
|
|
58
|
+
|
|
59
|
+
## Etiquette
|
|
60
|
+
|
|
61
|
+
CO-OPS requests are at least 1 s apart, and 403 and 429 answers are treated as throttling and retried with backoff.
|
|
62
|
+
ERDDAP requests are at least `min_interval` apart and stop at the run's `budget`. Both send the User-Agent given to
|
|
63
|
+
`identify`; include a way for the provider to reach you.
|
|
64
|
+
|
|
65
|
+
## Reference
|
|
66
|
+
|
|
67
|
+
Hobday, A. J., et al. (2016). A hierarchical approach to defining marine heatwaves. *Progress in Oceanography*, 141,
|
|
68
|
+
227-238. doi:10.1016/j.pocean.2015.12.014
|
|
69
|
+
|
|
70
|
+
## License
|
|
71
|
+
|
|
72
|
+
MIT. Copyright (c) 2026 Long Horizon Observatory.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "insitukit"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Fetch, summarise and publish in situ ocean observations: NOAA CO-OPS and ERDDAP clients, record statistics, and a compact JSON time-series format."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { file = "LICENSE" }
|
|
12
|
+
authors = [{ name = "Long Horizon Observatory", email = "dfry@lhzn.io" }]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 3 - Alpha",
|
|
15
|
+
"Intended Audience :: Science/Research",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Hydrology",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Information Analysis",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"numpy>=1.26",
|
|
23
|
+
"pandas>=2.2",
|
|
24
|
+
"requests>=2.32",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
# Store.root of gs://bucket/prefix. Cloudflare Workers KV (kv://) needs only requests.
|
|
29
|
+
gcs = ["google-cloud-storage>=2.18"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Issues = "https://github.com/lhzn-io/insitukit/issues"
|
|
33
|
+
Source = "https://github.com/lhzn-io/insitukit"
|
|
34
|
+
|
|
35
|
+
[tool.hatch.build.targets.wheel]
|
|
36
|
+
packages = ["src/insitukit"]
|
|
37
|
+
|
|
38
|
+
[tool.ruff]
|
|
39
|
+
line-length = 120
|
|
40
|
+
|
|
41
|
+
[tool.ruff.lint]
|
|
42
|
+
# The rule set forcingkit pins (pyflakes and the syntax-level pycodestyle rules), chosen explicitly so a ruff
|
|
43
|
+
# upgrade cannot silently change what "clean" means.
|
|
44
|
+
select = ["E4", "E7", "E9", "F"]
|
|
45
|
+
|
|
46
|
+
[dependency-groups]
|
|
47
|
+
dev = [
|
|
48
|
+
"pre-commit>=4.5.1",
|
|
49
|
+
"pytest>=8",
|
|
50
|
+
"ruff>=0.16.9",
|
|
51
|
+
]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Fetch, summarise and publish in situ ocean observations.
|
|
2
|
+
|
|
3
|
+
- ``insitukit.coops``: NOAA CO-OPS (Tides and Currents) observations, tide predictions and station metadata.
|
|
4
|
+
- ``insitukit.erddap``: a paced, budgeted ERDDAP tabledap client for any server.
|
|
5
|
+
- ``insitukit.stats``: daily means with a coverage rule, exceedance, joint histograms, marine heatwave climatology.
|
|
6
|
+
- ``insitukit.series``: the compact JSON form of a regular time series, and back.
|
|
7
|
+
- ``insitukit.store``: where the JSON goes (a directory, Cloudflare Workers KV or Google Cloud Storage), behind a
|
|
8
|
+
guard on object names and sizes.
|
|
9
|
+
|
|
10
|
+
Call ``identify`` once at start-up so every request names your application and a contact.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.0"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def identify(user_agent: str, application: str | None = None) -> None:
|
|
17
|
+
"""Name the caller on every request: ``user_agent`` as the HTTP User-Agent (include a URL or an address a data
|
|
18
|
+
provider can reach you at), and ``application`` as CO-OPS's ``application`` parameter (default: unchanged)."""
|
|
19
|
+
from . import coops
|
|
20
|
+
|
|
21
|
+
coops.USER_AGENT = user_agent
|
|
22
|
+
coops._session.headers["User-Agent"] = user_agent
|
|
23
|
+
if application is not None:
|
|
24
|
+
coops.APPLICATION = application
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""Fetch observations and tide predictions from the NOAA CO-OPS data API (Tides and Currents).
|
|
2
|
+
|
|
3
|
+
Every request asks for metric units and GMT; converting to other units is the caller's. The API answers HTTP 200 with an ``{"error": ...}`` body for a bad request, and "No data
|
|
4
|
+
was found" when a station lacks the sensor or the window falls in a gap: that is an empty result,
|
|
5
|
+
not a failure. Server errors and timeouts retry with backoff.
|
|
6
|
+
|
|
7
|
+
Per-request range limits (checked against the live API, 2026-10):
|
|
8
|
+
|
|
9
|
+
- 6-minute observations (``water_temperature``, ``air_temperature``, ``wind``, ``air_pressure``,
|
|
10
|
+
``water_level``): 31 days. We ask for 30-day spans.
|
|
11
|
+
- the same products with ``interval=h`` (the reading at the top of each hour), and ``hourly_height``:
|
|
12
|
+
365 days, compared by calendar date, so one calendar year (including a leap year) fits one request.
|
|
13
|
+
|
|
14
|
+
Requests carry ``USER_AGENT`` and ``APPLICATION``; set your own with ``insitukit.identify``.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import logging
|
|
20
|
+
import time
|
|
21
|
+
from datetime import datetime, timedelta, timezone
|
|
22
|
+
|
|
23
|
+
import numpy as np
|
|
24
|
+
import pandas as pd
|
|
25
|
+
import requests
|
|
26
|
+
|
|
27
|
+
from . import __version__
|
|
28
|
+
|
|
29
|
+
BASE_URL = "https://api.tidesandcurrents.noaa.gov/api/prod/datagetter"
|
|
30
|
+
META_URL = "https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations/{station}.json"
|
|
31
|
+
USER_AGENT = f"insitukit/{__version__} (+https://github.com/lhzn-io/insitukit)"
|
|
32
|
+
APPLICATION = "insitukit"
|
|
33
|
+
SIX_MINUTE_SPAN = timedelta(days=30)
|
|
34
|
+
# Value fields per product, renamed to the column names returned by fetch().
|
|
35
|
+
FIELDS = {
|
|
36
|
+
"wind": {"s": "wind_speed", "d": "wind_direction", "g": "wind_gust"},
|
|
37
|
+
}
|
|
38
|
+
DATUM_PRODUCTS = {"water_level", "hourly_height", "high_low", "predictions"}
|
|
39
|
+
|
|
40
|
+
# The API sits behind a gateway that answers 403 "Forbidden" (not 429) once a client sends too many
|
|
41
|
+
# requests in a short period; a burst of about 300 requests in 10 minutes was enough. We pace every
|
|
42
|
+
# request and treat 403 and 429 as throttling, retried with backoff like a server error.
|
|
43
|
+
MIN_INTERVAL = 1.0
|
|
44
|
+
THROTTLED = {403, 429}
|
|
45
|
+
|
|
46
|
+
log = logging.getLogger(__name__)
|
|
47
|
+
_session = requests.Session()
|
|
48
|
+
_session.headers["User-Agent"] = USER_AGENT
|
|
49
|
+
_last_request = 0.0
|
|
50
|
+
# Requests sent since import (or since the caller reset it); used for logging and cost checks.
|
|
51
|
+
request_count = 0
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class CoopsError(RuntimeError):
|
|
55
|
+
"""The API rejected the request (bad station, product or date range)."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _pace() -> None:
|
|
59
|
+
"""Keep at least MIN_INTERVAL seconds between requests."""
|
|
60
|
+
global _last_request
|
|
61
|
+
wait = _last_request + MIN_INTERVAL - time.monotonic()
|
|
62
|
+
if wait > 0:
|
|
63
|
+
time.sleep(wait)
|
|
64
|
+
_last_request = time.monotonic()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _get(params: dict, retries: int = 4, timeout: int = 120) -> dict | None:
|
|
68
|
+
"""GET the data API with retries; return None when the API reports no data for the window."""
|
|
69
|
+
global request_count
|
|
70
|
+
query = {"units": "metric", "time_zone": "gmt", "format": "json", "application": APPLICATION, **params}
|
|
71
|
+
delay = 5.0
|
|
72
|
+
for attempt in range(1, retries + 1):
|
|
73
|
+
request_count += 1
|
|
74
|
+
_pace()
|
|
75
|
+
try:
|
|
76
|
+
resp = _session.get(BASE_URL, params=query, timeout=timeout)
|
|
77
|
+
except requests.RequestException as exc:
|
|
78
|
+
log.warning("request failed (%s), attempt %d: %s", exc.__class__.__name__, attempt, params)
|
|
79
|
+
else:
|
|
80
|
+
if resp.status_code == 200:
|
|
81
|
+
body = resp.json()
|
|
82
|
+
err = (body.get("error") or {}).get("message") if isinstance(body, dict) else None
|
|
83
|
+
if err is None:
|
|
84
|
+
return body
|
|
85
|
+
if "no data was found" in err.lower():
|
|
86
|
+
return None
|
|
87
|
+
raise CoopsError(f"{err.strip()} ({params})")
|
|
88
|
+
if resp.status_code < 500 and resp.status_code not in THROTTLED:
|
|
89
|
+
raise CoopsError(f"HTTP {resp.status_code} ({params})")
|
|
90
|
+
log.warning("HTTP %d, attempt %d: %s", resp.status_code, attempt, params)
|
|
91
|
+
if attempt < retries:
|
|
92
|
+
time.sleep(delay)
|
|
93
|
+
delay *= 2
|
|
94
|
+
raise RuntimeError(f"giving up after {retries} attempts: {params}")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _stamp(ts: datetime) -> str:
|
|
98
|
+
return ts.astimezone(timezone.utc).strftime("%Y%m%d %H:%M")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def spans(start: datetime, end: datetime, hourly: bool) -> list[tuple[datetime, datetime]]:
|
|
102
|
+
"""Request windows covering ``[start, end)``: calendar years for hourly data, 30 days otherwise."""
|
|
103
|
+
out = []
|
|
104
|
+
lo = start
|
|
105
|
+
while lo < end:
|
|
106
|
+
if hourly:
|
|
107
|
+
hi = min(datetime(lo.year + 1, 1, 1, tzinfo=timezone.utc), end)
|
|
108
|
+
else:
|
|
109
|
+
hi = min(lo + SIX_MINUTE_SPAN, end)
|
|
110
|
+
out.append((lo, hi))
|
|
111
|
+
lo = hi
|
|
112
|
+
return out
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def fetch(
|
|
116
|
+
product: str,
|
|
117
|
+
station: str,
|
|
118
|
+
start: datetime,
|
|
119
|
+
end: datetime,
|
|
120
|
+
interval: str | None = None,
|
|
121
|
+
datum: str = "MLLW",
|
|
122
|
+
) -> pd.DataFrame:
|
|
123
|
+
"""Observations for ``start <= time < end`` with a UTC ``time`` column and float value columns.
|
|
124
|
+
|
|
125
|
+
Single-value products return one column named after the product; ``wind`` returns
|
|
126
|
+
``wind_speed`` (m/s), ``wind_direction`` (degrees true) and ``wind_gust`` (m/s). Readings the
|
|
127
|
+
API flags as missing come back as NaN. Long windows are split to respect the range limits.
|
|
128
|
+
"""
|
|
129
|
+
fields = FIELDS.get(product, {"v": product})
|
|
130
|
+
hourly = interval == "h" or product == "hourly_height"
|
|
131
|
+
frames = []
|
|
132
|
+
for lo, hi in spans(start, end, hourly):
|
|
133
|
+
# The API's end_date is inclusive; stop one minute short so adjacent spans do not overlap.
|
|
134
|
+
params = {"product": product, "station": station, "begin_date": _stamp(lo), "end_date": _stamp(hi - timedelta(minutes=1))}
|
|
135
|
+
if interval:
|
|
136
|
+
params["interval"] = interval
|
|
137
|
+
if product in DATUM_PRODUCTS:
|
|
138
|
+
params["datum"] = datum
|
|
139
|
+
body = _get(params)
|
|
140
|
+
rows = (body or {}).get("data") or []
|
|
141
|
+
log.info("%s %s %s..%s: %d rows", station, product, params["begin_date"], params["end_date"], len(rows))
|
|
142
|
+
if rows:
|
|
143
|
+
frames.append(pd.DataFrame(rows))
|
|
144
|
+
cols = ["time", *fields.values()]
|
|
145
|
+
if not frames:
|
|
146
|
+
return pd.DataFrame({c: pd.Series(dtype="datetime64[ns, UTC]" if c == "time" else float) for c in cols})
|
|
147
|
+
raw = pd.concat(frames, ignore_index=True)
|
|
148
|
+
df = pd.DataFrame({"time": pd.to_datetime(raw["t"], utc=True)})
|
|
149
|
+
for key, name in fields.items():
|
|
150
|
+
df[name] = pd.to_numeric(raw[key], errors="coerce") if key in raw.columns else np.nan
|
|
151
|
+
df = df[(df["time"] >= pd.Timestamp(start)) & (df["time"] < pd.Timestamp(end))]
|
|
152
|
+
return df.sort_values("time").drop_duplicates("time").reset_index(drop=True)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def tide_events(station: str, start: datetime, end: datetime, datum: str = "MLLW") -> list[dict]:
|
|
156
|
+
"""High and low tide predictions in ``[start, end)``: ``[{"time": Timestamp, "m": float, "type": "H"|"L"}]``."""
|
|
157
|
+
params = {
|
|
158
|
+
"product": "predictions",
|
|
159
|
+
"station": station,
|
|
160
|
+
"begin_date": _stamp(start),
|
|
161
|
+
"end_date": _stamp(end - timedelta(minutes=1)),
|
|
162
|
+
"interval": "hilo",
|
|
163
|
+
"datum": datum,
|
|
164
|
+
}
|
|
165
|
+
body = _get(params) or {}
|
|
166
|
+
out = []
|
|
167
|
+
for row in body.get("predictions") or []:
|
|
168
|
+
try:
|
|
169
|
+
out.append({"time": pd.Timestamp(row["t"], tz="UTC"), "m": float(row["v"]), "type": row["type"]})
|
|
170
|
+
except (KeyError, TypeError, ValueError):
|
|
171
|
+
continue
|
|
172
|
+
return out
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def predictions(station: str, start: datetime, end: datetime, datum: str = "MLLW") -> pd.Series:
|
|
176
|
+
"""6-minute tide predictions (m above ``datum``) in ``[start, end)``, indexed by UTC time. A station's own
|
|
177
|
+
predictions taken from its observed water level give the surge without the phase shift of a nearby station's."""
|
|
178
|
+
vals: dict[pd.Timestamp, float] = {}
|
|
179
|
+
for lo, hi in spans(start, end, hourly=False):
|
|
180
|
+
body = _get({"product": "predictions", "station": station, "datum": datum, "interval": "6",
|
|
181
|
+
"begin_date": _stamp(lo), "end_date": _stamp(hi - timedelta(minutes=1))}) or {}
|
|
182
|
+
for row in body.get("predictions") or []:
|
|
183
|
+
try:
|
|
184
|
+
vals[pd.Timestamp(row["t"], tz="UTC")] = float(row["v"])
|
|
185
|
+
except (KeyError, TypeError, ValueError):
|
|
186
|
+
continue
|
|
187
|
+
s = pd.Series(vals, dtype=float).sort_index()
|
|
188
|
+
return s[(s.index >= pd.Timestamp(start)) & (s.index < pd.Timestamp(end))]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def metadata(station: str) -> dict:
|
|
192
|
+
"""Station metadata with details and sensors (name, lat, lng, established, sensor list)."""
|
|
193
|
+
global request_count
|
|
194
|
+
request_count += 1
|
|
195
|
+
_pace()
|
|
196
|
+
resp = _session.get(META_URL.format(station=station), params={"expand": "details,sensors"}, timeout=60)
|
|
197
|
+
resp.raise_for_status()
|
|
198
|
+
return resp.json()["stations"][0]
|