usdata 0.18.0__tar.gz → 0.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- usdata-0.20.0/PKG-INFO +124 -0
- usdata-0.20.0/README.md +79 -0
- {usdata-0.18.0 → usdata-0.20.0}/pyproject.toml +17 -3
- {usdata-0.18.0 → usdata-0.20.0}/pyproject.toml.orig +24 -3
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_fetch.py +28 -3
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_grib.py +32 -11
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/app.py +22 -4
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/inspect.py +2 -1
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/data/registry.yaml +182 -41
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/doctor.py +3 -2
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/inspect.py +69 -20
- usdata-0.20.0/src/usdata/mirror.py +60 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/models.py +4 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/http.py +4 -2
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/listing.py +36 -10
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/base.py +29 -6
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/coastwatch.py +1 -1
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/coops.py +5 -1
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/grib_index.py +75 -26
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/hrrr.py +31 -10
- usdata-0.20.0/src/usdata/providers/noaa/ibtracs.py +200 -0
- usdata-0.20.0/src/usdata/providers/noaa/nbm.py +106 -0
- usdata-0.20.0/src/usdata/providers/noaa/rap.py +110 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/params.py +22 -0
- usdata-0.20.0/src/usdata/providers/usgs/earthquakes.py +147 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/pull.py +240 -80
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/query.py +19 -5
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/readers.py +100 -15
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/testing.py +6 -1
- usdata-0.18.0/PKG-INFO +0 -123
- usdata-0.18.0/README.md +0 -89
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/__init__.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/__main__.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_files.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_hurdat2.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_netcdf.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_progress.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_radar.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cache.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cache_ops.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cite.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/__init__.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/cache.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/cite.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/doctor.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/progress.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/data/nexrad_sites.csv +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/data/places.csv +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/data/places.sources.json +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/manifest.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/__init__.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/erddap.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/s3.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/provenance.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/__init__.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/http.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/__init__.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/gfs.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/ghcnd.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/glm.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/goes.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/gsom.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/gsoy.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/hurdat2.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/lcd.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/mrms.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/nexrad.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/nexrad_level3.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/normals.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/sites.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/spc.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/storm_events.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/usgs/__init__.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/usgs/daily.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/py.typed +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/registry.py +0 -0
- {usdata-0.18.0 → usdata-0.20.0}/src/usdata/selection.py +0 -0
usdata-0.20.0/PKG-INFO
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: usdata
|
|
3
|
+
Version: 0.20.0
|
|
4
|
+
Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
|
|
5
|
+
Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
|
|
6
|
+
Author: Jake Van Slyke
|
|
7
|
+
Author-email: Jake Van Slyke <jakervanslyke@gmail.com>
|
|
8
|
+
License-Expression: Apache-2.0
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
21
|
+
Requires-Dist: httpx>=0.28.1
|
|
22
|
+
Requires-Dist: pydantic>=2.7
|
|
23
|
+
Requires-Dist: pyyaml>=6.0
|
|
24
|
+
Requires-Dist: typer>=0.18
|
|
25
|
+
Requires-Dist: eccodes>=2.48 ; extra == 'grib'
|
|
26
|
+
Requires-Dist: eccodeslib>=2.48 ; sys_platform == 'linux' and extra == 'grib'
|
|
27
|
+
Requires-Dist: xarray>=2025.1 ; extra == 'grib'
|
|
28
|
+
Requires-Dist: numpy>=2 ; extra == 'grib'
|
|
29
|
+
Requires-Dist: xarray>=2025.1 ; extra == 'netcdf'
|
|
30
|
+
Requires-Dist: h5netcdf[h5py]>=1.8.1 ; extra == 'netcdf'
|
|
31
|
+
Requires-Dist: pandas>=2.2 ; extra == 'pandas'
|
|
32
|
+
Requires-Dist: xradar>=0.12.0 ; extra == 'radar'
|
|
33
|
+
Requires-Python: >=3.11
|
|
34
|
+
Project-URL: Homepage, https://usdata.dev/
|
|
35
|
+
Project-URL: Documentation, https://docs.usdata.dev/
|
|
36
|
+
Project-URL: Repository, https://github.com/jakeryderv/usdata
|
|
37
|
+
Project-URL: Examples, https://usdata.dev/examples/
|
|
38
|
+
Project-URL: Changelog, https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md
|
|
39
|
+
Project-URL: Issues, https://github.com/jakeryderv/usdata/issues
|
|
40
|
+
Provides-Extra: grib
|
|
41
|
+
Provides-Extra: netcdf
|
|
42
|
+
Provides-Extra: pandas
|
|
43
|
+
Provides-Extra: radar
|
|
44
|
+
Description-Content-Type: text/markdown
|
|
45
|
+
|
|
46
|
+
# usdata
|
|
47
|
+
|
|
48
|
+
[](https://pypi.org/project/usdata/)
|
|
49
|
+
[](https://pypi.org/project/usdata/)
|
|
50
|
+
[](https://github.com/jakeryderv/usdata/actions/workflows/ci.yml)
|
|
51
|
+
[](https://github.com/jakeryderv/usdata/actions/workflows/integration.yml)
|
|
52
|
+
[](LICENSE)
|
|
53
|
+
[](https://docs.usdata.dev/)
|
|
54
|
+
|
|
55
|
+
[](https://usdata.dev/)
|
|
56
|
+
|
|
57
|
+
Reproducible acquisition of U.S. public scientific data. One Python SDK and
|
|
58
|
+
CLI discovers curated NOAA and USGS datasets, fetches their files, and keeps a
|
|
59
|
+
record of every input: a manifest names them, a lockfile pins them by checksum,
|
|
60
|
+
and the record of what was fetched is what a methods section cites. Analysis
|
|
61
|
+
stays in pandas and xarray; usdata only acquires. If you need every product
|
|
62
|
+
one agency publishes, that agency's own library is the better tool; usdata is
|
|
63
|
+
for pinning inputs across sources and proving later that they have not changed.
|
|
64
|
+
|
|
65
|
+
```sh
|
|
66
|
+
pip install "usdata[pandas]"
|
|
67
|
+
usdata search precipitation --location Oklahoma
|
|
68
|
+
usdata fetch noaa:ghcn-daily -p stations=USW00013967 --start 2024-05-06 --end 2024-05-07
|
|
69
|
+
usdata pull dataset.yaml && usdata cite dataset.yaml
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The same four steps from Python:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from usdata import build_query, fetch, get, pull
|
|
76
|
+
|
|
77
|
+
query = build_query(start="2024-05-06", end="2024-05-07", stations="USW00013967")
|
|
78
|
+
items = fetch(get("noaa:ghcn-daily"), query)
|
|
79
|
+
frame = items[0].open() # pandas DataFrame, units and provenance in frame.attrs
|
|
80
|
+
result = pull("dataset.yaml") # every manifest source, pinned in dataset.lock.json
|
|
81
|
+
print(result.fetched[0].provenance.checksum) # what usdata cite reports
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
**Alpha.** Releases are tested and lockfiles are stable, but the Python API
|
|
85
|
+
and CLI can change between minor versions; the [changelog](CHANGELOG.md) marks
|
|
86
|
+
breaking changes.
|
|
87
|
+
|
|
88
|
+
## Where to look
|
|
89
|
+
|
|
90
|
+
| | |
|
|
91
|
+
| --- | --- |
|
|
92
|
+
| [usdata.dev](https://usdata.dev/) | What it is for: the [dataset browser](https://usdata.dev/datasets/) and [worked examples](https://usdata.dev/examples/) with saved results |
|
|
93
|
+
| [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
|
|
94
|
+
| [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
|
|
95
|
+
|
|
96
|
+
Twenty-three datasets are available today and twenty-one more are planned, grouped
|
|
97
|
+
by agency and product family in the [catalog](docs/providers/README.md).
|
|
98
|
+
|
|
99
|
+
## How this compares
|
|
100
|
+
|
|
101
|
+
usdata resolves a place, a time window, and a few parameters to the exact
|
|
102
|
+
files that satisfy them, then pins those files so the same inputs can be
|
|
103
|
+
restored and verified later. Neighbouring tools each do part of that.
|
|
104
|
+
|
|
105
|
+
| Tool | What it does | What usdata adds |
|
|
106
|
+
| --- | --- | --- |
|
|
107
|
+
| [pooch](https://www.fatiando.org/pooch/) | Downloads and verifies files from a registry of names, hashes, and a base URL you write | Discovery: the query resolves to the URLs, and the lockfile is written, not authored |
|
|
108
|
+
| [intake](https://intake.readthedocs.io/) | Describes datasets in catalogs and opens them through drivers | Checksums, lockfiles, and provenance; usdata stops at handing a file to pandas or xarray |
|
|
109
|
+
| [DVC](https://dvc.org/) | Versions data files and can track one URL by its ETag or hash | Knowing which URLs a scientific question needs, across agencies |
|
|
110
|
+
| [Herbie](https://herbie.readthedocs.io/) | Deep access to model output, including subsetting GRIB2 through index files | One interface across observations, radar, satellite, and models, with a lockfile; usdata's index-file fetch reproduces Herbie's for that reason |
|
|
111
|
+
| [dataretrieval](https://github.com/DOI-USGS/dataretrieval-python) | Full coverage of USGS water data | Cross-agency pinning; use dataretrieval for USGS breadth |
|
|
112
|
+
|
|
113
|
+
## Contributing
|
|
114
|
+
|
|
115
|
+
Setup, commands, checks, and the release procedure are in
|
|
116
|
+
[CONTRIBUTING.md](CONTRIBUTING.md). The project record lives beside it:
|
|
117
|
+
[architecture](docs/architecture.md), [testing](docs/testing.md),
|
|
118
|
+
[decisions](docs/adr/README.md), [roadmap](docs/roadmap.md), and
|
|
119
|
+
[versioning](docs/versioning.md). The main way the project grows is a new
|
|
120
|
+
dataset, following [adding a dataset](docs/guides/adding-a-dataset.md).
|
|
121
|
+
|
|
122
|
+
## License
|
|
123
|
+
|
|
124
|
+
[Apache-2.0](LICENSE).
|
usdata-0.20.0/README.md
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# usdata
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/usdata/)
|
|
4
|
+
[](https://pypi.org/project/usdata/)
|
|
5
|
+
[](https://github.com/jakeryderv/usdata/actions/workflows/ci.yml)
|
|
6
|
+
[](https://github.com/jakeryderv/usdata/actions/workflows/integration.yml)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
[](https://docs.usdata.dev/)
|
|
9
|
+
|
|
10
|
+
[](https://usdata.dev/)
|
|
11
|
+
|
|
12
|
+
Reproducible acquisition of U.S. public scientific data. One Python SDK and
|
|
13
|
+
CLI discovers curated NOAA and USGS datasets, fetches their files, and keeps a
|
|
14
|
+
record of every input: a manifest names them, a lockfile pins them by checksum,
|
|
15
|
+
and the record of what was fetched is what a methods section cites. Analysis
|
|
16
|
+
stays in pandas and xarray; usdata only acquires. If you need every product
|
|
17
|
+
one agency publishes, that agency's own library is the better tool; usdata is
|
|
18
|
+
for pinning inputs across sources and proving later that they have not changed.
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
pip install "usdata[pandas]"
|
|
22
|
+
usdata search precipitation --location Oklahoma
|
|
23
|
+
usdata fetch noaa:ghcn-daily -p stations=USW00013967 --start 2024-05-06 --end 2024-05-07
|
|
24
|
+
usdata pull dataset.yaml && usdata cite dataset.yaml
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
The same four steps from Python:
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
from usdata import build_query, fetch, get, pull
|
|
31
|
+
|
|
32
|
+
query = build_query(start="2024-05-06", end="2024-05-07", stations="USW00013967")
|
|
33
|
+
items = fetch(get("noaa:ghcn-daily"), query)
|
|
34
|
+
frame = items[0].open() # pandas DataFrame, units and provenance in frame.attrs
|
|
35
|
+
result = pull("dataset.yaml") # every manifest source, pinned in dataset.lock.json
|
|
36
|
+
print(result.fetched[0].provenance.checksum) # what usdata cite reports
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
**Alpha.** Releases are tested and lockfiles are stable, but the Python API
|
|
40
|
+
and CLI can change between minor versions; the [changelog](CHANGELOG.md) marks
|
|
41
|
+
breaking changes.
|
|
42
|
+
|
|
43
|
+
## Where to look
|
|
44
|
+
|
|
45
|
+
| | |
|
|
46
|
+
| --- | --- |
|
|
47
|
+
| [usdata.dev](https://usdata.dev/) | What it is for: the [dataset browser](https://usdata.dev/datasets/) and [worked examples](https://usdata.dev/examples/) with saved results |
|
|
48
|
+
| [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
|
|
49
|
+
| [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
|
|
50
|
+
|
|
51
|
+
Twenty-three datasets are available today and twenty-one more are planned, grouped
|
|
52
|
+
by agency and product family in the [catalog](docs/providers/README.md).
|
|
53
|
+
|
|
54
|
+
## How this compares
|
|
55
|
+
|
|
56
|
+
usdata resolves a place, a time window, and a few parameters to the exact
|
|
57
|
+
files that satisfy them, then pins those files so the same inputs can be
|
|
58
|
+
restored and verified later. Neighbouring tools each do part of that.
|
|
59
|
+
|
|
60
|
+
| Tool | What it does | What usdata adds |
|
|
61
|
+
| --- | --- | --- |
|
|
62
|
+
| [pooch](https://www.fatiando.org/pooch/) | Downloads and verifies files from a registry of names, hashes, and a base URL you write | Discovery: the query resolves to the URLs, and the lockfile is written, not authored |
|
|
63
|
+
| [intake](https://intake.readthedocs.io/) | Describes datasets in catalogs and opens them through drivers | Checksums, lockfiles, and provenance; usdata stops at handing a file to pandas or xarray |
|
|
64
|
+
| [DVC](https://dvc.org/) | Versions data files and can track one URL by its ETag or hash | Knowing which URLs a scientific question needs, across agencies |
|
|
65
|
+
| [Herbie](https://herbie.readthedocs.io/) | Deep access to model output, including subsetting GRIB2 through index files | One interface across observations, radar, satellite, and models, with a lockfile; usdata's index-file fetch reproduces Herbie's for that reason |
|
|
66
|
+
| [dataretrieval](https://github.com/DOI-USGS/dataretrieval-python) | Full coverage of USGS water data | Cross-agency pinning; use dataretrieval for USGS breadth |
|
|
67
|
+
|
|
68
|
+
## Contributing
|
|
69
|
+
|
|
70
|
+
Setup, commands, checks, and the release procedure are in
|
|
71
|
+
[CONTRIBUTING.md](CONTRIBUTING.md). The project record lives beside it:
|
|
72
|
+
[architecture](docs/architecture.md), [testing](docs/testing.md),
|
|
73
|
+
[decisions](docs/adr/README.md), [roadmap](docs/roadmap.md), and
|
|
74
|
+
[versioning](docs/versioning.md). The main way the project grows is a new
|
|
75
|
+
dataset, following [adding a dataset](docs/guides/adding-a-dataset.md).
|
|
76
|
+
|
|
77
|
+
## License
|
|
78
|
+
|
|
79
|
+
[Apache-2.0](LICENSE).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "usdata"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.20.0"
|
|
4
4
|
description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -8,16 +8,27 @@ requires-python = ">=3.11"
|
|
|
8
8
|
keywords = [
|
|
9
9
|
"noaa",
|
|
10
10
|
"usgs",
|
|
11
|
-
"nasa",
|
|
12
11
|
"open-data",
|
|
13
12
|
"scientific-data",
|
|
14
13
|
"provenance",
|
|
14
|
+
"reproducible-research",
|
|
15
|
+
"weather",
|
|
16
|
+
"climate",
|
|
17
|
+
"meteorology",
|
|
15
18
|
]
|
|
16
19
|
classifiers = [
|
|
17
|
-
"Development Status ::
|
|
20
|
+
"Development Status :: 3 - Alpha",
|
|
21
|
+
"Environment :: Console",
|
|
18
22
|
"Intended Audience :: Science/Research",
|
|
23
|
+
"Operating System :: OS Independent",
|
|
19
24
|
"Programming Language :: Python :: 3",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Programming Language :: Python :: 3.13",
|
|
28
|
+
"Programming Language :: Python :: 3.14",
|
|
20
29
|
"Topic :: Scientific/Engineering",
|
|
30
|
+
"Topic :: Scientific/Engineering :: Atmospheric Science",
|
|
31
|
+
"Topic :: Scientific/Engineering :: Hydrology",
|
|
21
32
|
]
|
|
22
33
|
dependencies = [
|
|
23
34
|
"httpx>=0.28.1",
|
|
@@ -48,6 +59,9 @@ grib = [
|
|
|
48
59
|
Homepage = "https://usdata.dev/"
|
|
49
60
|
Documentation = "https://docs.usdata.dev/"
|
|
50
61
|
Repository = "https://github.com/jakeryderv/usdata"
|
|
62
|
+
Examples = "https://usdata.dev/examples/"
|
|
63
|
+
Changelog = "https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md"
|
|
64
|
+
Issues = "https://github.com/jakeryderv/usdata/issues"
|
|
51
65
|
|
|
52
66
|
[project.scripts]
|
|
53
67
|
usdata = "usdata.cli:app"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "usdata"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.20.0"
|
|
4
4
|
description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -8,12 +8,30 @@ authors = [
|
|
|
8
8
|
{ name = "Jake Van Slyke", email = "jakervanslyke@gmail.com" }
|
|
9
9
|
]
|
|
10
10
|
requires-python = ">=3.11"
|
|
11
|
-
keywords = [
|
|
11
|
+
keywords = [
|
|
12
|
+
"noaa",
|
|
13
|
+
"usgs",
|
|
14
|
+
"open-data",
|
|
15
|
+
"scientific-data",
|
|
16
|
+
"provenance",
|
|
17
|
+
"reproducible-research",
|
|
18
|
+
"weather",
|
|
19
|
+
"climate",
|
|
20
|
+
"meteorology",
|
|
21
|
+
]
|
|
12
22
|
classifiers = [
|
|
13
|
-
"Development Status ::
|
|
23
|
+
"Development Status :: 3 - Alpha",
|
|
24
|
+
"Environment :: Console",
|
|
14
25
|
"Intended Audience :: Science/Research",
|
|
26
|
+
"Operating System :: OS Independent",
|
|
15
27
|
"Programming Language :: Python :: 3",
|
|
28
|
+
"Programming Language :: Python :: 3.11",
|
|
29
|
+
"Programming Language :: Python :: 3.12",
|
|
30
|
+
"Programming Language :: Python :: 3.13",
|
|
31
|
+
"Programming Language :: Python :: 3.14",
|
|
16
32
|
"Topic :: Scientific/Engineering",
|
|
33
|
+
"Topic :: Scientific/Engineering :: Atmospheric Science",
|
|
34
|
+
"Topic :: Scientific/Engineering :: Hydrology",
|
|
17
35
|
]
|
|
18
36
|
dependencies = [
|
|
19
37
|
"httpx>=0.28.1",
|
|
@@ -38,6 +56,9 @@ grib = [
|
|
|
38
56
|
Homepage = "https://usdata.dev/"
|
|
39
57
|
Documentation = "https://docs.usdata.dev/"
|
|
40
58
|
Repository = "https://github.com/jakeryderv/usdata"
|
|
59
|
+
Examples = "https://usdata.dev/examples/"
|
|
60
|
+
Changelog = "https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md"
|
|
61
|
+
Issues = "https://github.com/jakeryderv/usdata/issues"
|
|
41
62
|
|
|
42
63
|
[project.scripts]
|
|
43
64
|
usdata = "usdata.cli:app"
|
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import os
|
|
6
6
|
from collections.abc import Mapping
|
|
7
|
+
from datetime import UTC, datetime
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from typing import TYPE_CHECKING, Any
|
|
9
10
|
|
|
@@ -129,6 +130,8 @@ def _fetch_asset(
|
|
|
129
130
|
and prov.provider == dataset.provider
|
|
130
131
|
and prov.source_url == asset.href
|
|
131
132
|
and prov.size == path.stat().st_size
|
|
133
|
+
# A listing that now reports another size describes a rebuilt file.
|
|
134
|
+
and (asset.size is None or prov.size == asset.size)
|
|
132
135
|
and (asset.checksum is None or prov.checksum == asset.checksum)
|
|
133
136
|
# Trust an untouched cached file; hash whenever anything is unclear.
|
|
134
137
|
and (_sidecar_not_older(path) or sha256_file(path) == prov.checksum)
|
|
@@ -137,7 +140,10 @@ def _fetch_asset(
|
|
|
137
140
|
return FetchedAsset(asset=asset, path=path, provenance=prov, from_cache=True)
|
|
138
141
|
with staged_path(path) as tmp:
|
|
139
142
|
partial = adapter.prepare_fetch(asset, pinned)
|
|
140
|
-
|
|
143
|
+
if partial is None:
|
|
144
|
+
adapter.fetch(asset, tmp)
|
|
145
|
+
else:
|
|
146
|
+
adapter.fetch_partial(asset, tmp, partial)
|
|
141
147
|
prov = provenance.record(dataset, asset, tmp, partial)
|
|
142
148
|
if asset.checksum and prov.checksum != asset.checksum:
|
|
143
149
|
raise ChecksumMismatch(f"{asset.id}: expected {asset.checksum}, got {prov.checksum}")
|
|
@@ -169,12 +175,31 @@ def _fetch_with(
|
|
|
169
175
|
root: Path | None = None,
|
|
170
176
|
force: bool = False,
|
|
171
177
|
) -> list[FetchedAsset]:
|
|
172
|
-
"""Run the loop on an adapter the caller opened, so one adapter can serve many queries.
|
|
173
|
-
|
|
178
|
+
"""Run the loop on an adapter the caller opened, so one adapter can serve many queries.
|
|
179
|
+
|
|
180
|
+
The listing is put in the order every result promises, by ``asset.time.start``
|
|
181
|
+
then id, so no adapter has to sort and no caller has to sort defensively.
|
|
182
|
+
"""
|
|
183
|
+
assets = ordered(adapter.list_assets(query))
|
|
174
184
|
_progress.batch([asset.size for asset in assets])
|
|
175
185
|
return [_fetch_asset(dataset, a, adapter, root=root, force=force) for a in assets]
|
|
176
186
|
|
|
177
187
|
|
|
188
|
+
def ordered(assets: list[Asset]) -> list[Asset]:
|
|
189
|
+
"""``assets`` by start time, then id; assets without a start time come last, by id."""
|
|
190
|
+
return sorted(
|
|
191
|
+
assets,
|
|
192
|
+
key=lambda asset: (
|
|
193
|
+
asset.time is None or asset.time.start is None,
|
|
194
|
+
asset.time.start if asset.time is not None and asset.time.start is not None else _NEVER,
|
|
195
|
+
asset.id,
|
|
196
|
+
),
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
_NEVER = datetime.max.replace(tzinfo=UTC)
|
|
201
|
+
|
|
202
|
+
|
|
178
203
|
def fetch(
|
|
179
204
|
dataset: Dataset,
|
|
180
205
|
query: Query,
|
|
@@ -81,6 +81,9 @@ def _modules() -> tuple[Any, Any, Any]:
|
|
|
81
81
|
except (RuntimeError, OSError) as error:
|
|
82
82
|
# eccodes imports, then findlibs fails to locate the shared library.
|
|
83
83
|
raise MissingReaderDependency(f"{LIBRARY_HINT} ({error})") from error
|
|
84
|
+
# A GRIB2 message may hold several fields (RAP packs wind components that way);
|
|
85
|
+
# without this ecCodes yields only the first and silently drops the rest.
|
|
86
|
+
eccodes.codes_grib_multi_support_on()
|
|
84
87
|
return eccodes, xarray, numpy
|
|
85
88
|
|
|
86
89
|
|
|
@@ -92,9 +95,15 @@ def _handle(eccodes: Any, handle: int) -> Iterator[int]:
|
|
|
92
95
|
eccodes.codes_release(handle)
|
|
93
96
|
|
|
94
97
|
|
|
95
|
-
def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[int]:
|
|
96
|
-
"""
|
|
98
|
+
def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[tuple[int, int]]:
|
|
99
|
+
"""Message index and handle for each field in an in-memory GRIB byte string.
|
|
100
|
+
|
|
101
|
+
Messages are walked by their section 0 lengths; an in-memory message yields
|
|
102
|
+
its first field only, which is every field of the single-field products
|
|
103
|
+
that arrive compressed.
|
|
104
|
+
"""
|
|
97
105
|
offset = 0
|
|
106
|
+
index = 0
|
|
98
107
|
while offset < len(data):
|
|
99
108
|
if data[offset : offset + 4] != b"GRIB":
|
|
100
109
|
raise ValueError(f"not a GRIB message at byte {offset}")
|
|
@@ -104,12 +113,18 @@ def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[int]:
|
|
|
104
113
|
if length < 16 or offset + length > len(data):
|
|
105
114
|
raise ValueError(f"truncated GRIB message at byte {offset}")
|
|
106
115
|
with _handle(eccodes, eccodes.codes_new_from_message(data[offset : offset + length])) as h:
|
|
107
|
-
yield h
|
|
116
|
+
yield index, h
|
|
108
117
|
offset += length
|
|
118
|
+
index += 1
|
|
109
119
|
|
|
110
120
|
|
|
111
|
-
def _messages(eccodes: Any, path: Path) -> Iterator[int]:
|
|
112
|
-
"""
|
|
121
|
+
def _messages(eccodes: Any, path: Path) -> Iterator[tuple[int, int]]:
|
|
122
|
+
"""Message index and handle for each field, decompressing gzip in memory.
|
|
123
|
+
|
|
124
|
+
The index counts GRIB2 messages from zero, so the fields of one message
|
|
125
|
+
that holds several share it; that is the number a partial fetch's record
|
|
126
|
+
pairs with, and the ``file_index`` the reader and ``inspect`` report.
|
|
127
|
+
"""
|
|
113
128
|
with path.open("rb") as probe:
|
|
114
129
|
compressed = probe.read(2) == b"\x1f\x8b"
|
|
115
130
|
if compressed:
|
|
@@ -117,12 +132,18 @@ def _messages(eccodes: Any, path: Path) -> Iterator[int]:
|
|
|
117
132
|
yield from _messages_from_bytes(eccodes, data)
|
|
118
133
|
return
|
|
119
134
|
# ecCodes reads through the descriptor, so it gets a file object it alone positions.
|
|
135
|
+
index = -1
|
|
136
|
+
last_offset: int | None = None
|
|
120
137
|
with path.open("rb") as raw:
|
|
121
138
|
while (handle := eccodes.codes_grib_new_from_file(raw)) is not None:
|
|
122
139
|
with _handle(eccodes, handle) as h:
|
|
123
140
|
if eccodes.codes_get(h, "edition") != 2:
|
|
124
141
|
raise ValueError("only GRIB edition 2 is supported")
|
|
125
|
-
|
|
142
|
+
offset = _get(eccodes, h, "offset", int)
|
|
143
|
+
if offset is None or offset != last_offset:
|
|
144
|
+
index += 1
|
|
145
|
+
last_offset = offset
|
|
146
|
+
yield index, h
|
|
126
147
|
|
|
127
148
|
|
|
128
149
|
def _get(eccodes: Any, h: int, key: str, ktype: type | None = None) -> Any:
|
|
@@ -203,7 +224,7 @@ def inventory(path: Path) -> list[GribMessage]:
|
|
|
203
224
|
"""Every message in a local GRIB2 file, in file order, without decoding its values."""
|
|
204
225
|
eccodes = _modules()[0]
|
|
205
226
|
messages = []
|
|
206
|
-
for index, h in
|
|
227
|
+
for index, h in _messages(eccodes, path):
|
|
207
228
|
keys = {key: _get(eccodes, h, key, str) for key in INVENTORY_KEYS}
|
|
208
229
|
messages.append(
|
|
209
230
|
GribMessage(
|
|
@@ -373,7 +394,7 @@ def open_grib2(
|
|
|
373
394
|
needs_select = select is None and not fetched.provenance.is_partial
|
|
374
395
|
object_messages = fetched.provenance.object_messages
|
|
375
396
|
selectors = fetched.provenance.selectors if object_messages else []
|
|
376
|
-
for h in _messages(eccodes, fetched.path):
|
|
397
|
+
for index, h in _messages(eccodes, fetched.path):
|
|
377
398
|
count += 1
|
|
378
399
|
if count > 1 and needs_select:
|
|
379
400
|
continue
|
|
@@ -415,15 +436,15 @@ def open_grib2(
|
|
|
415
436
|
if stamp:
|
|
416
437
|
attrs[label] = stamp
|
|
417
438
|
message: dict[str, Any] = {
|
|
418
|
-
"file_index":
|
|
419
|
-
"object_index": object_messages[
|
|
439
|
+
"file_index": index,
|
|
440
|
+
"object_index": object_messages[index] if object_messages else None,
|
|
420
441
|
"shortName": _get(eccodes, h, "shortName", str),
|
|
421
442
|
"typeOfLevel": attrs.get("typeOfLevel"),
|
|
422
443
|
"level": attrs.get("level"),
|
|
423
444
|
"step": attrs.get("step"),
|
|
424
445
|
}
|
|
425
446
|
if selectors:
|
|
426
|
-
message["selector"] = selectors[
|
|
447
|
+
message["selector"] = selectors[index]
|
|
427
448
|
short = message["shortName"]
|
|
428
449
|
if short in (None, "", "unknown", "~"):
|
|
429
450
|
short = _product_name(fetched.asset.id) or (
|
|
@@ -184,7 +184,10 @@ def search(
|
|
|
184
184
|
typer.Option("--location", "--state", help="State, 'County, ST', or quoted FIPS code."),
|
|
185
185
|
] = None,
|
|
186
186
|
start: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
|
|
187
|
-
end: Annotated[
|
|
187
|
+
end: Annotated[
|
|
188
|
+
str | None,
|
|
189
|
+
typer.Option(help="ISO date or datetime; a date alone runs to the end of that UTC day."),
|
|
190
|
+
] = None,
|
|
188
191
|
planned: Annotated[
|
|
189
192
|
bool, typer.Option("--planned", help="Include planned datasets that have no adapter yet.")
|
|
190
193
|
] = False,
|
|
@@ -321,7 +324,10 @@ def fetch(
|
|
|
321
324
|
lon: Annotated[float | None, typer.Option()] = None,
|
|
322
325
|
radius_km: Annotated[float, typer.Option(help="Radius around --lat/--lon.")] = 50.0,
|
|
323
326
|
start: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
|
|
324
|
-
end: Annotated[
|
|
327
|
+
end: Annotated[
|
|
328
|
+
str | None,
|
|
329
|
+
typer.Option(help="ISO date or datetime; a date alone runs to the end of that UTC day."),
|
|
330
|
+
] = None,
|
|
325
331
|
variables: Annotated[
|
|
326
332
|
str | None, typer.Option("--vars", help="Comma-separated variable names.")
|
|
327
333
|
] = None,
|
|
@@ -507,16 +513,28 @@ def pull(
|
|
|
507
513
|
except (httpx.HTTPError, ChecksumMismatch) as e:
|
|
508
514
|
typer.secho(f"fetch failed: {e}", err=True, fg="red")
|
|
509
515
|
raise typer.Exit(code=4) from None
|
|
510
|
-
updated = set(result.updated)
|
|
516
|
+
updated, mirrored = set(result.updated), set(result.mirrored)
|
|
511
517
|
if not quiet:
|
|
512
518
|
for f in result.fetched:
|
|
513
|
-
|
|
519
|
+
if f.asset.id in updated:
|
|
520
|
+
tag = "updated"
|
|
521
|
+
elif f.asset.id in mirrored:
|
|
522
|
+
tag = "mirrored"
|
|
523
|
+
else:
|
|
524
|
+
tag = "cached" if f.from_cache else "fetched"
|
|
514
525
|
typer.echo(f"{f.path}\t{tag}\t{f.provenance.size} bytes")
|
|
515
526
|
if result.updated:
|
|
516
527
|
mode = f"updated {len(result.updated)} pin(s) in"
|
|
517
528
|
else:
|
|
518
529
|
mode = "restored from" if result.from_lockfile else "wrote"
|
|
519
530
|
typer.echo(f"{len(result.fetched)} asset(s); {mode} {result.lockfile_path}", err=True)
|
|
531
|
+
if result.mirrored:
|
|
532
|
+
typer.secho(
|
|
533
|
+
f"{len(result.mirrored)} asset(s) changed upstream and were restored from the "
|
|
534
|
+
"mirror; the lockfile still pins the source. Pass --update to accept new bytes.",
|
|
535
|
+
err=True,
|
|
536
|
+
fg="yellow",
|
|
537
|
+
)
|
|
520
538
|
|
|
521
539
|
|
|
522
540
|
@app.command()
|
|
@@ -94,7 +94,8 @@ def _echo_detail(summary: Summary, width: int) -> None:
|
|
|
94
94
|
else:
|
|
95
95
|
typer.echo(" columns:")
|
|
96
96
|
for name in frame.columns:
|
|
97
|
-
|
|
97
|
+
unit = frame.units.get(name)
|
|
98
|
+
typer.echo(f" {name} ({unit})" if unit else f" {name}")
|
|
98
99
|
if (scene := summary.netcdf) is not None:
|
|
99
100
|
_echo_table(
|
|
100
101
|
"variables",
|