usdata 0.18.0__tar.gz → 0.20.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. usdata-0.20.0/PKG-INFO +124 -0
  2. usdata-0.20.0/README.md +79 -0
  3. {usdata-0.18.0 → usdata-0.20.0}/pyproject.toml +17 -3
  4. {usdata-0.18.0 → usdata-0.20.0}/pyproject.toml.orig +24 -3
  5. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_fetch.py +28 -3
  6. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_grib.py +32 -11
  7. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/app.py +22 -4
  8. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/inspect.py +2 -1
  9. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/data/registry.yaml +182 -41
  10. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/doctor.py +3 -2
  11. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/inspect.py +69 -20
  12. usdata-0.20.0/src/usdata/mirror.py +60 -0
  13. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/models.py +4 -0
  14. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/http.py +4 -2
  15. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/listing.py +36 -10
  16. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/base.py +29 -6
  17. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/coastwatch.py +1 -1
  18. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/coops.py +5 -1
  19. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/grib_index.py +75 -26
  20. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/hrrr.py +31 -10
  21. usdata-0.20.0/src/usdata/providers/noaa/ibtracs.py +200 -0
  22. usdata-0.20.0/src/usdata/providers/noaa/nbm.py +106 -0
  23. usdata-0.20.0/src/usdata/providers/noaa/rap.py +110 -0
  24. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/params.py +22 -0
  25. usdata-0.20.0/src/usdata/providers/usgs/earthquakes.py +147 -0
  26. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/pull.py +240 -80
  27. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/query.py +19 -5
  28. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/readers.py +100 -15
  29. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/testing.py +6 -1
  30. usdata-0.18.0/PKG-INFO +0 -123
  31. usdata-0.18.0/README.md +0 -89
  32. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/__init__.py +0 -0
  33. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/__main__.py +0 -0
  34. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_files.py +0 -0
  35. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_hurdat2.py +0 -0
  36. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_netcdf.py +0 -0
  37. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_progress.py +0 -0
  38. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/_radar.py +0 -0
  39. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cache.py +0 -0
  40. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cache_ops.py +0 -0
  41. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cite.py +0 -0
  42. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/__init__.py +0 -0
  43. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/cache.py +0 -0
  44. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/cite.py +0 -0
  45. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/doctor.py +0 -0
  46. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/cli/progress.py +0 -0
  47. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/data/nexrad_sites.csv +0 -0
  48. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/data/places.csv +0 -0
  49. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/data/places.sources.json +0 -0
  50. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/manifest.py +0 -0
  51. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/__init__.py +0 -0
  52. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/erddap.py +0 -0
  53. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/protocols/s3.py +0 -0
  54. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/provenance.py +0 -0
  55. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/__init__.py +0 -0
  56. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/http.py +0 -0
  57. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/__init__.py +0 -0
  58. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/gfs.py +0 -0
  59. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/ghcnd.py +0 -0
  60. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/glm.py +0 -0
  61. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/goes.py +0 -0
  62. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/gsom.py +0 -0
  63. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/gsoy.py +0 -0
  64. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/hurdat2.py +0 -0
  65. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/lcd.py +0 -0
  66. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/mrms.py +0 -0
  67. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/nexrad.py +0 -0
  68. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/nexrad_level3.py +0 -0
  69. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/normals.py +0 -0
  70. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/sites.py +0 -0
  71. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/spc.py +0 -0
  72. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/noaa/storm_events.py +0 -0
  73. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/usgs/__init__.py +0 -0
  74. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/providers/usgs/daily.py +0 -0
  75. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/py.typed +0 -0
  76. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/registry.py +0 -0
  77. {usdata-0.18.0 → usdata-0.20.0}/src/usdata/selection.py +0 -0
usdata-0.20.0/PKG-INFO ADDED
@@ -0,0 +1,124 @@
1
+ Metadata-Version: 2.4
2
+ Name: usdata
3
+ Version: 0.20.0
4
+ Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
5
+ Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
6
+ Author: Jake Van Slyke
7
+ Author-email: Jake Van Slyke <jakervanslyke@gmail.com>
8
+ License-Expression: Apache-2.0
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Environment :: Console
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Programming Language :: Python :: 3.14
18
+ Classifier: Topic :: Scientific/Engineering
19
+ Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
20
+ Classifier: Topic :: Scientific/Engineering :: Hydrology
21
+ Requires-Dist: httpx>=0.28.1
22
+ Requires-Dist: pydantic>=2.7
23
+ Requires-Dist: pyyaml>=6.0
24
+ Requires-Dist: typer>=0.18
25
+ Requires-Dist: eccodes>=2.48 ; extra == 'grib'
26
+ Requires-Dist: eccodeslib>=2.48 ; sys_platform == 'linux' and extra == 'grib'
27
+ Requires-Dist: xarray>=2025.1 ; extra == 'grib'
28
+ Requires-Dist: numpy>=2 ; extra == 'grib'
29
+ Requires-Dist: xarray>=2025.1 ; extra == 'netcdf'
30
+ Requires-Dist: h5netcdf[h5py]>=1.8.1 ; extra == 'netcdf'
31
+ Requires-Dist: pandas>=2.2 ; extra == 'pandas'
32
+ Requires-Dist: xradar>=0.12.0 ; extra == 'radar'
33
+ Requires-Python: >=3.11
34
+ Project-URL: Homepage, https://usdata.dev/
35
+ Project-URL: Documentation, https://docs.usdata.dev/
36
+ Project-URL: Repository, https://github.com/jakeryderv/usdata
37
+ Project-URL: Examples, https://usdata.dev/examples/
38
+ Project-URL: Changelog, https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md
39
+ Project-URL: Issues, https://github.com/jakeryderv/usdata/issues
40
+ Provides-Extra: grib
41
+ Provides-Extra: netcdf
42
+ Provides-Extra: pandas
43
+ Provides-Extra: radar
44
+ Description-Content-Type: text/markdown
45
+
46
+ # usdata
47
+
48
+ [![PyPI](https://img.shields.io/pypi/v/usdata)](https://pypi.org/project/usdata/)
49
+ [![Python](https://img.shields.io/pypi/pyversions/usdata)](https://pypi.org/project/usdata/)
50
+ [![CI](https://github.com/jakeryderv/usdata/actions/workflows/ci.yml/badge.svg)](https://github.com/jakeryderv/usdata/actions/workflows/ci.yml)
51
+ [![Live checks](https://github.com/jakeryderv/usdata/actions/workflows/integration.yml/badge.svg)](https://github.com/jakeryderv/usdata/actions/workflows/integration.yml)
52
+ [![License](https://img.shields.io/github/license/jakeryderv/usdata)](LICENSE)
53
+ [![Docs](https://img.shields.io/badge/docs-docs.usdata.dev-2f5bea)](https://docs.usdata.dev/)
54
+
55
+ [![usdata.dev: public science, reproducible inputs](https://usdata.dev/og.png)](https://usdata.dev/)
56
+
57
+ Reproducible acquisition of U.S. public scientific data. One Python SDK and
58
+ CLI discovers curated NOAA and USGS datasets, fetches their files, and keeps a
59
+ record of every input: a manifest names them, a lockfile pins them by checksum,
60
+ and the record of what was fetched is what a methods section cites. Analysis
61
+ stays in pandas and xarray; usdata only acquires. If you need every product
62
+ one agency publishes, that agency's own library is the better tool; usdata is
63
+ for pinning inputs across sources and proving later that they have not changed.
64
+
65
+ ```sh
66
+ pip install "usdata[pandas]"
67
+ usdata search precipitation --location Oklahoma
68
+ usdata fetch noaa:ghcn-daily -p stations=USW00013967 --start 2024-05-06 --end 2024-05-07
69
+ usdata pull dataset.yaml && usdata cite dataset.yaml
70
+ ```
71
+
72
+ The same four steps from Python:
73
+
74
+ ```python
75
+ from usdata import build_query, fetch, get, pull
76
+
77
+ query = build_query(start="2024-05-06", end="2024-05-07", stations="USW00013967")
78
+ items = fetch(get("noaa:ghcn-daily"), query)
79
+ frame = items[0].open() # pandas DataFrame, units and provenance in frame.attrs
80
+ result = pull("dataset.yaml") # every manifest source, pinned in dataset.lock.json
81
+ print(result.fetched[0].provenance.checksum) # what usdata cite reports
82
+ ```
83
+
84
+ **Alpha.** Releases are tested and lockfiles are stable, but the Python API
85
+ and CLI can change between minor versions; the [changelog](CHANGELOG.md) marks
86
+ breaking changes.
87
+
88
+ ## Where to look
89
+
90
+ | | |
91
+ | --- | --- |
92
+ | [usdata.dev](https://usdata.dev/) | What it is for: the [dataset browser](https://usdata.dev/datasets/) and [worked examples](https://usdata.dev/examples/) with saved results |
93
+ | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
94
+ | [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
95
+
96
+ Twenty-three datasets are available today and twenty-one more are planned, grouped
97
+ by agency and product family in the [catalog](docs/providers/README.md).
98
+
99
+ ## How this compares
100
+
101
+ usdata resolves a place, a time window, and a few parameters to the exact
102
+ files that satisfy them, then pins those files so the same inputs can be
103
+ restored and verified later. Neighbouring tools each do part of that.
104
+
105
+ | Tool | What it does | What usdata adds |
106
+ | --- | --- | --- |
107
+ | [pooch](https://www.fatiando.org/pooch/) | Downloads and verifies files from a registry of names, hashes, and a base URL you write | Discovery: the query resolves to the URLs, and the lockfile is written, not authored |
108
+ | [intake](https://intake.readthedocs.io/) | Describes datasets in catalogs and opens them through drivers | Checksums, lockfiles, and provenance; usdata stops at handing a file to pandas or xarray |
109
+ | [DVC](https://dvc.org/) | Versions data files and can track one URL by its ETag or hash | Knowing which URLs a scientific question needs, across agencies |
110
+ | [Herbie](https://herbie.readthedocs.io/) | Deep access to model output, including subsetting GRIB2 through index files | One interface across observations, radar, satellite, and models, with a lockfile; usdata's index-file fetch reproduces Herbie's for that reason |
111
+ | [dataretrieval](https://github.com/DOI-USGS/dataretrieval-python) | Full coverage of USGS water data | Cross-agency pinning; use dataretrieval for USGS breadth |
112
+
113
+ ## Contributing
114
+
115
+ Setup, commands, checks, and the release procedure are in
116
+ [CONTRIBUTING.md](CONTRIBUTING.md). The project record lives beside it:
117
+ [architecture](docs/architecture.md), [testing](docs/testing.md),
118
+ [decisions](docs/adr/README.md), [roadmap](docs/roadmap.md), and
119
+ [versioning](docs/versioning.md). The main way the project grows is a new
120
+ dataset, following [adding a dataset](docs/guides/adding-a-dataset.md).
121
+
122
+ ## License
123
+
124
+ [Apache-2.0](LICENSE).
@@ -0,0 +1,79 @@
1
+ # usdata
2
+
3
+ [![PyPI](https://img.shields.io/pypi/v/usdata)](https://pypi.org/project/usdata/)
4
+ [![Python](https://img.shields.io/pypi/pyversions/usdata)](https://pypi.org/project/usdata/)
5
+ [![CI](https://github.com/jakeryderv/usdata/actions/workflows/ci.yml/badge.svg)](https://github.com/jakeryderv/usdata/actions/workflows/ci.yml)
6
+ [![Live checks](https://github.com/jakeryderv/usdata/actions/workflows/integration.yml/badge.svg)](https://github.com/jakeryderv/usdata/actions/workflows/integration.yml)
7
+ [![License](https://img.shields.io/github/license/jakeryderv/usdata)](LICENSE)
8
+ [![Docs](https://img.shields.io/badge/docs-docs.usdata.dev-2f5bea)](https://docs.usdata.dev/)
9
+
10
+ [![usdata.dev: public science, reproducible inputs](https://usdata.dev/og.png)](https://usdata.dev/)
11
+
12
+ Reproducible acquisition of U.S. public scientific data. One Python SDK and
13
+ CLI discovers curated NOAA and USGS datasets, fetches their files, and keeps a
14
+ record of every input: a manifest names them, a lockfile pins them by checksum,
15
+ and the record of what was fetched is what a methods section cites. Analysis
16
+ stays in pandas and xarray; usdata only acquires. If you need every product
17
+ one agency publishes, that agency's own library is the better tool; usdata is
18
+ for pinning inputs across sources and proving later that they have not changed.
19
+
20
+ ```sh
21
+ pip install "usdata[pandas]"
22
+ usdata search precipitation --location Oklahoma
23
+ usdata fetch noaa:ghcn-daily -p stations=USW00013967 --start 2024-05-06 --end 2024-05-07
24
+ usdata pull dataset.yaml && usdata cite dataset.yaml
25
+ ```
26
+
27
+ The same four steps from Python:
28
+
29
+ ```python
30
+ from usdata import build_query, fetch, get, pull
31
+
32
+ query = build_query(start="2024-05-06", end="2024-05-07", stations="USW00013967")
33
+ items = fetch(get("noaa:ghcn-daily"), query)
34
+ frame = items[0].open() # pandas DataFrame, units and provenance in frame.attrs
35
+ result = pull("dataset.yaml") # every manifest source, pinned in dataset.lock.json
36
+ print(result.fetched[0].provenance.checksum) # what usdata cite reports
37
+ ```
38
+
39
+ **Alpha.** Releases are tested and lockfiles are stable, but the Python API
40
+ and CLI can change between minor versions; the [changelog](CHANGELOG.md) marks
41
+ breaking changes.
42
+
43
+ ## Where to look
44
+
45
+ | | |
46
+ | --- | --- |
47
+ | [usdata.dev](https://usdata.dev/) | What it is for: the [dataset browser](https://usdata.dev/datasets/) and [worked examples](https://usdata.dev/examples/) with saved results |
48
+ | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
49
+ | [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
50
+
51
+ Twenty-three datasets are available today and twenty-one more are planned, grouped
52
+ by agency and product family in the [catalog](docs/providers/README.md).
53
+
54
+ ## How this compares
55
+
56
+ usdata resolves a place, a time window, and a few parameters to the exact
57
+ files that satisfy them, then pins those files so the same inputs can be
58
+ restored and verified later. Neighbouring tools each do part of that.
59
+
60
+ | Tool | What it does | What usdata adds |
61
+ | --- | --- | --- |
62
+ | [pooch](https://www.fatiando.org/pooch/) | Downloads and verifies files from a registry of names, hashes, and a base URL you write | Discovery: the query resolves to the URLs, and the lockfile is written, not authored |
63
+ | [intake](https://intake.readthedocs.io/) | Describes datasets in catalogs and opens them through drivers | Checksums, lockfiles, and provenance; usdata stops at handing a file to pandas or xarray |
64
+ | [DVC](https://dvc.org/) | Versions data files and can track one URL by its ETag or hash | Knowing which URLs a scientific question needs, across agencies |
65
+ | [Herbie](https://herbie.readthedocs.io/) | Deep access to model output, including subsetting GRIB2 through index files | One interface across observations, radar, satellite, and models, with a lockfile; usdata's index-file fetch reproduces Herbie's for that reason |
66
+ | [dataretrieval](https://github.com/DOI-USGS/dataretrieval-python) | Full coverage of USGS water data | Cross-agency pinning; use dataretrieval for USGS breadth |
67
+
68
+ ## Contributing
69
+
70
+ Setup, commands, checks, and the release procedure are in
71
+ [CONTRIBUTING.md](CONTRIBUTING.md). The project record lives beside it:
72
+ [architecture](docs/architecture.md), [testing](docs/testing.md),
73
+ [decisions](docs/adr/README.md), [roadmap](docs/roadmap.md), and
74
+ [versioning](docs/versioning.md). The main way the project grows is a new
75
+ dataset, following [adding a dataset](docs/guides/adding-a-dataset.md).
76
+
77
+ ## License
78
+
79
+ [Apache-2.0](LICENSE).
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "usdata"
3
- version = "0.18.0"
3
+ version = "0.20.0"
4
4
  description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -8,16 +8,27 @@ requires-python = ">=3.11"
8
8
  keywords = [
9
9
  "noaa",
10
10
  "usgs",
11
- "nasa",
12
11
  "open-data",
13
12
  "scientific-data",
14
13
  "provenance",
14
+ "reproducible-research",
15
+ "weather",
16
+ "climate",
17
+ "meteorology",
15
18
  ]
16
19
  classifiers = [
17
- "Development Status :: 2 - Pre-Alpha",
20
+ "Development Status :: 3 - Alpha",
21
+ "Environment :: Console",
18
22
  "Intended Audience :: Science/Research",
23
+ "Operating System :: OS Independent",
19
24
  "Programming Language :: Python :: 3",
25
+ "Programming Language :: Python :: 3.11",
26
+ "Programming Language :: Python :: 3.12",
27
+ "Programming Language :: Python :: 3.13",
28
+ "Programming Language :: Python :: 3.14",
20
29
  "Topic :: Scientific/Engineering",
30
+ "Topic :: Scientific/Engineering :: Atmospheric Science",
31
+ "Topic :: Scientific/Engineering :: Hydrology",
21
32
  ]
22
33
  dependencies = [
23
34
  "httpx>=0.28.1",
@@ -48,6 +59,9 @@ grib = [
48
59
  Homepage = "https://usdata.dev/"
49
60
  Documentation = "https://docs.usdata.dev/"
50
61
  Repository = "https://github.com/jakeryderv/usdata"
62
+ Examples = "https://usdata.dev/examples/"
63
+ Changelog = "https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md"
64
+ Issues = "https://github.com/jakeryderv/usdata/issues"
51
65
 
52
66
  [project.scripts]
53
67
  usdata = "usdata.cli:app"
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "usdata"
3
- version = "0.18.0"
3
+ version = "0.20.0"
4
4
  description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -8,12 +8,30 @@ authors = [
8
8
  { name = "Jake Van Slyke", email = "jakervanslyke@gmail.com" }
9
9
  ]
10
10
  requires-python = ">=3.11"
11
- keywords = ["noaa", "usgs", "nasa", "open-data", "scientific-data", "provenance"]
11
+ keywords = [
12
+ "noaa",
13
+ "usgs",
14
+ "open-data",
15
+ "scientific-data",
16
+ "provenance",
17
+ "reproducible-research",
18
+ "weather",
19
+ "climate",
20
+ "meteorology",
21
+ ]
12
22
  classifiers = [
13
- "Development Status :: 2 - Pre-Alpha",
23
+ "Development Status :: 3 - Alpha",
24
+ "Environment :: Console",
14
25
  "Intended Audience :: Science/Research",
26
+ "Operating System :: OS Independent",
15
27
  "Programming Language :: Python :: 3",
28
+ "Programming Language :: Python :: 3.11",
29
+ "Programming Language :: Python :: 3.12",
30
+ "Programming Language :: Python :: 3.13",
31
+ "Programming Language :: Python :: 3.14",
16
32
  "Topic :: Scientific/Engineering",
33
+ "Topic :: Scientific/Engineering :: Atmospheric Science",
34
+ "Topic :: Scientific/Engineering :: Hydrology",
17
35
  ]
18
36
  dependencies = [
19
37
  "httpx>=0.28.1",
@@ -38,6 +56,9 @@ grib = [
38
56
  Homepage = "https://usdata.dev/"
39
57
  Documentation = "https://docs.usdata.dev/"
40
58
  Repository = "https://github.com/jakeryderv/usdata"
59
+ Examples = "https://usdata.dev/examples/"
60
+ Changelog = "https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md"
61
+ Issues = "https://github.com/jakeryderv/usdata/issues"
41
62
 
42
63
  [project.scripts]
43
64
  usdata = "usdata.cli:app"
@@ -4,6 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  import os
6
6
  from collections.abc import Mapping
7
+ from datetime import UTC, datetime
7
8
  from pathlib import Path
8
9
  from typing import TYPE_CHECKING, Any
9
10
 
@@ -129,6 +130,8 @@ def _fetch_asset(
129
130
  and prov.provider == dataset.provider
130
131
  and prov.source_url == asset.href
131
132
  and prov.size == path.stat().st_size
133
+ # A listing that now reports another size describes a rebuilt file.
134
+ and (asset.size is None or prov.size == asset.size)
132
135
  and (asset.checksum is None or prov.checksum == asset.checksum)
133
136
  # Trust an untouched cached file; hash whenever anything is unclear.
134
137
  and (_sidecar_not_older(path) or sha256_file(path) == prov.checksum)
@@ -137,7 +140,10 @@ def _fetch_asset(
137
140
  return FetchedAsset(asset=asset, path=path, provenance=prov, from_cache=True)
138
141
  with staged_path(path) as tmp:
139
142
  partial = adapter.prepare_fetch(asset, pinned)
140
- adapter.fetch(asset, tmp)
143
+ if partial is None:
144
+ adapter.fetch(asset, tmp)
145
+ else:
146
+ adapter.fetch_partial(asset, tmp, partial)
141
147
  prov = provenance.record(dataset, asset, tmp, partial)
142
148
  if asset.checksum and prov.checksum != asset.checksum:
143
149
  raise ChecksumMismatch(f"{asset.id}: expected {asset.checksum}, got {prov.checksum}")
@@ -169,12 +175,31 @@ def _fetch_with(
169
175
  root: Path | None = None,
170
176
  force: bool = False,
171
177
  ) -> list[FetchedAsset]:
172
- """Run the loop on an adapter the caller opened, so one adapter can serve many queries."""
173
- assets = adapter.list_assets(query)
178
+ """Run the loop on an adapter the caller opened, so one adapter can serve many queries.
179
+
180
+ The listing is put in the order every result promises, by ``asset.time.start``
181
+ then id, so no adapter has to sort and no caller has to sort defensively.
182
+ """
183
+ assets = ordered(adapter.list_assets(query))
174
184
  _progress.batch([asset.size for asset in assets])
175
185
  return [_fetch_asset(dataset, a, adapter, root=root, force=force) for a in assets]
176
186
 
177
187
 
188
+ def ordered(assets: list[Asset]) -> list[Asset]:
189
+ """``assets`` by start time, then id; assets without a start time come last, by id."""
190
+ return sorted(
191
+ assets,
192
+ key=lambda asset: (
193
+ asset.time is None or asset.time.start is None,
194
+ asset.time.start if asset.time is not None and asset.time.start is not None else _NEVER,
195
+ asset.id,
196
+ ),
197
+ )
198
+
199
+
200
+ _NEVER = datetime.max.replace(tzinfo=UTC)
201
+
202
+
178
203
  def fetch(
179
204
  dataset: Dataset,
180
205
  query: Query,
@@ -81,6 +81,9 @@ def _modules() -> tuple[Any, Any, Any]:
81
81
  except (RuntimeError, OSError) as error:
82
82
  # eccodes imports, then findlibs fails to locate the shared library.
83
83
  raise MissingReaderDependency(f"{LIBRARY_HINT} ({error})") from error
84
+ # A GRIB2 message may hold several fields (RAP packs wind components that way);
85
+ # without this ecCodes yields only the first and silently drops the rest.
86
+ eccodes.codes_grib_multi_support_on()
84
87
  return eccodes, xarray, numpy
85
88
 
86
89
 
@@ -92,9 +95,15 @@ def _handle(eccodes: Any, handle: int) -> Iterator[int]:
92
95
  eccodes.codes_release(handle)
93
96
 
94
97
 
95
- def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[int]:
96
- """Handles for each message in an in-memory GRIB byte string, using section 0 lengths."""
98
+ def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[tuple[int, int]]:
99
+ """Message index and handle for each field in an in-memory GRIB byte string.
100
+
101
+ Messages are walked by their section 0 lengths; an in-memory message yields
102
+ its first field only, which is every field of the single-field products
103
+ that arrive compressed.
104
+ """
97
105
  offset = 0
106
+ index = 0
98
107
  while offset < len(data):
99
108
  if data[offset : offset + 4] != b"GRIB":
100
109
  raise ValueError(f"not a GRIB message at byte {offset}")
@@ -104,12 +113,18 @@ def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[int]:
104
113
  if length < 16 or offset + length > len(data):
105
114
  raise ValueError(f"truncated GRIB message at byte {offset}")
106
115
  with _handle(eccodes, eccodes.codes_new_from_message(data[offset : offset + length])) as h:
107
- yield h
116
+ yield index, h
108
117
  offset += length
118
+ index += 1
109
119
 
110
120
 
111
- def _messages(eccodes: Any, path: Path) -> Iterator[int]:
112
- """Handles for each message, decompressing gzip in memory without changing cached bytes."""
121
+ def _messages(eccodes: Any, path: Path) -> Iterator[tuple[int, int]]:
122
+ """Message index and handle for each field, decompressing gzip in memory.
123
+
124
+ The index counts GRIB2 messages from zero, so the fields of one message
125
+ that holds several share it; that is the number a partial fetch's record
126
+ pairs with, and the ``file_index`` the reader and ``inspect`` report.
127
+ """
113
128
  with path.open("rb") as probe:
114
129
  compressed = probe.read(2) == b"\x1f\x8b"
115
130
  if compressed:
@@ -117,12 +132,18 @@ def _messages(eccodes: Any, path: Path) -> Iterator[int]:
117
132
  yield from _messages_from_bytes(eccodes, data)
118
133
  return
119
134
  # ecCodes reads through the descriptor, so it gets a file object it alone positions.
135
+ index = -1
136
+ last_offset: int | None = None
120
137
  with path.open("rb") as raw:
121
138
  while (handle := eccodes.codes_grib_new_from_file(raw)) is not None:
122
139
  with _handle(eccodes, handle) as h:
123
140
  if eccodes.codes_get(h, "edition") != 2:
124
141
  raise ValueError("only GRIB edition 2 is supported")
125
- yield h
142
+ offset = _get(eccodes, h, "offset", int)
143
+ if offset is None or offset != last_offset:
144
+ index += 1
145
+ last_offset = offset
146
+ yield index, h
126
147
 
127
148
 
128
149
  def _get(eccodes: Any, h: int, key: str, ktype: type | None = None) -> Any:
@@ -203,7 +224,7 @@ def inventory(path: Path) -> list[GribMessage]:
203
224
  """Every message in a local GRIB2 file, in file order, without decoding its values."""
204
225
  eccodes = _modules()[0]
205
226
  messages = []
206
- for index, h in enumerate(_messages(eccodes, path)):
227
+ for index, h in _messages(eccodes, path):
207
228
  keys = {key: _get(eccodes, h, key, str) for key in INVENTORY_KEYS}
208
229
  messages.append(
209
230
  GribMessage(
@@ -373,7 +394,7 @@ def open_grib2(
373
394
  needs_select = select is None and not fetched.provenance.is_partial
374
395
  object_messages = fetched.provenance.object_messages
375
396
  selectors = fetched.provenance.selectors if object_messages else []
376
- for h in _messages(eccodes, fetched.path):
397
+ for index, h in _messages(eccodes, fetched.path):
377
398
  count += 1
378
399
  if count > 1 and needs_select:
379
400
  continue
@@ -415,15 +436,15 @@ def open_grib2(
415
436
  if stamp:
416
437
  attrs[label] = stamp
417
438
  message: dict[str, Any] = {
418
- "file_index": count - 1,
419
- "object_index": object_messages[count - 1] if object_messages else None,
439
+ "file_index": index,
440
+ "object_index": object_messages[index] if object_messages else None,
420
441
  "shortName": _get(eccodes, h, "shortName", str),
421
442
  "typeOfLevel": attrs.get("typeOfLevel"),
422
443
  "level": attrs.get("level"),
423
444
  "step": attrs.get("step"),
424
445
  }
425
446
  if selectors:
426
- message["selector"] = selectors[count - 1]
447
+ message["selector"] = selectors[index]
427
448
  short = message["shortName"]
428
449
  if short in (None, "", "unknown", "~"):
429
450
  short = _product_name(fetched.asset.id) or (
@@ -184,7 +184,10 @@ def search(
184
184
  typer.Option("--location", "--state", help="State, 'County, ST', or quoted FIPS code."),
185
185
  ] = None,
186
186
  start: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
187
- end: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
187
+ end: Annotated[
188
+ str | None,
189
+ typer.Option(help="ISO date or datetime; a date alone runs to the end of that UTC day."),
190
+ ] = None,
188
191
  planned: Annotated[
189
192
  bool, typer.Option("--planned", help="Include planned datasets that have no adapter yet.")
190
193
  ] = False,
@@ -321,7 +324,10 @@ def fetch(
321
324
  lon: Annotated[float | None, typer.Option()] = None,
322
325
  radius_km: Annotated[float, typer.Option(help="Radius around --lat/--lon.")] = 50.0,
323
326
  start: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
324
- end: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
327
+ end: Annotated[
328
+ str | None,
329
+ typer.Option(help="ISO date or datetime; a date alone runs to the end of that UTC day."),
330
+ ] = None,
325
331
  variables: Annotated[
326
332
  str | None, typer.Option("--vars", help="Comma-separated variable names.")
327
333
  ] = None,
@@ -507,16 +513,28 @@ def pull(
507
513
  except (httpx.HTTPError, ChecksumMismatch) as e:
508
514
  typer.secho(f"fetch failed: {e}", err=True, fg="red")
509
515
  raise typer.Exit(code=4) from None
510
- updated = set(result.updated)
516
+ updated, mirrored = set(result.updated), set(result.mirrored)
511
517
  if not quiet:
512
518
  for f in result.fetched:
513
- tag = "updated" if f.asset.id in updated else "cached" if f.from_cache else "fetched"
519
+ if f.asset.id in updated:
520
+ tag = "updated"
521
+ elif f.asset.id in mirrored:
522
+ tag = "mirrored"
523
+ else:
524
+ tag = "cached" if f.from_cache else "fetched"
514
525
  typer.echo(f"{f.path}\t{tag}\t{f.provenance.size} bytes")
515
526
  if result.updated:
516
527
  mode = f"updated {len(result.updated)} pin(s) in"
517
528
  else:
518
529
  mode = "restored from" if result.from_lockfile else "wrote"
519
530
  typer.echo(f"{len(result.fetched)} asset(s); {mode} {result.lockfile_path}", err=True)
531
+ if result.mirrored:
532
+ typer.secho(
533
+ f"{len(result.mirrored)} asset(s) changed upstream and were restored from the "
534
+ "mirror; the lockfile still pins the source. Pass --update to accept new bytes.",
535
+ err=True,
536
+ fg="yellow",
537
+ )
520
538
 
521
539
 
522
540
  @app.command()
@@ -94,7 +94,8 @@ def _echo_detail(summary: Summary, width: int) -> None:
94
94
  else:
95
95
  typer.echo(" columns:")
96
96
  for name in frame.columns:
97
- typer.echo(f" {name}")
97
+ unit = frame.units.get(name)
98
+ typer.echo(f" {name} ({unit})" if unit else f" {name}")
98
99
  if (scene := summary.netcdf) is not None:
99
100
  _echo_table(
100
101
  "variables",