usdata 0.1.0__tar.gz → 0.2.0__tar.gz

usdata-0.2.0/PKG-INFO

@@ -0,0 +1,113 @@
+Metadata-Version: 2.4
+Name: usdata
+Version: 0.2.0
+Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
+Keywords: noaa,usgs,nasa,open-data,scientific-data,provenance
+Author: Jake Van Slyke
+Author-email: Jake Van Slyke <jakervanslyke@gmail.com>
+License-Expression: Apache-2.0
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: typer>=0.12
+Requires-Python: >=3.11
+Project-URL: Homepage, https://github.com/jakeryderv/usdata
+Project-URL: Repository, https://github.com/jakeryderv/usdata
+Description-Content-Type: text/markdown
+
+# usdata
+
+Unified Python SDK and CLI for discovering, fetching, and tracking the
+provenance of U.S. public scientific data (NOAA, USGS, NASA, and more).
+
+> Status: pre-alpha. `noaa:ghcn-daily` and `noaa:nexrad-level2` fetch real
+> data with provenance; other registry entries are stubs.
+> See [docs/roadmap.md](docs/roadmap.md).
+
+## Install
+
+```sh
+pip install usdata        # or: uv add usdata
+```
+
+## Usage
+
+```python
+from usdata import build_query, get, search
+from usdata.fetch import fetch
+
+for r in search("precipitation", location="Oklahoma"):
+    print(r.dataset.id, r.dataset.title)
+
+ds = get("noaa:ghcn-daily")
+query = build_query(
+    lat=35.39,
+    lon=-97.60,
+    radius_km=15,
+    start="2024-05-06",
+    end="2024-05-07",
+    variables=["PRCP", "TMAX"],
+)
+for item in fetch(ds, query):
+    print(item.path, item.provenance.checksum)
+```
+
+```sh
+usdata search "tornado radar" --state OK
+usdata info noaa:ghcn-daily
+usdata fetch noaa:ghcn-daily --lat 35.39 --lon -97.60 --radius-km 15 \
+    --start 2024-05-06 --end 2024-05-07 --vars PRCP,TMAX
+usdata fetch noaa:ghcn-daily -p stations=USW00013967 --start 2024-01-01 --end 2024-12-31
+usdata fetch noaa:nexrad-level2 --lat 35.47 --lon -97.52 \
+    --start 2024-05-06T20:00 --end 2024-05-06T23:00        # nearest radar (KTLX)
+usdata fetch noaa:nexrad-level2 -p site=KTLX --start 2024-05-06T20:00 --end 2024-05-06T20:30 --dry-run
+usdata pull dataset.yaml
+```
+
+Fetched files land in `~/.cache/usdata/<provider>/<dataset>/` (override with
+`USDATA_CACHE_DIR` or `--cache-dir`), each with a `.provenance.json` sidecar
+recording source URL, retrieval time, checksum, size, and license.
+
+A manifest declares every input a project needs; `pull` fetches them and writes
+a lockfile with checksums and provenance so the inputs can be reproduced:
+
+```yaml
+name: tornado-environment
+sources:
+  - dataset: noaa:nexrad-level2
+    location: oklahoma
+    start: 2024-05-06
+    end: 2024-05-07
+  - dataset: noaa:ghcn-daily
+    location: oklahoma
+    start: 2024-05-01
+    end: 2024-05-31
+```
+
+## Development
+
+Requires [uv](https://docs.astral.sh/uv/) and [just](https://just.systems/).
+
+```sh
+git clone https://github.com/jakeryderv/usdata && cd usdata
+just setup     # install toolchain and dependencies
+just test      # unit tests
+just check     # format, lint, typecheck, tests (what CI runs)
+just run search radar
+```
+
+Integration tests that hit live services run with `just test-integration`.
+
+Releases are published to PyPI by tagging: create a GitHub release and the
+`publish.yml` workflow uploads via trusted publishing.
+
+See [docs/architecture.md](docs/architecture.md) for how the pieces fit and
+[docs/adr/](docs/adr/) for why.
+
+## License
+
+Apache-2.0

usdata-0.2.0/README.md

@@ -0,0 +1,92 @@
+# usdata
+
+Unified Python SDK and CLI for discovering, fetching, and tracking the
+provenance of U.S. public scientific data (NOAA, USGS, NASA, and more).
+
+> Status: pre-alpha. `noaa:ghcn-daily` and `noaa:nexrad-level2` fetch real
+> data with provenance; other registry entries are stubs.
+> See [docs/roadmap.md](docs/roadmap.md).
+
+## Install
+
+```sh
+pip install usdata        # or: uv add usdata
+```
+
+## Usage
+
+```python
+from usdata import build_query, get, search
+from usdata.fetch import fetch
+
+for r in search("precipitation", location="Oklahoma"):
+    print(r.dataset.id, r.dataset.title)
+
+ds = get("noaa:ghcn-daily")
+query = build_query(
+    lat=35.39,
+    lon=-97.60,
+    radius_km=15,
+    start="2024-05-06",
+    end="2024-05-07",
+    variables=["PRCP", "TMAX"],
+)
+for item in fetch(ds, query):
+    print(item.path, item.provenance.checksum)
+```
+
+```sh
+usdata search "tornado radar" --state OK
+usdata info noaa:ghcn-daily
+usdata fetch noaa:ghcn-daily --lat 35.39 --lon -97.60 --radius-km 15 \
+    --start 2024-05-06 --end 2024-05-07 --vars PRCP,TMAX
+usdata fetch noaa:ghcn-daily -p stations=USW00013967 --start 2024-01-01 --end 2024-12-31
+usdata fetch noaa:nexrad-level2 --lat 35.47 --lon -97.52 \
+    --start 2024-05-06T20:00 --end 2024-05-06T23:00        # nearest radar (KTLX)
+usdata fetch noaa:nexrad-level2 -p site=KTLX --start 2024-05-06T20:00 --end 2024-05-06T20:30 --dry-run
+usdata pull dataset.yaml
+```
+
+Fetched files land in `~/.cache/usdata/<provider>/<dataset>/` (override with
+`USDATA_CACHE_DIR` or `--cache-dir`), each with a `.provenance.json` sidecar
+recording source URL, retrieval time, checksum, size, and license.
+
+A manifest declares every input a project needs; `pull` fetches them and writes
+a lockfile with checksums and provenance so the inputs can be reproduced:
+
+```yaml
+name: tornado-environment
+sources:
+  - dataset: noaa:nexrad-level2
+    location: oklahoma
+    start: 2024-05-06
+    end: 2024-05-07
+  - dataset: noaa:ghcn-daily
+    location: oklahoma
+    start: 2024-05-01
+    end: 2024-05-31
+```
+
+## Development
+
+Requires [uv](https://docs.astral.sh/uv/) and [just](https://just.systems/).
+
+```sh
+git clone https://github.com/jakeryderv/usdata && cd usdata
+just setup     # install toolchain and dependencies
+just test      # unit tests
+just check     # format, lint, typecheck, tests (what CI runs)
+just run search radar
+```
+
+Integration tests that hit live services run with `just test-integration`.
+
+Releases are published to PyPI by tagging: create a GitHub release and the
+`publish.yml` workflow uploads via trusted publishing.
+
+See [docs/architecture.md](docs/architecture.md) for how the pieces fit and
+[docs/adr/](docs/adr/) for why.
+
+## License
+
+Apache-2.0

usdata-0.2.0/pyproject.toml

@@ -0,0 +1,84 @@
+[project]
+name = "usdata"
+version = "0.2.0"
+description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
+readme = "README.md"
+license = "Apache-2.0"
+requires-python = ">=3.11"
+keywords = [
+    "noaa",
+    "usgs",
+    "nasa",
+    "open-data",
+    "scientific-data",
+    "provenance",
+]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering",
+]
+dependencies = [
+    "httpx>=0.28.1",
+    "pydantic>=2.7",
+    "pyyaml>=6.0",
+    "typer>=0.12",
+]
+
+[[project.authors]]
+name = "Jake Van Slyke"
+email = "jakervanslyke@gmail.com"
+
+[project.urls]
+Homepage = "https://github.com/jakeryderv/usdata"
+Repository = "https://github.com/jakeryderv/usdata"
+
+[project.scripts]
+usdata = "usdata.cli:app"
+
+[dependency-groups]
+dev = [
+    "pyright>=1.1.380",
+    "pytest>=8.0",
+    "respx>=0.23.1",
+    "ruff>=0.6",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.5,<0.13.0"]
+build-backend = "uv_build"
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+src = [
+    "src",
+    "tests",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "I",
+    "UP",
+    "B",
+    "SIM",
+    "RUF",
+]
+
+[tool.pyright]
+include = [
+    "src",
+    "tests",
+]
+pythonVersion = "3.11"
+typeCheckingMode = "standard"
+venvPath = "."
+venv = ".venv"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = ["integration: hits live services; skipped unless --run-integration is passed"]
+addopts = "-ra"

usdata-0.2.0/pyproject.toml.orig

@@ -0,0 +1,64 @@
+[project]
+name = "usdata"
+version = "0.2.0"
+description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
+readme = "README.md"
+license = "Apache-2.0"
+authors = [
+    { name = "Jake Van Slyke", email = "jakervanslyke@gmail.com" }
+]
+requires-python = ">=3.11"
+keywords = ["noaa", "usgs", "nasa", "open-data", "scientific-data", "provenance"]
+classifiers = [
+    "Development Status :: 2 - Pre-Alpha",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering",
+]
+dependencies = [
+    "httpx>=0.28.1",
+    "pydantic>=2.7",
+    "pyyaml>=6.0",
+    "typer>=0.12",
+]
+
+[project.urls]
+Homepage = "https://github.com/jakeryderv/usdata"
+Repository = "https://github.com/jakeryderv/usdata"
+
+[project.scripts]
+usdata = "usdata.cli:app"
+
+[dependency-groups]
+dev = [
+    "pyright>=1.1.380",
+    "pytest>=8.0",
+    "respx>=0.23.1",
+    "ruff>=0.6",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.5,<0.13.0"]
+build-backend = "uv_build"
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM", "RUF"]
+
+[tool.pyright]
+include = ["src", "tests"]
+pythonVersion = "3.11"
+typeCheckingMode = "standard"
+venvPath = "."
+venv = ".venv"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+markers = [
+    "integration: hits live services; skipped unless --run-integration is passed",
+]
+addopts = "-ra"

usdata-0.2.0/src/usdata/__init__.py

@@ -0,0 +1,42 @@
+"""usdata: unified access and provenance for U.S. public scientific data."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+from typing import Any
+
+try:
+    __version__ = version("usdata")
+except PackageNotFoundError:  # running from a source tree without an install
+    __version__ = "0.0.0"
+
+from usdata.models import Asset, BBox, Dataset, Provenance, Query, TimeRange
+from usdata.query import build_query
+from usdata.registry import DatasetNotFound, Registry, SearchResult, default_registry
+
+__all__ = [
+    "Asset",
+    "BBox",
+    "Dataset",
+    "DatasetNotFound",
+    "Provenance",
+    "Query",
+    "Registry",
+    "SearchResult",
+    "TimeRange",
+    "__version__",
+    "build_query",
+    "default_registry",
+    "get",
+    "search",
+]
+
+
+def search(text: str | None = None, **kwargs: Any) -> list[SearchResult]:
+    """Search the curated registry. Keyword arguments match ``build_query``."""
+    return default_registry().search(build_query(text, **kwargs))
+
+
+def get(dataset_id: str) -> Dataset:
+    """Look up a dataset by ``provider:name`` id."""
+    return default_registry().get(dataset_id)

usdata-0.2.0/src/usdata/cache.py

@@ -0,0 +1,33 @@
+"""Local file cache. Layout: ``<cache_dir>/<provider>/<dataset name>/<asset id>``."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from pathlib import Path
+
+from usdata.models import Asset
+
+ENV_VAR = "USDATA_CACHE_DIR"
+
+
+def cache_dir() -> Path:
+    if override := os.environ.get(ENV_VAR):
+        return Path(override).expanduser()
+    xdg = os.environ.get("XDG_CACHE_HOME")
+    base = Path(xdg).expanduser() if xdg else Path.home() / ".cache"
+    return base / "usdata"
+
+
+def asset_path(asset: Asset, root: Path | None = None) -> Path:
+    provider, name = asset.dataset_id.split(":", 1)
+    safe_id = asset.id.replace("/", "_")
+    return (root or cache_dir()) / provider / name / safe_id
+
+
+def sha256_file(path: Path, chunk_size: int = 1 << 20) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        while chunk := f.read(chunk_size):
+            h.update(chunk)
+    return f"sha256:{h.hexdigest()}"

usdata-0.2.0/src/usdata/cli/__init__.py

@@ -0,0 +1,3 @@
+from usdata.cli.app import app
+
+__all__ = ["app"]

usdata-0.2.0/src/usdata/cli/app.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import httpx
+import typer
+
+from usdata import __version__, build_query, default_registry
+from usdata.fetch import fetch_asset
+from usdata.manifest import Manifest
+from usdata.providers import load_adapter
+from usdata.providers.base import NotImplementedProvider
+from usdata.query import UnknownPlace
+from usdata.registry import DatasetNotFound
+
+app = typer.Typer(
+    name="usdata",
+    help="Discover, fetch, and track provenance of U.S. public scientific data.",
+    no_args_is_help=True,
+)
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(f"usdata {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: Annotated[
+        bool | None,
+        typer.Option("--version", callback=_version_callback, is_eager=True, help="Show version."),
+    ] = None,
+) -> None:
+    pass
+
+
+@app.command()
+def search(
+    text: Annotated[str | None, typer.Argument(help="Free-text keywords.")] = None,
+    provider: Annotated[
+        str | None, typer.Option(help="Restrict to one provider, e.g. noaa.")
+    ] = None,
+    state: Annotated[str | None, typer.Option(help="State name or postal code.")] = None,
+    start: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
+    end: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
+) -> None:
+    """Search the curated dataset registry."""
+    try:
+        query = build_query(text, provider=provider, location=state, start=start, end=end)
+    except UnknownPlace as e:
+        typer.secho(f"Unknown place: {e}", err=True, fg="red")
+        raise typer.Exit(code=2) from None
+    results = default_registry().search(query)
+    if not results:
+        typer.echo("No datasets matched.")
+        raise typer.Exit(code=1)
+    width = max(len(r.dataset.id) for r in results)
+    for r in results:
+        typer.echo(f"{r.dataset.id:<{width}}  {r.dataset.title}")
+
+
+@app.command()
+def info(
+    dataset_id: Annotated[str, typer.Argument(help="Dataset id, e.g. noaa:nexrad-level2")],
+) -> None:
+    """Show details for one dataset."""
+    try:
+        ds = default_registry().get(dataset_id)
+    except DatasetNotFound:
+        typer.secho(f"Unknown dataset: {dataset_id}", err=True, fg="red")
+        raise typer.Exit(code=2) from None
+    typer.echo(f"{ds.id}\n  {ds.title}\n")
+    typer.echo(f"  {ds.description.strip()}\n")
+    typer.echo(f"  provider:  {ds.provider}")
+    typer.echo(f"  protocol:  {ds.protocol.value}")
+    typer.echo(f"  license:   {ds.license or 'unknown'}")
+    typer.echo(f"  homepage:  {ds.homepage or '-'}")
+    caps = ", ".join(k for k, v in ds.capabilities.model_dump().items() if v) or "none"
+    typer.echo(f"  subsetting: {caps}")
+    if ds.spatial_extent:
+        typer.echo(f"  extent:    {ds.spatial_extent.as_tuple()}")
+    if ds.temporal_extent:
+        typer.echo(
+            f"  time:      {ds.temporal_extent.start} .. {ds.temporal_extent.end or 'present'}"
+        )
+
+
+@app.command()
+def fetch(
+    dataset_id: Annotated[str, typer.Argument(help="Dataset id, e.g. noaa:ghcn-daily")],
+    state: Annotated[str | None, typer.Option(help="State name or postal code.")] = None,
+    bbox: Annotated[str | None, typer.Option(help="west,south,east,north in degrees.")] = None,
+    lat: Annotated[float | None, typer.Option()] = None,
+    lon: Annotated[float | None, typer.Option()] = None,
+    radius_km: Annotated[float, typer.Option(help="Radius around --lat/--lon.")] = 50.0,
+    start: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
+    end: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
+    variables: Annotated[
+        str | None, typer.Option("--vars", help="Comma-separated variable names.")
+    ] = None,
+    param: Annotated[
+        list[str] | None,
+        typer.Option("--param", "-p", help="Provider-specific key=value, repeatable."),
+    ] = None,
+    cache_dir: Annotated[Path | None, typer.Option(help="Override the cache directory.")] = None,
+    force: Annotated[bool, typer.Option(help="Re-download even if cached.")] = False,
+    dry_run: Annotated[
+        bool, typer.Option(help="List matching assets without downloading.")
+    ] = False,
+) -> None:
+    """Resolve a query against one dataset and download the matching assets."""
+    params: dict[str, str] = {}
+    for item in param or []:
+        key, sep, value = item.partition("=")
+        if not sep:
+            typer.secho(f"--param expects key=value, got {item!r}", err=True, fg="red")
+            raise typer.Exit(code=2)
+        params[key] = value
+    box = None
+    if bbox:
+        try:
+            w, s, e, n = (float(x) for x in bbox.split(","))
+        except ValueError:
+            typer.secho("--bbox expects west,south,east,north", err=True, fg="red")
+            raise typer.Exit(code=2) from None
+        box = (w, s, e, n)
+    try:
+        ds = default_registry().get(dataset_id)
+        query = build_query(
+            location=state,
+            bbox=box,
+            lat=lat,
+            lon=lon,
+            radius_km=radius_km,
+            start=start,
+            end=end,
+            variables=[v.strip() for v in variables.split(",")] if variables else None,
+            **params,
+        )
+        adapter = load_adapter(ds)
+        assets = adapter.list_assets(query)
+        if dry_run:
+            for a in assets:
+                typer.echo(f"{a.id}\t{a.href}")
+            typer.echo(f"{len(assets)} asset(s) matched", err=True)
+            return
+        fetched = [fetch_asset(ds, a, root=cache_dir, force=force) for a in assets]
+    except (DatasetNotFound, UnknownPlace, ValueError) as e:
+        typer.secho(str(e), err=True, fg="red")
+        raise typer.Exit(code=2) from None
+    except NotImplementedProvider as e:
+        typer.secho(str(e), err=True, fg="yellow")
+        raise typer.Exit(code=3) from None
+    except httpx.HTTPError as e:
+        typer.secho(f"request failed: {e}", err=True, fg="red")
+        raise typer.Exit(code=4) from None
+    if not fetched:
+        typer.echo("No assets matched.", err=True)
+        raise typer.Exit(code=1)
+    for f in fetched:
+        tag = "cached" if f.from_cache else "fetched"
+        typer.echo(f"{f.path}\t{tag}\t{f.provenance.size} bytes")
+
+
+@app.command()
+def pull(manifest: Annotated[Path, typer.Argument(exists=True, dir_okay=False)]) -> None:
+    """Fetch every source in a manifest and write a lockfile."""
+    m = Manifest.load(manifest)
+    missing = m.validate_against()
+    if missing:
+        typer.secho(f"Unknown datasets in manifest: {', '.join(missing)}", err=True, fg="red")
+        raise typer.Exit(code=2)
+    typer.echo(f"{m.name} v{m.version}: {len(m.sources)} source(s) validated")
+    typer.secho("pull is not implemented yet; no data was fetched.", err=True, fg="yellow")
+    raise typer.Exit(code=3)