eo-fetch-api 1.0.0__py3-none-any.whl

eo_fetch/__init__.py

@@ -0,0 +1,48 @@
+"""eo-fetch: one search-and-download interface for Earth-observation and climate data.
+
+import eo_fetch as eof
+"""
+
+__version__ = "1.0.0"
+
+from eo_fetch.api import (
+    Results,
+    collections,
+    convert,
+    doctor,
+    download,
+    login,
+    plot,
+    products,
+    provider,
+    providers,
+    search,
+    to_earthkit,
+    verify,
+)
+from eo_fetch.api import open_data as open
+from eo_fetch.discovery import ProductMatch
+from eo_fetch.downloader import Result
+from eo_fetch.models import Item, Query
+
+__all__ = [
+    "Item",
+    "ProductMatch",
+    "Query",
+    "Result",
+    "Results",
+    "__version__",
+    "collections",
+    "convert",
+    "doctor",
+    "download",
+    "login",
+    "open",
+    "plot",
+    "products",
+    "provider",
+    "providers",
+    "search",
+    "to_earthkit",
+    "verify",
+]

eo_fetch/__main__.py

@@ -0,0 +1,3 @@
+from eo_fetch.cli import main
+
+raise SystemExit(main())

eo_fetch/api.py

@@ -0,0 +1,288 @@
+"""The Python API: ``import eo_fetch as eof``.
+
+    import eo_fetch as eof
+
+    found = eof.search("dwd", "radolan-yw", limit=2)
+    eof.download(found, "data/dwd")
+    ds = eof.open("data/dwd/raa01-yw_10000-2611032225-dwd---bin.hdf5")
+
+Everything the command line does is available here, with the same names and rules.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Mapping
+from datetime import UTC, date, datetime
+from pathlib import Path
+
+from eo_fetch.backends import backend_factory, get_backend, param_names
+from eo_fetch.discovery import ProductMatch, find_products
+from eo_fetch.downloader import Result, download_items
+from eo_fetch.manifest import build_manifest, verify_manifest, write_manifest
+from eo_fetch.models import Item, Query
+from eo_fetch.registry import ProviderInfo, all_providers, get_provider
+
+
+def parse_time(value: str | date | datetime) -> datetime:
+    """An ISO string, date or datetime as a timezone-aware datetime (naive values are UTC)."""
+    if isinstance(value, datetime):
+        moment = value
+    elif isinstance(value, date):
+        moment = datetime(value.year, value.month, value.day)
+    else:
+        moment = datetime.fromisoformat(str(value))
+    return moment if moment.tzinfo else moment.replace(tzinfo=UTC)
+
+
+def param_text(value: object) -> str:
+    """A search parameter as text; lists and tuples become comma-separated values."""
+    if isinstance(value, list | tuple | set | frozenset):
+        return ",".join(str(part) for part in value)
+    return str(value)
+
+
+def make_backend(info: ProviderInfo, user_params: Mapping[str, str]):
+    """Create the provider's backend; unknown parameter names raise ValueError."""
+    constructor, search_names = param_names(info.backend)
+    unknown = set(user_params) - constructor - search_names
+    if getattr(backend_factory(info.backend), "open_params", False):
+        unknown = set()  # every other parameter is a search facet
+    if unknown:
+        accepted = ", ".join(sorted(constructor | search_names)) or "none"
+        raise ValueError(
+            f"Unknown --param {', '.join(sorted(unknown))} for provider {info.name} "
+            f"(accepted: {accepted})"
+        )
+    return get_backend(info.backend, {**info.backend_params, **user_params})
+
+
+class Results(list[Item]):
+    """The items a search found, plus what is needed to download them.
+
+    It is a normal list of :class:`Item`. Slices and :meth:`where` keep the download context, so
+    ``eof.download(found[:3], dest)`` and ``eof.download(found.where(...), dest)`` work.
+    """
+
+    def __init__(self, items: Iterable[Item], *, info: ProviderInfo, backend, query: Query):
+        super().__init__(items)
+        self.info = info
+        self.backend = backend
+        self.query = query
+
+    def __getitem__(self, index):
+        picked = super().__getitem__(index)
+        return self.derive(picked) if isinstance(index, slice) else picked
+
+    def derive(self, items: Iterable[Item]) -> Results:
+        return Results(items, info=self.info, backend=self.backend, query=self.query)
+
+    def where(self, keep: Callable[[Item], bool]) -> Results:
+        """The items for which ``keep(item)`` is true."""
+        return self.derive(item for item in self if keep(item))
+
+    @property
+    def megabytes(self) -> float:
+        """Total size in megabytes (unknown sizes count as 0)."""
+        return sum(item.expected_size or 0 for item in self) / 1e6
+
+    def download(self, dest, **options) -> list[Result]:
+        """Same as ``eof.download(self, dest, **options)``."""
+        return download(self, dest, **options)
+
+
+def providers() -> list[ProviderInfo]:
+    """All provider cards (name, access type, licence, last verified, notes)."""
+    return all_providers()
+
+
+def provider(name: str) -> ProviderInfo:
+    """The card of one provider."""
+    return get_provider(name)
+
+
+def products(
+    text: str = "",
+    provider: str | None = None,
+    *,
+    live: bool | None = None,
+    browse: bool | None = None,
+    on_error: Callable[[ProviderInfo, Exception], None] | None = None,
+) -> list[ProductMatch]:
+    """Find the product names to use in :func:`search`.
+
+    ``products()`` lists every alias of every provider; ``products("precipitation")`` keeps those
+    whose name or description contains the word(s); ``products(provider="dwd")`` lists one
+    provider (and, for file-tree providers, its top-level folders). With a ``provider`` and a
+    ``text`` the live catalogue of the provider is searched too (``live=True`` does so for all
+    providers, which needs the internet and, for some, credentials). Each result's ``product``
+    is what you pass to :func:`search`. A provider whose live catalogue cannot be reached is
+    skipped with a warning (or ``on_error(info, exception)`` if given).
+    """
+    return find_products(
+        text, provider, live=live, browse=browse, make_backend=make_backend, on_error=on_error
+    )
+
+
+def search(
+    provider: str,
+    product: str = "",
+    *,
+    start=None,
+    end=None,
+    bbox=None,
+    text: str = "",
+    limit: int | None = None,
+    params: Mapping[str, object] | None = None,
+    **extra: object,
+) -> Results:
+    """Find items. ``start``/``end`` take ISO strings, dates or datetimes (``end`` is exclusive).
+
+    ``bbox`` is (west, south, east, north) in degrees. Provider-specific options are keyword
+    arguments (or a ``params`` dict), for example ``assets="B04,B08"`` or
+    ``variable_id=["tas", "pr"]``; they are the ``--param`` options of the command line.
+    """
+    info = get_provider(provider)
+    user_params = {
+        key: param_text(value)
+        for key, value in {**(params or {}), **extra}.items()
+        if value is not None
+    }
+    backend = make_backend(info, user_params)
+    query = Query(
+        provider=info.name,
+        product=info.products.get(product, product),
+        start=parse_time(start) if start is not None else None,
+        end=parse_time(end) if end is not None else None,
+        bbox=tuple(float(value) for value in bbox) if bbox else None,
+        text=text,
+        limit=limit,
+        params=user_params,
+    )
+    return Results(backend.search(query), info=info, backend=backend, query=query)
+
+
+def download(
+    results: Results,
+    dest,
+    *,
+    workers: int | None = None,
+    retries: int = 3,
+    manifest=None,
+    on_result: Callable[[Result], None] | None = None,
+) -> list[Result]:
+    """Download the items of a search into ``dest`` and write the manifest.
+
+    Files are verified (size and checksum when the provider announces them), retried on errors,
+    and never exceed the connections the service allows. The manifest goes to
+    ``dest/manifest.json`` unless ``manifest`` is given. Failures do not raise: check
+    ``result.status`` (``downloaded``, ``skipped`` or ``failed``) and ``result.error``.
+    """
+    if not isinstance(results, Results):
+        raise TypeError(
+            "Pass the result of eof.search() (or a slice or .where() of it), not a plain list"
+        )
+    if not results:
+        return []
+    dest = Path(dest)
+    outcome = download_items(
+        results.backend, list(results), dest, workers=workers, retries=retries, on_result=on_result
+    )
+    path = Path(manifest) if manifest else dest / "manifest.json"
+    write_manifest(path, build_manifest(results.query, results.info, outcome))
+    return outcome
+
+
+def verify(manifest, root=None) -> list[str]:
+    """Check downloaded files against a manifest; returns the problems (empty when all is well)."""
+    manifest = Path(manifest)
+    return verify_manifest(manifest, Path(root) if root else manifest.parent)
+
+
+def collections(provider: str, text: str = "", **params: object) -> list[tuple[str, str]]:
+    """List the collections or datasets a provider offers, optionally filtered by ``text``."""
+    info = get_provider(provider)
+    backend = make_backend(info, {k: param_text(v) for k, v in params.items()})
+    lister = getattr(backend, "collections", None)
+    if lister is None:
+        raise NotImplementedError(f"The {info.backend} backend has no collection listing")
+    return lister(text)
+
+
+def login(provider: str) -> None:
+    """Ask for the provider's credentials (hidden prompts) and save them in your user folder."""
+    info = get_provider(provider)
+    backend = make_backend(info, {})
+    login_step = getattr(backend, "login", None)
+    if login_step is None:
+        raise NotImplementedError(f"{info.name} has no login step; see doctor({info.name!r})")
+    login_step()
+
+
+def doctor(provider: str, **params: object) -> bool:
+    """Print what is needed to access a provider and whether it works; True when all is well."""
+    info = get_provider(provider)
+    backend = make_backend(info, {k: param_text(v) for k, v in params.items()})
+    print(f"{info.name}: {info.title}")
+    print(f"  access: {info.auth}; latency: {info.latency or 'not stated'}")
+    print(f"  licence: {info.licence or 'not stated'}")
+    check = getattr(backend, "doctor", None)
+    if check is None:
+        print("  no access diagnostics implemented for this backend yet")
+        return True
+    return bool(check())
+
+
+def open_data(path, format: str | None = None, **options):
+    """Open a downloaded file as an ``xarray.Dataset`` (NetCDF, GRIB, GeoTIFF, SEVIRI, ...)."""
+    from eo_fetch.readers import open_data as reader
+
+    return reader(path, format=format, **options)
+
+
+def plot(
+    source,
+    output=None,
+    *,
+    variable: str | None = None,
+    style: str | None = None,
+    domain=None,
+    resolution: float | None = None,
+    bbox=None,
+    title: str | None = None,
+    **options,
+) -> Path:
+    """Draw a file or Dataset on a map with the shared styles; returns the image path."""
+    from eo_fetch import plotting
+
+    if output is None:
+        if not isinstance(source, str | Path):
+            raise ValueError("Give output=... when plotting a Dataset")
+        output = Path("plots") / (Path(source).stem[:60] + ".png")
+    plotting.plot(
+        source,
+        Path(output),
+        variable=variable,
+        style=style,
+        domain=domain,
+        resolution=resolution,
+        bbox=tuple(bbox) if bbox else None,
+        title=title,
+        **options,
+    )
+    return Path(output)
+
+
+def convert(source, output, *, resolution: float | None = None, bbox=None, **options) -> Path:
+    """Write a readable file as NetCDF (``.nc``) or Zarr (``.zarr``), optionally on lat/lon."""
+    from eo_fetch import bridge
+
+    return bridge.convert(
+        source, Path(output), resolution=resolution, bbox=tuple(bbox) if bbox else None, **options
+    )
+
+
+def to_earthkit(source, variable: str | None = None, resolution=None, bbox=None, **options):
+    """An earthkit-data object for a file, ready for earthkit-plots, -regrid or -transforms."""
+    from eo_fetch import bridge
+
+    return bridge.to_earthkit(source, variable, resolution, bbox, **options)

eo_fetch/backends/__init__.py

@@ -0,0 +1,71 @@
+"""Backends and the registry that builds them by name."""
+
+from __future__ import annotations
+
+import inspect
+from collections.abc import Callable, Mapping
+
+from eo_fetch.backends.base import Backend
+from eo_fetch.backends.cds import CdsBackend
+from eo_fetch.backends.ecmwf import EcmwfOpenDataBackend
+from eo_fetch.backends.esgf import EsgfBackend
+from eo_fetch.backends.eumetsat import EumetsatBackend
+from eo_fetch.backends.http_index import HttpIndexBackend
+from eo_fetch.backends.local import LocalBackend
+from eo_fetch.backends.marine import MarineBackend
+from eo_fetch.backends.nasa import NasaBackend
+from eo_fetch.backends.s3 import S3Backend
+from eo_fetch.backends.stac import StacBackend
+
+# Backends import their third-party library only when first used, so listing is always cheap.
+BACKENDS: dict[str, Callable[..., Backend]] = {
+    "local": LocalBackend,
+    "eumetsat": EumetsatBackend,
+    "s3": S3Backend,
+    "http-index": HttpIndexBackend,
+    "nasa": NasaBackend,
+    "stac": StacBackend,
+    "ecmwf-opendata": EcmwfOpenDataBackend,
+    "cds": CdsBackend,
+    "marine": MarineBackend,
+    "esgf": EsgfBackend,
+}
+
+
+def backend_factory(name: str) -> Callable[..., Backend]:
+    try:
+        return BACKENDS[name]
+    except KeyError:
+        known = ", ".join(sorted(BACKENDS))
+        raise KeyError(f"Unknown backend {name!r}. Available: {known}") from None
+
+
+def param_names(name: str) -> tuple[set[str], set[str]]:
+    """Return (constructor parameters, search parameters) a backend accepts."""
+    factory = backend_factory(name)
+    return set(inspect.signature(factory).parameters), set(getattr(factory, "query_params", ()))
+
+
+def get_backend(name: str, params: Mapping[str, str] | None = None) -> Backend:
+    """Build a backend by name; only the parameters its constructor accepts are passed on."""
+    accepted, _ = param_names(name)
+    return backend_factory(name)(**{k: v for k, v in (params or {}).items() if k in accepted})
+
+
+__all__ = [
+    "BACKENDS",
+    "Backend",
+    "CdsBackend",
+    "EcmwfOpenDataBackend",
+    "EsgfBackend",
+    "EumetsatBackend",
+    "HttpIndexBackend",
+    "LocalBackend",
+    "MarineBackend",
+    "NasaBackend",
+    "S3Backend",
+    "StacBackend",
+    "backend_factory",
+    "get_backend",
+    "param_names",
+]

eo_fetch/backends/base.py

@@ -0,0 +1,28 @@
+"""The interface every backend implements."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from eo_fetch.models import Item, Query
+
+
+class Backend(Protocol):
+    """A wrapper around one access method (a library or a protocol).
+
+    ``max_connections`` is the number of parallel downloads the service tolerates. ``fetch``
+    must write the complete object to ``target``; the downloader passes a temporary name and
+    renames it after verification.
+    """
+
+    name: str
+    max_connections: int
+
+    def search(self, query: Query) -> list[Item]:
+        """Return the items matching the query, sorted by time then key."""
+        ...
+
+    def fetch(self, item: Item, target: Path) -> None:
+        """Write the item to ``target``."""
+        ...

eo_fetch/backends/cds.py

@@ -0,0 +1,174 @@
+"""Copernicus Climate Data Store backend (ERA5 and other hourly datasets), via cdsapi.
+
+``search`` plans one request per month of the time range; ``fetch`` submits it to the CDS queue
+and downloads the result. Requests can wait in the queue for minutes or hours.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from calendar import monthrange
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+
+from eo_fetch.models import Item, Query
+
+DEFAULT_HOURS = ("00:00", "06:00", "12:00", "18:00")
+CDS_URL = "https://cds.climate.copernicus.eu/api"
+
+
+def split_list(text: str) -> list[str]:
+    return [part.strip() for part in text.split(",") if part.strip()]
+
+
+def month_requests(start: datetime, end: datetime) -> list[tuple[int, int, list[int]]]:
+    """(year, month, days) for every month touched by [start, end); only the days inside."""
+    result = []
+    day = datetime(start.year, start.month, start.day, tzinfo=UTC)
+    while day < end:
+        last = monthrange(day.year, day.month)[1]
+        days = []
+        cursor = day
+        while cursor < end and cursor.month == day.month:
+            days.append(cursor.day)
+            cursor += timedelta(days=1)
+        result.append((day.year, day.month, days))
+        day = datetime(day.year, day.month, last, tzinfo=UTC) + timedelta(days=1)
+    return result
+
+
+def build_request(
+    year: int,
+    month: int,
+    days: list[int],
+    variables: list[str],
+    hours: list[str],
+    bbox,
+    data_format: str,
+    product_type: str,
+    levels: list[str],
+    extra: dict,
+) -> dict:
+    """The CDS request for one month (ERA5 style; ``extra`` can add or replace keys)."""
+    request: dict = {
+        "variable": variables,
+        "year": f"{year:04d}",
+        "month": f"{month:02d}",
+        "day": [f"{d:02d}" for d in days],
+        "time": hours,
+        "data_format": data_format,
+        "download_format": "unarchived",
+    }
+    if product_type:
+        request["product_type"] = [product_type]
+    if levels:
+        request["pressure_level"] = levels
+    if bbox:
+        west, south, east, north = bbox
+        request["area"] = [north, west, south, east]  # the CDS wants N, W, S, E
+    request.update(extra)
+    return request
+
+
+class CdsBackend:
+    """Plan and fetch hourly datasets of the Copernicus Climate Data Store (ERA5 and others).
+
+    ``query.product`` is the dataset name (``reanalysis-era5-single-levels``). Search needs
+    ``--start`` and ``--end`` (end exclusive). Parameters: ``variable`` (required, CDS names
+    such as ``2m_temperature,total_precipitation``), ``hours`` (default 00:00,06:00,12:00,18:00),
+    ``format`` (``grib`` or ``netcdf``; default grib), ``product_type`` (default reanalysis),
+    ``level`` (pressure levels in hPa), and ``extra`` (a JSON object merged into the request).
+    ``--bbox`` becomes the request's area. Credentials are read by cdsapi from ``~/.cdsapirc``
+    (or CDSAPI_URL and CDSAPI_KEY) and never stored by eo-fetch. Accept each dataset's licence
+    once on the dataset's page of cds.climate.copernicus.eu.
+    """
+
+    name = "cds"
+    max_connections = 2
+    query_params = frozenset({"variable", "hours", "format", "product_type", "level", "extra"})
+
+    def __init__(self) -> None:
+        self._client = None
+
+    def login(self) -> None:
+        """Ask for the CDS personal access token (hidden) and save it to ~/.cdsapirc."""
+        from eo_fetch.backends.credentials import ask_secret, confirm_overwrite, write_private
+
+        path = Path.home() / ".cdsapirc"
+        if not confirm_overwrite(path):
+            print("Left unchanged.")
+            return
+        print("Find your token on your profile page at https://cds.climate.copernicus.eu")
+        key = ask_secret("CDS personal access token (input hidden): ")
+        write_private(path, f"url: {CDS_URL}\nkey: {key}\n")
+        print(f"Saved to {path} (plain text, in your user folder). Delete the file to log out.")
+        print("Remember to accept each dataset's licence on its page before downloading.")
+
+    def client(self):
+        if self._client is None:
+            import cdsapi
+
+            self._client = cdsapi.Client()
+        return self._client
+
+    def search(self, query: Query) -> list[Item]:
+        if not query.product:
+            raise ValueError("CDS search needs --product with a dataset name")
+        if not (query.start and query.end):
+            raise ValueError("CDS search needs both --start and --end (end is exclusive)")
+        variables = split_list(query.params.get("variable") or "")
+        if not variables:
+            raise ValueError("Give the variables with --param variable=2m_temperature,...")
+        hours = split_list(query.params.get("hours") or "") or list(DEFAULT_HOURS)
+        data_format = query.params.get("format") or "grib"
+        if data_format not in {"grib", "netcdf"}:
+            raise ValueError("--param format must be grib or netcdf")
+        extra = json.loads(query.params.get("extra") or "{}")
+        product_type = query.params.get("product_type", "reanalysis")
+        levels = split_list(query.params.get("level") or "")
+        suffix = "grib" if data_format == "grib" else "nc"
+        label = re.sub(r"[^A-Za-z0-9]+", "-", "-".join(variables))[:60].strip("-")
+        items = []
+        for year, month, days in month_requests(query.start, query.end):
+            request = build_request(
+                year=year,
+                month=month,
+                days=days,
+                variables=variables,
+                hours=hours,
+                bbox=query.bbox,
+                data_format=data_format,
+                product_type=product_type,
+                levels=levels,
+                extra=extra,
+            )
+            items.append(
+                Item(
+                    provider=query.provider,
+                    product=query.product,
+                    key=f"{query.product}_{year:04d}{month:02d}_{label}.{suffix}",
+                    time=datetime(year, month, days[0], tzinfo=UTC),
+                    meta={"request": json.dumps(request)},
+                )
+            )
+        return items[: query.limit] if query.limit else items
+
+    def fetch(self, item: Item, target: Path) -> None:
+        request = json.loads(item.meta["request"])
+        self.client().retrieve(item.product, request, str(target))
+
+    def doctor(self, collection_ids: tuple[str, ...] = ()) -> bool:
+        rc = Path.home() / ".cdsapirc"
+        if os.environ.get("CDSAPI_KEY") or rc.is_file():
+            print("  credentials: found (CDSAPI_KEY or ~/.cdsapirc), tested when a request runs")
+        else:
+            print(
+                "  credentials: NOT FOUND. Create a free account at "
+                "https://cds.climate.copernicus.eu, then run `eo-fetch login --provider cds` "
+                "(or write ~/.cdsapirc yourself, or set CDSAPI_URL and CDSAPI_KEY)."
+            )
+            return False
+        print("  licences: accept each dataset's licence on its page, or requests will be refused")
+        return True

eo_fetch/backends/credentials.py

@@ -0,0 +1,31 @@
+"""Helpers for the ``eo-fetch login`` commands: hidden prompts and private credential files."""
+
+from __future__ import annotations
+
+import contextlib
+import getpass
+from pathlib import Path
+
+
+def ask_secret(prompt: str) -> str:
+    """Ask for a secret without echoing it; an empty answer is an error."""
+    value = getpass.getpass(prompt).strip()
+    if not value:
+        raise ValueError("Nothing entered")
+    return value
+
+
+def confirm_overwrite(path: Path) -> bool:
+    """True when the file does not exist, or the user agrees to replace it."""
+    if not path.exists():
+        return True
+    answer = input(f"{path} exists. Replace it? [y/N] ").strip().lower()
+    return answer in {"y", "yes"}
+
+
+def write_private(path: Path, text: str) -> None:
+    """Write a credentials file (plain text) readable only by the user where the OS allows."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(text, encoding="utf-8")
+    with contextlib.suppress(OSError):  # Windows ignores POSIX modes
+        path.chmod(0o600)