dwdopen 0.2.1__py3-none-any.whl

dwdopen/__init__.py

@@ -0,0 +1,61 @@
+"""Python access to DWD Open Data."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+
+from dwdopen.client import DWD, NWP
+from dwdopen.exceptions import (
+    AmbiguousSelectionError,
+    CatalogueError,
+    CatalogueUnavailableError,
+    DownloadError,
+    DWDOpenError,
+    IncompleteRunError,
+    InvalidSelectorError,
+    MissingAssetError,
+    NoMatchingRunError,
+    ResolutionError,
+    RunExpiredError,
+    SelectionError,
+    UnknownModelError,
+    UnknownParameterError,
+)
+from dwdopen.nwp.durations import hours, minutes
+from dwdopen.nwp.model import Model, ParameterInfo
+from dwdopen.nwp.request import DownloadResult
+from dwdopen.nwp.run import Run
+from dwdopen.nwp.selectors import Between, Every, LevelType
+
+__all__ = [
+    "DWD",
+    "NWP",
+    "AmbiguousSelectionError",
+    "Between",
+    "CatalogueError",
+    "CatalogueUnavailableError",
+    "DWDOpenError",
+    "DownloadError",
+    "DownloadResult",
+    "Every",
+    "IncompleteRunError",
+    "InvalidSelectorError",
+    "LevelType",
+    "MissingAssetError",
+    "Model",
+    "NoMatchingRunError",
+    "ParameterInfo",
+    "ResolutionError",
+    "Run",
+    "RunExpiredError",
+    "SelectionError",
+    "UnknownModelError",
+    "UnknownParameterError",
+    "hours",
+    "minutes",
+]
+
+try:
+    __version__ = version("dwdopen")
+except PackageNotFoundError:  # running from a source tree, not installed
+    __version__ = "0.0.0.dev0"

dwdopen/_fileserver/__init__.py

@@ -0,0 +1,4 @@
+"""Catalogue implementation backed by DWD's Open Data file server.
+"""
+
+from __future__ import annotations

dwdopen/_fileserver/download.py

@@ -0,0 +1,265 @@
+"""Fetching assets over HTTP, in parallel, onto disk."""
+
+from __future__ import annotations
+
+import logging
+import os
+import secrets
+import shutil
+import time
+from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from dwdopen._fileserver.http import HttpClient
+from dwdopen._fileserver.naming import already_complete, local_name, temp_name
+from dwdopen.exceptions import DownloadError
+from dwdopen.nwp.request import Asset, CombineMode, Fetched, human_size
+from dwdopen.nwp.selectors import LevelType
+
+__all__ = ["HttpDownloader"]
+
+logger = logging.getLogger("dwdopen")
+
+LARGE_DOWNLOAD = 2 * 1024**3
+"""Log when a user plans to download large amounts of data already in the
+planning stage.
+"""
+
+RETRY_STATUS = {404, 408, 425, 429, 500, 502, 503, 504}
+"""HTTP error statuses worth a retry. 404 is handled separately."""
+
+
+class HttpDownloader:
+    """Fetches assets concurrently and writes them atomically.
+    Takes an HttpClient so transport configuration stays in one place.
+    """
+
+    def __init__(
+        self,
+        http: HttpClient,
+        *,
+        max_workers: int = 16,
+        max_attempts: int = 4,
+        backoff: float = 0.5
+    ) -> None:
+        """
+        max_workers
+            Concurrent downloads. Bounded by the client's connection pool.
+        max_attempts
+            Attempts per asset for a failure.
+        backoff
+            Seconds before the second attempt; doubles each further attempt.
+        """
+        self._http = http
+        self._max_workers = max_workers
+        self._max_attempts = max_attempts
+        self._backoff = backoff
+
+    def fetch(
+        self,
+        assets: Sequence[Asset],
+        destination: Path,
+        *,
+        combine: CombineMode = "all",
+        temp_dir: Path | None = None,
+    ) -> Fetched:
+        """Fetches the given assets to the destination path. Calls either fetch
+        separately or combined depending on the combine flag.
+        """
+        destination = Path(destination)
+        if not assets:
+            logger.info("nothing to download")
+            return Fetched(files=(), bytes_downloaded=0)
+
+        self._announce(assets, destination, combine)
+        if combine == "none":
+            fetched = self._fetch_separately(assets, destination, temp_dir)
+        else:
+            fetched = self._fetch_combined(assets, destination, temp_dir)
+        self._report(fetched, combine)
+        return fetched
+
+    def _fetch_separately(
+        self, assets: Sequence[Asset], destination: Path, temp_dir: Path | None
+    ) -> Fetched:
+        """One file per asset, named after its keys, inside one directory."""
+        destination.mkdir(parents=True, exist_ok=True)
+        staging = destination if temp_dir is None else temp_dir
+
+        target_paths = [destination / local_name(asset.keys) for asset in assets]
+        sizes = self._fetch_all(assets, target_paths, staging)
+        # A zero size asset means the file was already there, thus it was skipped.
+        return Fetched(
+            files=tuple(target_paths),
+            bytes_downloaded=sum(sizes),
+            skipped=sum(1 for size in sizes if size == 0),
+        )
+
+    def _fetch_combined(
+        self, assets: Sequence[Asset], destination: Path, temp_dir: Path | None
+    ) -> Fetched:
+        """Every message concatenated into one GRIB2 file.
+        GRIB2 messages are self-delimiting, each carrying its own length and
+        terminator, so joining the bytes of several files is a valid file.
+        The parts are downloaded into temp_dir if given, before concatenating.
+        """
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        parent = destination.parent if temp_dir is None else temp_dir
+        staging_dir = self._private_directory(parent)
+
+        try:
+            parts = [staging_dir / f"{index:08d}.part" for index in range(len(assets))]
+            sizes = self._fetch_all(assets, parts, staging_dir)
+
+            # Merge into the staging directory first, never straight into
+            # the destination. Writing there directly would leave a partial
+            # file under the final name if the process died mid-merge.
+            merged = staging_dir / "merged"
+            with merged.open("wb") as out:
+                for part in parts:
+                    with part.open("rb") as chunk:
+                        shutil.copyfileobj(chunk, out)
+
+            # os.replace is atomic within a filesystem: the destination holds
+            # either the old file or the whole new one, never half.
+            os.replace(merged, destination)
+        finally:
+            # Remove temp files.
+            shutil.rmtree(staging_dir, ignore_errors=True)
+
+        return Fetched(files=(destination,), bytes_downloaded=sum(sizes))
+
+    def _fetch_all(
+        self, assets: Sequence[Asset], targets: Sequence[Path], staging: Path
+    ) -> list[int]:
+        """Fetch every asset into its target, in parallel."""
+        if len(assets) != len(targets):
+            raise ValueError(
+                f"{len(assets)} assets but {len(targets)} targets"
+            )
+
+        # Num workers: As specified, or number of assets to get.
+        workers = max(1, min(self._max_workers, len(assets)))
+
+        def fetch(asset: Asset, target: Path) -> int:
+            return self._fetch_one(asset, target, staging)
+
+        with ThreadPoolExecutor(max_workers=workers) as pool:
+            # Returns list of download sizes for the assets.
+            return list(pool.map(fetch, assets, targets))
+
+    def _fetch_one(self, asset: Asset, target: Path, staging: Path) -> int:
+        """Fetch one asset, write it to the specified target.
+        Returns the bytes transferred, so zero means the file was already there
+        and nothing had to be fetched.
+        """
+        if already_complete(target, asset.size):
+            logger.debug("skipping %s, already downloaded", target)
+            return 0
+
+        # Write into a partial/staging directory instead of the target path.
+        # This makes sure two processes downloading the same asset to not have
+        # a race condition on the file access and write. After downloading,
+        # os.replace() makes an atomic move into the target file.
+        payload = self._get_with_retries(asset)
+        partial = staging / temp_name(target.name)
+        partial.write_bytes(payload)
+        os.replace(partial, target)
+        logger.debug("wrote %s (%d bytes)", target, len(payload))
+        return len(payload)
+
+    def _get_with_retries(self, asset: Asset) -> bytes:
+        """Fetch one path, retrying failures with exponential backoff."""
+        attempt = 1
+        while True:
+            try:
+                return self._http.get_asset(asset.path)
+            except DownloadError as exc:
+                # Get limit of retries for the error code.
+                limit = 1
+                if exc.status is None or exc.status in RETRY_STATUS:
+                    # Set limit based on error code, if retrying makes actual sense.
+                    limit = self._max_attempts
+                if attempt >= limit:
+                    raise # Abort after N retries
+                delay = self._backoff * 2 ** (attempt - 1)
+                logger.warning(
+                    "attempt %d/%d for %s failed (%s), retrying in %.1fs",
+                    attempt, limit, asset.path, exc, delay,
+                )
+                time.sleep(delay)
+                attempt += 1
+
+    # Private helpers
+
+    def _private_directory(self, parent: Path) -> Path:
+        """A staging directory private to this process."""
+        parent.mkdir(parents=True, exist_ok=True)
+        staging = parent / f".dwdopen-{temp_name('merge')}-{secrets.token_hex(4)}"
+        staging.mkdir()
+        return staging
+
+    def _announce(
+        self, assets: Sequence[Asset], destination: Path, combine: CombineMode
+    ) -> None:
+        """Announcing the download before it starts (e.g. inform user about size)."""
+        asset_sizes = [asset.size for asset in assets if asset.size is not None]
+        total = sum(asset_sizes) if len(asset_sizes) == len(assets) else None
+        size = "unknown size" if total is None else human_size(total)
+
+        message = "downloading %d assets (%s) to %s, combine=%s"
+        args = (len(assets), size, destination, combine)
+
+        if total is not None and total >= LARGE_DOWNLOAD:
+            # Inform user about a large request.
+            logger.info(message + " - this is a large request.", *args)
+        else:
+            logger.info(message, *args)
+
+        # Also warn the user if there are multiple level types in the request and
+        # combine="all" is set: Multiple level types in a single GRIB file is valid
+        # but makes many problems with software reading GRIB files.
+        if combine == "all":
+            self._warn_about_mixed_level_types(assets)
+
+    @staticmethod
+    def _report(fetched: Fetched, combine: CombineMode) -> None:
+        """Report the finished download.
+        """
+        size = human_size(fetched.bytes_downloaded)
+        if fetched.skipped:
+            size += f", {fetched.skipped} already downloaded"
+        if combine == "none":
+            logger.info(
+                "saved %d files (%s) to %s",
+                len(fetched.files), size, fetched.files[0].parent,
+            )
+        else:
+            logger.info("saved %s (%s)", fetched.files[0], size)
+
+    @staticmethod
+    def _warn_about_mixed_level_types(assets: Sequence[Asset]) -> None:
+        """Warn when one file will hold several vertical coordinate types.
+        Concatenating them is legal GRIB2 and stays allowed, but readers do
+        struggle with it. CDO in particular wants one vertical axis per file.
+        """
+        kinds = {
+            asset.level_type.code
+            for asset in assets
+            if asset.level_type is not None
+        }
+        # A 2-D field alongside a 3-D one is also problematic.
+        if any(asset.level_type is None for asset in assets):
+            kinds.add(-1)
+        if len(kinds) > 1:
+            named = ", ".join(
+                "surface" if code == -1 else str(LevelType.of(code))
+                for code in sorted(kinds)
+            )
+            logger.warning(
+                "combining several level types into one file (%s). This is valid "
+                "GRIB2, but some readers want one vertical axis per file. "
+                "cdo splitzaxis, or combine=\"none\", separates them",
+                named,
+            )

dwdopen/_fileserver/http.py

@@ -0,0 +1,60 @@
+"""HTTP access to the Open Data file server."""
+
+from __future__ import annotations
+
+import httpx
+
+from dwdopen.exceptions import CatalogueUnavailableError, DownloadError
+
+__all__ = ["HttpClient"]
+
+
+class HttpClient:
+    """Fetches paths below a base URL and maps HTTP failures onto dwdopen errors.
+    Contains separate methods for fetching a listing and fetching an actual file,
+    to separate potential errors.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        *,
+        timeout: float = 30.0,
+        max_connections: int = 16,
+    ) -> None:
+        self._client = httpx.Client(
+            base_url=base_url.rstrip("/"),
+            timeout=timeout,
+            follow_redirects=True,
+            headers={"User-Agent": "dwdopen"},
+            limits=httpx.Limits(max_connections=max_connections),
+        )
+
+    def get_listing(self, path: str) -> str:
+        """Fetch a directory listing as text."""
+        try:
+            response = self._client.get(path)
+            response.raise_for_status()
+        except httpx.HTTPError as exc:
+            raise CatalogueUnavailableError(f"could not read {path}: {exc}") from exc
+        return response.text
+
+    def get_asset(self, path: str) -> bytes:
+        """Fetch one file.
+        The status is put in the error so the caller can decide how to recover
+        from the error based on the failed state.
+        """
+        try:
+            response = self._client.get(path)
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            raise DownloadError(
+                f"could not download {path}: {exc}",
+                status=exc.response.status_code,
+            ) from exc
+        except httpx.HTTPError as exc:
+            raise DownloadError(f"could not download {path}: {exc}") from exc
+        return response.content
+
+    def close(self) -> None:
+        self._client.close()

dwdopen/_fileserver/listing.py

@@ -0,0 +1,108 @@
+"""Parsing of nginx directory listings.
+
+This is the only module in the package that looks at the raw HTML
+returned by the OpenData DWD server.
+To the abstract layers, only ListingEntries are exposed.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from urllib.parse import unquote
+
+__all__ = ["ListingEntry", "parse_listing"]
+
+
+@dataclass(frozen=True)
+class ListingEntry:
+    """One row of a directory listing.
+    """
+
+    name: str
+    """The URL-decoded path."""
+
+    is_dir: bool
+
+    modified: datetime | None = None
+    """In UTC. Used to see if files have settled: We only accept a list entry
+    if it already exists on the server for a minute or so. Otherwise, the writing
+    might not have finished on the server, or, as DWD has multiple servers that
+    are not exactly in sync, we might later want to request that file from a
+    server that still does not have the file yet.
+
+     Might be not available if entry is a directory."""
+
+    size: int | None = None
+    """Bytes. None for directories."""
+
+
+# opendata.dwd.de serves nginx autoindex: one entry per line inside a single
+# <pre>:
+#
+#   <a href="2026-09-15T06%3A00/">2026-09-15T06:00/</a>   15-Sep-2026 08:42:25   -
+#   <a href="PT000H00M.grib2">PT000H00M.grib2</a>         16-Sep-2026 02:40:29   877797
+#
+# The href name should be parsed, as the displayed name might be truncated.
+# Date and size are optional in this pattern. Thanks to Claude for the regex :)
+_ENTRY = re.compile(
+    r'<a\s+href="(?P<href>[^"]*)"[^>]*>.*?</a>'
+    r"(?:\s+(?P<date>\d{2}-[A-Za-z]{3}-\d{4}\s+\d{2}:\d{2}:\d{2})"
+    r"\s+(?P<size>-|\d+))?",
+    re.IGNORECASE,
+)
+
+# Define months, can't parse them from strptime as that one is locale based.
+_MONTHS = {
+    "jan": 1, "feb": 2, "mar": 3, "apr": 4, "may": 5, "jun": 6,
+    "jul": 7, "aug": 8, "sep": 9, "oct": 10, "nov": 11, "dec": 12,
+}
+
+
+def parse_listing(html: str) -> list[ListingEntry]:
+    """Parse one nginx autoindex page.
+
+    The parent dir entry is skipped; every other entry is a child of the listed
+    directory.
+    An unreadable page yields an empty list. The caller should know which
+    directories have children. An empty page might be as valid, if there are
+    no files yet for the chosen run.
+    """
+    entries: list[ListingEntry] = []
+    # Iterate through regex occurrences; each match is a file or folder.
+    for match in _ENTRY.finditer(html):
+        href = match.group("href")
+        if href == "../":
+            continue
+
+        size = match.group("size")
+        entries.append(
+            ListingEntry(
+                name=unquote(href.rstrip("/")),
+                is_dir=href.endswith("/"),
+                modified=_parse_date(match.group("date")),
+                size=None if size in (None, "-") else int(size), # None for directories
+            )
+        )
+    return entries
+
+
+def _parse_date(value: str | None) -> datetime | None:
+    """Parse a listing date, given for example by "16-Sep-2026 02:40:29" as UTC.
+
+    We assume the timestamps are always UTC and not locale dependent.
+    Returns None if parsing fails.
+    """
+    if value is None:
+        return None
+    day, month, rest = value.split("-", 2)
+    year, clock = rest.split(maxsplit=1)
+    hour, minute, second = clock.split(":")
+    try:
+        return datetime(
+            int(year), _MONTHS[month.lower()], int(day),
+            int(hour), int(minute), int(second), tzinfo=UTC,
+        )
+    except (KeyError, ValueError):
+        return None

dwdopen/_fileserver/naming.py

@@ -0,0 +1,70 @@
+"""Turning a server path into a local file name.
+The name is derived from the asset's key/token pairs, so it is deterministic.
+"""
+
+from __future__ import annotations
+
+import os
+import secrets
+from pathlib import Path
+
+from dwdopen._fileserver.paths import Segment
+
+__all__ = ["already_complete", "local_name", "temp_name"]
+
+_SAFE = set(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-"
+)
+"""Allowed characters for file names. Characters that both Linux and Windows
+accept and also removed some weird chars that might be awkward in shells.
+(In practice, this is mainly about the ':' character on Windows)
+"""
+
+
+def local_name(keys: tuple[Segment, ...]) -> str:
+    """Build the file name for one asset from its path keys.
+
+    m/icon-eu, p/T_2M, r/2026-09-18T09:00, s/PT006H00M.grib2 becomes
+    ``m-icon-eu_p-T_2M_r-2026-09-18T09%3A00_s-PT006H00M.grib2``.
+    """
+    return "_".join(f"{key}-{_escape(token)}" for key, token in keys)
+
+
+def temp_name(final: str) -> str:
+    """Build a partial-download name that no other writer will pick.
+    The temporary name carries the process id and a random token, so
+    multiple download processes won't interfere.
+    """
+    return f"{final}.{os.getpid()}.{secrets.token_hex(4)}.part"
+
+
+def _escape(token: str) -> str:
+    """Escape characters not in _SAFE list."""
+    return "".join(c if c in _SAFE else f"%{ord(c):02X}" for c in token)
+
+
+GRIB_MAGIC = b"GRIB"
+GRIB_TERMINATOR = b"7777"
+
+
+def already_complete(target: Path, expected_size: int | None) -> bool:
+    """Whether a file on disk can be trusted as a finished download.
+    Two checks: The size has to match what the catalogue listed,
+    and the file has to look like GRIB2: every message opens with "GRIB" and
+    closes with "7777".
+    """
+    if expected_size is None:
+        return False
+    try:
+        if target.stat().st_size != expected_size:
+            return False
+        if expected_size < len(GRIB_MAGIC) + len(GRIB_TERMINATOR):
+            return False
+        with target.open("rb") as handle:
+            if handle.read(4) != GRIB_MAGIC:
+                return False
+            handle.seek(-4, os.SEEK_END)
+            return handle.read(4) == GRIB_TERMINATOR
+    except OSError:
+        # Missing or unreadable.
+        return False

dwdopen/_fileserver/paths.py

@@ -0,0 +1,48 @@
+"""Building URL paths for accessing the server from ordered key/token pairs."""
+
+from __future__ import annotations
+
+__all__ = ["Segment", "build_path"]
+
+Segment = tuple[str, str]
+"""One path key and its token value, e.g. ("lvt1", "100")."""
+
+V1_ROOT = "weather/nwp/v1"
+
+
+def build_path(
+    *segments: Segment,
+    key: str | None = None,
+    directory: bool = True,
+) -> str:
+    """Join key/value pairs into a path below the v1 root.
+    Each key value pair describes one file attribute, for example the model,
+    level type, ensemble member and so on.
+    Adding a key without a token value returns a directory listing all possible
+    values for that key: build_path(("m", "icon-eu"), ("p", "T"),
+    key="lvt1") gives the directory whose entries are the available level type codes.
+
+    The builder does not know or check the order of the keys. DWD's order is
+    m, p, [wvl1], [lvt1, lv1], r, [e], s, where [] keys are optional, for example
+    wvl1 is the wavelength for ICON ART, lvt1/lv1 is the level type and level value
+    for 3-D grids (absent for 2-D), and e is the ensemble member for ensemble forecasts.
+
+    DWD's newsletter of 2 September 2026 documents only m, p, lvt1, lv1, r, e, s
+    and states that lvt1/lv1 are omitted for single-level parameters. wvl1 is
+    not in that list at all: it appears under ICON-ART parameters on the server
+    but is documented not in that document.
+    Therefore, we allow unknown keys to be passed through rather than validating against
+    a fixed set of keys (m, r, e, s...). This also allows us in future to
+    support further
+    keys directly without having to change any code here.
+
+    Directories get a trailing slash because nginx answers 301 without one.
+    """
+    parts = [V1_ROOT]
+    for segment_key, token in segments:
+        parts.append(segment_key)
+        parts.append(token)
+    if key is not None:
+        parts.append(key)
+    path = "/".join(parts)
+    return f"{path}/" if directory else path