parseo 0.4.4__py3-none-any.whl

parseo/schemas/eumetsat/metop_filename_v1_0_0.json

@@ -0,0 +1,68 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "schema_id": "eumetsat:metop",
+  "schema_version": "1.0.0",
+  "status": "current",
+  "description": "EUMETSAT Metop product filename (extension optional).",
+  "fields": {
+    "platform": {
+      "type": "string",
+      "enum": [
+        "M01",
+        "M02",
+        "M03"
+      ],
+      "description": "Spacecraft unit"
+    },
+    "instrument": {
+      "type": "string",
+      "enum": [
+        "AVHR",
+        "IASI",
+        "ASCAT"
+      ],
+      "description": "Instrument short name"
+    },
+    "processing_level": {
+      "type": "string",
+      "enum": [
+        "L1B",
+        "L2"
+      ],
+      "description": "Processing level"
+    },
+    "start_datetime": {
+      "type": "string",
+      "pattern": "^\\d{8}T\\d{6}Z$",
+      "description": "Acquisition start time (UTC, YYYYMMDDTHHMMSSZ)"
+    },
+    "end_datetime": {
+      "type": "string",
+      "pattern": "^\\d{8}T\\d{6}Z$",
+      "description": "Acquisition end time (UTC, YYYYMMDDTHHMMSSZ)"
+    },
+    "orbit_number": {
+      "type": "string",
+      "pattern": "^O\\d{6}$",
+      "description": "Orbit number"
+    },
+    "collection": {
+      "type": "string",
+      "pattern": "^C\\d{2}$",
+      "description": "Collection identifier"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d{2}$",
+      "description": "Product version"
+    },
+    "extension": {
+      "type": "string",
+      "enum": [
+        "nc"
+      ],
+      "description": "File extension without leading dot"
+    }
+  },
+  "template": "{platform}_{instrument}_{processing_level}_{start_datetime}_{end_datetime}_{orbit_number}_{collection}_V{version}[.{extension}]"
+}

parseo/schemas/eumetsat/mtg_filename_v1_0_0.json

@@ -0,0 +1,65 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "schema_id": "eumetsat:mtg",
+  "schema_version": "1.0.0",
+  "status": "current",
+  "description": "EUMETSAT MTG product filename (extension optional).",
+  "fields": {
+    "platform": {
+      "type": "string",
+      "enum": [
+        "MTG-I1",
+        "MTG-I2",
+        "MTG-S1",
+        "MTG-S2"
+      ],
+      "description": "Spacecraft unit"
+    },
+    "instrument": {
+      "type": "string",
+      "enum": [
+        "FCI",
+        "LI",
+        "IRS",
+        "UVN"
+      ],
+      "description": "Instrument short name"
+    },
+    "product_type": {
+      "type": "string",
+      "pattern": "^[A-Z0-9_]{2,10}$",
+      "description": "Product short name"
+    },
+    "processing_level": {
+      "type": "string",
+      "enum": [
+        "L1",
+        "L2"
+      ],
+      "description": "Processing level"
+    },
+    "start_datetime": {
+      "type": "string",
+      "pattern": "^\\d{8}T\\d{6}Z$",
+      "description": "Acquisition start time (UTC, YYYYMMDDTHHMMSSZ)"
+    },
+    "end_datetime": {
+      "type": "string",
+      "pattern": "^\\d{8}T\\d{6}Z$",
+      "description": "Acquisition end time (UTC, YYYYMMDDTHHMMSSZ)"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^\\d{2}$",
+      "description": "Product version"
+    },
+    "extension": {
+      "type": "string",
+      "enum": [
+        "nc"
+      ],
+      "description": "File extension without leading dot"
+    }
+  },
+  "template": "{platform}_{instrument}_{product_type}_{processing_level}_{start_datetime}_{end_datetime}_V{version}[.{extension}]"
+}

parseo/schemas/nasa/modis_filename_v1_0_0.json

@@ -0,0 +1,54 @@
+{
+  "schema_id": "nasa:modis",
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "schema_version": "1.0.0",
+  "status": "current",
+  "stac_version": "1.1.0",
+
+  "description": "MODIS product filename (extension optional).",
+  "fields": {
+    "platform": {
+      "type": "string",
+      "enum": ["MOD","MYD","MCD"],
+      "description": "Spacecraft platform (MOD=Terra, MYD=Aqua, MCD=Combined)",
+      "stac_map": {
+        "MOD": {
+          "platform": "Terra",
+          "instruments": ["MODIS"]
+        },
+        "MYD": {
+          "platform": "Aqua",
+          "instruments": ["MODIS"]
+        },
+        "MCD": {
+          "platform": "Combined",
+          "instruments": ["MODIS"]
+        }
+      }
+    },
+    "product": {
+      "type": "string",
+      "pattern": "^\\d{2}$",
+      "description": "Two-digit product code (e.g., '09' for Surface Reflectance)"
+    },
+    "variant": {
+      "type": "string",
+      "pattern": "^[A-Za-z0-9]+$",
+      "description": "Product variant or sub-type (e.g., GA, A1)"
+    },
+    "acq_date": {"type": "string", "pattern": "^\\d{7}$", "description": "Acquisition date (YYYYDDD)"},
+    "tile_id": {"type": "string", "pattern": "^h\\d{2}v\\d{2}$", "description": "MODIS sinusoidal tile"},
+    "collection": {"type": "string", "pattern": "^\\d{3}$", "description": "Collection number"},
+    "proc_date": {"type": "string", "pattern": "^\\d{13}$", "description": "Production date (YYYYDDDHHMMSS)"},
+    "extension": {"type": "string", "pattern": "^[A-Za-z0-9]+$", "description": "File extension."}
+  },
+  "template": "{platform}{product}{variant}.A{acq_date}.{tile_id}.{collection}.{proc_date}[.{extension}]",
+  "examples": [
+    "MOD09GA.A2021123.h18v04.006.2021132234506.hdf",
+    "MYD11A1.A2021001.h21v09.061.2021015190034"
+  ],
+  "counter_examples": [
+    "MOD09GA.X2021123.h18v04.006.2021132234506.hdf",
+    "MOD09GA.A2021123.H18V04.006.2021132234506.hdf"
+  ]
+}

parseo/schemas/usgs/landsat/landsat_filename_v1_0_0.json

@@ -0,0 +1,50 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "schema_id": "usgs:landsat:landsat",
+  "schema_version": "1.0.0",
+  "status": "current",
+  "stac_version": "1.1.0",
+
+  "description": "Landsat Collection 2 scene identifier (extension optional).",
+  "fields": {
+    "platform": {
+      "type": "string",
+      "pattern": "^(LT04|LT05|LE07|LC08|LC09)$",
+      "description": "Platform and sensor code",
+      "stac_map": {
+        "LT04": {
+          "platform": "landsat-4",
+          "instruments": ["TM"]
+        },
+        "LT05": {
+          "platform": "landsat-5",
+          "instruments": ["TM"]
+        },
+        "LE07": {
+          "platform": "landsat-7",
+          "instruments": ["ETM+"]
+        },
+        "LC08": {
+          "platform": "landsat-8",
+          "instruments": ["OLI_TIRS"]
+        },
+        "LC09": {
+          "platform": "landsat-9",
+          "instruments": ["OLI_TIRS"]
+        }
+      }
+    },
+    "processing_level": {"type": "string", "enum": ["L1TP", "L1GT", "L1GS", "L2SP", "L2SR"], "description": "Processing level"},
+    "wrs_path": {"type": "string", "pattern": "^\\d{3}$", "description": "WRS-2 path"},
+    "wrs_row": {"type": "string", "pattern": "^\\d{3}$", "description": "WRS-2 row"},
+    "acq_date": {"type": "string", "pattern": "^\\d{8}$", "description": "Acquisition date (YYYYMMDD)"},
+    "proc_date": {"type": "string", "pattern": "^\\d{8}$", "description": "Processing date (YYYYMMDD)"},
+    "collection_number": {"type": "string", "pattern": "^\\d{2}$", "description": "Collection number"},
+    "tier": {"type": "string", "enum": ["T1", "T2", "RT"], "description": "Tier classification"},
+    "extension": {"type": "string", "pattern": "^[A-Za-z0-9]+$", "description": "File extension without leading dot"}
+  },
+  "template": "{platform}_{processing_level}_{wrs_path}{wrs_row}_{acq_date}_{proc_date}_{collection_number}_{tier}[.{extension}]",
+  "examples": [
+    "LC08_L1TP_190026_20200101_20200114_02_T1.tar"
+  ]
+}

parseo/stac_http.py

@@ -0,0 +1,267 @@
+"""Helpers for querying STAC APIs via the standard library.
+
+The routines in this module intentionally rely only on :mod:`urllib` from the
+Python standard library to avoid pulling in heavier dependencies.  For a
+``pystac-client`` based alternative see :mod:`parseo.stac_scraper`.  All helper
+functions require explicitly passing the ``base_url`` of the STAC service.
+"""
+from __future__ import annotations
+
+from collections.abc import Iterable
+from functools import lru_cache
+import itertools
+import json
+from pathlib import Path
+import re
+from string import Template
+from typing import Union
+import urllib.error
+from urllib.parse import urljoin
+from urllib.parse import urlparse
+import urllib.request
+
+
+class StacClientError(IOError):
+    """Raised when a STAC API request fails."""
+
+
+def _norm_collection_id(collection_id: str, *, base_url: str) -> str:
+    """Resolve ``collection_id`` to the official ID from the STAC API."""
+
+    norm = re.sub(r"[^A-Za-z0-9]", "", collection_id).upper()
+    for cid in _list_collections_cached(base_url):
+        if re.sub(r"[^A-Za-z0-9]", "", cid).upper() == norm:
+            return cid
+    return collection_id
+
+
+def _norm_base(base_url: str) -> str:
+    """Return ``base_url`` with exactly one trailing slash."""
+    return base_url.rstrip("/") + "/"
+
+
+def _read_json(url: str) -> dict:
+    try:
+        with urllib.request.urlopen(url) as resp:  # type: ignore[call-arg]
+            return json.load(resp)
+    except urllib.error.URLError as err:
+        raise StacClientError(f"Could not connect to {url}: {err.reason}") from err
+
+
+def list_collections_http(base_url: str, *, deep: bool = False) -> list[str]:
+    """Return available collection IDs from the STAC API using ``urllib``.
+
+    This lightweight helper performs raw HTTP requests without requiring
+    third-party libraries.  If ``deep`` is ``True`` the function follows
+    ``rel='child'`` links and gathers collection IDs from nested catalogs as
+    well.  For a variant powered by ``pystac-client`` see
+    :func:`parseo.stac_scraper.list_collections_client`.
+    """
+    base = _norm_base(base_url)
+
+    # First, fetch the standard ``/collections`` endpoint which should expose
+    # top-level collections for STAC APIs.
+    url = urljoin(base, "collections")
+    try:
+        data = _read_json(url)
+    except urllib.error.HTTPError as err:
+        raise StacClientError(f"HTTP error {err.code} for {err.geturl()}") from err
+
+    collections = {c["id"] for c in data.get("collections", [])}
+
+    if not deep:
+        return sorted(collections)
+
+    # Breadth-first traversal of child links starting from the catalog root.
+    to_visit = [base]
+    visited: set[str] = set()
+
+    while to_visit:
+        cur = to_visit.pop()
+        if cur in visited:
+            continue
+        visited.add(cur)
+        try:
+            data = _read_json(cur)
+        except urllib.error.HTTPError as err:
+            raise StacClientError(f"HTTP error {err.code} for {err.geturl()}") from err
+
+        # Collect IDs if this document represents a collection or includes
+        # embedded collections.
+        if data.get("type") == "Collection":
+            cid = data.get("id")
+            if cid:
+                collections.add(cid)
+        for coll in data.get("collections", []):
+            cid = coll.get("id")
+            if cid:
+                collections.add(cid)
+
+        # Queue any child links for further traversal.
+        for link in data.get("links", []):
+            if link.get("rel") == "child":
+                href = link.get("href")
+                if href:
+                    base_cur = cur if cur.endswith("/") else cur + "/"
+                    to_visit.append(urljoin(base_cur, href))
+
+    return sorted(collections)
+
+
+@lru_cache(maxsize=32)
+def _list_collections_cached(base_url: str) -> tuple[str, ...]:
+    """Cached helper returning collection IDs for ``base_url``."""
+    # Use a deep listing to include collections nested in child catalogs.
+    # This ensures that `_norm_collection_id` can resolve aliases for
+    # collections not present at the top-level of the STAC service.
+    return tuple(list_collections_http(base_url, deep=True))
+
+
+def iter_asset_filenames(
+    collection_id: str,
+    *,
+    base_url: str,
+    limit: int = 100,
+    asset_role: Union[str, None] = None,
+) -> Iterable[str]:
+    """Yield asset filenames from items of a collection.
+
+    Pagination links (``rel="next"``) are followed until all pages are
+    exhausted or ``limit`` filenames have been yielded.  When ``asset_role`` is
+    provided, only assets declaring that role are considered.  Resulting
+    filenames are sanitized: directory components are stripped and any
+    characters outside ``[A-Za-z0-9._-]`` are replaced with ``_``.
+    """
+    base = _norm_base(base_url)
+    collection_id = _norm_collection_id(collection_id, base_url=base)
+    url = urljoin(base, f"collections/{collection_id}/items?limit={limit}")
+    remaining = limit
+    first_request = True
+    while url and remaining > 0:
+        try:
+            data = _read_json(url)
+        except urllib.error.HTTPError as err:
+            if first_request and err.code == 404:
+                raise StacClientError(
+                    f"Collection '{collection_id}' not found at {base}. "
+                    "Use `parseo stac-sample <collection> --stac-url <url>` with a valid collection ID."
+                ) from err
+            raise StacClientError(f"HTTP error {err.code} for {err.geturl()}") from err
+        first_request = False
+        for feat in data.get("features", []):
+            props = feat.get("properties", {})
+            assets = feat.get("assets", {})
+            for asset in assets.values():
+                if asset_role and asset_role not in (asset.get("roles") or []):
+                    continue
+                title = asset.get("title")
+                href = asset.get("href")
+                filename = None
+                # Many assets use a generic title like "Product" which does not
+                # convey the actual filename.  In such cases prefer extracting
+                # the name from the href.  Only fall back to the title if it is
+                # present and not the generic "Product".
+                if title and title.strip().lower() != "product":
+                    filename = title
+                elif href:
+                    if "$" in href:
+                        href_sub = Template(href).safe_substitute(props)
+                        if re.search(r"\$(?!value\b)\w+", href_sub):
+                            continue
+                        href = href_sub
+                    m = re.search(r"Products\('([^']+)'\)", href)
+                    if m:
+                        filename = m.group(1)
+                    else:
+                        parsed_url = urlparse(href)
+                        path = Path(parsed_url.path)
+                        filename = path.name
+                        if not path.suffix:
+                            filename += ".dat"
+                else:
+                    continue
+                if filename.startswith("$"):
+                    continue
+                filename = Path(filename).name
+                filename = re.sub(r"[^A-Za-z0-9._-]", "_", filename)
+                yield filename
+                remaining -= 1
+                if remaining == 0:
+                    return
+        url = None
+        for link in data.get("links", []):
+            if link.get("rel") == "next":
+                url = link.get("href")
+                break
+
+def iter_collection_tree(
+    collection_id: str,
+    *,
+    base_url: str,
+    limit: int = 100,
+    asset_role: Union[str, None] = None,
+) -> Iterable[tuple[str, str]]:
+    """Yield ``(collection_id, filename)`` pairs for all leaf collections.
+
+    The function follows ``links`` with ``rel="child"`` starting from
+    ``collection_id`` until it reaches leaf collections.  For each leaf it
+    yields filenames from :func:`iter_asset_filenames`.  The ``asset_role``
+    parameter is forwarded to :func:`iter_asset_filenames`.
+    """
+    base = _norm_base(base_url)
+    collection_id = _norm_collection_id(collection_id, base_url=base)
+    url = urljoin(base, f"collections/{collection_id}")
+    try:
+        data = _read_json(url)
+    except urllib.error.HTTPError as err:
+        if err.code == 404:
+            raise StacClientError(
+                f"Collection '{collection_id}' not found at {base}. "
+                "Use `parseo stac-sample <collection> --stac-url <url>` with a valid collection ID."
+            ) from err
+        raise StacClientError(f"HTTP error {err.code} for {err.geturl()}") from err
+
+    children = [
+        link.get("href", "").rstrip("/").split("/")[-1]
+        for link in data.get("links", [])
+        if link.get("rel") == "child"
+    ]
+
+    if children:
+        for child in children:
+            yield from iter_collection_tree(
+                child, base_url=base, limit=limit, asset_role=asset_role
+            )
+    else:
+        for fn in itertools.islice(
+            iter_asset_filenames(
+                collection_id, base_url=base, limit=limit, asset_role=asset_role
+            ),
+            limit,
+        ):
+            yield collection_id, fn
+
+
+def sample_collection_filenames(
+    collection_id: str,
+    samples: int = 5,
+    *,
+    base_url: str,
+    asset_role: Union[str, None] = None,
+) -> dict[str, list[str]]:
+    """Return ``samples`` filenames for each leaf collection.
+
+    ``collection_id`` may be the official STAC ID or any case/format variant
+    resolvable via :func:`list_collections_http`.  When ``collection_id`` has
+    child collections, a sample is collected from each leaf.  Only assets whose
+    ``roles`` include ``asset_role`` are returned when the parameter is
+    supplied.
+    """
+    out: dict[str, list[str]] = {}
+    for cid, fn in iter_collection_tree(
+        collection_id, base_url=base_url, limit=samples, asset_role=asset_role
+    ):
+        lst = out.setdefault(cid, [])
+        if len(lst) < samples:
+            lst.append(fn)
+    return out

parseo/stac_scraper.py

@@ -0,0 +1,115 @@
+"""STAC helpers backed by ``pystac-client``.
+
+This module mirrors the utilities in :mod:`parseo.stac_http` but relies on
+``pystac-client`` for STAC catalog traversal.  Use these helpers when the extra
+features of ``pystac-client`` are required.  For a lightweight alternative that
+only depends on the Python standard library see :mod:`parseo.stac_http`.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Union
+from urllib.parse import urlparse
+
+def list_collections_client(base_url: str, *, deep: bool = False) -> list[str]:
+    """Return collection IDs from a STAC API using ``pystac-client``.
+
+    Parameters mirror :func:`parseo.stac_http.list_collections_http` but
+    this variant requires the optional ``pystac-client`` dependency.  It is
+    suitable when more advanced STAC handling is needed, at the cost of pulling
+    in the external library.
+
+    Raises
+    ------
+    ImportError
+        If ``pystac-client`` is not installed.
+    """
+    try:
+        from pystac_client import Client
+    except Exception as exc:  # pragma: no cover - exercised when dependency missing
+        raise ImportError(
+            "pystac-client is required for list_collections_client"
+        ) from exc
+
+    client = Client.open(base_url)
+    collections = {c.id for c in client.get_collections()}
+    if not deep:
+        return sorted(collections)
+
+    # Breadth-first traversal of child catalogs.
+    to_visit = list(client.get_children())
+    visited: set[str] = set()
+    while to_visit:
+        child = to_visit.pop()
+        href = getattr(child, "href", None) or getattr(child, "target", None)
+        if not href or href in visited:
+            continue
+        visited.add(href)
+        sub_client = Client.open(href)
+        collections.update(c.id for c in sub_client.get_collections())
+        to_visit.extend(sub_client.get_children())
+
+    return sorted(collections)
+
+
+def search_stac_and_download(
+    *,
+    stac_url: str,
+    collections: list[str],
+    bbox: Union[list[float], tuple[float, float, float, float]],
+    datetime: str,
+    dest_dir: Union[str, Path],
+) -> Path:
+    """Download the first asset matching a STAC search.
+
+    The search is performed via :mod:`pystac-client` and the asset is retrieved
+    with :mod:`requests`. ``dest_dir`` is created if needed and the path to the
+    downloaded file is returned.
+
+    Raises
+    ------
+    ImportError
+        If ``pystac-client`` or ``requests`` is not installed.
+    FileNotFoundError
+        If the STAC search yields no downloadable assets or all downloads
+        fail.
+    """
+
+    try:
+        from pystac_client import Client
+    except Exception as exc:  # pragma: no cover - exercised when dependency missing
+        raise ImportError(
+            "pystac-client is required for search_stac_and_download"
+        ) from exc
+
+    try:
+        import requests
+    except Exception as exc:  # pragma: no cover - exercised when dependency missing
+        raise ImportError(
+            "requests is required for search_stac_and_download"
+        ) from exc
+
+    client = Client.open(stac_url)
+    search = client.search(collections=collections, bbox=bbox, datetime=datetime)
+    for item in search.items():
+        for asset in item.assets.values():
+            href = getattr(asset, "href", None)
+            if not href:
+                continue
+            name = getattr(asset, "title", None)
+            if not name:
+                name = Path(urlparse(href).path).name
+            dest_dir_path = Path(dest_dir)
+            dest_dir_path.mkdir(parents=True, exist_ok=True)
+            dest_path = dest_dir_path / name
+            try:
+                with requests.get(href, stream=True) as resp:
+                    resp.raise_for_status()
+                    with open(dest_path, "wb") as fh:
+                        for chunk in resp.iter_content(chunk_size=8192):
+                            if chunk:
+                                fh.write(chunk)
+                return dest_path
+            except requests.HTTPError:
+                continue
+    raise FileNotFoundError("No matching assets found")