mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/harvest/stac.py

@@ -0,0 +1,384 @@
+"""Parse one STAC ``/collections`` response into public-source records.
+
+Records declare JSON because it is the supported STAC response encoding. Asset encodings remain
+in descriptive text. Missing optional fields are accepted. Text and identifiers are bounded by
+the shared harvest protocol; unusable records are skipped.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from mostlyright.data_harness.canonical import CanonicalJSONError, parse_foreign_json
+from mostlyright.data_harness.sources.catalog.harvest.ckan import MAX_HARVEST_RECORDS
+from mostlyright.data_harness.sources.catalog.harvest.protocol import (
+    MAX_RECORD_PUBLISHER,
+    MAX_RECORD_TEXT,
+    MAX_RECORD_TITLE,
+    CatalogHarvestError,
+    HarvestCursor,
+    HarvestedRecord,
+    HarvesterDescriptor,
+    HarvestPage,
+    HarvestResponseEvidence,
+    bounded_record_text,
+    carries_a_zone_offset,
+    declared_timestamp,
+    harvest_limits,
+    usable_https_uri,
+    usable_license_id,
+    usable_record_id,
+)
+from mostlyright.data_harness.sources.contracts import EvidenceReference
+
+STAC_LIMITS = harvest_limits("application/json", max_response_bytes=2 * 1024 * 1024)
+
+SUPPORTED_STAC_MAJOR = "1."
+
+ATTRIBUTION = "attribution"
+
+# SPDX identifiers, which is a different vocabulary from CKAN's licence register -- hence a second
+# table rather than a shared one. The mapping idiom is the same; the identifiers genuinely differ.
+STAC_LICENCE_RIGHTS: dict[str, tuple[str, tuple[str, ...], str | None]] = {
+    "cc0-1.0": ("approved", ("analysis", "publication"), None),
+    "public-domain": ("approved", ("analysis", "publication"), None),
+    "pddl-1.0": ("approved", ("analysis", "publication"), None),
+    "cc-by-4.0": ("conditional", ("analysis", "publication"), ATTRIBUTION),
+    "cc-by-sa-4.0": ("conditional", ("analysis", "publication"), ATTRIBUTION),
+    "odc-by-1.0": ("conditional", ("analysis", "publication"), ATTRIBUTION),
+}
+
+UNCLEAR_LICENCE_IDS = frozenset({"", "proprietary", "various", "other", "unknown"})
+
+
+def stac_licence_rights(license_id: str | None) -> tuple[str, tuple[str, ...], str | None]:
+    """Map a STAC SPDX licence identifier onto a rights status, allowed uses and an obligation."""
+
+    if license_id is None:
+        return ("unclear", (), None)
+    normalized = license_id.strip().lower()
+    if normalized in UNCLEAR_LICENCE_IDS:
+        return ("unclear", (), None)
+    return STAC_LICENCE_RIGHTS.get(normalized, ("unclear", (), None))
+
+
+class StacHarvester:
+    """Read a STAC ``/collections`` document. Takes bytes; never fetches."""
+
+    _DESCRIPTOR = HarvesterDescriptor(
+        protocol="stac",
+        harvester_id="stac.collections",
+        harvester_version="1.0.0",
+    )
+
+    @property
+    def descriptor(self) -> HarvesterDescriptor:
+        return self._DESCRIPTOR
+
+    def parse(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+    ) -> tuple[HarvestedRecord, ...]:
+        page = self.parse_page(
+            payload,
+            uri=uri,
+            observed_at=observed_at,
+            evidence=evidence,
+            response_evidence=None,
+            cursor=None,
+        )
+        if not page.records:
+            raise CatalogHarvestError(
+                "HARVEST_SHAPE", "stac.collections", "no collection carried a usable identity"
+            )
+        return page.records
+
+    def parse_page(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+        response_evidence: HarvestResponseEvidence | None,
+        cursor: HarvestCursor | None,
+    ) -> HarvestPage:
+        if cursor is not None and (cursor.protocol != "stac" or cursor.value != uri):
+            raise CatalogHarvestError("HARVEST_CURSOR", "stac.cursor", "cursor mismatch")
+        document = _document(payload)
+        collections = document.get("collections")
+        if not isinstance(collections, list):
+            raise CatalogHarvestError(
+                "HARVEST_SHAPE",
+                "stac.collections",
+                "a STAC collections response must carry a collections array",
+            )
+        if len(collections) > MAX_HARVEST_RECORDS:
+            raise CatalogHarvestError(
+                "HARVEST_RECORD_LIMIT",
+                "stac.collections",
+                f"a single response may not carry more than {MAX_HARVEST_RECORDS} collections",
+            )
+        skipped: list[str] = []
+        records: list[HarvestedRecord] = []
+        for position, collection in enumerate(collections):
+            if not isinstance(collection, dict):
+                raise CatalogHarvestError(
+                    "HARVEST_SHAPE",
+                    f"stac.collections[{position}]",
+                    "each collection must be an object",
+                )
+            _require_supported_version(collection, document, position)
+            record = _record(
+                collection,
+                uri=uri,
+                observed_at=observed_at,
+                evidence=evidence,
+                skipped=skipped,
+            )
+            if record is not None:
+                records.append(record)
+        next_uri = _next_link(document)
+        provider_count = _matched_count(document)
+        if not collections and next_uri is not None:
+            raise CatalogHarvestError(
+                "HARVEST_EMPTY_PAGE", "stac.collections", "an empty page may not continue"
+            )
+        return HarvestPage(
+            protocol="stac",
+            records=tuple(records),
+            next_cursor=(
+                HarvestCursor(protocol="stac", kind="next", value=next_uri)
+                if next_uri is not None
+                else None
+            ),
+            provider_count=provider_count,
+            count_basis="reported_total" if provider_count is not None else "not_reported",
+            skipped_record_ids=tuple(sorted(skipped)),
+            evidence=evidence,
+            response_evidence=response_evidence,
+        )
+
+
+def _next_link(document: dict[str, Any]) -> str | None:
+    links = document.get("links")
+    if links is None:
+        return None
+    if not isinstance(links, list):
+        raise CatalogHarvestError("HARVEST_SHAPE", "stac.links", "links must be an array")
+    found: list[str] = []
+    for position, link in enumerate(links):
+        if not isinstance(link, dict):
+            raise CatalogHarvestError(
+                "HARVEST_SHAPE", f"stac.links[{position}]", "each link must be an object"
+            )
+        if link.get("rel") != "next":
+            continue
+        href = usable_https_uri(link.get("href"))
+        if href is None:
+            raise CatalogHarvestError(
+                "HARVEST_CURSOR", f"stac.links[{position}].href", "next must be bounded HTTPS"
+            )
+        found.append(href)
+    if len(found) > 1:
+        raise CatalogHarvestError("HARVEST_CURSOR", "stac.links", "at most one next link is valid")
+    return found[0] if found else None
+
+
+def _matched_count(document: dict[str, Any]) -> int | None:
+    value = document.get("numberMatched")
+    context = document.get("context")
+    if value is None and isinstance(context, dict):
+        value = context.get("matched")
+    if value is None:
+        return None
+    if type(value) is not int or value < 0:
+        raise CatalogHarvestError("HARVEST_COUNT", "stac.matched", "must be nonnegative integer")
+    return value
+
+
+def _document(payload: bytes) -> dict[str, Any]:
+    # Fraction-tolerant on purpose. A STAC Collection is required by its own specification to carry
+    # `extent.spatial.bbox`, and every real bbox is fractional; so are `gsd`, `center_wavelength`,
+    # `proj:transform` and most `summaries` values. Parsing these with the integer-only canonical
+    # parser refuses every genuine /collections response. `parse_foreign_json` keeps the rest of the
+    # hardening -- duplicate keys, byte-order marks, strict UTF-8, NaN/Infinity, integer range --
+    # and no float reaches a `HarvestedRecord`: every field below is read through `_string`.
+    try:
+        document = parse_foreign_json(payload)
+    except CanonicalJSONError as error:
+        raise CatalogHarvestError(
+            "HARVEST_SHAPE",
+            "stac.response",
+            f"the response is not strict JSON: {error.detail}",
+        ) from None
+    if not isinstance(document, dict):
+        raise CatalogHarvestError(
+            "HARVEST_SHAPE",
+            "stac.response",
+            "a STAC response must be a JSON object",
+        )
+    return document
+
+
+def _require_supported_version(
+    collection: dict[str, Any],
+    document: dict[str, Any],
+    position: int,
+) -> None:
+    declared = collection.get("stac_version") or document.get("stac_version")
+    if not isinstance(declared, str) or not declared.startswith(SUPPORTED_STAC_MAJOR):
+        raise CatalogHarvestError(
+            "HARVEST_STAC_VERSION",
+            f"stac.collections[{position}].stac_version",
+            f"this harvester reads STAC {SUPPORTED_STAC_MAJOR}x, found {declared!r}",
+        )
+
+
+def _record(
+    collection: dict[str, Any],
+    *,
+    uri: str,
+    observed_at: str,
+    evidence: EvidenceReference,
+    skipped: list[str],
+) -> HarvestedRecord | None:
+    collection_id = usable_record_id(collection.get("id"))
+    if collection_id is None:
+        skipped.append("<collection without a usable id>")
+        return None
+    landing = usable_https_uri(_self_link(collection)) or usable_https_uri(
+        _collection_uri(uri, collection_id)
+    )
+    if landing is None:
+        skipped.append(collection_id)
+        return None
+    encodings = _asset_encodings(collection)
+    description = _string(collection.get("description")) or "No description published."
+    if encodings:
+        description = (
+            f"{description} Published asset encodings: {', '.join(encodings)}. "
+            "The harness reads this catalogue as JSON metadata; those encodings need a Reader."
+        )
+    title = _string(collection.get("title"))
+    return HarvestedRecord(
+        protocol="stac",
+        record_id=collection_id,
+        title=bounded_record_text(title, maximum=MAX_RECORD_TITLE) if title else collection_id,
+        description=bounded_record_text(description, maximum=MAX_RECORD_TEXT),
+        publisher=bounded_record_text(_publisher(collection), maximum=MAX_RECORD_PUBLISHER),
+        landing_uri=landing,
+        data_formats=("json",),
+        evidence=evidence,
+        license_id=usable_license_id(collection.get("license")),
+        license_uri=None,
+        declared_updated_at=_temporal_end(collection),
+    )
+
+
+def _publisher(collection: dict[str, Any]) -> str:
+    providers = collection.get("providers")
+    if isinstance(providers, list):
+        for provider in providers:
+            if isinstance(provider, dict):
+                name = _string(provider.get("name"))
+                if name is not None:
+                    return name
+    return "Unnamed publisher"
+
+
+def _self_link(collection: dict[str, Any]) -> str | None:
+    links = collection.get("links")
+    if not isinstance(links, list):
+        return None
+    for link in links:
+        if not isinstance(link, dict) or link.get("rel") != "self":
+            continue
+        href = _string(link.get("href"))
+        if href is not None and href.startswith("https://"):
+            return href
+    return None
+
+
+def _collection_uri(uri: str, collection_id: str) -> str | None:
+    if not uri.startswith("https://"):
+        return None
+    base = uri.split("?", 1)[0].rstrip("/")
+    return f"{base}/{collection_id}"
+
+
+def _asset_encodings(collection: dict[str, Any]) -> tuple[str, ...]:
+    """The media types this collection's item assets really publish, in declared order."""
+
+    assets = collection.get("item_assets")
+    if not isinstance(assets, dict):
+        return ()
+    encodings: list[str] = []
+    for value in assets.values():
+        if not isinstance(value, dict):
+            continue
+        media_type = _string(value.get("type"))
+        if media_type is not None and media_type not in encodings:
+            encodings.append(media_type)
+    return tuple(encodings[:16])
+
+
+def _temporal_end(collection: dict[str, Any]) -> str | None:
+    """The end of the first temporal interval, when the publisher declares a concrete one.
+
+    An open-ended interval leaves this ``None``. Reading it as "now" would invent a fact the
+    catalogue did not state, and cadence is one of the things the facts gate decides on.
+    """
+
+    extent = collection.get("extent")
+    if not isinstance(extent, dict):
+        return None
+    temporal = extent.get("temporal")
+    if not isinstance(temporal, dict):
+        return None
+    interval = temporal.get("interval")
+    if not isinstance(interval, list) or not interval:
+        return None
+    first = interval[0]
+    if not isinstance(first, list) or len(first) < 2:
+        return None
+    end = _string(first[1])
+    if end is None:
+        return None
+    return _stac_timestamp(end)
+
+
+def _stac_timestamp(value: str) -> str | None:
+    """Read a temporal-interval bound as a canonical UTC timestamp, or not at all.
+
+    STAC requires UTC, but the interval is publisher text and a real instance spells it however it
+    likes. The normalised result goes through ``declared_timestamp`` and becomes an undeclared
+    update time when it is not canonical -- without that, one collection's zone-bearing or
+    malformed bound refused every other collection in the same response. See that function.
+
+    The offset check runs before the fractional second is cut, because the offset rides on the
+    fraction and was being cut away with it -- see ``carries_a_zone_offset``.
+    """
+
+    if carries_a_zone_offset(value.strip()):
+        return None
+    text = value.strip()
+    if text.endswith("Z"):
+        text = text[:-1]
+    if "." in text:
+        head, _, fraction = text.partition(".")
+        text = f"{head}.{fraction[:6]}" if fraction[:6].isdigit() else head
+    if len(text) < 19:
+        return None
+    return declared_timestamp(f"{text}Z")
+
+
+def _string(value: Any) -> str | None:
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    return text or None