mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/harvest/__init__.py

@@ -0,0 +1,7 @@
+"""Harvesters that read open catalogue endpoints into structured public-source facts.
+
+This package carries no re-exports. Consumers import the concrete module -- ``harvest.protocol``
+for the transport, record shape and registry, ``harvest.ckan`` and the rest for one endpoint
+family each. Keeping ``__init__`` empty means adding a harvester never contends with an existing
+file, and it stops the transport and a parser from sharing a namespace.
+"""

mostlyright/data_harness/sources/catalog/harvest/ckan.py

@@ -0,0 +1,384 @@
+"""The CKAN harvester: read one ``package_search`` response into public-source records.
+
+CKAN is the most common open data-catalogue software, and its API is close to uniform across
+instances -- but only close. The base path is per-instance configuration rather than a constant:
+``data.gov.uk`` serves ``/api/action/`` and 301s on ``/api/3/action/``, while ``open.canada.ca``
+and ``data.humdata.org`` serve ``/api/3/action/``. This module therefore never constructs an
+endpoint URL; the caller supplies one and the transport validates it.
+
+Three field-level judgements are worth stating because all three are deliberate losses of
+information.
+
+**An unrecognised payload format is dropped, not carried.** ``DATA_FORMATS`` is a closed
+vocabulary of things the harness can actually read. An entry claiming a format the harness cannot
+open is worse than one claiming none, because the claim is what a later reader would trust.
+
+**An unrecognised licence maps to unclear.** Unclear escalates to a human; it never admits. The
+alternative -- guessing that an unfamiliar licence id probably means attribution -- would put the
+harness in the position of having granted permission nobody granted.
+
+**An out-of-bounds field degrades that field or that record, never the response.** CKAN's ``notes``
+has no server-side length limit, and a multi-thousand-character description is ordinary on
+data.gov.uk and open.canada.ca. Passing one to ``HarvestedRecord`` raised, the raise escaped
+``parse``, and one publisher's prose refused every other package on the page -- so free text is cut
+to the record bound and an unusable id, landing page or licence id degrades to ``skipped`` or to
+``None``. The rule and the reasoning live in ``protocol.bounded_record_text`` and
+``protocol.usable_record_id``, next to the timestamp case that first stated it.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from urllib.parse import parse_qs, urlsplit
+
+from mostlyright.data_harness.canonical import CanonicalJSONError, parse_foreign_json
+from mostlyright.data_harness.formats import FORMAT_MEDIA_TYPES
+from mostlyright.data_harness.sources.catalog.harvest.protocol import (
+    MAX_RECORD_PUBLISHER,
+    MAX_RECORD_TEXT,
+    MAX_RECORD_TITLE,
+    CatalogHarvestError,
+    HarvestCursor,
+    HarvestedRecord,
+    HarvesterDescriptor,
+    HarvestPage,
+    HarvestResponseEvidence,
+    bounded_record_text,
+    carries_a_zone_offset,
+    declared_timestamp,
+    harvest_limits,
+    usable_https_uri,
+    usable_license_id,
+    usable_record_id,
+)
+from mostlyright.data_harness.sources.contracts import DATA_FORMATS, EvidenceReference
+
+CKAN_LIMITS = harvest_limits("application/json", max_response_bytes=2 * 1024 * 1024)
+
+MAX_HARVEST_RECORDS = 1_000
+
+# Publisher spellings a CKAN instance uses that the format table does not carry. Each is a
+# non-canonical synonym rather than a format's real media type: ``application/csv`` and
+# ``application/json-lines`` are vendor spellings no registry lists, and ``jsonl`` is a bare
+# suffix published in the ``format`` field. They belong here because they are facts about how
+# publishers write, not about what the harness can read.
+_PUBLISHER_FORMAT_SYNONYMS = {
+    "application/csv": "csv",
+    "application/json-lines": "ndjson",
+    "jsonl": "ndjson",
+}
+
+# Every canonical media type is derived from ``FORMAT_MEDIA_TYPES`` rather than re-spelled. A
+# format's media types are the format table's to declare; restating them here is a second
+# declaration of the table, and one that drifts silently the moment a Reader family lands.
+_FORMAT_ALIASES: dict[str, str] = {
+    **{
+        media_type: name
+        for name, media_types in FORMAT_MEDIA_TYPES.items()
+        for media_type in media_types
+    },
+    **_PUBLISHER_FORMAT_SYNONYMS,
+}
+
+# One licence id, one rights disposition. `conditional` requires at least one obligation by
+# contract, which is exactly right for an attribution licence: the obligation is the attribution.
+ATTRIBUTION = "attribution"
+CKAN_LICENCE_RIGHTS: dict[str, tuple[str, tuple[str, ...], str | None]] = {
+    "cc-zero": ("approved", ("analysis", "publication"), None),
+    "cc0-1.0": ("approved", ("analysis", "publication"), None),
+    "odc-pddl": ("approved", ("analysis", "publication"), None),
+    "cc-by": ("conditional", ("analysis", "publication"), ATTRIBUTION),
+    "cc-by-4.0": ("conditional", ("analysis", "publication"), ATTRIBUTION),
+    "cc-by-sa": ("conditional", ("analysis", "publication"), ATTRIBUTION),
+    "uk-ogl": ("conditional", ("analysis", "publication"), ATTRIBUTION),
+    "odc-by": ("conditional", ("analysis", "publication"), ATTRIBUTION),
+}
+
+UNCLEAR_LICENCE_IDS = frozenset({"", "notspecified", "other-open", "other-closed", "unknown"})
+
+
+def ckan_licence_rights(license_id: str | None) -> tuple[str, tuple[str, ...], str | None]:
+    """Map a CKAN licence id onto a rights status, allowed uses and an obligation kind.
+
+    An absent, empty, ``notspecified`` or unrecognised licence maps to ``unclear``. That is the
+    status the facts gate escalates rather than guesses at.
+    """
+
+    if license_id is None:
+        return ("unclear", (), None)
+    normalized = license_id.strip().lower()
+    if normalized in UNCLEAR_LICENCE_IDS:
+        return ("unclear", (), None)
+    return CKAN_LICENCE_RIGHTS.get(normalized, ("unclear", (), None))
+
+
+def normalize_data_format(value: str) -> str | None:
+    """Return the closed-vocabulary format this publisher spelling means, or ``None``."""
+
+    if not isinstance(value, str):
+        return None
+    candidate = value.strip().lower().lstrip(".")
+    candidate = _FORMAT_ALIASES.get(candidate, candidate)
+    return candidate if candidate in DATA_FORMATS else None
+
+
+class CkanHarvester:
+    """Read a CKAN ``package_search`` response. Takes bytes; never fetches."""
+
+    _DESCRIPTOR = HarvesterDescriptor(
+        protocol="ckan",
+        harvester_id="ckan.package_search",
+        harvester_version="1.0.0",
+    )
+
+    @property
+    def descriptor(self) -> HarvesterDescriptor:
+        return self._DESCRIPTOR
+
+    def parse(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+    ) -> tuple[HarvestedRecord, ...]:
+        page = self.parse_page(
+            payload,
+            uri=uri,
+            observed_at=observed_at,
+            evidence=evidence,
+            response_evidence=None,
+            cursor=None,
+            _require_provider_count=False,
+        )
+        if not page.records:
+            raise CatalogHarvestError(
+                "HARVEST_SHAPE",
+                "ckan.result.results",
+                "no package carried a usable public landing uri; "
+                f"skipped={list(page.skipped_record_ids)}",
+            )
+        return page.records
+
+    def parse_page(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+        response_evidence: HarvestResponseEvidence | None,
+        cursor: HarvestCursor | None,
+        _require_provider_count: bool = True,
+    ) -> HarvestPage:
+        document = _document(payload)
+        if document.get("success") is not True:
+            raise CatalogHarvestError(
+                "HARVEST_UNSUCCESSFUL",
+                "ckan.success",
+                "the CKAN instance reported an unsuccessful response",
+            )
+        result = document.get("result")
+        if not isinstance(result, dict) or not isinstance(result.get("results"), list):
+            raise CatalogHarvestError(
+                "HARVEST_SHAPE",
+                "ckan.result.results",
+                "a CKAN package_search response must carry result.results",
+            )
+        packages = result["results"]
+        provider_count = result.get("count")
+        if provider_count is None and not _require_provider_count:
+            provider_count = len(packages)
+        if type(provider_count) is not int or provider_count < 0:
+            raise CatalogHarvestError(
+                "HARVEST_COUNT",
+                "ckan.result.count",
+                "a paged CKAN response must carry a nonnegative integer count",
+            )
+        if len(packages) > MAX_HARVEST_RECORDS:
+            raise CatalogHarvestError(
+                "HARVEST_RECORD_LIMIT",
+                "ckan.result.results",
+                f"a single response may not carry more than {MAX_HARVEST_RECORDS} packages",
+            )
+        skipped: list[str] = []
+        records: list[HarvestedRecord] = []
+        for position, package in enumerate(packages):
+            if not isinstance(package, dict):
+                raise CatalogHarvestError(
+                    "HARVEST_SHAPE",
+                    f"ckan.result.results[{position}]",
+                    "each package must be an object",
+                )
+            record = _record(package, observed_at=observed_at, evidence=evidence, skipped=skipped)
+            if record is not None:
+                records.append(record)
+        start, rows = _page_coordinate(uri, cursor, len(packages))
+        next_start = start + len(packages)
+        next_cursor = (
+            HarvestCursor(protocol="ckan", kind="offset", value=f"{next_start}:{rows}")
+            if packages and next_start < provider_count
+            else None
+        )
+        if not packages and start < provider_count:
+            raise CatalogHarvestError(
+                "HARVEST_EMPTY_PAGE",
+                "ckan.result.results",
+                "the provider count says records remain but the page is empty",
+            )
+        return HarvestPage(
+            protocol="ckan",
+            records=tuple(records),
+            next_cursor=next_cursor,
+            provider_count=provider_count,
+            count_basis="reported_total",
+            skipped_record_ids=tuple(sorted(skipped)),
+            evidence=evidence,
+            response_evidence=response_evidence,
+        )
+
+
+def _page_coordinate(
+    uri: str, cursor: HarvestCursor | None, returned_records: int
+) -> tuple[int, int]:
+    if cursor is not None:
+        if cursor.protocol != "ckan" or cursor.kind != "offset":
+            raise CatalogHarvestError("HARVEST_CURSOR", "ckan.cursor", "cursor mismatch")
+        start_text, rows_text = cursor.value.split(":", 1)
+        start, rows = int(start_text), int(rows_text)
+    else:
+        query = parse_qs(urlsplit(uri).query, keep_blank_values=True)
+        try:
+            start = int(query.get("start", ["0"])[0])
+            rows = int(query.get("rows", [str(max(1, returned_records))])[0])
+        except (TypeError, ValueError):
+            raise CatalogHarvestError(
+                "HARVEST_CURSOR", "ckan.uri", "start and rows must be canonical integers"
+            ) from None
+    if start < 0 or not 1 <= rows <= MAX_HARVEST_RECORDS:
+        raise CatalogHarvestError(
+            "HARVEST_CURSOR", "ckan.cursor", "start/rows are outside the page bounds"
+        )
+    if returned_records > rows:
+        raise CatalogHarvestError(
+            "HARVEST_CURSOR", "ckan.result.results", "response exceeds its requested rows"
+        )
+    return start, rows
+
+
+def _document(payload: bytes) -> dict[str, Any]:
+    # `parse_foreign_json` is the repository's one parsing authority in its third-party mode. It
+    # refuses duplicate keys, byte-order marks, non-strict UTF-8, NaN/Infinity and out-of-range
+    # integers exactly as the canonical parser does, and differs only in admitting finite fractional
+    # numbers -- which a publisher's document legitimately carries (a resource size, a search score,
+    # a coordinate) and which the strict parser would refuse, taking the whole response with it. No
+    # float reaches a `HarvestedRecord`: every field below is read through `_string` or the closed
+    # format vocabulary, and `canonical_json_bytes` still refuses a float, so nothing read here can
+    # be digested. Never `json.loads` here.
+    try:
+        document = parse_foreign_json(payload)
+    except CanonicalJSONError as error:
+        raise CatalogHarvestError(
+            "HARVEST_SHAPE",
+            "ckan.response",
+            f"the response is not strict JSON: {error.detail}",
+        ) from None
+    if not isinstance(document, dict):
+        raise CatalogHarvestError(
+            "HARVEST_SHAPE",
+            "ckan.response",
+            "a CKAN response must be a JSON object",
+        )
+    return document
+
+
+def _record(
+    package: dict[str, Any],
+    *,
+    observed_at: str,
+    evidence: EvidenceReference,
+    skipped: list[str],
+) -> HarvestedRecord | None:
+    # Every field is absent-by-default. A CKAN instance may omit any of them, and a KeyError
+    # escaping this parser would be a crash where a typed refusal belongs. Every field is also
+    # bounded-by-default: an id or a uri that is not one degrades this record, and free text is cut
+    # to the record bound. Neither may reach `HarvestedRecord` unchecked, because a raise there
+    # escapes `parse` and takes every sibling package in the response with it.
+    record_id = usable_record_id(package.get("id")) or usable_record_id(package.get("name"))
+    if record_id is None:
+        skipped.append("<package without a usable id>")
+        return None
+    landing = usable_https_uri(package.get("url"))
+    if landing is None:
+        # Recorded, never silently included: a non-HTTPS landing page is not a public citation.
+        skipped.append(record_id)
+        return None
+    formats: list[str] = []
+    resources = package.get("resources")
+    if isinstance(resources, list):
+        for resource in resources:
+            if not isinstance(resource, dict):
+                continue
+            normalized = normalize_data_format(_string(resource.get("format")) or "")
+            if normalized is not None and normalized not in formats:
+                formats.append(normalized)
+    organization = package.get("organization")
+    publisher = None
+    if isinstance(organization, dict):
+        publisher = _string(organization.get("title")) or _string(organization.get("name"))
+    title = _string(package.get("title"))
+    description = _string(package.get("notes"))
+    return HarvestedRecord(
+        protocol="ckan",
+        record_id=record_id,
+        title=bounded_record_text(title, maximum=MAX_RECORD_TITLE) if title else record_id,
+        description=(
+            bounded_record_text(description, maximum=MAX_RECORD_TEXT)
+            if description
+            else "No description published."
+        ),
+        publisher=(
+            bounded_record_text(publisher, maximum=MAX_RECORD_PUBLISHER)
+            if publisher
+            else "Unnamed publisher"
+        ),
+        landing_uri=landing,
+        data_formats=tuple(sorted(formats)),
+        evidence=evidence,
+        license_id=usable_license_id(package.get("license_id")),
+        license_uri=usable_https_uri(package.get("license_url")),
+        declared_updated_at=_ckan_timestamp(_string(package.get("metadata_modified"))),
+    )
+
+
+def _string(value: Any) -> str | None:
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    return text or None
+
+
+def _ckan_timestamp(value: str | None) -> str | None:
+    """CKAN emits ``2025-12-31T11:33:28.123456`` with no zone. Read it as UTC or not at all.
+
+    A CKAN instance is free to put anything in ``metadata_modified``, so the normalised result goes
+    through ``declared_timestamp`` and becomes an undeclared update time when it is not a canonical
+    UTC timestamp. Without that, one package's bad field refused every other package in the same
+    response -- see that function.
+
+    The offset check runs before the fractional second is cut, because the offset rides on the
+    fraction and was being cut away with it -- see ``carries_a_zone_offset``.
+    """
+
+    if value is None:
+        return None
+    if carries_a_zone_offset(value):
+        return None
+    text = value.rstrip("Z")
+    if "." in text:
+        head, _, fraction = text.partition(".")
+        text = f"{head}.{fraction[:6]}" if fraction[:6].isdigit() else head
+    if len(text) < 19:
+        return None
+    return declared_timestamp(f"{text}Z")