mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/harvest/sdmx.py

@@ -0,0 +1,445 @@
+"""The SDMX harvester: read one 2.1 dataflow structure message into public-source records.
+
+XML is the one genuinely new attack surface in the harvest package, and it is hardened in three
+ordered steps.
+
+**First, a byte-level pre-check, before any parser object exists.** A payload declaring
+``<!DOCTYPE`` or ``<!ENTITY`` is refused outright. That single check closes XXE, external-entity
+SSRF and entity-expansion (the billion-laughs family) together, because all three require a
+document type or an entity declaration. Refusing rather than sanitising is right here: a
+legitimate SDMX structure message has no reason to declare either, so there is nothing to lose.
+
+The check walks the whole prolog rather than scanning a fixed-width prefix of it, and it walks it
+in the encoding the parser will read. Both halves are load-bearing, and each was found missing in
+turn.
+
+A prefix scan looks equivalent to a walk and is not: comments, processing instructions and
+whitespace may all legally precede the root element at any length, so padding past the window
+would carry a declaration through to the parser. The walk is bounded by ``MAX_PROLOG_BYTES`` and
+fails closed, because walking an unbounded prolog is itself attacker-controlled work.
+
+A UTF-8-only scan looks equivalent to an encoding-aware one and is not: expat auto-detects UTF-16
+and UTF-32 from a byte-order mark or from the byte spelling of the mandatory leading ``<``, so a
+declaration written in either was invisible to a search for the ASCII pattern and reached the
+parser unseen. The payload is therefore normalised to UTF-8 before the walk, by the parser's own
+detection rules. This was not theoretical: a UTF-16 payload well inside the byte cap defined an
+entity that expanded to sixteen million characters, which the element budget cannot see because
+one element with an enormous body is still one element.
+
+Nothing behind the pre-check would catch either evasion -- ``ElementTree`` expands a defined
+internal entity, and the bundled expat's amplification limit is a build-dependent accident rather
+than a control this module may lean on.
+
+**Second, an element budget enforced during a pull parse.** ``MAX_XML_ELEMENTS`` stops work in
+progress. A one-shot ``fromstring`` would build the entire tree before anyone could count it,
+which turns a size-capped response into an unbounded allocation.
+
+**Third, typed failure.** A raw ``ParseError`` escaping a trust boundary is an untyped failure at
+exactly the place a caller needs to distinguish "this endpoint is broken" from "this endpoint is
+hostile", so parser errors are wrapped.
+
+No XML dependency is added. The three checks above are what a hardening library would do for this
+document shape, and pulling one in would change the governed dependency closure for no gain.
+
+**Query shape.** ``dataflow/{agency}/{id}/latest`` is what this harvester expects. ``all/latest``
+is expected to exceed the byte cap on a large provider -- the observed Eurostat response is
+37,166,239 bytes against a 4 MiB cap. That is a usage constraint, not a defect: an agent asking a
+statistical office for its entire structure catalogue in one request is asking the wrong question.
+The endpoint path itself is per-provider configuration, so this module never constructs one.
+
+**Payload formats.** A structure message describes dataflows, not payloads, so every record here
+carries ``data_formats=()``. That is honest and it is also a scope limit worth stating plainly: an
+SDMX record cannot be composed into a catalog entry on the record's own evidence, because an entry
+must declare at least one format the harness can read. The author supplies them at the composition
+seam -- ``compose_catalog_entry(..., data_formats=("csv",))`` -- and the harvested record remains
+the cited evidence for everything else. Guessing a format here would be this module claiming a
+capability the structure message never stated.
+
+**Licences.** An SDMX structure message carries none, so every record has ``license_id=None``.
+That is honest, and it maps to ``unclear`` rights, which the facts gate escalates to a human
+rather than guessing at -- the correct outcome for a source whose terms are genuinely not
+machine-readable.
+"""
+
+from __future__ import annotations
+
+import re
+from xml.etree.ElementTree import ParseError, XMLPullParser
+
+from mostlyright.data_harness.sources.catalog.harvest.ckan import MAX_HARVEST_RECORDS
+from mostlyright.data_harness.sources.catalog.harvest.protocol import (
+    MAX_RECORD_PUBLISHER,
+    MAX_RECORD_TEXT,
+    MAX_RECORD_TITLE,
+    CatalogHarvestError,
+    HarvestCursor,
+    HarvestedRecord,
+    HarvesterDescriptor,
+    HarvestPage,
+    HarvestResponseEvidence,
+    bounded_record_text,
+    harvest_limits,
+    usable_https_uri,
+    usable_record_id,
+)
+from mostlyright.data_harness.sources.contracts import EvidenceReference
+
+SDMX_LIMITS = harvest_limits(
+    "application/vnd.sdmx.structure+xml",
+    max_response_bytes=4 * 1024 * 1024,
+)
+
+MAX_XML_ELEMENTS = 200_000
+
+# How much prolog the declaration pre-check will walk before refusing. A declaration must precede
+# the root element, but the prolog itself is unbounded -- comments, processing instructions and
+# whitespace may all be arbitrarily long -- so the walk is bounded and fails closed. A real SDMX
+# structure message carries a prolog of a few dozen bytes.
+MAX_PROLOG_BYTES = 64 * 1024
+
+# Feed size for the bounded pull parse.
+FEED_CHUNK_BYTES = 64 * 1024
+
+SDMX_STRUCTURE_NS = "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/structure"
+SDMX_MESSAGE_NS = "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/message"
+SDMX_COMMON_NS = "http://www.sdmx.org/resources/sdmxml/schemas/v2_1/common"
+XML_NS = "http://www.w3.org/XML/1998/namespace"
+
+_DATAFLOW_TAG = f"{{{SDMX_STRUCTURE_NS}}}Dataflow"
+_NAME_TAG = f"{{{SDMX_COMMON_NS}}}Name"
+_DESCRIPTION_TAG = f"{{{SDMX_COMMON_NS}}}Description"
+_LANG_ATTR = f"{{{XML_NS}}}lang"
+
+_DECLARATION = re.compile(rb"(?i)<!\s*(doctype|entity)")
+
+# How a conforming XML parser works out the byte order of a document before it has read the
+# encoding declaration -- XML 1.0 appendix F, and what expat implements. Longest patterns first,
+# because a UTF-32LE mark begins with a UTF-16LE one. The four unmarked patterns are the byte
+# spellings of the mandatory leading `<`.
+_AUTODETECTED_ENCODINGS = (
+    (b"\x00\x00\xfe\xff", "utf-32"),
+    (b"\xff\xfe\x00\x00", "utf-32"),
+    (b"\x00\x00\x00\x3c", "utf-32-be"),
+    (b"\x3c\x00\x00\x00", "utf-32-le"),
+    (b"\xfe\xff", "utf-16"),
+    (b"\xff\xfe", "utf-16"),
+    (b"\x00\x3c", "utf-16-be"),
+    (b"\x3c\x00", "utf-16-le"),
+)
+
+
+class SdmxHarvester:
+    """Read an SDMX 2.1 dataflow structure message. Takes bytes; never fetches."""
+
+    _DESCRIPTOR = HarvesterDescriptor(
+        protocol="sdmx",
+        harvester_id="sdmx.dataflow",
+        harvester_version="1.0.0",
+    )
+
+    @property
+    def descriptor(self) -> HarvesterDescriptor:
+        return self._DESCRIPTOR
+
+    def parse(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+    ) -> tuple[HarvestedRecord, ...]:
+        page = self.parse_page(
+            payload,
+            uri=uri,
+            observed_at=observed_at,
+            evidence=evidence,
+            response_evidence=None,
+            cursor=None,
+        )
+        if not page.records:
+            raise CatalogHarvestError(
+                "HARVEST_SHAPE", "sdmx.Structures.Dataflows", "message carries no dataflow"
+            )
+        return page.records
+
+    def parse_page(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+        response_evidence: HarvestResponseEvidence | None,
+        cursor: HarvestCursor | None,
+    ) -> HarvestPage:
+        if cursor is not None:
+            raise CatalogHarvestError(
+                "HARVEST_CURSOR", "sdmx.cursor", "SDMX dataflow harvest is one-shot"
+            )
+        if not isinstance(payload, bytes):
+            raise CatalogHarvestError("TYPE", "sdmx.payload", "payload must be bytes")
+        require_no_xml_declarations(payload)
+        root = _parse_bounded(payload)
+        dataflows = [element for element in root.iter() if element.tag == _DATAFLOW_TAG]
+        if len(dataflows) > MAX_HARVEST_RECORDS:
+            raise CatalogHarvestError(
+                "HARVEST_RECORD_LIMIT",
+                "sdmx.Structures.Dataflows",
+                f"a single message may not carry more than {MAX_HARVEST_RECORDS} dataflows",
+            )
+        records: list[HarvestedRecord] = []
+        skipped: list[str] = []
+        for position, dataflow in enumerate(dataflows):
+            record = _record(
+                dataflow,
+                position=position,
+                uri=uri,
+                evidence=evidence,
+                skipped=skipped,
+            )
+            if record is not None:
+                records.append(record)
+        return HarvestPage(
+            protocol="sdmx",
+            records=tuple(records),
+            next_cursor=None,
+            provider_count=len(dataflows),
+            count_basis="one_shot_response",
+            skipped_record_ids=tuple(sorted(skipped)),
+            evidence=evidence,
+            response_evidence=response_evidence,
+        )
+
+
+def require_no_xml_declarations(payload: bytes) -> None:
+    """Refuse a payload declaring a document type or an entity, before any parser exists.
+
+    The whole prolog is walked, not a fixed-width prefix of it. A prefix scan is an evadable
+    control: comments, processing instructions and whitespace are all legal before the root element
+    and all three can be padded to any length, so a declaration hidden behind enough padding would
+    reach the parser -- and ElementTree does expand a defined internal entity, so the parser is not
+    a fallback control. The walk consumes exactly the constructs that may legally precede the root
+    element and stops at the first thing that is not one of them.
+
+    The walk runs over UTF-8 bytes, whichever byte order the sender chose. expat auto-detects
+    UTF-16 and UTF-32 from a byte-order mark or from the first character's byte pattern, so a
+    declaration spelled in either was invisible to a scan for the ASCII pattern and reached the
+    parser unseen -- a walk that only reads one encoding is an evadable control for the same
+    reason a prefix scan is. Normalising rather than refusing keeps the refusal in one place and
+    keeps the reported code the same whatever the encoding.
+    """
+
+    payload = _utf8_normalised(payload)
+    cursor = _skip_bom_and_space(payload, 0)
+    while cursor < len(payload):
+        if cursor > MAX_PROLOG_BYTES:
+            raise _declaration_refused(
+                f"the prolog exceeds {MAX_PROLOG_BYTES} bytes before any element begins, so it "
+                "cannot be cleared of declarations within its budget"
+            )
+        window = payload[cursor : cursor + 32]
+        match = _DECLARATION.match(window)
+        if match is not None:
+            raise _declaration_refused(
+                f"the document declares a {match.group(1).decode('ascii').lower()}; document type "
+                "and entity declarations are refused outright, because an SDMX structure message "
+                "has no reason to carry one"
+            )
+        if window.startswith(b"<!--"):
+            end = payload.find(b"-->", cursor + 4)
+            if end < 0:
+                # Unterminated: nothing can follow it, so no declaration can be hiding behind it.
+                return
+            cursor = _skip_bom_and_space(payload, end + 3)
+            continue
+        if window.startswith(b"<?"):
+            end = payload.find(b"?>", cursor + 2)
+            if end < 0:
+                return
+            cursor = _skip_bom_and_space(payload, end + 2)
+            continue
+        # Anything else is the root element or malformed input; either way the prolog is over and
+        # no declaration may legally appear past this point.
+        return
+
+
+def _utf8_normalised(payload: bytes) -> bytes:
+    """Return the payload as UTF-8 bytes, whatever byte order it arrived in.
+
+    The detection rules are the parser's own -- XML 1.0 appendix F, which is what expat implements:
+    a byte-order mark if there is one, otherwise the byte pattern of the mandatory leading ``<``.
+    Reading it any other way would leave a gap between what this pre-check inspects and what the
+    parser will act on, which is precisely the defect this closes.
+
+    A payload that announces one of these encodings and then is not valid in it is refused rather
+    than passed along, because at that point nothing can say what the parser would make of it.
+    """
+
+    encoding = _autodetected_encoding(payload)
+    if encoding is None:
+        return payload
+    try:
+        text = payload.decode(encoding)
+    except UnicodeDecodeError:
+        raise CatalogHarvestError(
+            "HARVEST_XML_ENCODING",
+            "sdmx.payload",
+            f"the payload announces {encoding} and is not valid in it, so what the parser would "
+            "make of it cannot be established before parsing it",
+        ) from None
+    return text.encode("utf-8")
+
+
+def _autodetected_encoding(payload: bytes) -> str | None:
+    """The encoding a conforming XML parser would detect from the first four bytes.
+
+    ``None`` means UTF-8 or another ASCII-compatible single-byte encoding, for which the ASCII
+    declaration pattern is already directly readable in the bytes. The byte-order-mark codecs are
+    named without an endianness suffix on purpose: those forms consume the mark, so the decoded
+    text starts at the first real character.
+    """
+
+    prefix = payload[:4]
+    for pattern, encoding in _AUTODETECTED_ENCODINGS:
+        if prefix.startswith(pattern):
+            return encoding
+    return None
+
+
+def _skip_bom_and_space(payload: bytes, cursor: int) -> int:
+    if cursor == 0 and payload.startswith(b"\xef\xbb\xbf"):
+        cursor = 3
+    while cursor < len(payload) and payload[cursor : cursor + 1].isspace():
+        cursor += 1
+    return cursor
+
+
+def _declaration_refused(detail: str) -> CatalogHarvestError:
+    return CatalogHarvestError("HARVEST_XML_DOCTYPE", "sdmx.payload", detail)
+
+
+def _parse_bounded(payload: bytes):  # type: ignore[no-untyped-def]
+    # Fed in chunks and drained between them, so the budget stops work in progress rather than
+    # after the whole tree has already been built.
+    parser = XMLPullParser(events=("start",))
+    root = None
+    elements = 0
+    try:
+        for offset in range(0, len(payload), FEED_CHUNK_BYTES):
+            parser.feed(payload[offset : offset + FEED_CHUNK_BYTES])
+            for _event, element in parser.read_events():
+                elements += 1
+                if root is None:
+                    root = element
+                if elements > MAX_XML_ELEMENTS:
+                    raise CatalogHarvestError(
+                        "HARVEST_XML_ELEMENTS",
+                        "sdmx.payload",
+                        f"the document exceeds {MAX_XML_ELEMENTS} elements",
+                    )
+        parser.close()
+        for _event, element in parser.read_events():
+            elements += 1
+            if root is None:
+                root = element
+    except ParseError as error:
+        raise CatalogHarvestError(
+            "HARVEST_SHAPE",
+            "sdmx.payload",
+            f"the response is not well-formed XML: {error}",
+        ) from None
+    if root is None:
+        raise CatalogHarvestError(
+            "HARVEST_SHAPE",
+            "sdmx.payload",
+            "the response carries no XML element",
+        )
+    return root
+
+
+def _record(
+    dataflow,  # type: ignore[no-untyped-def]
+    *,
+    position: int,
+    uri: str,
+    evidence: EvidenceReference,
+    skipped: list[str],
+) -> HarvestedRecord | None:
+    flow_id = _attribute(dataflow, "id")
+    agency = _attribute(dataflow, "agencyID")
+    version = _attribute(dataflow, "version") or "1.0"
+    if flow_id is None or agency is None:
+        skipped.append(f"<dataflow {position} without id or agencyID>")
+        return None
+    # The dataflow's natural coordinate is an identifier, so it degrades this record rather than
+    # being cut: an agency, id or version outside the record grammar would raise out of `parse` and
+    # take every other dataflow in the message with it.
+    record_id = usable_record_id(f"{agency}:{flow_id}({version})")
+    if record_id is None:
+        skipped.append(f"<dataflow {position} whose coordinate is not a usable record id>")
+        return None
+    landing = usable_https_uri(_structure_uri(uri, agency, flow_id, version))
+    if landing is None:
+        skipped.append(record_id)
+        return None
+    name = _localised(dataflow, _NAME_TAG) or flow_id
+    description = _localised(dataflow, _DESCRIPTION_TAG) or (
+        f"SDMX dataflow {agency}:{flow_id}({version}) published as a 2.1 structure message."
+    )
+    return HarvestedRecord(
+        protocol="sdmx",
+        record_id=record_id,
+        title=bounded_record_text(name, maximum=MAX_RECORD_TITLE),
+        description=bounded_record_text(description, maximum=MAX_RECORD_TEXT),
+        publisher=bounded_record_text(agency, maximum=MAX_RECORD_PUBLISHER),
+        landing_uri=landing,
+        data_formats=(),
+        evidence=evidence,
+        # A structure message states no licence. Saying so honestly maps to unclear rights, which
+        # escalates rather than admits.
+        license_id=None,
+        license_uri=None,
+        declared_updated_at=None,
+    )
+
+
+def _structure_uri(uri: str, agency: str, flow_id: str, version: str) -> str | None:
+    if not uri.startswith("https://"):
+        return None
+    base = uri.split("?", 1)[0].rstrip("/")
+    marker = "/dataflow"
+    if marker in base:
+        base = base[: base.index(marker) + len(marker)]
+        return f"{base}/{agency}/{flow_id}/{version}"
+    return base
+
+
+def _attribute(element, name: str) -> str | None:  # type: ignore[no-untyped-def]
+    value = element.get(name)
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    return text or None
+
+
+def _localised(element, tag: str) -> str | None:  # type: ignore[no-untyped-def]
+    """The English text of this child, or the first declared one, deterministically.
+
+    Falling back to "whichever came out of the iterator" would make the harvested title depend on
+    document order in a way nobody documented, so the fallback is explicit and ordered.
+    """
+
+    candidates = [child for child in element if child.tag == tag]
+    if not candidates:
+        return None
+    for child in candidates:
+        if (child.get(_LANG_ATTR) or "").lower().startswith("en"):
+            text = (child.text or "").strip()
+            if text:
+                return text
+    for child in candidates:
+        text = (child.text or "").strip()
+        if text:
+            return text
+    return None