mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/readers/json_tabular.py

@@ -0,0 +1,485 @@
+"""Declaratively project bounded JSON or NDJSON documents into canonical tabular CSV.
+
+The Reader deliberately implements selection rather than inference. A recipe identifies the
+record collection, any arrays to expand, and every output column with RFC 6901 JSON Pointers.
+The same implementation therefore handles a top-level array, an API response envelope, and
+nested observations without source-specific code or executable expressions.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from mostlyright.data_harness.formats import (
+    FORMAT_MEDIA_TYPES,
+    FORMAT_SUFFIXES,
+    READER_CONTRACT_VERSION,
+    READER_OUTPUT_FORMATS,
+)
+from mostlyright.data_harness.readers.contracts import (
+    ReaderBudgets,
+    ReaderError,
+    ReaderPin,
+    ReaderResult,
+    bulk_default_budgets,
+)
+from mostlyright.data_harness.readers.tabular import (
+    FALLBACK_STEM,
+    encode_canonical_csv,
+    sealed_filename,
+)
+
+__all__ = ["JsonTabularReader"]
+
+(_OUTPUT_FORMAT,) = READER_OUTPUT_FORMATS
+_OUTPUT_MEDIA_TYPE = sorted(FORMAT_MEDIA_TYPES[_OUTPUT_FORMAT])[0]
+_OUTPUT_SUFFIX = sorted(FORMAT_SUFFIXES[_OUTPUT_FORMAT])[0]
+
+_DOCUMENT_FORMATS = ("json", "ndjson")
+_OPTION_KEYS = frozenset({"document_format", "records_pointer", "expand", "columns"})
+_COLUMN_KEYS = frozenset({"name", "pointer", "required"})
+_SAFE_COLUMN = re.compile(r"^[^\x00-\x1f\x7f]{1,256}$")
+_MAX_POINTER_BYTES = 1_024
+_MAX_EXPANSIONS = 16
+_MAX_JSON_DEPTH = 64
+_MISSING = object()
+
+# JSON expansion retains one small immutable view per output row and expansion level. Keeping the
+# family ceiling below the shared million-row default makes even the worst 16-level chain bounded
+# well inside the 512 MiB clean-room boundary, while ordinary non-expanded tables remain large.
+_JSON_DEFAULT_BUDGETS = bulk_default_budgets()
+_MAX_OVERLAY_RECORDS = 200_000
+
+
+def _refuse_unknown(mapping: Mapping[str, Any], admitted: frozenset[str], subject: str) -> None:
+    unknown = sorted(str(key) for key in mapping if key not in admitted)
+    if unknown:
+        raise ReaderError(
+            "READER_OPTIONS",
+            subject,
+            f"names no such setting: {', '.join(unknown)}; admitted settings are "
+            f"{', '.join(sorted(admitted))}",
+        )
+
+
+def _pointer(value: Any, subject: str) -> tuple[str, ...]:
+    if not isinstance(value, str):
+        raise ReaderError(
+            "READER_OPTIONS", subject, "must be an RFC 6901 JSON Pointer of at most 1024 bytes"
+        )
+    try:
+        encoded = value.encode("utf-8")
+    except UnicodeEncodeError:
+        raise ReaderError(
+            "READER_OPTIONS", subject, "must contain valid Unicode scalar values"
+        ) from None
+    if len(encoded) > _MAX_POINTER_BYTES:
+        raise ReaderError(
+            "READER_OPTIONS", subject, "must be an RFC 6901 JSON Pointer of at most 1024 bytes"
+        )
+    if value == "":
+        return ()
+    if not value.startswith("/"):
+        raise ReaderError("READER_OPTIONS", subject, "must be empty or start with '/'")
+    tokens: list[str] = []
+    for raw in value[1:].split("/"):
+        index = 0
+        decoded: list[str] = []
+        while index < len(raw):
+            if raw[index] != "~":
+                decoded.append(raw[index])
+                index += 1
+                continue
+            if index + 1 >= len(raw) or raw[index + 1] not in "01":
+                raise ReaderError(
+                    "READER_OPTIONS", subject, "contains an invalid RFC 6901 '~' escape"
+                )
+            decoded.append("~" if raw[index + 1] == "0" else "/")
+            index += 2
+        tokens.append("".join(decoded))
+    return tuple(tokens)
+
+
+def _validated_options(options: Any, budgets: ReaderBudgets) -> dict[str, Any]:
+    subject = "reader.json.tabular.decode_options"
+    if not isinstance(options, Mapping):
+        raise ReaderError("READER_OPTIONS", subject, "must be an object")
+    _refuse_unknown(options, _OPTION_KEYS, subject)
+
+    document_format = options.get("document_format", "json")
+    if document_format not in _DOCUMENT_FORMATS:
+        raise ReaderError(
+            "READER_OPTIONS",
+            f"{subject}.document_format",
+            f"must be one of {', '.join(_DOCUMENT_FORMATS)}",
+        )
+    records_pointer = options.get("records_pointer", "")
+    _pointer(records_pointer, f"{subject}.records_pointer")
+
+    expand = options.get("expand", [])
+    if not isinstance(expand, list) or len(expand) > _MAX_EXPANSIONS:
+        raise ReaderError(
+            "READER_OPTIONS",
+            f"{subject}.expand",
+            f"must be an array of at most {_MAX_EXPANSIONS} JSON Pointers",
+        )
+    for index, item in enumerate(expand):
+        _pointer(item, f"{subject}.expand[{index}]")
+
+    columns = options.get("columns")
+    if not isinstance(columns, list) or not columns or len(columns) > budgets.max_columns:
+        raise ReaderError(
+            "READER_OPTIONS",
+            f"{subject}.columns",
+            "must be a non-empty array within the column budget",
+        )
+    admitted_columns: list[dict[str, Any]] = []
+    names: list[str] = []
+    for index, column in enumerate(columns):
+        item_subject = f"{subject}.columns[{index}]"
+        if not isinstance(column, Mapping):
+            raise ReaderError("READER_OPTIONS", item_subject, "must be an object")
+        _refuse_unknown(column, _COLUMN_KEYS, item_subject)
+        name = column.get("name")
+        if not isinstance(name, str) or _SAFE_COLUMN.fullmatch(name) is None:
+            raise ReaderError(
+                "READER_OPTIONS",
+                f"{item_subject}.name",
+                "must be bounded text without control characters",
+            )
+        pointer = column.get("pointer")
+        _pointer(pointer, f"{item_subject}.pointer")
+        required = column.get("required", True)
+        if not isinstance(required, bool):
+            raise ReaderError("READER_OPTIONS", f"{item_subject}.required", "must be boolean")
+        names.append(name)
+        admitted_columns.append({"name": name, "pointer": pointer, "required": required})
+    if len(set(names)) != len(names):
+        raise ReaderError("READER_OPTIONS", f"{subject}.columns", "column names must be unique")
+
+    return {
+        "document_format": document_format,
+        "records_pointer": records_pointer,
+        "expand": list(expand),
+        "columns": admitted_columns,
+    }
+
+
+def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    value: dict[str, Any] = {}
+    for key, item in pairs:
+        if key in value:
+            raise ReaderError(
+                "READER_ADMISSION",
+                "reader.json.tabular.content",
+                f"JSON object repeats key {key!r}",
+            )
+        value[key] = item
+    return value
+
+
+def _decode_documents(content: bytes, document_format: str, budgets: ReaderBudgets) -> list[Any]:
+    if not isinstance(content, (bytes, bytearray)):
+        raise ReaderError("READER_ADMISSION", "reader.json.tabular.content", "must be exact bytes")
+    if not content or len(content) > budgets.max_input_bytes:
+        raise ReaderError(
+            "READER_BUDGET",
+            "reader.json.tabular.content",
+            "is empty or exceeds the input byte budget",
+        )
+    try:
+        text = bytes(content).removeprefix(b"\xef\xbb\xbf").decode("utf-8", errors="strict")
+    except UnicodeDecodeError:
+        raise ReaderError(
+            "READER_ADMISSION", "reader.json.tabular.content", "must be strict UTF-8"
+        ) from None
+
+    decoder = json.JSONDecoder(
+        object_pairs_hook=_unique_object,
+        parse_constant=lambda value: (_ for _ in ()).throw(
+            ReaderError(
+                "READER_ADMISSION",
+                "reader.json.tabular.content",
+                f"non-finite JSON number {value!r} is not admitted",
+            )
+        ),
+    )
+    try:
+        if document_format == "json":
+            documents = [decoder.decode(text)]
+        else:
+            documents = [decoder.decode(line) for line in text.splitlines() if line.strip()]
+    except ReaderError:
+        raise
+    except (json.JSONDecodeError, ValueError):
+        raise ReaderError(
+            "READER_DECODE", "reader.json.tabular.content", "JSON syntax is malformed"
+        ) from None
+    except RecursionError:
+        raise ReaderError(
+            "READER_BUDGET", "reader.json.tabular.content", "JSON nesting exceeds the depth limit"
+        ) from None
+    if not documents:
+        raise ReaderError(
+            "READER_ADMISSION", "reader.json.tabular.content", "contains no JSON documents"
+        )
+    return documents
+
+
+def _validate_tree(value: Any) -> None:
+    pending: list[tuple[Any, int]] = [(value, 0)]
+    while pending:
+        item, depth = pending.pop()
+        if depth > _MAX_JSON_DEPTH:
+            raise ReaderError(
+                "READER_BUDGET",
+                "reader.json.tabular.content",
+                f"JSON nesting exceeds the fixed {_MAX_JSON_DEPTH}-level limit",
+            )
+        if isinstance(item, dict):
+            for key in item:
+                try:
+                    key.encode("utf-8")
+                except UnicodeEncodeError:
+                    raise ReaderError(
+                        "READER_ADMISSION",
+                        "reader.json.tabular.content",
+                        "contains an object key that is not valid Unicode scalar text",
+                    ) from None
+            pending.extend((child, depth + 1) for child in item.values())
+        elif isinstance(item, list):
+            pending.extend((child, depth + 1) for child in item)
+        elif isinstance(item, float) and not math.isfinite(item):
+            raise ReaderError(
+                "READER_ADMISSION",
+                "reader.json.tabular.content",
+                "contains a number outside finite binary64 range",
+            )
+        elif isinstance(item, str):
+            try:
+                item.encode("utf-8")
+            except UnicodeEncodeError:
+                raise ReaderError(
+                    "READER_ADMISSION",
+                    "reader.json.tabular.content",
+                    "contains a string that is not valid Unicode scalar text",
+                ) from None
+        elif item is not None and not isinstance(item, str | bool | int | float):
+            raise ReaderError(
+                "READER_ADMISSION",
+                "reader.json.tabular.content",
+                f"contains unsupported {type(item).__name__} value",
+            )
+
+
+def _array_index(token: str, subject: str) -> int:
+    if (
+        token == "-"
+        or not token.isascii()
+        or not token.isdigit()
+        or (len(token) > 1 and token.startswith("0"))
+    ):
+        raise ReaderError("READER_ADMISSION", subject, "does not name an RFC 6901 array index")
+    return int(token)
+
+
+@dataclass(frozen=True, slots=True)
+class _ExpandedRecord:
+    """One constant-size overlay; parents and wide source containers are never copied."""
+
+    parent: Any
+    pointer: str
+    replacement: Any
+
+
+def _overlay(value: Any, pointer: str) -> tuple[Any, tuple[str, ...]]:
+    tokens = _pointer(pointer, "reader.json.tabular.pointer")
+    current = value
+    while isinstance(current, _ExpandedRecord):
+        replacement_tokens = _pointer(current.pointer, "reader.json.tabular.expand")
+        if tokens[: len(replacement_tokens)] == replacement_tokens:
+            return current.replacement, tokens[len(replacement_tokens) :]
+        current = current.parent
+    return current, tokens
+
+
+def _resolve(value: Any, pointer: str, subject: str, *, missing: Any = _MISSING) -> Any:
+    current, tokens = _overlay(value, pointer)
+    for token in tokens:
+        if isinstance(current, dict):
+            if token not in current:
+                if missing is not _MISSING:
+                    return missing
+                raise ReaderError("READER_ADMISSION", subject, f"does not exist at {pointer!r}")
+            current = current[token]
+        elif isinstance(current, list):
+            index = _array_index(token, subject)
+            if index >= len(current):
+                if missing is not _MISSING:
+                    return missing
+                raise ReaderError("READER_ADMISSION", subject, f"does not exist at {pointer!r}")
+            current = current[index]
+        else:
+            raise ReaderError("READER_ADMISSION", subject, f"crosses a scalar at {token!r}")
+    return current
+
+
+def _selected_records(documents: Sequence[Any], pointer: str, budgets: ReaderBudgets) -> list[Any]:
+    selected: list[Any] = []
+    for index, document in enumerate(documents):
+        _validate_tree(document)
+        value = _resolve(
+            document, pointer, f"reader.json.tabular.documents[{index}].records_pointer"
+        )
+        if isinstance(value, list):
+            if len(selected) + len(value) > budgets.max_rows:
+                raise ReaderError(
+                    "READER_BUDGET",
+                    f"reader.json.tabular.documents[{index}].records_pointer",
+                    "selection exceeds the row budget",
+                )
+            selected.extend(value)
+        elif isinstance(value, dict):
+            if len(selected) >= budgets.max_rows:
+                raise ReaderError(
+                    "READER_BUDGET",
+                    f"reader.json.tabular.documents[{index}].records_pointer",
+                    "selection exceeds the row budget",
+                )
+            selected.append(value)
+        else:
+            raise ReaderError(
+                "READER_ADMISSION",
+                f"reader.json.tabular.documents[{index}].records_pointer",
+                "must select an object or an array of records",
+            )
+    return selected
+
+
+def _expanded_records(
+    records: list[Any], pointers: Sequence[str], budgets: ReaderBudgets
+) -> list[Any]:
+    current = records
+    overlay_records = 0
+    for expansion_index, pointer in enumerate(pointers):
+        expanded: list[Any] = []
+        for record_index, record in enumerate(current):
+            subject = f"reader.json.tabular.expand[{expansion_index}].records[{record_index}]"
+            values = _resolve(record, pointer, subject)
+            if not isinstance(values, list):
+                raise ReaderError("READER_ADMISSION", subject, "must select an array")
+            if len(expanded) + len(values) > budgets.max_rows:
+                raise ReaderError("READER_BUDGET", subject, "expansion exceeds the row budget")
+            if overlay_records + len(values) > _MAX_OVERLAY_RECORDS:
+                raise ReaderError(
+                    "READER_BUDGET", subject, "expansion exceeds the cumulative overlay budget"
+                )
+            expanded.extend(_ExpandedRecord(record, pointer, item) for item in values)
+            overlay_records += len(values)
+        current = expanded
+    if len(current) > budgets.max_rows:
+        raise ReaderError("READER_BUDGET", "reader.json.tabular.records", "exceeds the row budget")
+    return current
+
+
+def _project(records: Sequence[Any], columns: Sequence[Mapping[str, Any]]) -> Iterator[list[Any]]:
+    for row_index, record in enumerate(records):
+        row: list[Any] = []
+        for column in columns:
+            subject = f"reader.json.tabular.rows[{row_index}].{column['name']}"
+            value = _resolve(
+                record,
+                str(column["pointer"]),
+                subject,
+                missing=_MISSING if bool(column["required"]) else None,
+            )
+            if isinstance(value, dict | list):
+                raise ReaderError(
+                    "READER_ADMISSION",
+                    subject,
+                    "selects an object or array; expand it or select a scalar child",
+                )
+            row.append(value)
+        yield row
+
+
+@dataclass(frozen=True)
+class JsonTabularReader:
+    """A generic, versioned JSON/NDJSON-to-table Reader."""
+
+    family_id: str = "json.tabular"
+    family_version: str = "1.0.0"
+    contract_version: str = READER_CONTRACT_VERSION
+    output_format: str = _OUTPUT_FORMAT
+    accepted_media_types: tuple[str, ...] = ("application/json", "application/x-ndjson")
+    default_budgets: ReaderBudgets = field(default_factory=lambda: _JSON_DEFAULT_BUDGETS)
+
+    def validate_options(self, options: Mapping[str, Any]) -> Mapping[str, Any]:
+        return _validated_options(options, self.default_budgets)
+
+    def decode(self, content: bytes, pin: ReaderPin, budgets: ReaderBudgets) -> ReaderResult:
+        options = _validated_options(pin.decode_options, budgets)
+        documents = _decode_documents(content, str(options["document_format"]), budgets)
+        records = _selected_records(documents, str(options["records_pointer"]), budgets)
+        records = _expanded_records(records, list(options["expand"]), budgets)
+        columns = list(options["columns"])
+        if len(records) * len(columns) > budgets.max_declared_cells:
+            raise ReaderError(
+                "READER_BUDGET", "reader.json.tabular.records", "exceeds the cell budget"
+            )
+        names = tuple(str(column["name"]) for column in columns)
+        return ReaderResult(
+            content=encode_canonical_csv(names, _project(records, columns), budgets=budgets),
+            data_format=_OUTPUT_FORMAT,
+            media_type=_OUTPUT_MEDIA_TYPE,
+            filename=sealed_filename(FALLBACK_STEM, suffix=_OUTPUT_SUFFIX),
+            row_count=len(records),
+            column_names=names,
+            declared_cell_count=len(records) * len(names),
+        )
+
+
+@dataclass(frozen=True)
+class JsonTabularReaderV1_1(JsonTabularReader):
+    """``json.tabular@1.1.0`` admits the weak labels, and decodes nothing new.
+
+    A publisher that serves a JSON document under ``text/plain`` is not describing a different
+    document; it is declining to describe the document at all.
+    ``acquisition/http.py`` already writes that rule down for the byte-stream spelling -- a weak
+    label "says only that a server declined to say anything", and the control is the structural
+    check behind the sandbox boundary, never the header.  Version ``1.0.0`` admitted the two
+    labels that name the format outright, which refused whole archives that were never unfit:
+    a large share of public agency JSON and NDJSON endpoints answer under ``text/plain``.
+
+    This is an admission change and not a decode change.  ``decode`` is inherited untouched, so
+    the bytes a recipe seals under this coordinate are the bytes ``1.0.0`` would have sealed;
+    what moves is only which responses reach the decoder.  It is a new coordinate for the reason
+    ``delimited_text@1.1.0`` and ``archive.zip@1.2.0`` are: ``1.0.0`` stays closed to the weak
+    labels, so an already-approved recipe cannot silently begin admitting responses its review
+    never saw.
+
+    The label is the only thing widened.  ``text/html`` remains outside every allowlist, which is
+    what keeps the bot-wall refusal intact, and the decoder's own refusals -- a document that is
+    not valid JSON or NDJSON, a pointer that selects nothing, a column that resolves to an object
+    or array, and every budget bound -- are unchanged and remain the whole admission.  Archive and
+    executable bytes arriving under these labels are refused as invalid JSON rather than by
+    prefix: the magic table is enforced in ``acquisition/parsing`` and in the container families,
+    and this family imports neither.
+
+    ``application/octet-stream`` is deliberately not added here.  It is the weak label an object
+    store uses for a stored document, and admitting it is a separate widening with its own
+    evidence, taken the way ``archive.zip`` took its two: one label at a time, each its own
+    coordinate.
+    """
+
+    family_version: str = "1.1.0"
+    accepted_media_types: tuple[str, ...] = (
+        "application/json",
+        "application/x-ndjson",
+        "text/plain",
+    )