mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/authoring_policy.py

@@ -0,0 +1,701 @@
+"""The closed Data.gov v4 metadata authoring policy.
+
+This module is the only place where a normalized Data.gov v4 provider record becomes a v2 catalog
+entry, and it is deliberately a table rather than a program.  Every known fact it authors is a
+value the provider literally declared at an allowlisted JSON path, carried together with the exact
+record digest that stated it.  Everything else is ``unknown``.  There is no inference here: no
+heuristic, no model, no title-derived identity, no format guessing, and no resource fetch.  The
+module imports no transport and is given bytes-derived dictionaries, never a retriever.
+
+Three separations are load-bearing.
+
+*Semantic fields versus observations.*  A normalized record has a semantic
+``fields`` partition and a volatile ``observations`` partition.  ``last_harvested_date`` is a
+catalog-ingestion timestamp that Data.gov rewrites on every re-harvest, and ``popularity`` is a
+per-sweep counter; neither is a fact about the source.  The policy reads ``fields`` only, re-derives
+the record's own semantic digest before authoring anything from it, and refuses outright any record
+whose semantic ``fields`` carry an observation path.  Locator values -- landing pages, access and
+download URLs, harvest-record links, organization logos -- do stay in ``fields`` as inert bounded
+text; the policy simply never reads them, so no fact and no provenance can ever cite one.
+
+*Facts versus layer text.*  A fact keeps the complete declared value.  The four retrieval layer
+texts are a rendering of those facts and are capped at :data:`MAX_LAYER_TEXT_BYTES` UTF-8 bytes
+with a Unicode-boundary-safe truncation plus an explicit per-entry disposition.  The cap is what
+keeps a 1,000-entry range/layer aggregate inside the 8 MiB ``encode_many`` bound downstream
+(1000 * 8192 = 8,192,000 <= 8,388,608), so exactly one encode call per range/layer/backend stays
+exact.  The pinned MiniLM tokenizer truncates far below this cap, so retrieval is unaffected.
+
+*Admission versus evidence.*  Only an exact reviewed license mapping may state rights.  Declared
+rights prose, a federal publisher, or a public access level never grants one; ambiguity is flagged
+and queued, never guessed.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any
+
+from mostlyright.data_harness.canonical import canonical_sha256, sha256_bytes
+from mostlyright.data_harness.sources.catalog.contracts import (
+    EMBEDDING_LAYERS,
+    MAX_DECLARED_NAME,
+    MAX_DECLARED_VOCABULARY,
+    MAX_DESCRIPTION,
+    MAX_SPATIAL_SCOPE,
+)
+from mostlyright.data_harness.sources.catalog.entry_v2 import (
+    CATALOG_ENTRY_V2_CONTRACT_VERSION,
+    PROVIDER_CONTRACTS,
+    CatalogEntryV2,
+    CatalogFact,
+    CatalogObservation,
+    ProviderFactProvenance,
+    ProviderRecordIdentity,
+)
+from mostlyright.data_harness.sources.catalog.harvest.datagov_v4 import (
+    DATAGOV_V4_RECORD_VERSION,
+)
+from mostlyright.data_harness.sources.catalog.harvest.protocol import (
+    DATAGOV_V4_HARVESTER_COORDINATE,
+)
+from mostlyright.data_harness.sources.contracts import (
+    DATA_FORMATS,
+    RIGHTS_STATUSES,
+    SourceContractError,
+    _CanonicalContract,
+)
+
+AUTHORING_POLICY_SCHEMA = "mr-data-catalog-authoring-policy.v1"
+DATAGOV_V4_POLICY_ID = "datagov-v4-policy.v1"
+
+# One <=1000-entry range holds at most 1000 * 8192 = 8,192,000 bytes per layer, inside the
+# 8,388,608-byte ``encode_many`` bound. Widening this constant breaks that equation.
+MAX_LAYER_TEXT_BYTES = 8_192
+
+# ``CatalogEntryV2`` bounds a known title at 200 characters. A longer or unsafe title cannot be
+# carried, and truncating identity prose would author something the provider never declared, so
+# such a record is skipped rather than reshaped.
+MAX_TITLE = 200
+
+LAYER_TEXT_UNKNOWN = "unknown"
+LAYER_TEXT_NOT_APPLICABLE = "not_applicable"
+
+AUTHORING_DISPOSITIONS = ("authored", "flagged", "skipped", "failed")
+
+AUTHORING_REASON_CODES = frozenset(
+    {
+        "AUTHOR_OK",
+        "AUTHOR_RIGHTS_ABSENT",
+        "AUTHOR_RIGHTS_CONDITIONAL",
+        "AUTHOR_RIGHTS_PROHIBITED",
+        "AUTHOR_RIGHTS_PROSE",
+        "AUTHOR_RIGHTS_UNMAPPED",
+        "AUTHOR_TITLE_ABSENT",
+        "AUTHOR_TITLE_LIMIT",
+        "AUTHOR_TITLE_UNSAFE",
+        "AUTHOR_IDENTIFIER_UNSAFE",
+        "AUTHOR_RECORD_SCHEMA",
+        "AUTHOR_RECORD_DIGEST",
+        "AUTHOR_RECORD_FIELDS",
+        "AUTHOR_ENTRY_CONTRACT",
+    }
+)
+
+_RECORD_MEMBERS = frozenset(
+    {
+        "schema_version",
+        "protocol",
+        "record_id",
+        "fields",
+        "observations",
+        "raw_response_sha256",
+        "normalized_sha256",
+    }
+)
+_REQUIRED_RECORD_MEMBERS = _RECORD_MEMBERS - {"observations"}
+
+# The exact provider paths this policy may read. Adding one is a reviewed source edit.
+_SOURCE_JSON_PATHS = (
+    "$.dcat.distribution[*].format",
+    "$.dcat.distribution[*].mediaType",
+    "$.dcat.license",
+    "$.dcat.rights",
+    "$.dcat.spatial",
+    "$.dcat.spatial[*]",
+    "$.description",
+    "$.identifier",
+    "$.keyword[*]",
+    "$.publisher",
+    "$.theme[*]",
+    "$.title",
+)
+
+# Catalog-ingestion observations belong in a separate ``observations`` partition
+# precisely because they are not facts about a source, so a semantic ``fields`` partition that
+# carries one has been tampered with and the record is refused outright.
+_OBSERVATION_JSON_PATHS = ("$.last_harvested_date", "$.popularity")
+
+# Locator and catalog-ingestion paths this policy never reads. The raw evidence retains the
+# locator values in raw and normalized evidence as inert bounded text; they are simply never facts,
+# never provenance, and never fetched.
+_REFUSED_JSON_PATHS = (
+    "$.dcat.describedBy",
+    "$.dcat.distribution[*].accessURL",
+    "$.dcat.distribution[*].describedBy",
+    "$.dcat.distribution[*].downloadURL",
+    "$.dcat.landingPage",
+    "$.dcat.references",
+    "$.harvest_record",
+    "$.harvest_record_raw",
+    "$.harvest_record_transformed",
+    "$.last_harvested_date",
+    "$.organization.logo",
+    "$.popularity",
+)
+
+# Exact declared license values only. Public-domain dedications carry no obligation and may be
+# ``approved``; attribution licenses are ``conditional`` and therefore still flagged, because v2
+# has no obligation fact that could carry the attribution requirement.
+_RIGHTS_MAP = (
+    ("http://creativecommons.org/licenses/by/4.0/", "conditional"),
+    ("http://creativecommons.org/publicdomain/zero/1.0/", "approved"),
+    ("http://www.usa.gov/publicdomain/label/1.0/", "approved"),
+    ("https://creativecommons.org/licenses/by/4.0/", "conditional"),
+    ("https://creativecommons.org/publicdomain/zero/1.0/", "approved"),
+    ("https://www.usa.gov/publicdomain/label/1.0/", "approved"),
+)
+
+# Exact declared distribution ``format``/``mediaType`` spellings that name one wire encoding the
+# harness can actually read. Anything else -- HTML, PDF, ZIP, API, XML, GeoJSON -- is deliberately
+# unmapped: an unrecognized format is not a format we may claim.
+_FORMAT_MAP = (
+    ("CSV", "csv"),
+    ("JSON", "json"),
+    ("NDJSON", "ndjson"),
+    ("PARQUET", "parquet"),
+    ("application/json", "json"),
+    ("application/vnd.apache.parquet", "parquet"),
+    ("application/x-ndjson", "ndjson"),
+    ("csv", "csv"),
+    ("json", "json"),
+    ("ndjson", "ndjson"),
+    ("parquet", "parquet"),
+    ("text/csv", "csv"),
+)
+
+
+class AuthoringPolicyRefused(SourceContractError):
+    """A stable refusal of an authoring policy coordinate or contract."""
+
+
+@dataclass(frozen=True)
+class AuthoringPolicy(_CanonicalContract):
+    """One reviewed, allowlisted, digest-bound provider authoring policy."""
+
+    policy_id: str
+    provider_id: str
+    harvester_coordinate: str
+    record_schema_version: str
+    entry_schema_version: str
+    entry_id_prefix: str
+    identifier_json_path: str
+    source_json_paths: tuple[str, ...]
+    refused_json_paths: tuple[str, ...]
+    observation_json_paths: tuple[str, ...]
+    rights_map: tuple[tuple[str, str], ...]
+    format_map: tuple[tuple[str, str], ...]
+    max_layer_text_bytes: int
+    schema_version: str = AUTHORING_POLICY_SCHEMA
+
+    def __post_init__(self) -> None:
+        contract = PROVIDER_CONTRACTS.get(self.provider_id)
+        if (
+            self.schema_version != AUTHORING_POLICY_SCHEMA
+            or contract is None
+            or self.harvester_coordinate != contract.harvester_coordinate
+            or self.identifier_json_path != contract.identifier_json_path
+        ):
+            raise AuthoringPolicyRefused(
+                "AUTHOR_POLICY_CONTRACT",
+                "policy.provider_id",
+                "policy must bind one registered provider contract",
+            )
+        source = set(self.source_json_paths)
+        refused = set(self.refused_json_paths)
+        if (
+            self.source_json_paths != tuple(sorted(source))
+            or self.refused_json_paths != tuple(sorted(refused))
+            or self.observation_json_paths != tuple(sorted(set(self.observation_json_paths)))
+            or not source
+            or not refused
+            or not self.observation_json_paths
+            or not source.isdisjoint(refused)
+            or not set(self.observation_json_paths) <= refused
+            or self.identifier_json_path not in source
+        ):
+            raise AuthoringPolicyRefused(
+                "AUTHOR_POLICY_CONTRACT",
+                "policy.source_json_paths",
+                "source and refused paths must be sorted, unique, and disjoint",
+            )
+        if self.rights_map != tuple(sorted(set(self.rights_map))) or any(
+            status not in RIGHTS_STATUSES for _value, status in self.rights_map
+        ):
+            raise AuthoringPolicyRefused(
+                "AUTHOR_POLICY_CONTRACT",
+                "policy.rights_map",
+                "rights mappings must be sorted, unique, and closed",
+            )
+        if self.format_map != tuple(sorted(set(self.format_map))) or any(
+            data_format not in DATA_FORMATS for _value, data_format in self.format_map
+        ):
+            raise AuthoringPolicyRefused(
+                "AUTHOR_POLICY_CONTRACT",
+                "policy.format_map",
+                "format mappings must be sorted, unique, and closed",
+            )
+        if type(self.max_layer_text_bytes) is not int or not 0 < self.max_layer_text_bytes <= (
+            MAX_LAYER_TEXT_BYTES
+        ):
+            raise AuthoringPolicyRefused(
+                "AUTHOR_POLICY_CONTRACT",
+                "policy.max_layer_text_bytes",
+                "layer text cap must stay inside the shared encode bound",
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "policy_id": self.policy_id,
+            "provider_id": self.provider_id,
+            "harvester_coordinate": self.harvester_coordinate,
+            "record_schema_version": self.record_schema_version,
+            "entry_schema_version": self.entry_schema_version,
+            "entry_id_prefix": self.entry_id_prefix,
+            "identifier_json_path": self.identifier_json_path,
+            "source_json_paths": list(self.source_json_paths),
+            "refused_json_paths": list(self.refused_json_paths),
+            "observation_json_paths": list(self.observation_json_paths),
+            "rights_map": [
+                {"declared": value, "status": status} for value, status in self.rights_map
+            ],
+            "format_map": [
+                {"declared": value, "data_format": data_format}
+                for value, data_format in self.format_map
+            ],
+            "max_layer_text_bytes": self.max_layer_text_bytes,
+        }
+
+    def rights_for(self, declared: str) -> str | None:
+        return dict(self.rights_map).get(declared)
+
+    def data_format_for(self, declared: str) -> str | None:
+        return dict(self.format_map).get(declared)
+
+
+DATAGOV_V4_AUTHORING_POLICY = AuthoringPolicy(
+    policy_id=DATAGOV_V4_POLICY_ID,
+    provider_id="datagov_v4",
+    harvester_coordinate=DATAGOV_V4_HARVESTER_COORDINATE,
+    record_schema_version=DATAGOV_V4_RECORD_VERSION,
+    entry_schema_version=CATALOG_ENTRY_V2_CONTRACT_VERSION,
+    entry_id_prefix="datagov_v4.",
+    identifier_json_path="$.identifier",
+    source_json_paths=_SOURCE_JSON_PATHS,
+    refused_json_paths=_REFUSED_JSON_PATHS,
+    observation_json_paths=_OBSERVATION_JSON_PATHS,
+    rights_map=_RIGHTS_MAP,
+    format_map=_FORMAT_MAP,
+    max_layer_text_bytes=MAX_LAYER_TEXT_BYTES,
+)
+
+AUTHORING_POLICIES: Mapping[str, AuthoringPolicy] = MappingProxyType(
+    {DATAGOV_V4_POLICY_ID: DATAGOV_V4_AUTHORING_POLICY}
+)
+
+
+def resolve_authoring_policy(policy_id: Any) -> AuthoringPolicy:
+    """Return the exact reviewed policy for one coordinate; never infer a near match."""
+
+    policy = AUTHORING_POLICIES.get(policy_id) if isinstance(policy_id, str) else None
+    if policy is None:
+        raise AuthoringPolicyRefused(
+            "AUTHOR_POLICY_UNKNOWN",
+            "policy_id",
+            "policy coordinate is not one reviewed allowlisted policy",
+        )
+    return policy
+
+
+@dataclass(frozen=True)
+class AuthoredRecord:
+    """One record's exact authoring outcome: an entry or an explicit refusal, never a guess."""
+
+    provider_record_id: str
+    disposition: str
+    reason_code: str
+    entry: CatalogEntryV2 | None
+    layer_texts: tuple[tuple[str, str], ...]
+    truncated_layers: tuple[str, ...]
+
+    @property
+    def entry_id(self) -> str | None:
+        return None if self.entry is None else self.entry.entry_id
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "provider_record_id": self.provider_record_id,
+            "disposition": self.disposition,
+            "reason_code": self.reason_code,
+            "entry": None if self.entry is None else self.entry.to_dict(),
+            "layer_texts": [{"layer": layer, "text": text} for layer, text in self.layer_texts],
+            "layer_text_truncated": list(self.truncated_layers),
+        }
+
+
+def author_catalog_record(
+    record: Mapping[str, Any],
+    *,
+    policy: AuthoringPolicy,
+    observed_at: str,
+    page_evidence_sha256: str,
+) -> AuthoredRecord:
+    """Map one normalized provider record onto one honest v2 entry and its four layer texts."""
+
+    envelope = _record_envelope(record, policy)
+    if envelope is not None:
+        return _refusal(record, "failed", envelope)
+    record_id = record["record_id"]
+    if canonical_sha256(_semantic_payload(record)) != record["normalized_sha256"]:
+        return _refusal(record, "failed", "AUTHOR_RECORD_DIGEST")
+    observations = {_base_path(path) for path in policy.observation_json_paths}
+    values: dict[str, Any] = {}
+    for field in record["fields"]:
+        if _base_path(field["path"]) in observations:
+            return _refusal(record, "failed", "AUTHOR_RECORD_FIELDS")
+        # A ``typed_json_ast.v1`` value carries a provider number too large for exact JSON. No fact
+        # this policy authors is numeric, so such a value simply declares nothing here.
+        if field.get("encoding", "canonical_json") == "canonical_json":
+            values[field["path"]] = field["value"]
+
+    if values.get(policy.identifier_json_path) != record_id or not _safe_text(
+        record_id, maximum=512
+    ):
+        return _refusal(record, "skipped", "AUTHOR_IDENTIFIER_UNSAFE")
+
+    title = values.get("$.title")
+    if not isinstance(title, str) or not title:
+        return _refusal(record, "skipped", "AUTHOR_TITLE_ABSENT")
+    if not _safe_text(title, maximum=len(title)):
+        return _refusal(record, "skipped", "AUTHOR_TITLE_UNSAFE")
+    if len(title) > MAX_TITLE:
+        return _refusal(record, "skipped", "AUTHOR_TITLE_LIMIT")
+
+    provenance = _provenance_factory(policy, record)
+    rights_fact, rights_reason = _rights(values, policy, provenance)
+    facts = {
+        "title": _known_text_fact(title, ("$.title",), provenance),
+        "publisher": _text_fact(values, "$.publisher", maximum=200, provenance=provenance),
+        "description": _text_fact(
+            values, "$.description", maximum=MAX_DESCRIPTION, provenance=provenance
+        ),
+        "spatial_scope": _text_tuple_fact(
+            values,
+            ("$.dcat.spatial", "$.dcat.spatial[*]"),
+            maximum=MAX_SPATIAL_SCOPE,
+            provenance=provenance,
+        ),
+        "data_formats": _data_formats(values, policy, provenance),
+        "declared_vocabulary": _text_tuple_fact(
+            values,
+            ("$.keyword[*]", "$.theme[*]"),
+            maximum=MAX_DECLARED_VOCABULARY,
+            provenance=provenance,
+        ),
+        "rights": rights_fact,
+        # No v4 field asserts any of these, so a closed policy states exactly that. ``accessLevel``
+        # is a disclosure classification, not one of the harness access kinds; the provider
+        # declares no column list, row count, profile, or dataset authentication requirement.
+        "access_kind": CatalogFact(state="unknown"),
+        "authentication_required": CatalogFact(state="unknown"),
+        "declared_columns": CatalogFact(state="unknown"),
+        "declared_row_count": CatalogFact(state="unknown"),
+        "profiles": CatalogFact(state="unknown"),
+    }
+    try:
+        entry = CatalogEntryV2(
+            entry_id=f"{policy.entry_id_prefix}{sha256_bytes(record_id.encode('utf-8'))}",
+            entry_version=1,
+            provider_record=ProviderRecordIdentity(
+                provider_id=policy.provider_id,
+                provider_record_id=record_id,
+                provider_record_sha256=record["normalized_sha256"],
+                identifier_json_path=policy.identifier_json_path,
+            ),
+            observations=(
+                CatalogObservation(
+                    harvester_coordinate=policy.harvester_coordinate,
+                    observed_at=observed_at,
+                    response_evidence_sha256=record["raw_response_sha256"],
+                    page_evidence_sha256=page_evidence_sha256,
+                ),
+            ),
+            **facts,
+        )
+    except SourceContractError:
+        return _refusal(record, "failed", "AUTHOR_ENTRY_CONTRACT")
+
+    texts, truncated = _layer_texts(entry, policy)
+    admitted = rights_fact.state == "known" and rights_fact.value == "approved"
+    return AuthoredRecord(
+        provider_record_id=record_id,
+        disposition="authored" if admitted else "flagged",
+        reason_code=rights_reason,
+        entry=entry,
+        layer_texts=texts,
+        truncated_layers=truncated,
+    )
+
+
+def _record_envelope(record: Mapping[str, Any], policy: AuthoringPolicy) -> str | None:
+    if not isinstance(record, Mapping) or not _REQUIRED_RECORD_MEMBERS <= set(record) <= (
+        _RECORD_MEMBERS
+    ):
+        return "AUTHOR_RECORD_SCHEMA"
+    if (
+        record["schema_version"] != policy.record_schema_version
+        or record["protocol"] != policy.provider_id
+        or not isinstance(record["record_id"], str)
+        or not record["record_id"]
+        or not isinstance(record["fields"], list)
+        or not record["fields"]
+        or not _is_digest(record["raw_response_sha256"])
+        or not _is_digest(record["normalized_sha256"])
+        or not isinstance(record.get("observations", []), list)
+    ):
+        return "AUTHOR_RECORD_SCHEMA"
+    for field in record["fields"]:
+        if (
+            not isinstance(field, Mapping)
+            or not {"path", "value"} <= set(field) <= {"path", "value", "encoding"}
+            or not isinstance(field["path"], str)
+        ):
+            return "AUTHOR_RECORD_SCHEMA"
+    return None
+
+
+def _semantic_payload(record: Mapping[str, Any]) -> dict[str, Any]:
+    return {
+        "schema_version": record["schema_version"],
+        "protocol": record["protocol"],
+        "record_id": record["record_id"],
+        "fields": record["fields"],
+    }
+
+
+def _refusal(record: Mapping[str, Any], disposition: str, reason_code: str) -> AuthoredRecord:
+    identifier = record.get("record_id") if isinstance(record, Mapping) else None
+    return AuthoredRecord(
+        provider_record_id=identifier if isinstance(identifier, str) else "",
+        disposition=disposition,
+        reason_code=reason_code,
+        entry=None,
+        layer_texts=(),
+        truncated_layers=(),
+    )
+
+
+def _provenance_factory(policy: AuthoringPolicy, record: Mapping[str, Any]):
+    def build(paths: tuple[str, ...]) -> ProviderFactProvenance:
+        return ProviderFactProvenance(
+            provider_id=policy.provider_id,
+            provider_record_id=record["record_id"],
+            provider_record_sha256=record["normalized_sha256"],
+            json_paths=tuple(sorted(set(paths))),
+        )
+
+    return build
+
+
+def _known_text_fact(value: str, paths: tuple[str, ...], provenance) -> CatalogFact:
+    return CatalogFact(state="known", value=value, provenance=provenance(paths))
+
+
+def _text_fact(values: Mapping[str, Any], path: str, *, maximum: int, provenance) -> CatalogFact:
+    value = values.get(path)
+    if not isinstance(value, str) or not _safe_text(value, maximum=maximum):
+        return CatalogFact(state="unknown")
+    return _known_text_fact(value, (path,), provenance)
+
+
+def _text_tuple_fact(
+    values: Mapping[str, Any], paths: tuple[str, ...], *, maximum: int, provenance
+) -> CatalogFact:
+    """Admit exactly the declared values that satisfy the contract; an empty subset is unknown."""
+
+    admitted: set[str] = set()
+    cited: list[str] = []
+    for path in paths:
+        declared = _declared_strings(values.get(path))
+        safe = {item for item in declared if _safe_text(item, maximum=MAX_DECLARED_NAME)}
+        if safe:
+            admitted |= safe
+            cited.append(path)
+    if not admitted or len(admitted) > maximum:
+        return CatalogFact(state="unknown")
+    return CatalogFact(
+        state="known", value=tuple(sorted(admitted)), provenance=provenance(tuple(cited))
+    )
+
+
+def _data_formats(values: Mapping[str, Any], policy: AuthoringPolicy, provenance) -> CatalogFact:
+    admitted: set[str] = set()
+    cited: list[str] = []
+    for path in ("$.dcat.distribution[*].format", "$.dcat.distribution[*].mediaType"):
+        mapped = {
+            policy.data_format_for(item)
+            for item in _declared_strings(values.get(path))
+            if policy.data_format_for(item) is not None
+        }
+        if mapped:
+            admitted |= {value for value in mapped if value is not None}
+            cited.append(path)
+    if not admitted:
+        return CatalogFact(state="unknown")
+    return CatalogFact(
+        state="known", value=tuple(sorted(admitted)), provenance=provenance(tuple(cited))
+    )
+
+
+def _rights(
+    values: Mapping[str, Any], policy: AuthoringPolicy, provenance
+) -> tuple[CatalogFact, str]:
+    """Map an exact declared license, or flag. Declared prose never authorizes anything."""
+
+    prose = values.get("$.dcat.rights")
+    if isinstance(prose, str) and prose.strip():
+        return CatalogFact(state="unknown"), "AUTHOR_RIGHTS_PROSE"
+    declared = values.get("$.dcat.license")
+    if not isinstance(declared, str) or not declared:
+        return CatalogFact(state="unknown"), "AUTHOR_RIGHTS_ABSENT"
+    status = policy.rights_for(declared)
+    if status is None:
+        return CatalogFact(state="unknown"), "AUTHOR_RIGHTS_UNMAPPED"
+    fact = CatalogFact(state="known", value=status, provenance=provenance(("$.dcat.license",)))
+    reason = {
+        "approved": "AUTHOR_OK",
+        "conditional": "AUTHOR_RIGHTS_CONDITIONAL",
+        "prohibited": "AUTHOR_RIGHTS_PROHIBITED",
+        "unclear": "AUTHOR_RIGHTS_UNMAPPED",
+    }[status]
+    return fact, reason
+
+
+def _layer_texts(
+    entry: CatalogEntryV2, policy: AuthoringPolicy
+) -> tuple[tuple[tuple[str, str], ...], tuple[str, ...]]:
+    """Render exactly four layer texts, using canonical state tokens instead of invented prose."""
+
+    rendered = {
+        "description": f"{entry.title.value}\n{_one(entry.description)}",
+        "metadata": " ".join(
+            (
+                _one(entry.publisher),
+                *_many(entry.spatial_scope),
+                *_many(entry.data_formats),
+                _one(entry.access_kind),
+                _one(entry.rights),
+            )
+        ),
+        "columns": " ".join((*_many(entry.declared_columns), *_many(entry.declared_vocabulary))),
+        "profiles": " ".join(
+            (
+                "rows",
+                _one(entry.declared_row_count),
+                "authentication",
+                _one(entry.authentication_required),
+                "profiles",
+                *_many(entry.profiles),
+                "formats",
+                *_many(entry.data_formats),
+            )
+        ),
+    }
+    texts: list[tuple[str, str]] = []
+    truncated: list[str] = []
+    for layer in EMBEDDING_LAYERS:
+        text, was_truncated = _truncate_utf8(rendered[layer], policy.max_layer_text_bytes)
+        texts.append((layer, text))
+        if was_truncated:
+            truncated.append(layer)
+    return tuple(texts), tuple(truncated)
+
+
+def _one(fact: CatalogFact) -> str:
+    if fact.state == "not_applicable":
+        return LAYER_TEXT_NOT_APPLICABLE
+    if fact.state != "known":
+        return LAYER_TEXT_UNKNOWN
+    if type(fact.value) is bool:
+        return "true" if fact.value else "false"
+    return str(fact.value)
+
+
+def _many(fact: CatalogFact) -> tuple[str, ...]:
+    if fact.state == "not_applicable":
+        return (LAYER_TEXT_NOT_APPLICABLE,)
+    if fact.state != "known" or not isinstance(fact.value, tuple):
+        return (LAYER_TEXT_UNKNOWN,)
+    return fact.value
+
+
+def _truncate_utf8(value: str, maximum: int) -> tuple[str, bool]:
+    """Cut to at most ``maximum`` UTF-8 bytes without ever splitting a code point."""
+
+    encoded = value.encode("utf-8")
+    if len(encoded) <= maximum:
+        return value, False
+    cut = encoded[:maximum]
+    while cut:
+        try:
+            return cut.decode("utf-8"), True
+        except UnicodeDecodeError:
+            cut = cut[:-1]
+    return "", True
+
+
+def _declared_strings(value: Any) -> tuple[str, ...]:
+    """Read a declared scalar or array of strings; every other shape declares nothing."""
+
+    if isinstance(value, str):
+        return (value,)
+    if isinstance(value, list):
+        return tuple(item for item in value if isinstance(item, str))
+    return ()
+
+
+def _safe_text(value: str, *, maximum: int) -> bool:
+    if not value or len(value) > maximum:
+        return False
+    for character in value:
+        codepoint = ord(character)
+        if (
+            0xD800 <= codepoint <= 0xDFFF
+            or codepoint < 0x20
+            or 0x7F <= codepoint <= 0x9F
+            or (character.isspace() and character != " ")
+        ):
+            return False
+    return True
+
+
+def _base_path(path: str) -> str:
+    return path[:-3] if path.endswith("[*]") else path
+
+
+def _is_digest(value: Any) -> bool:
+    return (
+        isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
+    )