mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/gating.py

@@ -0,0 +1,374 @@
+"""The facts gate: decide whether a catalog entry may be used, on facts alone.
+
+Every decision here is a function of structured facts -- the rights status and its obligations,
+whether the source needs authentication, which payload formats it publishes, how often it updates,
+and whether it is answering. None of those is a matter of degree, and none of them is negotiable
+by how well a source matched a question.
+
+This module has no way to consult a similarity number. It does not import the module that
+computes one, or any module that touches a sealed embedding; its inputs cannot represent one; and
+its own function signatures have no parameter that could carry one. That is a structural property
+rather than a convention, and a test proves it by reading the import closure and the syntax tree
+rather than by reading this paragraph.
+
+Rights are not re-adjudicated here either. ``sources.rights_rule.rights_disposition`` is the one
+authority for what a rights status means, and ``evaluate_obligations`` is the one obligation
+evaluator. A second copy could disagree with the first, and the disagreement
+would be about whether someone may lawfully use a dataset.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from mostlyright.data_harness.sources.catalog.contracts import CatalogEntry
+from mostlyright.data_harness.sources.contracts import (
+    DATA_FORMATS,
+    OBLIGATION_LIFECYCLES,
+    SourceContractError,
+    _CanonicalContract,
+    evaluate_obligations,
+)
+from mostlyright.data_harness.sources.rights_rule import rights_disposition
+
+CATALOG_REQUIREMENT_CONTRACT_VERSION = "harness-source-catalog-requirement.v1"
+CATALOG_VERDICT_CONTRACT_VERSION = "harness-source-catalog-verdict.v1"
+
+# Ordered. A verdict's check list is digested, so its order must be fixed.
+CATALOG_CHECKS = (
+    "rights",
+    "obligations",
+    "authentication",
+    "data_format",
+    "cadence",
+    "liveness",
+)
+
+CATALOG_DISPOSITIONS = ("admitted", "refused", "human_escalation_required")
+
+MAX_DETAIL = 500
+MAX_REQUIRED_FORMATS = 16
+MAX_ATTACHED_DIGESTS = 32
+MAX_FREQUENCY_SECONDS = 10 * 365 * 24 * 60 * 60
+
+
+@dataclass(frozen=True)
+class CatalogRequirement(_CanonicalContract):
+    """What a caller needs from a source, expressed entirely as facts.
+
+    There is deliberately no threshold field. A threshold would be an admission decision taken by
+    degree of textual match, which is the one thing this gate exists to make impossible.
+    """
+
+    intended_use: str
+    lifecycle: str
+    required_data_formats: tuple[str, ...]
+    max_update_frequency_seconds: int
+    require_open_access: bool = True
+    require_live: bool = True
+    attached_license_artifact_digests: tuple[str, ...] = ()
+    schema_version: str = CATALOG_REQUIREMENT_CONTRACT_VERSION
+
+    def __post_init__(self) -> None:
+        _constant(
+            self.schema_version,
+            CATALOG_REQUIREMENT_CONTRACT_VERSION,
+            "requirement.schema_version",
+        )
+        _text(self.intended_use, "requirement.intended_use", maximum=200)
+        _choice(self.lifecycle, OBLIGATION_LIFECYCLES, "requirement.lifecycle")
+        _choices(
+            self.required_data_formats,
+            DATA_FORMATS,
+            "requirement.required_data_formats",
+            maximum=MAX_REQUIRED_FORMATS,
+        )
+        _integer(
+            self.max_update_frequency_seconds,
+            "requirement.max_update_frequency_seconds",
+            minimum=0,
+            maximum=MAX_FREQUENCY_SECONDS,
+        )
+        _boolean(self.require_open_access, "requirement.require_open_access")
+        _boolean(self.require_live, "requirement.require_live")
+        _digests(
+            self.attached_license_artifact_digests,
+            "requirement.attached_license_artifact_digests",
+            maximum=MAX_ATTACHED_DIGESTS,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "intended_use": self.intended_use,
+            "lifecycle": self.lifecycle,
+            "required_data_formats": list(self.required_data_formats),
+            "max_update_frequency_seconds": self.max_update_frequency_seconds,
+            "require_open_access": self.require_open_access,
+            "require_live": self.require_live,
+            "attached_license_artifact_digests": list(self.attached_license_artifact_digests),
+        }
+
+
+@dataclass(frozen=True)
+class CatalogCheck(_CanonicalContract):
+    """One named fact check and how it came out."""
+
+    check: str
+    outcome: str
+    detail: str
+
+    def __post_init__(self) -> None:
+        _choice(self.check, frozenset(CATALOG_CHECKS), "check.check")
+        _choice(self.outcome, frozenset({"pass", "fail"}), "check.outcome")
+        _text(self.detail, "check.detail", maximum=MAX_DETAIL)
+
+    def to_dict(self) -> dict[str, str]:
+        return {"check": self.check, "outcome": self.outcome, "detail": self.detail}
+
+
+@dataclass(frozen=True)
+class CatalogGateVerdict(_CanonicalContract):
+    """One entry's decision, with the citations that make it checkable later.
+
+    Three digests, because a decision has three inputs and citing two of them makes the record
+    ambiguous. ``entry_digest`` and ``rights_digest`` cite what was decided about;
+    ``requirement_digest`` cites what it was decided against, which supplies the other half of
+    every check here -- the lifecycle, the intended use, the required formats, the cadence ceiling
+    and the open-access and liveness switches. Without it the same entry yields ``admitted`` under
+    one requirement and ``refused`` under another and the two stored verdicts are indistinguishable
+    in their citations. ``registry.acquire``'s ``REQUEST_BINDING`` check is the same idea one layer
+    down.
+    """
+
+    entry_coordinate: str
+    disposition: str
+    reason: str
+    checks: tuple[CatalogCheck, ...]
+    entry_digest: str
+    rights_digest: str
+    requirement_digest: str
+    schema_version: str = CATALOG_VERDICT_CONTRACT_VERSION
+
+    def __post_init__(self) -> None:
+        _constant(
+            self.schema_version,
+            CATALOG_VERDICT_CONTRACT_VERSION,
+            "verdict.schema_version",
+        )
+        _text(self.entry_coordinate, "verdict.entry_coordinate", maximum=200)
+        _choice(self.disposition, frozenset(CATALOG_DISPOSITIONS), "verdict.disposition")
+        _text(self.reason, "verdict.reason", maximum=MAX_DETAIL)
+        if not isinstance(self.checks, tuple):
+            raise SourceContractError("TYPE", "verdict.checks", "must be an immutable tuple")
+        if tuple(item.check for item in self.checks) != CATALOG_CHECKS:
+            raise SourceContractError(
+                "CATALOG_CHECK_SET",
+                "verdict.checks",
+                f"must contain each of {list(CATALOG_CHECKS)} exactly once, in order",
+            )
+        _digest(self.entry_digest, "verdict.entry_digest")
+        _digest(self.rights_digest, "verdict.rights_digest")
+        _digest(self.requirement_digest, "verdict.requirement_digest")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "entry_coordinate": self.entry_coordinate,
+            "disposition": self.disposition,
+            "reason": self.reason,
+            "checks": [item.to_dict() for item in self.checks],
+            "entry_digest": self.entry_digest,
+            "rights_digest": self.rights_digest,
+            "requirement_digest": self.requirement_digest,
+        }
+
+
+def gate_entries(
+    entries: tuple[CatalogEntry, ...],
+    *,
+    requirement: CatalogRequirement,
+) -> tuple[CatalogGateVerdict, ...]:
+    """Decide each entry against the requirement. A pure function of facts.
+
+    Note what this signature cannot express: there is no argument through which a degree of
+    textual match could arrive, and neither ``CatalogEntry`` nor ``CatalogRequirement`` has a
+    field able to carry one.
+    """
+
+    if not isinstance(entries, tuple):
+        raise SourceContractError("TYPE", "entries", "must be an immutable tuple")
+    if not isinstance(requirement, CatalogRequirement):
+        raise SourceContractError("TYPE", "requirement", "must be a CatalogRequirement")
+    verdicts: list[CatalogGateVerdict] = []
+    for position, entry in enumerate(entries):
+        if not isinstance(entry, CatalogEntry):
+            raise SourceContractError(
+                "TYPE",
+                f"entries[{position}]",
+                "must be a CatalogEntry",
+            )
+        verdicts.append(_gate_one(entry, requirement))
+    return tuple(verdicts)
+
+
+def _gate_one(entry: CatalogEntry, requirement: CatalogRequirement) -> CatalogGateVerdict:
+    disposition_of_rights = rights_disposition(entry.rights.status)
+    outcomes: dict[str, tuple[bool, str]] = {}
+
+    outcomes["rights"] = (
+        disposition_of_rights == "admit",
+        f"rights status {entry.rights.status!r} is {disposition_of_rights}",
+    )
+
+    if disposition_of_rights == "admit":
+        try:
+            applicable = evaluate_obligations(
+                rights=entry.rights,
+                lifecycle=requirement.lifecycle,
+                intended_use=requirement.intended_use,
+                attached_artifact_digests=requirement.attached_license_artifact_digests,
+            )
+        except SourceContractError as error:
+            outcomes["obligations"] = (
+                False,
+                f"licence obligations are not satisfied: {error.detail}"[:MAX_DETAIL],
+            )
+        else:
+            outcomes["obligations"] = (
+                True,
+                f"{len(applicable)} licence obligation(s) apply and are satisfied",
+            )
+    else:
+        outcomes["obligations"] = (
+            True,
+            "not evaluated: the rights check already decided this entry",
+        )
+
+    needs_login = requirement.require_open_access and entry.authentication_required
+    outcomes["authentication"] = (
+        not needs_login,
+        "the source requires authentication and open access was required"
+        if needs_login
+        else "no authentication is required",
+    )
+
+    shared_formats = sorted(set(entry.data_formats) & set(requirement.required_data_formats))
+    outcomes["data_format"] = (
+        bool(shared_formats),
+        f"published formats {sorted(entry.data_formats)} "
+        f"{'include' if shared_formats else 'do not include'} "
+        f"a required format from {sorted(requirement.required_data_formats)}",
+    )
+
+    observed_frequency = entry.observation.update_frequency_seconds
+    outcomes["cadence"] = (
+        observed_frequency <= requirement.max_update_frequency_seconds,
+        f"the source updates every {observed_frequency}s against a required "
+        f"{requirement.max_update_frequency_seconds}s",
+    )
+
+    live_status = entry.observation.live_status
+    outcomes["liveness"] = (
+        not (requirement.require_live and live_status != "live"),
+        f"the last probe found the source {live_status!r}",
+    )
+
+    checks = tuple(
+        CatalogCheck(
+            check=name,
+            outcome="pass" if outcomes[name][0] else "fail",
+            detail=outcomes[name][1],
+        )
+        for name in CATALOG_CHECKS
+    )
+    failed = [item.check for item in checks if item.outcome != "pass"]
+
+    # Precedence mirrors the fitness adjudicator exactly: escalation first, then any failure.
+    if disposition_of_rights == "escalate":
+        disposition = "human_escalation_required"
+        reason = "rights are unclear and require human legal/product escalation"
+    elif failed:
+        disposition = "refused"
+        reason = f"failed checks: {', '.join(failed)}"
+    else:
+        disposition = "admitted"
+        reason = "every required fact check passes"
+
+    return CatalogGateVerdict(
+        entry_coordinate=entry.coordinate,
+        disposition=disposition,
+        reason=reason,
+        checks=checks,
+        entry_digest=entry.digest,
+        rights_digest=entry.rights.digest,
+        requirement_digest=requirement.digest,
+    )
+
+
+def _text(value: Any, path: str, *, maximum: int) -> str:
+    if not isinstance(value, str) or not value or len(value) > maximum:
+        raise SourceContractError(
+            "TEXT",
+            path,
+            f"must be a non-empty string no longer than {maximum} characters",
+        )
+    return value
+
+
+def _constant(value: Any, expected: str, path: str) -> None:
+    if value != expected:
+        raise SourceContractError("VERSION", path, f"must equal {expected!r}")
+
+
+def _choice(value: Any, allowed: frozenset[str] | set[str], path: str) -> str:
+    text = _text(value, path, maximum=128)
+    if text not in allowed:
+        raise SourceContractError("ENUM", path, f"must be one of {sorted(allowed)}")
+    return text
+
+
+def _choices(values: Any, allowed: frozenset[str], path: str, *, maximum: int) -> None:
+    if not isinstance(values, tuple):
+        raise SourceContractError("TYPE", path, "must be an immutable tuple")
+    if not values:
+        raise SourceContractError("EMPTY", path, "must not be empty")
+    if len(values) > maximum:
+        raise SourceContractError("LIMIT", path, f"must contain at most {maximum} values")
+    for position, value in enumerate(values):
+        _choice(value, allowed, f"{path}[{position}]")
+    if len(set(values)) != len(values):
+        raise SourceContractError("DUPLICATE", path, "values must be unique")
+
+
+def _digest(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=64)
+    if len(text) != 64 or any(char not in "0123456789abcdef" for char in text):
+        raise SourceContractError("SHA256", path, "must be a lowercase 64-character SHA-256")
+    return text
+
+
+def _digests(values: Any, path: str, *, maximum: int) -> None:
+    if not isinstance(values, tuple):
+        raise SourceContractError("TYPE", path, "must be an immutable tuple")
+    if len(values) > maximum:
+        raise SourceContractError("LIMIT", path, f"must contain at most {maximum} values")
+    for position, value in enumerate(values):
+        _digest(value, f"{path}[{position}]")
+
+
+def _integer(value: Any, path: str, *, minimum: int, maximum: int) -> int:
+    if type(value) is not int or not minimum <= value <= maximum:
+        raise SourceContractError(
+            "INTEGER",
+            path,
+            f"must be an integer in [{minimum}, {maximum}]",
+        )
+    return value
+
+
+def _boolean(value: Any, path: str) -> bool:
+    if type(value) is not bool:
+        raise SourceContractError("BOOLEAN", path, "must be a boolean")
+    return value