mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/rights_decisions.py

@@ -0,0 +1,254 @@
+"""Human rights decisions that move flagged provider records out of the queue.
+
+A provider that states no recognized licence puts every one of its records into the rights
+queue, and the queue has no exit inside the machine, because rights are a human question the
+harvest cannot answer. This module is the exit. One decision document carries, per record, a
+complete rights claim a person authored, the digest of the exact published facts that person
+looked at, a reference to where the decision was made, and when it was made. The delta
+classifier then admits an authored candidate for a cleared record only when the candidate's
+rights are exactly the decided rights, and refuses the moment the provider's published facts
+drift from what the person saw -- a decision is about one spelling of a record's facts, never
+about a record id forever. The binding is the record's ``facts_sha256``, deliberately not its
+full observation digest: a byte-identical record re-observed at a new time, or sharing a page
+with an unrelated changed record, does not void a standing decision, because the person
+decided about the facts and the facts have not moved.
+
+Two decisions exist and both are decisions. ``cleared`` says a person established a lawful
+basis; the record leaves the queue and demands a candidate. ``prohibited`` says a person
+established the opposite; the record stays out of admission with the decision on it, so the
+queue distinguishes "nobody has looked" from "somebody looked and said no". A decision whose
+rights say ``unclear`` is refused outright: deciding nothing is not a decision, and the queue
+already records that state without a document.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import stat
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import parse_canonical_json
+from mostlyright.data_harness.sources.catalog.contracts import rights_evidence_from_dict
+from mostlyright.data_harness.sources.contracts import (
+    RightsEvidence,
+    SourceContractError,
+    require_canonical_timestamp,
+)
+from mostlyright.data_harness.sources.rights_rule import rights_disposition
+
+RIGHTS_DECISIONS_SCHEMA = "mr-data-catalog-rights-decisions.v1"
+
+#: Decision documents are per-record prose and evidence references; four MiB holds thousands
+#: of reviewed decisions, and a document that cannot fit here is not a reviewed set of them.
+MAX_RIGHTS_DECISIONS_BYTES = 4 * 1024 * 1024
+
+_DIGEST_LENGTH = 64
+_MAX_RECORD_ID = 200
+_MAX_REFERENCE = 200
+
+
+def _is_digest(value: Any) -> bool:
+    return (
+        isinstance(value, str)
+        and len(value) == _DIGEST_LENGTH
+        and all(character in "0123456789abcdef" for character in value)
+    )
+
+
+def _bounded_text(value: Any, path: str, maximum: int) -> str:
+    if not isinstance(value, str) or not value.strip() or len(value) > maximum:
+        raise SourceContractError(
+            "RIGHTS_DECISION_FIELD",
+            path,
+            f"must be nonempty text of at most {maximum} characters",
+        )
+    return value
+
+
+@dataclass(frozen=True)
+class RightsDecision:
+    """One human rights decision, bound to the exact provider record it was made against."""
+
+    record_id: str
+    record_facts_sha256: str
+    human_decision_reference: str
+    decided_at: str
+    rights: RightsEvidence
+
+    def __post_init__(self) -> None:
+        _bounded_text(self.record_id, "rights_decision.record_id", _MAX_RECORD_ID)
+        if not _is_digest(self.record_facts_sha256):
+            raise SourceContractError(
+                "RIGHTS_DECISION_FIELD",
+                "rights_decision.record_facts_sha256",
+                "must be the lowercase SHA-256 of the decided record's published facts",
+            )
+        _bounded_text(
+            self.human_decision_reference,
+            "rights_decision.human_decision_reference",
+            _MAX_REFERENCE,
+        )
+        require_canonical_timestamp(self.decided_at, "rights_decision.decided_at")
+        if not isinstance(self.rights, RightsEvidence):
+            raise SourceContractError(
+                "RIGHTS_DECISION_FIELD",
+                "rights_decision.rights",
+                "must be a complete RightsEvidence",
+            )
+        disposition = rights_disposition(self.rights.status)
+        if disposition == "escalate":
+            raise SourceContractError(
+                "RIGHTS_DECISION_UNDECIDED",
+                "rights_decision.rights.status",
+                "a decision that says unclear decides nothing; the queue already records "
+                "that state without a document",
+            )
+        if disposition == "admit":
+            # An admitting claim carries the approval authority inside the rights, exactly as
+            # the entry it will become must carry it, and the two spellings must be one.
+            if self.rights.human_decision_reference != self.human_decision_reference:
+                raise SourceContractError(
+                    "RIGHTS_DECISION_BINDING",
+                    "rights_decision.rights.human_decision_reference",
+                    "the decided rights must carry exactly this decision's reference",
+                )
+        # A prohibiting claim cannot embed approval authority at all: the rights contract
+        # refuses that spelling at construction (RIGHTS_AUTHORITY_CONFLICT), so the
+        # prohibiting decision's reference lives on the decision alone by construction.
+
+    @property
+    def cleared(self) -> bool:
+        """True when this decision admits the record; false when it prohibits it."""
+
+        return rights_disposition(self.rights.status) == "admit"
+
+
+def _read_bounded_regular(path: Path) -> bytes:
+    flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(path, flags)
+    except OSError as error:
+        raise SourceContractError(
+            "RIGHTS_DECISIONS_UNAVAILABLE",
+            "rights_decisions",
+            "the decision document could not be opened as a regular file",
+        ) from error
+    try:
+        metadata = os.fstat(descriptor)
+        if (
+            not stat.S_ISREG(metadata.st_mode)
+            or metadata.st_nlink != 1
+            or metadata.st_size > MAX_RIGHTS_DECISIONS_BYTES
+        ):
+            raise SourceContractError(
+                "RIGHTS_DECISIONS_UNAVAILABLE",
+                "rights_decisions",
+                "the decision document must be one bounded single-link regular file",
+            )
+        chunks: list[bytes] = []
+        while True:
+            try:
+                chunk = os.read(descriptor, 1 << 16)
+            except OSError as error:
+                raise SourceContractError(
+                    "RIGHTS_DECISIONS_UNAVAILABLE",
+                    "rights_decisions",
+                    "the decision document could not be read",
+                ) from error
+            if not chunk:
+                break
+            chunks.append(chunk)
+            if sum(len(item) for item in chunks) > MAX_RIGHTS_DECISIONS_BYTES:
+                raise SourceContractError(
+                    "RIGHTS_DECISIONS_UNAVAILABLE",
+                    "rights_decisions",
+                    "the decision document must be one bounded single-link regular file",
+                )
+        return b"".join(chunks)
+    except OSError as error:
+        raise SourceContractError(
+            "RIGHTS_DECISIONS_UNAVAILABLE",
+            "rights_decisions",
+            "the decision document could not be read",
+        ) from error
+    finally:
+        os.close(descriptor)
+
+
+def load_rights_decisions(path: Path, expected_sha256: str) -> Mapping[str, RightsDecision]:
+    """Load one digest-bound decision document into typed decisions by record id."""
+
+    if not _is_digest(expected_sha256):
+        raise SourceContractError(
+            "RIGHTS_DECISIONS_DIGEST",
+            "rights_decisions_sha256",
+            "must be a lowercase SHA-256",
+        )
+    raw = _read_bounded_regular(path)
+    if hashlib.sha256(raw).hexdigest() != expected_sha256:
+        raise SourceContractError(
+            "RIGHTS_DECISIONS_DIGEST",
+            "rights_decisions",
+            "decision bytes differ from the required digest",
+        )
+    value = parse_canonical_json(raw)
+    expected_fields = {"schema_version", "decisions_by_record_id"}
+    if (
+        not isinstance(value, dict)
+        or set(value) != expected_fields
+        or value["schema_version"] != RIGHTS_DECISIONS_SCHEMA
+    ):
+        raise SourceContractError(
+            "RIGHTS_DECISIONS_SCHEMA",
+            "rights_decisions",
+            "decision fields or version differ",
+        )
+    raw_decisions = value["decisions_by_record_id"]
+    if not isinstance(raw_decisions, dict):
+        raise SourceContractError(
+            "RIGHTS_DECISIONS_SCHEMA",
+            "rights_decisions.decisions_by_record_id",
+            "must be a mapping of record ids to decisions",
+        )
+    decisions: dict[str, RightsDecision] = {}
+    for key, payload in raw_decisions.items():
+        if not isinstance(key, str) or not isinstance(payload, dict):
+            raise SourceContractError(
+                "RIGHTS_DECISIONS_SCHEMA",
+                "rights_decisions.decisions_by_record_id",
+                "keys must be record ids and values must be decision objects",
+            )
+        expected_decision_fields = {
+            "record_id",
+            "record_facts_sha256",
+            "human_decision_reference",
+            "decided_at",
+            "rights",
+        }
+        if set(payload) != expected_decision_fields:
+            raise SourceContractError(
+                "RIGHTS_DECISIONS_SCHEMA",
+                f"rights_decisions.decisions_by_record_id.{key}",
+                "decision fields differ",
+            )
+        if payload["record_id"] != key:
+            raise SourceContractError(
+                "RIGHTS_DECISIONS_SCHEMA",
+                f"rights_decisions.decisions_by_record_id.{key}",
+                "the mapping key must equal the decision's record id",
+            )
+        decisions[key] = RightsDecision(
+            record_id=payload["record_id"],
+            record_facts_sha256=payload["record_facts_sha256"],
+            human_decision_reference=payload["human_decision_reference"],
+            decided_at=payload["decided_at"],
+            rights=rights_evidence_from_dict(
+                payload["rights"],
+                path=f"rights_decisions.decisions_by_record_id.{key}.rights",
+            ),
+        )
+    return decisions