mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/fitness.py

@@ -0,0 +1,162 @@
+"""Deterministic source-fitness report construction and comparison."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from mostlyright.data_harness.local_contracts import SourceProposal
+from mostlyright.data_harness.sources.contracts import (
+    FITNESS_DIMENSIONS,
+    EvidenceReference,
+    FitnessEvidence,
+    FitnessReport,
+    RightsEvidence,
+    SourceContractError,
+    SourceObservation,
+    evaluate_obligations,
+)
+from mostlyright.data_harness.sources.rights_rule import rights_disposition
+
+
+@dataclass(frozen=True)
+class FitnessSignals:
+    """Worker-calculated or exact-review inputs for every required dimension."""
+
+    outcomes: tuple[tuple[str, bool], ...]
+    summaries: tuple[tuple[str, str], ...]
+    evidence: tuple[EvidenceReference, ...]
+
+    def __post_init__(self) -> None:
+        outcome_keys = [key for key, _ in self.outcomes]
+        summary_keys = [key for key, _ in self.summaries]
+        if set(outcome_keys) != FITNESS_DIMENSIONS or len(outcome_keys) != len(FITNESS_DIMENSIONS):
+            raise SourceContractError(
+                "FITNESS_SIGNALS",
+                "signals.outcomes",
+                "must contain each required fitness dimension exactly once",
+            )
+        if set(summary_keys) != FITNESS_DIMENSIONS or len(summary_keys) != len(FITNESS_DIMENSIONS):
+            raise SourceContractError(
+                "FITNESS_SIGNALS",
+                "signals.summaries",
+                "must contain each required fitness dimension exactly once",
+            )
+        if not self.evidence:
+            raise SourceContractError(
+                "FITNESS_EVIDENCE",
+                "signals.evidence",
+                "fitness requires immutable cited evidence",
+            )
+
+    def outcome_map(self) -> dict[str, bool]:
+        return dict(self.outcomes)
+
+    def summary_map(self) -> dict[str, str]:
+        return dict(self.summaries)
+
+
+def build_fitness_report(
+    *,
+    report_id: str,
+    question_id: str,
+    proposal: SourceProposal,
+    observation: SourceObservation,
+    rights: RightsEvidence,
+    intended_use: str,
+    attached_license_artifact_digests: tuple[str, ...],
+    signals: FitnessSignals,
+    require_timely: bool,
+    max_publication_delay_seconds: int,
+) -> FitnessReport:
+    """Build a complete cited report and derive its decision."""
+
+    if proposal.source_id != observation.source_id:
+        raise SourceContractError(
+            "SOURCE_BINDING",
+            "proposal.source_id",
+            "proposal and observation source IDs must match",
+        )
+    if proposal.observed_at != observation.observed_at:
+        raise SourceContractError(
+            "OBSERVATION_TIME_BINDING",
+            "proposal.observed_at",
+            "proposal and worker observation must refer to the same observation time",
+        )
+    if max_publication_delay_seconds < 0:
+        raise SourceContractError(
+            "DELAY_LIMIT",
+            "max_publication_delay_seconds",
+            "must not be negative",
+        )
+    if rights_disposition(rights.status) == "admit":
+        evaluate_obligations(
+            rights=rights,
+            lifecycle="candidate",
+            intended_use=intended_use,
+            attached_artifact_digests=attached_license_artifact_digests,
+        )
+
+    outcomes = signals.outcome_map()
+    summaries = signals.summary_map()
+    cited_digests = tuple(item.content_sha256 for item in signals.evidence)
+    if rights_disposition(rights.status) != "admit":
+        outcomes["rights"] = False
+        summaries["rights"] = f"rights status {rights.status} blocks source selection"
+    if require_timely and (
+        observation.live_status != "live"
+        or observation.publication_delay_seconds > max_publication_delay_seconds
+    ):
+        outcomes["liveness"] = False
+        outcomes["publication_delay"] = False
+        summaries["liveness"] = (
+            f"timely source required; observed endpoint is {observation.live_status}"
+        )
+        summaries["publication_delay"] = (
+            f"observed delay {observation.publication_delay_seconds}s exceeds "
+            f"{max_publication_delay_seconds}s"
+        )
+
+    dimensions = tuple(
+        FitnessEvidence(
+            dimension=dimension,
+            outcome="pass" if outcomes[dimension] else "fail",
+            summary=summaries[dimension],
+            evidence_digests=cited_digests,
+        )
+        for dimension in sorted(FITNESS_DIMENSIONS)
+    )
+    failed = [item.dimension for item in dimensions if item.outcome != "pass"]
+    if rights_disposition(rights.status) == "escalate":
+        decision = "human_escalation_required"
+        reason = "rights are unclear and require human legal/product escalation"
+    elif failed:
+        decision = "rejected"
+        reason = f"failed fitness dimensions: {', '.join(failed)}"
+    else:
+        decision = "selected"
+        reason = "all required fitness dimensions pass"
+    return FitnessReport(
+        report_id=report_id,
+        question_id=question_id,
+        source_id=proposal.source_id,
+        observed_at=observation.observed_at,
+        intended_use=intended_use,
+        dimensions=dimensions,
+        decision=decision,
+        decision_reason=reason,
+        rights_digest=rights.digest,
+        observation_digest=observation.digest,
+    )
+
+
+def select_one_source(reports: tuple[FitnessReport, ...]) -> FitnessReport:
+    """Require one and only one fully selected source."""
+
+    selected = tuple(report for report in reports if report.decision == "selected")
+    if len(selected) != 1:
+        raise SourceContractError(
+            "SOURCE_SELECTION",
+            "fitness_reports",
+            f"expected exactly one selected source, found {len(selected)}",
+        )
+    return selected[0]

mostlyright/data_harness/sources/governance.py

@@ -0,0 +1,163 @@
+"""Pre-persistence classification and governance controls."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from mostlyright.data_harness.canonical import sha256_bytes
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+# Sized to the hosted contract's normalized-output maximum: the classification scan runs
+# over the exact bytes a hosted acquisition persists, and a year-scale normalized source
+# is gigabytes.  The scan stays one bounded pass; measured at ~10 MB/s on digit-heavy
+# tables (the payment-card scan dominates), so the ceiling costs minutes, not hours --
+# the guard is against unbounded input, not against large input.
+MAX_CLASSIFICATION_SCAN_BYTES = 4_294_967_296
+
+_SECRET_PATTERNS = (
+    ("private_key", re.compile(rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")),
+    ("aws_access_key", re.compile(rb"\bAKIA[0-9A-Z]{16}\b")),
+    ("github_token", re.compile(rb"\bgh[pousr]_[A-Za-z0-9_]{30,255}\b")),
+    ("bearer_token", re.compile(rb"(?i)\bauthorization\s*:\s*bearer\s+\S{16,}")),
+    (
+        "credential_assignment",
+        re.compile(
+            rb"""(?ix)
+            \b(?:password|passwd|api[_-]?key|secret)\b
+            ["']?\s*[=:]\s*["']?\S{8,}
+            """
+        ),
+    ),
+)
+_HIGHLY_SENSITIVE_PATTERNS = (
+    (
+        "us_social_security_number",
+        re.compile(rb"(?<![0-9])(?:[0-6][0-9]{2}|7[0-6][0-9])-[0-9]{2}-[0-9]{4}(?![0-9])"),
+    ),
+    (
+        "health_record_field",
+        re.compile(rb"(?i)\b(?:diagnosis|medical_record|patient_id|health_condition)\b"),
+    ),
+)
+# The boundaries exclude adjacent letters as well as digits: a primary account number is
+# never embedded inside a longer alphanumeric token, while a SHA-256 digest or a hex
+# identifier routinely carries a thirteen-plus digit run that happens to pass Luhn -- the
+# ECB ICP flow's structure digest opens with sixteen such digits, and a digit-only boundary
+# refused a catalog entry over its own evidence digest. A separator-grouped or
+# punctuation-adjacent number still matches, because quotes, commas, and whitespace are not
+# alphanumeric.
+_PAYMENT_CARD_PATTERN = re.compile(rb"(?<![0-9A-Za-z])(?:[0-9][ -]?){13,19}(?![0-9A-Za-z])")
+
+
+def _luhn_valid(value: bytes) -> bool:
+    digits = [byte - ord("0") for byte in value if ord("0") <= byte <= ord("9")]
+    if not 13 <= len(digits) <= 19:
+        return False
+    total = 0
+    parity = len(digits) % 2
+    for index, digit in enumerate(digits):
+        if index % 2 == parity:
+            digit *= 2
+            if digit > 9:
+                digit -= 9
+        total += digit
+    return total % 10 == 0
+
+
+@dataclass(frozen=True)
+class ClassificationFinding:
+    code: str
+    byte_offset: int
+
+    def __post_init__(self) -> None:
+        if not self.code or type(self.byte_offset) is not int or self.byte_offset < 0:
+            raise SourceContractError(
+                "CLASSIFICATION_FINDING",
+                "classification.finding",
+                "finding code and byte offset must be valid",
+            )
+
+
+@dataclass(frozen=True)
+class ClassificationResult:
+    content_sha256: str
+    classification: str
+    admitted: bool
+    findings: tuple[ClassificationFinding, ...]
+
+
+def classify_before_persistence(
+    content: bytes,
+    *,
+    declared_classification: str,
+    lawful_basis_declared: bool,
+) -> ClassificationResult:
+    """Scan bounded bytes before an admitted snapshot is persisted."""
+
+    if not isinstance(content, bytes):
+        raise SourceContractError("TYPE", "content", "classification content must be bytes")
+    if len(content) > MAX_CLASSIFICATION_SCAN_BYTES:
+        raise SourceContractError(
+            "CLASSIFICATION_LIMIT",
+            "content",
+            f"classification input exceeds {MAX_CLASSIFICATION_SCAN_BYTES} bytes",
+        )
+    if declared_classification not in {"public", "internal", "confidential", "restricted"}:
+        raise SourceContractError(
+            "CLASSIFICATION",
+            "declared_classification",
+            "must be public, internal, confidential, or restricted",
+        )
+    findings: list[ClassificationFinding] = []
+    for code, pattern in (*_SECRET_PATTERNS, *_HIGHLY_SENSITIVE_PATTERNS):
+        match = pattern.search(content)
+        if match:
+            findings.append(ClassificationFinding(code=code, byte_offset=match.start()))
+    payment_match = next(
+        (match for match in _PAYMENT_CARD_PATTERN.finditer(content) if _luhn_valid(match.group())),
+        None,
+    )
+    if payment_match is not None:
+        findings.append(
+            ClassificationFinding(
+                code="payment_card_candidate",
+                byte_offset=payment_match.start(),
+            )
+        )
+    if not lawful_basis_declared:
+        findings.append(ClassificationFinding(code="lawful_basis_missing", byte_offset=0))
+    admitted = not findings
+    return ClassificationResult(
+        content_sha256=sha256_bytes(content),
+        classification=declared_classification if admitted else "rejected",
+        admitted=admitted,
+        findings=tuple(findings),
+    )
+
+
+def require_admissible(result: ClassificationResult) -> None:
+    if not result.admitted:
+        codes = sorted(item.code for item in result.findings)
+        raise SourceContractError(
+            "DATA_CLASSIFICATION_REJECTED",
+            "classification",
+            f"content must not persist; findings={codes}",
+        )
+
+
+def classify_and_require_admissible(
+    content: bytes,
+    *,
+    declared_classification: str,
+    lawful_basis_declared: bool,
+) -> ClassificationResult:
+    """Return the sole admissible classification evidence for these exact bytes."""
+
+    result = classify_before_persistence(
+        content,
+        declared_classification=declared_classification,
+        lawful_basis_declared=lawful_basis_declared,
+    )
+    require_admissible(result)
+    return result

mostlyright/data_harness/sources/hosted.py

@@ -0,0 +1,173 @@
+"""Studio-hosted public HTTPS adapter.
+
+Kept outside ``sources.adapters`` because the generated Studio boundary imports recipe contracts,
+which in turn import the core source package. This leaf module may depend on both without making
+the core adapter import graph cyclic.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from uuid import UUID
+
+from mostlyright.data_harness.acquisition.http import seal_content_addressed_snapshot
+from mostlyright.data_harness.canonical import canonical_sha256
+from mostlyright.data_harness.local_contracts import ContractError, validate_source_locator
+from mostlyright.data_harness.sources._adapter_steps import (
+    DecodeRecord,
+    _authorize_candidate,
+    _output,
+)
+from mostlyright.data_harness.sources.adapters import (
+    AdapterGovernance,
+    _bind_request,
+    _https_query,
+)
+from mostlyright.data_harness.sources.collections import COLLECTION_ADAPTER_ID
+from mostlyright.data_harness.sources.contracts import (
+    AcquisitionRequest,
+    AdapterDescriptor,
+    SourceContractError,
+)
+from mostlyright.data_harness.sources.governance import (
+    classify_before_persistence,
+    require_admissible,
+)
+from mostlyright.data_harness.sources.registry import AcquisitionContext, AcquisitionOutput
+from mostlyright.data_harness.ux.hosted_acquisition import (
+    StudioExchange,
+    acquire_public_https,
+    source_authority_digest,
+)
+
+
+@dataclass(frozen=True)
+class HostedPublicHttpsAdapter:
+    """Public HTTPS adapter executed by Studio's credential-free hosted clean room."""
+
+    exchange: StudioExchange
+    workspace_id: UUID
+    governance: AdapterGovernance
+    descriptor: AdapterDescriptor
+
+    def __post_init__(self) -> None:
+        if self.descriptor.adapter_id == COLLECTION_ADAPTER_ID:
+            raise SourceContractError(
+                "HOSTED_COLLECTION_UNSUPPORTED",
+                "adapter.descriptor",
+                "bounded collections require the local attested acquisition route",
+            )
+        if self.descriptor.source_class not in {"user_url", "user_api", "external_adapter"}:
+            raise SourceContractError(
+                "ADAPTER_CLASS",
+                "adapter.descriptor.source_class",
+                "hosted public HTTPS adapter requires user_url, user_api, or external_adapter",
+            )
+
+    def acquire(
+        self,
+        request: AcquisitionRequest,
+        context: AcquisitionContext,
+    ) -> AcquisitionOutput:
+        _bind_request(request, self.descriptor)
+        if request.credential_reference_id is not None:
+            raise SourceContractError(
+                "AUTHENTICATED_SOURCE_REQUIRES_TRUSTED_GATEWAY",
+                "request.credential_reference_id",
+                "hosted public acquisition receives no source credential",
+            )
+        url, data_format, filename = _https_query(request.query)
+        try:
+            validate_source_locator(url, "https_url", "request.query.url")
+        except ContractError as error:
+            raise SourceContractError(
+                "HTTPS_URL",
+                "request.query.url",
+                "hosted public acquisition requires a credential-free canonical URL",
+            ) from error
+        _authorize_candidate(request, self.governance)
+        query = {
+            "url": url,
+            "data_format": data_format,
+            "filename": filename,
+            "reader_pin": None if request.reader_pin is None else dict(request.reader_pin),
+            "resource_caps": (
+                None if request.resource_caps is None else dict(request.resource_caps)
+            ),
+            "limits": {
+                "max_source_bytes": context.max_source_bytes,
+                "max_normalized_bytes": (
+                    context.max_source_bytes
+                    if context.max_normalized_bytes is None
+                    else context.max_normalized_bytes
+                ),
+                "max_rows": context.max_rows,
+                "max_columns": context.max_columns,
+            },
+        }
+        acquired = acquire_public_https(
+            exchange=self.exchange,
+            workspace_id=self.workspace_id,
+            source_id=request.source_id,
+            source_authority_digest=source_authority_digest(request.source_id, query),
+            acquisition_attempt_id=context.attempt_id,
+            query=query,
+        )
+        classification = classify_before_persistence(
+            acquired.content,
+            declared_classification=self.governance.declared_classification,
+            lawful_basis_declared=self.governance.lawful_basis_declared,
+        )
+        require_admissible(classification)
+        snapshot_path = seal_content_addressed_snapshot(
+            context.snapshot_root,
+            content=acquired.content,
+            suffix=data_format,
+        )
+        decode = None
+        if acquired.family_id is not None:
+            if acquired.family_version is None or acquired.decode_options_digest is None:
+                raise SourceContractError(
+                    "HOSTED_RESULT_BINDING",
+                    "hosted.decode",
+                    "hosted acquisition returned an incomplete Reader identity",
+                )
+            decode = DecodeRecord(
+                fetched_content_sha256=acquired.raw_content_sha256,
+                family_id=acquired.family_id,
+                family_version=acquired.family_version,
+                decode_options_digest=acquired.decode_options_digest,
+                flags=acquired.decode_flags,
+            )
+        transport_envelope = canonical_sha256(
+            {
+                "schema_version": "hosted-public-https-transport-envelope.v1",
+                "hosted_request_digest": acquired.request_digest,
+                "crawler_transport_evidence_digest": acquired.transport_evidence_digest,
+                "egress_policy_attestation": acquired.egress_policy_attestation,
+                "clean_room": {
+                    "host_platform": "linux-amd64",
+                    "parser_network_syscalls": "denied",
+                    "parser_process_creation": "denied",
+                    "no_new_privs": True,
+                },
+            }
+        )
+        return _output(
+            request=request,
+            descriptor=self.descriptor,
+            governance=self.governance,
+            snapshot_path=snapshot_path,
+            content=acquired.content,
+            data_format=acquired.data_format,
+            media_type=acquired.media_type,
+            source_uri=acquired.source_url,
+            final_uri=acquired.final_url,
+            classification=classification.classification,
+            transport_evidence_digest=transport_envelope,
+            parsed_schema_digest=acquired.parsed_schema_digest,
+            decode=decode,
+        )
+
+
+__all__ = ["HostedPublicHttpsAdapter"]