mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/__init__.py

@@ -0,0 +1,209 @@
+"""Versioned source adapter and evidence contracts.
+
+Acquisition runs in three steps. A caller builds an ``AcquisitionRequest`` and passes it with an
+``AcquisitionContext`` to ``AdapterRegistry.acquire``. The registry resolves the allowlisted
+adapter for the requested adapter id and version and calls its ``acquire`` method. The adapter
+returns an ``AcquisitionOutput`` holding the digest-bound receipt and the source observation, plus
+the optional parsed schema digest, content-addressed snapshot path, and next stream watermark.
+
+``docs/SOURCES-AND-ACQUISITION.md`` describes that boundary in full, including catalog-provider
+ownership, the sports comparison, stream batches, and deletion.
+"""
+
+from mostlyright.data_harness.sources.adapters import (
+    AdapterGovernance,
+    LocalFileAdapter,
+    ObservationPolicy,
+    PublicHttpsAdapter,
+    RangeReaderHttpsAdapter,
+)
+from mostlyright.data_harness.sources.cadence import (
+    SOURCE_CADENCE_ALGORITHM_DIGEST,
+    SOURCE_CADENCE_ALGORITHM_VERSION,
+    SOURCE_CADENCE_INFERENCE_SCHEMA,
+    SOURCE_CADENCE_OBSERVATION_SCHEMA,
+    CadenceContractError,
+    CadenceInference,
+    EmpiricalSourceObservation,
+    PartitionDigest,
+    SourceFrontier,
+    infer_source_cadence,
+    parse_cadence_inference,
+    parse_empirical_source_observation,
+    source_cadence_algorithm_spec,
+    validate_observation_history,
+)
+from mostlyright.data_harness.sources.collections import (
+    CollectionMember,
+    PublicHttpsCollectionAdapter,
+    collection_members,
+)
+from mostlyright.data_harness.sources.contracts import (
+    AcquisitionReceipt,
+    AcquisitionReceiptV2,
+    AcquisitionReceiptV3,
+    AcquisitionRequest,
+    AdapterDescriptor,
+    CollectionMemberEvidence,
+    DeletionTombstone,
+    EvidenceReference,
+    FetchedMember,
+    FitnessEvidence,
+    FitnessReport,
+    LicenseObligation,
+    RetentionPolicy,
+    RightsEvidence,
+    SnapshotReference,
+    SourceContractError,
+    SourceObservation,
+    evaluate_obligations,
+    parse_adapter_descriptor,
+)
+from mostlyright.data_harness.sources.deletion import (
+    DeletionCoordinator,
+    DeletionOutcome,
+    RetentionDecision,
+    load_sealed_tombstone,
+    project_availability,
+    require_source_available,
+    retention_decision,
+)
+from mostlyright.data_harness.sources.fitness import (
+    FitnessSignals,
+    build_fitness_report,
+    select_one_source,
+)
+from mostlyright.data_harness.sources.governance import (
+    ClassificationFinding,
+    ClassificationResult,
+    classify_and_require_admissible,
+    classify_before_persistence,
+    require_admissible,
+)
+from mostlyright.data_harness.sources.integration import (
+    AcquiredOneToOneJoinEvidence,
+    OneToOneJoinEvidence,
+    join_acquired_sources_one_to_one,
+    join_source_tables_one_to_one,
+)
+from mostlyright.data_harness.sources.registry import (
+    AcquisitionContext,
+    AcquisitionOutput,
+    AdapterRegistry,
+    SourceAdapter,
+)
+from mostlyright.data_harness.sources.sports import (
+    GovernedSportsPackage,
+    SportsComparison,
+    compare_recorded_sports_sources,
+    create_governed_sports_package,
+    openligadb_adapter,
+    openligadb_descriptor,
+    openligadb_rights,
+)
+from mostlyright.data_harness.sources.stream import (
+    StreamBatch,
+    StreamBatchAdapter,
+    StreamGateway,
+    StreamState,
+    StreamWatermarkStore,
+)
+from mostlyright.data_harness.sources.stream_connector import (
+    KNOWN_STREAM_CONNECTORS,
+    STREAM_CONNECTOR_IDS,
+    StreamConnectorConfig,
+    stream_connector,
+)
+from mostlyright.data_harness.sources.stream_recorder import (
+    CaptureManifest,
+    RecordedStreamGateway,
+    read_capture_manifest,
+    seal_recorded_capture,
+)
+
+__all__ = [
+    "KNOWN_STREAM_CONNECTORS",
+    "SOURCE_CADENCE_ALGORITHM_DIGEST",
+    "SOURCE_CADENCE_ALGORITHM_VERSION",
+    "SOURCE_CADENCE_INFERENCE_SCHEMA",
+    "SOURCE_CADENCE_OBSERVATION_SCHEMA",
+    "STREAM_CONNECTOR_IDS",
+    "AcquiredOneToOneJoinEvidence",
+    "AcquisitionContext",
+    "AcquisitionOutput",
+    "AcquisitionReceipt",
+    "AcquisitionReceiptV2",
+    "AcquisitionReceiptV3",
+    "AcquisitionRequest",
+    "AdapterDescriptor",
+    "AdapterGovernance",
+    "AdapterRegistry",
+    "CadenceContractError",
+    "CadenceInference",
+    "CaptureManifest",
+    "ClassificationFinding",
+    "ClassificationResult",
+    "CollectionMember",
+    "CollectionMemberEvidence",
+    "DeletionCoordinator",
+    "DeletionOutcome",
+    "DeletionTombstone",
+    "EmpiricalSourceObservation",
+    "EvidenceReference",
+    "FetchedMember",
+    "FitnessEvidence",
+    "FitnessReport",
+    "FitnessSignals",
+    "GovernedSportsPackage",
+    "LicenseObligation",
+    "LocalFileAdapter",
+    "ObservationPolicy",
+    "OneToOneJoinEvidence",
+    "PartitionDigest",
+    "PublicHttpsAdapter",
+    "PublicHttpsCollectionAdapter",
+    "RangeReaderHttpsAdapter",
+    "RecordedStreamGateway",
+    "RetentionDecision",
+    "RetentionPolicy",
+    "RightsEvidence",
+    "SnapshotReference",
+    "SourceAdapter",
+    "SourceContractError",
+    "SourceFrontier",
+    "SourceObservation",
+    "SportsComparison",
+    "StreamBatch",
+    "StreamBatchAdapter",
+    "StreamConnectorConfig",
+    "StreamGateway",
+    "StreamState",
+    "StreamWatermarkStore",
+    "build_fitness_report",
+    "classify_and_require_admissible",
+    "classify_before_persistence",
+    "collection_members",
+    "compare_recorded_sports_sources",
+    "create_governed_sports_package",
+    "evaluate_obligations",
+    "infer_source_cadence",
+    "join_acquired_sources_one_to_one",
+    "join_source_tables_one_to_one",
+    "load_sealed_tombstone",
+    "openligadb_adapter",
+    "openligadb_descriptor",
+    "openligadb_rights",
+    "parse_adapter_descriptor",
+    "parse_cadence_inference",
+    "parse_empirical_source_observation",
+    "project_availability",
+    "read_capture_manifest",
+    "require_admissible",
+    "require_source_available",
+    "retention_decision",
+    "seal_recorded_capture",
+    "select_one_source",
+    "source_cadence_algorithm_spec",
+    "stream_connector",
+    "validate_observation_history",
+]

mostlyright/data_harness/sources/_adapter_steps.py

@@ -0,0 +1,213 @@
+"""Adapter steps shared by every source adapter in this package.
+
+The file, HTTPS, external-provider, and stream adapters all authorize the candidate use, derive
+sandbox parse limits, require the coordinator-pinned sandbox policy, and build the receipt and
+observation the same way. Those four steps live here so no adapter can drift from them.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from mostlyright.data_harness.acquisition.http import ProbeTransport
+from mostlyright.data_harness.acquisition.parsing import ParseLimits
+from mostlyright.data_harness.canonical import sha256_bytes
+from mostlyright.data_harness.sources.contracts import (
+    AcquisitionReceipt,
+    AcquisitionRequest,
+    AdapterDescriptor,
+    SnapshotReference,
+    SourceContractError,
+    SourceObservation,
+    decode_options_digest,
+    evaluate_obligations,
+)
+from mostlyright.data_harness.sources.registry import (
+    AcquisitionContext,
+    AcquisitionOutput,
+)
+
+if TYPE_CHECKING:
+    from mostlyright.data_harness.sources.adapters import AdapterGovernance
+
+
+@dataclass(frozen=True)
+class DecodeRecord:
+    """What a Reader reports back to the adapter that ran it.
+
+    Local to ``sources/`` rather than imported from ``readers``, so the evidence layer
+    stays independent of the Toolbox implementation and the seam can be exercised with a
+    stub.  It carries the digest of the fetched bytes rather than the bytes themselves:
+    the sealed bytes are already in the adapter's hand and a second copy of the input
+    would only be a second thing that could go stale.
+    """
+
+    fetched_content_sha256: str
+    family_id: str
+    family_version: str
+    decode_options_digest: str
+    flags: tuple[str, ...] = ()
+
+
+def _require_decode_matches_pin(
+    request: AcquisitionRequest,
+    decode: DecodeRecord | None,
+) -> None:
+    """Refuse an acquisition whose decode contradicts what the request entitled.
+
+    The adapter-side twin of ``recipe._validate_reader_pin_binding``, and deliberately not
+    a replacement for it.  This one fires earlier and nearer the decode, which is where
+    the diagnosis is cheapest; the coordinator's own check stays as an independent second
+    gate reading the sealed receipt.  A bug then has to defeat both.
+
+    It fires in the same three directions the coordinator check covers: the request
+    entitles a decoder and none ran, a decoder ran that the request entitled none, and
+    both are stated but disagree.
+    """
+
+    pin = request.reader_pin
+    if pin is None and decode is None:
+        return
+    if pin is None:
+        raise SourceContractError(
+            "DECODE_PIN_MISMATCH",
+            "receipt.family_id",
+            "a decode was recorded for a request that entitles no Reader family",
+        )
+    if decode is None:
+        raise SourceContractError(
+            "DECODE_PIN_MISMATCH",
+            "request.reader_pin",
+            "the request entitles a Reader family that this acquisition did not run",
+        )
+    if (decode.family_id, decode.family_version) != (pin["family_id"], pin["family_version"]):
+        raise SourceContractError(
+            "DECODE_PIN_MISMATCH",
+            "receipt.family_id",
+            "the decode names a Reader family the request did not entitle",
+        )
+    if decode.decode_options_digest != decode_options_digest(pin["decode_options"]):
+        raise SourceContractError(
+            "DECODE_PIN_MISMATCH",
+            "receipt.decode_options_digest",
+            "the decode ran with settings other than the ones the request entitled",
+        )
+
+
+def _output(
+    *,
+    request: AcquisitionRequest,
+    descriptor: AdapterDescriptor,
+    governance: AdapterGovernance,
+    snapshot_path: Path,
+    content: bytes,
+    data_format: str,
+    media_type: str,
+    source_uri: str,
+    final_uri: str,
+    classification: str,
+    transport_evidence_digest: str,
+    parsed_schema_digest: str,
+    next_watermark: str | None = None,
+    decode: DecodeRecord | None = None,
+    probe_transport: ProbeTransport | None = None,
+    parsed_row_count: int | None = None,
+    parsed_row_digest: str | None = None,
+) -> AcquisitionOutput:
+    _require_decode_matches_pin(request, decode)
+    observation = SourceObservation(
+        source_id=request.source_id,
+        observed_at=governance.observation.observed_at,
+        event_time_field=governance.observation.event_time_field,
+        available_at_field=governance.observation.available_at_field,
+        historical_start=governance.observation.historical_start,
+        historical_end=governance.observation.historical_end,
+        live_status=governance.observation.live_status,
+        publication_delay_seconds=governance.observation.publication_delay_seconds,
+        update_frequency_seconds=governance.observation.update_frequency_seconds,
+        response_status=governance.observation.response_status,
+        evidence=governance.observation.evidence,
+    )
+    digest = sha256_bytes(content)
+    receipt = AcquisitionReceipt(
+        receipt_id=f"receipt.{request.digest[:24]}",
+        request_digest=request.digest,
+        source_id=request.source_id,
+        adapter=descriptor,
+        query_digest=request.query_digest,
+        snapshot=SnapshotReference(
+            content_sha256=digest,
+            size_bytes=len(content),
+            media_type=media_type,
+            data_format=data_format,
+            storage_name=snapshot_path.name,
+        ),
+        # The two hashes coincide for a direct fetch and diverge for a Reader-decoded
+        # acquisition: the fetched bytes are what arrived, the sealed bytes are what the
+        # Reader normalized them into.  That is the middle link of the three-link identity
+        # chain ``AcquisitionReceipt`` describes, and keeping both is what gives a
+        # normalized artifact a way back to the bytes that produced it.  On the direct
+        # path the snapshot digest is reused rather than the content hashed twice.
+        fetched_content_sha256=digest if decode is None else decode.fetched_content_sha256,
+        family_id=None if decode is None else decode.family_id,
+        family_version=None if decode is None else decode.family_version,
+        decode_options_digest=None if decode is None else decode.decode_options_digest,
+        decode_flags=() if decode is None else decode.flags,
+        source_uri=source_uri,
+        final_uri=final_uri,
+        event_time_field=governance.observation.event_time_field,
+        observed_at=governance.observation.observed_at,
+        available_at=governance.observation.available_at,
+        ingested_at=request.requested_at,
+        classification=classification,
+        rights_digest=governance.rights.digest,
+        retention_digest=governance.retention.digest,
+        observation_digest=observation.digest,
+        transport_evidence_digest=transport_evidence_digest,
+    )
+    return AcquisitionOutput(
+        receipt=receipt,
+        observation=observation,
+        parsed_schema_digest=parsed_schema_digest,
+        snapshot_path=snapshot_path,
+        next_watermark=next_watermark,
+        probe_transport=probe_transport,
+        parsed_row_count=parsed_row_count,
+        parsed_row_digest=parsed_row_digest,
+    )
+
+
+def _authorize_candidate(request: AcquisitionRequest, governance: AdapterGovernance) -> None:
+    evaluate_obligations(
+        rights=governance.rights,
+        lifecycle="candidate",
+        intended_use=request.intended_use,
+        attached_artifact_digests=governance.attached_license_artifact_digests,
+    )
+
+
+def _parse_limits(context: AcquisitionContext) -> ParseLimits:
+    return ParseLimits(
+        max_input_bytes=context.max_source_bytes,
+        max_uncompressed_bytes=min(max(context.max_source_bytes, 1) * 100, 1 << 36),
+        max_rows=context.max_rows,
+        max_columns=context.max_columns,
+        max_total_cells=min(context.max_rows * context.max_columns, 100_000_000),
+    )
+
+
+def _require_sandbox_binding(policy_digest: str, context: AcquisitionContext) -> None:
+    if context.crawler_sandbox_attestation_digest is None:
+        raise SourceContractError(
+            "SANDBOX_ATTESTATION_REQUIRED",
+            "context.crawler_sandbox_attestation_digest",
+            "acquisition requires the coordinator-pinned crawler policy digest",
+        )
+    if policy_digest != context.crawler_sandbox_attestation_digest:
+        raise SourceContractError(
+            "SANDBOX_ATTESTATION_MISMATCH",
+            "sandbox.policy_digest",
+            "sandbox result does not match the coordinator-pinned policy",
+        )