keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/models.py

@@ -0,0 +1,473 @@
+"""Persistence-owned mutable models and structural conversions."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from decimal import Decimal
+from enum import StrEnum
+from typing import Literal, TypeAlias
+from uuid import UUID, uuid5
+
+from keble_helpers.typings import PydanticModelConfig
+from pydantic import AwareDatetime, BaseModel, Field, SecretStr
+
+from keble_scraper_contract import (
+    ProxyCandidateSource,
+    ProxyCandidateState,
+    ProxyHealthObservation,
+    ScrapeArtifactRef,
+    ScrapeAttemptView,
+    ScrapeCompletionEvent,
+    ScrapeCost,
+    ScrapeFailure,
+    ScrapeJobCreate,
+    ScrapeJobState,
+    ScrapeJobView,
+    ScrapeLocalFailureCode,
+    ScrapeRetentionClass,
+    ScrapeTransportPolicy,
+    ScrapeTransportRoute,
+)
+
+_COMPLETION_EVENT_NAMESPACE = UUID("6fe96d16-f8d8-4742-ae72-a20922cf72fd")
+
+
+class PersistenceModel(BaseModel):
+    """Mutable internal model with strict snake-case persistence fields.
+
+    Side effects if changes:
+    - all Mongo document validation and update serialization.
+    """
+
+    model_config = PydanticModelConfig.default(extra="forbid", validate_default=True)
+
+
+class ScrapeJobMongoObject(PersistenceModel):
+    """Canonical mutable lifecycle document for `scrape_jobs`.
+
+    Steps:
+    1. retain immutable command, attempt limit, and deadline facts;
+    2. retain mutable lease/retry/terminal lifecycle state;
+    3. convert with immutable attempts into the public view.
+
+    Side effects if changes:
+    - idempotent creation, leases, optimistic cancellation, and job reads.
+    """
+
+    job_id: UUID
+    caller_identity: str
+    create: ScrapeJobCreate
+    request_hash: str
+    state: ScrapeJobState
+    revision: int = Field(ge=1)
+    attempt_count: int = Field(default=0, ge=0)
+    attempt_limit: int = Field(ge=1, le=10)
+    deadline_at: datetime
+    artifact_ids: list[UUID] = Field(default_factory=list)
+    total_cost: ScrapeCost = Field(default_factory=ScrapeCost)
+    created_at: datetime
+    updated_at: datetime
+    next_attempt_at: datetime | None = None
+    lease_owner: str | None = None
+    lease_expires_at: datetime | None = None
+    terminal_failure: ScrapeFailure | None = None
+    history_expires_at: datetime | None = None
+
+    def to_view(self, *, attempts: tuple[ScrapeAttemptView, ...]) -> ScrapeJobView:
+        """Convert Mongo lifecycle plus immutable attempts into the wire view.
+
+        Steps:
+        1. copy immutable command and stored budget facts;
+        2. attach caller-ordered immutable attempts;
+        3. expose terminal/retry/lease state without persistence-only ownership fields.
+
+        Side effects if changes:
+        - API create/read/cancel responses and Shopify polling.
+        """
+
+        return ScrapeJobView(
+            job_id=self.job_id,
+            caller_identity=self.caller_identity,
+            create=self.create,
+            state=self.state,
+            revision=self.revision,
+            attempt_limit=self.attempt_limit,
+            deadline_at=self.deadline_at,
+            attempts=attempts,
+            artifact_ids=tuple(self.artifact_ids),
+            total_cost=self.total_cost,
+            created_at=self.created_at,
+            updated_at=self.updated_at,
+            next_attempt_at=self.next_attempt_at,
+            lease_expires_at=self.lease_expires_at,
+            terminal_failure=self.terminal_failure,
+        )
+
+
+class ScrapeAttemptMongoObject(PersistenceModel):
+    """Immutable attempt row stored in `scrape_attempts`.
+
+    Side effects if changes:
+    - attempt ordering, retry evidence, and cost reconciliation.
+    """
+
+    job_id: UUID
+    attempt: ScrapeAttemptView
+
+
+class ScrapeCacheEntryMongoObject(PersistenceModel):
+    """Validated transport cache pointer stored in `scrape_cache_entries`.
+
+    Side effects if changes:
+    - cache-mode behavior, conditional headers, and artifact retention.
+    """
+
+    request_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
+    artifact_ids: list[UUID]
+    etag: str | None = None
+    last_modified: str | None = None
+    response_status: int = Field(ge=100, le=599)
+    content_type: str
+    expires_at: datetime
+    updated_at: datetime
+
+
+class ArtifactBatchLookup(PersistenceModel):
+    """Resolve ordered cache artifacts without converting absence into an exception.
+
+    Steps:
+    1. retain artifacts in the exact cache-pointer order;
+    2. retain every missing identity separately;
+    3. let the execution service invalidate stale cache state before fetching.
+
+    Side effects if changes:
+    - stale cache pointers must never enter paid retry crash loops;
+    - cache completion and 304 handling consume this exact result.
+    """
+
+    artifacts: tuple[ScrapeArtifactRef, ...]
+    missing_artifact_ids: tuple[UUID, ...]
+
+
+class ScrapeArtifactMongoObject(PersistenceModel):
+    """Private object metadata and reference count in `scrape_artifacts`.
+
+    Side effects if changes:
+    - content deduplication, signed downloads, pins, and orphan deletion.
+    """
+
+    artifact: ScrapeArtifactRef
+    object_key: str
+    reference_count: int = Field(default=0, ge=0)
+    deletion_lease_id: UUID | None = None
+    deletion_lease_expires_at: datetime | None = None
+
+
+class ScrapeArtifactPinMongoObject(PersistenceModel):
+    """Version-fenced downstream retention reference for one artifact.
+
+    Steps:
+    1. Persist one stable artifact-and-owner identity exactly once.
+    2. Advance expiry only with a newer retention revision.
+    3. Retain creation and last-policy-change timestamps for audit evidence.
+
+    Side effects if changes:
+    - artifact reference counts and orphan eligibility;
+    - Shopify latest-to-archive retention synchronization.
+    """
+
+    artifact_id: UUID
+    owner_identity: str
+    retention_class: ScrapeRetentionClass
+    retention_revision: int = Field(ge=1)
+    expires_at: datetime | None = None
+    created_at: datetime
+    updated_at: datetime
+
+
+class ProxyCandidateMongoObject(PersistenceModel):
+    """Encrypted proxy candidate and bounded health summary.
+
+    Steps:
+    1. retain a redacted endpoint beside encrypted credentials;
+    2. retain bounded health observations and score;
+    3. schedule the next quarantined health check.
+
+    Side effects if changes:
+    - experimental proxy selection, health decay, and credential redaction.
+    """
+
+    candidate_id: UUID
+    endpoint_redacted: str
+    encrypted_proxy_url: str
+    source: ProxyCandidateSource
+    source_reference: str
+    state: ProxyCandidateState
+    health_score: Decimal = Field(ge=0, le=1)
+    observations: list[ProxyHealthObservation] = Field(
+        default_factory=list, max_length=20
+    )
+    next_health_at: datetime
+    created_at: datetime
+    updated_at: datetime
+
+
+class OutboxState(StrEnum):
+    """Finite callback outbox lifecycle.
+
+    Side effects if changes:
+    - callback retry, replay identity, and stale delivery repair.
+    """
+
+    PENDING = "PENDING"
+    LEASED = "LEASED"
+    DELIVERED = "DELIVERED"
+    DEAD_LETTER = "DEAD_LETTER"
+
+
+class ScrapeOutboxMongoObject(PersistenceModel):
+    """Durable signed callback delivery row in `scrape_outbox`.
+
+    Steps:
+    1. retain one deterministic completion event identity;
+    2. own immutable attempt limit and mutable lease state;
+    3. retain due/delivery timing for recovery and acknowledgement.
+
+    Side effects if changes:
+    - webhook delivery retries and exactly-once event identity.
+    """
+
+    event: ScrapeCompletionEvent
+    state: OutboxState = OutboxState.PENDING
+    attempt_count: int = Field(default=0, ge=0)
+    attempt_limit: int = Field(ge=1, le=10)
+    next_attempt_at: datetime
+    lease_owner: str | None = None
+    lease_expires_at: datetime | None = None
+    delivered_at: datetime | None = None
+    history_expires_at: datetime | None = None
+
+
+class DispatchableJob(PersistenceModel):
+    """Minimal due-job projection required for queue publication.
+
+    Steps:
+    1. retain the opaque job identity;
+    2. retain only the queue-selecting transport policy;
+    3. exclude attempts, artifacts, callbacks, and request bodies.
+
+    Side effects if changes:
+    - maintenance batch reads and Celery queue selection share this projection.
+    """
+
+    job_id: UUID
+    transport_policy: ScrapeTransportPolicy
+
+
+class LeaseRecoverySummary(PersistenceModel):
+    """Count retryable and terminalized rows from one bounded recovery sweep.
+
+    Steps:
+    1. count leases returned to durable retry state;
+    2. count rows terminalized by immutable attempt/deadline policy;
+    3. expose no individual job payloads to maintenance metrics.
+
+    Side effects if changes:
+    - maintenance telemetry and recovery acceptance tests use these counters.
+    """
+
+    recovered_count: int = Field(default=0, ge=0)
+    terminalized_count: int = Field(default=0, ge=0)
+
+
+class HistoryRetentionSummary(PersistenceModel):
+    """Report one singleton, bounded history-retention transaction.
+
+    Steps:
+    1. disclose whether this caller acquired the retention lease;
+    2. count deleted terminal parents and immutable attempt children;
+    3. count independently expired delivered callbacks and audit facts.
+
+    Side effects if changes:
+    - maintenance telemetry and retention-capacity tests consume these counters.
+    """
+
+    lease_acquired: bool
+    jobs_deleted: int = Field(default=0, ge=0)
+    attempts_deleted: int = Field(default=0, ge=0)
+    delivered_outbox_deleted: int = Field(default=0, ge=0)
+    audit_events_deleted: int = Field(default=0, ge=0)
+
+
+class AuditAction(StrEnum):
+    """Audited operator/service side-effect family.
+
+    Side effects if changes:
+    - compliance exports and raw-download accountability.
+    """
+
+    JOB_CREATE = "JOB_CREATE"
+    JOB_CANCEL = "JOB_CANCEL"
+    ARTIFACT_DOWNLOAD = "ARTIFACT_DOWNLOAD"
+    ARTIFACT_PIN = "ARTIFACT_PIN"
+    PROXY_IMPORT = "PROXY_IMPORT"
+    CLEANUP_RUN = "CLEANUP_RUN"
+    BUDGET_UPDATE = "BUDGET_UPDATE"
+
+
+class ScrapeAuditEventMongoObject(PersistenceModel):
+    """Immutable operator/service audit fact in `scrape_audit_events`.
+
+    Steps:
+    1. retain the server-owned actor/action/target evidence;
+    2. retain one occurrence time and absolute history-expiry clock;
+    3. expose no credential or arbitrary provider payload.
+
+    Side effects if changes:
+    - security investigations, operations UI history, and bounded audit storage.
+    """
+
+    audit_id: UUID
+    actor_id: str
+    actor_role: str
+    action: AuditAction
+    target_identity: str
+    reason: str
+    request_id: str
+    occurred_at: datetime
+    history_expires_at: datetime | None = None
+
+
+class FetchLease(PersistenceModel):
+    """Worker-owned job lease and immutable attempt ordinal.
+
+    Side effects if changes:
+    - late acknowledgement and lease-expiry recovery semantics.
+    """
+
+    job: ScrapeJobMongoObject
+    attempt_number: int = Field(ge=1)
+    lease_owner: str
+
+
+class RouteReservation(PersistenceModel):
+    """Secret-bearing internal transport decision excluded from API models.
+
+    Side effects if changes:
+    - fetch egress, affinity, trust, and budget accounting.
+    """
+
+    route: ScrapeTransportRoute
+    proxy_url: SecretStr | None = None
+    session_id: str | None = None
+    reserved_cost_usd: Decimal = Field(default=Decimal("0"), ge=0)
+
+
+class RouteReservationGranted(PersistenceModel):
+    """Successful preflight decision containing one priced route reservation.
+
+    Steps:
+    1. discriminate the granted path explicitly;
+    2. retain the secret-bearing reservation only inside the worker process;
+    3. exclude failure/error objects from normal control flow.
+
+    Side effects if changes:
+    - execution and route tests consume this exact granted branch.
+    """
+
+    decision: Literal["GRANTED"] = "GRANTED"
+    reservation: RouteReservation
+
+
+class RouteReservationDenied(PersistenceModel):
+    """Expected local route/cost denial represented without an exception object.
+
+    Steps:
+    1. discriminate the denied path explicitly;
+    2. retain the attempted route and finite local failure code;
+    3. retain one bounded safe message for the typed attempt conversion.
+
+    Side effects if changes:
+    - zero-provider-attempt audit rows and bounded retry decisions depend on this branch.
+    """
+
+    decision: Literal["DENIED"] = "DENIED"
+    route: ScrapeTransportRoute
+    code: ScrapeLocalFailureCode
+    safe_message: str = Field(min_length=1, max_length=512)
+    retry_at: AwareDatetime | None = None
+
+
+RouteReservationOutcome: TypeAlias = RouteReservationGranted | RouteReservationDenied
+
+
+class CallbackTransportDisposition(StrEnum):
+    """Finite registered-callback HTTP outcome returned without exceptions-as-data.
+
+    Steps:
+    1. distinguish acknowledged delivery from HTTP retryability;
+    2. keep permanent receiver rejection separate from transient capacity;
+    3. leave client/runtime exceptions outside this expected-result vocabulary.
+
+    Side effects if changes:
+    - outbox acknowledgement, backoff, and dead-letter timing.
+    """
+
+    DELIVERED = "DELIVERED"
+    RETRYABLE = "RETRYABLE"
+    PERMANENT = "PERMANENT"
+
+
+class CallbackTransportResult(PersistenceModel):
+    """One sanitized HTTP callback result with an optional provider horizon.
+
+    Steps:
+    1. retain only a finite disposition and bounded status evidence;
+    2. retain a parsed future retry time when the receiver supplies one;
+    3. omit response bodies, URLs, and credentials from durable control flow.
+
+    Side effects if changes:
+    - callback retries consume this exact expected transport contract.
+    """
+
+    disposition: CallbackTransportDisposition
+    status_code: int = Field(ge=100, le=599)
+    retry_at: datetime | None = None
+
+
+def build_completion_event(
+    *,
+    job: ScrapeJobMongoObject,
+    state: ScrapeJobState,
+    artifacts: tuple[ScrapeArtifactRef, ...],
+    failure: ScrapeFailure | None,
+    occurred_at: datetime,
+) -> ScrapeCompletionEvent | None:
+    """Convert one terminal stored job into its deterministic callback event.
+
+    Steps:
+    1. return no event when the command registered no callback audience;
+    2. derive one stable event identity from job and terminal state;
+    3. copy only registered correlation, artifact, and typed-failure evidence.
+
+    Side effects if changes:
+    - success, retry exhaustion, and lease recovery must emit the same outbox identity;
+    - callback HMAC replay protection depends on deterministic event IDs.
+    """
+
+    command = job.create
+    if command.callback_identity is None or command.callback_correlation_id is None:
+        return None
+    return ScrapeCompletionEvent(
+        event_id=uuid5(
+            _COMPLETION_EVENT_NAMESPACE,
+            f"{job.job_id}:{state.value}",
+        ),
+        audience=command.callback_identity,
+        job_id=job.job_id,
+        correlation_id=command.callback_correlation_id,
+        state=state,
+        artifacts=artifacts,
+        failure=failure,
+        occurred_at=occurred_at,
+    )