keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/infrastructure/mongo.py

@@ -0,0 +1,2345 @@
+"""Mongo persistence, indexes, transactions, and structural conversions."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Mapping, Sequence
+from datetime import UTC, datetime, timedelta
+from decimal import Decimal
+import re
+from typing import Any
+from uuid import UUID, uuid4, uuid5
+
+from bson.codec_options import CodecOptions, TypeCodec, TypeRegistry
+from bson.binary import UuidRepresentation
+from bson.decimal128 import Decimal128
+from pydantic import SecretStr
+from pymongo import (
+    ASCENDING,
+    DESCENDING,
+    AsyncMongoClient,
+    IndexModel,
+    ReturnDocument,
+)
+from pymongo.read_concern import ReadConcern
+from pymongo.write_concern import WriteConcern
+
+from keble_scraper_contract import (
+    ProxyCandidateCreate,
+    ProxyCandidateState,
+    ProxyCandidateView,
+    ProxyHealthObservation,
+    ScrapeArtifactPin,
+    ScrapeArtifactRef,
+    ScrapeArtifactTrust,
+    ScrapeAttemptView,
+    ScrapeCacheMode,
+    ScrapeCompletionEvent,
+    ScrapeFailure,
+    ScrapeJobCancel,
+    ScrapeJobCreate,
+    ScrapeJobState,
+    ScrapeJobView,
+    ScrapeRetentionClass,
+    ScrapeTransportPolicy,
+    ScrapeLocalFailureCode,
+    utc_now,
+)
+
+from keble_scraper_api.artifact_namespace import ArtifactStoreIdentity
+from keble_scraper_api.application.failures import build_local_failure
+from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
+from keble_scraper_api.models import (
+    ArtifactBatchLookup,
+    DispatchableJob,
+    FetchLease,
+    HistoryRetentionSummary,
+    LeaseRecoverySummary,
+    OutboxState,
+    ProxyCandidateMongoObject,
+    ScrapeArtifactMongoObject,
+    ScrapeArtifactPinMongoObject,
+    ScrapeAttemptMongoObject,
+    ScrapeAuditEventMongoObject,
+    ScrapeCacheEntryMongoObject,
+    ScrapeJobMongoObject,
+    ScrapeOutboxMongoObject,
+    build_completion_event,
+)
+from keble_scraper_api.settings import ScraperSettings
+
+from .credentials import ProxyCredentialCipher
+
+_JOB_NAMESPACE = UUID("99034655-a296-4a6a-aa9d-69e17998ec15")
+_PROXY_NAMESPACE = UUID("b6404416-e7d4-4481-8b34-7484f5a10fab")
+_PIN_NAMESPACE = UUID("5c190785-8648-44ac-89e2-a413707f8058")
+_ARTIFACT_NAMESPACE_MARKER_ID = "artifact-store-namespace-v1"
+_ARTIFACT_NAMESPACE_PROBE_HASH = "0" * 64
+_HISTORY_RETENTION_LEASE_ID = "history-retention-v1"
+
+
+class _DecimalCodec(TypeCodec):
+    """Round-trip exact Python Decimal values through BSON Decimal128.
+
+    Side effects if changes:
+    - vendor cost arithmetic and budget/audit persistence precision.
+    """
+
+    @property
+    def python_type(self) -> type[Decimal]:
+        """Return the exact application type encoded by this codec.
+
+        Side effects if changes:
+        - Mongo type-registry selection for every Decimal value.
+        """
+
+        return Decimal
+
+    @property
+    def bson_type(self) -> type[Decimal128]:
+        """Return the exact BSON type decoded by this codec.
+
+        Side effects if changes:
+        - Mongo type-registry decoding for every Decimal128 value.
+        """
+
+        return Decimal128
+
+    def transform_python(self, value: Decimal) -> Decimal128:
+        """Encode one exact decimal before Mongo writes.
+
+        Side effects if changes:
+        - all embedded `ScrapeCost` values written by this repository.
+        """
+
+        return Decimal128(value)
+
+    def transform_bson(self, value: Decimal128) -> Decimal:
+        """Decode one BSON decimal into Pydantic-compatible Decimal.
+
+        Side effects if changes:
+        - job cost views and attempt history validation.
+        """
+
+        return value.to_decimal()
+
+
+SCRAPER_INDEXES: dict[str, tuple[IndexModel, ...]] = {
+    "scrape_jobs": (
+        IndexModel(
+            [("caller_identity", ASCENDING), ("create.idempotency_key", ASCENDING)],
+            unique=True,
+            name="caller_idempotency_unique_idx",
+        ),
+        IndexModel(
+            [("state", ASCENDING), ("next_attempt_at", ASCENDING)],
+            name="state_next_attempt_idx",
+        ),
+        IndexModel(
+            [("state", ASCENDING), ("lease_expires_at", ASCENDING)],
+            name="state_lease_expiry_idx",
+        ),
+        IndexModel([("created_at", DESCENDING)], name="created_at_desc_idx"),
+        IndexModel(
+            [("history_expires_at", ASCENDING)],
+            name="history_expires_at_idx",
+        ),
+    ),
+    "scrape_attempts": (
+        IndexModel(
+            [("job_id", ASCENDING), ("attempt.attempt_number", ASCENDING)],
+            unique=True,
+            name="job_attempt_number_unique_idx",
+        ),
+        IndexModel(
+            [("job_id", ASCENDING), ("attempt.observed_at", DESCENDING)],
+            name="job_observed_at_desc_idx",
+        ),
+        IndexModel(
+            [("attempt.route", ASCENDING), ("attempt.observed_at", DESCENDING)],
+            name="route_observed_at_desc_idx",
+        ),
+    ),
+    "scrape_cache_entries": (
+        IndexModel(
+            [("request_hash", ASCENDING)], unique=True, name="request_hash_unique_idx"
+        ),
+        IndexModel([("expires_at", ASCENDING)], name="expires_at_idx"),
+    ),
+    "scrape_artifacts": (
+        IndexModel(
+            [("artifact.content_sha256", ASCENDING), ("artifact.kind", ASCENDING)],
+            unique=True,
+            name="content_kind_unique_idx",
+        ),
+        IndexModel(
+            [
+                ("reference_count", ASCENDING),
+                ("artifact.expires_at", ASCENDING),
+                ("deletion_lease_expires_at", ASCENDING),
+            ],
+            name="orphan_expiry_idx",
+        ),
+    ),
+    "scrape_artifact_pins": (
+        IndexModel(
+            [("artifact_id", ASCENDING), ("owner_identity", ASCENDING)],
+            unique=True,
+            name="artifact_owner_unique_idx",
+        ),
+        IndexModel([("expires_at", ASCENDING)], name="expires_at_idx"),
+    ),
+    "proxy_candidates": (
+        IndexModel(
+            [("endpoint_redacted", ASCENDING)], unique=True, name="endpoint_unique_idx"
+        ),
+        IndexModel(
+            [("state", ASCENDING), ("health_score", DESCENDING)],
+            name="state_health_desc_idx",
+        ),
+        IndexModel(
+            [("state", ASCENDING), ("next_health_at", ASCENDING)],
+            name="state_next_health_idx",
+        ),
+        IndexModel(
+            [("health_score", DESCENDING), ("created_at", DESCENDING)],
+            name="health_created_desc_idx",
+        ),
+    ),
+    "scrape_outbox": (
+        IndexModel(
+            [("event.event_id", ASCENDING)], unique=True, name="event_unique_idx"
+        ),
+        IndexModel(
+            [("state", ASCENDING), ("next_attempt_at", ASCENDING)],
+            name="state_next_attempt_idx",
+        ),
+        IndexModel(
+            [("state", ASCENDING), ("lease_expires_at", ASCENDING)],
+            name="state_lease_expiry_idx",
+        ),
+        IndexModel(
+            [("history_expires_at", ASCENDING)],
+            name="history_expires_at_idx",
+        ),
+    ),
+    "scrape_audit_events": (
+        IndexModel(
+            [("history_expires_at", ASCENDING)],
+            name="history_expires_at_idx",
+        ),
+    ),
+}
+
+OBSOLETE_SCRAPER_INDEX_NAMES: dict[str, tuple[str, ...]] = {
+    "scrape_audit_events": (
+        "actor_occurred_at_desc_idx",
+        "action_occurred_at_desc_idx",
+        "target_text_idx",
+    ),
+}
+
+
+class MongoScraperRepository:
+    """Implement scraper persistence through indexed Mongo query shapes.
+
+    Steps:
+    1. own all typed scraper collections and their indexes;
+    2. fence lifecycle transitions with transactions and lease ownership;
+    3. convert every persisted document through strict Pydantic models.
+
+    Side effects if changes:
+    - all job/cache/artifact/proxy/outbox/audit persistence and transactions.
+    """
+
+    def __init__(
+        self,
+        *,
+        client: AsyncMongoClient[dict[str, Any]],
+        settings: ScraperSettings,
+        proxy_cipher: ProxyCredentialCipher,
+    ) -> None:
+        """Bind one client, database ownership, and proxy secret cipher.
+
+        Steps:
+        1. configure timezone/UUID/Decimal codecs and majority writes;
+        2. retain immutable job attempt/deadline creation policy;
+        3. bind every owned collection by stable name.
+
+        Side effects if changes:
+        - database selection and collection/index ownership.
+        """
+
+        codec_options = CodecOptions(
+            type_registry=TypeRegistry([_DecimalCodec()]),
+            tz_aware=True,
+            tzinfo=UTC,
+            uuid_representation=UuidRepresentation.STANDARD,
+        )
+        self._client = client
+        self._database = client.get_database(
+            settings.mongo_database,
+            codec_options=codec_options,
+            write_concern=WriteConcern("majority"),
+        )
+        self._cipher = proxy_cipher
+        self._attempt_limit = settings.max_attempts
+        self._job_deadline_seconds = settings.job_deadline_seconds
+        self._terminal_history_retention = timedelta(
+            days=settings.terminal_history_retention_days
+        )
+        self._delivered_outbox_retention = timedelta(
+            days=settings.delivered_outbox_retention_days
+        )
+        self._audit_history_retention = timedelta(
+            days=settings.audit_history_retention_days
+        )
+        self._history_retention_lease = timedelta(
+            seconds=settings.history_retention_lease_seconds
+        )
+        self.jobs = self._database["scrape_jobs"]
+        self.attempts = self._database["scrape_attempts"]
+        self.cache = self._database["scrape_cache_entries"]
+        self.artifacts = self._database["scrape_artifacts"]
+        self.artifact_pins = self._database["scrape_artifact_pins"]
+        self.metadata = self._database["scrape_metadata"]
+        self.proxies = self._database["proxy_candidates"]
+        self.outbox = self._database["scrape_outbox"]
+        self.audit = self._database["scrape_audit_events"]
+
+    @classmethod
+    def build(
+        cls,
+        *,
+        settings: ScraperSettings,
+        proxy_cipher: ProxyCredentialCipher,
+    ) -> "MongoScraperRepository":
+        """Construct a standard-UUID async Mongo client from typed settings.
+
+        Side effects if changes:
+        - process connection pools and replica-set transaction behavior.
+        """
+
+        client: AsyncMongoClient[dict[str, Any]] = AsyncMongoClient(
+            settings.mongo_uri.get_secret_value(),
+            uuidRepresentation="standard",
+            tz_aware=True,
+        )
+        return cls(client=client, settings=settings, proxy_cipher=proxy_cipher)
+
+    async def ensure_indexes(self) -> None:
+        """Create live indexes and remove indexes with no supported read surface.
+
+        Steps:
+        1. inspect each owned collection's exact index names;
+        2. remove only explicitly obsolete deterministic names;
+        3. create the current non-empty index set idempotently.
+
+        Side effects if changes:
+        - startup readiness and every heavy list/lease/cleanup query.
+        """
+
+        collections = {
+            "scrape_jobs": self.jobs,
+            "scrape_attempts": self.attempts,
+            "scrape_cache_entries": self.cache,
+            "scrape_artifacts": self.artifacts,
+            "scrape_artifact_pins": self.artifact_pins,
+            "proxy_candidates": self.proxies,
+            "scrape_outbox": self.outbox,
+            "scrape_audit_events": self.audit,
+        }
+        for name, collection in collections.items():
+            existing = await collection.index_information()
+            for obsolete_name in OBSOLETE_SCRAPER_INDEX_NAMES.get(name, ()):
+                if obsolete_name in existing:
+                    await collection.drop_index(obsolete_name)
+            indexes = SCRAPER_INDEXES[name]
+            if indexes:
+                await collection.create_indexes(list(indexes))
+        await self.metadata.update_one(
+            {"_id": _HISTORY_RETENTION_LEASE_ID},
+            {
+                "$setOnInsert": {
+                    "lease_owner": None,
+                    "lease_expires_at": datetime.fromtimestamp(0, tz=UTC),
+                    "fence_revision": 0,
+                }
+            },
+            upsert=True,
+        )
+
+    async def close(self) -> None:
+        """Close the async Mongo pool during application shutdown.
+
+        Side effects if changes:
+        - API/worker resource cleanup.
+        """
+
+        await self._client.close()
+
+    async def readiness(self) -> None:
+        """Ping Mongo for readiness without exposing topology details.
+
+        Side effects if changes:
+        - deployment health and rollout gating.
+        """
+
+        await self._client.admin.command("ping")
+
+    async def ensure_artifact_store_namespace(
+        self,
+        *,
+        probe_object_key: str,
+        store_identity: ArtifactStoreIdentity,
+        namespace_owner: str,
+    ) -> None:
+        """Bind artifact metadata to one complete backend/location identity.
+
+        Steps:
+        1. validate the store's deterministic zero-hash probe shape;
+        2. reject legacy/unmarked rows and keys outside the configured prefix;
+        3. atomically retain backend, endpoint/bucket or directory, prefix, and owner;
+        4. reject every later backend/location/owner drift before serving work.
+
+        Side effects if changes:
+        - API/worker startup fails before reads/writes when store wiring drifts;
+        - one artifact identity cannot resolve to two backend object keys.
+        """
+
+        parts = probe_object_key.rsplit("/", 2)
+        if len(parts) != 3 or parts[1:] != ["00", _ARTIFACT_NAMESPACE_PROBE_HASH]:
+            raise ValueError(
+                "artifact store probe key must end with the zero hash path"
+            )
+        namespace_prefix = parts[0]
+        if store_identity.prefix != namespace_prefix:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                message="artifact store identity prefix differs from its probe key",
+            )
+        store_identity_payload = store_identity.model_dump(mode="json")
+        marker = await self.metadata.find_one({"_id": _ARTIFACT_NAMESPACE_MARKER_ID})
+        if marker is not None:
+            if (
+                marker.get("probe_object_key") != probe_object_key
+                or marker.get("store_identity") != store_identity_payload
+                or marker.get("namespace_owner") != namespace_owner
+            ):
+                raise ScraperDomainError(
+                    code=ScraperErrorCode.CONFIGURATION_INVALID,
+                    message=(
+                        "artifact store backend, location, or owner changed for this database"
+                    ),
+                )
+            return
+        mismatch = await self.artifacts.find_one(
+            {"object_key": {"$not": re.compile(f"^{re.escape(namespace_prefix)}/")}},
+            {"_id": 1},
+        )
+        if mismatch is not None:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                message="artifact store namespace differs from persisted artifact keys",
+            )
+        existing_artifact = await self.artifacts.find_one({}, {"_id": 1})
+        if existing_artifact is not None:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                message="artifact store identity is missing for persisted artifact rows",
+            )
+        await self.metadata.update_one(
+            {"_id": _ARTIFACT_NAMESPACE_MARKER_ID},
+            {
+                "$setOnInsert": {
+                    "probe_object_key": probe_object_key,
+                    "store_identity": store_identity_payload,
+                    "namespace_owner": namespace_owner,
+                }
+            },
+            upsert=True,
+        )
+        marker = await self.metadata.find_one({"_id": _ARTIFACT_NAMESPACE_MARKER_ID})
+        if (
+            marker is None
+            or marker.get("probe_object_key") != probe_object_key
+            or marker.get("store_identity") != store_identity_payload
+            or marker.get("namespace_owner") != namespace_owner
+        ):
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                message="artifact store backend, location, or owner changed for this database",
+            )
+
+    async def create_or_get(
+        self,
+        *,
+        caller_identity: str,
+        command: ScrapeJobCreate,
+        request_hash: str,
+    ) -> tuple[ScrapeJobMongoObject, bool]:
+        """Atomically create or reuse one deterministic idempotency identity.
+
+        Steps:
+        1. derive the caller-scoped deterministic job identity;
+        2. insert immutable command/budget/deadline facts exactly once;
+        3. reject any reuse whose request hash differs.
+
+        Collection/query/index:
+        - `scrape_jobs`, deterministic `_id` plus `caller_idempotency_unique_idx`.
+
+        Side effects if changes:
+        - duplicate queue dispatch and idempotency conflict behavior.
+        """
+
+        job_id = uuid5(_JOB_NAMESPACE, f"{caller_identity}:{command.idempotency_key}")
+        now = utc_now()
+        job = ScrapeJobMongoObject(
+            job_id=job_id,
+            caller_identity=caller_identity,
+            create=command,
+            request_hash=request_hash,
+            state=ScrapeJobState.PENDING,
+            revision=1,
+            attempt_limit=self._attempt_limit,
+            deadline_at=now + timedelta(seconds=self._job_deadline_seconds),
+            created_at=now,
+            updated_at=now,
+            next_attempt_at=now,
+        )
+        before = await self.jobs.find_one_and_update(
+            {"_id": job_id},
+            {"$setOnInsert": self._job_document(job)},
+            upsert=True,
+            return_document=ReturnDocument.BEFORE,
+        )
+        stored = job if before is None else self._job_model(before)
+        if stored.request_hash != request_hash:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.REVISION_CONFLICT,
+                message="idempotency key was already used for a different request",
+            )
+        return stored, before is not None
+
+    async def get_view(self, *, job_id: UUID) -> ScrapeJobView:
+        """Read one job plus ordered immutable attempts by supporting indexes.
+
+        Steps:
+        1. load the canonical job by primary identity;
+        2. load attempts through the ordered unique index;
+        3. validate and convert both layers into one public view.
+
+        Collection/query/index:
+        - `scrape_jobs` `_id`; `scrape_attempts` `job_attempt_number_unique_idx`.
+
+        Side effects if changes:
+        - job polling and admin diagnostics.
+        """
+
+        raw = await self.jobs.find_one({"_id": job_id})
+        if raw is None:
+            raise self._not_found("scrape job")
+        attempt_docs = (
+            await self.attempts.find({"job_id": job_id})
+            .sort("attempt.attempt_number", ASCENDING)
+            .to_list(length=100)
+        )
+        attempts = tuple(
+            ScrapeAttemptMongoObject.model_validate(self._without_id(item)).attempt
+            for item in attempt_docs
+        )
+        return self._job_model(raw).to_view(attempts=attempts)
+
+    async def list_views(
+        self,
+        *,
+        limit: int,
+        before: datetime | None,
+    ) -> tuple[ScrapeJobView, ...]:
+        """List newest jobs through the `created_at_desc_idx` cursor shape.
+
+        Steps:
+        1. apply optional created-time cursor and bounded limit;
+        2. load rows in descending creation order;
+        3. hydrate each selected job into its complete view.
+
+        Side effects if changes:
+        - operations dashboard pagination and Mongo load.
+        """
+
+        query: dict[str, object] = {}
+        if before is not None:
+            query["created_at"] = {"$lt": before}
+        rows = (
+            await self.jobs.find(query)
+            .sort("created_at", DESCENDING)
+            .limit(limit)
+            .to_list(length=limit)
+        )
+        views = await asyncio.gather(
+            *(self.get_view(job_id=self._job_model(row).job_id) for row in rows)
+        )
+        return tuple(views)
+
+    async def cancel(self, *, job_id: UUID, command: ScrapeJobCancel) -> ScrapeJobView:
+        """Optimistically transition a non-terminal job to CANCELLED.
+
+        Steps:
+        1. match job identity, expected revision, and cancellable state;
+        2. write finite cancellation evidence and clear lease/retry fields;
+        3. return the winner or raise a typed conflict/not-found result.
+
+        Collection/query/index:
+        - `scrape_jobs` `_id` and revision in one atomic update.
+
+        Side effects if changes:
+        - worker late acknowledgement and cancellation audit semantics.
+        """
+
+        now = utc_now()
+        cancellation = build_local_failure(
+            code=ScrapeLocalFailureCode.OPERATOR_CANCELLED,
+            safe_message="Scrape job was cancelled by an operator.",
+            observed_at=now,
+            physical_attempt_count=0,
+        )
+        raw = await self.jobs.find_one_and_update(
+            {
+                "_id": job_id,
+                "revision": command.expected_revision,
+                "state": {
+                    "$in": [
+                        ScrapeJobState.PENDING.value,
+                        ScrapeJobState.LEASED.value,
+                        ScrapeJobState.RETRY_SCHEDULED.value,
+                    ]
+                },
+            },
+            {
+                "$set": {
+                    "state": ScrapeJobState.CANCELLED.value,
+                    "updated_at": now,
+                    "history_expires_at": now + self._terminal_history_retention,
+                    "terminal_failure": cancellation.model_dump(
+                        mode="python",
+                        by_alias=False,
+                    ),
+                },
+                "$inc": {"revision": 1},
+                "$unset": {
+                    "lease_owner": "",
+                    "lease_expires_at": "",
+                    "next_attempt_at": "",
+                },
+            },
+            return_document=ReturnDocument.AFTER,
+        )
+        if raw is None:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.REVISION_CONFLICT,
+                message="job revision or lifecycle no longer permits cancellation",
+            )
+        return await self.get_view(job_id=job_id)
+
+    async def lease_due(
+        self,
+        *,
+        job_id: UUID,
+        worker_id: str,
+        lease_seconds: int,
+    ) -> FetchLease | None:
+        """Lease one due job only inside its immutable paid-work budget.
+
+        Steps:
+        1. require a pending/retry row whose due time has arrived;
+        2. reject rows at their stored attempt limit or deadline;
+        3. atomically assign owner/expiry and increment the attempt ordinal.
+
+        Collection/query/index:
+        - `scrape_jobs`, `_id` plus `state_next_attempt_idx` lifecycle fields.
+
+        Side effects if changes:
+        - worker exclusivity and late acknowledgement prevention.
+        """
+
+        now = utc_now()
+        raw = await self.jobs.find_one_and_update(
+            {
+                "_id": job_id,
+                "state": {
+                    "$in": [
+                        ScrapeJobState.PENDING.value,
+                        ScrapeJobState.RETRY_SCHEDULED.value,
+                    ]
+                },
+                "next_attempt_at": {"$lte": now},
+                "deadline_at": {"$gt": now},
+                "$expr": {"$lt": ["$attempt_count", "$attempt_limit"]},
+            },
+            {
+                "$set": {
+                    "state": ScrapeJobState.LEASED.value,
+                    "lease_owner": worker_id,
+                    "lease_expires_at": now + timedelta(seconds=lease_seconds),
+                    "updated_at": now,
+                },
+                "$inc": {"revision": 1, "attempt_count": 1},
+            },
+            return_document=ReturnDocument.AFTER,
+        )
+        if raw is None:
+            return None
+        job = self._job_model(raw)
+        return FetchLease(
+            job=job, attempt_number=job.attempt_count, lease_owner=worker_id
+        )
+
+    async def append_attempt(self, *, job_id: UUID, attempt: ScrapeAttemptView) -> None:
+        """Atomically append attempt evidence and add measured job cost.
+
+        Steps:
+        1. insert the immutable attempt under its unique ordinal;
+        2. add exact estimated/actual/usage units to the job aggregate;
+        3. commit both writes in one snapshot transaction.
+
+        Collection/query/index:
+        - `scrape_attempts` `job_attempt_number_unique_idx`; `scrape_jobs` `_id`.
+
+        Side effects if changes:
+        - attempts remain append-only and aggregate cost stays ACID-consistent.
+        """
+
+        row = ScrapeAttemptMongoObject(job_id=job_id, attempt=attempt)
+        async with self._client.start_session() as session:
+            async with await session.start_transaction(
+                read_concern=ReadConcern("snapshot"),
+                write_concern=WriteConcern("majority"),
+            ):
+                await self.attempts.insert_one(
+                    {
+                        "_id": attempt.attempt_id,
+                        **row.model_dump(mode="python", by_alias=False),
+                    },
+                    session=session,
+                )
+                await self.jobs.update_one(
+                    {"_id": job_id},
+                    {
+                        "$inc": {
+                            "total_cost.estimated_amount": attempt.cost.estimated_amount,
+                            "total_cost.actual_amount": attempt.cost.actual_amount,
+                            "total_cost.bytes_transferred": attempt.cost.bytes_transferred,
+                            "total_cost.browser_seconds": attempt.cost.browser_seconds,
+                        }
+                    },
+                    session=session,
+                )
+
+    async def complete(
+        self,
+        *,
+        lease: FetchLease,
+        artifacts: tuple[ScrapeArtifactRef, ...],
+        event: ScrapeCompletionEvent | None,
+    ) -> ScrapeJobView:
+        """Commit terminal success and callback outbox in one transaction.
+
+        Steps:
+        1. fence the exact leased revision and owner;
+        2. publish terminal artifacts/cost state;
+        3. insert the deterministic callback event in the same transaction.
+
+        Side effects if changes:
+        - job truth, artifact references, and Shopify callbacks must stay atomic.
+        """
+
+        now = utc_now()
+        terminal_values: dict[str, object] = {
+            "state": ScrapeJobState.SUCCEEDED.value,
+            "artifact_ids": [item.artifact_id for item in artifacts],
+            "updated_at": now,
+        }
+        if event is None:
+            terminal_values["history_expires_at"] = (
+                now + self._terminal_history_retention
+            )
+        async with self._client.start_session() as session:
+            async with await session.start_transaction(
+                read_concern=ReadConcern("snapshot"),
+                write_concern=WriteConcern("majority"),
+            ):
+                updated = await self.jobs.update_one(
+                    {
+                        "_id": lease.job.job_id,
+                        "state": ScrapeJobState.LEASED.value,
+                        "lease_owner": lease.lease_owner,
+                        "revision": lease.job.revision,
+                    },
+                    {
+                        "$set": terminal_values,
+                        "$inc": {"revision": 1},
+                        "$unset": {
+                            "lease_owner": "",
+                            "lease_expires_at": "",
+                            "next_attempt_at": "",
+                            "terminal_failure": "",
+                        },
+                    },
+                    session=session,
+                )
+                if updated.modified_count != 1:
+                    raise self._late_acknowledgement()
+                if event is not None:
+                    outbox = ScrapeOutboxMongoObject(
+                        event=event,
+                        attempt_limit=lease.job.attempt_limit,
+                        next_attempt_at=now,
+                    )
+                    await self.outbox.insert_one(
+                        {
+                            "_id": event.event_id,
+                            **outbox.model_dump(mode="python", by_alias=False),
+                        },
+                        session=session,
+                    )
+        return await self.get_view(job_id=lease.job.job_id)
+
+    async def schedule_retry(
+        self,
+        *,
+        lease: FetchLease,
+        failure: ScrapeFailure,
+        next_attempt_at: datetime,
+        dead_letter: bool,
+        event: ScrapeCompletionEvent | None,
+    ) -> ScrapeJobView:
+        """Commit retry/dead-letter state and typed outbox consistently.
+
+        Steps:
+        1. fence the exact leased revision and owner;
+        2. clear the lease into due retry or terminal dead-letter state;
+        3. insert a terminal callback only when the job is final.
+
+        Side effects if changes:
+        - retry timing, stale leases, and failure callbacks.
+        """
+
+        state = (
+            ScrapeJobState.DEAD_LETTER
+            if dead_letter
+            else ScrapeJobState.RETRY_SCHEDULED
+        )
+        now = utc_now()
+        terminal_failure = (
+            failure.model_dump(mode="python", by_alias=False) if dead_letter else None
+        )
+        lifecycle_values: dict[str, object] = {
+            "state": state.value,
+            "next_attempt_at": None if dead_letter else next_attempt_at,
+            "updated_at": now,
+            "terminal_failure": terminal_failure,
+        }
+        if dead_letter and event is None:
+            lifecycle_values["history_expires_at"] = (
+                now + self._terminal_history_retention
+            )
+        async with self._client.start_session() as session:
+            async with await session.start_transaction(
+                read_concern=ReadConcern("snapshot"),
+                write_concern=WriteConcern("majority"),
+            ):
+                updated = await self.jobs.update_one(
+                    {
+                        "_id": lease.job.job_id,
+                        "state": ScrapeJobState.LEASED.value,
+                        "lease_owner": lease.lease_owner,
+                        "revision": lease.job.revision,
+                    },
+                    {
+                        "$set": lifecycle_values,
+                        "$inc": {"revision": 1},
+                        "$unset": {"lease_owner": "", "lease_expires_at": ""},
+                    },
+                    session=session,
+                )
+                if updated.modified_count != 1:
+                    raise self._late_acknowledgement()
+                if event is not None:
+                    outbox = ScrapeOutboxMongoObject(
+                        event=event,
+                        attempt_limit=lease.job.attempt_limit,
+                        next_attempt_at=now,
+                    )
+                    await self.outbox.insert_one(
+                        {
+                            "_id": event.event_id,
+                            **outbox.model_dump(mode="python", by_alias=False),
+                        },
+                        session=session,
+                    )
+        return await self.get_view(job_id=lease.job.job_id)
+
+    async def recover_expired_leases(
+        self,
+        *,
+        now: datetime,
+        limit: int = 1_000,
+    ) -> LeaseRecoverySummary:
+        """Recover expired worker leases without bypassing stored work budgets.
+
+        Steps:
+        1. read a bounded expiry-ordered batch through ``state_lease_expiry_idx``;
+        2. return still-eligible rows to durable retry state with exact revision fencing;
+        3. terminalize attempt/deadline-exhausted rows and their callback outbox atomically.
+
+        Side effects if changes:
+        - crashed workers may retry a paid route only inside the immutable job budget;
+        - exhausted jobs and Shopify callbacks become durable terminal facts.
+        """
+
+        rows = (
+            await self.jobs.find(
+                {
+                    "state": ScrapeJobState.LEASED.value,
+                    "lease_expires_at": {"$lte": now},
+                }
+            )
+            .sort("lease_expires_at", ASCENDING)
+            .limit(limit)
+            .to_list(length=limit)
+        )
+        recovered_count = 0
+        terminalized_count = 0
+        for raw in rows:
+            job = self._job_model(raw)
+            if self._job_budget_exhausted(job=job, now=now):
+                terminalized_count += int(
+                    await self._terminalize_exhausted_job(job=job, now=now)
+                )
+                continue
+            recovered = await self.jobs.update_one(
+                {
+                    "_id": job.job_id,
+                    "state": ScrapeJobState.LEASED.value,
+                    "revision": job.revision,
+                    "lease_owner": job.lease_owner,
+                    "lease_expires_at": {"$lte": now},
+                },
+                {
+                    "$set": {
+                        "state": ScrapeJobState.RETRY_SCHEDULED.value,
+                        "next_attempt_at": now,
+                        "updated_at": now,
+                    },
+                    "$inc": {"revision": 1},
+                    "$unset": {"lease_owner": "", "lease_expires_at": ""},
+                },
+            )
+            recovered_count += recovered.modified_count
+        return LeaseRecoverySummary(
+            recovered_count=recovered_count,
+            terminalized_count=terminalized_count,
+        )
+
+    async def terminalize_exhausted_due_jobs(
+        self,
+        *,
+        now: datetime,
+        limit: int = 1_000,
+    ) -> int:
+        """Terminalize non-leased jobs whose immutable budget is already spent.
+
+        Steps:
+        1. select due pending/retry rows at their attempt limit or deadline;
+        2. re-check the exact revision inside each majority transaction;
+        3. commit dead-letter state and deterministic callback outbox together.
+
+        Side effects if changes:
+        - queued stale messages cannot restart paid work after policy exhaustion;
+        - due-job dispatch excludes every terminalized identity.
+        """
+
+        rows = (
+            await self.jobs.find(
+                {
+                    "state": {
+                        "$in": [
+                            ScrapeJobState.PENDING.value,
+                            ScrapeJobState.RETRY_SCHEDULED.value,
+                        ]
+                    },
+                    "next_attempt_at": {"$lte": now},
+                    "$or": [
+                        {"deadline_at": {"$lte": now}},
+                        {"$expr": {"$gte": ["$attempt_count", "$attempt_limit"]}},
+                    ],
+                }
+            )
+            .sort("next_attempt_at", ASCENDING)
+            .limit(limit)
+            .to_list(length=limit)
+        )
+        terminalized_count = 0
+        for raw in rows:
+            terminalized_count += int(
+                await self._terminalize_exhausted_job(
+                    job=self._job_model(raw),
+                    now=now,
+                )
+            )
+        return terminalized_count
+
+    async def due_jobs(
+        self,
+        *,
+        now: datetime,
+        limit: int,
+    ) -> tuple[DispatchableJob, ...]:
+        """Project all queue fields for one bounded eligible due-job batch.
+
+        Steps:
+        1. use ``state_next_attempt_idx`` for due lifecycle ordering;
+        2. exclude attempt/deadline-exhausted work before broker publication;
+        3. project only identity and transport policy in one Mongo query.
+
+        Side effects if changes:
+        - maintenance avoids per-job reads and publishes only eligible work.
+        """
+
+        rows = (
+            await self.jobs.find(
+                {
+                    "state": {
+                        "$in": [
+                            ScrapeJobState.PENDING.value,
+                            ScrapeJobState.RETRY_SCHEDULED.value,
+                        ]
+                    },
+                    "next_attempt_at": {"$lte": now},
+                    "deadline_at": {"$gt": now},
+                    "$expr": {"$lt": ["$attempt_count", "$attempt_limit"]},
+                },
+                {"_id": 1, "create.transport_policy": 1},
+            )
+            .sort("next_attempt_at", ASCENDING)
+            .limit(limit)
+            .to_list(length=limit)
+        )
+        return tuple(
+            DispatchableJob(
+                job_id=row["_id"],
+                transport_policy=ScrapeTransportPolicy(
+                    row["create"]["transport_policy"]
+                ),
+            )
+            for row in rows
+        )
+
+    async def _terminalize_exhausted_job(
+        self,
+        *,
+        job: ScrapeJobMongoObject,
+        now: datetime,
+    ) -> bool:
+        """Commit one exact exhausted job and callback in a majority transaction.
+
+        Steps:
+        1. distinguish deadline expiry from attempt-budget exhaustion;
+        2. build one typed terminal failure and deterministic callback event;
+        3. fence the current revision/state before writing both records atomically.
+
+        Side effects if changes:
+        - terminal job truth, outbox delivery, and late-worker rejection stay consistent.
+        """
+
+        deadline_exceeded = job.deadline_at <= now
+        code = (
+            ScrapeLocalFailureCode.JOB_DEADLINE_EXCEEDED
+            if deadline_exceeded
+            else ScrapeLocalFailureCode.ATTEMPT_BUDGET_EXHAUSTED
+        )
+        failure = build_local_failure(
+            code=code,
+            safe_message=(
+                "Scraper job deadline expired before safe completion."
+                if deadline_exceeded
+                else "Automatic scraper attempt budget is exhausted."
+            ),
+            observed_at=now,
+            physical_attempt_count=job.attempt_count,
+        )
+        event = build_completion_event(
+            job=job,
+            state=ScrapeJobState.DEAD_LETTER,
+            artifacts=(),
+            failure=failure,
+            occurred_at=now,
+        )
+        terminal_values: dict[str, object] = {
+            "state": ScrapeJobState.DEAD_LETTER.value,
+            "terminal_failure": failure.model_dump(
+                mode="python",
+                by_alias=False,
+            ),
+            "next_attempt_at": None,
+            "updated_at": now,
+        }
+        if event is None:
+            terminal_values["history_expires_at"] = (
+                now + self._terminal_history_retention
+            )
+        async with self._client.start_session() as session:
+            async with await session.start_transaction(
+                read_concern=ReadConcern("snapshot"),
+                write_concern=WriteConcern("majority"),
+            ):
+                updated = await self.jobs.update_one(
+                    {
+                        "_id": job.job_id,
+                        "revision": job.revision,
+                        "state": job.state.value,
+                    },
+                    {
+                        "$set": terminal_values,
+                        "$inc": {"revision": 1},
+                        "$unset": {"lease_owner": "", "lease_expires_at": ""},
+                    },
+                    session=session,
+                )
+                if updated.modified_count != 1:
+                    return False
+                if event is not None:
+                    outbox = ScrapeOutboxMongoObject(
+                        event=event,
+                        attempt_limit=job.attempt_limit,
+                        next_attempt_at=now,
+                    )
+                    await self.outbox.insert_one(
+                        {
+                            "_id": event.event_id,
+                            **outbox.model_dump(mode="python", by_alias=False),
+                        },
+                        session=session,
+                    )
+        return True
+
+    @staticmethod
+    def _job_budget_exhausted(
+        *,
+        job: ScrapeJobMongoObject,
+        now: datetime,
+    ) -> bool:
+        """Return whether a stored attempt limit or deadline forbids more work.
+
+        Steps:
+        1. compare the durable attempt ordinal with its immutable limit;
+        2. compare the current time with the immutable job deadline;
+        3. return one boolean used by both lease and due-state recovery.
+
+        Side effects if changes:
+        - paid retry admission and terminal recovery share this exact predicate.
+        """
+
+        return job.attempt_count >= job.attempt_limit or job.deadline_at <= now
+
+    async def get_valid(
+        self,
+        *,
+        request_hash: str,
+        mode: ScrapeCacheMode,
+        now: datetime,
+    ) -> ScrapeCacheEntryMongoObject | None:
+        """Read one cache pointer through `request_hash_unique_idx`.
+
+        Side effects if changes:
+        - reuse, conditional revalidation, and bypass semantics.
+        """
+
+        if mode is ScrapeCacheMode.BYPASS:
+            return None
+        query: dict[str, object] = {"request_hash": request_hash}
+        if mode is ScrapeCacheMode.REUSE_VALID:
+            query["expires_at"] = {"$gt": now}
+        raw = await self.cache.find_one(query)
+        return (
+            ScrapeCacheEntryMongoObject.model_validate(self._without_id(raw))
+            if raw is not None
+            else None
+        )
+
+    async def upsert(self, *, entry: ScrapeCacheEntryMongoObject) -> None:
+        """Upsert one complete cache pointer by request identity.
+
+        Steps:
+        1. key the pointer by canonical request hash;
+        2. replace all validators/artifact identities atomically;
+        3. retain its bounded expiry and compare-delete timestamp.
+
+        Side effects if changes:
+        - future cache hits and conditional validators.
+        """
+
+        await self.cache.replace_one(
+            {"request_hash": entry.request_hash},
+            {
+                "_id": entry.request_hash,
+                **entry.model_dump(mode="python", by_alias=False),
+            },
+            upsert=True,
+        )
+
+    async def delete_current(
+        self,
+        *,
+        request_hash: str,
+        updated_at: datetime,
+    ) -> bool:
+        """Delete only the exact stale cache revision observed by one worker.
+
+        Steps:
+        1. match the request identity and observed cache timestamp;
+        2. delete at most that exact pointer revision;
+        3. preserve any concurrently refreshed pointer.
+
+        Side effects if changes:
+        - missing artifact recovery cannot erase a newer cache write.
+        """
+
+        deleted = await self.cache.delete_one(
+            {
+                "request_hash": request_hash,
+                "updated_at": updated_at,
+            }
+        )
+        return deleted.deleted_count == 1
+
+    async def delete_expired(self, *, now: datetime) -> int:
+        """Delete expired cache pointers through `expires_at_idx`.
+
+        Side effects if changes:
+        - cache retention and later artifact orphan eligibility.
+        """
+
+        result = await self.cache.delete_many({"expires_at": {"$lte": now}})
+        return result.deleted_count
+
+    async def register(
+        self, *, artifact: ScrapeArtifactRef, object_key: str
+    ) -> ScrapeArtifactRef:
+        """Atomically register or strengthen content metadata outside deletion leases.
+
+        Steps:
+        1. insert the stable content-addressed row if absent;
+        2. strengthen trust/retention/expiry without downgrade;
+        3. reject active deletion leases and validate the stored winner.
+
+        Merge policy:
+        1. trust can promote from untrusted to verified but never downgrade;
+        2. retention can strengthen from transient to cache to archive;
+        3. expiry extends and creation time preserves the earliest observation;
+        4. active cleanup leases reject recapture before object bytes are written.
+
+        Side effects if changes:
+        - content deduplication, cleanup races, retention, and download resolution.
+        """
+
+        row = ScrapeArtifactMongoObject(artifact=artifact, object_key=object_key)
+        await self.artifacts.update_one(
+            {"_id": artifact.artifact_id},
+            {"$setOnInsert": row.model_dump(mode="python", by_alias=False)},
+            upsert=True,
+        )
+        now = utc_now()
+        set_values: dict[str, object] = {
+            "deletion_lease_id": None,
+            "deletion_lease_expires_at": None,
+        }
+        if artifact.trust is ScrapeArtifactTrust.VERIFIED:
+            set_values["artifact.trust"] = ScrapeArtifactTrust.VERIFIED
+        merged = await self.artifacts.update_one(
+            {
+                "_id": artifact.artifact_id,
+                "$or": [
+                    {"deletion_lease_id": None},
+                    {"deletion_lease_id": {"$exists": False}},
+                    {"deletion_lease_expires_at": {"$lte": now}},
+                ],
+            },
+            {
+                "$set": set_values,
+                "$max": {"artifact.expires_at": artifact.expires_at},
+                "$min": {"artifact.created_at": artifact.created_at},
+            },
+        )
+        if merged.matched_count != 1:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.REVISION_CONFLICT,
+                message="artifact is currently leased for deletion",
+            )
+        if artifact.retention_class is ScrapeRetentionClass.REFERENCED_ARCHIVE:
+            await self.artifacts.update_one(
+                {"_id": artifact.artifact_id},
+                {"$set": {"artifact.retention_class": artifact.retention_class}},
+            )
+        elif artifact.retention_class is ScrapeRetentionClass.CACHE_7D:
+            await self.artifacts.update_one(
+                {
+                    "_id": artifact.artifact_id,
+                    "artifact.retention_class": ScrapeRetentionClass.TRANSIENT_24H,
+                },
+                {"$set": {"artifact.retention_class": artifact.retention_class}},
+            )
+        raw = await self.artifacts.find_one({"_id": artifact.artifact_id})
+        if raw is None:
+            raise self._not_found("scrape artifact")
+        stored = ScrapeArtifactMongoObject.model_validate(self._without_id(raw))
+        if stored.object_key != object_key:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                message="artifact identity resolved to a different object key",
+            )
+        return stored.artifact
+
+    async def get(self, *, artifact_id: UUID) -> ScrapeArtifactRef:
+        """Read one artifact by primary key.
+
+        Side effects if changes:
+        - download authorization and downstream pins.
+        """
+
+        raw = await self.artifacts.find_one({"_id": artifact_id})
+        if raw is None:
+            raise self._not_found("scrape artifact")
+        return ScrapeArtifactMongoObject.model_validate(self._without_id(raw)).artifact
+
+    async def get_many_existing(
+        self,
+        *,
+        artifact_ids: Sequence[UUID],
+    ) -> ArtifactBatchLookup:
+        """Resolve one ordered artifact batch without raising for stale identities.
+
+        Steps:
+        1. fetch all distinct requested IDs in one indexed ``_id`` query;
+        2. validate and map each persisted artifact by identity;
+        3. rebuild requested order and report every missing ID explicitly.
+
+        Side effects if changes:
+        - cache completion performs one Mongo read and invalidates stale pointers safely.
+        """
+
+        distinct_ids = tuple(dict.fromkeys(artifact_ids))
+        rows = await self.artifacts.find({"_id": {"$in": distinct_ids}}).to_list(
+            length=len(distinct_ids)
+        )
+        by_id = {
+            row["_id"]: ScrapeArtifactMongoObject.model_validate(
+                self._without_id(row)
+            ).artifact
+            for row in rows
+        }
+        missing = tuple(
+            artifact_id for artifact_id in artifact_ids if artifact_id not in by_id
+        )
+        artifacts = tuple(
+            by_id[artifact_id] for artifact_id in artifact_ids if artifact_id in by_id
+        )
+        return ArtifactBatchLookup(
+            artifacts=artifacts,
+            missing_artifact_ids=missing,
+        )
+
+    async def object_key(self, *, artifact_id: UUID) -> str:
+        """Resolve the private object key without exposing it in API models.
+
+        Side effects if changes:
+        - signed downloads and orphan deletion.
+        """
+
+        raw = await self.artifacts.find_one({"_id": artifact_id}, {"object_key": 1})
+        if raw is None:
+            raise self._not_found("scrape artifact")
+        return str(raw["object_key"])
+
+    async def pin(self, *, pin: ScrapeArtifactPin) -> None:
+        """Synchronize one monotonic retention intent and reference exactly once.
+
+        Steps:
+        1. Validate the artifact exists and is not under a live deletion lease.
+        2. Insert the stable owner pin and increment its reference only once.
+        3. Ignore stale revisions, accept exact replays, and reject split-brain reuse.
+        4. Compare-and-set a newer expiry and revision in the same transaction.
+
+        Side effects if changes:
+        - downstream revision retention and orphan eligibility;
+        - Shopify out-of-order outbox convergence and storage cost.
+        """
+
+        pin_id = uuid5(_PIN_NAMESPACE, f"{pin.artifact_id}:{pin.owner_identity}")
+        now = utc_now()
+        async with self._client.start_session() as session:
+            async with await session.start_transaction(
+                read_concern=ReadConcern("snapshot"),
+                write_concern=WriteConcern("majority"),
+            ):
+                existing_artifact = await self.artifacts.find_one(
+                    {"_id": pin.artifact_id},
+                    session=session,
+                )
+                if existing_artifact is None:
+                    raise self._not_found("scrape artifact")
+                artifact_model = self._artifact_model(existing_artifact)
+                if (
+                    artifact_model.deletion_lease_expires_at is not None
+                    and artifact_model.deletion_lease_expires_at > now
+                ):
+                    raise ScraperDomainError(
+                        code=ScraperErrorCode.REVISION_CONFLICT,
+                        message="scrape artifact cleanup is already in progress",
+                    )
+                row = ScrapeArtifactPinMongoObject(
+                    artifact_id=pin.artifact_id,
+                    owner_identity=pin.owner_identity,
+                    retention_class=artifact_model.artifact.retention_class,
+                    retention_revision=pin.retention_revision,
+                    expires_at=pin.expires_at,
+                    created_at=now,
+                    updated_at=now,
+                )
+                before = await self.artifact_pins.find_one_and_update(
+                    {"_id": pin_id},
+                    {"$setOnInsert": row.model_dump(mode="python", by_alias=False)},
+                    upsert=True,
+                    return_document=ReturnDocument.BEFORE,
+                    session=session,
+                )
+                if before is None:
+                    await self.artifacts.update_one(
+                        {"_id": pin.artifact_id},
+                        {
+                            "$inc": {"reference_count": 1},
+                            "$unset": {
+                                "deletion_lease_id": "",
+                                "deletion_lease_expires_at": "",
+                            },
+                        },
+                        session=session,
+                    )
+                else:
+                    persisted = ScrapeArtifactPinMongoObject.model_validate(
+                        self._without_id(before)
+                    )
+                    if pin.retention_revision < persisted.retention_revision:
+                        return
+                    if pin.retention_revision == persisted.retention_revision:
+                        if not self._same_pin_expiry(
+                            left=pin.expires_at,
+                            right=persisted.expires_at,
+                        ):
+                            raise ScraperDomainError(
+                                code=ScraperErrorCode.REVISION_CONFLICT,
+                                message=(
+                                    "artifact retention revision was reused with "
+                                    "different expiry policy"
+                                ),
+                            )
+                        return
+                    advanced = await self.artifact_pins.update_one(
+                        {
+                            "_id": pin_id,
+                            "retention_revision": persisted.retention_revision,
+                        },
+                        {
+                            "$set": {
+                                "retention_revision": pin.retention_revision,
+                                "expires_at": pin.expires_at,
+                                "updated_at": now,
+                            }
+                        },
+                        session=session,
+                    )
+                    if advanced.modified_count != 1:
+                        raise ScraperDomainError(
+                            code=ScraperErrorCode.REVISION_CONFLICT,
+                            message="artifact retention changed during compare-and-set",
+                        )
+
+    @staticmethod
+    def _same_pin_expiry(
+        *,
+        left: datetime | None,
+        right: datetime | None,
+    ) -> bool:
+        """Compare expiry values at MongoDB's millisecond datetime precision.
+
+        Steps:
+        1. Preserve `None` as the explicit indefinite-retention policy.
+        2. Truncate aware finite timestamps to persisted millisecond precision.
+        3. Return equality only for one replay-equivalent policy.
+
+        Side effects if changes:
+        - same-revision pin replay acceptance;
+        - false split-brain conflict detection after Mongo round trips.
+        """
+
+        if left is None or right is None:
+            return left is right
+        left_millisecond = left.replace(microsecond=(left.microsecond // 1_000) * 1_000)
+        right_millisecond = right.replace(
+            microsecond=(right.microsecond // 1_000) * 1_000
+        )
+        return left_millisecond == right_millisecond
+
+    async def release_expired_pins(self, *, now: datetime) -> int:
+        """Delete expired pins and decrement artifact references transactionally.
+
+        Steps:
+        1. select a bounded indexed set of expired owner pins;
+        2. delete pins and decrement each artifact reference count atomically;
+        3. return the exact released pin count.
+
+        Side effects if changes:
+        - twelve-month retention and orphan eligibility.
+        """
+
+        rows = (
+            await self.artifact_pins.find({"expires_at": {"$lte": now}})
+            .limit(1_000)
+            .to_list(length=1_000)
+        )
+        if not rows:
+            return 0
+        async with self._client.start_session() as session:
+            async with await session.start_transaction(
+                read_concern=ReadConcern("snapshot"),
+                write_concern=WriteConcern("majority"),
+            ):
+                await self.artifact_pins.delete_many(
+                    {"_id": {"$in": [row["_id"] for row in rows]}},
+                    session=session,
+                )
+                for row in rows:
+                    await self.artifacts.update_one(
+                        {"_id": row["artifact_id"], "reference_count": {"$gt": 0}},
+                        {"$inc": {"reference_count": -1}},
+                        session=session,
+                    )
+        return len(rows)
+
+    async def lease_orphan_object_keys(
+        self,
+        *,
+        now: datetime,
+        limit: int,
+        lease_id: UUID,
+        lease_seconds: int,
+    ) -> tuple[str, ...]:
+        """Atomically lease expired unreferenced objects through `orphan_expiry_idx`.
+
+        Side effects if changes:
+        - maintenance object deletion exclusivity and storage cost.
+        """
+
+        object_keys: list[str] = []
+        for _ in range(limit):
+            row = await self.artifacts.find_one_and_update(
+                {
+                    "reference_count": 0,
+                    "artifact.expires_at": {"$lte": now},
+                    "$or": [
+                        {"deletion_lease_expires_at": {"$exists": False}},
+                        {"deletion_lease_expires_at": None},
+                        {"deletion_lease_expires_at": {"$lte": now}},
+                    ],
+                },
+                {
+                    "$set": {
+                        "deletion_lease_id": lease_id,
+                        "deletion_lease_expires_at": now
+                        + timedelta(seconds=lease_seconds),
+                    }
+                },
+                sort=[("artifact.expires_at", ASCENDING)],
+                return_document=ReturnDocument.AFTER,
+            )
+            if row is None:
+                break
+            object_keys.append(str(row["object_key"]))
+        return tuple(object_keys)
+
+    async def complete_orphan_deletion(
+        self,
+        *,
+        object_keys: Sequence[str],
+        lease_id: UUID,
+    ) -> int:
+        """Delete metadata only after the same lease deleted object-store bytes.
+
+        Side effects if changes:
+        - orphan cleanup ordering and private object accounting.
+        """
+
+        if not object_keys:
+            return 0
+        result = await self.artifacts.delete_many(
+            {
+                "object_key": {"$in": list(object_keys)},
+                "reference_count": 0,
+                "deletion_lease_id": lease_id,
+            }
+        )
+        return result.deleted_count
+
+    async def release_orphan_deletion(
+        self,
+        *,
+        object_keys: Sequence[str],
+        lease_id: UUID,
+    ) -> None:
+        """Release object leases after an idempotent store deletion failure.
+
+        Side effects if changes:
+        - orphan retry eligibility and pin availability.
+        """
+
+        if not object_keys:
+            return
+        await self.artifacts.update_many(
+            {
+                "object_key": {"$in": list(object_keys)},
+                "deletion_lease_id": lease_id,
+            },
+            {
+                "$unset": {
+                    "deletion_lease_id": "",
+                    "deletion_lease_expires_at": "",
+                }
+            },
+        )
+
+    async def import_candidate(
+        self, *, command: ProxyCandidateCreate
+    ) -> ProxyCandidateView:
+        """Encrypt and insert one candidate in mandatory quarantine.
+
+        Steps:
+        1. normalize/redact and encrypt the submitted proxy credential;
+        2. insert a deterministic quarantined candidate exactly once;
+        3. return only the credential-free operational view.
+
+        Side effects if changes:
+        - future experimental egress and proxy audit visibility.
+        """
+
+        redacted, encrypted = self._cipher.encode(command=command)
+        now = utc_now()
+        candidate_id = uuid5(_PROXY_NAMESPACE, redacted)
+        candidate = ProxyCandidateMongoObject(
+            candidate_id=candidate_id,
+            endpoint_redacted=redacted,
+            encrypted_proxy_url=encrypted,
+            source=command.source,
+            source_reference=command.source_reference,
+            state=ProxyCandidateState.QUARANTINED,
+            health_score=Decimal("0"),
+            next_health_at=now,
+            created_at=now,
+            updated_at=now,
+        )
+        raw = await self.proxies.find_one_and_update(
+            {"_id": candidate_id},
+            {"$setOnInsert": candidate.model_dump(mode="python", by_alias=False)},
+            upsert=True,
+            return_document=ReturnDocument.AFTER,
+        )
+        if raw is None:
+            raise self._not_found("proxy candidate")
+        return self._proxy_view(self._proxy_model(raw))
+
+    async def list_candidates(self, *, limit: int) -> tuple[ProxyCandidateView, ...]:
+        """List best candidates through `health_created_desc_idx`.
+
+        Steps:
+        1. bound the query by operator limit;
+        2. order by health then recency through the supporting index;
+        3. convert every encrypted row to a redacted view.
+
+        Side effects if changes:
+        - proxy dashboard ordering.
+        """
+
+        rows = (
+            await self.proxies.find({})
+            .sort([("health_score", DESCENDING), ("created_at", DESCENDING)])
+            .limit(limit)
+            .to_list(length=limit)
+        )
+        return tuple(self._proxy_view(self._proxy_model(row)) for row in rows)
+
+    async def choose_healthy(self) -> SecretStr | None:
+        """Decrypt the best promoted experimental candidate at dispatch time.
+
+        Side effects if changes:
+        - public experimental route egress.
+        """
+
+        raw = await self.proxies.find_one(
+            {"state": ProxyCandidateState.HEALTHY_EXPERIMENTAL.value},
+            sort=[("health_score", DESCENDING)],
+        )
+        return (
+            self._cipher.decode(encrypted_proxy_url=str(raw["encrypted_proxy_url"]))
+            if raw is not None
+            else None
+        )
+
+    async def credential_for(self, *, candidate_id: UUID) -> SecretStr:
+        """Decrypt one candidate credential solely for a health/dispatch worker.
+
+        Steps:
+        1. project only the encrypted credential by candidate identity;
+        2. reject a missing candidate with a typed not-found error;
+        3. decrypt into write-only in-memory request material.
+
+        Side effects if changes:
+        - public proxy credentials enter one worker's in-memory request config.
+        """
+
+        raw = await self.proxies.find_one(
+            {"_id": candidate_id}, {"encrypted_proxy_url": 1}
+        )
+        if raw is None:
+            raise self._not_found("proxy candidate")
+        return self._cipher.decode(encrypted_proxy_url=str(raw["encrypted_proxy_url"]))
+
+    async def due_candidates(
+        self,
+        *,
+        now: datetime,
+        limit: int,
+    ) -> tuple[ProxyCandidateView, ...]:
+        """Read due candidates through `state_next_health_idx`.
+
+        Steps:
+        1. restrict to health-managed states and due timestamps;
+        2. order and bound through the compound index;
+        3. return credential-free candidate views.
+
+        Side effects if changes:
+        - proxy health worker load and quarantine freshness.
+        """
+
+        rows = (
+            await self.proxies.find(
+                {
+                    "state": {
+                        "$in": [
+                            ProxyCandidateState.QUARANTINED.value,
+                            ProxyCandidateState.HEALTHY_EXPERIMENTAL.value,
+                            ProxyCandidateState.DEGRADED.value,
+                        ]
+                    },
+                    "next_health_at": {"$lte": now},
+                }
+            )
+            .sort("next_health_at", ASCENDING)
+            .limit(limit)
+            .to_list(length=limit)
+        )
+        return tuple(self._proxy_view(self._proxy_model(row)) for row in rows)
+
+    async def record_health(
+        self,
+        *,
+        candidate_id: UUID,
+        observation: ProxyHealthObservation,
+        next_health_at: datetime,
+    ) -> ProxyCandidateView:
+        """Append bounded health history and derive promotion/decay state.
+
+        Steps:
+        1. load and validate the current candidate row;
+        2. append one observation within the bounded history window;
+        3. derive score/state and persist the next check time.
+
+        Side effects if changes:
+        - experimental route eligibility and proxy trust.
+        """
+
+        current = await self.proxies.find_one({"_id": candidate_id})
+        if current is None:
+            raise self._not_found("proxy candidate")
+        model = self._proxy_model(current)
+        recent = [*model.observations[-19:], observation]
+        success_count = sum(1 for item in recent if item.succeeded and item.tls_valid)
+        score = (Decimal(success_count) / Decimal(len(recent))).quantize(
+            Decimal("0.001")
+        )
+        state = (
+            ProxyCandidateState.HEALTHY_EXPERIMENTAL
+            if len(recent) >= 5 and score >= Decimal("0.8")
+            else ProxyCandidateState.DEGRADED
+            if len(recent) >= 5
+            else ProxyCandidateState.QUARANTINED
+        )
+        raw = await self.proxies.find_one_and_update(
+            {"_id": candidate_id},
+            {
+                "$set": {
+                    "observations": [
+                        item.model_dump(mode="python", by_alias=False)
+                        for item in recent
+                    ],
+                    "health_score": score,
+                    "state": state.value,
+                    "next_health_at": next_health_at,
+                    "updated_at": utc_now(),
+                }
+            },
+            return_document=ReturnDocument.AFTER,
+        )
+        if raw is None:
+            raise self._not_found("proxy candidate")
+        return self._proxy_view(self._proxy_model(raw))
+
+    async def append(self, *, event: ScrapeAuditEventMongoObject) -> None:
+        """Insert one immutable audit fact with its absolute retention clock.
+
+        Steps:
+        1. derive the expiry solely from the server-owned occurrence time;
+        2. preserve the immutable caller/action/reason evidence;
+        3. insert the exact audit identity once.
+
+        Side effects if changes:
+        - compliance records remain queryable for the configured audit window;
+        - bounded maintenance eventually removes only expired audit facts.
+        """
+
+        stored = event.model_copy(
+            update={
+                "history_expires_at": (
+                    event.occurred_at + self._audit_history_retention
+                )
+            }
+        )
+        await self.audit.insert_one(
+            {
+                "_id": stored.audit_id,
+                **stored.model_dump(mode="python", by_alias=False),
+            }
+        )
+
+    async def lease_callback_due(
+        self,
+        *,
+        worker_id: str,
+        now: datetime,
+        lease_seconds: int,
+    ) -> ScrapeOutboxMongoObject | None:
+        """Lease one due callback inside its immutable delivery budget.
+
+        Steps:
+        1. select pending due work below its stored attempt limit;
+        2. assign an exact worker owner and expiry;
+        3. increment the durable callback attempt ordinal atomically.
+
+        Side effects if changes:
+        - webhook delivery concurrency and replay identity.
+        """
+
+        raw = await self.outbox.find_one_and_update(
+            {
+                "state": OutboxState.PENDING.value,
+                "next_attempt_at": {"$lte": now},
+                "$expr": {"$lt": ["$attempt_count", "$attempt_limit"]},
+            },
+            {
+                "$set": {
+                    "state": OutboxState.LEASED.value,
+                    "lease_owner": worker_id,
+                    "lease_expires_at": now + timedelta(seconds=lease_seconds),
+                },
+                "$inc": {"attempt_count": 1},
+            },
+            sort=[("next_attempt_at", ASCENDING)],
+            return_document=ReturnDocument.AFTER,
+        )
+        return (
+            ScrapeOutboxMongoObject.model_validate(self._without_id(raw))
+            if raw is not None
+            else None
+        )
+
+    async def recover_expired_callback_leases(
+        self,
+        *,
+        now: datetime,
+        limit: int = 1_000,
+    ) -> LeaseRecoverySummary:
+        """Recover crashed callback owners or terminalize spent delivery budgets.
+
+        Steps:
+        1. read a bounded expired-lease batch through ``state_lease_expiry_idx``;
+        2. return rows below their immutable limit to pending state;
+        3. dead-letter rows whose last leased delivery spent the limit.
+
+        Side effects if changes:
+        - crashed callback workers cannot strand terminal Shopify events forever;
+        - callback delivery never exceeds its stored attempt budget.
+        """
+
+        rows = (
+            await self.outbox.find(
+                {
+                    "state": OutboxState.LEASED.value,
+                    "lease_expires_at": {"$lte": now},
+                }
+            )
+            .sort("lease_expires_at", ASCENDING)
+            .limit(limit)
+            .to_list(length=limit)
+        )
+        recovered_count = 0
+        terminalized_count = 0
+        for raw in rows:
+            row = ScrapeOutboxMongoObject.model_validate(self._without_id(raw))
+            dead_letter = row.attempt_count >= row.attempt_limit
+            updated = await self.outbox.update_one(
+                {
+                    "_id": row.event.event_id,
+                    "state": OutboxState.LEASED.value,
+                    "lease_owner": row.lease_owner,
+                    "lease_expires_at": {"$lte": now},
+                    "attempt_count": row.attempt_count,
+                },
+                {
+                    "$set": {
+                        "state": (
+                            OutboxState.DEAD_LETTER.value
+                            if dead_letter
+                            else OutboxState.PENDING.value
+                        ),
+                        "next_attempt_at": now,
+                    },
+                    "$unset": {"lease_owner": "", "lease_expires_at": ""},
+                },
+            )
+            if dead_letter:
+                terminalized_count += updated.modified_count
+            else:
+                recovered_count += updated.modified_count
+        return LeaseRecoverySummary(
+            recovered_count=recovered_count,
+            terminalized_count=terminalized_count,
+        )
+
+    async def mark_delivered(
+        self,
+        *,
+        event_id: UUID,
+        worker_id: str,
+        delivered_at: datetime,
+    ) -> None:
+        """Mark one callback delivered and start dependent retention clocks.
+
+        Steps:
+        1. require the event to remain leased by the acknowledging worker;
+        2. retain the delivered callback until its independent replay horizon;
+        3. start terminal Job history retention only after durable delivery.
+
+        Side effects if changes:
+        - duplicate callback prevention and callback replay evidence;
+        - terminal Job/Attempt history becomes eligible for ordered purge.
+        """
+
+        outbox_expires_at = delivered_at + self._delivered_outbox_retention
+        job_expires_at = delivered_at + self._terminal_history_retention
+        async with self._client.start_session() as session:
+            async with await session.start_transaction(
+                read_concern=ReadConcern("snapshot"),
+                write_concern=WriteConcern("majority"),
+            ):
+                raw = await self.outbox.find_one_and_update(
+                    {
+                        "_id": event_id,
+                        "state": OutboxState.LEASED.value,
+                        "lease_owner": worker_id,
+                    },
+                    {
+                        "$set": {
+                            "state": OutboxState.DELIVERED.value,
+                            "delivered_at": delivered_at,
+                            "history_expires_at": outbox_expires_at,
+                        },
+                        "$unset": {"lease_owner": "", "lease_expires_at": ""},
+                    },
+                    return_document=ReturnDocument.AFTER,
+                    session=session,
+                )
+                if raw is None:
+                    raise self._late_acknowledgement()
+                outbox = ScrapeOutboxMongoObject.model_validate(self._without_id(raw))
+                job_update = await self.jobs.update_one(
+                    {
+                        "_id": outbox.event.job_id,
+                        "state": {
+                            "$in": [
+                                ScrapeJobState.SUCCEEDED.value,
+                                ScrapeJobState.FAILED.value,
+                                ScrapeJobState.CANCELLED.value,
+                                ScrapeJobState.DEAD_LETTER.value,
+                            ]
+                        },
+                        "history_expires_at": None,
+                    },
+                    {"$set": {"history_expires_at": job_expires_at}},
+                    session=session,
+                )
+                if job_update.modified_count != 1:
+                    raise RuntimeError(
+                        "delivered callback does not reference one retained terminal job"
+                    )
+
+    async def purge_expired_history(
+        self,
+        *,
+        now: datetime,
+        limit: int,
+    ) -> HistoryRetentionSummary:
+        """Delete one bounded dependency-ordered history batch atomically.
+
+        Steps:
+        1. acquire one expiring Mongo singleton lease without blocking a peer;
+        2. select each due collection through `history_expires_at_idx`;
+        3. delete Attempt children with terminal Job parents in one transaction;
+        4. delete independently expired delivered callbacks and audit facts;
+        5. release only the exact lease owner after the transaction commits.
+
+        Collection/query/index:
+        - `scrape_jobs`, `scrape_outbox`, and `scrape_audit_events` use
+          `history_expires_at_idx`; attempts use `job_attempt_number_unique_idx`.
+
+        Side effects if changes:
+        - financial/source evidence and callback/audit replay horizons;
+        - Mongo working-set size, backup volume, and maintenance throughput.
+        """
+
+        if limit < 1:
+            raise ValueError("history retention limit must be positive")
+        lease_owner = uuid4()
+        lease = await self.metadata.find_one_and_update(
+            {
+                "_id": _HISTORY_RETENTION_LEASE_ID,
+                "lease_expires_at": {"$lte": now},
+            },
+            {
+                "$set": {
+                    "lease_owner": lease_owner,
+                    "lease_expires_at": now + self._history_retention_lease,
+                }
+            },
+            return_document=ReturnDocument.AFTER,
+        )
+        if lease is None:
+            return HistoryRetentionSummary(lease_acquired=False)
+
+        jobs_deleted = 0
+        attempts_deleted = 0
+        delivered_outbox_deleted = 0
+        audit_events_deleted = 0
+        async with self._client.start_session() as session:
+            async with await session.start_transaction(
+                read_concern=ReadConcern("snapshot"),
+                write_concern=WriteConcern("majority"),
+            ):
+                lease_fence = await self.metadata.update_one(
+                    {
+                        "_id": _HISTORY_RETENTION_LEASE_ID,
+                        "lease_owner": lease_owner,
+                        "lease_expires_at": {"$gt": utc_now()},
+                    },
+                    {"$inc": {"fence_revision": 1}},
+                    session=session,
+                )
+                if lease_fence.modified_count != 1:
+                    raise RuntimeError(
+                        "history retention lease expired or was replaced before purge"
+                    )
+                job_rows = await (
+                    self.jobs.find(
+                        {
+                            "state": {
+                                "$in": [
+                                    ScrapeJobState.SUCCEEDED.value,
+                                    ScrapeJobState.FAILED.value,
+                                    ScrapeJobState.CANCELLED.value,
+                                    ScrapeJobState.DEAD_LETTER.value,
+                                ]
+                            },
+                            "history_expires_at": {"$lte": now},
+                        },
+                        {"_id": 1},
+                        session=session,
+                    )
+                    .sort("history_expires_at", ASCENDING)
+                    .limit(limit)
+                    .to_list(length=limit)
+                )
+                job_ids = [row["_id"] for row in job_rows]
+                if job_ids:
+                    attempt_result = await self.attempts.delete_many(
+                        {"job_id": {"$in": job_ids}},
+                        session=session,
+                    )
+                    job_result = await self.jobs.delete_many(
+                        {
+                            "_id": {"$in": job_ids},
+                            "state": {
+                                "$in": [
+                                    ScrapeJobState.SUCCEEDED.value,
+                                    ScrapeJobState.FAILED.value,
+                                    ScrapeJobState.CANCELLED.value,
+                                    ScrapeJobState.DEAD_LETTER.value,
+                                ]
+                            },
+                            "history_expires_at": {"$lte": now},
+                        },
+                        session=session,
+                    )
+                    if job_result.deleted_count != len(job_ids):
+                        raise RuntimeError(
+                            "terminal history changed during atomic retention purge"
+                        )
+                    attempts_deleted = attempt_result.deleted_count
+                    jobs_deleted = job_result.deleted_count
+
+                outbox_rows = await (
+                    self.outbox.find(
+                        {
+                            "state": OutboxState.DELIVERED.value,
+                            "history_expires_at": {"$lte": now},
+                        },
+                        {"_id": 1},
+                        session=session,
+                    )
+                    .sort("history_expires_at", ASCENDING)
+                    .limit(limit)
+                    .to_list(length=limit)
+                )
+                outbox_ids = [row["_id"] for row in outbox_rows]
+                if outbox_ids:
+                    outbox_result = await self.outbox.delete_many(
+                        {
+                            "_id": {"$in": outbox_ids},
+                            "state": OutboxState.DELIVERED.value,
+                            "history_expires_at": {"$lte": now},
+                        },
+                        session=session,
+                    )
+                    if outbox_result.deleted_count != len(outbox_ids):
+                        raise RuntimeError(
+                            "delivered callback history changed during atomic purge"
+                        )
+                    delivered_outbox_deleted = outbox_result.deleted_count
+
+                audit_rows = await (
+                    self.audit.find(
+                        {"history_expires_at": {"$lte": now}},
+                        {"_id": 1},
+                        session=session,
+                    )
+                    .sort("history_expires_at", ASCENDING)
+                    .limit(limit)
+                    .to_list(length=limit)
+                )
+                audit_ids = [row["_id"] for row in audit_rows]
+                if audit_ids:
+                    audit_result = await self.audit.delete_many(
+                        {
+                            "_id": {"$in": audit_ids},
+                            "history_expires_at": {"$lte": now},
+                        },
+                        session=session,
+                    )
+                    if audit_result.deleted_count != len(audit_ids):
+                        raise RuntimeError(
+                            "audit history changed during atomic retention purge"
+                        )
+                    audit_events_deleted = audit_result.deleted_count
+
+        await self.metadata.update_one(
+            {
+                "_id": _HISTORY_RETENTION_LEASE_ID,
+                "lease_owner": lease_owner,
+            },
+            {
+                "$set": {"lease_expires_at": now},
+                "$unset": {"lease_owner": ""},
+            },
+        )
+        return HistoryRetentionSummary(
+            lease_acquired=True,
+            jobs_deleted=jobs_deleted,
+            attempts_deleted=attempts_deleted,
+            delivered_outbox_deleted=delivered_outbox_deleted,
+            audit_events_deleted=audit_events_deleted,
+        )
+
+    async def reschedule(
+        self,
+        *,
+        event_id: UUID,
+        worker_id: str,
+        next_attempt_at: datetime,
+        dead_letter: bool,
+    ) -> None:
+        """Reschedule one exact worker-owned callback or dead-letter it.
+
+        Steps:
+        1. require the current lease owner before any state change;
+        2. retain the supplied durable retry horizon or terminal decision;
+        3. reject stale owners after recovery or replacement.
+
+        Side effects if changes:
+        - callback retry cadence and operational gaps.
+        """
+
+        updated = await self.outbox.update_one(
+            {
+                "_id": event_id,
+                "state": OutboxState.LEASED.value,
+                "lease_owner": worker_id,
+            },
+            {
+                "$set": {
+                    "state": (
+                        OutboxState.DEAD_LETTER.value
+                        if dead_letter
+                        else OutboxState.PENDING.value
+                    ),
+                    "next_attempt_at": next_attempt_at,
+                },
+                "$unset": {"lease_owner": "", "lease_expires_at": ""},
+            },
+        )
+        if updated.modified_count != 1:
+            raise self._late_acknowledgement()
+
+    @staticmethod
+    def _job_document(job: ScrapeJobMongoObject) -> dict[str, object]:
+        """Convert the Pydantic job into its primary-key Mongo document.
+
+        Side effects if changes:
+        - canonical job storage shape.
+        """
+
+        document = job.model_dump(
+            mode="python",
+            by_alias=False,
+            exclude={"create"},
+        )
+        document["create"] = job.create.model_dump(mode="json", by_alias=False)
+        return {"_id": job.job_id, **document}
+
+    @staticmethod
+    def _job_model(raw: Mapping[str, Any]) -> ScrapeJobMongoObject:
+        """Validate one Mongo job document before application use.
+
+        Steps:
+        1. remove the storage-only `_id` duplicate;
+        2. validate every lifecycle/budget field through Pydantic;
+        3. return the canonical persistence model.
+
+        Side effects if changes:
+        - corruption detection on every job read.
+        """
+
+        return ScrapeJobMongoObject.model_validate(
+            MongoScraperRepository._without_id(raw)
+        )
+
+    @staticmethod
+    def _artifact_model(raw: Mapping[str, Any]) -> ScrapeArtifactMongoObject:
+        """Validate one artifact metadata document before retention decisions.
+
+        Side effects if changes:
+        - pin, cleanup lease, and private object consistency checks.
+        """
+
+        return ScrapeArtifactMongoObject.model_validate(
+            MongoScraperRepository._without_id(raw)
+        )
+
+    @staticmethod
+    def _proxy_model(raw: Mapping[str, Any]) -> ProxyCandidateMongoObject:
+        """Validate one encrypted proxy Mongo document.
+
+        Steps:
+        1. remove the storage-only `_id` duplicate;
+        2. validate encrypted/redacted/health fields;
+        3. return the canonical persistence model.
+
+        Side effects if changes:
+        - proxy list, health, and route selection.
+        """
+
+        return ProxyCandidateMongoObject.model_validate(
+            MongoScraperRepository._without_id(raw)
+        )
+
+    @staticmethod
+    def _proxy_view(model: ProxyCandidateMongoObject) -> ProxyCandidateView:
+        """Drop encrypted proxy material from the operational view.
+
+        Side effects if changes:
+        - API and frontend credential redaction.
+        """
+
+        return ProxyCandidateView(
+            candidate_id=model.candidate_id,
+            endpoint_redacted=model.endpoint_redacted,
+            source=model.source,
+            source_reference=model.source_reference,
+            state=model.state,
+            health_score=model.health_score,
+            observations=tuple(model.observations),
+            created_at=model.created_at,
+        )
+
+    @staticmethod
+    def _without_id(raw: Mapping[str, Any]) -> dict[str, Any]:
+        """Copy a Mongo document without its storage-only `_id` key.
+
+        Side effects if changes:
+        - every persistence-to-Pydantic conversion.
+        """
+
+        output = dict(raw)
+        output.pop("_id", None)
+        return output
+
+    @staticmethod
+    def _not_found(object_name: str) -> ScraperDomainError:
+        """Build the shared typed missing-object failure.
+
+        Side effects if changes:
+        - API 404 mapping and worker diagnostics.
+        """
+
+        return ScraperDomainError(
+            code=ScraperErrorCode.NOT_FOUND,
+            message=f"{object_name} was not found",
+        )
+
+    @staticmethod
+    def _late_acknowledgement() -> ScraperDomainError:
+        """Build the typed stale-worker acknowledgement failure.
+
+        Side effects if changes:
+        - duplicate completion and cancellation race safety.
+        """
+
+        return ScraperDomainError(
+            code=ScraperErrorCode.REVISION_CONFLICT,
+            message="job lease is stale or no longer owns this transition",
+        )