keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/application/jobs.py

@@ -0,0 +1,891 @@
+"""Canonical job submission, execution, retry, cache, and audit services."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timedelta
+from decimal import Decimal
+from uuid import UUID, uuid4, uuid5
+
+from keble_helpers import UpstreamFailureDisposition
+from keble_scraper_contract import (
+    OperatorSession,
+    ScrapeArtifactRef,
+    ScrapeArtifactTrust,
+    ScrapeAttemptOutcome,
+    ScrapeAttemptView,
+    ScrapeCacheMode,
+    ScrapeCost,
+    ScrapeJobAccepted,
+    ScrapeJobCancel,
+    ScrapeJobCreate,
+    ScrapeLocalFailureCode,
+    ScrapeJobState,
+    ScrapeJobView,
+    ScrapeRetentionClass,
+    ScrapeTransportRoute,
+    utc_now,
+)
+
+from keble_scraper_api.application.artifacts import ArtifactService
+from keble_scraper_api.application.fetching import FetchResult
+from keble_scraper_api.application.failures import (
+    build_fetch_failure,
+    build_local_failure,
+    terminalize_failure,
+)
+from keble_scraper_api.application.scheduling import ceil_persistence_millisecond
+from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
+from keble_scraper_api.models import (
+    AuditAction,
+    DispatchableJob,
+    FetchLease,
+    RouteReservation,
+    RouteReservationDenied,
+    ScrapeAuditEventMongoObject,
+    ScrapeCacheEntryMongoObject,
+    build_completion_event,
+)
+from keble_scraper_api.protocols import (
+    AuditRepositoryProtocol,
+    BrowserFetcherProtocol,
+    CacheRepositoryProtocol,
+    HistoryRetentionRepositoryProtocol,
+    HttpFetcherProtocol,
+    JobDispatcherProtocol,
+    JobRepositoryProtocol,
+    OutboxRepositoryProtocol,
+    ProxyRouterProtocol,
+    RequestProfileProtocol,
+)
+from keble_scraper_api.settings import ScraperSettings
+
+_ATTEMPT_NAMESPACE = UUID("d020a447-8384-4c34-a546-3f701ea524d7")
+
+
+class AuditService:
+    """Build immutable audit facts for every sensitive action.
+
+    Side effects if changes:
+    - compliance history for mutations and raw downloads.
+    """
+
+    def __init__(self, *, repository: AuditRepositoryProtocol) -> None:
+        """Bind the immutable audit repository.
+
+        Side effects if changes:
+        - every API mutation writes through this instance.
+        """
+
+        self._repository = repository
+
+    async def record(
+        self,
+        *,
+        actor_id: str,
+        actor_role: str,
+        action: AuditAction,
+        target_identity: str,
+        reason: str,
+        request_id: str,
+        audit_id: UUID | None = None,
+    ) -> UUID:
+        """Append one server-timestamped audit event and return its identity.
+
+        Steps:
+        1. accept a reserved server identity when an external action must carry it;
+        2. otherwise create the immutable identity at this canonical boundary;
+        3. append the typed event before returning the persisted identity.
+
+        Side effects if changes:
+        - Mongo audit storage, investigations, and OSS-log correlation.
+        """
+
+        resolved_audit_id = audit_id or uuid4()
+        await self._repository.append(
+            event=ScrapeAuditEventMongoObject(
+                audit_id=resolved_audit_id,
+                actor_id=actor_id,
+                actor_role=actor_role,
+                action=action,
+                target_identity=target_identity,
+                reason=reason,
+                request_id=request_id,
+                occurred_at=utc_now(),
+            )
+        )
+        return resolved_audit_id
+
+
+class ScrapeJobService:
+    """Own idempotent API submission/read/cancel behavior.
+
+    Side effects if changes:
+    - Mongo lifecycle, Celery dispatch, and mutation audit records.
+    """
+
+    def __init__(
+        self,
+        *,
+        repository: JobRepositoryProtocol,
+        dispatcher: JobDispatcherProtocol,
+        profiles: RequestProfileProtocol,
+        audit: AuditService,
+    ) -> None:
+        """Bind canonical lifecycle dependencies without infrastructure lookup.
+
+        Side effects if changes:
+        - all service and operator job endpoints.
+        """
+
+        self._repository = repository
+        self._dispatcher = dispatcher
+        self._profiles = profiles
+        self._audit = audit
+
+    async def create(
+        self,
+        *,
+        caller_identity: str,
+        command: ScrapeJobCreate,
+        request_id: str,
+        actor_role: str = "SERVICE",
+    ) -> ScrapeJobAccepted:
+        """Create/reuse one job and dispatch only the newly inserted identity.
+
+        Side effects if changes:
+        - queue work, idempotency, and service audit history.
+        """
+
+        prepared = self._profiles.prepare(command=command)
+        stored, reused = await self._repository.create_or_get(
+            caller_identity=caller_identity,
+            command=command,
+            request_hash=prepared.request_identity,
+        )
+        if not reused:
+            await self._dispatcher.dispatch(
+                job_id=stored.job_id,
+                transport_policy=command.transport_policy,
+            )
+            await self._audit.record(
+                actor_id=caller_identity,
+                actor_role=actor_role,
+                action=AuditAction.JOB_CREATE,
+                target_identity=str(stored.job_id),
+                reason="idempotent scrape job submission",
+                request_id=request_id,
+            )
+        return ScrapeJobAccepted(
+            job=await self._repository.get_view(job_id=stored.job_id),
+            reused=reused,
+        )
+
+    async def get(self, *, job_id: UUID) -> ScrapeJobView:
+        """Read one current job projection and attempt history.
+
+        Side effects if changes:
+        - service polling and operations UI.
+        """
+
+        return await self._repository.get_view(job_id=job_id)
+
+    async def list_admin(
+        self,
+        *,
+        limit: int,
+        before: datetime | None,
+    ) -> tuple[ScrapeJobView, ...]:
+        """List bounded newest jobs for authenticated operators.
+
+        Side effects if changes:
+        - dashboard Mongo read load.
+        """
+
+        return await self._repository.list_views(limit=limit, before=before)
+
+    async def cancel(
+        self,
+        *,
+        job_id: UUID,
+        command: ScrapeJobCancel,
+        actor: OperatorSession,
+        request_id: str,
+    ) -> ScrapeJobView:
+        """Authorize/commit optimistic cancellation and append audit evidence.
+
+        Side effects if changes:
+        - cancellation lifecycle and operator accountability.
+        """
+
+        if not actor.can_mutate():
+            raise ScraperDomainError(
+                code=ScraperErrorCode.AUTH_FORBIDDEN,
+                message="operator role does not permit mutations",
+            )
+        view = await self._repository.cancel(job_id=job_id, command=command)
+        await self._audit.record(
+            actor_id=actor.user_id,
+            actor_role=actor.role.value,
+            action=AuditAction.JOB_CANCEL,
+            target_identity=str(job_id),
+            reason=command.reason,
+            request_id=request_id,
+        )
+        return view
+
+    async def cancel_service(
+        self,
+        *,
+        job_id: UUID,
+        command: ScrapeJobCancel,
+        caller_identity: str,
+        request_id: str,
+    ) -> ScrapeJobView:
+        """Cancel a caller-owned job through the service credential boundary.
+
+        Side effects if changes:
+        - service cancellation lifecycle and audit identity.
+        """
+
+        current = await self._repository.get_view(job_id=job_id)
+        if current.caller_identity != caller_identity:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.AUTH_FORBIDDEN,
+                message="service caller does not own this job",
+            )
+        view = await self._repository.cancel(job_id=job_id, command=command)
+        await self._audit.record(
+            actor_id=caller_identity,
+            actor_role="SERVICE",
+            action=AuditAction.JOB_CANCEL,
+            target_identity=str(job_id),
+            reason=command.reason,
+            request_id=request_id,
+        )
+        return view
+
+
+class ScrapeExecutionService:
+    """Execute leased jobs through cache, budget, fetch, artifact, and outbox stages.
+
+    Steps:
+    1. lease one durable immutable job budget;
+    2. resolve cache and route preflight before physical work;
+    3. append typed evidence and commit retry or terminal state atomically.
+
+    Side effects if changes:
+    - external network/vendor calls, Mongo attempts, OSS artifacts, and callbacks.
+    """
+
+    def __init__(
+        self,
+        *,
+        settings: ScraperSettings,
+        jobs: JobRepositoryProtocol,
+        cache: CacheRepositoryProtocol,
+        artifacts: ArtifactService,
+        router: ProxyRouterProtocol,
+        profiles: RequestProfileProtocol,
+        http_fetcher: HttpFetcherProtocol,
+        browser_fetcher: BrowserFetcherProtocol,
+    ) -> None:
+        """Bind one worker dependency graph from typed components.
+
+        Side effects if changes:
+        - every fetch/render worker execution.
+        """
+
+        self._settings = settings
+        self._jobs = jobs
+        self._cache = cache
+        self._artifacts = artifacts
+        self._router = router
+        self._profiles = profiles
+        self._http_fetcher = http_fetcher
+        self._browser_fetcher = browser_fetcher
+
+    async def process(self, *, job_id: UUID, worker_id: str) -> ScrapeJobView | None:
+        """Lease and fully process one idempotent job identity.
+
+        Steps:
+        1. return without side effects when another worker/cancel owns the job;
+        2. reuse valid cache or reserve a priced/rate-bounded route;
+        3. append the fetch attempt before terminal/retry state transition;
+        4. commit artifacts and callback outbox with the job terminal state.
+
+        Side effects if changes:
+        - queue acknowledgement safety and all scraper external/persistent work.
+        """
+
+        lease = await self._jobs.lease_due(
+            job_id=job_id,
+            worker_id=worker_id,
+            lease_seconds=self._settings.worker_lease_seconds,
+        )
+        if lease is None:
+            return None
+        command = lease.job.create
+        prepared = self._profiles.prepare(command=command)
+        cache_entry = await self._cache.get_valid(
+            request_hash=prepared.request_identity,
+            mode=command.cache_mode,
+            now=utc_now(),
+        )
+        if (
+            command.cache_mode is ScrapeCacheMode.REUSE_VALID
+            and cache_entry is not None
+        ):
+            cached = await self._complete_from_cache(
+                lease=lease,
+                entry=cache_entry,
+            )
+            if cached is not None:
+                return cached
+            cache_entry = None
+        reservation_result = await self._router.reserve(
+            policy=command.transport_policy,
+            affinity_key=prepared.request_identity,
+            estimated_bytes=self._settings.response_policy(
+                command.policy_class.value
+            ).max_bytes,
+            attempt_number=lease.attempt_number,
+        )
+        if isinstance(reservation_result, RouteReservationDenied):
+            return await self._record_preflight_failure(
+                lease=lease,
+                denial=reservation_result,
+                final_url=prepared.url,
+            )
+        reservation = reservation_result.reservation
+        conditionals = self._conditional_headers(entry=cache_entry)
+        response_policy = self._settings.response_policy(command.policy_class.value)
+        fetch_call = (
+            self._browser_fetcher.fetch(
+                request=prepared, response_policy=response_policy
+            )
+            if reservation.route is ScrapeTransportRoute.CLOUDFLARE_BROWSER_RUN
+            else self._http_fetcher.fetch(
+                request=prepared,
+                reservation=reservation,
+                response_policy=response_policy,
+                conditional_headers=conditionals,
+            )
+        )
+        fetched = await fetch_call
+        actual_cost = await self._router.reconcile(
+            reservation=reservation,
+            actual_bytes=len(fetched.body),
+            browser_seconds=float(fetched.browser_seconds),
+        )
+        attempt = self._attempt(
+            lease=lease,
+            result=fetched,
+            reservation=reservation,
+            actual_cost=actual_cost,
+        )
+        await self._jobs.append_attempt(job_id=job_id, attempt=attempt)
+        if fetched.succeeded:
+            return await self._commit_success(
+                lease=lease,
+                result=fetched,
+                cache_entry=cache_entry,
+                prepared_identity=prepared.request_identity,
+            )
+        await self._cache_negative(
+            result=fetched,
+            request_hash=prepared.request_identity,
+        )
+        return await self._commit_failure(lease=lease, result=fetched)
+
+    async def _complete_from_cache(
+        self,
+        *,
+        lease: FetchLease,
+        entry: ScrapeCacheEntryMongoObject,
+    ) -> ScrapeJobView | None:
+        """Complete a sound cache hit or invalidate one stale pointer.
+
+        Steps:
+        1. replay stable negative metadata without inventing provider usage;
+        2. resolve every positive artifact in one ordered batch read;
+        3. compare-and-delete a pointer with any missing artifact before fetching.
+
+        Side effects if changes:
+        - job terminal state and callback outbox receive only complete artifacts;
+        - stale cache entries cannot create lease-expiry retry loops.
+        """
+
+        if entry.response_status >= 400:
+            synthetic = FetchResult(
+                outcome=ScrapeAttemptOutcome.HTTP_ERROR,
+                route=ScrapeTransportRoute.DIRECT,
+                trust=ScrapeArtifactTrust.VERIFIED,
+                final_url="https://cached.invalid/",
+                status_code=entry.response_status,
+                content_type=entry.content_type,
+                elapsed_ms=0,
+                error_code=f"CACHED_HTTP_{entry.response_status}",
+                observed_at=utc_now(),
+            )
+            return await self._commit_failure(
+                lease=lease, result=synthetic, force_terminal=True
+            )
+        lookup = await self._artifacts.get_many_existing(
+            artifact_ids=entry.artifact_ids,
+        )
+        if lookup.missing_artifact_ids:
+            await self._cache.delete_current(
+                request_hash=entry.request_hash,
+                updated_at=entry.updated_at,
+            )
+            return None
+        event = build_completion_event(
+            job=lease.job,
+            state=ScrapeJobState.SUCCEEDED,
+            artifacts=lookup.artifacts,
+            failure=None,
+            occurred_at=utc_now(),
+        )
+        return await self._jobs.complete(
+            lease=lease,
+            artifacts=lookup.artifacts,
+            event=event,
+        )
+
+    async def _commit_success(
+        self,
+        *,
+        lease: FetchLease,
+        result: FetchResult,
+        cache_entry: ScrapeCacheEntryMongoObject | None,
+        prepared_identity: str,
+    ) -> ScrapeJobView:
+        """Capture/cache content and commit terminal success with outbox.
+
+        Steps:
+        1. validate every conditional-cache artifact before accepting a 304;
+        2. capture new content before publishing a retention-bounded cache pointer;
+        3. atomically commit success and its deterministic callback event.
+
+        Side effects if changes:
+        - OSS/Mongo artifacts, cache pointers, job state, and callback event.
+        """
+
+        if result.status_code == 304:
+            if cache_entry is None:
+                missing = result.model_copy(
+                    update={
+                        "outcome": ScrapeAttemptOutcome.HTTP_ERROR,
+                        "error_code": "CACHE_VALIDATOR_WITHOUT_ENTRY",
+                    }
+                )
+                return await self._commit_failure(
+                    lease=lease, result=missing, force_terminal=True
+                )
+            lookup = await self._artifacts.get_many_existing(
+                artifact_ids=cache_entry.artifact_ids,
+            )
+            if lookup.missing_artifact_ids:
+                await self._cache.delete_current(
+                    request_hash=cache_entry.request_hash,
+                    updated_at=cache_entry.updated_at,
+                )
+                missing = result.model_copy(
+                    update={
+                        "outcome": ScrapeAttemptOutcome.HTTP_ERROR,
+                        "error_code": "CACHE_VALIDATOR_WITHOUT_ENTRY",
+                    }
+                )
+                return await self._commit_failure(
+                    lease=lease,
+                    result=missing,
+                )
+            artifacts = lookup.artifacts
+        else:
+            artifacts = await self._artifacts.capture(
+                result=result,
+                capture_mode=lease.job.create.capture_mode,
+                retention_class=lease.job.create.retention_class,
+            )
+            cache_updated_at = utc_now()
+            await self._cache.upsert(
+                entry=ScrapeCacheEntryMongoObject(
+                    request_hash=prepared_identity,
+                    artifact_ids=[item.artifact_id for item in artifacts],
+                    etag=result.etag,
+                    last_modified=result.last_modified,
+                    response_status=result.status_code or 200,
+                    content_type=result.content_type or "application/octet-stream",
+                    expires_at=self._cache_expiry(
+                        artifacts=artifacts,
+                        retention_class=lease.job.create.retention_class,
+                        now=cache_updated_at,
+                    ),
+                    updated_at=cache_updated_at,
+                )
+            )
+        event = build_completion_event(
+            job=lease.job,
+            state=ScrapeJobState.SUCCEEDED,
+            artifacts=artifacts,
+            failure=None,
+            occurred_at=utc_now(),
+        )
+        return await self._jobs.complete(
+            lease=lease,
+            artifacts=artifacts,
+            event=event,
+        )
+
+    async def _commit_failure(
+        self,
+        *,
+        lease: FetchLease,
+        result: FetchResult,
+        force_terminal: bool = False,
+    ) -> ScrapeJobView:
+        """Apply bounded retry policy or terminal dead-letter with callback.
+
+        Steps:
+        1. classify the typed fetch result and compute local exponential backoff;
+        2. honor the later provider retry horizon without crossing the job deadline;
+        3. terminalize against immutable stored attempt/deadline policy and emit callback.
+
+        Side effects if changes:
+        - retry timing, vendor fallback, dead letters, and Shopify gap state.
+        """
+
+        failure = build_fetch_failure(result=result)
+        retryable = failure.disposition is UpstreamFailureDisposition.RETRYABLE
+        now = utc_now()
+        delay = self._settings.retry_base_seconds * (
+            2 ** max(0, lease.attempt_number - 1)
+        )
+        local_next_attempt_at = now + timedelta(seconds=delay)
+        next_attempt_at = ceil_persistence_millisecond(
+            value=max(
+                local_next_attempt_at,
+                failure.retry_at or local_next_attempt_at,
+            )
+        )
+        attempt_budget_exhausted = lease.attempt_number >= lease.job.attempt_limit
+        deadline_exhausted = (
+            lease.job.deadline_at <= now or next_attempt_at >= lease.job.deadline_at
+        )
+        dead_letter = (
+            force_terminal
+            or not retryable
+            or attempt_budget_exhausted
+            or deadline_exhausted
+        )
+        if deadline_exhausted and retryable and not force_terminal:
+            terminal_failure = build_local_failure(
+                code=ScrapeLocalFailureCode.JOB_DEADLINE_EXCEEDED,
+                safe_message="Scraper job deadline expired before safe retry.",
+                observed_at=now,
+                physical_attempt_count=failure.physical_attempt_count,
+            )
+        elif dead_letter:
+            terminal_failure = terminalize_failure(failure=failure)
+        else:
+            terminal_failure = failure
+        event = (
+            build_completion_event(
+                job=lease.job,
+                state=ScrapeJobState.DEAD_LETTER,
+                artifacts=(),
+                failure=terminal_failure,
+                occurred_at=now,
+            )
+            if dead_letter
+            else None
+        )
+        return await self._jobs.schedule_retry(
+            lease=lease,
+            failure=terminal_failure,
+            next_attempt_at=next_attempt_at,
+            dead_letter=dead_letter,
+            event=event,
+        )
+
+    async def _record_preflight_failure(
+        self,
+        *,
+        lease: FetchLease,
+        denial: RouteReservationDenied,
+        final_url: str,
+    ) -> ScrapeJobView:
+        """Append an expected typed route/budget denial before retry policy.
+
+        Steps:
+        1. convert the finite preflight denial into a zero-physical-call fetch result;
+        2. append one immutable zero-cost attempt using the selected route;
+        3. retry only local capacity denial while terminalizing other policy denials.
+
+        Side effects if changes:
+        - pre-call attempts and budget-exhaustion retry behavior.
+        """
+
+        result = FetchResult(
+            outcome=ScrapeAttemptOutcome.DENIED,
+            route=denial.route,
+            trust=(
+                ScrapeArtifactTrust.UNTRUSTED
+                if denial.route is ScrapeTransportRoute.PUBLIC_POOL_EXPERIMENT
+                else ScrapeArtifactTrust.VERIFIED
+            ),
+            final_url=final_url,
+            elapsed_ms=0,
+            error_code=denial.code.value,
+            retry_at=denial.retry_at,
+            observed_at=utc_now(),
+        )
+        attempt = self._attempt(
+            lease=lease,
+            result=result,
+            reservation=RouteReservation(route=denial.route),
+            actual_cost=Decimal("0"),
+        )
+        await self._jobs.append_attempt(job_id=lease.job.job_id, attempt=attempt)
+        force_terminal = denial.code not in {
+            ScrapeLocalFailureCode.BUDGET_EXHAUSTED,
+            ScrapeLocalFailureCode.ROUTE_UNAVAILABLE,
+        }
+        return await self._commit_failure(
+            lease=lease,
+            result=result,
+            force_terminal=force_terminal,
+        )
+
+    async def _cache_negative(self, *, result: FetchResult, request_hash: str) -> None:
+        """Cache stable negative HTTP responses briefly without retaining bodies.
+
+        Side effects if changes:
+        - repeated missing/password/denied request traffic.
+        """
+
+        if result.status_code not in {401, 403, 404, 410}:
+            return
+        await self._cache.upsert(
+            entry=ScrapeCacheEntryMongoObject(
+                request_hash=request_hash,
+                artifact_ids=[],
+                response_status=result.status_code,
+                content_type=result.content_type or "application/octet-stream",
+                expires_at=utc_now() + timedelta(hours=1),
+                updated_at=utc_now(),
+            )
+        )
+
+    @staticmethod
+    def _cache_expiry(
+        *,
+        artifacts: tuple[ScrapeArtifactRef, ...],
+        retention_class: ScrapeRetentionClass,
+        now: datetime,
+    ) -> datetime:
+        """Bound a cache pointer by both cache policy and artifact availability.
+
+        Steps:
+        1. derive the retention-class horizon for metadata-only captures;
+        2. cap reusable pointers at seven days;
+        3. select the earliest finite artifact expiry when artifacts exist.
+
+        Side effects if changes:
+        - cache reuse must never outlive the shortest referenced artifact;
+        - transient 24-hour captures cannot create six-day stale-pointer loops.
+        """
+
+        retention_expiry = {
+            ScrapeRetentionClass.TRANSIENT_24H: now + timedelta(hours=24),
+            ScrapeRetentionClass.CACHE_7D: now + timedelta(days=7),
+            ScrapeRetentionClass.REFERENCED_ARCHIVE: now + timedelta(days=7),
+        }[retention_class]
+        artifact_expiries = tuple(
+            artifact.expires_at
+            for artifact in artifacts
+            if artifact.expires_at is not None
+        )
+        return min((retention_expiry, *artifact_expiries))
+
+    def _attempt(
+        self,
+        *,
+        lease: FetchLease,
+        result: FetchResult,
+        reservation: RouteReservation,
+        actual_cost: Decimal,
+    ) -> ScrapeAttemptView:
+        """Convert one fetch result into immutable append-only attempt evidence.
+
+        Side effects if changes:
+        - job history, cost totals, and operational metrics.
+        """
+
+        attempt_id = uuid5(
+            _ATTEMPT_NAMESPACE,
+            f"{lease.job.job_id}:{lease.attempt_number}",
+        )
+        return ScrapeAttemptView(
+            attempt_id=attempt_id,
+            attempt_number=lease.attempt_number,
+            route=result.route,
+            outcome=result.outcome,
+            trust=result.trust,
+            http_status=result.status_code,
+            bytes_received=len(result.body),
+            elapsed_ms=result.elapsed_ms,
+            failure=(None if result.succeeded else build_fetch_failure(result=result)),
+            cost=ScrapeCost(
+                estimated_amount=reservation.reserved_cost_usd,
+                actual_amount=actual_cost,
+                bytes_transferred=len(result.body),
+                browser_seconds=result.browser_seconds,
+            ),
+            observed_at=result.observed_at,
+        )
+
+    @staticmethod
+    def _conditional_headers(
+        *,
+        entry: ScrapeCacheEntryMongoObject | None,
+    ) -> dict[str, str]:
+        """Build only server-owned ETag/Last-Modified validators.
+
+        Side effects if changes:
+        - conditional request bytes and 304 cache reuse.
+        """
+
+        if entry is None:
+            return {}
+        return {
+            key: value
+            for key, value in {
+                "If-None-Match": entry.etag,
+                "If-Modified-Since": entry.last_modified,
+            }.items()
+            if value is not None
+        }
+
+
+class MaintenanceService:
+    """Run bounded cleanup, recovery, and due-job redispatch loops.
+
+    Steps:
+    1. recover expired job and callback leases under immutable budgets;
+    2. clean bounded cache/pin/orphan state in ownership order;
+    3. project and publish due jobs with bounded concurrency.
+
+    Side effects if changes:
+    - cache/pin/object retention and crashed-worker recovery.
+    """
+
+    def __init__(
+        self,
+        *,
+        jobs: JobRepositoryProtocol,
+        outbox: OutboxRepositoryProtocol,
+        cache: CacheRepositoryProtocol,
+        artifacts: ArtifactService,
+        history: HistoryRetentionRepositoryProtocol,
+        dispatcher: JobDispatcherProtocol,
+        dispatch_concurrency: int,
+        history_retention_batch_size: int,
+    ) -> None:
+        """Bind maintenance repositories and bounded queue publication policy.
+
+        Steps:
+        1. retain separate job and callback recovery protocols;
+        2. retain cache/artifact and dependency-ordered history cleanup owners;
+        3. validate positive dispatch and history batch widths.
+
+        Side effects if changes:
+        - every beat-triggered maintenance run.
+        """
+
+        if dispatch_concurrency < 1:
+            raise ValueError("dispatch_concurrency must be positive")
+        if history_retention_batch_size < 1:
+            raise ValueError("history retention batch size must be positive")
+        self._jobs = jobs
+        self._outbox = outbox
+        self._cache = cache
+        self._artifacts = artifacts
+        self._history = history
+        self._dispatcher = dispatcher
+        self._dispatch_concurrency = dispatch_concurrency
+        self._history_retention_batch_size = history_retention_batch_size
+
+    async def run(self) -> dict[str, int]:
+        """Recover leases, evict cache/pins, delete orphans, and redispatch due jobs.
+
+        Steps:
+        1. recover/terminalize expired job and callback leases before new work;
+        2. clean expired cache, pins, orphan objects, and bounded history in owner order;
+        3. terminalize exhausted due jobs, then publish one bounded due projection batch.
+
+        Side effects if changes:
+        - Mongo/OSS deletions and Celery task publication.
+        """
+
+        now = utc_now()
+        job_recovery = await self._jobs.recover_expired_leases(now=now)
+        callback_recovery = await self._outbox.recover_expired_callback_leases(
+            now=now,
+        )
+        cache_deleted = await self._cache.delete_expired(now=now)
+        pins_released = await self._artifacts.release_expired_pins()
+        orphans_deleted = await self._artifacts.cleanup_orphans()
+        history = await self._history.purge_expired_history(
+            now=now,
+            limit=self._history_retention_batch_size,
+        )
+        exhausted_due = await self._jobs.terminalize_exhausted_due_jobs(
+            now=now,
+        )
+        due = await self._jobs.due_jobs(now=now, limit=1_000)
+        await self._dispatch_due(due=due)
+        return {
+            "recoveredJobLeases": job_recovery.recovered_count,
+            "terminalizedJobLeases": job_recovery.terminalized_count,
+            "recoveredCallbackLeases": callback_recovery.recovered_count,
+            "terminalizedCallbackLeases": callback_recovery.terminalized_count,
+            "terminalizedDueJobs": exhausted_due,
+            "expiredCacheEntries": cache_deleted,
+            "releasedPins": pins_released,
+            "deletedOrphans": orphans_deleted,
+            "historyRetentionLeaseAcquired": int(history.lease_acquired),
+            "expiredJobs": history.jobs_deleted,
+            "expiredAttempts": history.attempts_deleted,
+            "expiredDeliveredCallbacks": history.delivered_outbox_deleted,
+            "expiredAuditEvents": history.audit_events_deleted,
+            "redispatchedJobs": len(due),
+        }
+
+    async def _dispatch_due(
+        self,
+        *,
+        due: tuple[DispatchableJob, ...],
+    ) -> None:
+        """Publish due jobs in bounded fail-fast task groups.
+
+        Steps:
+        1. partition the minimal due projection by configured concurrency;
+        2. publish one partition through fail-fast structured concurrency;
+        3. start the next partition only after the current one succeeds.
+
+        Side effects if changes:
+        - Celery broker load and maintenance failure visibility.
+        """
+
+        for offset in range(0, len(due), self._dispatch_concurrency):
+            batch = due[offset : offset + self._dispatch_concurrency]
+            async with asyncio.TaskGroup() as group:
+                for job in batch:
+                    group.create_task(
+                        self._dispatcher.dispatch(
+                            job_id=job.job_id,
+                            transport_policy=job.transport_policy,
+                        )
+                    )