keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/application/failures.py

@@ -0,0 +1,427 @@
+"""Canonical conversion from scraper runtime evidence to wire-safe failures."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from keble_helpers import (
+    UpstreamFailureDisposition,
+    UpstreamFailureKind,
+    UpstreamOperatorAction,
+)
+from keble_scraper_contract import (
+    ScrapeAttemptOutcome,
+    ScrapeFailure,
+    ScrapeLocalFailure,
+    ScrapeLocalFailureCode,
+    ScrapeLocalFailureOrigin,
+    ScrapeTransportRoute,
+    ScrapeUpstreamFailure,
+    ScrapeUpstreamOrigin,
+)
+
+from keble_scraper_api.application.fetching import FetchResult
+from keble_scraper_api.application.scheduling import ceil_persistence_millisecond
+from keble_scraper_api.errors import ScraperDomainError
+
+
+_LOCAL_RECOVERY = {
+    ScrapeLocalFailureCode.AUTH_INVALID: (
+        UpstreamFailureDisposition.OPERATOR_ACTION,
+        UpstreamOperatorAction.ROTATE_CREDENTIAL,
+    ),
+    ScrapeLocalFailureCode.AUTH_FORBIDDEN: (
+        UpstreamFailureDisposition.OPERATOR_ACTION,
+        UpstreamOperatorAction.REQUEST_PERMISSION,
+    ),
+    ScrapeLocalFailureCode.NOT_FOUND: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.NONE,
+    ),
+    ScrapeLocalFailureCode.REVISION_CONFLICT: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.INVALID_TRANSITION: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.VALIDATION_ERROR: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.URL_DENIED: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.DNS_FAILED: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.REDIRECT_LIMIT: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.TIMEOUT: (
+        UpstreamFailureDisposition.RETRYABLE,
+        UpstreamOperatorAction.WAIT,
+    ),
+    ScrapeLocalFailureCode.NETWORK_ERROR: (
+        UpstreamFailureDisposition.RETRYABLE,
+        UpstreamOperatorAction.WAIT,
+    ),
+    ScrapeLocalFailureCode.RESPONSE_TOO_LARGE: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.CONTENT_TYPE_REJECTED: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.BUDGET_EXHAUSTED: (
+        UpstreamFailureDisposition.RETRYABLE,
+        UpstreamOperatorAction.WAIT,
+    ),
+    ScrapeLocalFailureCode.ROUTE_UNAVAILABLE: (
+        UpstreamFailureDisposition.RETRYABLE,
+        UpstreamOperatorAction.WAIT,
+    ),
+    ScrapeLocalFailureCode.REPLAY_REJECTED: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.FIX_REQUEST,
+    ),
+    ScrapeLocalFailureCode.CONFIGURATION_INVALID: (
+        UpstreamFailureDisposition.OPERATOR_ACTION,
+        UpstreamOperatorAction.CONTACT_PROVIDER,
+    ),
+    ScrapeLocalFailureCode.CACHE_ARTIFACT_MISSING: (
+        UpstreamFailureDisposition.RETRYABLE,
+        UpstreamOperatorAction.WAIT,
+    ),
+    ScrapeLocalFailureCode.CALLBACK_DELIVERY_FAILED: (
+        UpstreamFailureDisposition.RETRYABLE,
+        UpstreamOperatorAction.WAIT,
+    ),
+    ScrapeLocalFailureCode.ATTEMPT_BUDGET_EXHAUSTED: (
+        UpstreamFailureDisposition.OPERATOR_ACTION,
+        UpstreamOperatorAction.CONTACT_PROVIDER,
+    ),
+    ScrapeLocalFailureCode.JOB_DEADLINE_EXCEEDED: (
+        UpstreamFailureDisposition.OPERATOR_ACTION,
+        UpstreamOperatorAction.CONTACT_PROVIDER,
+    ),
+    ScrapeLocalFailureCode.OPERATOR_CANCELLED: (
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.NONE,
+    ),
+    ScrapeLocalFailureCode.UNKNOWN_RUNTIME: (
+        UpstreamFailureDisposition.OPERATOR_ACTION,
+        UpstreamOperatorAction.CONTACT_PROVIDER,
+    ),
+}
+
+_LOCAL_ORIGIN = {
+    ScrapeLocalFailureCode.AUTH_INVALID: ScrapeLocalFailureOrigin.CALLER_AUTH,
+    ScrapeLocalFailureCode.AUTH_FORBIDDEN: ScrapeLocalFailureOrigin.CALLER_AUTH,
+    ScrapeLocalFailureCode.VALIDATION_ERROR: ScrapeLocalFailureOrigin.REQUEST_POLICY,
+    ScrapeLocalFailureCode.URL_DENIED: ScrapeLocalFailureOrigin.REQUEST_POLICY,
+    ScrapeLocalFailureCode.DNS_FAILED: ScrapeLocalFailureOrigin.REQUEST_POLICY,
+    ScrapeLocalFailureCode.REDIRECT_LIMIT: ScrapeLocalFailureOrigin.REQUEST_POLICY,
+    ScrapeLocalFailureCode.RESPONSE_TOO_LARGE: ScrapeLocalFailureOrigin.REQUEST_POLICY,
+    ScrapeLocalFailureCode.CONTENT_TYPE_REJECTED: ScrapeLocalFailureOrigin.REQUEST_POLICY,
+    ScrapeLocalFailureCode.OPERATOR_CANCELLED: ScrapeLocalFailureOrigin.OPERATOR,
+}
+
+_FETCH_LOCAL_CODES = {
+    "URL_DENIED": ScrapeLocalFailureCode.URL_DENIED,
+    "DNS_FAILED": ScrapeLocalFailureCode.DNS_FAILED,
+    "REDIRECT_LIMIT": ScrapeLocalFailureCode.REDIRECT_LIMIT,
+    "RESPONSE_TOO_LARGE": ScrapeLocalFailureCode.RESPONSE_TOO_LARGE,
+    "CONTENT_TYPE_REJECTED": ScrapeLocalFailureCode.CONTENT_TYPE_REJECTED,
+    "ROUTE_UNAVAILABLE": ScrapeLocalFailureCode.ROUTE_UNAVAILABLE,
+    "BUDGET_EXHAUSTED": ScrapeLocalFailureCode.BUDGET_EXHAUSTED,
+    "CACHE_VALIDATOR_WITHOUT_ENTRY": ScrapeLocalFailureCode.CACHE_ARTIFACT_MISSING,
+}
+
+
+def build_local_failure(
+    *,
+    code: ScrapeLocalFailureCode,
+    safe_message: str,
+    observed_at: datetime,
+    physical_attempt_count: int,
+    retry_at: datetime | None = None,
+) -> ScrapeLocalFailure:
+    """Build one local failure from the exhaustive recovery policy tables.
+
+    Steps:
+    1. select the code's finite disposition and operator action;
+    2. select its local origin or default to scraper runtime;
+    3. retain a future horizon only for retryable local capacity;
+    4. construct the sanitized versioned contract payload.
+
+    Side effects if changes:
+    - API errors, job attempts, cancellation, and callbacks share this mapping;
+    - adding a contract code requires an explicit recovery entry above.
+    """
+
+    disposition, action = _LOCAL_RECOVERY[code]
+    origin = _LOCAL_ORIGIN.get(code, ScrapeLocalFailureOrigin.SCRAPER_RUNTIME)
+    durable_retry_at = (
+        ceil_persistence_millisecond(value=retry_at)
+        if retry_at is not None and disposition is UpstreamFailureDisposition.RETRYABLE
+        else None
+    )
+    return ScrapeLocalFailure(
+        origin=origin,
+        code=code,
+        disposition=disposition,
+        operator_action=action,
+        safe_message=safe_message,
+        observed_at=observed_at,
+        retry_at=durable_retry_at,
+        mapping_version="scraper-local-v1",
+        physical_attempt_count=physical_attempt_count,
+    )
+
+
+def build_domain_failure(
+    *,
+    error: ScraperDomainError,
+    observed_at: datetime,
+) -> ScrapeLocalFailure:
+    """Convert one API-local domain exception into the public failure type.
+
+    Steps:
+    1. retain the already finite contract-owned local code;
+    2. retain only the domain error's sanitized message;
+    3. mark that no target/provider request occurred.
+
+    Side effects if changes:
+    - FastAPI error envelopes and preflight attempts use the same conversion;
+    - raw exception types and tracebacks never cross the contract boundary.
+    """
+
+    return build_local_failure(
+        code=error.code,
+        safe_message=error.message,
+        observed_at=observed_at,
+        physical_attempt_count=0,
+    )
+
+
+def _upstream_origin(*, route: ScrapeTransportRoute) -> ScrapeUpstreamOrigin:
+    """Map physical browser calls separately from target HTTP responses.
+
+    Steps:
+    1. classify Cloudflare browser rendering as a paid browser provider;
+    2. classify ordinary direct/proxy HTTP status as target-resource evidence;
+    3. avoid treating a target response through Bright as Bright billing state.
+
+    Side effects if changes:
+    - payment evidence and supplier cost attribution depend on this distinction.
+    """
+
+    if route is ScrapeTransportRoute.CLOUDFLARE_BROWSER_RUN:
+        return ScrapeUpstreamOrigin.BROWSER_PROVIDER
+    return ScrapeUpstreamOrigin.TARGET_RESOURCE
+
+
+def _http_classification(
+    *,
+    status_code: int,
+) -> tuple[
+    UpstreamFailureKind,
+    UpstreamFailureDisposition,
+    UpstreamOperatorAction,
+    str,
+]:
+    """Classify one target/provider HTTP status without inferring payment.
+
+    Steps:
+    1. distinguish auth, permission, not-found, rate, timeout, and availability;
+    2. keep ambiguous 402 as invalid request rather than supplier payment;
+    3. return a bounded safe message with the finite recovery decision.
+
+    Side effects if changes:
+    - job retry scheduling and Shopify recovery behavior use this classification;
+    - supplier payment still requires a separate documented native-code mapper.
+    """
+
+    if status_code == 401:
+        return (
+            UpstreamFailureKind.AUTHENTICATION_REQUIRED,
+            UpstreamFailureDisposition.PERMANENT,
+            UpstreamOperatorAction.FIX_REQUEST,
+            "Target resource requires authentication not present in this request profile.",
+        )
+    if status_code == 403:
+        return (
+            UpstreamFailureKind.TEMPORARILY_UNAVAILABLE,
+            UpstreamFailureDisposition.RETRYABLE,
+            UpstreamOperatorAction.WAIT,
+            "Target or browser provider temporarily refused the request.",
+        )
+    if status_code in {404, 410}:
+        return (
+            UpstreamFailureKind.NOT_FOUND,
+            UpstreamFailureDisposition.PERMANENT,
+            UpstreamOperatorAction.NONE,
+            "Target resource is not available.",
+        )
+    if status_code == 408:
+        return (
+            UpstreamFailureKind.TIMEOUT,
+            UpstreamFailureDisposition.RETRYABLE,
+            UpstreamOperatorAction.WAIT,
+            "Target request timed out.",
+        )
+    if status_code in {425, 429}:
+        return (
+            UpstreamFailureKind.RATE_LIMITED,
+            UpstreamFailureDisposition.RETRYABLE,
+            UpstreamOperatorAction.WAIT,
+            "Target or browser provider requested a bounded delay.",
+        )
+    if status_code >= 500:
+        return (
+            UpstreamFailureKind.TEMPORARILY_UNAVAILABLE,
+            UpstreamFailureDisposition.RETRYABLE,
+            UpstreamOperatorAction.WAIT,
+            "Target or browser provider is temporarily unavailable.",
+        )
+    if 400 <= status_code < 500:
+        return (
+            UpstreamFailureKind.INVALID_REQUEST,
+            UpstreamFailureDisposition.PERMANENT,
+            UpstreamOperatorAction.FIX_REQUEST,
+            "Target or browser provider rejected the request.",
+        )
+    return (
+        UpstreamFailureKind.INVALID_RESPONSE,
+        UpstreamFailureDisposition.PERMANENT,
+        UpstreamOperatorAction.CONTACT_PROVIDER,
+        "Target or browser provider returned an invalid response status.",
+    )
+
+
+def build_fetch_failure(*, result: FetchResult) -> ScrapeFailure:
+    """Convert one non-success fetch result into local or upstream evidence.
+
+    Steps:
+    1. map known policy/capacity codes to finite local failures;
+    2. map HTTP, timeout, and network outcomes to upstream target/provider truth;
+    3. map any remaining runtime outcome to a finite unknown local failure.
+
+    Side effects if changes:
+    - append-only attempts, retry decisions, and terminal snapshots use it;
+    - arbitrary fetcher error strings cannot escape onto the public wire.
+    """
+
+    local_code = _FETCH_LOCAL_CODES.get(result.error_code or "")
+    if local_code is not None:
+        return build_local_failure(
+            code=local_code,
+            safe_message="Scraper request could not proceed under local policy.",
+            observed_at=result.observed_at,
+            physical_attempt_count=(
+                1
+                if local_code
+                in {
+                    ScrapeLocalFailureCode.RESPONSE_TOO_LARGE,
+                    ScrapeLocalFailureCode.CONTENT_TYPE_REJECTED,
+                }
+                else 0
+            ),
+            retry_at=result.retry_at,
+        )
+
+    if result.status_code is not None and result.status_code >= 400:
+        kind, disposition, action, message = _http_classification(
+            status_code=result.status_code
+        )
+        return ScrapeUpstreamFailure(
+            origin=_upstream_origin(route=result.route),
+            kind=kind,
+            disposition=disposition,
+            operator_action=action,
+            safe_message=message,
+            observed_at=result.observed_at,
+            mapping_version="scraper-http-v1",
+            physical_attempt_count=1,
+            retry_at=(
+                ceil_persistence_millisecond(value=result.retry_at)
+                if result.retry_at is not None
+                and disposition is UpstreamFailureDisposition.RETRYABLE
+                else None
+            ),
+            upstream_http_status=result.status_code,
+            upstream_code=result.upstream_code or f"HTTP_{result.status_code}",
+        )
+
+    if result.outcome is ScrapeAttemptOutcome.TIMEOUT:
+        kind = UpstreamFailureKind.TIMEOUT
+        message = "Target request timed out."
+    elif result.outcome is ScrapeAttemptOutcome.NETWORK_ERROR:
+        kind = UpstreamFailureKind.TEMPORARILY_UNAVAILABLE
+        message = "Target resource is temporarily unreachable."
+    else:
+        return build_local_failure(
+            code=ScrapeLocalFailureCode.UNKNOWN_RUNTIME,
+            safe_message="Scraper attempt failed without classified upstream evidence.",
+            observed_at=result.observed_at,
+            physical_attempt_count=0,
+        )
+    return ScrapeUpstreamFailure(
+        origin=_upstream_origin(route=result.route),
+        kind=kind,
+        disposition=UpstreamFailureDisposition.RETRYABLE,
+        operator_action=UpstreamOperatorAction.WAIT,
+        safe_message=message,
+        observed_at=result.observed_at,
+        mapping_version="scraper-transport-v1",
+        physical_attempt_count=1,
+        retry_at=(
+            ceil_persistence_millisecond(value=result.retry_at)
+            if result.retry_at is not None
+            else None
+        ),
+    )
+
+
+def terminalize_failure(*, failure: ScrapeFailure) -> ScrapeFailure:
+    """Convert a retryable cause into terminal attempt-budget evidence.
+
+    Steps:
+    1. preserve already terminal/operator failures unchanged;
+    2. retain upstream origin/kind/evidence while clearing retry timing;
+    3. require operator inspection after the bounded attempt budget is spent.
+
+    Side effects if changes:
+    - dead-letter job/callback snapshots remain honest about their final cause;
+    - immutable attempt rows keep the original retryable classification.
+    """
+
+    if failure.disposition is not UpstreamFailureDisposition.RETRYABLE:
+        return failure
+    if isinstance(failure, ScrapeLocalFailure):
+        return build_local_failure(
+            code=ScrapeLocalFailureCode.ATTEMPT_BUDGET_EXHAUSTED,
+            safe_message="Automatic scraper attempt budget is exhausted.",
+            observed_at=failure.observed_at,
+            physical_attempt_count=failure.physical_attempt_count,
+        )
+    return ScrapeUpstreamFailure(
+        origin=failure.origin,
+        kind=failure.kind,
+        disposition=UpstreamFailureDisposition.OPERATOR_ACTION,
+        operator_action=UpstreamOperatorAction.CONTACT_PROVIDER,
+        safe_message="Automatic scraper attempt budget is exhausted after upstream failure.",
+        observed_at=failure.observed_at,
+        mapping_version=failure.mapping_version,
+        physical_attempt_count=failure.physical_attempt_count,
+        capacity=failure.capacity,
+        upstream_http_status=failure.upstream_http_status,
+        upstream_code=failure.upstream_code,
+        upstream_request_id=failure.upstream_request_id,
+    )

keble_scraper_api/application/fetching.py

@@ -0,0 +1,72 @@
+"""Application-owned prepared request and fetch-result types."""
+
+from __future__ import annotations
+
+from decimal import Decimal
+
+from keble_helpers.typings import PydanticModelConfig
+from pydantic import AwareDatetime, BaseModel, Field
+
+from keble_scraper_contract import (
+    ScrapeArtifactTrust,
+    ScrapeAttemptOutcome,
+    ScrapeTransportRoute,
+)
+
+
+class PreparedFetch(BaseModel):
+    """Server-assembled outbound request with no caller credential surface.
+
+    Side effects if changes:
+    - URL policy, HTTP fetching, cache identity, and Shopify GraphQL safety.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="forbid")
+
+    url: str
+    method: str
+    headers: dict[str, str] = Field(default_factory=dict)
+    body: bytes | None = None
+    request_identity: str
+
+
+class FetchResult(BaseModel):
+    """Typed expected transport outcome with unexpected faults left to supervision.
+
+    Steps:
+    1. retain the physical route/status/body and measured duration/cost units;
+    2. retain only bounded local/native codes plus a parsed future retry horizon;
+    3. convert expected evidence into the public discriminated failure contract;
+    4. let unexpected client/runtime faults propagate without becoming result data.
+
+    Side effects if changes:
+    - retry scheduling, immutable attempts, cache writes, and artifact creation.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="forbid")
+
+    outcome: ScrapeAttemptOutcome
+    route: ScrapeTransportRoute
+    trust: ScrapeArtifactTrust
+    final_url: str
+    status_code: int | None = None
+    content_type: str | None = None
+    body: bytes = b""
+    etag: str | None = None
+    last_modified: str | None = None
+    elapsed_ms: int = Field(ge=0)
+    browser_seconds: Decimal = Field(default=Decimal("0"), ge=0)
+    error_code: str | None = None
+    upstream_code: str | None = Field(default=None, max_length=128)
+    retry_at: AwareDatetime | None = None
+    observed_at: AwareDatetime
+
+    @property
+    def succeeded(self) -> bool:
+        """Return whether content/metadata may proceed to artifact handling.
+
+        Side effects if changes:
+        - worker success, cache update, and retry behavior.
+        """
+
+        return self.outcome is ScrapeAttemptOutcome.SUCCEEDED