keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/infrastructure/proxy_router.py

@@ -0,0 +1,460 @@
+"""Redis-backed route rate/spend reservation and Bright session routing."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+from collections.abc import Awaitable
+from datetime import UTC, datetime, timedelta
+from decimal import ROUND_CEILING, Decimal
+from typing import Protocol, cast
+from urllib.parse import quote
+
+from pydantic import SecretStr
+from redis.asyncio import Redis
+
+from keble_scraper_contract import (
+    RouteBudgetView,
+    ScrapeTransportPolicy,
+    ScrapeTransportRoute,
+)
+
+from keble_scraper_api.errors import ScraperErrorCode
+from keble_scraper_api.models import (
+    RouteReservation,
+    RouteReservationDenied,
+    RouteReservationGranted,
+    RouteReservationOutcome,
+)
+from keble_scraper_api.protocols import ProxyRepositoryProtocol
+from keble_scraper_api.settings import ScraperSettings
+
+_RESERVE_SCRIPT = """
+local used = tonumber(redis.call('GET', KEYS[2]) or '0')
+local requested = tonumber(ARGV[3])
+local budget = tonumber(ARGV[4])
+if used + requested > budget then
+  return {0, 2, 0}
+end
+local now_ms = tonumber(ARGV[1])
+local interval_ms = tonumber(ARGV[2])
+local next_at_ms = tonumber(redis.call('GET', KEYS[1]) or now_ms)
+local admitted_at_ms = math.max(now_ms, next_at_ms)
+local wait_ms = admitted_at_ms - now_ms
+redis.call('SET', KEYS[1], admitted_at_ms + interval_ms)
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[5]))
+redis.call('INCRBY', KEYS[2], requested)
+redis.call('EXPIRE', KEYS[2], tonumber(ARGV[5]))
+return {1, used + requested, wait_ms}
+"""
+
+
+class ProxyRouteSleeper(Protocol):
+    """Injectable worker suspension for a reserved future route slot.
+
+    Side effects if changes:
+    - cross-worker request pacing and deterministic router tests.
+    """
+
+    async def sleep(self, *, seconds: float) -> None:
+        """Suspend the current worker until its atomic route slot is due.
+
+        Side effects if changes:
+        - provider request start times and worker lease duration.
+        """
+
+        ...
+
+
+class AsyncioProxyRouteSleeper:
+    """Production route pacing backed by cancellable asyncio suspension.
+
+    Side effects if changes:
+    - live direct, Bright, browser, and public-proxy request timing.
+    """
+
+    async def sleep(self, *, seconds: float) -> None:
+        """Wait without blocking another coroutine or worker process.
+
+        Side effects if changes:
+        - route throughput and Celery worker utilization.
+        """
+
+        await asyncio.sleep(seconds)
+
+
+class RedisProxyRouter:
+    """Choose egress and atomically reserve rate plus daily vendor spend.
+
+    Steps:
+    1. select a finite route from policy and attempt ordinal;
+    2. derive credentials/affinity without exposing secrets;
+    3. atomically reserve pacing and daily spend or return typed denial.
+
+    Side effects if changes:
+    - direct/Bright/browser/public route selection and vendor budget stops.
+    """
+
+    def __init__(
+        self,
+        *,
+        redis: Redis,
+        settings: ScraperSettings,
+        proxy_repository: ProxyRepositoryProtocol,
+        sleeper: ProxyRouteSleeper | None = None,
+    ) -> None:
+        """Bind the shared Redis state and quarantined proxy repository.
+
+        Side effects if changes:
+        - all workers share the same rate, spend, and session policy.
+        """
+
+        self._redis = redis
+        self._settings = settings
+        self._proxy_repository = proxy_repository
+        self._sleeper = sleeper or AsyncioProxyRouteSleeper()
+
+    async def reserve(
+        self,
+        *,
+        policy: ScrapeTransportPolicy,
+        affinity_key: str,
+        estimated_bytes: int,
+        attempt_number: int,
+    ) -> RouteReservationOutcome:
+        """Reserve one route or return a finite expected preflight denial.
+
+        Steps:
+        1. select a finite route from policy and attempt ordinal;
+        2. derive Bright affinity locally while Bright rotates the real IP;
+        3. atomically enforce second-level rate and UTC-day spend ceilings.
+
+        Side effects if changes:
+        - worker dispatch may spend vendor money and consume rate capacity.
+        """
+
+        route = self._select_route(policy=policy, attempt_number=attempt_number)
+        proxy_url: SecretStr | None = None
+        session_id: str | None = None
+        if route is ScrapeTransportRoute.BRIGHT_RESIDENTIAL:
+            bright = self._bright_proxy(affinity_key=affinity_key)
+            if bright is None:
+                return RouteReservationDenied(
+                    route=route,
+                    code=ScraperErrorCode.CONFIGURATION_INVALID,
+                    safe_message="Bright route is not configured.",
+                )
+            proxy_url, session_id = bright
+        if route is ScrapeTransportRoute.PUBLIC_POOL_EXPERIMENT:
+            if not self._settings.public_proxy_experiment_enabled:
+                return RouteReservationDenied(
+                    route=route,
+                    code=ScraperErrorCode.CONFIGURATION_INVALID,
+                    safe_message="Public proxy experiment is disabled.",
+                )
+            proxy_url = await self._proxy_repository.choose_healthy()
+            if proxy_url is None:
+                return RouteReservationDenied(
+                    route=route,
+                    code=ScraperErrorCode.ROUTE_UNAVAILABLE,
+                    safe_message="No healthy quarantined proxy is available.",
+                    retry_at=datetime.now(UTC)
+                    + timedelta(
+                        seconds=self._settings.upstream_rate_limit_default_seconds
+                    ),
+                )
+        estimated_cost = self._cost(
+            route=route,
+            byte_count=estimated_bytes,
+            browser_seconds=30.0
+            if route is ScrapeTransportRoute.CLOUDFLARE_BROWSER_RUN
+            else 0.0,
+        )
+        budget_denial = await self._reserve_budget(
+            route=route,
+            amount=estimated_cost,
+        )
+        if budget_denial is not None:
+            return budget_denial
+        return RouteReservationGranted(
+            reservation=RouteReservation(
+                route=route,
+                proxy_url=proxy_url,
+                session_id=session_id,
+                reserved_cost_usd=estimated_cost,
+            )
+        )
+
+    async def reconcile(
+        self,
+        *,
+        reservation: RouteReservation,
+        actual_bytes: int,
+        browser_seconds: float,
+    ) -> Decimal:
+        """Replace estimated spend with measured route units after an attempt.
+
+        Side effects if changes:
+        - remaining daily Bright/Cloudflare budget and benchmark cost truth.
+        """
+
+        actual = self._cost(
+            route=reservation.route,
+            byte_count=actual_bytes,
+            browser_seconds=browser_seconds,
+        )
+        delta = actual - reservation.reserved_cost_usd
+        if delta != 0:
+            key = self._budget_key(route=reservation.route)
+            await self._redis.incrby(key, self._micro_usd(delta))
+        return actual
+
+    async def budgets(self) -> tuple[RouteBudgetView, ...]:
+        """Return credential-free current UTC-day budget views.
+
+        Side effects if changes:
+        - operations cost dashboard and capacity approvals.
+        """
+
+        views: list[RouteBudgetView] = []
+        for route in ScrapeTransportRoute:
+            raw = await self._redis.get(self._budget_key(route=route))
+            consumed = Decimal(str(raw or 0)) / Decimal("1000000")
+            limit, rate, enabled = self._policy(route=route)
+            views.append(
+                RouteBudgetView(
+                    route=route,
+                    daily_limit_usd=limit,
+                    consumed_today_usd=max(Decimal("0"), consumed),
+                    request_rate_per_second=rate,
+                    enabled=enabled,
+                )
+            )
+        return tuple(views)
+
+    def _select_route(
+        self,
+        *,
+        policy: ScrapeTransportPolicy,
+        attempt_number: int,
+    ) -> ScrapeTransportRoute:
+        """Map caller policy plus retry ordinal to one actual route.
+
+        Side effects if changes:
+        - direct-to-Bright fallback and exception-renderer activation.
+        """
+
+        if policy is ScrapeTransportPolicy.DIRECT_ONLY:
+            return ScrapeTransportRoute.DIRECT
+        if policy is ScrapeTransportPolicy.DIRECT_THEN_BRIGHT:
+            return (
+                ScrapeTransportRoute.DIRECT
+                if attempt_number == 1
+                else ScrapeTransportRoute.BRIGHT_RESIDENTIAL
+            )
+        if policy is ScrapeTransportPolicy.BRIGHT_ONLY:
+            return ScrapeTransportRoute.BRIGHT_RESIDENTIAL
+        if policy is ScrapeTransportPolicy.BROWSER_EXCEPTION:
+            return ScrapeTransportRoute.CLOUDFLARE_BROWSER_RUN
+        return ScrapeTransportRoute.PUBLIC_POOL_EXPERIMENT
+
+    def _bright_proxy(self, *, affinity_key: str) -> tuple[SecretStr, str] | None:
+        """Build configured Bright credentials or return expected absence.
+
+        Steps:
+        1. return absence when any required server-owned credential is missing;
+        2. derive deterministic daily affinity without exposing the password;
+        3. return the secret-wrapped URL plus non-secret session identity.
+
+        Side effects if changes:
+        - Bright session stickiness and actual residential egress billing.
+        """
+
+        host = self._settings.bright_proxy_host
+        customer = self._settings.bright_customer
+        zone = self._settings.bright_zone
+        password_secret = self._settings.bright_password
+        if host is None or customer is None or zone is None or password_secret is None:
+            return None
+        session_seed = f"{affinity_key}:{datetime.now(UTC).date().isoformat()}".encode()
+        session_id = hashlib.sha256(session_seed).hexdigest()[:16]
+        username = f"brd-customer-{customer}-zone-{zone}-session-{session_id}"
+        password = password_secret.get_secret_value()
+        proxy = (
+            f"http://{quote(username, safe='')}:{quote(password, safe='')}@"
+            f"{host}:{self._settings.bright_proxy_port}"
+        )
+        return SecretStr(proxy), session_id
+
+    async def _reserve_budget(
+        self,
+        *,
+        route: ScrapeTransportRoute,
+        amount: Decimal,
+    ) -> RouteReservationDenied | None:
+        """Reserve one future slot or return an expected route/budget denial.
+
+        Steps:
+        1. reject disabled route policy as typed local data;
+        2. atomically reserve distributed rate and daily spend state;
+        3. return budget exhaustion as typed data or await the granted slot.
+
+        The Redis script assigns each worker a globally ordered start time. A
+        worker waits for that slot instead of treating normal burst pressure as
+        a route failure or consuming a paid fallback attempt.
+
+        Side effects if changes:
+        - concurrent workers are paced or stopped before external calls.
+        """
+
+        limit, rate, enabled = self._policy(route=route)
+        if not enabled:
+            return RouteReservationDenied(
+                route=route,
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                safe_message=f"{route.value} route is disabled.",
+            )
+        result = await cast(
+            Awaitable[list[int]],
+            self._redis.eval(
+                _RESERVE_SCRIPT,
+                2,
+                self._rate_key(route=route),
+                self._budget_key(route=route),
+                str(self._now_milliseconds()),
+                str(self._rate_interval_milliseconds(rate=rate)),
+                str(self._micro_usd(amount)),
+                str(self._micro_usd(limit)),
+                "172800",
+            ),
+        )
+        if int(result[0]) == 0:
+            now = datetime.now(UTC)
+            return RouteReservationDenied(
+                route=route,
+                code=ScraperErrorCode.BUDGET_EXHAUSTED,
+                safe_message="Route daily spend limit is exhausted.",
+                retry_at=now.replace(
+                    hour=0,
+                    minute=0,
+                    second=0,
+                    microsecond=0,
+                )
+                + timedelta(days=1),
+            )
+        wait_seconds = int(result[2]) / 1_000
+        if wait_seconds > 0:
+            await self._sleeper.sleep(seconds=wait_seconds)
+        return None
+
+    def _policy(
+        self,
+        *,
+        route: ScrapeTransportRoute,
+    ) -> tuple[Decimal, Decimal, bool]:
+        """Resolve server-owned daily cost/rate/enabled policy for one route.
+
+        Side effects if changes:
+        - preflight capacity and every route reservation.
+        """
+
+        policies = {
+            ScrapeTransportRoute.DIRECT: (
+                Decimal("0"),
+                self._settings.direct_requests_per_second,
+                True,
+            ),
+            ScrapeTransportRoute.BRIGHT_RESIDENTIAL: (
+                self._settings.bright_daily_budget_usd,
+                self._settings.bright_requests_per_second,
+                self._settings.bright_proxy_host is not None,
+            ),
+            ScrapeTransportRoute.CLOUDFLARE_BROWSER_RUN: (
+                self._settings.cloudflare_daily_budget_usd,
+                self._settings.cloudflare_requests_per_second,
+                self._settings.cloudflare_account_id is not None,
+            ),
+            ScrapeTransportRoute.PUBLIC_POOL_EXPERIMENT: (
+                Decimal("0"),
+                Decimal("1"),
+                self._settings.public_proxy_experiment_enabled,
+            ),
+        }
+        return policies[route]
+
+    def _cost(
+        self,
+        *,
+        route: ScrapeTransportRoute,
+        byte_count: int,
+        browser_seconds: float,
+    ) -> Decimal:
+        """Price measured vendor units without guessing unconfigured calls.
+
+        Side effects if changes:
+        - cost ledger, budget exhaustion, and rollout extrapolation.
+        """
+
+        if route is ScrapeTransportRoute.BRIGHT_RESIDENTIAL:
+            return (
+                Decimal(byte_count)
+                / Decimal("1000000000")
+                * self._settings.bright_cost_per_gb_usd
+            ).quantize(Decimal("0.00000001"))
+        if route is ScrapeTransportRoute.CLOUDFLARE_BROWSER_RUN:
+            return (
+                Decimal(str(browser_seconds))
+                / Decimal("3600")
+                * self._settings.cloudflare_browser_cost_per_hour_usd
+            ).quantize(Decimal("0.00000001"))
+        return Decimal("0")
+
+    def _rate_key(self, *, route: ScrapeTransportRoute) -> str:
+        """Build one stable per-route next-admission timestamp key.
+
+        Side effects if changes:
+        - cross-worker rate pacing and Redis state ownership.
+        """
+
+        return f"{self._settings.redis_namespace}:route:{route.value}:rate"
+
+    def _budget_key(self, *, route: ScrapeTransportRoute) -> str:
+        """Build one per-route UTC-day spend key.
+
+        Side effects if changes:
+        - budget reconciliation and dashboard reads.
+        """
+
+        day = datetime.now(UTC).date().isoformat()
+        return f"{self._settings.redis_namespace}:route:{route.value}:budget:{day}"
+
+    @staticmethod
+    def _micro_usd(amount: Decimal) -> int:
+        """Convert exact USD decimals to atomic Redis micro-dollar integers.
+
+        Side effects if changes:
+        - rate/budget atomicity and reconciliation rounding.
+        """
+
+        return int((amount * Decimal("1000000")).to_integral_value())
+
+    @staticmethod
+    def _now_milliseconds() -> int:
+        """Return current UTC epoch milliseconds for Redis admission ordering.
+
+        Side effects if changes:
+        - route wait duration and distributed worker ordering.
+        """
+
+        return int(datetime.now(UTC).timestamp() * 1_000)
+
+    @staticmethod
+    def _rate_interval_milliseconds(*, rate: Decimal) -> int:
+        """Convert requests per second into a conservative integer interval.
+
+        Side effects if changes:
+        - actual route throughput and provider rate-limit compliance.
+        """
+
+        interval = (Decimal("1000") / rate).to_integral_value(rounding=ROUND_CEILING)
+        return max(1, int(interval))

keble_scraper_api/infrastructure/request_profiles.py

@@ -0,0 +1,220 @@
+"""Server-owned request preparation and Shopify GraphQL templates."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+
+from keble_scraper_contract import (
+    PublicHttpRequest,
+    ScrapeJobCreate,
+    ShopifyStorefrontGraphqlRequest,
+    ShopifyStorefrontOperation,
+)
+
+from keble_scraper_api.application.fetching import PreparedFetch
+
+_PRODUCT_IDENTITIES_QUERY = """
+query ProductIdentities($first: Int!, $after: String, $query: String) {
+  products(first: $first, after: $after, sortKey: UPDATED_AT, query: $query) {
+    nodes { id handle updatedAt onlineStoreUrl }
+    pageInfo { hasNextPage endCursor }
+  }
+}
+""".strip()
+
+_PRODUCT_DETAIL_QUERY = """
+query ProductDetail($id: ID!) {
+  product(id: $id) {
+    id handle title description descriptionHtml productType vendor tags updatedAt
+    onlineStoreUrl
+    featuredImage { id url altText width height }
+    variants(first: 100) {
+      nodes {
+        id title sku availableForSale
+        price { amount currencyCode }
+        compareAtPrice { amount currencyCode }
+        selectedOptions { name value }
+        image { id url altText width height }
+      }
+      pageInfo { hasNextPage endCursor }
+    }
+    media(first: 100) {
+      nodes {
+        id mediaContentType alt previewImage { id url altText width height }
+        ... on MediaImage { image { id url altText width height } }
+        ... on Video { sources { url mimeType } }
+        ... on ExternalVideo { embeddedUrl }
+        ... on Model3d { sources { url mimeType } }
+      }
+      pageInfo { hasNextPage endCursor }
+    }
+  }
+}
+""".strip()
+
+_PRODUCT_VARIANTS_QUERY = """
+query ProductVariants($id: ID!, $first: Int!, $after: String) {
+  product(id: $id) {
+    id
+    variants(first: $first, after: $after) {
+      nodes {
+        id title sku availableForSale
+        price { amount currencyCode }
+        compareAtPrice { amount currencyCode }
+        selectedOptions { name value }
+        image { id url altText width height }
+      }
+      pageInfo { hasNextPage endCursor }
+    }
+  }
+}
+""".strip()
+
+_PRODUCT_MEDIA_QUERY = """
+query ProductMedia($id: ID!, $first: Int!, $after: String) {
+  product(id: $id) {
+    id
+    media(first: $first, after: $after) {
+      nodes {
+        id mediaContentType alt previewImage { id url altText width height }
+        ... on MediaImage { image { id url altText width height } }
+        ... on Video { sources { url mimeType } }
+        ... on ExternalVideo { embeddedUrl }
+        ... on Model3d { sources { url mimeType } }
+      }
+      pageInfo { hasNextPage endCursor }
+    }
+  }
+}
+""".strip()
+
+
+class RequestProfileRegistry:
+    """Convert contract requests into credential-free outbound bytes.
+
+    Side effects if changes:
+    - cache identity and all generic/Shopify transport requests.
+    """
+
+    def prepare(self, *, command: ScrapeJobCreate) -> PreparedFetch:
+        """Dispatch one discriminated request to its reviewed builder.
+
+        Side effects if changes:
+        - arbitrary POST/header prevention and request deduplication.
+        """
+
+        request = command.request
+        if isinstance(request, PublicHttpRequest):
+            return self._prepare_public(request=request)
+        return self._prepare_shopify(request=request)
+
+    @staticmethod
+    def _prepare_public(*, request: PublicHttpRequest) -> PreparedFetch:
+        """Prepare bounded generic headers without caller auth/host values.
+
+        Side effects if changes:
+        - generic GET/HEAD wire behavior and cache identity.
+        """
+
+        headers = {
+            key: value
+            for key, value in {
+                "Accept": request.accept,
+                "Accept-Language": request.accept_language,
+            }.items()
+            if value is not None
+        }
+        identity = RequestProfileRegistry._identity(
+            method=request.method,
+            url=str(request.url),
+            headers=headers,
+            body=None,
+        )
+        return PreparedFetch(
+            url=str(request.url),
+            method=request.method,
+            headers=headers,
+            request_identity=identity,
+        )
+
+    @staticmethod
+    def _prepare_shopify(*, request: ShopifyStorefrontGraphqlRequest) -> PreparedFetch:
+        """Build one allowlisted Storefront POST query and typed variables.
+
+        Side effects if changes:
+        - Shopify query complexity, pagination, and API-version compatibility.
+        """
+
+        query_by_operation = {
+            ShopifyStorefrontOperation.PRODUCT_IDENTITIES: _PRODUCT_IDENTITIES_QUERY,
+            ShopifyStorefrontOperation.PRODUCT_DETAIL: _PRODUCT_DETAIL_QUERY,
+            ShopifyStorefrontOperation.PRODUCT_VARIANTS: _PRODUCT_VARIANTS_QUERY,
+            ShopifyStorefrontOperation.PRODUCT_MEDIA: _PRODUCT_MEDIA_QUERY,
+        }
+        query = query_by_operation[request.operation]
+        variables: dict[str, object]
+        if request.operation is ShopifyStorefrontOperation.PRODUCT_IDENTITIES:
+            variables = {
+                "first": request.first,
+                "after": request.after,
+                "query": (
+                    f"updated_at:>{request.updated_after.isoformat()}"
+                    if request.updated_after is not None
+                    else None
+                ),
+            }
+        elif request.operation is ShopifyStorefrontOperation.PRODUCT_DETAIL:
+            variables = {"id": request.product_id}
+        else:
+            variables = {
+                "id": request.product_id,
+                "first": request.first,
+                "after": request.after,
+            }
+        body = json.dumps(
+            {"query": query, "variables": variables},
+            sort_keys=True,
+            separators=(",", ":"),
+        ).encode()
+        url = f"https://{request.store_domain}/api/2026-07/graphql.json"
+        headers = {"Accept": "application/json", "Content-Type": "application/json"}
+        identity = RequestProfileRegistry._identity(
+            method="POST",
+            url=url,
+            headers=headers,
+            body=body,
+        )
+        return PreparedFetch(
+            url=url,
+            method="POST",
+            headers=headers,
+            body=body,
+            request_identity=identity,
+        )
+
+    @staticmethod
+    def _identity(
+        *,
+        method: str,
+        url: str,
+        headers: dict[str, str],
+        body: bytes | None,
+    ) -> str:
+        """Hash the complete normalized request identity for cache/idempotency.
+
+        Side effects if changes:
+        - cache collisions, conditional validation, and job request hashes.
+        """
+
+        payload = json.dumps(
+            {
+                "method": method,
+                "url": url,
+                "headers": headers,
+                "bodySha256": hashlib.sha256(body or b"").hexdigest(),
+            },
+            sort_keys=True,
+            separators=(",", ":"),
+        ).encode()
+        return hashlib.sha256(payload).hexdigest()