keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/infrastructure/dispatcher.py

@@ -0,0 +1,57 @@
+"""Celery-backed asynchronous job dispatcher."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime
+from uuid import UUID
+
+from celery import Celery
+
+from keble_scraper_contract import ScrapeTransportPolicy
+
+from keble_scraper_api.settings import ScraperSettings
+
+
+class CeleryJobDispatcher:
+    """Publish only opaque job identities to the owning worker queue.
+
+    Side effects if changes:
+    - RabbitMQ messages, queue isolation, and render/fetch worker load.
+    """
+
+    def __init__(self, *, celery: Celery, settings: ScraperSettings) -> None:
+        """Bind one broker client and settings-owned queue names.
+
+        Side effects if changes:
+        - all API/maintenance task publication.
+        """
+
+        self._celery = celery
+        self._settings = settings
+
+    async def dispatch(
+        self,
+        *,
+        job_id: UUID,
+        transport_policy: ScrapeTransportPolicy,
+        not_before: datetime | None = None,
+    ) -> None:
+        """Publish a fetch/render task with optional broker ETA.
+
+        Side effects if changes:
+        - one RabbitMQ task message and worker scheduling.
+        """
+
+        queue = (
+            self._settings.render_queue
+            if transport_policy is ScrapeTransportPolicy.BROWSER_EXCEPTION
+            else self._settings.fetch_queue
+        )
+        await asyncio.to_thread(
+            self._celery.send_task,
+            "keble_scraper.execute_job",
+            args=[str(job_id)],
+            queue=queue,
+            eta=not_before,
+        )

keble_scraper_api/infrastructure/http_fetcher.py

@@ -0,0 +1,351 @@
+"""TLS-validating, IP-pinned, redirect-revalidating HTTP fetcher."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime
+from time import monotonic
+from urllib.parse import urljoin
+
+import aiohttp
+from keble_data_infra_contract import parse_retry_after
+from keble_helpers.typings import PydanticModelConfig
+from pydantic import BaseModel
+
+from keble_scraper_contract import (
+    ScrapeArtifactTrust,
+    ScrapeAttemptOutcome,
+    ScrapeTransportRoute,
+)
+
+from keble_scraper_api.application.fetching import FetchResult, PreparedFetch
+from keble_scraper_api.models import RouteReservation
+from keble_scraper_api.settings import ResponsePolicy, ScraperSettings
+
+from .url_policy import PublicUrlPolicy, ValidatedTarget
+
+
+class _HopResult(BaseModel):
+    """One IP-pinned HTTP hop before redirect orchestration.
+
+    Side effects if changes:
+    - redirect validation and final typed fetch outcomes.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="forbid")
+
+    result: FetchResult
+    redirect_location: str | None = None
+
+
+class SecureHttpFetcher:
+    """Fetch static resources through direct/Bright/public routes safely.
+
+    Steps:
+    1. validate and pin every initial/redirect URL hop;
+    2. reuse one pooled client while retaining per-request SNI/proxy policy;
+    3. stream bytes within response limits into typed outcomes.
+
+    Side effects if changes:
+    - TLS validation, SSRF pinning, limits, redirects, and retry outcomes.
+    """
+
+    def __init__(
+        self, *, settings: ScraperSettings, url_policy: PublicUrlPolicy
+    ) -> None:
+        """Bind immutable server policy and resolver.
+
+        Steps:
+        1. retain typed fetch and URL policies;
+        2. initialize a lazy pooled-session slot;
+        3. initialize a lock that serializes first session construction.
+
+        Side effects if changes:
+        - every worker fetch shares this response and URL policy.
+        """
+
+        self._settings = settings
+        self._url_policy = url_policy
+        self._session: aiohttp.ClientSession | None = None
+        self._session_lock = asyncio.Lock()
+
+    async def close(self) -> None:
+        """Close the process-owned pooled HTTP session exactly once.
+
+        Steps:
+        1. return when no outbound request created the lazy session;
+        2. close all pooled direct/proxy connections;
+        3. clear the reference so repeated shutdown remains idempotent.
+
+        Side effects if changes:
+        - API/worker shutdown and socket/connector leak prevention.
+        """
+
+        if self._session is None:
+            return
+        await self._session.close()
+        self._session = None
+
+    async def fetch(
+        self,
+        *,
+        request: PreparedFetch,
+        reservation: RouteReservation,
+        response_policy: ResponsePolicy,
+        conditional_headers: dict[str, str] | None = None,
+    ) -> FetchResult:
+        """Follow at most five independently validated and pinned hops.
+
+        Steps:
+        1. resolve and validate every URL before opening a connection;
+        2. connect to the chosen IP while retaining original Host/TLS SNI;
+        3. re-run policy for each redirect and stream within server byte limits.
+
+        Side effects if changes:
+        - cache validators, immutable attempt outcomes, and artifacts.
+        """
+
+        current_url = request.url
+        for redirect_count in range(self._settings.max_redirects + 1):
+            target = await self._url_policy.validate_and_resolve(url=current_url)
+            hop = await self._fetch_hop(
+                request=request,
+                target=target,
+                reservation=reservation,
+                response_policy=response_policy,
+                conditional_headers=conditional_headers or {},
+            )
+            if hop.redirect_location is None:
+                return hop.result
+            if redirect_count == self._settings.max_redirects:
+                return FetchResult(
+                    outcome=ScrapeAttemptOutcome.DENIED,
+                    route=reservation.route,
+                    trust=self._trust(reservation.route),
+                    final_url=current_url,
+                    elapsed_ms=hop.result.elapsed_ms,
+                    error_code="REDIRECT_LIMIT",
+                    observed_at=datetime.now(UTC),
+                )
+            current_url = urljoin(target.normalized_url, hop.redirect_location)
+        raise AssertionError("bounded redirect loop must return")
+
+    async def _fetch_hop(
+        self,
+        *,
+        request: PreparedFetch,
+        target: ValidatedTarget,
+        reservation: RouteReservation,
+        response_policy: ResponsePolicy,
+        conditional_headers: dict[str, str],
+    ) -> _HopResult:
+        """Execute one hop while allowing implementation/transport faults to supervise.
+
+        Steps:
+        1. construct the pinned request and bounded timeout policy;
+        2. call the pooled client without converting exceptions into business data;
+        3. attach measured elapsed time to a successfully typed transport result.
+
+        Side effects if changes:
+        - IP pinning, TLS verification, response limits, and route accounting.
+        """
+
+        started = monotonic()
+        address = target.addresses[0]
+        headers = {
+            "User-Agent": self._settings.identified_user_agent,
+            "Host": target.hostname,
+            **request.headers,
+            **conditional_headers,
+        }
+        timeout = aiohttp.ClientTimeout(
+            total=self._settings.connect_timeout_seconds
+            + self._settings.read_timeout_seconds,
+            connect=self._settings.connect_timeout_seconds,
+            sock_read=self._settings.read_timeout_seconds,
+        )
+        raw = await self._send(
+            method=request.method,
+            url=target.pinned_url(address=address),
+            original_url=target.normalized_url,
+            server_hostname=target.hostname,
+            headers=headers,
+            body=request.body,
+            proxy_url=(
+                reservation.proxy_url.get_secret_value()
+                if reservation.proxy_url is not None
+                else self._development_proxy_url(reservation=reservation)
+            ),
+            timeout=timeout,
+            response_policy=response_policy,
+            route=reservation.route,
+        )
+        elapsed_ms = max(0, int((monotonic() - started) * 1000))
+        result, redirect = raw
+        return _HopResult(
+            result=result.model_copy(update={"elapsed_ms": elapsed_ms}),
+            redirect_location=redirect,
+        )
+
+    def _development_proxy_url(self, *, reservation: RouteReservation) -> str | None:
+        """Return explicit local egress only for the verified DIRECT route.
+
+        Side effects if changes:
+        - development network path and production direct-route isolation.
+        """
+
+        if (
+            reservation.route is ScrapeTransportRoute.DIRECT
+            and self._settings.development_http_proxy_url is not None
+        ):
+            return self._settings.development_http_proxy_url.get_secret_value()
+        return None
+
+    async def _send(
+        self,
+        *,
+        method: str,
+        url: str,
+        original_url: str,
+        server_hostname: str,
+        headers: dict[str, str],
+        body: bytes | None,
+        proxy_url: str | None,
+        timeout: aiohttp.ClientTimeout,
+        response_policy: ResponsePolicy,
+        route: ScrapeTransportRoute,
+    ) -> tuple[FetchResult, str | None]:
+        """Stream one pooled aiohttp response within its byte ceiling.
+
+        Steps:
+        1. reuse the process-owned client session for connection pooling;
+        2. preserve the declared content type and parse shared Retry-After evidence;
+        3. stream bounded bytes into one finite HTTP/content/size outcome.
+
+        Side effects if changes:
+        - actual network egress and response artifact bytes.
+        """
+
+        session = await self._client_session()
+        async with session.request(
+            method,
+            url,
+            headers=headers,
+            data=body,
+            proxy=proxy_url,
+            allow_redirects=False,
+            server_hostname=server_hostname,
+            timeout=timeout,
+        ) as response:
+            status = response.status
+            location = (
+                response.headers.get("Location")
+                if status in {301, 302, 303, 307, 308}
+                else None
+            )
+            content_type = response.headers.get(
+                "Content-Type",
+                "application/octet-stream",
+            ).strip()
+            media_type = content_type.split(";", 1)[0].strip().lower()
+            observed_at = datetime.now(UTC)
+            retry_at = parse_retry_after(
+                value=response.headers.get("Retry-After"),
+                observed_at=observed_at,
+            )
+            base = {
+                "route": route,
+                "trust": self._trust(route),
+                "final_url": original_url,
+                "status_code": status,
+                "content_type": content_type,
+                "etag": response.headers.get("ETag"),
+                "last_modified": response.headers.get("Last-Modified"),
+                "elapsed_ms": 0,
+                "retry_at": retry_at,
+                "observed_at": observed_at,
+            }
+            if location is not None:
+                return FetchResult(
+                    outcome=ScrapeAttemptOutcome.SUCCEEDED, **base
+                ), location
+            if status >= 400:
+                return FetchResult(
+                    outcome=ScrapeAttemptOutcome.HTTP_ERROR,
+                    error_code="HTTP_ERROR",
+                    **base,
+                ), None
+            if status == 304 or method == "HEAD":
+                return FetchResult(outcome=ScrapeAttemptOutcome.SUCCEEDED, **base), None
+            if not any(
+                media_type.startswith(prefix)
+                for prefix in response_policy.content_type_prefixes
+            ):
+                return FetchResult(
+                    outcome=ScrapeAttemptOutcome.CONTENT_TYPE_REJECTED,
+                    error_code="CONTENT_TYPE_REJECTED",
+                    **base,
+                ), None
+            declared_length = response.headers.get("Content-Length", "")
+            if (
+                declared_length.isdigit()
+                and int(declared_length) > response_policy.max_bytes
+            ):
+                return FetchResult(
+                    outcome=ScrapeAttemptOutcome.SIZE_LIMIT,
+                    error_code="RESPONSE_TOO_LARGE",
+                    **base,
+                ), None
+            chunks: list[bytes] = []
+            received = 0
+            async for chunk in response.content.iter_chunked(65_536):
+                received += len(chunk)
+                if received > response_policy.max_bytes:
+                    return FetchResult(
+                        outcome=ScrapeAttemptOutcome.SIZE_LIMIT,
+                        error_code="RESPONSE_TOO_LARGE",
+                        **base,
+                    ), None
+                chunks.append(chunk)
+            return FetchResult(
+                outcome=ScrapeAttemptOutcome.SUCCEEDED,
+                body=b"".join(chunks),
+                **base,
+            ), None
+
+    async def _client_session(self) -> aiohttp.ClientSession:
+        """Return one lazy process-owned pooled aiohttp session.
+
+        Steps:
+        1. return the open session on the hot path;
+        2. serialize first construction for concurrent callers;
+        3. configure no ambient proxy trust and no automatic status exceptions.
+
+        Side effects if changes:
+        - connection reuse, DNS/socket volume, and shutdown ownership.
+        """
+
+        if self._session is not None and not self._session.closed:
+            return self._session
+        async with self._session_lock:
+            if self._session is None or self._session.closed:
+                self._session = aiohttp.ClientSession(
+                    trust_env=False,
+                    auto_decompress=True,
+                    raise_for_status=False,
+                )
+            return self._session
+
+    @staticmethod
+    def _trust(route: ScrapeTransportRoute) -> ScrapeArtifactTrust:
+        """Map public-proxy experiments to non-indexable trust.
+
+        Side effects if changes:
+        - Shopify production artifact acceptance.
+        """
+
+        return (
+            ScrapeArtifactTrust.UNTRUSTED
+            if route is ScrapeTransportRoute.PUBLIC_POOL_EXPERIMENT
+            else ScrapeArtifactTrust.VERIFIED
+        )