keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/infrastructure/url_policy.py

@@ -0,0 +1,341 @@
+"""DNS-resolving SSRF policy with validated connection targets."""
+
+from __future__ import annotations
+
+import asyncio
+import ipaddress
+import socket
+from time import monotonic
+from urllib.parse import urlsplit
+
+import httpx
+from keble_helpers.typings import PydanticModelConfig
+from pydantic import AliasChoices, AnyHttpUrl, BaseModel, Field, SecretStr, TypeAdapter
+
+from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
+from keble_scraper_api.protocols import HostResolverProtocol
+
+_HTTP_URL_ADAPTER = TypeAdapter(AnyHttpUrl)
+_EXPLICITLY_DENIED = frozenset(
+    {
+        ipaddress.ip_address("169.254.169.254"),
+        ipaddress.ip_address("100.100.100.200"),
+        ipaddress.ip_address("fd00:ec2::254"),
+    }
+)
+
+
+class ValidatedTarget(BaseModel):
+    """Public resolved address set pinned for one outbound hop.
+
+    Side effects if changes:
+    - direct/Bright/public connections and redirect revalidation.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="forbid")
+
+    normalized_url: str
+    hostname: str
+    port: int
+    addresses: tuple[str, ...]
+
+    def pinned_url(self, *, address: str) -> str:
+        """Replace the hostname with one validated IP while preserving request bytes.
+
+        Side effects if changes:
+        - TLS SNI/Host pairing and DNS-rebinding resistance.
+        """
+
+        parsed = urlsplit(self.normalized_url)
+        host = f"[{address}]" if ":" in address else address
+        authority = f"{host}:{self.port}"
+        path = parsed.path or "/"
+        query = f"?{parsed.query}" if parsed.query else ""
+        return f"{parsed.scheme}://{authority}{path}{query}"
+
+
+class AsyncioHostResolver:
+    """Production resolver returning every A/AAAA address from the event loop.
+
+    Side effects if changes:
+    - real DNS resolution and the address set inspected by SSRF policy.
+    """
+
+    async def resolve(self, *, hostname: str, port: int) -> tuple[str, ...]:
+        """Resolve one host without filtering so policy can fail closed on mixing.
+
+        Side effects if changes:
+        - DNS traffic and IP-pinning candidates.
+        """
+
+        loop = asyncio.get_running_loop()
+        entries = await loop.getaddrinfo(
+            hostname,
+            port,
+            family=socket.AF_UNSPEC,
+            type=socket.SOCK_STREAM,
+            proto=socket.IPPROTO_TCP,
+        )
+        return tuple(sorted({str(entry[4][0]) for entry in entries}))
+
+
+class _DnsOverHttpsAnswer(BaseModel):
+    """One typed IP answer from the configured JSON DoH endpoint.
+
+    Side effects if changes:
+    - development resolver response validation only.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="ignore")
+
+    record_type: int = Field(validation_alias=AliasChoices("type", "record_type"))
+    ttl: int = Field(default=300, validation_alias=AliasChoices("TTL", "ttl"), ge=0)
+    data: str = Field(min_length=1, max_length=512)
+
+
+class _DnsOverHttpsResponse(BaseModel):
+    """Typed subset of one RFC 8484 JSON response.
+
+    Side effects if changes:
+    - development DNS success and answer extraction.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="ignore")
+
+    status: int = Field(validation_alias=AliasChoices("Status", "status"))
+    answers: tuple[_DnsOverHttpsAnswer, ...] = Field(
+        default=(),
+        validation_alias=AliasChoices("Answer", "answers"),
+    )
+
+
+class _DnsOverHttpsResolution(BaseModel):
+    """Validated IP answers and their bounded local cache lifetime.
+
+    Side effects if changes:
+    - local resolver cache behavior.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="forbid")
+
+    addresses: tuple[str, ...]
+    ttl_seconds: int = Field(ge=30, le=3_600)
+
+
+class DnsOverHttpsHostResolver:
+    """Resolve real public IPs through an explicit development HTTP proxy.
+
+    Steps:
+    1. cache bounded host answers by provider TTL;
+    2. query A and AAAA families concurrently with fail-fast supervision;
+    3. return only validated IP strings to the SSRF policy.
+
+    Side effects if changes:
+    - local DoH traffic, SSRF address validation inputs, and DNS cache volume.
+    """
+
+    def __init__(
+        self,
+        *,
+        endpoint: AnyHttpUrl,
+        proxy_url: SecretStr,
+        timeout_seconds: float = 10,
+    ) -> None:
+        """Bind one reviewed DoH endpoint and secret-redacted local proxy URL.
+
+        Side effects if changes:
+        - development DNS egress and maximum resolver wait.
+        """
+
+        self._endpoint = endpoint
+        self._proxy_url = proxy_url
+        self._timeout_seconds = timeout_seconds
+        self._cache: dict[str, tuple[float, tuple[str, ...]]] = {}
+        self._lock = asyncio.Lock()
+
+    async def resolve(self, *, hostname: str, port: int) -> tuple[str, ...]:
+        """Resolve A/AAAA answers once per TTL while retaining fail-closed parsing.
+
+        Steps:
+        1. return an unexpired host-scoped cache entry when present;
+        2. query A and AAAA through the explicit proxy without ambient settings;
+        3. validate IP answer types and cache only the merged bounded result.
+
+        Side effects if changes:
+        - local DoH request count and the addresses inspected by `PublicUrlPolicy`.
+        """
+
+        del port
+        now = monotonic()
+        cached = self._cache.get(hostname)
+        if cached is not None and cached[0] > now:
+            return cached[1]
+        async with self._lock:
+            now = monotonic()
+            cached = self._cache.get(hostname)
+            if cached is not None and cached[0] > now:
+                return cached[1]
+            async with asyncio.TaskGroup() as group:
+                ipv4_task = group.create_task(
+                    self._query(hostname=hostname, record_type=1)
+                )
+                ipv6_task = group.create_task(
+                    self._query(hostname=hostname, record_type=28)
+                )
+            resolutions = (ipv4_task.result(), ipv6_task.result())
+            addresses = tuple(
+                sorted(
+                    {
+                        address
+                        for resolution in resolutions
+                        for address in resolution.addresses
+                    }
+                )
+            )
+            if not addresses:
+                return ()
+            ttl_seconds = min(resolution.ttl_seconds for resolution in resolutions)
+            self._cache[hostname] = (now + ttl_seconds, addresses)
+            return addresses
+
+    async def _query(
+        self,
+        *,
+        hostname: str,
+        record_type: int,
+    ) -> _DnsOverHttpsResolution:
+        """Fetch and parse one JSON DoH record family through the local proxy.
+
+        Side effects if changes:
+        - one explicit development network request on a DNS cache miss.
+        """
+
+        async with httpx.AsyncClient(
+            proxy=self._proxy_url.get_secret_value(),
+            timeout=self._timeout_seconds,
+            trust_env=False,
+        ) as client:
+            response = await client.get(
+                str(self._endpoint),
+                params={"name": hostname, "type": str(record_type)},
+                headers={"Accept": "application/dns-json"},
+            )
+            response.raise_for_status()
+        return self.parse_response(raw=response.content, record_type=record_type)
+
+    @staticmethod
+    def parse_response(*, raw: bytes, record_type: int) -> _DnsOverHttpsResolution:
+        """Convert one JSON DoH payload into validated A or AAAA addresses.
+
+        Side effects if changes:
+        - malformed or mismatched DNS answers admitted to SSRF validation.
+        """
+
+        response = _DnsOverHttpsResponse.model_validate_json(raw)
+        if response.status != 0:
+            return _DnsOverHttpsResolution(addresses=(), ttl_seconds=30)
+        matching = tuple(
+            answer for answer in response.answers if answer.record_type == record_type
+        )
+        addresses = tuple(str(ipaddress.ip_address(answer.data)) for answer in matching)
+        ttl_seconds = (
+            max(30, min(3_600, min(answer.ttl for answer in matching)))
+            if matching
+            else 30
+        )
+        return _DnsOverHttpsResolution(
+            addresses=addresses,
+            ttl_seconds=ttl_seconds,
+        )
+
+
+class PublicUrlPolicy:
+    """Validate public HTTP(S) URLs and resolve every hop before dispatch.
+
+    Steps:
+    1. validate typed URL syntax and explicit scheme/authority policy;
+    2. resolve every address for the selected host/port;
+    3. reject the whole answer set when any address is non-public.
+
+    Side effects if changes:
+    - generic URL acceptance, Shopify domains, redirects, and SSRF guarantees.
+    """
+
+    def __init__(self, *, resolver: HostResolverProtocol | None = None) -> None:
+        """Bind an injectable resolver while defaulting to real event-loop DNS.
+
+        Side effects if changes:
+        - all worker URL validation and deterministic policy tests.
+        """
+
+        self._resolver = resolver or AsyncioHostResolver()
+
+    async def validate_and_resolve(self, *, url: str) -> ValidatedTarget:
+        """Parse, deny private ranges, and return one stable address set.
+
+        Steps:
+        1. validate through Pydantic without trusting free-form parser recovery;
+        2. allow only HTTP(S), ports 80/443, no credentials/fragments;
+        3. resolve all A/AAAA answers and reject the entire hostname if any
+           answer is non-global or explicitly metadata-reserved.
+
+        Side effects if changes:
+        - connection pinning and redirect security.
+        """
+
+        parsed = _HTTP_URL_ADAPTER.validate_python(url)
+        if parsed.scheme not in {"http", "https"}:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.URL_DENIED,
+                message="only HTTP(S) resources are permitted",
+            )
+        if (
+            parsed.username is not None
+            or parsed.password is not None
+            or parsed.fragment
+        ):
+            raise ScraperDomainError(
+                code=ScraperErrorCode.URL_DENIED,
+                message="URL credentials and fragments are forbidden",
+            )
+        port = parsed.port or (443 if parsed.scheme == "https" else 80)
+        if port not in {80, 443}:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.URL_DENIED,
+                message="only ports 80 and 443 are permitted",
+            )
+        if parsed.host is None:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.URL_DENIED,
+                message="URL hostname is required",
+            )
+        hostname = parsed.host.rstrip(".").lower()
+        addresses = await self._resolver.resolve(hostname=hostname, port=port)
+        if not addresses:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.DNS_FAILED,
+                message="public hostname resolution failed",
+            )
+        denied = [address for address in addresses if not self._is_public(address)]
+        if denied:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.URL_DENIED,
+                message="hostname resolves to a denied network range",
+            )
+        return ValidatedTarget(
+            normalized_url=str(parsed),
+            hostname=hostname,
+            port=port,
+            addresses=addresses,
+        )
+
+    @staticmethod
+    def _is_public(address: str) -> bool:
+        """Return whether one resolver-produced address is globally routable.
+
+        Side effects if changes:
+        - private, loopback, CGNAT, documentation, multicast, and metadata denial.
+        """
+
+        candidate = ipaddress.ip_address(address)
+        return candidate.is_global and candidate not in _EXPLICITLY_DENIED

keble_scraper_api/main.py

@@ -0,0 +1,162 @@
+"""FastAPI application and process entrypoint."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from pathlib import Path
+from types import TracebackType
+from typing import cast
+
+import uvicorn
+from cryptography.fernet import InvalidToken
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from starlette.types import ExceptionHandler
+
+from keble_scraper_api.api.dependencies import ContainerProvider
+from keble_scraper_api.api.errors import invalid_session_handler, scraper_error_handler
+from keble_scraper_api.api.router import create_router
+from keble_scraper_api.container import ScraperContainer
+from keble_scraper_api.errors import ScraperDomainError
+from keble_scraper_api.settings import ScraperSettings
+from keble_scraper_api.telemetry import TelemetryRuntime, configure_sentry
+
+
+class ContainerLifecycle:
+    """Ensure process resources close through async-context semantics.
+
+    Steps:
+    1. start one explicitly owned dependency graph;
+    2. expose it only after readiness completes;
+    3. clear request access and close pools on normal lifespan exit.
+
+    Side effects if changes:
+    - FastAPI startup indexes/readiness and graceful connection cleanup.
+    """
+
+    def __init__(
+        self,
+        *,
+        container: ScraperContainer,
+        provider: ContainerProvider,
+    ) -> None:
+        """Bind one unstarted container and its static route provider.
+
+        Side effects if changes:
+        - application lifespan ownership.
+        """
+
+        self._container = container
+        self._provider = provider
+
+    async def __aenter__(self) -> ScraperContainer:
+        """Start dependencies then expose the bound graph to requests.
+
+        Steps:
+        1. await startup directly so faults retain their traceback;
+        2. bind request dependencies only after startup succeeds;
+        3. return the singular process container.
+
+        Side effects if changes:
+        - Mongo indexes, readiness calls, and route availability.
+        """
+
+        await self._container.start()
+        self._provider.bind(container=self._container)
+        return self._container
+
+    async def __aexit__(
+        self,
+        exception_type: type[BaseException] | None,
+        exception: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        """Clear route access and close process-owned pools on every exit.
+
+        Steps:
+        1. make request dependency resolution fail closed;
+        2. close every process-owned pool in container order;
+        3. retain the original lifespan exception for framework supervision.
+
+        Side effects if changes:
+        - shutdown request behavior and connection leakage.
+        """
+
+        self._provider.clear()
+        await self._container.close()
+
+
+def create_app(*, settings: ScraperSettings | None = None) -> FastAPI:
+    """Build the API without connecting until lifespan startup.
+
+    Steps:
+    1. resolve typed settings and configure process telemetry;
+    2. compose one lazy lifespan-owned container and API router;
+    3. register finite error mappings and optional static frontend serving.
+
+    Side effects if changes:
+    - OpenAPI, Sentry configuration, routes, and optional frontend serving.
+    """
+
+    resolved = settings or ScraperSettings()
+    configure_sentry(settings=resolved, runtime=TelemetryRuntime.API)
+    provider = ContainerProvider()
+
+    @asynccontextmanager
+    async def lifespan(application: FastAPI) -> AsyncIterator[None]:
+        """Start and close one explicit container around request service.
+
+        Steps:
+        1. build a process-local graph without hidden globals;
+        2. enter its startup/readiness lifecycle;
+        3. yield request service until framework shutdown.
+
+        Side effects if changes:
+        - all API process infrastructure resources.
+        """
+
+        container = ScraperContainer.build(settings=resolved)
+        async with ContainerLifecycle(container=container, provider=provider):
+            yield
+
+    app = FastAPI(
+        title="Keble Scraper API",
+        version="0.2.0",
+        lifespan=lifespan,
+    )
+    app.add_exception_handler(
+        ScraperDomainError,
+        cast(ExceptionHandler, scraper_error_handler),
+    )
+    app.add_exception_handler(
+        InvalidToken,
+        cast(ExceptionHandler, invalid_session_handler),
+    )
+    app.include_router(create_router(provider=provider))
+    frontend_dist = Path(resolved.frontend_dist)
+    if frontend_dist.is_dir():
+        app.mount(
+            "/",
+            StaticFiles(directory=frontend_dist, html=True),
+            name="scraper-frontend",
+        )
+    return app
+
+
+def run() -> None:
+    """Run the configured FastAPI process under Uvicorn.
+
+    Side effects if changes:
+    - binds the scraper API network listener.
+    """
+
+    settings = ScraperSettings()
+    uvicorn.run(
+        create_app(settings=settings),
+        host=settings.host,
+        port=settings.port,
+    )
+
+
+app = create_app()

keble_scraper_api/maintenance/__init__.py

@@ -0,0 +1 @@
+"""Guarded operator maintenance services and typed command contracts."""