keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/application/proxy_health.py

@@ -0,0 +1,126 @@
+"""Quarantined public-proxy health checking."""
+
+from __future__ import annotations
+
+from datetime import timedelta
+
+from keble_scraper_contract import (
+    ProxyHealthObservation,
+    ScrapeTransportRoute,
+    utc_now,
+)
+from keble_helpers.typings import PydanticModelConfig
+from pydantic import BaseModel, IPvAnyAddress
+
+from keble_scraper_api.application.fetching import FetchResult, PreparedFetch
+from keble_scraper_api.models import RouteReservation
+from keble_scraper_api.protocols import HttpFetcherProtocol, ProxyRepositoryProtocol
+from keble_scraper_api.settings import ScraperSettings
+
+
+class ProxyHealthPayload(BaseModel):
+    """Validated egress identity returned by the fixed health endpoint.
+
+    Side effects if changes:
+    - proxy health evidence and operator diagnostics.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="ignore")
+
+    ip: IPvAnyAddress
+
+
+class ProxyHealthService:
+    """Test quarantined proxies against one fixed public TLS endpoint.
+
+    Steps:
+    1. inspect only a bounded set of due quarantined candidates;
+    2. use the shared secure fetcher against a fixed endpoint;
+    3. persist typed health evidence only after response validation.
+
+    Side effects if changes:
+    - experimental proxy traffic and promotion/decay state.
+    """
+
+    def __init__(
+        self,
+        *,
+        repository: ProxyRepositoryProtocol,
+        fetcher: HttpFetcherProtocol,
+        settings: ScraperSettings,
+    ) -> None:
+        """Bind proxy persistence and the same secure HTTP transport.
+
+        Side effects if changes:
+        - all proxy-health worker runs.
+        """
+
+        self._repository = repository
+        self._fetcher = fetcher
+        self._settings = settings
+
+    async def run_due(self, *, limit: int = 100) -> int:
+        """Check due candidates sequentially and append bounded observations.
+
+        Steps:
+        1. load bounded due candidates and their secret credentials;
+        2. fetch the fixed endpoint while allowing implementation faults to supervise;
+        3. record only typed HTTP/payload outcomes as health evidence.
+
+        Side effects if changes:
+        - public network calls and candidate health state updates.
+        """
+
+        candidates = await self._repository.due_candidates(now=utc_now(), limit=limit)
+        for candidate in candidates:
+            credential = await self._repository.credential_for(
+                candidate_id=candidate.candidate_id
+            )
+            request = PreparedFetch(
+                url="https://api.ipify.org/?format=json",
+                method="GET",
+                headers={"Accept": "application/json"},
+                request_identity=f"proxy-health:{candidate.candidate_id}",
+            )
+            result = await self._fetcher.fetch(
+                request=request,
+                reservation=RouteReservation(
+                    route=ScrapeTransportRoute.PUBLIC_POOL_EXPERIMENT,
+                    proxy_url=credential,
+                ),
+                response_policy=self._settings.response_policy("JSON"),
+            )
+            parsed = await self._parse_payload(result=result)
+            succeeded = parsed is not None
+            observation = ProxyHealthObservation(
+                succeeded=succeeded,
+                latency_ms=result.elapsed_ms,
+                egress_ip=str(parsed.ip) if parsed is not None else None,
+                tls_valid=succeeded,
+            )
+            await self._repository.record_health(
+                candidate_id=candidate.candidate_id,
+                observation=observation,
+                next_health_at=utc_now() + timedelta(minutes=15),
+            )
+        return len(candidates)
+
+    @staticmethod
+    async def _parse_payload(
+        *,
+        result: FetchResult,
+    ) -> ProxyHealthPayload | None:
+        """Accept only a successful typed response containing one valid IP.
+
+        Steps:
+        1. treat a finite non-success transport result as failed health evidence;
+        2. validate a success body through the strict Pydantic payload;
+        3. let malformed success bodies surface as provider/implementation faults.
+
+        Side effects if changes:
+        - proxy promotion and recorded egress identity.
+        """
+
+        if not result.succeeded:
+            return None
+        return ProxyHealthPayload.model_validate_json(result.body)

keble_scraper_api/application/scheduling.py

@@ -0,0 +1,24 @@
+"""Durable scheduling precision shared by jobs, failures, and callbacks."""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+
+def ceil_persistence_millisecond(*, value: datetime) -> datetime:
+    """Round one due horizon upward to MongoDB BSON millisecond precision.
+
+    Steps:
+    1. retain timestamps already aligned to an exact millisecond;
+    2. add only the missing microseconds to reach the next millisecond;
+    3. preserve timezone information and never schedule before the input horizon.
+
+    Side effects if changes:
+    - job and callback retries must not run before provider/local wait horizons;
+    - append-only failure evidence must equal its persisted scheduling precision.
+    """
+
+    remainder = value.microsecond % 1_000
+    if remainder == 0:
+        return value
+    return value + timedelta(microseconds=1_000 - remainder)

keble_scraper_api/artifact_namespace.py

@@ -0,0 +1,76 @@
+"""Strongly typed artifact-store identity used by Mongo and binary stores."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+from keble_helpers.typings import PydanticModelConfig
+from pydantic import BaseModel, Field, model_validator
+
+
+class ArtifactStoreBackend(StrEnum):
+    """Enumerate storage backends whose object keys are not interchangeable.
+
+    Steps:
+    1. distinguish durable OSS, shared local directory, and test memory stores.
+
+    Side effects if changes:
+    - Mongo namespace markers persist these values as compatibility evidence;
+    - every new backend must implement the artifact-store owner-claim protocol.
+    """
+
+    OSS = "oss"
+    LOCAL_DIRECTORY = "local_directory"
+    MEMORY = "memory"
+
+
+class ArtifactStoreIdentity(BaseModel):
+    """Describe every non-secret location component that determines stored bytes.
+
+    Steps:
+    1. retain backend kind plus its canonical object-key prefix;
+    2. require endpoint/bucket only for OSS and directory only for local storage;
+    3. forbid irrelevant fields so backend swaps cannot compare equal.
+
+    Side effects if changes:
+    - scraper startup compares this exact value with the Mongo namespace marker;
+    - changing OSS endpoint/bucket or local directory requires explicit migration;
+    - access credentials and download base URLs remain safely excluded.
+    """
+
+    model_config = PydanticModelConfig.default(frozen=True, extra="forbid")
+
+    backend: ArtifactStoreBackend
+    prefix: str = Field(min_length=1, max_length=512)
+    endpoint: str | None = Field(default=None, min_length=1, max_length=2_048)
+    bucket: str | None = Field(default=None, min_length=1, max_length=255)
+    directory: str | None = Field(default=None, min_length=1, max_length=4_096)
+
+    @model_validator(mode="after")
+    def validate_backend_fields(self) -> "ArtifactStoreIdentity":
+        """Require the exact location fields owned by the selected backend.
+
+        Steps:
+        1. validate endpoint/bucket for OSS;
+        2. validate directory for local storage;
+        3. require memory identity to contain only its fixed prefix.
+
+        Side effects if changes:
+        - malformed markers must fail before artifact reads or cleanup;
+        - identity equality remains a total backend/location comparison.
+        """
+
+        if self.backend is ArtifactStoreBackend.OSS:
+            if self.endpoint is None or self.bucket is None or self.directory is not None:
+                raise ValueError("OSS identity requires endpoint/bucket and forbids directory")
+        elif self.backend is ArtifactStoreBackend.LOCAL_DIRECTORY:
+            if self.directory is None or self.endpoint is not None or self.bucket is not None:
+                raise ValueError(
+                    "local artifact identity requires directory and forbids OSS fields"
+                )
+        elif any(value is not None for value in (self.endpoint, self.bucket, self.directory)):
+            raise ValueError("memory artifact identity forbids external location fields")
+        return self
+
+
+__all__ = ["ArtifactStoreBackend", "ArtifactStoreIdentity"]

keble_scraper_api/container.py

@@ -0,0 +1,361 @@
+"""Explicit process composition for API and worker runtimes."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+from dataclasses import dataclass
+
+from celery import Celery
+from pydantic import SecretStr
+from redis.asyncio import Redis
+
+from keble_scraper_api.application.admin import (
+    ArtifactAdminService,
+    MaintenanceAdminService,
+    ProxyAdminService,
+)
+from keble_scraper_api.application.artifacts import ArtifactService
+from keble_scraper_api.application.auth import (
+    OperatorAuthService,
+    SessionCipher,
+    StaticServiceKeyAuthorizer,
+)
+from keble_scraper_api.application.jobs import (
+    AuditService,
+    MaintenanceService,
+    ScrapeExecutionService,
+    ScrapeJobService,
+)
+from keble_scraper_api.application.proxy_health import ProxyHealthService
+from keble_scraper_api.infrastructure.artifact_store import (
+    InMemoryArtifactStore,
+    LocalDirectoryArtifactStore,
+    OssArtifactStore,
+)
+from keble_scraper_api.infrastructure.callbacks import (
+    CallbackOutboxService,
+    RegisteredHttpCompletionCallback,
+    WebhookSigner,
+)
+from keble_scraper_api.infrastructure.celery_factory import build_celery
+from keble_scraper_api.infrastructure.cloudflare import CloudflareBrowserFetcher
+from keble_scraper_api.infrastructure.credentials import ProxyCredentialCipher
+from keble_scraper_api.infrastructure.dispatcher import CeleryJobDispatcher
+from keble_scraper_api.infrastructure.http_fetcher import SecureHttpFetcher
+from keble_scraper_api.infrastructure.mongo import MongoScraperRepository
+from keble_scraper_api.infrastructure.proxy_router import RedisProxyRouter
+from keble_scraper_api.infrastructure.request_profiles import RequestProfileRegistry
+from keble_scraper_api.infrastructure.url_policy import (
+    DnsOverHttpsHostResolver,
+    PublicUrlPolicy,
+)
+from keble_scraper_api.protocols import ArtifactStoreProtocol
+from keble_scraper_api.settings import RuntimeEnvironment, ScraperSettings
+
+
+@dataclass(frozen=True)
+class ScraperContainer:
+    """Fully assembled process service graph with explicit resource ownership.
+
+    Steps:
+    1. construct one process graph from a singular settings object;
+    2. start indexes, namespace fences, and readiness before use;
+    3. close every pooled external resource in deterministic order.
+
+    Side effects if changes:
+    - API/worker startup order and every external infrastructure client.
+    """
+
+    settings: ScraperSettings
+    repository: MongoScraperRepository
+    redis: Redis
+    celery: Celery
+    artifact_store: ArtifactStoreProtocol
+    http_fetcher: SecureHttpFetcher
+    browser_fetcher: CloudflareBrowserFetcher
+    callback_transport: RegisteredHttpCompletionCallback
+    auth: OperatorAuthService
+    service_keys: StaticServiceKeyAuthorizer
+    jobs: ScrapeJobService
+    execution: ScrapeExecutionService
+    maintenance: MaintenanceService
+    proxy_health: ProxyHealthService
+    callbacks: CallbackOutboxService
+    proxy_admin: ProxyAdminService
+    artifact_admin: ArtifactAdminService
+    maintenance_admin: MaintenanceAdminService
+
+    @classmethod
+    def build(cls, *, settings: ScraperSettings | None = None) -> "ScraperContainer":
+        """Construct the complete graph once from one typed settings value.
+
+        Steps:
+        1. create shared Mongo/Redis/Rabbit clients and encrypted credential owner;
+        2. wire protocols into application services with no service locator;
+        3. select private OSS in configured environments and isolated memory only
+           for local/test operation without OSS credentials.
+
+        Side effects if changes:
+        - connections, vendor clients, queue publication, and auth behavior.
+        """
+
+        resolved = settings or ScraperSettings()
+        proxy_key = _stable_fernet_key(
+            configured=resolved.proxy_secret_fernet_key,
+            fallback=resolved.service_key_current,
+        )
+        proxy_cipher = ProxyCredentialCipher(fernet_key=proxy_key)
+        repository = MongoScraperRepository.build(
+            settings=resolved,
+            proxy_cipher=proxy_cipher,
+        )
+        redis = Redis.from_url(
+            resolved.redis_uri.get_secret_value(),
+            decode_responses=True,
+        )
+        celery = build_celery(settings=resolved)
+        dispatcher = CeleryJobDispatcher(celery=celery, settings=resolved)
+        profiles = RequestProfileRegistry()
+        url_policy = PublicUrlPolicy(
+            resolver=(
+                DnsOverHttpsHostResolver(
+                    endpoint=resolved.development_dns_over_https_url,
+                    proxy_url=resolved.development_http_proxy_url,
+                )
+                if resolved.development_dns_over_https_url is not None
+                and resolved.development_http_proxy_url is not None
+                else None
+            )
+        )
+        http_fetcher = SecureHttpFetcher(settings=resolved, url_policy=url_policy)
+        browser_fetcher = CloudflareBrowserFetcher(
+            settings=resolved,
+            url_policy=url_policy,
+        )
+        proxy_router = RedisProxyRouter(
+            redis=redis,
+            settings=resolved,
+            proxy_repository=repository,
+        )
+        artifact_store = build_artifact_store(settings=resolved)
+        artifacts = ArtifactService(repository=repository, store=artifact_store)
+        audit = AuditService(repository=repository)
+        jobs = ScrapeJobService(
+            repository=repository,
+            dispatcher=dispatcher,
+            profiles=profiles,
+            audit=audit,
+        )
+        execution = ScrapeExecutionService(
+            settings=resolved,
+            jobs=repository,
+            cache=repository,
+            artifacts=artifacts,
+            router=proxy_router,
+            profiles=profiles,
+            http_fetcher=http_fetcher,
+            browser_fetcher=browser_fetcher,
+        )
+        maintenance = MaintenanceService(
+            jobs=repository,
+            outbox=repository,
+            cache=repository,
+            artifacts=artifacts,
+            history=repository,
+            dispatcher=dispatcher,
+            dispatch_concurrency=resolved.maintenance_dispatch_concurrency,
+            history_retention_batch_size=resolved.history_retention_batch_size,
+        )
+        signer = WebhookSigner(key=resolved.callback_hmac_current)
+        callback = RegisteredHttpCompletionCallback(settings=resolved, signer=signer)
+        callbacks = CallbackOutboxService(
+            repository=repository,
+            callback=callback,
+            lease_seconds=resolved.worker_lease_seconds,
+            retry_base_seconds=resolved.retry_base_seconds,
+        )
+        proxy_health = ProxyHealthService(
+            repository=repository,
+            fetcher=http_fetcher,
+            settings=resolved,
+        )
+        session_key = (
+            resolved.session_fernet_key
+            if resolved.session_fernet_key is not None
+            else _stable_fernet_key(
+                configured=None,
+                fallback=resolved.callback_hmac_current,
+            )
+        )
+        auth = OperatorAuthService(
+            settings=resolved,
+            cipher=SessionCipher(key=session_key),
+        )
+        service_keys = StaticServiceKeyAuthorizer(
+            current=resolved.service_key_current,
+            next_key=resolved.service_key_next,
+        )
+        proxy_admin = ProxyAdminService(
+            repository=repository,
+            router=proxy_router,
+            audit=audit,
+            settings=resolved,
+        )
+        artifact_admin = ArtifactAdminService(
+            artifacts=artifacts,
+            audit=audit,
+            settings=resolved,
+        )
+        maintenance_admin = MaintenanceAdminService(
+            maintenance=maintenance,
+            audit=audit,
+        )
+        return cls(
+            settings=resolved,
+            repository=repository,
+            redis=redis,
+            celery=celery,
+            artifact_store=artifact_store,
+            http_fetcher=http_fetcher,
+            browser_fetcher=browser_fetcher,
+            callback_transport=callback,
+            auth=auth,
+            service_keys=service_keys,
+            jobs=jobs,
+            execution=execution,
+            maintenance=maintenance,
+            proxy_health=proxy_health,
+            callbacks=callbacks,
+            proxy_admin=proxy_admin,
+            artifact_admin=artifact_admin,
+            maintenance_admin=maintenance_admin,
+        )
+
+    async def start(self) -> None:
+        """Create indexes and verify namespace/shared resources before work.
+
+        Steps:
+        1. create required Mongo indexes;
+        2. claim the binary-store namespace for one stable deployment owner;
+        3. bind Mongo artifact rows to the complete binary-store identity;
+        4. verify Mongo and Redis readiness only after both fences pass.
+
+        Side effects if changes:
+        - Mongo index/namespace state and process readiness.
+        """
+
+        await self.repository.ensure_indexes()
+        store_identity = self.artifact_store.namespace_identity()
+        namespace_owner = self.settings.resolved_artifact_namespace_owner
+        await self.artifact_store.ensure_namespace_owner(owner=namespace_owner)
+        await self.repository.ensure_artifact_store_namespace(
+            probe_object_key=self.artifact_store.object_key(content_sha256="0" * 64),
+            store_identity=store_identity,
+            namespace_owner=namespace_owner,
+        )
+        await self.repository.readiness()
+        await self.redis.ping()
+
+    async def close(self) -> None:
+        """Close pooled HTTP, Redis, then Mongo process-owned resources.
+
+        Steps:
+        1. close direct/proxy, Cloudflare, and callback pooled HTTP transports;
+        2. close the shared Redis connection pool;
+        3. close the Mongo client after no service can issue more work.
+
+        Side effects if changes:
+        - graceful API/worker shutdown and connection leakage.
+        """
+
+        await self.http_fetcher.close()
+        await self.browser_fetcher.close()
+        await self.callback_transport.close()
+        await self.redis.aclose()
+        await self.repository.close()
+
+    async def readiness(self) -> dict[str, str]:
+        """Verify Mongo/Redis without privileged vendor or service calls.
+
+        Side effects if changes:
+        - deployment health gates and rollback decisions.
+        """
+
+        await self.repository.readiness()
+        await self.redis.ping()
+        return {
+            "status": "ready",
+            "mongo": "ready",
+            "redis": "ready",
+            "artifactStore": (
+                "private-oss"
+                if self.settings.oss_bucket is not None
+                else "shared-local-directory"
+                if self.settings.local_artifact_directory is not None
+                else "ephemeral-development"
+            ),
+        }
+
+
+def _stable_fernet_key(
+    *, configured: SecretStr | None, fallback: SecretStr
+) -> SecretStr:
+    """Derive a deterministic URL-safe key only for non-production fallback use.
+
+    Side effects if changes:
+    - development proxy/session ciphertext compatibility across processes.
+    """
+
+    if configured is not None:
+        return configured
+    digest = hashlib.sha256(fallback.get_secret_value().encode()).digest()
+    return SecretStr(base64.urlsafe_b64encode(digest).decode())
+
+
+def build_artifact_store(*, settings: ScraperSettings) -> ArtifactStoreProtocol:
+    """Select private OSS or an explicit non-production ephemeral test store.
+
+    Steps:
+    1. prefer the complete private OSS tuple when configured;
+    2. otherwise select the explicit local-directory pair in local/test;
+    3. use process-memory storage only for credential-free local/test operation.
+
+    Side effects if changes:
+    - API, workers, and guarded maintenance commands share this sole selector;
+    - artifact durability and raw-download availability depend on the result.
+    """
+
+    configured = (
+        settings.oss_endpoint,
+        settings.oss_bucket,
+        settings.oss_access_key_id,
+        settings.oss_access_key_secret,
+    )
+    if all(value is not None for value in configured):
+        return OssArtifactStore(
+            endpoint=str(settings.oss_endpoint or ""),
+            bucket_name=settings.oss_bucket or "",
+            access_key_id=settings.oss_access_key_id or "",
+            access_key_secret=(
+                settings.oss_access_key_secret.get_secret_value()
+                if settings.oss_access_key_secret is not None
+                else ""
+            ),
+            prefix=settings.oss_prefix,
+        )
+    if (
+        settings.local_artifact_directory is not None
+        and settings.local_artifact_base_url is not None
+    ):
+        return LocalDirectoryArtifactStore(
+            directory=settings.local_artifact_directory,
+            base_url=str(settings.local_artifact_base_url),
+            prefix=settings.oss_prefix,
+        )
+    if settings.environment in {
+        RuntimeEnvironment.DEVELOPMENT,
+        RuntimeEnvironment.TEST,
+    }:
+        return InMemoryArtifactStore()
+    raise ValueError("private OSS configuration is required outside development/test")

keble_scraper_api/errors.py

@@ -0,0 +1,25 @@
+"""Typed sanitized scraper application failures."""
+
+from keble_scraper_contract import ScrapeLocalFailureCode
+
+
+ScraperErrorCode = ScrapeLocalFailureCode
+
+
+class ScraperDomainError(Exception):
+    """Sanitized typed failure crossing application and HTTP boundaries.
+
+    Side effects if changes:
+    - worker outcome conversion and FastAPI exception responses.
+    """
+
+    def __init__(self, *, code: ScraperErrorCode, message: str) -> None:
+        """Initialize one safe code/message pair without transport details.
+
+        Side effects if changes:
+        - every raised scraper domain failure serializes through this value.
+        """
+
+        super().__init__(message)
+        self.code = code
+        self.message = message