keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/__init__.py

@@ -0,0 +1,3 @@
+"""Keble scraper API, workers, persistence, and operations composition."""
+
+__version__ = "0.2.0"

keble_scraper_api/api/dependencies.py

@@ -0,0 +1,100 @@
+"""Runtime container binding and shared request authorization dependencies."""
+
+from __future__ import annotations
+
+from uuid import uuid4
+
+from fastapi import Request
+
+from keble_scraper_contract import OperatorSession
+
+from keble_scraper_api.container import ScraperContainer
+
+
+class ContainerProvider:
+    """Bind one lifespan-created service graph to static route definitions.
+
+    Side effects if changes:
+    - every FastAPI dependency resolves process-owned services here.
+    """
+
+    def __init__(self) -> None:
+        """Initialize an intentionally unbound provider for OpenAPI creation.
+
+        Side effects if changes:
+        - pre-start request behavior remains fail-closed.
+        """
+
+        self._container: ScraperContainer | None = None
+
+    def bind(self, *, container: ScraperContainer) -> None:
+        """Install exactly one initialized process container.
+
+        Side effects if changes:
+        - route execution becomes available after lifespan startup.
+        """
+
+        if self._container is not None:
+            raise RuntimeError("scraper container is already bound")
+        self._container = container
+
+    def clear(self) -> None:
+        """Remove the binding before process resources close.
+
+        Side effects if changes:
+        - shutdown requests fail closed.
+        """
+
+        self._container = None
+
+    def require(self) -> ScraperContainer:
+        """Return initialized services or reject pre-start/post-shutdown access.
+
+        Side effects if changes:
+        - every API endpoint dependency.
+        """
+
+        if self._container is None:
+            raise RuntimeError("scraper runtime is not initialized")
+        return self._container
+
+
+def request_id(*, request: Request) -> str:
+    """Use caller correlation metadata or generate a server request identity.
+
+    Side effects if changes:
+    - audit correlation; never job/idempotency identity.
+    """
+
+    return request.headers.get("x-request-id") or uuid4().hex
+
+
+def authorize_service(*, request: Request, provider: ContainerProvider) -> str:
+    """Authenticate the single Shopify service audience with current/next keys.
+
+    Side effects if changes:
+    - job submit/read/cancel service access.
+    """
+
+    container = provider.require()
+    container.service_keys.authorize(
+        supplied=request.headers.get("x-service-key")
+    )
+    return container.settings.service_caller_identity
+
+
+async def authorize_operator(
+    *,
+    request: Request,
+    provider: ContainerProvider,
+) -> OperatorSession:
+    """Resolve a backend-managed encrypted HttpOnly operator session.
+
+    Side effects if changes:
+    - every admin read/mutation authorization decision.
+    """
+
+    container = provider.require()
+    return await container.auth.session_from_cookie(
+        token=request.cookies.get(container.settings.session_cookie_name)
+    )

keble_scraper_api/api/errors.py

@@ -0,0 +1,72 @@
+"""Sanitized FastAPI exception-to-status conversion."""
+
+from cryptography.fernet import InvalidToken
+from fastapi import Request
+from fastapi.responses import JSONResponse
+
+from keble_scraper_api.api.schemas import ErrorResponse
+from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
+
+
+async def scraper_error_handler(
+    request: Request,
+    error: ScraperDomainError,
+) -> JSONResponse:
+    """Map one typed domain failure without exposing transport or secret details.
+
+    Steps:
+    1. select a stable HTTP status from the finite local code;
+    2. serialize only the bounded safe code and message;
+    3. omit exception details, URLs, and credentials.
+
+    Side effects if changes:
+    - all service/admin HTTP error status and bodies.
+    """
+
+    status_by_code = {
+        ScraperErrorCode.AUTH_INVALID: 401,
+        ScraperErrorCode.AUTH_FORBIDDEN: 403,
+        ScraperErrorCode.NOT_FOUND: 404,
+        ScraperErrorCode.REVISION_CONFLICT: 409,
+        ScraperErrorCode.INVALID_TRANSITION: 409,
+        ScraperErrorCode.VALIDATION_ERROR: 422,
+        ScraperErrorCode.URL_DENIED: 422,
+        ScraperErrorCode.DNS_FAILED: 422,
+        ScraperErrorCode.RESPONSE_TOO_LARGE: 413,
+        ScraperErrorCode.CONTENT_TYPE_REJECTED: 415,
+        ScraperErrorCode.BUDGET_EXHAUSTED: 429,
+        ScraperErrorCode.ROUTE_UNAVAILABLE: 503,
+        ScraperErrorCode.CONFIGURATION_INVALID: 503,
+    }
+    status_code = status_by_code.get(error.code, 400)
+    body = ErrorResponse(code=error.code.value, message=error.message)
+    return JSONResponse(
+        status_code=status_code,
+        content=body.model_dump(mode="json", by_alias=True),
+    )
+
+
+async def invalid_session_handler(
+    request: Request,
+    error: InvalidToken,
+) -> JSONResponse:
+    """Map authenticated-decryption rejection to the public auth contract.
+
+    Steps:
+    1. accept only cryptography's explicit invalid-token signal;
+    2. discard all opaque token/error detail;
+    3. return the same finite 401 body as a missing/expired session.
+
+    Side effects if changes:
+    - malformed or tampered admin cookies remain a safe authentication denial.
+    """
+
+    del request, error
+    body = ErrorResponse(
+        code=ScraperErrorCode.AUTH_INVALID.value,
+        message="operator session is invalid",
+    )
+    return JSONResponse(
+        status_code=401,
+        content=body.model_dump(mode="json", by_alias=True),
+    )

keble_scraper_api/api/router.py

@@ -0,0 +1,473 @@
+"""Thin authenticated FastAPI routes over canonical application services."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from uuid import UUID
+
+from fastapi import APIRouter, Query, Request, Response, status
+
+from keble_scraper_contract import (
+    GoogleLoginCommand,
+    OperatorAuthConfig,
+    OperatorSession,
+    ProxyCandidateCreate,
+    ProxyCandidateView,
+    RouteBudgetView,
+    ScrapeArtifactDownload,
+    ScrapeArtifactPin,
+    ScrapeArtifactRef,
+    ScrapeJobAccepted,
+    ScrapeJobCancel,
+    ScrapeJobCreate,
+    ScrapeJobView,
+)
+
+from keble_scraper_api.api.dependencies import (
+    ContainerProvider,
+    authorize_operator,
+    authorize_service,
+    request_id,
+)
+from keble_scraper_api.api.schemas import AdminReason
+from keble_scraper_api.application.admin import require_mutator
+from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
+
+
+def create_router(*, provider: ContainerProvider) -> APIRouter:
+    """Build service/auth/admin routes against one lifespan provider.
+
+    Side effects if changes:
+    - public OpenAPI and all scraper HTTP behavior.
+    """
+
+    router = APIRouter()
+
+    @router.get("/health", operation_id="scraper_health")
+    async def health() -> dict[str, str]:
+        """Report process liveness without privileged dependency calls.
+
+        Side effects if changes:
+        - container liveness probes.
+        """
+
+        return {"status": "ok"}
+
+    @router.get("/ready", operation_id="scraper_ready")
+    async def ready() -> dict[str, str]:
+        """Verify Mongo/Redis readiness without vendor calls.
+
+        Side effects if changes:
+        - deployment rollout and rollback decisions.
+        """
+
+        return await provider.require().readiness()
+
+    @router.post(
+        "/api/v1/auth/google",
+        response_model=OperatorSession,
+        operation_id="scraper_google_login",
+    )
+    async def google_login(
+        command: GoogleLoginCommand,
+        response: Response,
+    ) -> OperatorSession:
+        """Exchange Google through Truth and set the encrypted HttpOnly cookie.
+
+        Side effects if changes:
+        - external Truth auth and browser session cookie.
+        """
+
+        container = provider.require()
+        session, cookie = await container.auth.google_login(
+            credential=command.credential
+        )
+        _set_session_cookie(response=response, session=session, cookie=cookie, provider=provider)
+        return session
+
+    @router.get(
+        "/api/v1/auth/config",
+        response_model=OperatorAuthConfig,
+        operation_id="scraper_auth_config",
+    )
+    async def auth_config() -> OperatorAuthConfig:
+        """Return safe sign-in capabilities without credentials or secrets.
+
+        Side effects if changes:
+        - frontend Google/dev sign-in availability.
+        """
+
+        settings = provider.require().settings
+        return OperatorAuthConfig(
+            google_client_id=settings.google_client_id,
+            dev_auth_enabled=(
+                settings.environment.value == "development"
+                and settings.dev_auth_enabled
+            ),
+        )
+
+    @router.post(
+        "/api/v1/auth/dev",
+        response_model=OperatorSession,
+        operation_id="scraper_dev_login",
+    )
+    async def dev_login(response: Response) -> OperatorSession:
+        """Set a visibly local session under the explicit development-only gate.
+
+        Side effects if changes:
+        - local browser session cookie.
+        """
+
+        container = provider.require()
+        session, cookie = container.auth.dev_login()
+        _set_session_cookie(response=response, session=session, cookie=cookie, provider=provider)
+        return session
+
+    @router.get(
+        "/api/v1/auth/session",
+        response_model=OperatorSession,
+        operation_id="scraper_auth_session",
+    )
+    async def auth_session(request: Request) -> OperatorSession:
+        """Return the safe session view without bearer/cookie contents.
+
+        Side effects if changes:
+        - operations frontend bootstrap authorization.
+        """
+
+        return await authorize_operator(request=request, provider=provider)
+
+    @router.post(
+        "/api/v1/auth/logout",
+        status_code=status.HTTP_204_NO_CONTENT,
+        operation_id="scraper_logout",
+    )
+    async def logout(response: Response) -> None:
+        """Delete the backend-owned session cookie.
+
+        Side effects if changes:
+        - browser authentication state.
+        """
+
+        container = provider.require()
+        response.delete_cookie(
+            container.settings.session_cookie_name,
+            httponly=True,
+            samesite="strict",
+        )
+
+    @router.post(
+        "/api/v1/jobs",
+        response_model=ScrapeJobAccepted,
+        status_code=status.HTTP_202_ACCEPTED,
+        operation_id="create_scrape_job",
+    )
+    async def create_job(command: ScrapeJobCreate, request: Request) -> ScrapeJobAccepted:
+        """Create/reuse and asynchronously dispatch one service-owned job.
+
+        Side effects if changes:
+        - Mongo job/audit writes and RabbitMQ publication.
+        """
+
+        caller = authorize_service(request=request, provider=provider)
+        return await provider.require().jobs.create(
+            caller_identity=caller,
+            command=command,
+            request_id=request_id(request=request),
+        )
+
+    @router.get(
+        "/api/v1/jobs/{job_id}",
+        response_model=ScrapeJobView,
+        operation_id="get_scrape_job",
+    )
+    async def get_job(job_id: UUID, request: Request) -> ScrapeJobView:
+        """Read one service-authenticated job and verify caller ownership.
+
+        Side effects if changes:
+        - Shopify polling Mongo reads.
+        """
+
+        caller = authorize_service(request=request, provider=provider)
+        view = await provider.require().jobs.get(job_id=job_id)
+        if view.caller_identity != caller:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.NOT_FOUND,
+                message="scrape job was not found",
+            )
+        return view
+
+    @router.post(
+        "/api/v1/jobs/{job_id}:cancel",
+        response_model=ScrapeJobView,
+        operation_id="cancel_scrape_job",
+    )
+    async def cancel_job(
+        job_id: UUID,
+        command: ScrapeJobCancel,
+        request: Request,
+    ) -> ScrapeJobView:
+        """Optimistically cancel one caller-owned service job.
+
+        Side effects if changes:
+        - Mongo lifecycle and audit writes.
+        """
+
+        caller = authorize_service(request=request, provider=provider)
+        return await provider.require().jobs.cancel_service(
+            job_id=job_id,
+            command=command,
+            caller_identity=caller,
+            request_id=request_id(request=request),
+        )
+
+    @router.get(
+        "/api/v1/admin/jobs",
+        response_model=list[ScrapeJobView],
+        operation_id="admin_list_scrape_jobs",
+    )
+    async def admin_list_jobs(
+        request: Request,
+        limit: int = Query(default=50, ge=1, le=200),
+        before: datetime | None = None,
+    ) -> tuple[ScrapeJobView, ...]:
+        """List newest jobs for any authenticated operator role.
+
+        Side effects if changes:
+        - operations Mongo reads.
+        """
+
+        await authorize_operator(request=request, provider=provider)
+        return await provider.require().jobs.list_admin(limit=limit, before=before)
+
+    @router.post(
+        "/api/v1/admin/jobs",
+        response_model=ScrapeJobAccepted,
+        status_code=status.HTTP_202_ACCEPTED,
+        operation_id="admin_create_scrape_job",
+    )
+    async def admin_create_job(
+        command: ScrapeJobCreate,
+        request: Request,
+    ) -> ScrapeJobAccepted:
+        """Submit one job from the console using the operator as caller identity.
+
+        Side effects if changes:
+        - Mongo job/audit writes and RabbitMQ publication.
+        """
+
+        actor = await authorize_operator(request=request, provider=provider)
+        require_mutator(actor=actor)
+        return await provider.require().jobs.create(
+            caller_identity=f"operator:{actor.user_id}",
+            command=command,
+            request_id=request_id(request=request),
+            actor_role=actor.role.value,
+        )
+
+    @router.post(
+        "/api/v1/admin/jobs/{job_id}:cancel",
+        response_model=ScrapeJobView,
+        operation_id="admin_cancel_scrape_job",
+    )
+    async def admin_cancel_job(
+        job_id: UUID,
+        command: ScrapeJobCancel,
+        request: Request,
+    ) -> ScrapeJobView:
+        """Optimistically cancel one job through operator authorization.
+
+        Side effects if changes:
+        - Mongo lifecycle and audit writes.
+        """
+
+        actor = await authorize_operator(request=request, provider=provider)
+        return await provider.require().jobs.cancel(
+            job_id=job_id,
+            command=command,
+            actor=actor,
+            request_id=request_id(request=request),
+        )
+
+    @router.get(
+        "/api/v1/artifacts/{artifact_id}:download",
+        response_model=ScrapeArtifactDownload,
+        operation_id="download_scrape_artifact",
+    )
+    async def download_artifact(
+        artifact_id: UUID,
+        request: Request,
+        reason: str = Query(min_length=8, max_length=1_000),
+    ) -> ScrapeArtifactDownload:
+        """Issue one audited short-lived raw artifact download.
+
+        Side effects if changes:
+        - signed object access and audit write.
+        """
+
+        if request.headers.get("x-service-key") is not None:
+            caller = authorize_service(request=request, provider=provider)
+            return await provider.require().artifact_admin.download_service(
+                artifact_id=artifact_id,
+                caller_identity=caller,
+                request_id=request_id(request=request),
+                reason=reason,
+            )
+        actor = await authorize_operator(request=request, provider=provider)
+        return await provider.require().artifact_admin.download(
+            artifact_id=artifact_id,
+            actor=actor,
+            request_id=request_id(request=request),
+            reason=reason,
+        )
+
+    @router.post(
+        "/api/v1/artifacts/{artifact_id}:pin",
+        status_code=status.HTTP_204_NO_CONTENT,
+        operation_id="pin_scrape_artifact",
+    )
+    async def pin_artifact(
+        artifact_id: UUID,
+        command: ScrapeArtifactPin,
+        request: Request,
+    ) -> None:
+        """Synchronize one version-fenced service-owned retention pin.
+
+        Steps:
+        1. Reject a path/body artifact identity mismatch before authorization.
+        2. Authenticate the service and apply its ordered retention command.
+
+        Side effects if changes:
+        - Mongo pin/reference count and stale-delivery fencing;
+        - immutable service audit write.
+        """
+
+        if command.artifact_id != artifact_id:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.VALIDATION_ERROR,
+                message="artifact path identity does not match pin command",
+            )
+        caller = authorize_service(request=request, provider=provider)
+        await provider.require().artifact_admin.pin_service(
+            pin=command,
+            caller_identity=caller,
+            request_id=request_id(request=request),
+        )
+
+    @router.get(
+        "/api/v1/artifacts/{artifact_id}",
+        response_model=ScrapeArtifactRef,
+        operation_id="get_scrape_artifact",
+    )
+    async def get_artifact(artifact_id: UUID, request: Request) -> ScrapeArtifactRef:
+        """Return safe artifact metadata to the authenticated Shopify service.
+
+        Side effects if changes:
+        - scraper Mongo read for polling/callback recovery.
+        """
+
+        authorize_service(request=request, provider=provider)
+        return await provider.require().artifact_admin.get_service(
+            artifact_id=artifact_id
+        )
+
+    @router.get(
+        "/api/v1/admin/proxies",
+        response_model=list[ProxyCandidateView],
+        operation_id="admin_list_proxy_candidates",
+    )
+    async def list_proxies(
+        request: Request,
+        limit: int = Query(default=100, ge=1, le=500),
+    ) -> tuple[ProxyCandidateView, ...]:
+        """List quarantined proxy records without credentials.
+
+        Side effects if changes:
+        - proxy dashboard Mongo reads.
+        """
+
+        await authorize_operator(request=request, provider=provider)
+        return await provider.require().proxy_admin.list_candidates(limit=limit)
+
+    @router.post(
+        "/api/v1/admin/proxies",
+        response_model=ProxyCandidateView,
+        operation_id="admin_import_proxy_candidate",
+    )
+    async def import_proxy(
+        command: ProxyCandidateCreate,
+        request: Request,
+    ) -> ProxyCandidateView:
+        """Import an encrypted candidate into mandatory quarantine.
+
+        Side effects if changes:
+        - proxy Mongo/audit writes.
+        """
+
+        actor = await authorize_operator(request=request, provider=provider)
+        return await provider.require().proxy_admin.import_candidate(
+            command=command,
+            actor=actor,
+            request_id=request_id(request=request),
+        )
+
+    @router.get(
+        "/api/v1/admin/budgets",
+        response_model=list[RouteBudgetView],
+        operation_id="admin_list_route_budgets",
+    )
+    async def list_budgets(request: Request) -> tuple[RouteBudgetView, ...]:
+        """Return current route rate/spend gates to any operator.
+
+        Side effects if changes:
+        - Redis cost dashboard reads.
+        """
+
+        await authorize_operator(request=request, provider=provider)
+        return await provider.require().proxy_admin.budgets()
+
+    @router.post(
+        "/api/v1/admin/cleanup",
+        response_model=dict[str, int],
+        operation_id="admin_run_scraper_cleanup",
+    )
+    async def run_cleanup(command: AdminReason, request: Request) -> dict[str, int]:
+        """Run and audit one bounded manual maintenance pass.
+
+        Side effects if changes:
+        - Mongo/OSS cleanup and RabbitMQ redispatch.
+        """
+
+        actor = await authorize_operator(request=request, provider=provider)
+        return await provider.require().maintenance_admin.run(
+            actor=actor,
+            request_id=request_id(request=request),
+            reason=command.reason,
+        )
+
+    return router
+
+
+def _set_session_cookie(
+    *,
+    response: Response,
+    session: OperatorSession,
+    cookie: str,
+    provider: ContainerProvider,
+) -> None:
+    """Set one secure backend-managed session cookie and no browser token.
+
+    Side effects if changes:
+    - browser session persistence and CSRF/cookie security.
+    """
+
+    settings = provider.require().settings
+    response.set_cookie(
+        settings.session_cookie_name,
+        cookie,
+        max_age=max(1, int((session.expires_at - datetime.now(session.expires_at.tzinfo)).total_seconds())),
+        httponly=True,
+        secure=settings.environment.value in {"staging", "production"},
+        samesite="strict",
+        path="/",
+    )

keble_scraper_api/api/schemas.py

@@ -0,0 +1,26 @@
+"""FastAPI-only bounded command and error schemas."""
+
+from pydantic import Field
+
+from keble_scraper_contract import ScraperContractModel
+
+
+class AdminReason(ScraperContractModel):
+    """Audited reason required for one sensitive operator action.
+
+    Side effects if changes:
+    - cleanup/download audit evidence and frontend command forms.
+    """
+
+    reason: str = Field(min_length=8, max_length=1_000)
+
+
+class ErrorResponse(ScraperContractModel):
+    """Sanitized typed HTTP error body.
+
+    Side effects if changes:
+    - service clients and operations frontend error presentation.
+    """
+
+    code: str
+    message: str