keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/application/admin.py

@@ -0,0 +1,285 @@
+"""Authenticated proxy, artifact-download, and cleanup admin services."""
+
+from __future__ import annotations
+
+from uuid import UUID, uuid4
+
+from keble_scraper_contract import (
+    OperatorSession,
+    ProxyCandidateCreate,
+    ProxyCandidateSource,
+    ProxyCandidateView,
+    RouteBudgetView,
+    ScrapeArtifactDownload,
+    ScrapeArtifactPin,
+    ScrapeArtifactRef,
+)
+
+from keble_scraper_api.application.artifacts import ArtifactService
+from keble_scraper_api.application.jobs import AuditService, MaintenanceService
+from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
+from keble_scraper_api.models import AuditAction
+from keble_scraper_api.protocols import ProxyRepositoryProtocol, ProxyRouterProtocol
+from keble_scraper_api.settings import ScraperSettings
+
+
+def require_mutator(*, actor: OperatorSession) -> None:
+    """Enforce MEMBER read-only and ADMIN/SUPADMIN full parity.
+
+    Side effects if changes:
+    - every operations-console mutation and raw artifact download.
+    """
+
+    if not actor.can_mutate():
+        raise ScraperDomainError(
+            code=ScraperErrorCode.AUTH_FORBIDDEN,
+            message="operator role does not permit mutations",
+        )
+
+
+class ProxyAdminService:
+    """Manage quarantined candidates and credential-free route budgets.
+
+    Side effects if changes:
+    - future experimental egress, spend policy views, and audit records.
+    """
+
+    def __init__(
+        self,
+        *,
+        repository: ProxyRepositoryProtocol,
+        router: ProxyRouterProtocol,
+        audit: AuditService,
+        settings: ScraperSettings,
+    ) -> None:
+        """Bind proxy persistence/router, audit, and feed allowlist policy.
+
+        Side effects if changes:
+        - all proxy admin endpoints.
+        """
+
+        self._repository = repository
+        self._router = router
+        self._audit = audit
+        self._settings = settings
+
+    async def list_candidates(self, *, limit: int) -> tuple[ProxyCandidateView, ...]:
+        """Return bounded credential-free candidate views.
+
+        Side effects if changes:
+        - proxy dashboard Mongo reads.
+        """
+
+        return await self._repository.list_candidates(limit=limit)
+
+    async def import_candidate(
+        self,
+        *,
+        command: ProxyCandidateCreate,
+        actor: OperatorSession,
+        request_id: str,
+    ) -> ProxyCandidateView:
+        """Validate provenance, quarantine one encrypted candidate, and audit.
+
+        Side effects if changes:
+        - encrypted proxy Mongo write and potential future experiment egress.
+        """
+
+        require_mutator(actor=actor)
+        if command.source is ProxyCandidateSource.ALLOWLISTED_FEED:
+            allowlist = set(self._settings.public_proxy_feed_allowlist)
+            if command.source_reference not in allowlist:
+                raise ScraperDomainError(
+                    code=ScraperErrorCode.AUTH_FORBIDDEN,
+                    message="proxy feed is not allowlisted",
+                )
+        candidate = await self._repository.import_candidate(command=command)
+        await self._audit.record(
+            actor_id=actor.user_id,
+            actor_role=actor.role.value,
+            action=AuditAction.PROXY_IMPORT,
+            target_identity=str(candidate.candidate_id),
+            reason=f"proxy candidate imported from {command.source.value}",
+            request_id=request_id,
+        )
+        return candidate
+
+    async def budgets(self) -> tuple[RouteBudgetView, ...]:
+        """Return current route spend/rate gates.
+
+        Side effects if changes:
+        - cost dashboard Redis reads.
+        """
+
+        return await self._router.budgets()
+
+
+class ArtifactAdminService:
+    """Authorize, sign, and audit private artifact downloads.
+
+    Side effects if changes:
+    - raw object access and immutable audit evidence.
+    """
+
+    def __init__(
+        self,
+        *,
+        artifacts: ArtifactService,
+        audit: AuditService,
+        settings: ScraperSettings,
+    ) -> None:
+        """Bind artifact storage, audit, and signed-URL duration.
+
+        Side effects if changes:
+        - exact raw-download endpoint behavior.
+        """
+
+        self._artifacts = artifacts
+        self._audit = audit
+        self._settings = settings
+
+    async def download(
+        self,
+        *,
+        artifact_id: UUID,
+        actor: OperatorSession,
+        request_id: str,
+        reason: str,
+    ) -> ScrapeArtifactDownload:
+        """Issue one short-lived private download only to a mutating role.
+
+        Side effects if changes:
+        - signed OSS access and raw-download audit write.
+        """
+
+        require_mutator(actor=actor)
+        audit_id = uuid4()
+        download = await self._artifacts.download(
+            artifact_id=artifact_id,
+            expires_seconds=self._settings.artifact_download_seconds,
+            audit_id=audit_id,
+        )
+        await self._audit.record(
+            actor_id=actor.user_id,
+            actor_role=actor.role.value,
+            action=AuditAction.ARTIFACT_DOWNLOAD,
+            target_identity=str(artifact_id),
+            reason=reason,
+            request_id=request_id,
+            audit_id=audit_id,
+        )
+        return download
+
+    async def download_service(
+        self,
+        *,
+        artifact_id: UUID,
+        caller_identity: str,
+        request_id: str,
+        reason: str,
+    ) -> ScrapeArtifactDownload:
+        """Issue and audit one private download for the Shopify service audience.
+
+        Side effects if changes:
+        - signed OSS access and immutable service audit write.
+        """
+
+        audit_id = uuid4()
+        download = await self._artifacts.download(
+            artifact_id=artifact_id,
+            expires_seconds=self._settings.artifact_download_seconds,
+            audit_id=audit_id,
+        )
+        await self._audit.record(
+            actor_id=caller_identity,
+            actor_role="SERVICE",
+            action=AuditAction.ARTIFACT_DOWNLOAD,
+            target_identity=str(artifact_id),
+            reason=reason,
+            request_id=request_id,
+            audit_id=audit_id,
+        )
+        return download
+
+    async def get_service(self, *, artifact_id: UUID) -> ScrapeArtifactRef:
+        """Return safe artifact metadata to the authenticated Shopify service.
+
+        Side effects if changes:
+        - scraper Mongo metadata read; no object access is granted.
+        """
+
+        return await self._artifacts.get(artifact_id=artifact_id)
+
+    async def pin_service(
+        self,
+        *,
+        pin: ScrapeArtifactPin,
+        caller_identity: str,
+        request_id: str,
+    ) -> None:
+        """Synchronize and audit one version-fenced Shopify retention pin.
+
+        Steps:
+        1. Apply the retention command through the ACID artifact repository.
+        2. Record the accepted command revision and caller without secret data.
+
+        Side effects if changes:
+        - artifact reference count and orphan eligibility;
+        - immutable service audit evidence.
+        """
+
+        await self._artifacts.pin(pin=pin)
+        await self._audit.record(
+            actor_id=caller_identity,
+            actor_role="SERVICE",
+            action=AuditAction.ARTIFACT_PIN,
+            target_identity=str(pin.artifact_id),
+            reason=(
+                f"artifact retention revision {pin.retention_revision} "
+                f"synchronized by {pin.owner_identity}"
+            ),
+            request_id=request_id,
+        )
+
+
+class MaintenanceAdminService:
+    """Authorize and audit bounded manual cleanup runs.
+
+    Side effects if changes:
+    - lease recovery, cache/pin cleanup, object deletion, and task publication.
+    """
+
+    def __init__(self, *, maintenance: MaintenanceService, audit: AuditService) -> None:
+        """Bind maintenance engine and immutable audit service.
+
+        Side effects if changes:
+        - manual cleanup endpoint.
+        """
+
+        self._maintenance = maintenance
+        self._audit = audit
+
+    async def run(
+        self,
+        *,
+        actor: OperatorSession,
+        request_id: str,
+        reason: str,
+    ) -> dict[str, int]:
+        """Execute one bounded cleanup and record its operator identity.
+
+        Side effects if changes:
+        - persistent cleanup and audit history.
+        """
+
+        require_mutator(actor=actor)
+        result = await self._maintenance.run()
+        await self._audit.record(
+            actor_id=actor.user_id,
+            actor_role=actor.role.value,
+            action=AuditAction.CLEANUP_RUN,
+            target_identity="scraper-maintenance",
+            reason=reason,
+            request_id=request_id,
+        )
+        return result

keble_scraper_api/application/artifacts.py

@@ -0,0 +1,393 @@
+"""Artifact capture, sanitization, content identity, and download services."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from datetime import timedelta
+from uuid import UUID, uuid4, uuid5
+
+import nh3
+
+from keble_scraper_contract import (
+    ScrapeArtifactDownload,
+    ScrapeArtifactKind,
+    ScrapeArtifactPin,
+    ScrapeArtifactRef,
+    ScrapeCaptureMode,
+    ScrapeRetentionClass,
+    utc_now,
+)
+
+from keble_scraper_api.application.fetching import FetchResult
+from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
+from keble_scraper_api.models import ArtifactBatchLookup
+from keble_scraper_api.protocols import (
+    ArtifactRepositoryProtocol,
+    ArtifactStoreProtocol,
+)
+
+_ARTIFACT_NAMESPACE = UUID("2dc62d8c-619b-4bba-81b1-6fba8854e84e")
+_CHARSET_PATTERN = re.compile(
+    r"charset\s*=\s*[\"']?([A-Za-z0-9._-]+)",
+    flags=re.IGNORECASE,
+)
+_SUPPORTED_HTML_ENCODINGS = {
+    "utf-8": "utf-8",
+    "utf8": "utf-8",
+    "gbk": "gb18030",
+    "gb2312": "gb18030",
+    "gb18030": "gb18030",
+    "shift-jis": "shift_jis",
+    "shift_jis": "shift_jis",
+    "sjis": "shift_jis",
+    "euc-jp": "euc_jp",
+    "euc_jp": "euc_jp",
+    "iso-8859-1": "windows-1252",
+    "latin1": "windows-1252",
+    "windows-1252": "windows-1252",
+}
+
+
+class ArtifactService:
+    """Convert successful bounded fetches into private content-addressed artifacts.
+
+    Steps:
+    1. transform typed fetch results into capture-mode-specific bytes;
+    2. write content-addressed bytes before publishing metadata;
+    3. resolve, download, pin, and clean artifacts through owner protocols.
+
+    Side effects if changes:
+    - OSS writes, sanitizer output, Mongo metadata, and Shopify source evidence.
+    """
+
+    def __init__(
+        self,
+        *,
+        repository: ArtifactRepositoryProtocol,
+        store: ArtifactStoreProtocol,
+    ) -> None:
+        """Bind metadata and binary stores behind independent protocols.
+
+        Side effects if changes:
+        - all artifact creation and download paths.
+        """
+
+        self._repository = repository
+        self._store = store
+
+    async def capture(
+        self,
+        *,
+        result: FetchResult,
+        capture_mode: ScrapeCaptureMode,
+        retention_class: ScrapeRetentionClass,
+    ) -> tuple[ScrapeArtifactRef, ...]:
+        """Create exactly the artifacts required by one bounded capture mode.
+
+        Steps:
+        1. preserve metadata-only and empty/304 outcomes without object writes;
+        2. always hash/store raw body when requested;
+        3. sanitize HTML into a distinct immutable artifact without scripts,
+           style blocks, stylesheet links, event attributes, or inline CSS.
+
+        Side effects if changes:
+        - private object count, retention, and offline reprocessing fidelity.
+        """
+
+        if capture_mode is ScrapeCaptureMode.METADATA_ONLY or not result.body:
+            return ()
+        if capture_mode is ScrapeCaptureMode.RAW_BODY:
+            return (
+                await self._store_one(
+                    content=result.body,
+                    content_type=result.content_type or "application/octet-stream",
+                    kind=ScrapeArtifactKind.RAW_BODY,
+                    result=result,
+                    retention_class=retention_class,
+                ),
+            )
+        if result.content_type is None or not result.content_type.startswith(
+            "text/html"
+        ):
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONTENT_TYPE_REJECTED,
+                message="HTML capture mode requires an HTML response",
+            )
+        sanitized = self._sanitize_html(
+            content=result.body,
+            content_type=result.content_type,
+        )
+        if capture_mode is ScrapeCaptureMode.RENDERED_HTML:
+            return (
+                await self._store_one(
+                    content=sanitized,
+                    content_type="text/html; charset=utf-8",
+                    kind=ScrapeArtifactKind.RENDERED_HTML,
+                    result=result,
+                    retention_class=retention_class,
+                ),
+            )
+        raw = await self._store_one(
+            content=result.body,
+            content_type=result.content_type,
+            kind=ScrapeArtifactKind.RAW_BODY,
+            result=result,
+            retention_class=retention_class,
+        )
+        clean = await self._store_one(
+            content=sanitized,
+            content_type="text/html; charset=utf-8",
+            kind=ScrapeArtifactKind.SANITIZED_HTML,
+            result=result,
+            retention_class=retention_class,
+        )
+        return raw, clean
+
+    async def download(
+        self,
+        *,
+        artifact_id: UUID,
+        expires_seconds: int,
+        audit_id: UUID,
+    ) -> ScrapeArtifactDownload:
+        """Resolve metadata then issue an audit-correlated object signature.
+
+        Side effects if changes:
+        - raw artifact access; callers must append the matching audit event.
+        """
+
+        await self._repository.get(artifact_id=artifact_id)
+        object_key = await self._repository.object_key(artifact_id=artifact_id)
+        return await self._store.signed_download(
+            artifact_id=artifact_id,
+            object_key=object_key,
+            expires_seconds=expires_seconds,
+            audit_id=audit_id,
+        )
+
+    async def get(self, *, artifact_id: UUID) -> ScrapeArtifactRef:
+        """Return safe private-object metadata without issuing object access.
+
+        Side effects if changes:
+        - Shopify polling recovery and artifact hash/trust validation.
+        """
+
+        return await self._repository.get(artifact_id=artifact_id)
+
+    async def get_many_existing(
+        self,
+        *,
+        artifact_ids: list[UUID],
+    ) -> ArtifactBatchLookup:
+        """Resolve ordered cache artifacts while preserving explicit absence.
+
+        Steps:
+        1. delegate one primary-key batch read to the repository;
+        2. preserve cache order for existing artifacts;
+        3. return missing identities without exception laundering.
+
+        Side effects if changes:
+        - cached job completion, invalidation, and callback artifact order.
+        """
+
+        return await self._repository.get_many_existing(
+            artifact_ids=artifact_ids,
+        )
+
+    async def pin(self, *, pin: ScrapeArtifactPin) -> None:
+        """Create or strengthen one downstream artifact retention reference.
+
+        Side effects if changes:
+        - Mongo pin/reference count and orphan cleanup eligibility.
+        """
+
+        await self._repository.pin(pin=pin)
+
+    async def release_expired_pins(self) -> int:
+        """Release due downstream pins before orphan selection.
+
+        Side effects if changes:
+        - artifact reference counts and retention cleanup.
+        """
+
+        return await self._repository.release_expired_pins(now=utc_now())
+
+    async def cleanup_orphans(
+        self,
+        *,
+        limit: int = 1_000,
+        lease_seconds: int = 300,
+    ) -> int:
+        """Lease confirmed orphans, delete objects, then delete leased metadata.
+
+        Steps:
+        1. atomically lease only expired, unreferenced metadata;
+        2. let an object-store failure escape for supervision while its lease expires;
+        3. remove metadata only when the same lease still owns each row.
+
+        Side effects if changes:
+        - irreversible object removal and storage-cost reconciliation.
+        """
+
+        lease_id = uuid4()
+        object_keys = await self._repository.lease_orphan_object_keys(
+            now=utc_now(),
+            limit=limit,
+            lease_id=lease_id,
+            lease_seconds=lease_seconds,
+        )
+        await self._store.delete_many(object_keys=object_keys)
+        return await self._repository.complete_orphan_deletion(
+            object_keys=object_keys,
+            lease_id=lease_id,
+        )
+
+    async def _store_one(
+        self,
+        *,
+        content: bytes,
+        content_type: str,
+        kind: ScrapeArtifactKind,
+        result: FetchResult,
+        retention_class: ScrapeRetentionClass,
+    ) -> ScrapeArtifactRef:
+        """Store/register one deterministic content-plus-kind identity.
+
+        Steps:
+        1. derive immutable content and artifact identities plus retention metadata;
+        2. durably write/verify content-addressed bytes before visibility;
+        3. register metadata only after the object key matches the expected namespace.
+
+        Side effects if changes:
+        - OSS bytes and content metadata deduplication;
+        - metadata must never point at an object write that did not complete.
+        """
+
+        content_hash = hashlib.sha256(content).hexdigest()
+        artifact_id = uuid5(_ARTIFACT_NAMESPACE, f"{content_hash}:{kind.value}")
+        now = utc_now()
+        expires_at = {
+            ScrapeRetentionClass.TRANSIENT_24H: now + timedelta(hours=24),
+            ScrapeRetentionClass.CACHE_7D: now + timedelta(days=7),
+            ScrapeRetentionClass.REFERENCED_ARCHIVE: now + timedelta(days=365),
+        }[retention_class]
+        artifact = ScrapeArtifactRef(
+            artifact_id=artifact_id,
+            content_sha256=content_hash,
+            kind=kind,
+            content_type=content_type,
+            byte_length=len(content),
+            trust=result.trust,
+            retention_class=retention_class,
+            created_at=now,
+            expires_at=expires_at,
+        )
+        object_key = self._store.object_key(content_sha256=content_hash)
+        stored_key = await self._store.put_content_addressed(
+            content_sha256=content_hash,
+            content=content,
+            content_type=content_type,
+        )
+        if stored_key != object_key:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                message="artifact store returned an inconsistent content key",
+            )
+        return await self._repository.register(
+            artifact=artifact,
+            object_key=object_key,
+        )
+
+    @staticmethod
+    def _sanitize_html(*, content: bytes, content_type: str | None) -> bytes:
+        """Decode declared HTML correctly, then remove executable/style content.
+
+        Steps:
+        1. inspect the HTTP charset and the first HTML metadata window;
+        2. map only a finite reviewed charset family with UTF-8 fallback;
+        3. decode with replacement before sanitizing and emitting UTF-8 bytes.
+
+        Side effects if changes:
+        - sanitized artifact safety, international text fidelity, and downstream parsers.
+        """
+
+        encoding = ArtifactService._html_encoding(
+            content=content,
+            content_type=content_type,
+        )
+        text = content.decode(encoding, errors="replace")
+        sanitized = nh3.clean(
+            text,
+            tags={
+                "html",
+                "head",
+                "title",
+                "meta",
+                "body",
+                "main",
+                "section",
+                "article",
+                "div",
+                "span",
+                "p",
+                "h1",
+                "h2",
+                "h3",
+                "h4",
+                "h5",
+                "h6",
+                "ul",
+                "ol",
+                "li",
+                "a",
+                "img",
+                "picture",
+                "source",
+                "table",
+                "thead",
+                "tbody",
+                "tr",
+                "th",
+                "td",
+                "strong",
+                "em",
+                "br",
+            },
+            attributes={
+                "*": {"id", "class", "data-product-id", "data-product-handle"},
+                "a": {"href"},
+                "img": {"src", "srcset", "alt", "width", "height"},
+                "source": {"src", "srcset", "type"},
+                "meta": {"name", "property", "content"},
+            },
+            url_schemes={"http", "https"},
+            strip_comments=True,
+        )
+        return sanitized.encode()
+
+    @staticmethod
+    def _html_encoding(*, content: bytes, content_type: str | None) -> str:
+        """Resolve one safe supported HTML encoding from header then metadata.
+
+        Steps:
+        1. inspect the response Content-Type before document bytes;
+        2. inspect a Latin-1-safe metadata window when the header is silent;
+        3. normalize aliases through the finite supported encoding table.
+
+        Side effects if changes:
+        - GBK/Shift-JIS/Western HTML normalization and artifact hash identity.
+        """
+
+        sources = (
+            content_type or "",
+            content[:4_096].decode("latin-1", errors="ignore"),
+        )
+        for source in sources:
+            matched = _CHARSET_PATTERN.search(source)
+            if matched is not None:
+                normalized = matched.group(1).strip().lower()
+                supported = _SUPPORTED_HTML_ENCODINGS.get(normalized)
+                if supported is not None:
+                    return supported
+        return "utf-8"