keble-scraper-api 0.2.0__py3-none-any.whl

keble_scraper_api/infrastructure/artifact_store.py

@@ -0,0 +1,585 @@
+"""Private OSS, shared local-development, and in-memory artifact stores."""
+
+from __future__ import annotations
+
+import asyncio
+import fcntl
+import hashlib
+from collections.abc import Sequence
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from uuid import UUID, uuid4
+
+import oss2
+from pydantic import AnyHttpUrl
+
+from keble_scraper_contract import ScrapeArtifactDownload
+
+from keble_scraper_api.artifact_namespace import (
+    ArtifactStoreBackend,
+    ArtifactStoreIdentity,
+)
+from keble_scraper_api.errors import ScraperDomainError, ScraperErrorCode
+
+_AUDIT_QUERY_PARAMETER = "x-keble-audit-id"
+_CONTENT_SHA256_METADATA_HEADER = "x-oss-meta-keble-content-sha256"
+_NAMESPACE_OWNER_DIRECTORY = ".keble-namespace-owners"
+
+
+def _namespace_owner_filename(*, owner: str) -> str:
+    """Return a stable non-reversible filename for one configured database owner.
+
+    Steps:
+    1. hash the validated non-secret owner identity with SHA-256;
+    2. append a fixed canary suffix for deterministic listing.
+
+    Side effects if changes:
+    - OSS and local-directory stores must derive identical owner claim names;
+    - changing the derivation requires an explicit namespace migration.
+    """
+
+    if not owner or owner != owner.strip():
+        raise ValueError("artifact namespace owner must be a non-blank exact identity")
+    return f"{hashlib.sha256(owner.encode()).hexdigest()}.owner"
+
+
+def _validate_namespace_owner_keys(
+    *,
+    expected_key: str,
+    keys: Sequence[str],
+) -> None:
+    """Reject any namespace claim set not owned solely by the expected database.
+
+    Steps:
+    1. normalize the finite listed claim keys;
+    2. accept an empty pre-claim set or the one exact expected key;
+    3. reject every foreign or multiple-owner state.
+
+    Side effects if changes:
+    - two databases must never both authorize destructive cleanup in one prefix.
+    """
+
+    normalized = tuple(sorted(set(keys)))
+    if normalized not in {(), (expected_key,)}:
+        raise ScraperDomainError(
+            code=ScraperErrorCode.CONFIGURATION_INVALID,
+            message="artifact namespace is already claimed by another database owner",
+        )
+
+
+class OssArtifactStore:
+    """Store content-addressed objects in one private Alibaba OSS prefix.
+
+    Side effects if changes:
+    - raw/sanitized artifact durability, signed downloads, and storage deletion.
+    """
+
+    def __init__(
+        self,
+        *,
+        endpoint: str,
+        bucket_name: str,
+        access_key_id: str,
+        access_key_secret: str,
+        prefix: str,
+    ) -> None:
+        """Construct one private bucket client from secret-system values.
+
+        Steps:
+        1. build the authenticated OSS bucket client;
+        2. retain normalized non-secret endpoint/bucket/prefix identity fields.
+
+        Side effects if changes:
+        - external OSS connections and object namespace ownership.
+        """
+
+        auth = oss2.Auth(access_key_id, access_key_secret)
+        self._endpoint = endpoint.rstrip("/")
+        self._bucket_name = bucket_name
+        self._bucket = oss2.Bucket(auth, self._endpoint, bucket_name)
+        self._prefix = prefix.strip("/")
+
+    def namespace_identity(self) -> ArtifactStoreIdentity:
+        """Return the exact OSS endpoint, bucket, and prefix identity.
+
+        Steps:
+        1. expose normalized non-secret OSS location fields.
+
+        Side effects if changes:
+        - Mongo marker equality fences endpoint/bucket/prefix drift.
+        """
+
+        return ArtifactStoreIdentity(
+            backend=ArtifactStoreBackend.OSS,
+            endpoint=self._endpoint,
+            bucket=self._bucket_name,
+            prefix=self._prefix,
+        )
+
+    async def ensure_namespace_owner(self, *, owner: str) -> None:
+        """Claim one OSS prefix for one stable Mongo deployment identity.
+
+        Steps:
+        1. list the bounded owner-canary prefix and reject foreign claims;
+        2. idempotently write this owner's deterministic canary;
+        3. list and read back the claim before authorizing startup.
+
+        Side effects if changes:
+        - performs bounded OSS LIST, PUT, and GET operations at process startup;
+        - cross-database orphan deletion is blocked before serving work.
+        """
+
+        owner_prefix = f"{self._prefix}/{_NAMESPACE_OWNER_DIRECTORY}/"
+        expected_key = owner_prefix + _namespace_owner_filename(owner=owner)
+        before = await asyncio.to_thread(
+            self._list_namespace_owner_keys,
+            owner_prefix=owner_prefix,
+        )
+        _validate_namespace_owner_keys(expected_key=expected_key, keys=before)
+        await asyncio.to_thread(
+            self._bucket.put_object,
+            expected_key,
+            owner.encode(),
+            headers={"Content-Type": "text/plain; charset=utf-8"},
+        )
+        after = await asyncio.to_thread(
+            self._list_namespace_owner_keys,
+            owner_prefix=owner_prefix,
+        )
+        _validate_namespace_owner_keys(expected_key=expected_key, keys=after)
+        response = await asyncio.to_thread(self._bucket.get_object, expected_key)
+        recorded_owner = await asyncio.to_thread(response.read)
+        if recorded_owner != owner.encode():
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                message="artifact namespace owner canary content differs",
+            )
+
+    def _list_namespace_owner_keys(self, *, owner_prefix: str) -> tuple[str, ...]:
+        """List at most two OSS claim keys for split-owner detection.
+
+        Steps:
+        1. request the bounded owner prefix from OSS;
+        2. return exact object keys without response payload content.
+
+        Side effects if changes:
+        - startup LIST request count and cross-database collision detection.
+        """
+
+        result = self._bucket.list_objects_v2(prefix=owner_prefix, max_keys=2)
+        return tuple(item.key for item in result.object_list)
+
+    def object_key(self, *, content_sha256: str) -> str:
+        """Derive the prefix-scoped content key without external I/O.
+
+        Side effects if changes:
+        - Mongo object references and every OSS write/delete/sign operation.
+        """
+
+        return f"{self._prefix}/{content_sha256[:2]}/{content_sha256}"
+
+    async def put_content_addressed(
+        self,
+        *,
+        content_sha256: str,
+        content: bytes,
+        content_type: str,
+    ) -> str:
+        """Idempotently write one hash-keyed private object.
+
+        Side effects if changes:
+        - OSS bytes and later artifact metadata references.
+        """
+
+        object_key = self.object_key(content_sha256=content_sha256)
+        await asyncio.to_thread(
+            self._bucket.put_object,
+            object_key,
+            content,
+            headers=self.write_headers(
+                content_sha256=content_sha256,
+                content_type=content_type,
+            ),
+        )
+        return object_key
+
+    @staticmethod
+    def write_headers(*, content_sha256: str, content_type: str) -> dict[str, str]:
+        """Build object metadata that makes content identity auditable by HEAD.
+
+        Steps:
+        1. retain the transport content type used by current readers;
+        2. persist the canonical SHA-256 as private user metadata.
+
+        Side effects if changes:
+        - OSS metadata and rollout integrity probes for newly written objects.
+        """
+
+        return {
+            "Content-Type": content_type,
+            _CONTENT_SHA256_METADATA_HEADER: content_sha256,
+        }
+
+    async def signed_download(
+        self,
+        *,
+        artifact_id: UUID,
+        object_key: str,
+        expires_seconds: int,
+        audit_id: UUID,
+    ) -> ScrapeArtifactDownload:
+        """Generate one short-lived GET signature with an OSS-log audit identity.
+
+        Side effects if changes:
+        - private raw artifact access duration and exact access-log correlation.
+        """
+
+        url = await asyncio.to_thread(
+            self._bucket.sign_url,
+            "GET",
+            object_key,
+            expires_seconds,
+            params={_AUDIT_QUERY_PARAMETER: str(audit_id)},
+        )
+        return ScrapeArtifactDownload(
+            artifact_id=artifact_id,
+            download_url=AnyHttpUrl(url),
+            expires_at=datetime.now(UTC) + timedelta(seconds=expires_seconds),
+        )
+
+    async def delete_many(self, *, object_keys: Sequence[str]) -> None:
+        """Delete a bounded set of confirmed orphan objects.
+
+        Side effects if changes:
+        - irreversible OSS storage removal after repository eligibility checks.
+        """
+
+        if object_keys:
+            await asyncio.to_thread(
+                self._bucket.batch_delete_objects, list(object_keys)
+            )
+
+
+class LocalDirectoryArtifactStore:
+    """Share content-addressed development artifacts across local processes.
+
+    Side effects if changes:
+    - local scraper API/worker artifact interoperability and cleanup paths;
+    - localhost artifact URLs consumed by `ScraperApiClient.read_verified`.
+    """
+
+    def __init__(
+        self,
+        *,
+        directory: str,
+        base_url: str,
+        prefix: str,
+    ) -> None:
+        """Bind one isolated directory and its separately served localhost URL.
+
+        Steps:
+        1. normalize the configured directory and content-address prefix;
+        2. reject path-like prefixes that could escape the owned directory;
+        3. retain a URL root served by the local canary file server.
+
+        Side effects if changes:
+        - local filesystem namespace and development artifact download routing.
+        """
+
+        normalized_prefix = prefix.strip("/")
+        prefix_parts = normalized_prefix.split("/")
+        if not normalized_prefix or any(
+            part in {"", ".", ".."} for part in prefix_parts
+        ):
+            raise ValueError("local artifact prefix must contain safe path segments")
+        self._directory = Path(directory).expanduser().resolve()
+        self._base_url = base_url.rstrip("/")
+        self._prefix = normalized_prefix
+
+    def namespace_identity(self) -> ArtifactStoreIdentity:
+        """Return the exact shared directory and content prefix identity.
+
+        Steps:
+        1. expose the resolved directory and normalized key prefix.
+
+        Side effects if changes:
+        - Mongo marker equality fences directory/backend swaps.
+        """
+
+        return ArtifactStoreIdentity(
+            backend=ArtifactStoreBackend.LOCAL_DIRECTORY,
+            directory=str(self._directory),
+            prefix=self._prefix,
+        )
+
+    async def ensure_namespace_owner(self, *, owner: str) -> None:
+        """Claim one local directory/prefix for one database identity.
+
+        Steps:
+        1. create the private owner-canary directory;
+        2. reject existing foreign owner files before writing this canary;
+        3. re-list and verify exact owner content after the atomic file write.
+
+        Side effects if changes:
+        - local API/worker processes share one ownership fence;
+        - cross-database cleanup cannot enter a shared directory prefix.
+        """
+
+        await asyncio.to_thread(self._ensure_namespace_owner_sync, owner=owner)
+
+    def _ensure_namespace_owner_sync(self, *, owner: str) -> None:
+        """Perform the local claim under one filesystem lock.
+
+        Steps:
+        1. lock the owner directory across local processes;
+        2. validate, write, revalidate, and read back the exact owner claim.
+
+        Side effects if changes:
+        - creates one lock file plus one stable owner canary under the prefix.
+        """
+
+        owner_directory = self._directory / self._prefix / _NAMESPACE_OWNER_DIRECTORY
+        owner_directory.mkdir(parents=True, exist_ok=True)
+        expected_path = owner_directory / _namespace_owner_filename(owner=owner)
+        lock_path = owner_directory / ".claim.lock"
+        with lock_path.open("a+b") as lock_file:
+            fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
+            before = tuple(
+                str(path)
+                for path in owner_directory.iterdir()
+                if path.is_file() and path.name != lock_path.name
+            )
+            _validate_namespace_owner_keys(
+                expected_key=str(expected_path),
+                keys=before,
+            )
+            expected_path.write_text(owner, encoding="utf-8")
+            after = tuple(
+                str(path)
+                for path in owner_directory.iterdir()
+                if path.is_file() and path.name != lock_path.name
+            )
+            _validate_namespace_owner_keys(
+                expected_key=str(expected_path),
+                keys=after,
+            )
+            if expected_path.read_text(encoding="utf-8") != owner:
+                raise ScraperDomainError(
+                    code=ScraperErrorCode.CONFIGURATION_INVALID,
+                    message="artifact namespace owner canary content differs",
+                )
+
+    def object_key(self, *, content_sha256: str) -> str:
+        """Derive one prefix-scoped relative object key from a canonical hash.
+
+        Side effects if changes:
+        - Mongo object references, local paths, and localhost download URLs.
+        """
+
+        if len(content_sha256) != 64 or any(
+            character not in "0123456789abcdef" for character in content_sha256
+        ):
+            raise ValueError("content_sha256 must be a lowercase SHA-256 digest")
+        return f"{self._prefix}/{content_sha256[:2]}/{content_sha256}"
+
+    async def put_content_addressed(
+        self,
+        *,
+        content_sha256: str,
+        content: bytes,
+        content_type: str,
+    ) -> str:
+        """Atomically write one hash-keyed object into the shared directory.
+
+        Steps:
+        1. derive and validate the owned relative key;
+        2. create only its parent directories;
+        3. replace the final object atomically with equivalent hashed bytes.
+
+        Side effects if changes:
+        - shared local artifact bytes visible to API and worker processes.
+        """
+
+        del content_type
+        object_key = self.object_key(content_sha256=content_sha256)
+        if hashlib.sha256(content).hexdigest() != content_sha256:
+            raise ValueError(
+                "local artifact content does not match its SHA-256 identity"
+            )
+        path = self._owned_path(object_key=object_key)
+        await asyncio.to_thread(path.parent.mkdir, parents=True, exist_ok=True)
+        temporary = path.with_name(f".{path.name}.{uuid4().hex}.tmp")
+        await asyncio.to_thread(temporary.write_bytes, content)
+        await asyncio.to_thread(temporary.replace, path)
+        return object_key
+
+    async def signed_download(
+        self,
+        *,
+        artifact_id: UUID,
+        object_key: str,
+        expires_seconds: int,
+        audit_id: UUID,
+    ) -> ScrapeArtifactDownload:
+        """Return the isolated localhost URL for one existing development object.
+
+        Steps:
+        1. validate that the persisted key remains inside the configured root;
+        2. require the content-addressed file to exist;
+        3. expose its local-server URL with the same audit query used by OSS.
+
+        Side effects if changes:
+        - development-only raw artifact access and local log reconciliation.
+        """
+
+        path = self._owned_path(object_key=object_key)
+        if not await asyncio.to_thread(path.is_file):
+            raise FileNotFoundError(object_key)
+        return ScrapeArtifactDownload(
+            artifact_id=artifact_id,
+            download_url=AnyHttpUrl(
+                f"{self._base_url}/{object_key}?{_AUDIT_QUERY_PARAMETER}={audit_id}"
+            ),
+            expires_at=datetime.now(UTC) + timedelta(seconds=expires_seconds),
+        )
+
+    async def delete_many(self, *, object_keys: Sequence[str]) -> None:
+        """Delete exact owned local objects while preserving unrelated files.
+
+        Side effects if changes:
+        - local maintenance cleanup and retained canary evidence.
+        """
+
+        for object_key in object_keys:
+            path = self._owned_path(object_key=object_key)
+            await asyncio.to_thread(path.unlink, missing_ok=True)
+
+    def _owned_path(self, *, object_key: str) -> Path:
+        """Resolve one persisted key and reject paths outside the owned root.
+
+        Side effects if changes:
+        - local filesystem traversal protection for reads, writes, and cleanup.
+        """
+
+        path = (self._directory / object_key).resolve()
+        if not path.is_relative_to(self._directory):
+            raise ValueError(
+                "local artifact object key escaped its configured directory"
+            )
+        return path
+
+
+class InMemoryArtifactStore:
+    """Deterministic object store for offline tests only.
+
+    Side effects if changes:
+    - unit-test artifact content and cleanup assertions.
+    """
+
+    def __init__(self) -> None:
+        """Initialize isolated in-process bytes.
+
+        Steps:
+        1. create an empty content map and an unclaimed namespace owner slot.
+
+        Side effects if changes:
+        - each test must receive a fresh store to prevent leakage.
+        """
+
+        self.objects: dict[str, bytes] = {}
+        self._namespace_owner: str | None = None
+
+    def namespace_identity(self) -> ArtifactStoreIdentity:
+        """Return the fixed process-memory backend identity used only in tests.
+
+        Steps:
+        1. expose the non-durable memory key namespace.
+
+        Side effects if changes:
+        - Mongo markers distinguish memory from local and OSS backends.
+        """
+
+        return ArtifactStoreIdentity(
+            backend=ArtifactStoreBackend.MEMORY,
+            prefix="memory",
+        )
+
+    async def ensure_namespace_owner(self, *, owner: str) -> None:
+        """Retain one in-process owner for deterministic offline tests.
+
+        Steps:
+        1. initialize the owner on first use;
+        2. accept exact repeats and reject a second owner.
+
+        Side effects if changes:
+        - unit-test cleanup remains isolated to one logical database owner.
+        """
+
+        if self._namespace_owner is None:
+            self._namespace_owner = owner
+        elif self._namespace_owner != owner:
+            raise ScraperDomainError(
+                code=ScraperErrorCode.CONFIGURATION_INVALID,
+                message="artifact namespace is already claimed by another database owner",
+            )
+
+    def object_key(self, *, content_sha256: str) -> str:
+        """Derive the isolated test key without mutating stored bytes.
+
+        Side effects if changes:
+        - offline metadata/object identity and cleanup assertions.
+        """
+
+        return f"memory/{content_sha256[:2]}/{content_sha256}"
+
+    async def put_content_addressed(
+        self,
+        *,
+        content_sha256: str,
+        content: bytes,
+        content_type: str,
+    ) -> str:
+        """Store bytes by hash for deterministic test behavior.
+
+        Side effects if changes:
+        - offline artifact deduplication assertions.
+        """
+
+        object_key = self.object_key(content_sha256=content_sha256)
+        self.objects[object_key] = content
+        return object_key
+
+    async def signed_download(
+        self,
+        *,
+        artifact_id: UUID,
+        object_key: str,
+        expires_seconds: int,
+        audit_id: UUID,
+    ) -> ScrapeArtifactDownload:
+        """Return a non-network test URL only for an existing object.
+
+        Side effects if changes:
+        - offline raw-download tests.
+        """
+
+        if object_key not in self.objects:
+            raise KeyError(object_key)
+        return ScrapeArtifactDownload(
+            artifact_id=artifact_id,
+            download_url=AnyHttpUrl(
+                "https://artifacts.test/"
+                f"{object_key}?{_AUDIT_QUERY_PARAMETER}={audit_id}"
+            ),
+            expires_at=datetime.now(UTC) + timedelta(seconds=expires_seconds),
+        )
+
+    async def delete_many(self, *, object_keys: Sequence[str]) -> None:
+        """Delete exact test keys without affecting other test instances.
+
+        Side effects if changes:
+        - maintenance cleanup assertions.
+        """
+
+        for object_key in object_keys:
+            self.objects.pop(object_key, None)