rememberstack 0.1.0__py3-none-any.whl

rememberstack/adapters/selfhost/minio.py

@@ -0,0 +1,279 @@
+"""S3-compatible MinIO object storage for the self-host profile."""
+
+from typing import cast
+from typing import NotRequired
+from typing import Protocol
+from typing import TypedDict
+
+import boto3
+from botocore.client import Config
+from botocore.exceptions import ClientError
+from pydantic import SecretStr
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+from rememberstack.model import ObjectAlreadyExistsError
+from rememberstack.model import ObjectKey
+from rememberstack.model import ObjectKeyEscapesRootError
+
+
+class MinIOSettings(BaseSettings):
+    """Connection settings for the self-host S3-compatible object store."""
+
+    model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_MINIO_", extra="ignore")
+
+    endpoint_url: str
+    access_key: SecretStr
+    secret_key: SecretStr
+    region: str = "us-east-1"
+
+
+class _StreamingBody(Protocol):
+    """The two response-body operations used by this adapter."""
+
+    def read(self) -> bytes:
+        """Read the complete response body."""
+        ...
+
+    def close(self) -> None:
+        """Release the underlying HTTP connection."""
+        ...
+
+
+class _GetObjectOutput(TypedDict):
+    """The fields consumed from an S3 GetObject response."""
+
+    Body: _StreamingBody
+
+
+class _HeadObjectOutput(TypedDict):
+    """The fields consumed from an S3 HeadObject response."""
+
+    Metadata: NotRequired[dict[str, str]]
+
+
+class _ListedObject(TypedDict):
+    """One object identity returned by ListObjectsV2."""
+
+    Key: str
+
+
+class _ListObjectsOutput(TypedDict):
+    """The fields consumed from one ListObjectsV2 response page."""
+
+    Contents: NotRequired[list[_ListedObject]]
+    IsTruncated: NotRequired[bool]
+    NextContinuationToken: NotRequired[str]
+
+
+class _S3Client(Protocol):
+    """The narrow boto3 client subset the MinIO adapter owns."""
+
+    def head_bucket(self, *, Bucket: str) -> object:
+        """Check that one bucket is reachable."""
+        ...
+
+    def create_bucket(self, *, Bucket: str) -> object:
+        """Create one bucket."""
+        ...
+
+    def get_object(self, *, Bucket: str, Key: str) -> _GetObjectOutput:
+        """Read one object."""
+        ...
+
+    def put_object(
+        self,
+        *,
+        Bucket: str,
+        Key: str,
+        Body: bytes,
+        IfNoneMatch: str,
+        Metadata: dict[str, str],
+    ) -> object:
+        """Conditionally create one immutable object."""
+        ...
+
+    def head_object(self, *, Bucket: str, Key: str) -> _HeadObjectOutput:
+        """Read one object's metadata."""
+        ...
+
+    def delete_object(self, *, Bucket: str, Key: str) -> object:
+        """Delete one object idempotently."""
+        ...
+
+    def list_objects_v2(
+        self, *, Bucket: str, Prefix: str, ContinuationToken: str = ""
+    ) -> _ListObjectsOutput:
+        """List one page beneath a key prefix."""
+        ...
+
+
+class MinIOObjectStore:
+    """Immutable objects in one explicitly selected MinIO bucket."""
+
+    def __init__(
+        self,
+        *,
+        bucket: str,
+        settings: MinIOSettings | None = None,
+        client: _S3Client | None = None,
+    ) -> None:
+        """Bind one bucket to either injected test client or configured MinIO."""
+        if not bucket:
+            raise ValueError("a MinIO object store requires a non-empty bucket")
+        if client is None and settings is None:
+            raise ValueError("MinIO settings are required when no client is injected")
+        self._bucket = bucket
+        self._client = client or _client(settings=cast("MinIOSettings", settings))
+
+    def ensure_bucket(self) -> None:
+        """Provision the configured bucket if it does not exist."""
+        try:
+            self._client.head_bucket(Bucket=self._bucket)
+            return
+        except ClientError as error:
+            if _error_code(error=error) not in {"404", "NoSuchBucket", "NotFound"}:
+                raise
+        try:
+            self._client.create_bucket(Bucket=self._bucket)
+        except ClientError as error:
+            if _error_code(error=error) not in {
+                "BucketAlreadyExists",
+                "BucketAlreadyOwnedByYou",
+            }:
+                raise
+
+    def read_bytes(self, *, key: ObjectKey) -> bytes:
+        """Read all bytes stored at one validated object key."""
+        response = self._client.get_object(
+            Bucket=self._bucket, Key=_validated_key(key=key)
+        )
+        body = response["Body"]
+        try:
+            return body.read()
+        finally:
+            body.close()
+
+    def write_bytes(
+        self, *, key: ObjectKey, content: bytes, storage_class: str | None = None
+    ) -> None:
+        """Create immutable bytes atomically, refusing an occupied key."""
+        metadata = {} if storage_class is None else {"storage-class": storage_class}
+        try:
+            self._client.put_object(
+                Bucket=self._bucket,
+                Key=_validated_key(key=key),
+                Body=content,
+                IfNoneMatch="*",
+                Metadata=metadata,
+            )
+        except ClientError as error:
+            if _error_code(error=error) not in {
+                "409",
+                "412",
+                "ConditionalRequestConflict",
+                "PreconditionFailed",
+            }:
+                raise
+            raise ObjectAlreadyExistsError(
+                f"object key {key.root!r} is already occupied; objects are immutable"
+            ) from error
+
+    def storage_class_of(self, *, key: ObjectKey) -> str | None:
+        """Return the routing class recorded in object metadata, when present."""
+        response = self._client.head_object(
+            Bucket=self._bucket, Key=_validated_key(key=key)
+        )
+        return response.get("Metadata", {}).get("storage-class")
+
+    def purge_objects(
+        self, *, keys: tuple[ObjectKey, ...], prefixes: tuple[ObjectKey, ...]
+    ) -> None:
+        """Idempotently delete exact keys and prefix-boundary descendants."""
+        targets = {_validated_key(key=key) for key in keys}
+        for prefix in prefixes:
+            targets.update(self._keys_under(prefix=_validated_key(key=prefix)))
+        for target in sorted(targets):
+            self._client.delete_object(Bucket=self._bucket, Key=target)
+
+    def verify_objects_purged(
+        self, *, keys: tuple[ObjectKey, ...], prefixes: tuple[ObjectKey, ...]
+    ) -> None:
+        """Fail when any exact key or prefix-boundary descendant remains."""
+        remaining: list[str] = []
+        for key in keys:
+            normalized = _validated_key(key=key)
+            if self._exists(normalized=normalized):
+                remaining.append(normalized)
+        for prefix in prefixes:
+            remaining.extend(self._keys_under(prefix=_validated_key(key=prefix)))
+        if remaining:
+            raise RuntimeError(
+                f"object purge verification found: {sorted(remaining)!r}"
+            )
+
+    def _exists(self, *, normalized: str) -> bool:
+        """Return whether one exact object exists, propagating non-absence errors."""
+        try:
+            self._client.head_object(Bucket=self._bucket, Key=normalized)
+            return True
+        except ClientError as error:
+            if _error_code(error=error) in {"404", "NoSuchKey", "NotFound"}:
+                return False
+            raise
+
+    def _keys_under(self, *, prefix: str) -> tuple[str, ...]:
+        """Enumerate exact and descendant keys without matching sibling prefixes."""
+        boundary = prefix.rstrip("/")
+        result: list[str] = []
+        continuation = ""
+        while True:
+            page = (
+                self._client.list_objects_v2(
+                    Bucket=self._bucket, Prefix=boundary, ContinuationToken=continuation
+                )
+                if continuation
+                else self._client.list_objects_v2(Bucket=self._bucket, Prefix=boundary)
+            )
+            result.extend(
+                item["Key"]
+                for item in page.get("Contents", [])
+                if item["Key"] == boundary or item["Key"].startswith(f"{boundary}/")
+            )
+            if not page.get("IsTruncated", False):
+                return tuple(result)
+            continuation = page.get("NextContinuationToken", "")
+            if not continuation:
+                raise RuntimeError(
+                    "MinIO returned a truncated object page without a continuation token"
+                )
+
+
+def _client(*, settings: MinIOSettings) -> _S3Client:
+    """Construct the path-style S3 client supported by local MinIO."""
+    return cast(
+        "_S3Client",
+        boto3.client(
+            "s3",
+            endpoint_url=settings.endpoint_url,
+            aws_access_key_id=settings.access_key.get_secret_value(),
+            aws_secret_access_key=settings.secret_key.get_secret_value(),
+            region_name=settings.region,
+            config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
+        ),
+    )
+
+
+def _validated_key(*, key: ObjectKey) -> str:
+    """Reject absolute and parent-traversal keys for local/S3 parity."""
+    parts = key.root.split("/")
+    if key.root.startswith("/") or ".." in parts:
+        raise ObjectKeyEscapesRootError(
+            f"object key {key.root!r} escapes the store root"
+        )
+    return key.root
+
+
+def _error_code(*, error: ClientError) -> str:
+    """Return the provider's stable error code without trimming the exception."""
+    return str(error.response.get("Error", {}).get("Code", ""))

rememberstack/adapters/selfhost/mounts.py

@@ -0,0 +1,249 @@
+"""Local-directory mount publisher: the four D51 read-only views (e0 §5).
+
+Agents read the memory on their filesystem: the **corpus filesystem** they
+browse first, the **artifacts** they drill into from a stub, the **raw**
+originals — deliberately *off the navigation path* — and the Plane-K
+checkout. Every view is read-only; writes always go through the pipeline
+and Postgres stays the authority.
+
+The self-host tier materializes what the cloud tier gets from a bucket
+mount, and implements D51's three guardrails here rather than assuming
+them of infrastructure:
+
+1. **Raw is off-path.** Nothing in the corpus tree links into it; reaching
+   an original means following an explicit `raw_uri` from a stub or from
+   `document.md` frontmatter — a deliberate act, never a browse default.
+2. **Data-access audit logging is mandatory.** The audit property came
+   from logging, not from keeping raw unmounted (a mount read is still a
+   read), so originals are readable only through `AuditedRawReader`, which
+   refuses an unattributed request rather than logging a blank.
+3. **Storage class routes by mime.** Media a multimodal harness actually
+   reads stays hot; text/office originals kept only for audit and
+   re-conversion go cold — this kills the grep-the-archive cost bug at the
+   source. Per-deployment config, like the D38 converter router.
+
+The corpus view always serves the snapshot the registry marks latest, and
+swaps whole trees atomically, so a browsing agent never walks a half-built
+tree.
+"""
+
+from datetime import datetime
+from datetime import UTC
+import json
+import os
+from pathlib import Path
+import shutil
+from typing import Final
+from typing import Protocol
+from uuid import UUID
+from uuid import uuid4
+
+from rememberstack.core import storage_class_for
+from rememberstack.model import ObjectKey
+from rememberstack.model import PublishedMounts
+from rememberstack.ports.object_store import ObjectStorePort
+from rememberstack.spine.projection import ProjectionCatalog
+
+EMPTY_CORPUS_NOTE: Final = (
+    "# Corpus filesystem\n\n"
+    "> No P3 snapshot has been published yet. Run the corpus-filesystem\n"
+    "> builder; until then use the query API, which needs no projection.\n"
+)
+
+
+class RawAccessDenied(Exception):
+    """An unattributed raw read was refused (originals are audited)."""
+
+
+class MountAdmission(Protocol):
+    """The composition-owned barrier checked before publishing serving paths."""
+
+    def assert_available(self, *, deployment_id: UUID) -> None:
+        """Raise while D74 keeps the deployment fail-closed."""
+        ...
+
+
+class LocalMountPublisher:
+    """Publish the P3, artifact, raw, and Plane-K views as local trees."""
+
+    def __init__(
+        self,
+        *,
+        root: Path,
+        catalog: ProjectionCatalog | None = None,
+        corpusfs_store: ObjectStorePort | None = None,
+        artifacts_root: Path | None = None,
+        raw_root: Path | None = None,
+        knowledge_root: Path | None = None,
+        admission: MountAdmission,
+    ) -> None:
+        """Bind the publisher to its mount root, the P3 source, and the stores.
+
+        The artifact/raw/knowledge views point at the REAL store roots when
+        given (Codex review: an empty directory is not a usable mount);
+        without them the publisher provisions empty view roots — the
+        Phase-0 shape, still useful before any store exists.
+        """
+        self._root = root.resolve()
+        self._catalog = catalog
+        self._corpusfs_store = corpusfs_store
+        self._artifacts_root = artifacts_root
+        self._raw_root = raw_root
+        self._knowledge_root = knowledge_root
+        self._admission = admission
+
+    def publish(self, *, deployment_id: UUID) -> PublishedMounts:
+        """Publish and return the exact four read-only deployment views."""
+        self._admission.assert_available(deployment_id=deployment_id)
+        base = self._root / str(deployment_id)
+        base.mkdir(parents=True, exist_ok=True)
+        corpus = base / "p3"
+        self._materialize_corpus(deployment_id=deployment_id, link=corpus)
+        return PublishedMounts(
+            deployment_id=deployment_id,
+            p3=str(corpus),
+            artifacts=str(
+                self._view(base=base, name="artifacts", real=self._artifacts_root)
+            ),
+            # off the navigation path (D51): the tree never promotes raw —
+            # stubs carry an explicit pointer, and reads go through
+            # AuditedRawReader, which is the only audited path on this tier
+            raw=str(self._view(base=base, name="raw", real=self._raw_root)),
+            knowledge=str(
+                self._view(base=base, name="knowledge", real=self._knowledge_root)
+            ),
+            read_only=True,
+        )
+
+    def _view(self, *, base: Path, name: str, real: Path | None) -> Path:
+        """One view locator: the real store root when known, else an empty dir."""
+        if real is not None:
+            real.mkdir(parents=True, exist_ok=True)
+            return real.resolve()
+        placeholder = base / name
+        placeholder.mkdir(parents=True, exist_ok=True)
+        return placeholder
+
+    def _materialize_corpus(self, *, deployment_id: UUID, link: Path) -> None:
+        """Serve the LATEST PUBLISHED snapshot behind an ATOMIC pointer.
+
+        The mount path is a symlink to a versioned directory, and the swap
+        replaces that symlink with `os.replace` — atomic on POSIX. The
+        previous "rmtree then rename" left a window where the mount path
+        did not exist at all, so a reader could hit ENOENT mid-swap and two
+        publishers could delete each other's staging (Codex review).
+        """
+        if self._catalog is None or self._corpusfs_store is None:
+            link.mkdir(parents=True, exist_ok=True)
+            return
+        latest = self._catalog.latest_snapshot(
+            deployment_id=deployment_id, plane="P3_corpusfs"
+        )
+        if latest is None:
+            empty = link.parent / "p3-empty"
+            empty.mkdir(parents=True, exist_ok=True)
+            (empty / "llms.txt").write_text(EMPTY_CORPUS_NOTE, encoding="utf-8")
+            _point(link=link, target=empty)
+            return
+        version = str(latest["version"])
+        served = link.parent / f"p3-{version}"
+        if not (served / ".snapshot-version").exists():
+            prefix = str(latest["gcs_uri"])
+            manifest = json.loads(
+                self._corpusfs_store.read_bytes(
+                    key=ObjectKey(f"{prefix}/MANIFEST.json")
+                )
+            )
+            # a unique staging dir per publisher: concurrent publishes of
+            # the same version never delete each other's work
+            staging = link.parent / f".staging-{version}-{uuid4().hex[:8]}"
+            for relative in manifest["files"]:
+                destination = staging / relative
+                destination.parent.mkdir(parents=True, exist_ok=True)
+                destination.write_bytes(
+                    self._corpusfs_store.read_bytes(
+                        key=ObjectKey(f"{prefix}/{relative}")
+                    )
+                )
+            (staging / ".snapshot-version").write_text(version, encoding="utf-8")
+            try:
+                staging.rename(served)  # atomic: the version dir appears whole
+            except OSError:  # another publisher won the race — theirs is fine
+                shutil.rmtree(staging, ignore_errors=True)
+        _point(link=link, target=served)
+
+
+class AuditedRawReader:
+    """The ONLY way to read an original — because reads must be logged.
+
+    D51's audit property comes from logging, not from keeping raw
+    unmounted: a mount read is still a read. So raw access is offered
+    exclusively through this reader, which records the accessor and the
+    stated purpose before returning bytes and refuses an unattributed
+    request outright rather than logging a blank.
+    """
+
+    def __init__(self, *, raw_store: ObjectStorePort, audit_log: Path) -> None:
+        """Bind the reader to the raw store and its append-only audit log."""
+        self._raw_store = raw_store
+        self._audit_log = audit_log
+
+    def read(
+        self, *, deployment_id: UUID, raw_uri: str, accessor: str, purpose: str
+    ) -> bytes:
+        """Read one original, recording who read it and why."""
+        if not accessor.strip() or not purpose.strip():
+            raise RawAccessDenied(
+                "raw access requires an accessor and a stated purpose:"
+                " originals are audited, never anonymously readable"
+            )
+        content = self._raw_store.read_bytes(key=ObjectKey(raw_uri))
+        self._append(
+            {
+                "at": datetime.now(tz=UTC).isoformat(),
+                "deployment_id": str(deployment_id),
+                "raw_uri": raw_uri,
+                "accessor": accessor,
+                "purpose": purpose,
+                "bytes": len(content),
+            }
+        )
+        return content
+
+    def entries(self) -> tuple[dict[str, object], ...]:
+        """The audit trail, oldest first (the operator's read surface)."""
+        if not self._audit_log.exists():
+            return ()
+        return tuple(
+            json.loads(line)
+            for line in self._audit_log.read_text(encoding="utf-8").splitlines()
+            if line.strip()
+        )
+
+    def _append(self, entry: dict[str, object]) -> None:
+        """Append one audit record (the log is append-only by construction)."""
+        self._audit_log.parent.mkdir(parents=True, exist_ok=True)
+        with self._audit_log.open("a", encoding="utf-8") as handle:
+            handle.write(json.dumps(entry) + "\n")
+
+
+def _point(*, link: Path, target: Path) -> None:
+    """Atomically point the mount path at a versioned directory.
+
+    A symlink swapped with `os.replace` is atomic on POSIX: a reader either
+    sees the old snapshot or the new one, never a missing path.
+    """
+    staging_link = link.with_name(f".{link.name}-{uuid4().hex[:8]}")
+    staging_link.symlink_to(target, target_is_directory=True)
+    if link.exists() and not link.is_symlink():
+        shutil.rmtree(link)  # a legacy real directory: replaced once
+    os.replace(staging_link, link)
+
+
+__all__ = (
+    "AuditedRawReader",
+    "EMPTY_CORPUS_NOTE",
+    "LocalMountPublisher",
+    "RawAccessDenied",
+    "storage_class_for",  # re-exported: the adapter applies this policy
+)

rememberstack/adapters/selfhost/object_store.py

@@ -0,0 +1,130 @@
+"""Local-filesystem object store adapter: immutable bytes under one root (D61/D62)."""
+
+from pathlib import Path
+
+from rememberstack.model import ObjectAlreadyExistsError
+from rememberstack.model import ObjectKey
+from rememberstack.model import ObjectKeyEscapesRootError
+
+
+class LocalFSObjectStore:
+    """The self-host object store: one directory tree of immutable objects."""
+
+    def __init__(self, *, root: Path) -> None:
+        """Bind the store to its root directory, creating it if absent."""
+        self._root = root.resolve()
+        self._root.mkdir(parents=True, exist_ok=True)
+
+    def read_bytes(self, *, key: ObjectKey) -> bytes:
+        """Read all bytes stored under an existing object key."""
+        return self._path_for(key=key).read_bytes()
+
+    def write_bytes(
+        self, *, key: ObjectKey, content: bytes, storage_class: str | None = None
+    ) -> None:
+        """Create immutable bytes, failing rather than replacing an occupied key.
+
+        A local filesystem has no storage classes, so the D51 routing
+        decision is RECORDED beside the object instead of dropped — the
+        cloud adapter turns the same value into a real class, and either
+        way an operator can see what each original was routed to.
+        """
+        path = self._path_for(key=key)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            with path.open(mode="xb") as handle:
+                handle.write(content)
+        except FileExistsError as err:
+            raise ObjectAlreadyExistsError(
+                f"object key {key.root!r} is already occupied; objects are immutable"
+            ) from err
+        if storage_class is not None:
+            path.with_name(f"{path.name}.storage-class").write_text(
+                storage_class, encoding="utf-8"
+            )
+
+    def storage_class_of(self, *, key: ObjectKey) -> str | None:
+        """The class one object was routed to, when the writer declared it."""
+        marker = self._path_for(key=key).with_suffix("")
+        marker = self._path_for(key=key)
+        marker = marker.with_name(f"{marker.name}.storage-class")
+        return marker.read_text(encoding="utf-8") if marker.exists() else None
+
+    def purge_objects(
+        self, *, keys: tuple[ObjectKey, ...], prefixes: tuple[ObjectKey, ...]
+    ) -> None:
+        """Idempotently erase exact keys, storage markers, and prefix matches."""
+        exact_paths = tuple(self._path_for(key=key) for key in keys)
+        normalized_prefixes = tuple(
+            self._path_for(key=prefix).relative_to(self._root).as_posix()
+            for prefix in prefixes
+        )
+        for path in exact_paths:
+            path.unlink(missing_ok=True)
+            path.with_name(f"{path.name}.storage-class").unlink(missing_ok=True)
+        if normalized_prefixes:
+            for path in tuple(self._root.rglob("*")):
+                if not (path.is_file() or path.is_symlink()):
+                    continue
+                relative = path.relative_to(self._root).as_posix()
+                if any(
+                    relative == prefix.rstrip("/")
+                    or relative.startswith(f"{prefix.rstrip('/')}/")
+                    for prefix in normalized_prefixes
+                ):
+                    path.unlink(missing_ok=True)
+        self._remove_empty_directories()
+
+    def verify_objects_purged(
+        self, *, keys: tuple[ObjectKey, ...], prefixes: tuple[ObjectKey, ...]
+    ) -> None:
+        """Prove exact keys and prefix-boundary descendants are absent."""
+        remaining = [
+            key.root
+            for key in keys
+            if self._path_for(key=key).exists()
+            or self._path_for(key=key)
+            .with_name(f"{self._path_for(key=key).name}.storage-class")
+            .exists()
+        ]
+        normalized_prefixes = tuple(
+            self._path_for(key=prefix).relative_to(self._root).as_posix().rstrip("/")
+            for prefix in prefixes
+        )
+        for path in self._root.rglob("*"):
+            if not (path.is_file() or path.is_symlink()):
+                continue
+            relative = path.relative_to(self._root).as_posix()
+            if any(
+                relative == prefix or relative.startswith(f"{prefix}/")
+                for prefix in normalized_prefixes
+            ):
+                remaining.append(relative)
+        if remaining:
+            raise RuntimeError(
+                f"object purge verification found: {sorted(remaining)!r}"
+            )
+
+    def _path_for(self, *, key: ObjectKey) -> Path:
+        """Resolve a key to a path strictly inside the root (no traversal)."""
+        candidate = (self._root / key.root).resolve()
+        if not candidate.is_relative_to(self._root):
+            raise ObjectKeyEscapesRootError(
+                f"object key {key.root!r} escapes the store root"
+            )
+        return candidate
+
+    def _remove_empty_directories(self) -> None:
+        """Prune empty object-key parents without ever removing the store root."""
+        directories = sorted(
+            (
+                path
+                for path in self._root.rglob("*")
+                if path.is_dir() and not path.is_symlink()
+            ),
+            key=lambda path: len(path.parts),
+            reverse=True,
+        )
+        for directory in directories:
+            if not any(directory.iterdir()):
+                directory.rmdir()