rememberstack 0.1.0__py3-none-any.whl

rememberstack/__init__.py

@@ -0,0 +1,9 @@
+"""RememberStack: open memory infrastructure for AI agents."""
+
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version
+
+try:
+    __version__ = version("rememberstack")
+except PackageNotFoundError:  # running from a checkout without installation
+    __version__ = "0.0.0+uninstalled"

rememberstack/adapters/__init__.py

@@ -0,0 +1,42 @@
+"""Provider adapter package."""
+
+from typing import TYPE_CHECKING
+
+from rememberstack.adapters.codex_writer import CodexAgentAdapterSettings
+from rememberstack.adapters.codex_writer import CodexCLIAgentAdapter
+from rememberstack.adapters.codex_writer import CodexCLIWriterAdapter
+from rememberstack.adapters.codex_writer import CodexWriterAdapterSettings
+from rememberstack.adapters.openrouter import OpenRouterModelProvider
+from rememberstack.adapters.openrouter import OpenRouterProviderError
+from rememberstack.adapters.openrouter import OpenRouterSettings
+
+if TYPE_CHECKING:
+    from rememberstack.adapters.markitdown_converter import MARKITDOWN_CONVERTER_VERSION
+    from rememberstack.adapters.markitdown_converter import MarkitdownConverter
+
+__all__ = (
+    "CodexCLIAgentAdapter",
+    "CodexCLIWriterAdapter",
+    "CodexAgentAdapterSettings",
+    "CodexWriterAdapterSettings",
+    "MARKITDOWN_CONVERTER_VERSION",
+    "MarkitdownConverter",
+    "OpenRouterModelProvider",
+    "OpenRouterProviderError",
+    "OpenRouterSettings",
+)
+
+
+def __getattr__(name: str) -> object:
+    """Load the media converter only for processes that actually compose it."""
+    if name == "MARKITDOWN_CONVERTER_VERSION":
+        from rememberstack.adapters.markitdown_converter import (
+            MARKITDOWN_CONVERTER_VERSION,
+        )
+
+        return MARKITDOWN_CONVERTER_VERSION
+    if name == "MarkitdownConverter":
+        from rememberstack.adapters.markitdown_converter import MarkitdownConverter
+
+        return MarkitdownConverter
+    raise AttributeError(name)

rememberstack/adapters/codex_writer.py

@@ -0,0 +1,221 @@
+"""Sandboxed stock-Codex adapter for one declared-output Plane-K session."""
+
+import json
+from pathlib import Path
+import subprocess
+from tempfile import TemporaryDirectory
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+from rememberstack.model import KnowledgeAgentSessionRequest
+from rememberstack.model import KnowledgeAgentSessionResult
+
+
+class CodexAgentAdapterSettings(BaseSettings):
+    """The executable binding shared by Plane-K stock-harness seats."""
+
+    model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_K_CODEX_")
+
+    executable: str = Field(default="codex", min_length=1)
+
+
+class CodexWriterAdapterSettings(BaseSettings):
+    """Backward-compatible writer-specific executable settings."""
+
+    model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_K_WRITER_CODEX_")
+
+    executable: str = Field(default="codex", min_length=1)
+
+
+class CodexCLIAgentAdapter:
+    """Run Codex in an ephemeral non-git workspace and accept declared files only."""
+
+    def __init__(
+        self, *, settings: CodexAgentAdapterSettings | CodexWriterAdapterSettings
+    ) -> None:
+        """Bind the adapter to its settings-owned executable."""
+        self._settings = settings
+
+    def run_session(
+        self, *, request: KnowledgeAgentSessionRequest
+    ) -> KnowledgeAgentSessionResult:
+        """Run one sandboxed session and return raw outputs without accepting them."""
+        with TemporaryDirectory(prefix="rememberstack-k-writer-") as temporary:
+            workspace = Path(temporary)
+            for relative_path, content in request.input_files.items():
+                target = _workspace_path(
+                    workspace=workspace, relative_path=relative_path
+                )
+                target.parent.mkdir(parents=True, exist_ok=True)
+                target.write_text(content, encoding="utf-8")
+            for relative_path in request.sandbox.accepted_output_paths:
+                _workspace_path(
+                    workspace=workspace, relative_path=relative_path
+                ).parent.mkdir(parents=True, exist_ok=True)
+            writer_workdir = workspace / "output"
+            last_message = workspace / ".session" / "last-message.txt"
+            last_message.parent.mkdir(parents=True, exist_ok=True)
+            command = _codex_command(
+                executable=self._settings.executable,
+                request=request,
+                writer_workdir=writer_workdir,
+                last_message=last_message,
+            )
+            try:
+                completed = subprocess.run(  # noqa: S603 -- fixed argv, no shell
+                    command,
+                    input=request.prompt,
+                    capture_output=True,
+                    text=True,
+                    check=False,
+                    timeout=request.timeout_seconds,
+                )
+            except subprocess.TimeoutExpired as error:
+                stdout = _timeout_text(value=error.stdout)
+                stderr = _timeout_text(value=error.stderr)
+                return KnowledgeAgentSessionResult(
+                    session_id=request.session_id,
+                    exit_code=None,
+                    timed_out=True,
+                    output_files=_declared_outputs(
+                        workspace=workspace, paths=request.sandbox.accepted_output_paths
+                    ),
+                    transcript=_process_transcript(
+                        stdout=stdout, stderr=stderr, exit_code=None, timed_out=True
+                    ),
+                    tokens=_token_usage(stdout=stdout),
+                )
+            return KnowledgeAgentSessionResult(
+                session_id=request.session_id,
+                exit_code=completed.returncode,
+                output_files=_declared_outputs(
+                    workspace=workspace, paths=request.sandbox.accepted_output_paths
+                ),
+                transcript=_process_transcript(
+                    stdout=completed.stdout,
+                    stderr=completed.stderr,
+                    exit_code=completed.returncode,
+                    timed_out=False,
+                ),
+                tokens=_token_usage(stdout=completed.stdout),
+            )
+
+
+def _codex_command(
+    *,
+    executable: str,
+    request: KnowledgeAgentSessionRequest,
+    writer_workdir: Path,
+    last_message: Path,
+) -> list[str]:
+    """Build the fixed stock-harness command with no inherited tools or web search."""
+    return [
+        executable,
+        "exec",
+        "--strict-config",
+        "--ignore-user-config",
+        "--ignore-rules",
+        "--ephemeral",
+        "--skip-git-repo-check",
+        "--sandbox",
+        "workspace-write",
+        "-c",
+        "sandbox_workspace_write.network_access=false",
+        "-c",
+        "shell_environment_policy.inherit=none",
+        "-c",
+        "sandbox_workspace_write.exclude_tmpdir_env_var=true",
+        "-c",
+        "sandbox_workspace_write.exclude_slash_tmp=true",
+        "--model",
+        request.model,
+        "--json",
+        "--color",
+        "never",
+        "--output-last-message",
+        str(last_message),
+        "--cd",
+        str(writer_workdir),
+        "-",
+    ]
+
+
+def _declared_outputs(*, workspace: Path, paths: tuple[str, ...]) -> dict[str, str]:
+    """Read only contract-declared UTF-8 files and discard every other writer edit."""
+    outputs: dict[str, str] = {}
+    for relative_path in paths:
+        try:
+            target = _workspace_path(workspace=workspace, relative_path=relative_path)
+            if target.is_file():
+                outputs[relative_path] = target.read_text(encoding="utf-8")
+        except (OSError, UnicodeError, ValueError):
+            # The compiler rejects the missing declared output after it archives
+            # the process transcript; hostile links and unreadable bytes cannot
+            # make a completed session disappear from the ledger.
+            continue
+    return outputs
+
+
+def _workspace_path(*, workspace: Path, relative_path: str) -> Path:
+    """Resolve a validated relative path without following a workspace escape."""
+    target = workspace.joinpath(*relative_path.split("/"))
+    if target.is_symlink() or not target.resolve().is_relative_to(workspace.resolve()):
+        raise ValueError(f"writer workspace path escapes root: {relative_path}")
+    return target
+
+
+def _timeout_text(*, value: str | bytes | None) -> str:
+    """Normalize subprocess timeout fragments without discarding captured bytes."""
+    if value is None:
+        return ""
+    return (
+        value.decode("utf-8", errors="replace") if isinstance(value, bytes) else value
+    )
+
+
+def _process_transcript(
+    *, stdout: str, stderr: str, exit_code: int | None, timed_out: bool
+) -> str:
+    """Retain complete process streams and terminal status as one immutable ledger object."""
+    return json.dumps(
+        {
+            "stdout": stdout,
+            "stderr": stderr,
+            "exit_code": exit_code,
+            "timed_out": timed_out,
+        },
+        sort_keys=True,
+    )
+
+
+def _token_usage(*, stdout: str) -> int | None:
+    """Sum typed Codex turn usage without treating its prose as trusted output."""
+    total = 0
+    found = False
+    for line in stdout.splitlines():
+        try:
+            event = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(event, dict) or event.get("type") != "turn.completed":
+            continue
+        usage = event.get("usage")
+        if not isinstance(usage, dict):
+            continue
+        input_tokens = usage.get("input_tokens")
+        output_tokens = usage.get("output_tokens")
+        if (
+            not isinstance(input_tokens, int)
+            or input_tokens < 0
+            or not isinstance(output_tokens, int)
+            or output_tokens < 0
+        ):
+            continue
+        total += input_tokens + output_tokens
+        found = True
+    return total if found else None
+
+
+CodexCLIWriterAdapter = CodexCLIAgentAdapter

rememberstack/adapters/markitdown_converter.py

@@ -0,0 +1,42 @@
+"""The markitdown conversion route (D38): office/html/email formats → Markdown."""
+
+import io
+from typing import Final
+
+from markitdown import MarkItDown
+from markitdown import StreamInfo
+from markitdown._exceptions import MarkItDownException
+
+from rememberstack.model import ConversionError
+from rememberstack.model import ConversionResult
+
+MARKITDOWN_CONVERTER_VERSION: Final = "markitdown-0.1"
+"""Pins the markitdown library generation this route was validated against."""
+
+
+class MarkitdownConverter:
+    """The default local route for structured text formats (html, office, email)."""
+
+    def __init__(self) -> None:
+        """Build the converter once; markitdown instances are reusable."""
+        self._markitdown = MarkItDown(enable_plugins=False)
+
+    @property
+    def name(self) -> str:
+        """The route name recorded on representations."""
+        return "markitdown"
+
+    @property
+    def version(self) -> str:
+        """The pinned markitdown route version (D38)."""
+        return MARKITDOWN_CONVERTER_VERSION
+
+    def convert(self, *, content: bytes, mime: str) -> ConversionResult:
+        """Convert one input via markitdown; its failures become typed failures."""
+        try:
+            result = self._markitdown.convert_stream(
+                io.BytesIO(content), stream_info=StreamInfo(mimetype=mime)
+            )
+        except MarkItDownException as err:
+            raise ConversionError(f"markitdown could not convert {mime!r}") from err
+        return ConversionResult(document_md=result.text_content)

rememberstack/adapters/openrouter.py

@@ -0,0 +1,136 @@
+"""The OpenRouter model-provider adapter (D63/D70): the shipped default binding."""
+
+from decimal import Decimal
+from decimal import InvalidOperation
+import json
+import time
+from typing import Any
+from typing import TypeVar
+
+import httpx
+from pydantic import Field
+from pydantic_settings import BaseSettings
+from pydantic_settings import SettingsConfigDict
+
+from rememberstack.model import EmbeddingRequest
+from rememberstack.model import EmbeddingResponse
+from rememberstack.model import GeneratedResponse
+from rememberstack.model import ModelRequest
+from rememberstack.model import ProviderAccountingError
+from rememberstack.model import ProviderCallUsage
+from rememberstack.model import StructuredResponseModel
+
+ResponseT = TypeVar("ResponseT", bound=StructuredResponseModel)
+
+
+class OpenRouterSettings(BaseSettings):
+    """The OpenRouter binding: key and endpoint, per deployment (D61)."""
+
+    model_config = SettingsConfigDict(env_prefix="REMEMBERSTACK_OPENROUTER_")
+
+    api_key: str = Field(min_length=1)
+    base_url: str = Field(default="https://openrouter.ai/api/v1")
+    timeout_s: float = Field(default=120.0, gt=0)
+
+
+class OpenRouterProviderError(Exception):
+    """OpenRouter returned an error or an unusable response body."""
+
+
+class OpenRouterModelProvider:
+    """Structured generations and embeddings over the OpenRouter HTTP API."""
+
+    def __init__(self, *, settings: OpenRouterSettings) -> None:
+        """Bind one HTTP client to the configured endpoint and key."""
+        self._settings = settings
+        self._client = httpx.Client(
+            base_url=settings.base_url,
+            headers={"Authorization": f"Bearer {settings.api_key}"},
+            timeout=settings.timeout_s,
+        )
+
+    def generate(
+        self, *, request: ModelRequest, response_type: type[ResponseT]
+    ) -> GeneratedResponse[ResponseT]:
+        """One chat completion constrained to the caller's declared JSON schema."""
+        started_ns = time.monotonic_ns()
+        body = self._post(
+            path="/chat/completions",
+            payload={
+                "model": request.model,
+                "messages": [{"role": "user", "content": request.prompt}],
+                "response_format": {
+                    "type": "json_schema",
+                    "json_schema": {
+                        "name": response_type.__name__,
+                        "strict": True,
+                        "schema": response_type.model_json_schema(),
+                    },
+                },
+            },
+        )
+        usage = _usage(
+            body=body,
+            requested_model=request.model,
+            latency_ms=(time.monotonic_ns() - started_ns) // 1_000_000,
+        )
+        try:
+            content = body["choices"][0]["message"]["content"]
+            output = response_type.model_validate(json.loads(content))
+        except (KeyError, IndexError, ValueError) as err:
+            raise OpenRouterProviderError(
+                f"unusable completion body for {response_type.__name__}"
+            ) from err
+        return GeneratedResponse(output=output, usage=usage)
+
+    def embed(self, *, request: EmbeddingRequest) -> EmbeddingResponse:
+        """One embeddings call for the caller's batch."""
+        started_ns = time.monotonic_ns()
+        body = self._post(
+            path="/embeddings",
+            payload={"model": request.model, "input": list(request.texts)},
+        )
+        usage = _usage(
+            body=body,
+            requested_model=request.model,
+            latency_ms=(time.monotonic_ns() - started_ns) // 1_000_000,
+        )
+        try:
+            ordered = sorted(body["data"], key=lambda item: item["index"])
+            return EmbeddingResponse(
+                vectors=tuple(tuple(item["embedding"]) for item in ordered), usage=usage
+            )
+        except (KeyError, TypeError, ValueError) as err:
+            raise OpenRouterProviderError("unusable embeddings body") from err
+
+    def _post(self, *, path: str, payload: dict[str, object]) -> dict[str, Any]:
+        """POST one JSON request; non-2xx responses become typed errors."""
+        response = self._client.post(path, json=payload)
+        if response.status_code >= 400:
+            raise OpenRouterProviderError(
+                f"OpenRouter {path} returned {response.status_code}: "
+                f"{response.text[:500]}"
+            )
+        return response.json()
+
+
+def _usage(
+    *, body: dict[str, Any], requested_model: str, latency_ms: int
+) -> ProviderCallUsage:
+    """Validate OpenRouter accounting; missing usage must not silently disable budgets."""
+    raw = body.get("usage")
+    if not isinstance(raw, dict):
+        raise ProviderAccountingError("OpenRouter response carries no usage accounting")
+    model_name = body.get("model", requested_model)
+    try:
+        return ProviderCallUsage(
+            model_name=model_name,
+            tokens_in=raw["prompt_tokens"],
+            tokens_out=raw.get("completion_tokens", 0),
+            cost_usd=Decimal(str(raw["cost"])),
+            latency_ms=latency_ms,
+        )
+    except (InvalidOperation, KeyError, TypeError, ValueError) as err:
+        raise ProviderAccountingError(
+            "OpenRouter response carries invalid usage accounting"
+        ) from err

rememberstack/adapters/selfhost/__init__.py

@@ -0,0 +1,54 @@
+"""Self-host adapters: pg delivery shell, local-FS object store, local mounts (WP-0.4a)."""
+
+from typing import TYPE_CHECKING
+
+from rememberstack.adapters.selfhost.forget import LocalFSForgetManifestStore
+from rememberstack.adapters.selfhost.git import LocalGitRepository
+from rememberstack.adapters.selfhost.minio import MinIOObjectStore
+from rememberstack.adapters.selfhost.minio import MinIOSettings
+from rememberstack.adapters.selfhost.mounts import AuditedRawReader
+from rememberstack.adapters.selfhost.mounts import LocalMountPublisher
+from rememberstack.adapters.selfhost.mounts import RawAccessDenied
+from rememberstack.adapters.selfhost.mounts import storage_class_for
+from rememberstack.adapters.selfhost.object_store import LocalFSObjectStore
+from rememberstack.adapters.selfhost.object_store import ObjectAlreadyExistsError
+from rememberstack.adapters.selfhost.object_store import ObjectKeyEscapesRootError
+from rememberstack.adapters.selfhost.projection import SelfHostProjectionPurger
+from rememberstack.adapters.selfhost.queue import SelfHostTaskQueue
+from rememberstack.adapters.selfhost.queue import SelfHostWorkerLoop
+from rememberstack.adapters.selfhost.queue import TokenBucket
+from rememberstack.adapters.selfhost.telemetry import JsonLineTelemetry
+from rememberstack.adapters.selfhost.watcher import LocalDirectoryWatcher
+
+if TYPE_CHECKING:
+    from rememberstack.adapters.selfhost.lance import LanceChunkIndex
+
+__all__ = (
+    "LanceChunkIndex",
+    "LocalFSForgetManifestStore",
+    "LocalGitRepository",
+    "LocalDirectoryWatcher",
+    "JsonLineTelemetry",
+    "LocalFSObjectStore",
+    "AuditedRawReader",
+    "LocalMountPublisher",
+    "MinIOObjectStore",
+    "MinIOSettings",
+    "RawAccessDenied",
+    "storage_class_for",
+    "ObjectAlreadyExistsError",
+    "ObjectKeyEscapesRootError",
+    "SelfHostTaskQueue",
+    "SelfHostProjectionPurger",
+    "SelfHostWorkerLoop",
+    "TokenBucket",
+)
+
+
+def __getattr__(name: str) -> object:
+    """Load the heavy LanceDB adapter only when a composition actually needs it."""
+    if name == "LanceChunkIndex":
+        from rememberstack.adapters.selfhost.lance import LanceChunkIndex
+
+        return LanceChunkIndex
+    raise AttributeError(name)

rememberstack/adapters/selfhost/forget.py

@@ -0,0 +1,66 @@
+"""Dedicated append-only local store for portable D74 forget intent."""
+
+import os
+from pathlib import Path
+from uuid import UUID
+from uuid import uuid4
+
+from rememberstack.model import ForgetManifest
+from rememberstack.model import ForgetManifestConflictError
+
+
+class LocalFSForgetManifestStore:
+    """Persist content-free manifests outside the ordinary data restore root."""
+
+    def __init__(self, *, root: Path) -> None:
+        """Bind an explicitly provisioned manifest root; absence is unsafe."""
+        self._root = root.resolve(strict=True)
+        if not self._root.is_dir():
+            raise NotADirectoryError(self._root)
+
+    def append(self, *, manifest: ForgetManifest) -> None:
+        """Atomically append exact canonical bytes, idempotent by ``forget_id``."""
+        content = manifest.canonical_bytes()
+        destination = self._path(forget_id=manifest.forget_id)
+        temporary = self._root / f".{manifest.forget_id}.{uuid4()}.tmp"
+        try:
+            with temporary.open(mode="xb") as handle:
+                handle.write(content)
+                handle.flush()
+                os.fsync(handle.fileno())
+            try:
+                os.link(temporary, destination)
+            except FileExistsError:
+                if destination.read_bytes() != content:
+                    raise ForgetManifestConflictError(
+                        f"forget_id {manifest.forget_id} already has different bytes"
+                    ) from None
+            else:
+                self._sync_root()
+        finally:
+            temporary.unlink(missing_ok=True)
+
+    def manifests(self, *, deployment_id: UUID) -> tuple[ForgetManifest, ...]:
+        """Enumerate and validate this deployment's manifests by forget identity."""
+        result: list[ForgetManifest] = []
+        for path in sorted(self._root.glob("*.json")):
+            manifest = ForgetManifest.model_validate_json(path.read_bytes())
+            if path.stem != str(manifest.forget_id):
+                raise ForgetManifestConflictError(
+                    f"manifest filename {path.name!r} disagrees with its forget_id"
+                )
+            if manifest.deployment_id == deployment_id:
+                result.append(manifest)
+        return tuple(result)
+
+    def _path(self, *, forget_id: UUID) -> Path:
+        """Return the fixed UUID-only destination for one immutable manifest."""
+        return self._root / f"{forget_id}.json"
+
+    def _sync_root(self) -> None:
+        """Make the newly linked directory entry durable before acknowledging append."""
+        descriptor = os.open(self._root, os.O_RDONLY)
+        try:
+            os.fsync(descriptor)
+        finally:
+            os.close(descriptor)