compartment 1.15.0__py3-none-any.whl

compartment/__init__.py

@@ -0,0 +1,7 @@
+"""Compartment - high-security, fully offline, encrypted vector memory for AI agents."""
+
+__version__ = "1.15.0"
+
+from . import offline_guard as _og
+
+_og.activate_from_env()

compartment/acl.py

@@ -0,0 +1,92 @@
+"""Per-caller namespace access control + vault-adjacent settings.
+
+Config lives NEXT to the vault as `<vault>.config.json` (it contains no
+secrets - only ACLs and preferences). `packs/*` namespaces are ALWAYS
+read-only for every caller, including "*". Violations are hard errors.
+"""
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+
+from .crypto import CryptoError
+
+DEFAULT_CONFIG = {
+    "callers": {
+        "*": {"default_namespace": "main", "grants": {"*": "rw"}},
+    },
+    "settings": {
+        "auto_lock_minutes": 30,
+        "include_packs_in_search": True,
+        "unlock_tool_enabled": False,
+        "duplicate_threshold": 0.97,
+        "index_precision": "f32",
+    },
+}
+
+
+class AclError(CryptoError):
+    pass
+
+
+@dataclass
+class VaultConfig:
+    callers: dict = field(default_factory=lambda: DEFAULT_CONFIG["callers"].copy())
+    settings: dict = field(default_factory=lambda: DEFAULT_CONFIG["settings"].copy())
+
+    @staticmethod
+    def path_for(vault_path: str) -> str:
+        return vault_path + ".config.json"
+
+    @classmethod
+    def load(cls, vault_path: str) -> "VaultConfig":
+        p = cls.path_for(vault_path)
+        if not os.path.exists(p):
+            return cls()
+        with open(p, encoding="utf-8") as f:
+            data = json.load(f)
+        cfg = cls()
+        cfg.callers = data.get("callers", cfg.callers)
+        cfg.settings = {**cfg.settings, **data.get("settings", {})}
+        return cfg
+
+    def save(self, vault_path: str) -> None:
+        with open(self.path_for(vault_path), "w", encoding="utf-8") as f:
+            json.dump({"callers": self.callers, "settings": self.settings}, f, indent=2)
+
+    # -- ACL ---------------------------------------------------------------
+
+    def _caller_entry(self, caller: str) -> dict:
+        entry = self.callers.get(caller) or self.callers.get("*")
+        if entry is None:
+            raise AclError(f"Caller {caller!r} has no access to this vault")
+        return entry
+
+    def default_namespace(self, caller: str) -> str:
+        return self._caller_entry(caller).get("default_namespace", "main")
+
+    def grant_for(self, caller: str, namespace: str) -> str:
+        """Returns 'rw', 'ro', or raises AclError. packs/* is always ro."""
+        entry = self._caller_entry(caller)
+        grants = entry.get("grants", {})
+        grant = grants.get(namespace)
+        if grant is None:
+            for pattern, g in grants.items():
+                if pattern.endswith("*") and namespace.startswith(pattern[:-1]):
+                    grant = g
+                    break
+        if grant is None:
+            grant = grants.get("*")
+        if grant is None:
+            raise AclError(
+                f"Caller {caller!r} is not granted access to namespace {namespace!r}")
+        if namespace.startswith("packs/"):
+            return "ro"  # pack namespaces are immutable for everyone
+        return grant
+
+    def check(self, caller: str, namespace: str, write: bool) -> None:
+        grant = self.grant_for(caller, namespace)
+        if write and grant != "rw":
+            raise AclError(
+                f"Caller {caller!r} has read-only access to namespace {namespace!r}")

compartment/audit.py

@@ -0,0 +1,54 @@
+"""Hash-chained, tamper-evident audit log (stored inside the sealed payload).
+
+Each entry's hash covers the previous entry's hash, so any edit, deletion,
+or reordering of history breaks the chain at a detectable point.
+A failure to write the audit entry fails the operation (fail-fast), never
+the other way around.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+
+GENESIS = "GENESIS"
+
+
+def _entry_hash(prev_hash: str, ts: float, caller: str, op: str, detail: str) -> str:
+    body = json.dumps(
+        {"prev": prev_hash, "ts": ts, "caller": caller, "op": op, "detail": detail},
+        sort_keys=True, separators=(",", ":"),
+    ).encode()
+    return hashlib.sha256(body).hexdigest()
+
+
+def append(conn, caller: str, op: str, detail: str) -> str:
+    row = conn.execute("SELECT hash FROM audit ORDER BY seq DESC LIMIT 1").fetchone()
+    prev = row["hash"] if row else GENESIS
+    ts = time.time()
+    h = _entry_hash(prev, ts, caller, op, detail)
+    conn.execute(
+        "INSERT INTO audit (ts, caller, op, detail, prev_hash, hash) VALUES (?,?,?,?,?,?)",
+        (ts, caller, op, detail, prev, h),
+    )
+    return h
+
+
+def head(conn) -> str:
+    row = conn.execute("SELECT hash FROM audit ORDER BY seq DESC LIMIT 1").fetchone()
+    return row["hash"] if row else GENESIS
+
+
+def verify(conn) -> tuple[bool, int, str]:
+    """Walk the chain. Returns (ok, entries_checked, message)."""
+    prev = GENESIS
+    n = 0
+    for row in conn.execute("SELECT * FROM audit ORDER BY seq"):
+        if row["prev_hash"] != prev:
+            return False, n, f"chain break at seq {row['seq']}: prev_hash mismatch"
+        want = _entry_hash(prev, row["ts"], row["caller"], row["op"], row["detail"])
+        if row["hash"] != want:
+            return False, n, f"chain break at seq {row['seq']}: entry hash mismatch"
+        prev = row["hash"]
+        n += 1
+    return True, n, f"audit chain intact ({n} entries)"

compartment/bench.py

@@ -0,0 +1,94 @@
+"""Performance + RAM benchmark against the seed pack and a synthetic corpus.
+
+Reports embed/store/search latencies (p50/p95) and peak RSS, and checks the
+8GB-baseline budgets: search p95 < 100ms at the synthetic scale, total RSS
+< 1GB at 100k records with default settings.
+"""
+from __future__ import annotations
+
+import random
+import sys
+import time
+
+import numpy as np
+
+try:
+    import resource            # POSIX only; absent on Windows
+except ImportError:            # pragma: no cover - Windows
+    resource = None
+
+from .vindex import build_index
+
+WORDS = ("report vault memory agent record office data schedule market key "
+         "index search secure backup ledger review batch upload form note").split()
+
+
+def _rss_mb() -> float:
+    if resource is None:       # Windows: peak-RSS via getrusage is unavailable
+        return 0.0
+    rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+    return rss / (1024 * 1024) if sys.platform == "darwin" else rss / 1024
+
+
+def _pct(xs: list[float], p: float) -> float:
+    xs = sorted(xs)
+    return xs[min(len(xs) - 1, int(len(xs) * p))]
+
+
+def run(vault, synthetic_n: int = 20_000, queries: int = 50) -> dict:
+    rng = random.Random(42)
+    out: dict = {"synthetic_records": synthetic_n}
+
+    # 1) real embed+store latency on a small sample
+    t_store = []
+    for i in range(20):
+        text = f"benchmark memory {i}: " + " ".join(rng.choices(WORDS, k=10))
+        t0 = time.perf_counter()
+        vault.store(text, caller="bench", namespace="bench", tags=["bench"])
+        t_store.append((time.perf_counter() - t0) * 1000)
+    out["store_ms_p50"] = round(_pct(t_store, 0.50), 1)
+    out["store_ms_p95"] = round(_pct(t_store, 0.95), 1)
+
+    # 2) synthetic vector corpus at scale (index-only: isolates search speed)
+    dim = int(vault.header.model["dim"])
+    mat = np.random.default_rng(42).standard_normal((synthetic_n, dim)).astype(np.float32)
+    mat /= np.linalg.norm(mat, axis=1, keepdims=True)
+    t0 = time.perf_counter()
+    idx = build_index(dim, list(range(1, synthetic_n + 1)), mat,
+                      precision=vault.config.settings.get("index_precision", "f32"),
+                      force=None)
+    out["index_build_s"] = round(time.perf_counter() - t0, 2)
+    out["index_kind"] = idx.kind
+
+    t_search = []
+    for _ in range(queries):
+        q = mat[rng.randrange(synthetic_n)]
+        t0 = time.perf_counter()
+        idx.search(q, 10)
+        t_search.append((time.perf_counter() - t0) * 1000)
+    out["vector_search_ms_p50"] = round(_pct(t_search, 0.50), 2)
+    out["vector_search_ms_p95"] = round(_pct(t_search, 0.95), 2)
+
+    # 3) full hybrid search on the live vault (embed + vector + FTS + fuse)
+    t_hybrid = []
+    for _ in range(20):
+        t0 = time.perf_counter()
+        vault.search("benchmark memory " + rng.choice(WORDS), caller="bench", top_k=8)
+        t_hybrid.append((time.perf_counter() - t0) * 1000)
+    out["hybrid_search_ms_p50"] = round(_pct(t_hybrid, 0.50), 1)
+    out["hybrid_search_ms_p95"] = round(_pct(t_hybrid, 0.95), 1)
+
+    out["peak_rss_mb"] = round(_rss_mb(), 0)
+    out["budgets"] = {
+        "vector_search_p95_under_100ms": out["vector_search_ms_p95"] < 100,
+        "rss_under_1gb": out["peak_rss_mb"] < 1024,
+    }
+    # clean up bench records
+    rows = vault.db.conn.execute(
+        "SELECT id, ikey FROM records WHERE ns = 'bench'").fetchall()
+    for r in rows:
+        vault.db.delete(r["id"], shred=False)
+        vault.index.remove(r["ikey"])
+        vault._id_by_ikey.pop(r["ikey"], None)
+    vault.save()
+    return out

compartment/claude_hooks.py

@@ -0,0 +1,207 @@
+"""Deterministic capture: mirror Claude Code memory writes into the vault.
+
+Telling a model to prefer compartment is a request, and a host that declares its
+own memory in the system prompt outranks anything a tool says. A hook does
+not ask. When Claude Code writes a file into its memory directory, this hook
+runs and the fact lands in the vault whether or not the model ever thought
+about compartment.
+
+Two halves:
+
+* `install()` merges an compartment entry into ~/.claude/settings.json - additive,
+  idempotent, and it backs the file up first. Other people's hooks are left
+  exactly as they are; only compartment's own entries are replaced on re-install.
+* `capture()` is what the hook command runs. It reads Claude Code's hook JSON
+  on stdin, and if the written file is a memory file, stores it. It exits 0
+  no matter what: a memory tool must never break the user's editor.
+"""
+from __future__ import annotations
+
+from .home import env, home
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+
+from . import claude_memory
+
+SETTINGS = Path.home() / ".claude" / "settings.json"
+# Identifies OUR entries. Both tokens, not one literal string: the command
+# carries `--vault ...` between them when a vault is pinned, so a single
+# substring would stop matching and re-installing would duplicate the hook.
+MARKER = "hook capture"
+_OWNER = "compartment"
+EVENT = "PostToolUse"
+MATCHER = "Write|Edit|MultiEdit|NotebookEdit"
+TIMEOUT = 15
+
+
+def _entry(command: str) -> dict:
+    return {"matcher": MATCHER,
+            "hooks": [{"type": "command", "command": command,
+                       "timeout": TIMEOUT}]}
+
+
+def hook_command(compartment_bin: str | None = None, vault: str | None = None) -> str:
+    """The exact shell command the hook runs."""
+    exe = compartment_bin or shutil.which("compartment") or "compartment"
+    if " " in exe:
+        exe = f'"{exe}"'
+    vault_part = f' --vault "{vault}"' if vault else ""
+    return f"{exe}{vault_part} hook capture"
+
+
+def is_ours(command: object) -> bool:
+    """Whether a configured hook command belongs to compartment."""
+    c = str(command or "")
+    return MARKER in c and _OWNER in c
+
+
+def is_installed(settings: Path | None = None) -> bool:
+    data = _read(settings or SETTINGS)
+    for group in (data.get("hooks", {}) or {}).get(EVENT, []) or []:
+        for h in group.get("hooks", []) or []:
+            if is_ours(h.get("command")):
+                return True
+    return False
+
+
+def _read(path: Path) -> dict:
+    if not path.exists():
+        return {}
+    try:
+        text = path.read_text(encoding="utf-8").strip()
+        return json.loads(text) if text else {}
+    except (json.JSONDecodeError, OSError):
+        raise ValueError(f"{path} is not valid JSON; refusing to touch it")
+
+
+def install(compartment_bin: str | None = None, vault: str | None = None,
+            settings: Path | None = None) -> dict:
+    """Add (or refresh) compartment's capture hook. Never removes anyone else's."""
+    path = Path(settings) if settings else SETTINGS
+    data = _read(path)                                  # raises on malformed
+    backup = None
+    if path.exists():
+        backup = path.with_suffix(path.suffix + ".compartment-backup")
+        shutil.copy2(path, backup)
+
+    hooks = data.setdefault("hooks", {})
+    groups = hooks.setdefault(EVENT, [])
+    # drop only our own previous entries, so re-installing updates in place
+    kept = []
+    for group in groups:
+        inner = [h for h in (group.get("hooks") or [])
+                 if not is_ours(h.get("command"))]
+        if inner:
+            kept.append({**group, "hooks": inner})
+        elif not group.get("hooks"):
+            kept.append(group)
+    kept.append(_entry(hook_command(compartment_bin, vault)))
+    hooks[EVENT] = kept
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    tmp.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+    os.replace(tmp, path)
+    return {"settings": str(path), "backup": str(backup) if backup else None,
+            "event": EVENT, "matcher": MATCHER}
+
+
+def uninstall(settings: Path | None = None) -> bool:
+    path = Path(settings) if settings else SETTINGS
+    data = _read(path)
+    groups = (data.get("hooks", {}) or {}).get(EVENT)
+    if not groups:
+        return False
+    kept, removed = [], False
+    for group in groups:
+        inner = [h for h in (group.get("hooks") or [])
+                 if not is_ours(h.get("command"))]
+        if len(inner) != len(group.get("hooks") or []):
+            removed = True
+        if inner or not group.get("hooks"):
+            kept.append({**group, "hooks": inner})
+    if not removed:
+        return False
+    data["hooks"][EVENT] = kept
+    path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+    return True
+
+
+# ------------------------------------------------------------------ capture
+
+def _file_from_payload(payload: dict) -> str | None:
+    """Claude Code puts the edited path under tool_input; tolerate variants
+    rather than silently capturing nothing if a field is renamed."""
+    for key in ("tool_input", "toolInput", "input"):
+        block = payload.get(key)
+        if isinstance(block, dict):
+            for f in ("file_path", "filePath", "path", "notebook_path"):
+                if block.get(f):
+                    return str(block[f])
+    for f in ("file_path", "filePath", "path"):
+        if payload.get(f):
+            return str(payload[f])
+    return None
+
+
+def is_memory_file(path: str | Path) -> bool:
+    """True for Claude Code's own memory notes - and nothing else. The index
+    is a table of contents, so it is skipped like the importer skips it."""
+    p = Path(path)
+    if p.suffix.lower() != ".md" or p.name == claude_memory.INDEX_NAME:
+        return False
+    parts = [x.lower() for x in p.parts]
+    return "memory" in parts and ".claude" in parts
+
+
+def capture(stream=None, vault_path: str | None = None) -> dict:
+    """Run by the hook. Returns a result dict; never raises, never blocks."""
+    stream = stream or sys.stdin
+    try:
+        raw = stream.read()
+    except Exception:                                   # noqa: BLE001
+        return {"stored": False, "reason": "unreadable stdin"}
+    if not raw or not raw.strip():
+        return {"stored": False, "reason": "empty payload"}
+    try:
+        payload = json.loads(raw)
+    except json.JSONDecodeError:
+        return {"stored": False, "reason": "payload is not JSON"}
+
+    target = _file_from_payload(payload)
+    if not target:
+        return {"stored": False, "reason": "no file path in payload"}
+    if not is_memory_file(target):
+        return {"stored": False, "reason": "not a memory file"}
+    p = Path(target)
+    if not p.exists():
+        return {"stored": False, "reason": "file no longer exists"}
+
+    from .crypto import CryptoError
+    from .vault import Vault
+    vp = vault_path or env("VAULT", str(home() / "memory.vault"))
+    if not os.path.exists(vp):
+        return {"stored": False, "reason": "no vault"}
+    try:
+        pw, key = Vault.resolve_credential(vp)
+        v = Vault.unlock(vp, passphrase=pw, raw_key=key)
+    except CryptoError:
+        # A locked vault is the user's choice, not an error worth shouting
+        # about mid-edit; the next import-claude sweeps up whatever was missed.
+        return {"stored": False, "reason": "vault locked"}
+    try:
+        rec = claude_memory.parse(p)
+        out = v.store(rec["text"], caller="claude-code-hook",
+                      tags=rec["tags"], importance=rec["importance"])
+        v.save()
+    except Exception as exc:                            # noqa: BLE001
+        return {"stored": False, "reason": f"store failed: {exc}"}
+    return {"stored": not out.get("duplicate"), "id": out.get("id"),
+            "duplicate": bool(out.get("duplicate")), "name": rec["name"]}
+
+
+__all__ = ["install", "uninstall", "is_installed", "is_ours", "capture", "hook_command",
+           "is_memory_file", "SETTINGS", "MARKER", "EVENT", "MATCHER"]

compartment/claude_memory.py

@@ -0,0 +1,183 @@
+"""Import Claude Code's file-based memories into the vault.
+
+Claude Code keeps a per-project memory directory of Markdown files, each
+one a single fact with YAML-ish frontmatter:
+
+    ---
+    name: some-slug
+    description: one-line summary
+    metadata:
+      type: user | feedback | project | reference
+    ---
+    the fact itself
+
+That store is local to one project, unencrypted, and invisible to every
+other agent and host. This module moves those facts into the vault, where
+they are encrypted at rest and shared across every agent that speaks to
+compartment. Nothing is deleted: the source files are read, never modified, so
+an import is always safe to re-run.
+
+Re-running is a no-op by design - Vault.store's near-duplicate guard
+recognises text it has already embedded and returns the existing record
+instead of a second copy.
+"""
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+# Claude Code's per-project memory lives under <root>/<project-slug>/memory.
+DEFAULT_ROOT = Path.home() / ".claude" / "projects"
+INDEX_NAME = "MEMORY.md"          # a table of contents, not a fact
+
+# Frontmatter `type` -> Compartment importance. Mirrors the vault's own tiers:
+# who the user is and what they have decided outranks reference material.
+IMPORTANCE = {
+    "user": 0.85,
+    "feedback": 0.85,
+    "project": 0.75,
+    "reference": 0.60,
+}
+DEFAULT_IMPORTANCE = 0.70
+
+_FRONTMATTER = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?", re.DOTALL)
+
+
+def discover(root: Path | str | None = None) -> list[Path]:
+    """Every Claude Code memory file, sorted, index files excluded."""
+    root = Path(root) if root else DEFAULT_ROOT
+    if not root.exists():
+        return []
+    if root.name == "memory" and root.is_dir():      # a memory dir directly
+        dirs = [root]
+    else:
+        dirs = sorted(p for p in root.glob("*/memory") if p.is_dir())
+        if not dirs and (root / "memory").is_dir():
+            dirs = [root / "memory"]
+    out: list[Path] = []
+    for d in dirs:
+        out.extend(sorted(p for p in d.glob("*.md")
+                          if p.name != INDEX_NAME and p.is_file()))
+    return out
+
+
+def _parse_frontmatter(raw: str) -> tuple[dict, str]:
+    """Return (fields, body). Hand-rolled: the frontmatter Compartment cares
+    about is flat scalars plus a one-level `metadata:` block, so pulling in
+    a YAML dependency (and its parser CVEs) would be a poor trade."""
+    m = _FRONTMATTER.match(raw)
+    if not m:
+        return {}, raw.strip()
+    fields: dict[str, str] = {}
+    section = None
+    for line in m.group(1).splitlines():
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+        indented = line[:1].isspace()
+        key, sep, val = line.strip().partition(":")
+        if not sep:
+            continue
+        key, val = key.strip(), val.strip().strip("'\"")
+        if not val:                      # a nested block opens, e.g. metadata:
+            section = key
+            continue
+        fields[f"{section}.{key}" if indented and section else key] = val
+    return fields, raw[m.end():].strip()
+
+
+def parse(path: Path) -> dict:
+    """One memory file -> the record Compartment should store."""
+    raw = path.read_text(encoding="utf-8", errors="replace")
+    fields, body = _parse_frontmatter(raw)
+    name = fields.get("name") or path.stem
+    description = fields.get("description", "")
+    kind = (fields.get("metadata.type") or fields.get("type") or "").lower()
+
+    # The description is a standalone summary and the body is the fact; a
+    # record carrying both retrieves well on either phrasing. Skip the
+    # description when the body already opens with it.
+    text = body
+    if description and not body.lower().startswith(description.lower()[:40]):
+        text = f"{description}\n\n{body}".strip() if body else description
+    if not text.strip():
+        text = description or name
+
+    tags = ["claude-memory"]
+    if kind:
+        tags.append(kind)
+    if name and name not in tags:
+        tags.append(name)
+    return {
+        "text": text,
+        "tags": tags,
+        "importance": IMPORTANCE.get(kind, DEFAULT_IMPORTANCE),
+        "name": name,
+        "source": str(path),
+    }
+
+
+def import_files(vault, files: list[Path], caller: str = "import-claude",
+                 namespace: str | None = None, dry_run: bool = False) -> dict:
+    """Store each parsed memory. Returns counts plus per-file outcomes.
+
+    Duplicates are not an error: re-importing an unchanged directory stores
+    nothing new, so this is safe to run on every `compartment integrate claude`.
+    """
+    imported = duplicates = failed = 0
+    items, errors = [], []
+    for f in files:
+        try:
+            rec = parse(f)
+        except OSError as exc:
+            failed += 1
+            errors.append(f"{f.name}: {exc}")
+            continue
+        if dry_run:
+            imported += 1
+            items.append({"name": rec["name"], "importance": rec["importance"],
+                          "tags": rec["tags"], "chars": len(rec["text"])})
+            continue
+        try:
+            out = vault.store(rec["text"], caller=caller, namespace=namespace,
+                              tags=rec["tags"], importance=rec["importance"])
+        except Exception as exc:                       # noqa: BLE001
+            failed += 1
+            errors.append(f"{f.name}: {exc}")
+            continue
+        if out.get("duplicate"):
+            duplicates += 1
+        else:
+            imported += 1
+        items.append({"name": rec["name"], "id": out.get("id"),
+                      "duplicate": bool(out.get("duplicate"))})
+    return {"scanned": len(files), "imported": imported,
+            "duplicates": duplicates, "failed": failed,
+            "dry_run": dry_run, "items": items, "errors": errors}
+
+
+def pending(vault, root: Path | str | None = None) -> int:
+    """How many Claude Code memory files are not in the vault yet.
+
+    Cheap enough for `compartment status` to call: it compares file count against
+    records already tagged `claude-memory`, no embedding involved.
+    """
+    files = discover(root)
+    if not files:
+        return 0
+    try:
+        rows = vault.db.conn.execute(
+            "SELECT COUNT(*) AS n FROM records WHERE tags LIKE '%claude-memory%'"
+        ).fetchone()
+        have = int(rows["n"] if hasattr(rows, "keys") else rows[0])
+    except Exception:                                   # noqa: BLE001
+        return len(files)
+    return max(0, len(files) - have)
+
+
+def default_root_exists() -> bool:
+    return DEFAULT_ROOT.exists() and bool(discover())
+
+
+__all__ = ["DEFAULT_ROOT", "discover", "parse", "import_files", "pending",
+           "default_root_exists", "IMPORTANCE"]