compartment 2.1__tar.gz → 2.3__tar.gz

{compartment-2.1 → compartment-2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: compartment
-Version: 2.1
+Version: 2.3
 Summary: Compartment - high-security, fully offline, encrypted vector memory for AI agents (MCP)
 Project-URL: Homepage, https://github.com/MaxFreedomPollard/Compartment
 Project-URL: Source, https://github.com/MaxFreedomPollard/Compartment

{compartment-2.1 → compartment-2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "compartment"
-version = "2.1"
+version = "2.3"
 description = "Compartment - high-security, fully offline, encrypted vector memory for AI agents (MCP)"
 readme = "README.md"
 requires-python = ">=3.11"

{compartment-2.1 → compartment-2.3}/src/compartment/__init__.py

@@ -1,6 +1,6 @@
 """Compartment - high-security, fully offline, encrypted vector memory for AI agents."""
 
-__version__ = "2.1"
+__version__ = "2.3"
 
 from . import offline_guard as _og
 

compartment-2.3/src/compartment/audit.py

@@ -0,0 +1,103 @@
+"""Hash-chained, tamper-evident audit log (stored inside the sealed payload).
+
+Each entry's hash covers the previous entry's hash, so any edit, deletion,
+or reordering of history breaks the chain at a detectable point.
+A failure to write the audit entry fails the operation (fail-fast), never
+the other way around.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+
+GENESIS = "GENESIS"
+
+
+def _entry_hash(prev_hash: str, ts: float, caller: str, op: str, detail: str) -> str:
+    body = json.dumps(
+        {"prev": prev_hash, "ts": ts, "caller": caller, "op": op, "detail": detail},
+        sort_keys=True, separators=(",", ":"),
+    ).encode()
+    return hashlib.sha256(body).hexdigest()
+
+
+def append(conn, caller: str, op: str, detail: str, ts: float | None = None) -> str:
+    """Add an entry, linked to whatever is currently at the head.
+
+    `ts` is only passed when replaying a journalled operation, so the entry
+    keeps the time it actually happened. It is still linked to the head as it
+    stands at replay time, never to the head the original writer saw: that
+    head may never have reached disk (read operations are audited in RAM and
+    persist only on save), and chaining to an entry nobody else has is what
+    leaves a dangling link no future verify can resolve."""
+    row = conn.execute("SELECT hash FROM audit ORDER BY seq DESC LIMIT 1").fetchone()
+    prev = row["hash"] if row else GENESIS
+    ts = time.time() if ts is None else ts
+    h = _entry_hash(prev, ts, caller, op, detail)
+    conn.execute(
+        "INSERT INTO audit (ts, caller, op, detail, prev_hash, hash) VALUES (?,?,?,?,?,?)",
+        (ts, caller, op, detail, prev, h),
+    )
+    return h
+
+
+def head(conn) -> str:
+    row = conn.execute("SELECT hash FROM audit ORDER BY seq DESC LIMIT 1").fetchone()
+    return row["hash"] if row else GENESIS
+
+
+def verify(conn) -> tuple[bool, int, str]:
+    """Walk the chain. Returns (ok, entries_checked, message)."""
+    prev = GENESIS
+    n = 0
+    for row in conn.execute("SELECT * FROM audit ORDER BY seq"):
+        if row["prev_hash"] != prev:
+            return False, n, f"chain break at seq {row['seq']}: prev_hash mismatch"
+        want = _entry_hash(prev, row["ts"], row["caller"], row["op"], row["detail"])
+        if row["hash"] != want:
+            return False, n, f"chain break at seq {row['seq']}: entry hash mismatch"
+        prev = row["hash"]
+        n += 1
+    return True, n, f"audit chain intact ({n} entries)"
+
+
+def relink(conn) -> tuple[int, int | None]:
+    """Re-link entries whose prev_hash points at something not in the log.
+
+    Only for repairing damage done by builds before this one, which replayed a
+    journalled entry with the head its original writer saw rather than the head
+    the log actually has. The result was a permanent dangling link: verify
+    stopped at it and never reported anything after it again.
+
+    What this does NOT do is paper over an edit. Every entry's own hash covers
+    its content, so content tampering is caught by the self-hash check and this
+    refuses to touch a log that fails it. Only the links are rebuilt, and only
+    forward from the first break. Nothing is deleted, reordered or reworded:
+    ts, caller, op and detail are exactly what they were.
+
+    Returns (entries relinked, seq of the first break) - (0, None) if intact."""
+    rows = conn.execute("SELECT * FROM audit ORDER BY seq").fetchall()
+    for row in rows:
+        want = _entry_hash(row["prev_hash"], row["ts"], row["caller"], row["op"],
+                           row["detail"])
+        if row["hash"] != want:
+            raise ValueError(
+                f"audit entry seq {row['seq']} does not hash to its own content. "
+                "That is content tampering, not a dangling link, and relinking "
+                "would destroy the evidence. Refusing.")
+
+    prev, first, changed = GENESIS, None, 0
+    for row in rows:
+        if row["prev_hash"] != prev:
+            if first is None:
+                first = row["seq"]
+        if first is not None:                    # rewrite everything after it
+            h = _entry_hash(prev, row["ts"], row["caller"], row["op"], row["detail"])
+            conn.execute("UPDATE audit SET prev_hash = ?, hash = ? WHERE seq = ?",
+                         (prev, h, row["seq"]))
+            changed += 1
+            prev = h
+        else:
+            prev = row["hash"]
+    return changed, first

{compartment-2.1 → compartment-2.3}/src/compartment/cli.py

@@ -468,6 +468,25 @@ def cmd_audit(args) -> None:
         sys.exit(2)
 
 
+def cmd_audit_repair(args) -> None:
+    v = _open_vault(args)
+    try:
+        changed, first = audit.relink(v.db.conn)
+    except ValueError as exc:
+        _die(str(exc))
+    if not changed:
+        _print({"ok": True, "relinked": 0, "message": "audit chain already intact"})
+        return
+    v._audit_and_capture("user", "audit-repair",
+                         f"relinked {changed} entries from seq {first}")
+    v.save()
+    ok, n, msg = audit.verify(v.db.conn)
+    _print({"ok": ok, "relinked": changed, "first_break_at_seq": first,
+            "entries": n, "message": msg, "head": audit.head(v.db.conn)})
+    if not ok:
+        sys.exit(2)
+
+
 def cmd_verify(args) -> None:
     loaded = read_vault_file(args.vault)   # structure + format checks
     out = {"vault": args.vault, "format": "ok",
@@ -1165,6 +1184,8 @@ def main(argv: list[str] | None = None) -> None:
     pa_sub = pa.add_subparsers(dest="audit_cmd", required=True)
     p = pa_sub.add_parser("verify", help="verify the hash chain")
     p.set_defaults(fn=cmd_audit)
+    p = pa_sub.add_parser("repair", help="re-link a chain broken by an older build")
+    p.set_defaults(fn=cmd_audit_repair)
 
     p = sub.add_parser("verify", help="check vault structure + signed manifest")
     p.set_defaults(fn=cmd_verify)

{compartment-2.1 → compartment-2.3}/src/compartment/crypto.py

@@ -19,6 +19,8 @@ from nacl.bindings import (
     crypto_aead_xchacha20poly1305_ietf_encrypt,
 )
 
+from . import wire
+
 KEY_LEN = 32
 NONCE_LEN = 24
 
@@ -81,6 +83,35 @@ def unseal(key: bytes, blob: bytes, aad: bytes = b"") -> bytes:
         ) from exc
 
 
+def unseal_which(key: bytes, blob: bytes, aad: bytes,
+                 legacy: tuple[bytes, ...] = ()) -> tuple[bytes, bytes]:
+    """unseal(), accepting associated data written by an older build.
+
+    Returns (plaintext, the aad that actually authenticated), so a caller can
+    tell a migrated blob from one still carrying a legacy label and rewrite it.
+
+    The current spelling is tried first, so anything already migrated costs
+    exactly one AEAD open. Legacy spellings are tried in order only after that
+    fails, which is what lets a pre-rename vault open at all. If none of them
+    authenticate the data really is tampered with, or the key really is wrong,
+    and the original error is raised."""
+    try:
+        return unseal(key, blob, aad=aad), aad
+    except TamperError:
+        for old in legacy:
+            try:
+                return unseal(key, blob, aad=old), old
+            except TamperError:
+                continue
+        raise
+
+
+def unseal_any(key: bytes, blob: bytes, aad: bytes,
+               legacy: tuple[bytes, ...] = ()) -> bytes:
+    """unseal_which() when the caller does not care which label was used."""
+    return unseal_which(key, blob, aad, legacy)[0]
+
+
 def wipe(buf: bytearray) -> None:
     """Best-effort zeroization (Python cannot guarantee no copies exist)."""
     for i in range(len(buf)):
@@ -99,7 +130,7 @@ def make_passphrase_slot(master_key: bytes, passphrase: str, kdf: dict | None =
         "type": "passphrase",
         "kdf": kdf,
         "salt": salt.hex(),
-        "wrapped": seal(wrap_key, master_key, aad=b"engram-keyslot").hex(),
+        "wrapped": seal(wrap_key, master_key, aad=wire.KEYSLOT).hex(),
     }
 
 
@@ -118,7 +149,7 @@ def make_recovery_slot(master_key: bytes, kdf: dict | None = None) -> tuple[dict
 # Domain separator between the knowledge factor (passphrase) and the
 # possession factor (keyfile bytes) before the KDF. Both factors feed
 # Argon2id, so two-factor unlock is enforced by arithmetic, not policy.
-KEYFILE_SEP = b"\x1f engram-2fa \x1f"
+KEYFILE_SEP = wire.KEYFILE_SEP
 KEYFILE_LEN = 32
 
 
@@ -137,19 +168,57 @@ def make_keyfile_slot(master_key: bytes, passphrase: str, keyfile: bytes,
         "kdf": kdf,
         "salt": salt.hex(),
         "keyfile_id": sha256(keyfile)[:16],   # UX only: detect the WRONG file
-        "wrapped": seal(wrap_key, master_key, aad=b"engram-keyslot").hex(),
+        "wrapped": seal(wrap_key, master_key, aad=wire.KEYSLOT).hex(),
     }
 
 
-def open_slot(slot: dict, secret: str, keyfile: bytes | None = None) -> bytes:
-    """Unwrap the master key from one slot."""
+def _slot_secrets(slot: dict, secret: str, keyfile: bytes | None) -> list[bytes]:
+    """The KDF inputs worth trying for this slot, current spelling first.
+
+    Only a two-factor slot has more than one. Its separator is a KDF input
+    rather than associated data, so an older separator derives a different
+    wrap key and can only be ruled out by deriving again."""
     raw = secret.encode("utf-8")
-    if slot["type"] == "passphrase+keyfile":
-        if keyfile is None:
-            raise CryptoError("This keyslot requires a keyfile")
-        raw = raw + KEYFILE_SEP + keyfile
-    wrap_key = derive_key(raw, bytes.fromhex(slot["salt"]), slot["kdf"])
-    return unseal(wrap_key, bytes.fromhex(slot["wrapped"]), aad=b"engram-keyslot")
+    if slot["type"] != "passphrase+keyfile":
+        return [raw]
+    if keyfile is None:
+        raise CryptoError("This keyslot requires a keyfile")
+    return [raw + sep + keyfile
+            for sep in (wire.KEYFILE_SEP, *wire.KEYFILE_SEP_LEGACY)]
+
+
+def open_slot(slot: dict, secret: str, keyfile: bytes | None = None) -> bytes:
+    """Unwrap the master key from one slot.
+
+    A slot written before 2.2 carries the legacy associated data, and a
+    two-factor one the legacy keyfile separator. Both still open. The slot is
+    then rewritten in place under the current spelling, so it migrates the
+    first time its credential is used; the caller persists that by saving the
+    header. Re-sealing only happens when something was actually legacy, or
+    every unlock would dirty the header and force a needless vault rewrite."""
+    salt, kdf = bytes.fromhex(slot["salt"]), slot["kdf"]
+    wrapped = bytes.fromhex(slot["wrapped"])
+    failure: TamperError | None = None
+    for i, raw in enumerate(_slot_secrets(slot, secret, keyfile)):
+        wrap_key = derive_key(raw, salt, kdf)
+        try:
+            master, used = unseal_which(wrap_key, wrapped, wire.KEYSLOT,
+                                        wire.KEYSLOT_LEGACY)
+        except TamperError as exc:
+            failure = exc
+            continue
+        if i:
+            # legacy separator: the wrap key itself is stale, so the slot has
+            # to be built again from scratch (a second Argon2id pass, once).
+            fresh = make_keyfile_slot(master, secret, keyfile, kdf)
+            fresh["type"] = slot["type"]
+            slot.clear()
+            slot.update(fresh)
+        elif used != wire.KEYSLOT:
+            # same wrap key, only the associated data moves. Free.
+            slot["wrapped"] = seal(wrap_key, master, aad=wire.KEYSLOT).hex()
+        return master
+    raise failure if failure is not None else TamperError("Keyslot did not open")
 
 
 def unwrap_master(keyslots: list[dict], secret: str,
@@ -190,16 +259,20 @@ def normalize_recovery(text: str) -> str:
 # Per-record keys → crypto-shred
 # ---------------------------------------------------------------------------
 
+def wrap_record_key(master_key: bytes, record_id: str, record_key: bytes) -> bytes:
+    aad, _ = wire.record_key(record_id)
+    return seal(master_key, record_key, aad=aad)
+
+
 def new_record_key(master_key: bytes, record_id: str) -> tuple[bytes, bytes]:
     """Returns (record_key, wrapped_record_key). Destroying the wrapped key
     makes the record's ciphertext permanently undecryptable (crypto-shred)."""
     rk = new_key()
-    wrapped = seal(master_key, rk, aad=b"engram-record:" + record_id.encode())
-    return rk, wrapped
+    return rk, wrap_record_key(master_key, record_id, rk)
 
 
 def unwrap_record_key(master_key: bytes, record_id: str, wrapped: bytes) -> bytes:
-    return unseal(master_key, wrapped, aad=b"engram-record:" + record_id.encode())
+    return unseal_any(master_key, wrapped, *wire.record_key(record_id))
 
 
 # 256 short, common, distinct words for recovery phrases (16 words = 128 bits).

{compartment-2.1 → compartment-2.3}/src/compartment/menubar.py

@@ -397,6 +397,7 @@ def run(vault: str | None = None, show: bool = False,
                             NSFont, NSImage, NSMakeRect, NSMenu, NSMenuItem,
                             NSPanel, NSPopover, NSScreen, NSSegmentedControl,
                             NSClipView, NSScrollView,
+                            NSBitmapImageRep, NSGraphicsContext,
                             NSSecureTextField, NSStackView, NSStatusBar,
                             NSSwitch, NSTextField,
                             NSButton, NSView, NSViewController,
@@ -852,14 +853,33 @@ def run(vault: str | None = None, show: bool = False,
     _d("controller ready")
 
     if render_to:
-        # app.run() normally does this; without it the text system is not up
-        # and the snapshot comes out with controls but no glyphs at all.
+        # app.run() normally does this; without it the text system is not up.
         app.finishLaunching()
         body = ctrl.buildBody()
         body.layoutSubtreeIfNeeded()
         bounds = body.bounds()
-        rep = body.bitmapImageRepForCachingDisplayInRect_(bounds)
-        body.cacheDisplayInRect_toBitmapImageRep_(bounds, rep)
+        # Render through PDF, not cacheDisplayInRect. Caching a view that
+        # belongs to no window gives you the switches and the segmented
+        # control and not one glyph of text, because there is no window
+        # backing store for the text system to lay glyphs into. Ordering an
+        # offscreen window in does not help either: it draws as an inactive
+        # window, so even the controls lose their tint. The PDF path records
+        # glyphs vectorially with their fonts embedded and needs no window at
+        # all, so it captures the panel exactly as designed. Rasterise at 2x
+        # for a Retina-sharp asset.
+        pdf = body.dataWithPDFInsideRect_(bounds)
+        w, h = bounds.size.width, bounds.size.height
+        image = NSImage.alloc().initWithData_(pdf)
+        rep = (NSBitmapImageRep.alloc()
+               .initWithBitmapDataPlanes_pixelsWide_pixelsHigh_bitsPerSample_samplesPerPixel_hasAlpha_isPlanar_colorSpaceName_bytesPerRow_bitsPerPixel_(
+                   None, int(w * 2), int(h * 2), 8, 4, True, False,
+                   "NSCalibratedRGBColorSpace", 0, 0))
+        rep.setSize_((w, h))
+        NSGraphicsContext.saveGraphicsState()
+        NSGraphicsContext.setCurrentContext_(
+            NSGraphicsContext.graphicsContextWithBitmapImageRep_(rep))
+        image.drawInRect_(NSMakeRect(0, 0, w, h))
+        NSGraphicsContext.restoreGraphicsState()
         png = rep.representationUsingType_properties_(4, {})   # 4 = PNG
         ok = png.writeToFile_atomically_(render_to, True)
         print(f"rendered popover -> {render_to}" if ok

{compartment-2.1 → compartment-2.3}/src/compartment/packs.py

@@ -20,7 +20,7 @@ import struct
 import numpy as np
 from nacl.signing import SigningKey, VerifyKey
 
-from . import crypto
+from . import crypto, wire
 from .crypto import CryptoError, TamperError
 from .vaultfile import pack_sections, unpack_sections
 
@@ -71,7 +71,7 @@ def build_pack(*, name: str, version: str, description: str, records: list[dict]
     if passphrase is not None:
         salt = secrets.token_bytes(16)
         key = crypto.derive_key(passphrase.encode(), salt)
-        body = crypto.seal(key, body, aad=b"engram-pack:" + name.encode())
+        body = crypto.seal(key, body, aad=wire.pack(name)[0])
         header["kdf"] = crypto.DEFAULT_KDF
         header["salt"] = salt.hex()
     header["content_sha256"] = crypto.sha256(body)
@@ -119,7 +119,7 @@ def read_pack(blob: bytes, passphrase: str | None = None
             raise PackError(f"Pack {header['name']!r} is encrypted; passphrase required")
         key = crypto.derive_key(passphrase.encode(), bytes.fromhex(header["salt"]),
                                 header["kdf"])
-        body = crypto.unseal(key, body, aad=b"engram-pack:" + header["name"].encode())
+        body = crypto.unseal_any(key, body, *wire.pack(header["name"]))
 
     sections = unpack_sections(body)
     records = [json.loads(l) for l in

{compartment-2.1 → compartment-2.3}/src/compartment/session.py

@@ -25,7 +25,7 @@ import os
 import stat
 from pathlib import Path
 
-from . import crypto
+from . import crypto, wire
 from .crypto import CryptoError, TamperError
 from .platforms import boot_time, machine_id
 
@@ -49,17 +49,16 @@ def _uid() -> str:
     return str(getuid()) if getuid else os.environ.get("USERNAME", "user")
 
 
-def _boot_key() -> bytes:
+def _boot_key(token: str = wire.SESSION_TOKEN) -> bytes:
     """Wrap key valid only for this boot session of this user on this
     machine. Uses the stable hardware machine id, NOT the hostname -
     macOS renames the host per network, which must not relock the vault."""
-    token = "|".join((
-        "engram-session-v2",
+    return hashlib.sha256("|".join((
+        token,
         _boot_time(),
         _uid(),
         machine_id(),
-    ))
-    return hashlib.sha256(token.encode()).digest()
+    )).encode()).digest()
 
 
 def _canon(vault_path: str) -> str:
@@ -83,7 +82,7 @@ def store(vault_path: str, master_key: bytes) -> Path:
     """Persist a boot-bound unlock credential for this vault."""
     p = _file_for(vault_path)
     blob = crypto.seal(_boot_key(), master_key,
-                       aad=b"engram-session:" + _canon(vault_path).encode())
+                       aad=wire.session(_canon(vault_path))[0])
     p.write_text(json.dumps({"vault": _canon(vault_path),
                              "wrapped": blob.hex()}), encoding="utf-8")
     os.chmod(p, stat.S_IRUSR | stat.S_IWUSR)  # 0600
@@ -92,15 +91,29 @@ def store(vault_path: str, master_key: bytes) -> Path:
 
 def get(vault_path: str) -> bytes | None:
     """Return the master key if a credential exists AND we are still in the
-    same boot session; otherwise remove the stale file and return None."""
+    same boot session; otherwise remove the stale file and return None.
+
+    A credential written before 2.2 is bound to the older token and sealed
+    under the older associated data. It is accepted and then rewritten in the
+    current form, so upgrading Compartment does not relock a vault that was
+    already open."""
     p = _file_for(vault_path)
     if not p.is_file():
         return None
+    canon = _canon(vault_path)
+    aad, legacy_aad = wire.session(canon)
     try:
-        data = json.loads(p.read_text(encoding="utf-8"))
-        return crypto.unseal(
-            _boot_key(), bytes.fromhex(data["wrapped"]),
-            aad=b"engram-session:" + _canon(vault_path).encode())
+        blob = bytes.fromhex(json.loads(p.read_text(encoding="utf-8"))["wrapped"])
+        for token in (wire.SESSION_TOKEN, *wire.SESSION_TOKEN_LEGACY):
+            try:
+                key, used = crypto.unseal_which(_boot_key(token), blob,
+                                                aad, legacy_aad)
+            except TamperError:
+                continue
+            if token != wire.SESSION_TOKEN or used != aad:
+                store(vault_path, key)
+            return key
+        raise TamperError("No boot-session token opened the credential")
     except (TamperError, CryptoError, ValueError, KeyError, OSError):
         # different boot (restart/power loss) or corrupt file → locked
         try:

{compartment-2.1 → compartment-2.3}/src/compartment/store.py

@@ -14,7 +14,7 @@ import uuid
 
 import numpy as np
 
-from . import crypto
+from . import crypto, wire
 from .crypto import CryptoError
 
 SCHEMA = """
@@ -110,7 +110,7 @@ class Store:
         rid = record_id or uuid.uuid4().hex
         rk, wrapped = crypto.new_record_key(master_key, rid)
         ct = crypto.seal(rk, crypto.canonical_json({"text": text}),
-                         aad=b"engram-record-body:" + rid.encode())
+                         aad=wire.record_body(rid)[0])
         now = time.time()
         self.conn.execute(
             "INSERT INTO records (id, ikey, ns, ct, key_wrapped, vec, dim, tags, importance,"
@@ -129,9 +129,35 @@ class Store:
 
     def decrypt_text(self, row: sqlite3.Row, master_key: bytes) -> str:
         rk = crypto.unwrap_record_key(master_key, row["id"], row["key_wrapped"])
-        body = crypto.unseal(rk, row["ct"], aad=b"engram-record-body:" + row["id"].encode())
+        body = crypto.unseal_any(rk, row["ct"], *wire.record_body(row["id"]))
         return json.loads(body)["text"]
 
+    def migrate_wire(self, master_key: bytes) -> int:
+        """Rewrite every record under the current associated data.
+
+        Records are the one place the labels do not migrate for free: a row's
+        key wrap and ciphertext are written on insert and never touched again,
+        so without this pass every read of every pre-2.2 record would pay a
+        failed AEAD open before the one that works, forever. Vault.unlock
+        calls this once and records it in the header.
+
+        The record key is kept and re-wrapped rather than rotated, so the
+        plaintext is the only thing that round-trips. Nothing is written to
+        disk here: this mutates the in-memory database, and the caller's
+        save() is what makes it real, atomically. If any row fails to decrypt
+        the error propagates and no save happens, so a vault is never left
+        half-converted."""
+        rows = self.conn.execute("SELECT id, ct, key_wrapped FROM records").fetchall()
+        for row in rows:
+            rid = row["id"]
+            rk = crypto.unwrap_record_key(master_key, rid, row["key_wrapped"])
+            body = crypto.unseal_any(rk, row["ct"], *wire.record_body(rid))
+            self.conn.execute(
+                "UPDATE records SET ct = ?, key_wrapped = ? WHERE id = ?",
+                (crypto.seal(rk, body, aad=wire.record_body(rid)[0]),
+                 crypto.wrap_record_key(master_key, rid, rk), rid))
+        return len(rows)
+
     def delete(self, record_id: str, shred: bool) -> bool:
         """Delete a record. shred=True also VACUUMs so the content (including
         FTS tokens and freed pages) is gone from the next serialized image,