compartment 2.2__tar.gz → 3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compartment-2.2 → compartment-3.0}/PKG-INFO +11 -9
- {compartment-2.2 → compartment-3.0}/README.md +10 -8
- {compartment-2.2 → compartment-3.0}/pyproject.toml +1 -1
- {compartment-2.2 → compartment-3.0}/src/compartment/__init__.py +1 -1
- compartment-3.0/src/compartment/acl.py +140 -0
- compartment-3.0/src/compartment/audit.py +223 -0
- compartment-3.0/src/compartment/bench.py +158 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/claude_hooks.py +8 -1
- {compartment-2.2 → compartment-3.0}/src/compartment/cli.py +73 -14
- {compartment-2.2 → compartment-3.0}/src/compartment/crypto.py +59 -3
- {compartment-2.2 → compartment-3.0}/src/compartment/dash.py +285 -102
- {compartment-2.2 → compartment-3.0}/src/compartment/data/hermes-plugin/__init__.py +35 -22
- {compartment-2.2 → compartment-3.0}/src/compartment/data/starter.mpack +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/longmemeval.py +35 -14
- compartment-3.0/src/compartment/offline_guard.py +307 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/packs.py +100 -13
- {compartment-2.2 → compartment-3.0}/src/compartment/platforms.py +23 -8
- compartment-3.0/src/compartment/server.py +532 -0
- compartment-3.0/src/compartment/session.py +531 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/vault.py +104 -22
- {compartment-2.2 → compartment-3.0}/src/compartment/vaultfile.py +79 -10
- {compartment-2.2 → compartment-3.0}/src/compartment.egg-info/PKG-INFO +11 -9
- {compartment-2.2 → compartment-3.0}/src/compartment.egg-info/SOURCES.txt +8 -0
- {compartment-2.2 → compartment-3.0}/tests/test_acl_audit_packs.py +14 -8
- compartment-3.0/tests/test_acl_precedence.py +246 -0
- compartment-3.0/tests/test_attempt_pacing.py +96 -0
- compartment-3.0/tests/test_audit_chain.py +123 -0
- {compartment-2.2 → compartment-3.0}/tests/test_claude_hooks_and_integrate.py +33 -0
- compartment-3.0/tests/test_journal_framing.py +155 -0
- compartment-3.0/tests/test_offline_guard_paths.py +165 -0
- compartment-3.0/tests/test_offline_selftest_bench.py +171 -0
- {compartment-2.2 → compartment-3.0}/tests/test_ondisk_format.py +2 -1
- compartment-3.0/tests/test_pack_trust.py +188 -0
- compartment-3.0/tests/test_session_entropy.py +250 -0
- {compartment-2.2 → compartment-3.0}/tests/test_session_lock.py +20 -0
- compartment-3.0/tests/test_starter_data_quality.py +108 -0
- {compartment-2.2 → compartment-3.0}/tests/test_starter_packs.py +9 -9
- {compartment-2.2 → compartment-3.0}/tests/test_tamper.py +4 -3
- compartment-2.2/src/compartment/acl.py +0 -92
- compartment-2.2/src/compartment/audit.py +0 -54
- compartment-2.2/src/compartment/bench.py +0 -94
- compartment-2.2/src/compartment/offline_guard.py +0 -53
- compartment-2.2/src/compartment/server.py +0 -327
- compartment-2.2/src/compartment/session.py +0 -131
- compartment-2.2/tests/test_offline_selftest_bench.py +0 -81
- {compartment-2.2 → compartment-3.0}/setup.cfg +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/claude_memory.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/data/hermes-plugin/plugin.yaml +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/data/menubar.png +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/data/menubar@2x.png +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/data/tray.ico +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/embed.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/home.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/menubar.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/models/bge-small-en-v1.5-int8/model_quantized.onnx +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/models/bge-small-en-v1.5-int8/tokenizer.json +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/salience.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/selftest.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/store.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/systray.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/vindex.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment/wire.py +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment.egg-info/dependency_links.txt +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment.egg-info/entry_points.txt +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment.egg-info/requires.txt +0 -0
- {compartment-2.2 → compartment-3.0}/src/compartment.egg-info/top_level.txt +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_claude_memory_and_recent.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_crypto.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_dash.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_instructions.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_macos_bundle.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_menubar.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_platforms.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_relations.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_salience.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_systray.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_twofa.py +0 -0
- {compartment-2.2 → compartment-3.0}/tests/test_vault_ops.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: compartment
|
|
3
|
-
Version:
|
|
3
|
+
Version: 3.0
|
|
4
4
|
Summary: Compartment - high-security, fully offline, encrypted vector memory for AI agents (MCP)
|
|
5
5
|
Project-URL: Homepage, https://github.com/MaxFreedomPollard/Compartment
|
|
6
6
|
Project-URL: Source, https://github.com/MaxFreedomPollard/Compartment
|
|
@@ -51,7 +51,7 @@ Requires-Dist: usearch>=2.9; extra == "dev"
|
|
|
51
51
|
Your AI agent forgets you the moment the session ends. Compartment ends
|
|
52
52
|
that. With Compartment, your AI agent gets better with experience: it keeps
|
|
53
53
|
every decision, preference and detail you give it, permanently, encrypted,
|
|
54
|
-
on your own computer. Hermes, Claude,
|
|
54
|
+
on your own computer. Hermes, Claude, OpenClaw and other AI Agents can
|
|
55
55
|
install in one command. One fully-transferable memory store is shared
|
|
56
56
|
simultaneously by all agents on the computer. 100% offline: no network, no
|
|
57
57
|
API key, no cloud account, no telemetry. The embedding model ships inside
|
|
@@ -181,7 +181,7 @@ own - Claude Code keeps per-project Markdown files with an auto-loaded
|
|
|
181
181
|
index. Two memories means facts land in whichever one the model happened to
|
|
182
182
|
think of, and neither is complete. Compartment takes over on install: it imports
|
|
183
183
|
what the file memory already holds, and both the MCP handshake and the
|
|
184
|
-
managed CLAUDE.md block tell the model that
|
|
184
|
+
managed CLAUDE.md block tell the model that Compartment supersedes it - write
|
|
185
185
|
every new memory here, treat the files as a read-only archive. One vault,
|
|
186
186
|
encrypted, shared by every agent and project on the machine. Nothing is
|
|
187
187
|
deleted; the files stay exactly where they were.
|
|
@@ -190,7 +190,7 @@ deleted; the files stay exactly where they were.
|
|
|
190
190
|
and a host that declares its own memory in its system prompt outranks
|
|
191
191
|
anything a tool says. So `compartment integrate claude` also installs a
|
|
192
192
|
`PostToolUse` hook: when Claude Code writes a memory file, the fact lands in
|
|
193
|
-
the vault whether or not the model ever thought about
|
|
193
|
+
the vault whether or not the model ever thought about Compartment. The hook is
|
|
194
194
|
additive and idempotent (your other hooks are untouched, settings.json is
|
|
195
195
|
backed up first), it exits successfully no matter what - a memory tool must
|
|
196
196
|
never break your editor - and it stays quiet when the vault is locked.
|
|
@@ -327,7 +327,7 @@ and `compartment bench`.
|
|
|
327
327
|
| Peak RSS, model + vault + index resident | 319 MB |
|
|
328
328
|
| Store one memory (embed + encrypt + fsync journal) | ~40 ms |
|
|
329
329
|
| Wheel size, model included | ~30 MB |
|
|
330
|
-
| Test suite (crypto, tamper, crash, offline, concurrency, 2FA, graph, dash) |
|
|
330
|
+
| Test suite (crypto, tamper, crash, offline, concurrency, 2FA, graph, dash) | 249 tests, ~60 s |
|
|
331
331
|
|
|
332
332
|
A single network round-trip to a cloud memory API costs more than this
|
|
333
333
|
entire pipeline. The property that makes Compartment secure (no plaintext
|
|
@@ -397,10 +397,12 @@ The default unlock mode is convenience, not a cage: after a normal
|
|
|
397
397
|
unlock, the vault stays usable across processes, logouts, and logins -
|
|
398
398
|
for weeks or months if you leave it that way - until the next restart or
|
|
399
399
|
power loss, or until you lock it yourself. Restart/power loss always
|
|
400
|
-
locks it: the stored credential is the master key wrapped
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
400
|
+
locks it: the stored credential is the master key wrapped under a random
|
|
401
|
+
32-byte per-boot secret, held in a volatile kernel object that is never
|
|
402
|
+
written to any filesystem, so a restart destroys it and a new boot can
|
|
403
|
+
never open the old wrap. A copy of the credential file on its own is
|
|
404
|
+
useless, because the key it needs was never on the disk. That is
|
|
405
|
+
arithmetic, not a policy check.
|
|
404
406
|
|
|
405
407
|
If you prefer reboot-surviving unlock on macOS, that is an explicit
|
|
406
408
|
opt-in (`compartment unlock --keychain`), with the tradeoff documented. At any
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
Your AI agent forgets you the moment the session ends. Compartment ends
|
|
6
6
|
that. With Compartment, your AI agent gets better with experience: it keeps
|
|
7
7
|
every decision, preference and detail you give it, permanently, encrypted,
|
|
8
|
-
on your own computer. Hermes, Claude,
|
|
8
|
+
on your own computer. Hermes, Claude, OpenClaw and other AI Agents can
|
|
9
9
|
install in one command. One fully-transferable memory store is shared
|
|
10
10
|
simultaneously by all agents on the computer. 100% offline: no network, no
|
|
11
11
|
API key, no cloud account, no telemetry. The embedding model ships inside
|
|
@@ -135,7 +135,7 @@ own - Claude Code keeps per-project Markdown files with an auto-loaded
|
|
|
135
135
|
index. Two memories means facts land in whichever one the model happened to
|
|
136
136
|
think of, and neither is complete. Compartment takes over on install: it imports
|
|
137
137
|
what the file memory already holds, and both the MCP handshake and the
|
|
138
|
-
managed CLAUDE.md block tell the model that
|
|
138
|
+
managed CLAUDE.md block tell the model that Compartment supersedes it - write
|
|
139
139
|
every new memory here, treat the files as a read-only archive. One vault,
|
|
140
140
|
encrypted, shared by every agent and project on the machine. Nothing is
|
|
141
141
|
deleted; the files stay exactly where they were.
|
|
@@ -144,7 +144,7 @@ deleted; the files stay exactly where they were.
|
|
|
144
144
|
and a host that declares its own memory in its system prompt outranks
|
|
145
145
|
anything a tool says. So `compartment integrate claude` also installs a
|
|
146
146
|
`PostToolUse` hook: when Claude Code writes a memory file, the fact lands in
|
|
147
|
-
the vault whether or not the model ever thought about
|
|
147
|
+
the vault whether or not the model ever thought about Compartment. The hook is
|
|
148
148
|
additive and idempotent (your other hooks are untouched, settings.json is
|
|
149
149
|
backed up first), it exits successfully no matter what - a memory tool must
|
|
150
150
|
never break your editor - and it stays quiet when the vault is locked.
|
|
@@ -281,7 +281,7 @@ and `compartment bench`.
|
|
|
281
281
|
| Peak RSS, model + vault + index resident | 319 MB |
|
|
282
282
|
| Store one memory (embed + encrypt + fsync journal) | ~40 ms |
|
|
283
283
|
| Wheel size, model included | ~30 MB |
|
|
284
|
-
| Test suite (crypto, tamper, crash, offline, concurrency, 2FA, graph, dash) |
|
|
284
|
+
| Test suite (crypto, tamper, crash, offline, concurrency, 2FA, graph, dash) | 249 tests, ~60 s |
|
|
285
285
|
|
|
286
286
|
A single network round-trip to a cloud memory API costs more than this
|
|
287
287
|
entire pipeline. The property that makes Compartment secure (no plaintext
|
|
@@ -351,10 +351,12 @@ The default unlock mode is convenience, not a cage: after a normal
|
|
|
351
351
|
unlock, the vault stays usable across processes, logouts, and logins -
|
|
352
352
|
for weeks or months if you leave it that way - until the next restart or
|
|
353
353
|
power loss, or until you lock it yourself. Restart/power loss always
|
|
354
|
-
locks it: the stored credential is the master key wrapped
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
354
|
+
locks it: the stored credential is the master key wrapped under a random
|
|
355
|
+
32-byte per-boot secret, held in a volatile kernel object that is never
|
|
356
|
+
written to any filesystem, so a restart destroys it and a new boot can
|
|
357
|
+
never open the old wrap. A copy of the credential file on its own is
|
|
358
|
+
useless, because the key it needs was never on the disk. That is
|
|
359
|
+
arithmetic, not a policy check.
|
|
358
360
|
|
|
359
361
|
If you prefer reboot-surviving unlock on macOS, that is an explicit
|
|
360
362
|
opt-in (`compartment unlock --keychain`), with the tradeoff documented. At any
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "compartment"
|
|
7
|
-
version = "
|
|
7
|
+
version = "3.0"
|
|
8
8
|
description = "Compartment - high-security, fully offline, encrypted vector memory for AI agents (MCP)"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Per-caller namespace access control + vault-adjacent settings.
|
|
2
|
+
|
|
3
|
+
Config lives NEXT to the vault as `<vault>.config.json` (it contains no
|
|
4
|
+
secrets - only ACLs and preferences). `packs/*` namespaces are ALWAYS
|
|
5
|
+
read-only for every caller, including "*". Violations are hard errors.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import copy
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
|
|
14
|
+
from .crypto import CryptoError
|
|
15
|
+
|
|
16
|
+
# The whole grant vocabulary. Anything outside it is not a grant at all, so it
|
|
17
|
+
# denies both reads and writes rather than being waved through as "not rw".
|
|
18
|
+
ALLOWED_GRANTS = ("rw", "ro", "none")
|
|
19
|
+
PERMITTED_GRANTS = ("rw", "ro")
|
|
20
|
+
|
|
21
|
+
DEFAULT_CONFIG = {
|
|
22
|
+
"callers": {
|
|
23
|
+
"*": {"default_namespace": "main", "grants": {"*": "rw"}},
|
|
24
|
+
},
|
|
25
|
+
"settings": {
|
|
26
|
+
"auto_lock_minutes": 30,
|
|
27
|
+
"include_packs_in_search": True,
|
|
28
|
+
"unlock_tool_enabled": False,
|
|
29
|
+
"duplicate_threshold": 0.97,
|
|
30
|
+
"index_precision": "f32",
|
|
31
|
+
},
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class AclError(CryptoError):
|
|
36
|
+
pass
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass
|
|
40
|
+
class VaultConfig:
|
|
41
|
+
# Deep copies: a shallow copy would hand every VaultConfig in the process
|
|
42
|
+
# the same inner grants dict, so editing one vault's ACLs would silently
|
|
43
|
+
# edit the module default and every other open vault.
|
|
44
|
+
callers: dict = field(
|
|
45
|
+
default_factory=lambda: copy.deepcopy(DEFAULT_CONFIG["callers"]))
|
|
46
|
+
settings: dict = field(
|
|
47
|
+
default_factory=lambda: copy.deepcopy(DEFAULT_CONFIG["settings"]))
|
|
48
|
+
|
|
49
|
+
@staticmethod
|
|
50
|
+
def path_for(vault_path: str) -> str:
|
|
51
|
+
return vault_path + ".config.json"
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def load(cls, vault_path: str) -> "VaultConfig":
|
|
55
|
+
p = cls.path_for(vault_path)
|
|
56
|
+
if not os.path.exists(p):
|
|
57
|
+
return cls()
|
|
58
|
+
with open(p, encoding="utf-8") as f:
|
|
59
|
+
data = json.load(f)
|
|
60
|
+
cfg = cls()
|
|
61
|
+
cfg.callers = data.get("callers", cfg.callers)
|
|
62
|
+
cfg.settings = {**cfg.settings, **data.get("settings", {})}
|
|
63
|
+
return cfg
|
|
64
|
+
|
|
65
|
+
def save(self, vault_path: str) -> None:
|
|
66
|
+
with open(self.path_for(vault_path), "w", encoding="utf-8") as f:
|
|
67
|
+
json.dump({"callers": self.callers, "settings": self.settings}, f, indent=2)
|
|
68
|
+
|
|
69
|
+
# -- ACL ---------------------------------------------------------------
|
|
70
|
+
|
|
71
|
+
def _caller_entry(self, caller: str) -> dict:
|
|
72
|
+
# An entry that EXISTS is used as written, even when it is empty. Only
|
|
73
|
+
# a missing caller falls back to "*". Writing {"evil": {}} is how a
|
|
74
|
+
# caller gets locked down, so it must never inherit the wildcard.
|
|
75
|
+
if caller in self.callers:
|
|
76
|
+
entry = self.callers[caller]
|
|
77
|
+
else:
|
|
78
|
+
entry = self.callers.get("*")
|
|
79
|
+
if not isinstance(entry, dict):
|
|
80
|
+
raise AclError(f"Caller {caller!r} has no access to this vault")
|
|
81
|
+
return entry
|
|
82
|
+
|
|
83
|
+
def default_namespace(self, caller: str) -> str:
|
|
84
|
+
return self._caller_entry(caller).get("default_namespace", "main")
|
|
85
|
+
|
|
86
|
+
@staticmethod
|
|
87
|
+
def _match(grants: dict, namespace: str):
|
|
88
|
+
"""The most specific grant for `namespace`, or None if nothing matches.
|
|
89
|
+
|
|
90
|
+
Specificity, not dict order, decides. An exact namespace key beats
|
|
91
|
+
every wildcard, and among wildcards the longest matching prefix wins,
|
|
92
|
+
so "secret/*" beats "*" and "a/b/*" beats "a/*". Note that "*" is
|
|
93
|
+
itself a wildcard whose prefix is empty: it is the weakest possible
|
|
94
|
+
match, never the first one taken.
|
|
95
|
+
"""
|
|
96
|
+
if not isinstance(grants, dict):
|
|
97
|
+
return None
|
|
98
|
+
if namespace in grants:
|
|
99
|
+
return grants[namespace]
|
|
100
|
+
best_len = -1
|
|
101
|
+
best = None
|
|
102
|
+
for pattern, g in grants.items():
|
|
103
|
+
if not isinstance(pattern, str) or not pattern.endswith("*"):
|
|
104
|
+
continue
|
|
105
|
+
prefix = pattern[:-1]
|
|
106
|
+
if namespace.startswith(prefix) and len(prefix) > best_len:
|
|
107
|
+
best_len, best = len(prefix), g
|
|
108
|
+
return best
|
|
109
|
+
|
|
110
|
+
def grant_for(self, caller: str, namespace: str) -> str:
|
|
111
|
+
"""Returns 'rw' or 'ro'. Anything else raises AclError.
|
|
112
|
+
|
|
113
|
+
'none' and any value outside the documented rw/ro/none vocabulary deny
|
|
114
|
+
the namespace outright - reads included. packs/* is always ro.
|
|
115
|
+
"""
|
|
116
|
+
entry = self._caller_entry(caller)
|
|
117
|
+
grant = self._match(entry.get("grants", {}), namespace)
|
|
118
|
+
if grant is None:
|
|
119
|
+
raise AclError(
|
|
120
|
+
f"Caller {caller!r} is not granted access to namespace {namespace!r}")
|
|
121
|
+
if grant not in PERMITTED_GRANTS:
|
|
122
|
+
if grant in ALLOWED_GRANTS: # explicit "none"
|
|
123
|
+
raise AclError(
|
|
124
|
+
f"Caller {caller!r} is denied access to namespace "
|
|
125
|
+
f"{namespace!r}")
|
|
126
|
+
raise AclError(
|
|
127
|
+
f"Caller {caller!r} has an unrecognized grant {grant!r} for "
|
|
128
|
+
f"namespace {namespace!r}; expected one of "
|
|
129
|
+
f"{', '.join(ALLOWED_GRANTS)}")
|
|
130
|
+
if namespace.startswith("packs/"):
|
|
131
|
+
return "ro" # pack namespaces are immutable for everyone
|
|
132
|
+
return grant
|
|
133
|
+
|
|
134
|
+
def check(self, caller: str, namespace: str, write: bool) -> None:
|
|
135
|
+
# grant_for already denies reads for 'none' and for unknown values, so
|
|
136
|
+
# this only has to police the read-only case on writes.
|
|
137
|
+
grant = self.grant_for(caller, namespace)
|
|
138
|
+
if write and grant != "rw":
|
|
139
|
+
raise AclError(
|
|
140
|
+
f"Caller {caller!r} has read-only access to namespace {namespace!r}")
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Hash-chained, tamper-evident audit log (stored inside the sealed payload).
|
|
2
|
+
|
|
3
|
+
Each entry's hash covers the previous entry's hash, so an edit or a reordering
|
|
4
|
+
in the MIDDLE of the log breaks the chain at a detectable point.
|
|
5
|
+
|
|
6
|
+
A forward-only chain cannot catch a truncation of its own tail: lop the last
|
|
7
|
+
three entries off and what remains still verifies, just shorter. So the head
|
|
8
|
+
and the length are anchored outside the chain, in the meta table, every time
|
|
9
|
+
the vault is saved. verify() then requires the chain to EXTEND that anchor:
|
|
10
|
+
fewer entries than were anchored, or a different hash at the anchored
|
|
11
|
+
position, is reported as removal rather than passing as a clean shorter log.
|
|
12
|
+
|
|
13
|
+
The anchor is written on save and is absent in vaults written by older
|
|
14
|
+
builds. A missing anchor is not a failure - it is an unanchored log, reported
|
|
15
|
+
as such, and the next save anchors it.
|
|
16
|
+
|
|
17
|
+
A failure to write the audit entry fails the operation (fail-fast), never
|
|
18
|
+
the other way around.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import hashlib
|
|
23
|
+
import json
|
|
24
|
+
import time
|
|
25
|
+
|
|
26
|
+
GENESIS = "GENESIS"
|
|
27
|
+
|
|
28
|
+
# meta keys holding the anchor. Additive: a vault without them predates
|
|
29
|
+
# anchoring and still opens.
|
|
30
|
+
ANCHOR_HEAD = "audit_anchor_head"
|
|
31
|
+
ANCHOR_COUNT = "audit_anchor_count"
|
|
32
|
+
RELINK_LOG = "audit_relink_log"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _meta_get(conn, k: str) -> str | None:
|
|
36
|
+
row = conn.execute("SELECT v FROM meta WHERE k = ?", (k,)).fetchone()
|
|
37
|
+
return row["v"] if row else None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _meta_set(conn, k: str, v: str) -> None:
|
|
41
|
+
conn.execute(
|
|
42
|
+
"INSERT INTO meta (k, v) VALUES (?, ?) "
|
|
43
|
+
"ON CONFLICT(k) DO UPDATE SET v = excluded.v", (k, v))
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def count(conn) -> int:
|
|
47
|
+
return conn.execute("SELECT COUNT(*) c FROM audit").fetchone()["c"]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def anchor(conn) -> None:
|
|
51
|
+
"""Pin the current head and length. Called on every save."""
|
|
52
|
+
_meta_set(conn, ANCHOR_HEAD, head(conn))
|
|
53
|
+
_meta_set(conn, ANCHOR_COUNT, str(count(conn)))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def read_anchor(conn) -> tuple[str, int] | None:
|
|
57
|
+
"""The pinned (head, count), or None for a vault written before anchoring."""
|
|
58
|
+
h = _meta_get(conn, ANCHOR_HEAD)
|
|
59
|
+
c = _meta_get(conn, ANCHOR_COUNT)
|
|
60
|
+
if h is None or c is None:
|
|
61
|
+
return None
|
|
62
|
+
try:
|
|
63
|
+
return h, int(c)
|
|
64
|
+
except ValueError:
|
|
65
|
+
return None
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _entry_hash(prev_hash: str, ts: float, caller: str, op: str, detail: str) -> str:
|
|
69
|
+
body = json.dumps(
|
|
70
|
+
{"prev": prev_hash, "ts": ts, "caller": caller, "op": op, "detail": detail},
|
|
71
|
+
sort_keys=True, separators=(",", ":"),
|
|
72
|
+
).encode()
|
|
73
|
+
return hashlib.sha256(body).hexdigest()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def append(conn, caller: str, op: str, detail: str, ts: float | None = None) -> str:
|
|
77
|
+
"""Add an entry, linked to whatever is currently at the head.
|
|
78
|
+
|
|
79
|
+
`ts` is only passed when replaying a journalled operation, so the entry
|
|
80
|
+
keeps the time it actually happened. It is still linked to the head as it
|
|
81
|
+
stands at replay time, never to the head the original writer saw: that
|
|
82
|
+
head may never have reached disk (read operations are audited in RAM and
|
|
83
|
+
persist only on save), and chaining to an entry nobody else has is what
|
|
84
|
+
leaves a dangling link no future verify can resolve."""
|
|
85
|
+
row = conn.execute("SELECT hash FROM audit ORDER BY seq DESC LIMIT 1").fetchone()
|
|
86
|
+
prev = row["hash"] if row else GENESIS
|
|
87
|
+
ts = time.time() if ts is None else ts
|
|
88
|
+
h = _entry_hash(prev, ts, caller, op, detail)
|
|
89
|
+
conn.execute(
|
|
90
|
+
"INSERT INTO audit (ts, caller, op, detail, prev_hash, hash) VALUES (?,?,?,?,?,?)",
|
|
91
|
+
(ts, caller, op, detail, prev, h),
|
|
92
|
+
)
|
|
93
|
+
return h
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def head(conn) -> str:
|
|
97
|
+
row = conn.execute("SELECT hash FROM audit ORDER BY seq DESC LIMIT 1").fetchone()
|
|
98
|
+
return row["hash"] if row else GENESIS
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def verify(conn) -> tuple[bool, int, str]:
|
|
102
|
+
"""Walk the chain, then check it against the anchor.
|
|
103
|
+
|
|
104
|
+
Returns (ok, entries_checked, message). The walk catches edits and
|
|
105
|
+
reorderings; the anchor catches a truncated tail, which the walk alone
|
|
106
|
+
cannot see because a shortened chain is still internally consistent."""
|
|
107
|
+
prev = GENESIS
|
|
108
|
+
n = 0
|
|
109
|
+
hashes: list[str] = []
|
|
110
|
+
for row in conn.execute("SELECT * FROM audit ORDER BY seq"):
|
|
111
|
+
if row["prev_hash"] != prev:
|
|
112
|
+
return False, n, f"chain break at seq {row['seq']}: prev_hash mismatch"
|
|
113
|
+
want = _entry_hash(prev, row["ts"], row["caller"], row["op"], row["detail"])
|
|
114
|
+
if row["hash"] != want:
|
|
115
|
+
return False, n, f"chain break at seq {row['seq']}: entry hash mismatch"
|
|
116
|
+
prev = row["hash"]
|
|
117
|
+
hashes.append(prev)
|
|
118
|
+
n += 1
|
|
119
|
+
|
|
120
|
+
anc = read_anchor(conn)
|
|
121
|
+
if anc is None:
|
|
122
|
+
# Every vault is anchored at creation and re-anchored on every save.
|
|
123
|
+
# A missing anchor means the meta rows were removed, which is exactly
|
|
124
|
+
# the move someone makes to hide a truncation, so it is a failure and
|
|
125
|
+
# not a tolerated older shape.
|
|
126
|
+
return False, n, (
|
|
127
|
+
"audit log has no anchor. Every vault is anchored when it is "
|
|
128
|
+
"created and on every save, so an absent anchor means the meta "
|
|
129
|
+
"rows were removed. A truncated log cannot be detected without "
|
|
130
|
+
"it, so this is reported as tampering.")
|
|
131
|
+
a_head, a_count = anc
|
|
132
|
+
if n < a_count:
|
|
133
|
+
return False, n, (
|
|
134
|
+
f"audit log is SHORTER than its anchor: {n} entries present, "
|
|
135
|
+
f"{a_count} anchored at the last save. {a_count - n} entries were "
|
|
136
|
+
"removed from the end. A forward walk cannot see this, which is "
|
|
137
|
+
"why the length is pinned.")
|
|
138
|
+
if a_count == 0:
|
|
139
|
+
pinned_ok = True
|
|
140
|
+
else:
|
|
141
|
+
pinned_ok = len(hashes) >= a_count and hashes[a_count - 1] == a_head
|
|
142
|
+
if not pinned_ok:
|
|
143
|
+
return False, n, (
|
|
144
|
+
f"audit entry {a_count} does not match the anchored head: history "
|
|
145
|
+
"was rewritten at or before the last save, not merely appended to.")
|
|
146
|
+
return True, n, f"audit chain intact ({n} entries, anchored at {a_count})"
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def relink(conn) -> tuple[int, int | None]:
|
|
150
|
+
"""Re-link entries whose prev_hash points at something not in the log.
|
|
151
|
+
|
|
152
|
+
Only for repairing damage done by builds before this one, which replayed a
|
|
153
|
+
journalled entry with the head its original writer saw rather than the head
|
|
154
|
+
the log actually has. The result was a permanent dangling link: verify
|
|
155
|
+
stopped at it and never reported anything after it again.
|
|
156
|
+
|
|
157
|
+
What this does NOT do is paper over an edit. Every entry's own hash covers
|
|
158
|
+
its content, so content tampering is caught by the self-hash check and this
|
|
159
|
+
refuses to touch a log that fails it. Only the links are rebuilt, and only
|
|
160
|
+
forward from the first break. Nothing is deleted, reordered or reworded:
|
|
161
|
+
ts, caller, op and detail are exactly what they were.
|
|
162
|
+
|
|
163
|
+
Returns (entries relinked, seq of the first break) - (0, None) if intact."""
|
|
164
|
+
rows = conn.execute("SELECT * FROM audit ORDER BY seq").fetchall()
|
|
165
|
+
for row in rows:
|
|
166
|
+
want = _entry_hash(row["prev_hash"], row["ts"], row["caller"], row["op"],
|
|
167
|
+
row["detail"])
|
|
168
|
+
if row["hash"] != want:
|
|
169
|
+
raise ValueError(
|
|
170
|
+
f"audit entry seq {row['seq']} does not hash to its own content. "
|
|
171
|
+
"That is content tampering, not a dangling link, and relinking "
|
|
172
|
+
"would destroy the evidence. Refusing.")
|
|
173
|
+
|
|
174
|
+
# A dangling link is damage worth repairing. A missing entry is not: the
|
|
175
|
+
# self-hash check above passes for a log with rows deleted, because every
|
|
176
|
+
# surviving row still hashes to its own content, and relinking would then
|
|
177
|
+
# rewrite the chain around the hole and report it as intact. The anchor is
|
|
178
|
+
# the only thing that knows how long the log used to be.
|
|
179
|
+
anc = read_anchor(conn)
|
|
180
|
+
if anc is not None and len(rows) < anc[1]:
|
|
181
|
+
raise ValueError(
|
|
182
|
+
f"audit log has {len(rows)} entries but {anc[1]} were anchored at "
|
|
183
|
+
"the last save. Entries were deleted, not merely unlinked, and "
|
|
184
|
+
"relinking would rewrite the chain around the gap and call it "
|
|
185
|
+
"intact. Refusing.")
|
|
186
|
+
|
|
187
|
+
prev, first, changed = GENESIS, None, 0
|
|
188
|
+
for row in rows:
|
|
189
|
+
if row["prev_hash"] != prev:
|
|
190
|
+
if first is None:
|
|
191
|
+
first = row["seq"]
|
|
192
|
+
if first is not None: # rewrite everything after it
|
|
193
|
+
h = _entry_hash(prev, row["ts"], row["caller"], row["op"], row["detail"])
|
|
194
|
+
conn.execute("UPDATE audit SET prev_hash = ?, hash = ? WHERE seq = ?",
|
|
195
|
+
(prev, h, row["seq"]))
|
|
196
|
+
changed += 1
|
|
197
|
+
prev = h
|
|
198
|
+
else:
|
|
199
|
+
prev = row["hash"]
|
|
200
|
+
|
|
201
|
+
if changed:
|
|
202
|
+
# Leave a permanent, non-erasable record that the chain was rebuilt.
|
|
203
|
+
# It goes in meta rather than in the log itself: appending an entry
|
|
204
|
+
# would change the log's length, and the repair must not look like
|
|
205
|
+
# ordinary activity. Anyone auditing later can see that a repair
|
|
206
|
+
# happened, when, where it started, and what the head was before.
|
|
207
|
+
try:
|
|
208
|
+
prior = json.loads(_meta_get(conn, RELINK_LOG) or "[]")
|
|
209
|
+
except (ValueError, TypeError):
|
|
210
|
+
prior = []
|
|
211
|
+
prior.append({"when": time.time(), "first_break_seq": first,
|
|
212
|
+
"entries_relinked": changed,
|
|
213
|
+
"head_before": rows[-1]["hash"], "head_after": prev})
|
|
214
|
+
_meta_set(conn, RELINK_LOG, json.dumps(prior))
|
|
215
|
+
return changed, first
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def relink_history(conn) -> list[dict]:
|
|
219
|
+
"""Every repair ever performed on this log. Empty is the normal case."""
|
|
220
|
+
try:
|
|
221
|
+
return json.loads(_meta_get(conn, RELINK_LOG) or "[]")
|
|
222
|
+
except (ValueError, TypeError):
|
|
223
|
+
return []
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Performance + RAM benchmark against the live vault and a synthetic corpus.
|
|
2
|
+
|
|
3
|
+
Measures three things and reports p50/p95 (nearest-rank) for each: embed+store
|
|
4
|
+
latency for 20 real records written to the vault, vector-search latency over a
|
|
5
|
+
synthetic corpus of `synthetic_n` random unit vectors (default 20,000) that is
|
|
6
|
+
indexed only and never stored in the vault, and full hybrid search (embed +
|
|
7
|
+
vector + FTS + fuse) on the live vault. It also reports the peak RSS of this
|
|
8
|
+
process for the run, which on a seeded vault is dominated by the model and the
|
|
9
|
+
index rather than by the synthetic corpus.
|
|
10
|
+
|
|
11
|
+
Budgets checked: vector search p95 < 100ms at the synthetic scale, and peak RSS
|
|
12
|
+
< 1GB. Peak RSS is only measurable where the POSIX `resource` module exists; on
|
|
13
|
+
platforms without it both the measurement and its budget are reported as null
|
|
14
|
+
rather than as a pass.
|
|
15
|
+
|
|
16
|
+
The 20 records the run writes are deleted through the normal audited `forget`
|
|
17
|
+
path when the run finishes, including when it fails partway through. Records
|
|
18
|
+
that were already in the "bench" namespace are left untouched.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import math
|
|
23
|
+
import random
|
|
24
|
+
import sys
|
|
25
|
+
import time
|
|
26
|
+
|
|
27
|
+
import numpy as np
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
import resource # POSIX only; absent on Windows
|
|
31
|
+
except ImportError: # pragma: no cover - Windows
|
|
32
|
+
resource = None
|
|
33
|
+
|
|
34
|
+
from .crypto import CryptoError
|
|
35
|
+
from .vindex import build_index
|
|
36
|
+
|
|
37
|
+
WORDS = ("report vault memory agent record office data schedule market key "
|
|
38
|
+
"index search secure backup ledger review batch upload form note").split()
|
|
39
|
+
|
|
40
|
+
SAMPLES = 20 # store / hybrid-search timing samples
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _rss_mb() -> float | None:
|
|
44
|
+
"""Peak RSS of this process in MB, or None where it cannot be measured.
|
|
45
|
+
|
|
46
|
+
Returning None rather than 0.0 keeps the budget honest: a platform without
|
|
47
|
+
`resource` has no measurement, so it must not report one and must not pass
|
|
48
|
+
a budget on the strength of it.
|
|
49
|
+
"""
|
|
50
|
+
if resource is None: # Windows: peak-RSS via getrusage is unavailable
|
|
51
|
+
return None
|
|
52
|
+
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
|
53
|
+
return rss / (1024 * 1024) if sys.platform == "darwin" else rss / 1024
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _pct(xs: list[float], p: float) -> float:
|
|
57
|
+
"""Nearest-rank percentile: the smallest value at or above rank ceil(p*N).
|
|
58
|
+
|
|
59
|
+
`int(len(xs) * p)` overshoots by one rank, so a 20 sample p95 landed on
|
|
60
|
+
index 19, the maximum, and published a worst case as a p95.
|
|
61
|
+
"""
|
|
62
|
+
if not xs:
|
|
63
|
+
raise ValueError("percentile of an empty series")
|
|
64
|
+
xs = sorted(xs)
|
|
65
|
+
k = min(len(xs) - 1, max(0, math.ceil(p * len(xs)) - 1))
|
|
66
|
+
return xs[k]
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _teardown(vault, record_ids: list[str]) -> None:
|
|
70
|
+
"""Delete exactly the records this run created, through the audited path.
|
|
71
|
+
|
|
72
|
+
Deleting by namespace would take every record a user had stored under the
|
|
73
|
+
namespace "bench", which `compartment store --namespace bench` lets them
|
|
74
|
+
create. Going through `vault.forget` also leaves an audit entry, which a
|
|
75
|
+
direct `vault.db.delete` does not.
|
|
76
|
+
"""
|
|
77
|
+
removed = False
|
|
78
|
+
for rid in record_ids:
|
|
79
|
+
try:
|
|
80
|
+
vault.forget(rid, caller="bench", shred=False)
|
|
81
|
+
removed = True
|
|
82
|
+
except CryptoError:
|
|
83
|
+
# already gone, or no longer writable: nothing to clean up
|
|
84
|
+
continue
|
|
85
|
+
if removed:
|
|
86
|
+
vault.save()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def run(vault, synthetic_n: int = 20_000, queries: int = 50) -> dict:
|
|
90
|
+
if synthetic_n < 1:
|
|
91
|
+
raise CryptoError(
|
|
92
|
+
f"bench: --records must be 1 or more, got {synthetic_n}")
|
|
93
|
+
if queries < 1:
|
|
94
|
+
raise CryptoError(f"bench: queries must be 1 or more, got {queries}")
|
|
95
|
+
|
|
96
|
+
rng = random.Random(42)
|
|
97
|
+
out: dict = {"synthetic_records": synthetic_n}
|
|
98
|
+
written: list[str] = []
|
|
99
|
+
try:
|
|
100
|
+
# 1) real embed+store latency on a small sample
|
|
101
|
+
t_store = []
|
|
102
|
+
for i in range(SAMPLES):
|
|
103
|
+
text = f"benchmark memory {i}: " + " ".join(rng.choices(WORDS, k=10))
|
|
104
|
+
t0 = time.perf_counter()
|
|
105
|
+
res = vault.store(text, caller="bench", namespace="bench",
|
|
106
|
+
tags=["bench"])
|
|
107
|
+
t_store.append((time.perf_counter() - t0) * 1000)
|
|
108
|
+
# a deduplicated store returns somebody else's record id; only the
|
|
109
|
+
# ids this run actually created are ours to delete afterwards
|
|
110
|
+
if not res.get("duplicate"):
|
|
111
|
+
written.append(res["id"])
|
|
112
|
+
out["store_ms_p50"] = round(_pct(t_store, 0.50), 1)
|
|
113
|
+
out["store_ms_p95"] = round(_pct(t_store, 0.95), 1)
|
|
114
|
+
|
|
115
|
+
# 2) synthetic vector corpus at scale (index-only: isolates search speed)
|
|
116
|
+
dim = int(vault.header.model["dim"])
|
|
117
|
+
mat = np.random.default_rng(42).standard_normal(
|
|
118
|
+
(synthetic_n, dim)).astype(np.float32)
|
|
119
|
+
mat /= np.linalg.norm(mat, axis=1, keepdims=True)
|
|
120
|
+
t0 = time.perf_counter()
|
|
121
|
+
idx = build_index(dim, list(range(1, synthetic_n + 1)), mat,
|
|
122
|
+
precision=vault.config.settings.get("index_precision", "f32"),
|
|
123
|
+
force=None)
|
|
124
|
+
out["index_build_s"] = round(time.perf_counter() - t0, 2)
|
|
125
|
+
out["index_kind"] = idx.kind
|
|
126
|
+
|
|
127
|
+
t_search = []
|
|
128
|
+
for _ in range(queries):
|
|
129
|
+
q = mat[rng.randrange(synthetic_n)]
|
|
130
|
+
t0 = time.perf_counter()
|
|
131
|
+
idx.search(q, 10)
|
|
132
|
+
t_search.append((time.perf_counter() - t0) * 1000)
|
|
133
|
+
out["vector_search_ms_p50"] = round(_pct(t_search, 0.50), 2)
|
|
134
|
+
out["vector_search_ms_p95"] = round(_pct(t_search, 0.95), 2)
|
|
135
|
+
|
|
136
|
+
# 3) full hybrid search on the live vault (embed + vector + FTS + fuse)
|
|
137
|
+
t_hybrid = []
|
|
138
|
+
for _ in range(SAMPLES):
|
|
139
|
+
t0 = time.perf_counter()
|
|
140
|
+
vault.search("benchmark memory " + rng.choice(WORDS),
|
|
141
|
+
caller="bench", top_k=8)
|
|
142
|
+
t_hybrid.append((time.perf_counter() - t0) * 1000)
|
|
143
|
+
out["hybrid_search_ms_p50"] = round(_pct(t_hybrid, 0.50), 1)
|
|
144
|
+
out["hybrid_search_ms_p95"] = round(_pct(t_hybrid, 0.95), 1)
|
|
145
|
+
|
|
146
|
+
rss = _rss_mb()
|
|
147
|
+
out["peak_rss_mb"] = None if rss is None else round(rss, 0)
|
|
148
|
+
out["budgets"] = {
|
|
149
|
+
"vector_search_p95_under_100ms": out["vector_search_ms_p95"] < 100,
|
|
150
|
+
# null, not True: an unmeasured budget is not a met budget
|
|
151
|
+
"rss_under_1gb": None if rss is None else rss < 1024,
|
|
152
|
+
}
|
|
153
|
+
if rss is None:
|
|
154
|
+
out["peak_rss_mb_note"] = (
|
|
155
|
+
"not measured: the POSIX resource module is unavailable here")
|
|
156
|
+
finally:
|
|
157
|
+
_teardown(vault, written)
|
|
158
|
+
return out
|
|
@@ -126,7 +126,14 @@ def uninstall(settings: Path | None = None) -> bool:
|
|
|
126
126
|
if not removed:
|
|
127
127
|
return False
|
|
128
128
|
data["hooks"][EVENT] = kept
|
|
129
|
-
|
|
129
|
+
# Same care as install(): keep a recovery copy, then swap the new file in
|
|
130
|
+
# atomically. A crash or a full disk mid-write must not cost the user every
|
|
131
|
+
# other hook, permission and env var in this file.
|
|
132
|
+
if path.exists():
|
|
133
|
+
shutil.copy2(path, path.with_suffix(path.suffix + ".compartment-backup"))
|
|
134
|
+
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
135
|
+
tmp.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
|
|
136
|
+
os.replace(tmp, path)
|
|
130
137
|
return True
|
|
131
138
|
|
|
132
139
|
|