memwall 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
memwall/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+ """memwall: scanner for AI agent memory stores.
2
+
3
+ Detects memory poisoning (injected standing orders, exfiltration vectors,
4
+ hidden payloads) and hygiene issues (retained PII) in agent memory.
5
+ """
6
+
7
+ __version__ = "0.4.0"
8
+
9
+ from .adapters import load # noqa: F401
10
+ from .heuristics import scan_records # noqa: F401
memwall/__main__.py ADDED
@@ -0,0 +1,4 @@
1
+ from .cli import run
2
+
3
+ if __name__ == "__main__":
4
+ run()
@@ -0,0 +1,81 @@
1
+ """Store adapters: read memories from different sources into MemoryRecords."""
2
+ from __future__ import annotations
3
+
4
+ import os
5
+ import sys
6
+
7
+ from . import files, generic_json, mcp_memory, mem0_export
8
+
9
+ ADAPTERS = {
10
+ "files": files.load,
11
+ "mcp": mcp_memory.load,
12
+ "mem0": mem0_export.load,
13
+ "json": generic_json.load,
14
+ }
15
+
16
+ SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build", ".next"}
17
+ # Hidden dirs are skipped except the ones agents actually keep memory in.
18
+ KEEP_HIDDEN = {".claude", ".codex", ".cursor", ".gemini", ".github"}
19
+ KNOWN_NAMES = {"claude.md", "agents.md", "gemini.md", ".cursorrules", "memory.json", "memory.jsonl"}
20
+ EXTS = {".md", ".mdc", ".txt", ".json", ".jsonl"}
21
+
22
+
23
+ def detect_format(path: str) -> str:
24
+ ext = os.path.splitext(path)[1].lower()
25
+ if ext == ".jsonl":
26
+ return "mcp"
27
+ if ext == ".json":
28
+ try:
29
+ with open(path, encoding="utf-8-sig") as fh:
30
+ head = fh.read(4096).lstrip()
31
+ except OSError:
32
+ return "json"
33
+ # The MCP reference memory server writes JSONL into a .json file.
34
+ if head.startswith('{"type"') or '"observations"' in head:
35
+ return "mcp"
36
+ if '"memory"' in head or '"results"' in head:
37
+ return "mem0"
38
+ return "json"
39
+ return "files"
40
+
41
+
42
+ MAX_WALK_FILE_BYTES = 5 * 1024 * 1024
43
+
44
+
45
+ def _walkable(file_path: str, low: str, ext: str) -> bool:
46
+ try:
47
+ if os.path.getsize(file_path) > MAX_WALK_FILE_BYTES:
48
+ print(f"memwall: skipped {file_path}: larger than "
49
+ f"{MAX_WALK_FILE_BYTES // (1024 * 1024)} MB", file=sys.stderr)
50
+ return False
51
+ except OSError:
52
+ return False
53
+ if ext == ".json" and low not in KNOWN_NAMES:
54
+ # In directory walks, only ingest .json that looks like agent memory -
55
+ # otherwise every package.json and tsconfig.json becomes noise.
56
+ return "memor" in low or "mem0" in low or detect_format(file_path) != "json"
57
+ return True
58
+
59
+
60
+ def load(path: str, fmt: str = "auto") -> list:
61
+ if fmt != "auto":
62
+ return ADAPTERS[fmt](path)
63
+ if os.path.isdir(path):
64
+ records = []
65
+ for root, dirs, names in os.walk(path):
66
+ dirs[:] = [d for d in dirs
67
+ if d not in SKIP_DIRS and (not d.startswith(".") or d in KEEP_HIDDEN)]
68
+ for name in sorted(names):
69
+ low = name.lower()
70
+ ext = os.path.splitext(low)[1]
71
+ if low not in KNOWN_NAMES and ext not in EXTS:
72
+ continue
73
+ file_path = os.path.join(root, name)
74
+ if not _walkable(file_path, low, ext):
75
+ continue
76
+ try:
77
+ records.extend(load(file_path, "auto"))
78
+ except Exception as exc: # keep walking on malformed files
79
+ print(f"memwall: skipped {file_path}: {exc}", file=sys.stderr)
80
+ return records
81
+ return ADAPTERS[detect_format(path)](path)
@@ -0,0 +1,50 @@
1
+ """Adapter for plain text/markdown memory and instruction files.
2
+
3
+ Instruction files (CLAUDE.md, AGENTS.md, ...) are marked kind="instructions"
4
+ so the rule engine can skip imperative-phrasing rules that are normal there.
5
+ Every other text file is kind="notes". Each blank-line-separated block
6
+ becomes one MemoryRecord.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import os
11
+
12
+ from ..models import MemoryRecord
13
+
14
+ INSTRUCTION_NAMES = {
15
+ "claude.md", "agents.md", "gemini.md", ".cursorrules",
16
+ "copilot-instructions.md", "system.md",
17
+ }
18
+
19
+
20
+ def _blocks(text: str):
21
+ block, start = [], 1
22
+ line_no = 0
23
+ for line_no, line in enumerate(text.splitlines(), 1):
24
+ if line.strip():
25
+ if not block:
26
+ start = line_no
27
+ block.append(line)
28
+ elif block:
29
+ yield start, "\n".join(block)
30
+ block = []
31
+ if block:
32
+ yield start, "\n".join(block)
33
+
34
+
35
+ def load(path: str) -> list:
36
+ with open(path, encoding="utf-8-sig", errors="replace") as fh:
37
+ text = fh.read()
38
+ base = os.path.basename(path)
39
+ low = base.lower()
40
+ kind = "instructions" if low in INSTRUCTION_NAMES or low.endswith(".mdc") else "notes"
41
+ return [
42
+ MemoryRecord(
43
+ id=f"{base}:{line}",
44
+ content=content,
45
+ store="files",
46
+ source=path,
47
+ metadata={"kind": kind},
48
+ )
49
+ for line, content in _blocks(text)
50
+ ]
@@ -0,0 +1,39 @@
1
+ """Fallback adapter for generic JSON memory dumps."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+
6
+ from ..models import MemoryRecord
7
+
8
+ _CONTENT_FIELDS = ("content", "memory", "text", "observation", "value", "fact", "note")
9
+
10
+
11
+ def load(path: str) -> list:
12
+ with open(path, encoding="utf-8-sig") as fh:
13
+ data = json.load(fh)
14
+ if isinstance(data, dict):
15
+ for key in ("results", "items", "memories", "data", "records"):
16
+ if isinstance(data.get(key), list):
17
+ data = data[key]
18
+ break
19
+ else:
20
+ data = [data]
21
+ if not isinstance(data, list):
22
+ data = [data]
23
+
24
+ records = []
25
+ for idx, item in enumerate(data):
26
+ if isinstance(item, str):
27
+ content = item
28
+ elif isinstance(item, dict):
29
+ content = next((str(item[f]) for f in _CONTENT_FIELDS if item.get(f)), "")
30
+ else:
31
+ continue
32
+ if content:
33
+ records.append(MemoryRecord(
34
+ id=str(item.get("id", f"item-{idx}")) if isinstance(item, dict) else f"item-{idx}",
35
+ content=content,
36
+ store="json",
37
+ source=path,
38
+ ))
39
+ return records
@@ -0,0 +1,41 @@
1
+ """Adapter for the MCP reference memory server (JSONL of entities/relations)."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+
6
+ from ..models import MemoryRecord
7
+
8
+
9
+ def load(path: str) -> list:
10
+ records = []
11
+ with open(path, encoding="utf-8-sig") as fh:
12
+ for line_no, line in enumerate(fh, 1):
13
+ line = line.strip()
14
+ if not line:
15
+ continue
16
+ try:
17
+ obj = json.loads(line)
18
+ except json.JSONDecodeError:
19
+ continue
20
+ if not isinstance(obj, dict):
21
+ continue
22
+ if obj.get("type") == "entity":
23
+ name = obj.get("name", f"entity-{line_no}")
24
+ for idx, observation in enumerate(obj.get("observations", [])):
25
+ records.append(MemoryRecord(
26
+ id=f"{name}#obs{idx}",
27
+ content=str(observation),
28
+ store="mcp-memory",
29
+ source=path,
30
+ metadata={"entity": name, "entityType": obj.get("entityType")},
31
+ ))
32
+ elif obj.get("type") == "relation":
33
+ content = f"{obj.get('from')} {obj.get('relationType')} {obj.get('to')}"
34
+ records.append(MemoryRecord(
35
+ id=f"relation:{line_no}",
36
+ content=content,
37
+ store="mcp-memory",
38
+ source=path,
39
+ metadata={"relation": True},
40
+ ))
41
+ return records
@@ -0,0 +1,43 @@
1
+ """Adapter for mem0-style exports (JSON with a list of memory objects)."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+
6
+ from ..models import MemoryRecord
7
+
8
+ _META_KEYS = ("user_id", "agent_id", "run_id", "categories")
9
+
10
+
11
+ def load(path: str) -> list:
12
+ with open(path, encoding="utf-8-sig") as fh:
13
+ data = json.load(fh)
14
+ if isinstance(data, dict):
15
+ items = data.get("results") or data.get("memories") or data.get("items") or []
16
+ else:
17
+ items = data
18
+ if not isinstance(items, list):
19
+ items = [items]
20
+
21
+ records = []
22
+ for idx, item in enumerate(items):
23
+ created = None
24
+ metadata = {}
25
+ if isinstance(item, str):
26
+ content, rid = item, f"mem-{idx}"
27
+ elif isinstance(item, dict):
28
+ content = item.get("memory") or item.get("text") or item.get("content") or ""
29
+ rid = str(item.get("id", f"mem-{idx}"))
30
+ created = item.get("created_at")
31
+ metadata = {k: item[k] for k in _META_KEYS if k in item}
32
+ else:
33
+ continue
34
+ if content:
35
+ records.append(MemoryRecord(
36
+ id=rid,
37
+ content=str(content),
38
+ store="mem0",
39
+ source=path,
40
+ created_at=created,
41
+ metadata=metadata,
42
+ ))
43
+ return records
memwall/baseline.py ADDED
@@ -0,0 +1,31 @@
1
+ """Baseline files: accept known findings so CI only fails on new ones.
2
+
3
+ A fingerprint is derived from the rule id and the memory content, so a
4
+ baselined finding stays suppressed until the memory itself changes.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import hashlib
9
+ import json
10
+
11
+
12
+ def fingerprint(record, finding) -> str:
13
+ blob = f"{finding.rule_id}|{record.content}".encode("utf-8")
14
+ return hashlib.sha256(blob).hexdigest()[:16]
15
+
16
+
17
+ def load_baseline(path: str) -> set:
18
+ try:
19
+ with open(path, encoding="utf-8-sig") as fh:
20
+ data = json.load(fh)
21
+ except FileNotFoundError:
22
+ return set()
23
+ return set(data.get("fingerprints") or [])
24
+
25
+
26
+ def save_baseline(path: str, reports) -> int:
27
+ fingerprints = sorted({
28
+ fingerprint(rep.record, f) for rep in reports for f in rep.findings})
29
+ with open(path, "w", encoding="utf-8") as fh:
30
+ json.dump({"version": 1, "fingerprints": fingerprints}, fh, indent=2)
31
+ return len(fingerprints)
memwall/cli.py ADDED
@@ -0,0 +1,235 @@
1
+ """memwall command-line interface."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+
8
+ from . import __version__, adapters, heuristics
9
+ from . import judge as judge_mod
10
+ from . import report as report_mod
11
+ from .models import ScanResult, Severity
12
+
13
+
14
+ def _strip_dashes(argv) -> list:
15
+ argv = list(argv or [])
16
+ return argv[1:] if argv and argv[0] == "--" else argv
17
+
18
+
19
+ def main(argv=None) -> int:
20
+ parser = argparse.ArgumentParser(
21
+ prog="memwall",
22
+ description="Scan and guard AI agent memory stores against poisoning and hygiene issues.",
23
+ )
24
+ parser.add_argument("--version", action="version", version=f"memwall {__version__}")
25
+ sub = parser.add_subparsers(dest="command", required=True)
26
+
27
+ scan_p = sub.add_parser("scan", help="scan a memory store, export, file, or directory")
28
+ scan_p.add_argument("path", help="memory file, export, or directory to walk")
29
+ scan_p.add_argument("--format", choices=["auto", "files", "mcp", "mem0", "json"], default="auto")
30
+ scan_p.add_argument("--judge", action="store_true",
31
+ help="run the Claude judge on flagged records (needs ANTHROPIC_API_KEY)")
32
+ scan_p.add_argument("--judge-all", action="store_true",
33
+ help="judge every record, not just heuristic hits")
34
+ scan_p.add_argument("--model", default=None, help=f"judge model (default: {judge_mod.DEFAULT_MODEL})")
35
+ scan_p.add_argument("--json", dest="json_out", metavar="PATH", help="write a JSON report")
36
+ scan_p.add_argument("--markdown", dest="md_out", metavar="PATH", help="write a Markdown report")
37
+ scan_p.add_argument("--fail-on", choices=["info", "low", "medium", "high", "critical"], default="high",
38
+ help="exit 1 if any finding is at or above this severity (CI gate)")
39
+ scan_p.add_argument("--baseline", metavar="PATH",
40
+ help="baseline file of accepted findings to suppress")
41
+ scan_p.add_argument("--update-baseline", action="store_true",
42
+ help="write current findings to --baseline and exit 0")
43
+ scan_p.add_argument("-q", "--quiet", action="store_true",
44
+ help="print only the summary lines")
45
+
46
+ sub.add_parser("rules", help="list heuristic rules")
47
+
48
+ gw_p = sub.add_parser("gateway", help="run the MCP quarantine gateway in front of a memory server")
49
+ gw_p.add_argument("--state-dir", default=".memwall", help="where to keep the event log and quarantine")
50
+ gw_p.add_argument("--block-at", choices=["low", "medium", "high", "critical"], default="high",
51
+ help="quarantine writes with findings at or above this severity")
52
+ gw_p.add_argument("--policy", metavar="PATH",
53
+ help="JSON policy file with per-origin block thresholds (overrides --block-at)")
54
+ gw_p.add_argument("--judge", action="store_true",
55
+ help="consult the Claude judge before quarantining flagged writes (needs ANTHROPIC_API_KEY)")
56
+ gw_p.add_argument("--model", default=None, help="judge model for --judge")
57
+ gw_p.add_argument("backend", nargs=argparse.REMAINDER,
58
+ help="backend command after --, e.g.: -- npx -y @modelcontextprotocol/server-memory")
59
+
60
+ log_p = sub.add_parser("log", help="show the gateway's forensic event log")
61
+ log_p.add_argument("--state-dir", default=".memwall")
62
+ log_p.add_argument("--tail", type=int, default=20, help="show the last N events")
63
+
64
+ q_p = sub.add_parser("quarantine", help="review quarantined memory writes")
65
+ q_p.add_argument("--state-dir", default=".memwall")
66
+ q_sub = q_p.add_subparsers(dest="qcmd", required=True)
67
+ q_sub.add_parser("list", help="list quarantine entries")
68
+ q_show = q_sub.add_parser("show", help="show one entry in full")
69
+ q_show.add_argument("qid")
70
+ q_release = q_sub.add_parser("release", help="replay an approved entry to the backend")
71
+ q_release.add_argument("qid")
72
+ q_release.add_argument("backend", nargs=argparse.REMAINDER,
73
+ help="backend command after --, same as the gateway wraps")
74
+ q_drop = q_sub.add_parser("drop", help="discard an entry (kept in the audit trail)")
75
+ q_drop.add_argument("qid")
76
+
77
+ args = parser.parse_args(argv)
78
+
79
+ for stream in (sys.stdout, sys.stderr):
80
+ if hasattr(stream, "reconfigure"):
81
+ try:
82
+ stream.reconfigure(errors="replace")
83
+ except Exception:
84
+ pass
85
+
86
+ if args.command == "rules":
87
+ width = max(len(r.name) for r in heuristics.RULES)
88
+ for rule in heuristics.RULES:
89
+ print(f"{rule.id} {rule.name.ljust(width)} {rule.severity.name.ljust(8)} {rule.description}")
90
+ return 0
91
+
92
+ if args.command == "scan":
93
+ return _cmd_scan(args)
94
+ if args.command == "gateway":
95
+ return _cmd_gateway(args)
96
+ if args.command == "log":
97
+ return _cmd_log(args)
98
+ if args.command == "quarantine":
99
+ return _cmd_quarantine(args)
100
+ return 0
101
+
102
+
103
+ def _cmd_scan(args) -> int:
104
+ try:
105
+ records = adapters.load(args.path, args.format)
106
+ except FileNotFoundError:
107
+ print(f"memwall: path not found: {args.path}", file=sys.stderr)
108
+ return 2
109
+ if not records:
110
+ print(f"memwall: no memories found at {args.path}")
111
+ return 0
112
+
113
+ reports = heuristics.scan_records(records)
114
+
115
+ if args.judge or args.judge_all:
116
+ targets = reports if args.judge_all else [r for r in reports if r.findings]
117
+ if targets:
118
+ judge_mod.judge(targets, model=args.model or judge_mod.DEFAULT_MODEL)
119
+
120
+ if args.update_baseline and not args.baseline:
121
+ print("memwall: --update-baseline needs --baseline PATH", file=sys.stderr)
122
+ return 2
123
+
124
+ suppressed = 0
125
+ if args.baseline:
126
+ from . import baseline as baseline_mod
127
+ if args.update_baseline:
128
+ count = baseline_mod.save_baseline(args.baseline, reports)
129
+ print(f"baseline written: {args.baseline} ({count} fingerprints)")
130
+ return 0
131
+ accepted = baseline_mod.load_baseline(args.baseline)
132
+ for rep in reports:
133
+ kept = [f for f in rep.findings
134
+ if baseline_mod.fingerprint(rep.record, f) not in accepted]
135
+ suppressed += len(rep.findings) - len(kept)
136
+ rep.findings = kept
137
+
138
+ sources = sorted({r.record.source for r in reports})
139
+ result = ScanResult(reports=reports, scanned=len(reports), stores=sources)
140
+ print(report_mod.render_terminal(result, quiet=args.quiet))
141
+ if suppressed:
142
+ print(f"suppressed {suppressed} baselined finding(s)")
143
+ if args.json_out:
144
+ report_mod.write_json(result, args.json_out)
145
+ print(f"json report: {args.json_out}")
146
+ if args.md_out:
147
+ report_mod.write_markdown(result, args.md_out)
148
+ print(f"markdown report: {args.md_out}")
149
+
150
+ threshold = Severity[args.fail_on.upper()]
151
+ worst = max((f.severity for rep in reports for f in rep.findings), default=None)
152
+ return 1 if worst is not None and worst >= threshold else 0
153
+
154
+
155
+ def _cmd_gateway(args) -> int:
156
+ from .gateway import Gateway
157
+ from .policy import Policy
158
+ backend = _strip_dashes(args.backend)
159
+ if not backend:
160
+ print("memwall: gateway needs a backend command after --", file=sys.stderr)
161
+ return 2
162
+ if args.policy:
163
+ policy = Policy.from_file(args.policy)
164
+ else:
165
+ policy = Policy(default_block_at=Severity[args.block_at.upper()])
166
+ gateway = Gateway(backend, state_dir=args.state_dir, policy=policy,
167
+ judge_enabled=args.judge, judge_model=args.model)
168
+ return gateway.run()
169
+
170
+
171
+ def _cmd_log(args) -> int:
172
+ from .state import StateStore
173
+ events = StateStore(args.state_dir).events()
174
+ if not events:
175
+ print("no events")
176
+ return 0
177
+ for event in events[-args.tail:]:
178
+ rules = ",".join(event.get("findings") or []) or "-"
179
+ qid = f" qid={event['qid']}" if event.get("qid") else ""
180
+ origin = event.get("origin")
181
+ origin = f" origin={origin}" if origin and origin != "unknown" else ""
182
+ print(f"{event['ts']} {event['event']:<8} {event['tool']:<24} {rules}{origin}{qid}")
183
+ return 0
184
+
185
+
186
+ def _cmd_quarantine(args) -> int:
187
+ from .state import StateStore
188
+ store = StateStore(args.state_dir)
189
+ entries = {e["qid"]: e for e in store.quarantine_entries()}
190
+
191
+ if args.qcmd == "list":
192
+ if not entries:
193
+ print("quarantine is empty")
194
+ return 0
195
+ for entry in sorted(entries.values(), key=lambda e: e["ts"]):
196
+ rules = ",".join(sorted({f["rule"] for f in entry["findings"]}))
197
+ origin = entry.get("origin", "-")
198
+ print(f"{entry['qid']} {entry['status']:<9} {entry['ts']} {entry['tool']:<24} {origin:<9} {rules}")
199
+ return 0
200
+
201
+ entry = entries.get(args.qid)
202
+ if entry is None:
203
+ print(f"memwall: no quarantine entry {args.qid}", file=sys.stderr)
204
+ return 2
205
+
206
+ if args.qcmd == "show":
207
+ print(json.dumps(entry, indent=2, ensure_ascii=False))
208
+ return 0
209
+
210
+ if args.qcmd == "drop":
211
+ store.set_status(args.qid, "dropped")
212
+ print(f"dropped {args.qid}")
213
+ return 0
214
+
215
+ if args.qcmd == "release":
216
+ from .mcp_client import MiniClient
217
+ backend = _strip_dashes(args.backend)
218
+ if not backend:
219
+ print("memwall: release needs the backend command after --", file=sys.stderr)
220
+ return 2
221
+ params = entry["request"].get("params") or {}
222
+ client = MiniClient(backend)
223
+ try:
224
+ client.initialize()
225
+ client.call_tool(params.get("name", ""), params.get("arguments") or {})
226
+ finally:
227
+ client.close()
228
+ store.set_status(args.qid, "released")
229
+ print(f"released {args.qid} to backend")
230
+ return 0
231
+ return 0
232
+
233
+
234
+ def run() -> None:
235
+ sys.exit(main())