memory-passport 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memory_passport/__init__.py +8 -0
- memory_passport/cli.py +383 -0
- memory_passport/diff.py +81 -0
- memory_passport/exclusions.py +137 -0
- memory_passport/exporters/__init__.py +6 -0
- memory_passport/exporters/base.py +41 -0
- memory_passport/exporters/chatgpt.py +50 -0
- memory_passport/exporters/claude.py +28 -0
- memory_passport/exporters/claude_code.py +50 -0
- memory_passport/exporters/cursor.py +30 -0
- memory_passport/exporters/markdown.py +15 -0
- memory_passport/exporters/prompt.py +67 -0
- memory_passport/exporters/registry.py +42 -0
- memory_passport/importers/__init__.py +6 -0
- memory_passport/importers/base.py +56 -0
- memory_passport/importers/builder.py +87 -0
- memory_passport/importers/chatgpt.py +140 -0
- memory_passport/importers/claude.py +201 -0
- memory_passport/importers/copilot.py +16 -0
- memory_passport/importers/gemini.py +18 -0
- memory_passport/importers/markdown.py +74 -0
- memory_passport/importers/registry.py +36 -0
- memory_passport/importers/router.py +92 -0
- memory_passport/importers/text.py +75 -0
- memory_passport/importers/textlist.py +37 -0
- memory_passport/inspect_export.py +112 -0
- memory_passport/mcp_server.py +103 -0
- memory_passport/merge.py +147 -0
- memory_passport/model.py +290 -0
- memory_passport/schema.py +32 -0
- memory_passport/spec/frontmatter.schema.json +50 -0
- memory_passport/spec/manifest.schema.json +28 -0
- memory_passport/store.py +205 -0
- memory_passport/validate.py +270 -0
- memory_passport-0.2.0.dist-info/METADATA +221 -0
- memory_passport-0.2.0.dist-info/RECORD +39 -0
- memory_passport-0.2.0.dist-info/WHEEL +4 -0
- memory_passport-0.2.0.dist-info/entry_points.txt +18 -0
- memory_passport-0.2.0.dist-info/licenses/LICENSE +21 -0
memory_passport/cli.py
ADDED
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
"""``passport`` command-line interface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
|
|
11
|
+
from memory_passport import __version__
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(
|
|
14
|
+
name="passport",
|
|
15
|
+
help="Portable AI assistant memory: validate, import, export, merge and diff vaults.",
|
|
16
|
+
no_args_is_help=True,
|
|
17
|
+
rich_markup_mode=None,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _version(value: bool) -> None:
|
|
22
|
+
if value:
|
|
23
|
+
typer.echo(f"memory-passport {__version__}")
|
|
24
|
+
raise typer.Exit()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@app.callback()
|
|
28
|
+
def main(
|
|
29
|
+
version: Annotated[
|
|
30
|
+
bool, typer.Option("--version", callback=_version, is_eager=True, help="Show version.")
|
|
31
|
+
] = False,
|
|
32
|
+
) -> None:
|
|
33
|
+
"""memory-passport CLI."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@app.command()
|
|
37
|
+
def validate(
|
|
38
|
+
vault: Annotated[Path, typer.Argument(help="Vault directory to check.")],
|
|
39
|
+
strict: Annotated[bool, typer.Option(help="Treat warnings as failures.")] = False,
|
|
40
|
+
as_json: Annotated[bool, typer.Option("--json", help="Machine-readable output.")] = False,
|
|
41
|
+
stale: Annotated[
|
|
42
|
+
int | None,
|
|
43
|
+
typer.Option(help="Warn about observed/inferred facts older than this many days."),
|
|
44
|
+
] = None,
|
|
45
|
+
) -> None:
|
|
46
|
+
"""Check a vault against the spec. Exit code 1 on errors (or warnings with --strict)."""
|
|
47
|
+
from memory_passport.validate import validate_vault
|
|
48
|
+
|
|
49
|
+
report = validate_vault(vault, stale_days=stale)
|
|
50
|
+
if as_json:
|
|
51
|
+
typer.echo(
|
|
52
|
+
json.dumps(
|
|
53
|
+
{
|
|
54
|
+
"root": str(report.root),
|
|
55
|
+
"files": report.file_count,
|
|
56
|
+
"facts": report.fact_count,
|
|
57
|
+
"ok": report.ok(strict=strict),
|
|
58
|
+
"issues": [i.__dict__ for i in report.issues],
|
|
59
|
+
},
|
|
60
|
+
indent=2,
|
|
61
|
+
)
|
|
62
|
+
)
|
|
63
|
+
else:
|
|
64
|
+
for issue in report.issues:
|
|
65
|
+
typer.echo(str(issue))
|
|
66
|
+
summary = (
|
|
67
|
+
f"{report.file_count} file(s), {report.fact_count} fact(s), "
|
|
68
|
+
f"{len(report.errors)} error(s), {len(report.warnings)} warning(s)"
|
|
69
|
+
)
|
|
70
|
+
typer.echo(("OK " if report.ok(strict=strict) else "FAIL ") + summary)
|
|
71
|
+
raise typer.Exit(code=0 if report.ok(strict=strict) else 1)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@app.command("import")
|
|
75
|
+
def import_(
|
|
76
|
+
source: Annotated[Path, typer.Argument(help="Export file, folder or pasted memory text.")],
|
|
77
|
+
from_: Annotated[
|
|
78
|
+
str | None, typer.Option("--from", help="Importer name (see `passport importers`).")
|
|
79
|
+
] = None,
|
|
80
|
+
out: Annotated[Path, typer.Option("--out", "-o", help="Vault directory to write.")] = Path(
|
|
81
|
+
"passport"
|
|
82
|
+
),
|
|
83
|
+
memory_text: Annotated[
|
|
84
|
+
Path | None, typer.Option(help="Extra pasted-memory file to combine with the export.")
|
|
85
|
+
] = None,
|
|
86
|
+
route: Annotated[
|
|
87
|
+
bool, typer.Option(help="Sort facts into people/topics/areas/preferences.")
|
|
88
|
+
] = True,
|
|
89
|
+
allow_health: Annotated[
|
|
90
|
+
bool, typer.Option(help="Keep health facts and mark the vault as opted in.")
|
|
91
|
+
] = False,
|
|
92
|
+
force: Annotated[bool, typer.Option(help="Write into a non-empty directory.")] = False,
|
|
93
|
+
) -> None:
|
|
94
|
+
"""Build a passport vault from a product's export."""
|
|
95
|
+
from memory_passport.importers import ImportOptions, get_importer, list_importers
|
|
96
|
+
from memory_passport.model import VaultError, write_vault
|
|
97
|
+
|
|
98
|
+
if not source.exists():
|
|
99
|
+
typer.echo(f"error: {source} does not exist", err=True)
|
|
100
|
+
raise typer.Exit(2)
|
|
101
|
+
if from_ is None:
|
|
102
|
+
matches = [n for n, cls in list_importers().items() if cls().detect(source)]
|
|
103
|
+
if len(matches) != 1:
|
|
104
|
+
typer.echo(
|
|
105
|
+
f"error: could not tell which importer to use ({', '.join(matches) or 'none'} "
|
|
106
|
+
"matched); pass --from",
|
|
107
|
+
err=True,
|
|
108
|
+
)
|
|
109
|
+
raise typer.Exit(2)
|
|
110
|
+
from_ = matches[0]
|
|
111
|
+
try:
|
|
112
|
+
importer = get_importer(from_)
|
|
113
|
+
except KeyError as e:
|
|
114
|
+
typer.echo(f"error: {e}", err=True)
|
|
115
|
+
raise typer.Exit(2) from None
|
|
116
|
+
if out.exists() and any(out.iterdir()) and not force:
|
|
117
|
+
typer.echo(f"error: {out} is not empty; pass --force to write into it", err=True)
|
|
118
|
+
raise typer.Exit(2)
|
|
119
|
+
opts = ImportOptions(route=route, allow_health=allow_health, memory_text=memory_text)
|
|
120
|
+
try:
|
|
121
|
+
result = importer.load(source, opts)
|
|
122
|
+
except VaultError as e:
|
|
123
|
+
typer.echo(f"error: {e}", err=True)
|
|
124
|
+
raise typer.Exit(1) from None
|
|
125
|
+
write_vault(result.vault, out)
|
|
126
|
+
for n in result.notes:
|
|
127
|
+
typer.echo(f" {n}")
|
|
128
|
+
for cat, text in result.dropped:
|
|
129
|
+
typer.echo(f" dropped ({cat}): {text[:60]}{'…' if len(text) > 60 else ''}")
|
|
130
|
+
for _, text in result.redacted:
|
|
131
|
+
typer.echo(f" redacted: {text[:60]}{'…' if len(text) > 60 else ''}")
|
|
132
|
+
typer.echo(f"wrote {len(result.vault.files)} file(s), {result.fact_count} fact(s) to {out}")
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@app.command()
|
|
136
|
+
def export(
|
|
137
|
+
vault: Annotated[Path, typer.Argument(help="Vault directory.")],
|
|
138
|
+
to: Annotated[str, typer.Option("--to", help="Exporter name (see `passport exporters`).")],
|
|
139
|
+
out: Annotated[
|
|
140
|
+
Path | None,
|
|
141
|
+
typer.Option(
|
|
142
|
+
"--out",
|
|
143
|
+
"-o",
|
|
144
|
+
help="File or folder to write; default prints or uses the exporter's name.",
|
|
145
|
+
),
|
|
146
|
+
] = None,
|
|
147
|
+
) -> None:
|
|
148
|
+
"""Render a vault as the text or files a product accepts."""
|
|
149
|
+
from memory_passport.exporters import get_exporter
|
|
150
|
+
from memory_passport.model import VaultError, load_vault
|
|
151
|
+
|
|
152
|
+
try:
|
|
153
|
+
exporter = get_exporter(to)
|
|
154
|
+
v = load_vault(vault)
|
|
155
|
+
except (KeyError, VaultError) as e:
|
|
156
|
+
typer.echo(f"error: {e}", err=True)
|
|
157
|
+
raise typer.Exit(2) from None
|
|
158
|
+
result = exporter.render(v)
|
|
159
|
+
for n in result.notes:
|
|
160
|
+
typer.echo(f"note: {n}", err=True)
|
|
161
|
+
if result.single is not None and "-" in result.files:
|
|
162
|
+
if out is None:
|
|
163
|
+
typer.echo(result.single, nl=False)
|
|
164
|
+
else:
|
|
165
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
166
|
+
out.write_text(result.single, encoding="utf-8")
|
|
167
|
+
typer.echo(f"wrote {out}", err=True)
|
|
168
|
+
return
|
|
169
|
+
target = out or Path(exporter.default_out)
|
|
170
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
171
|
+
for rel, content in result.files.items():
|
|
172
|
+
p = target / rel
|
|
173
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
174
|
+
p.write_text(content, encoding="utf-8")
|
|
175
|
+
typer.echo(f"wrote {len(result.files)} file(s) to {target}", err=True)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@app.command()
|
|
179
|
+
def merge(
|
|
180
|
+
vault_a: Annotated[Path, typer.Argument()],
|
|
181
|
+
vault_b: Annotated[Path, typer.Argument()],
|
|
182
|
+
out: Annotated[
|
|
183
|
+
Path, typer.Option("--out", "-o", help="Directory for the merged vault.")
|
|
184
|
+
] = Path("merged"),
|
|
185
|
+
force: Annotated[bool, typer.Option(help="Write into a non-empty directory.")] = False,
|
|
186
|
+
) -> None:
|
|
187
|
+
"""Merge two vaults. Disagreements get conflict markers; nothing is resolved silently."""
|
|
188
|
+
from memory_passport.merge import merge_dirs
|
|
189
|
+
from memory_passport.model import VaultError, write_vault
|
|
190
|
+
|
|
191
|
+
if out.exists() and any(out.iterdir()) and not force:
|
|
192
|
+
typer.echo(f"error: {out} is not empty; pass --force to write into it", err=True)
|
|
193
|
+
raise typer.Exit(2)
|
|
194
|
+
try:
|
|
195
|
+
merged, report = merge_dirs(vault_a, vault_b)
|
|
196
|
+
except VaultError as e:
|
|
197
|
+
typer.echo(f"error: {e}", err=True)
|
|
198
|
+
raise typer.Exit(1) from None
|
|
199
|
+
write_vault(merged, out)
|
|
200
|
+
typer.echo(f"merged {report.files} file(s), {report.facts} fact(s) into {out}")
|
|
201
|
+
for path, what in report.conflicts:
|
|
202
|
+
typer.echo(f" CONFLICT {path}: {what}")
|
|
203
|
+
if report.conflicts:
|
|
204
|
+
typer.echo(
|
|
205
|
+
f"{len(report.conflicts)} conflict(s) need a human; "
|
|
206
|
+
"the vault will not validate until resolved"
|
|
207
|
+
)
|
|
208
|
+
raise typer.Exit(3)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@app.command()
|
|
212
|
+
def diff(
|
|
213
|
+
vault_a: Annotated[Path, typer.Argument()],
|
|
214
|
+
vault_b: Annotated[Path, typer.Argument()],
|
|
215
|
+
as_json: Annotated[bool, typer.Option("--json")] = False,
|
|
216
|
+
) -> None:
|
|
217
|
+
"""Show what changed between two vaults, fact by fact. Exit code 1 if they differ."""
|
|
218
|
+
from memory_passport.diff import diff_dirs
|
|
219
|
+
from memory_passport.model import VaultError
|
|
220
|
+
|
|
221
|
+
try:
|
|
222
|
+
d = diff_dirs(vault_a, vault_b)
|
|
223
|
+
except VaultError as e:
|
|
224
|
+
typer.echo(f"error: {e}", err=True)
|
|
225
|
+
raise typer.Exit(2) from None
|
|
226
|
+
if as_json:
|
|
227
|
+
typer.echo(
|
|
228
|
+
json.dumps(
|
|
229
|
+
{
|
|
230
|
+
"only_a": [str(p) for p in d.only_a],
|
|
231
|
+
"only_b": [str(p) for p in d.only_b],
|
|
232
|
+
"changed": [
|
|
233
|
+
{
|
|
234
|
+
"path": str(fd.path),
|
|
235
|
+
"added": [f.render() for f in fd.added],
|
|
236
|
+
"removed": [f.render() for f in fd.removed],
|
|
237
|
+
"retagged": [[a.tag, b.tag, b.text] for a, b in fd.retagged],
|
|
238
|
+
"frontmatter": {k: list(v) for k, v in fd.frontmatter.items()},
|
|
239
|
+
}
|
|
240
|
+
for fd in d.changed
|
|
241
|
+
],
|
|
242
|
+
},
|
|
243
|
+
indent=2,
|
|
244
|
+
default=str,
|
|
245
|
+
)
|
|
246
|
+
)
|
|
247
|
+
else:
|
|
248
|
+
typer.echo(d.render(), nl=False)
|
|
249
|
+
raise typer.Exit(0 if d.empty else 1)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
@app.command()
|
|
253
|
+
def show(
|
|
254
|
+
vault: Annotated[Path, typer.Argument(help="Vault directory.")],
|
|
255
|
+
subject: Annotated[
|
|
256
|
+
str | None, typer.Argument(help="profile, preferences, people/<slug>, a kind, or a name.")
|
|
257
|
+
] = None,
|
|
258
|
+
query: Annotated[str | None, typer.Option("--query", "-q", help="Words to search for.")] = None,
|
|
259
|
+
) -> None:
|
|
260
|
+
"""Print a vault's subjects, one subject, or the facts matching a query."""
|
|
261
|
+
from memory_passport.model import VaultError, load_vault
|
|
262
|
+
from memory_passport.store import render_file, render_subjects, search
|
|
263
|
+
|
|
264
|
+
try:
|
|
265
|
+
v = load_vault(vault)
|
|
266
|
+
except VaultError as e:
|
|
267
|
+
typer.echo(f"error: {e}", err=True)
|
|
268
|
+
raise typer.Exit(2) from None
|
|
269
|
+
if query:
|
|
270
|
+
hits = search(v, query, subject)
|
|
271
|
+
for mf, f in hits:
|
|
272
|
+
typer.echo(f"{mf.path}: {f.render()}")
|
|
273
|
+
typer.echo(f"{len(hits)} fact(s)", err=True)
|
|
274
|
+
raise typer.Exit(0 if hits else 1)
|
|
275
|
+
if subject is None:
|
|
276
|
+
typer.echo(render_subjects(v), nl=False)
|
|
277
|
+
return
|
|
278
|
+
matches = [mf for mf in v.files if _matches(mf, subject)]
|
|
279
|
+
if not matches:
|
|
280
|
+
typer.echo(f"error: no subject matches '{subject}'", err=True)
|
|
281
|
+
raise typer.Exit(1)
|
|
282
|
+
for mf in matches:
|
|
283
|
+
typer.echo(render_file(mf))
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _matches(mf, subject: str) -> bool:
|
|
287
|
+
from memory_passport.store import _file_matches
|
|
288
|
+
|
|
289
|
+
return _file_matches(mf, subject)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
@app.command()
|
|
293
|
+
def add(
|
|
294
|
+
vault: Annotated[Path, typer.Argument(help="Vault directory.")],
|
|
295
|
+
text: Annotated[str, typer.Argument(help="The fact, one sentence.")],
|
|
296
|
+
to: Annotated[
|
|
297
|
+
str | None,
|
|
298
|
+
typer.Option(
|
|
299
|
+
"--to",
|
|
300
|
+
help=(
|
|
301
|
+
"profile, preferences, people/<slug>, person:<Name>, topic:<Name>, "
|
|
302
|
+
"area:<Name>. Default: routed from the text."
|
|
303
|
+
),
|
|
304
|
+
),
|
|
305
|
+
] = None,
|
|
306
|
+
tag: Annotated[str, typer.Option(help="stated, observed or inferred.")] = "stated",
|
|
307
|
+
section: Annotated[str, typer.Option(help="## heading to file it under.")] = "",
|
|
308
|
+
) -> None:
|
|
309
|
+
"""Append one fact to a vault, dated today, with exclusions applied."""
|
|
310
|
+
from memory_passport.model import VaultError
|
|
311
|
+
from memory_passport.store import add_fact
|
|
312
|
+
|
|
313
|
+
if tag not in ("stated", "observed", "inferred"):
|
|
314
|
+
typer.echo("error: --tag must be stated, observed or inferred", err=True)
|
|
315
|
+
raise typer.Exit(2)
|
|
316
|
+
try:
|
|
317
|
+
r = add_fact(vault, text, subject=to, tag=tag, section=section) # type: ignore[arg-type]
|
|
318
|
+
except VaultError as e:
|
|
319
|
+
typer.echo(f"error: {e}", err=True)
|
|
320
|
+
raise typer.Exit(2) from None
|
|
321
|
+
if r.dropped:
|
|
322
|
+
typer.echo(
|
|
323
|
+
f"refused: looks like {r.dropped}, which the spec excludes (SPEC.md §7)", err=True
|
|
324
|
+
)
|
|
325
|
+
raise typer.Exit(1)
|
|
326
|
+
if r.duplicate:
|
|
327
|
+
typer.echo(f"already in {r.path}; nothing written")
|
|
328
|
+
return
|
|
329
|
+
typer.echo(f"{'created' if r.created_file else 'updated'} {r.path}: {r.fact.render()}")
|
|
330
|
+
if r.redacted:
|
|
331
|
+
typer.echo(" a sensitive span was redacted", err=True)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
@app.command()
|
|
335
|
+
def forget(
|
|
336
|
+
vault: Annotated[Path, typer.Argument(help="Vault directory.")],
|
|
337
|
+
text: Annotated[str, typer.Argument(help="The fact to remove (matched loosely).")],
|
|
338
|
+
) -> None:
|
|
339
|
+
"""Remove a fact from a vault by its text."""
|
|
340
|
+
from memory_passport.store import remove_fact
|
|
341
|
+
|
|
342
|
+
removed = remove_fact(vault, text)
|
|
343
|
+
if not removed:
|
|
344
|
+
typer.echo("nothing matched", err=True)
|
|
345
|
+
raise typer.Exit(1)
|
|
346
|
+
for p, n in removed:
|
|
347
|
+
typer.echo(f"removed {p}:{n}")
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
@app.command()
|
|
351
|
+
def inspect(
|
|
352
|
+
source: Annotated[Path, typer.Argument(help="An export zip, folder or file.")],
|
|
353
|
+
) -> None:
|
|
354
|
+
"""Report what an export contains, without importing it. Paste the output into bug reports."""
|
|
355
|
+
from memory_passport.inspect_export import inspect_path
|
|
356
|
+
|
|
357
|
+
if not source.exists():
|
|
358
|
+
typer.echo(f"error: {source} does not exist", err=True)
|
|
359
|
+
raise typer.Exit(2)
|
|
360
|
+
for line in inspect_path(source):
|
|
361
|
+
typer.echo(line)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
@app.command()
|
|
365
|
+
def importers() -> None:
|
|
366
|
+
"""List available importers (built-in and plugins)."""
|
|
367
|
+
from memory_passport.importers import list_importers
|
|
368
|
+
|
|
369
|
+
for name, cls in list_importers().items():
|
|
370
|
+
typer.echo(f"{name:12} {cls.help}")
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
@app.command()
|
|
374
|
+
def exporters() -> None:
|
|
375
|
+
"""List available exporters (built-in and plugins)."""
|
|
376
|
+
from memory_passport.exporters import list_exporters
|
|
377
|
+
|
|
378
|
+
for name, cls in list_exporters().items():
|
|
379
|
+
typer.echo(f"{name:12} {cls.help}")
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
if __name__ == "__main__":
|
|
383
|
+
app()
|
memory_passport/diff.py
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Compare two vaults fact by fact."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path, PurePosixPath
|
|
7
|
+
|
|
8
|
+
from memory_passport.model import Fact, Vault, load_vault
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class FileDiff:
|
|
13
|
+
path: PurePosixPath
|
|
14
|
+
added: list[Fact] = field(default_factory=list)
|
|
15
|
+
removed: list[Fact] = field(default_factory=list)
|
|
16
|
+
retagged: list[tuple[Fact, Fact]] = field(default_factory=list)
|
|
17
|
+
frontmatter: dict[str, tuple[object, object]] = field(default_factory=dict)
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def empty(self) -> bool:
|
|
21
|
+
return not (self.added or self.removed or self.retagged or self.frontmatter)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class VaultDiff:
|
|
26
|
+
only_a: list[PurePosixPath]
|
|
27
|
+
only_b: list[PurePosixPath]
|
|
28
|
+
changed: list[FileDiff]
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def empty(self) -> bool:
|
|
32
|
+
return not (self.only_a or self.only_b or self.changed)
|
|
33
|
+
|
|
34
|
+
def render(self) -> str:
|
|
35
|
+
out: list[str] = []
|
|
36
|
+
for p in self.only_a:
|
|
37
|
+
out.append(f"--- {p} (only in a)")
|
|
38
|
+
for p in self.only_b:
|
|
39
|
+
out.append(f"+++ {p} (only in b)")
|
|
40
|
+
for fd in self.changed:
|
|
41
|
+
out.append(f"=== {fd.path}")
|
|
42
|
+
for k, (a, b) in fd.frontmatter.items():
|
|
43
|
+
out.append(f" ~ {k}: {a!r} -> {b!r}")
|
|
44
|
+
for f in fd.removed:
|
|
45
|
+
out.append(f" - [{f.tag}] {f.text}")
|
|
46
|
+
for f in fd.added:
|
|
47
|
+
out.append(f" + [{f.tag}] {f.text}")
|
|
48
|
+
for a, b in fd.retagged:
|
|
49
|
+
out.append(f" ~ [{a.tag}] -> [{b.tag}] {b.text}")
|
|
50
|
+
return "\n".join(out) + ("\n" if out else "no differences\n")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
_IGNORED = {"updated"}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def diff_vaults(a: Vault, b: Vault) -> VaultDiff:
|
|
57
|
+
fa = {f.path: f for f in a.files}
|
|
58
|
+
fb = {f.path: f for f in b.files}
|
|
59
|
+
changed: list[FileDiff] = []
|
|
60
|
+
for p in sorted(fa.keys() & fb.keys()):
|
|
61
|
+
fd = FileDiff(p)
|
|
62
|
+
ka = {f.key: f for f in fa[p].facts}
|
|
63
|
+
kb = {f.key: f for f in fb[p].facts}
|
|
64
|
+
fd.removed = [ka[k] for k in ka if k not in kb]
|
|
65
|
+
fd.added = [kb[k] for k in kb if k not in ka]
|
|
66
|
+
fd.retagged = [(ka[k], kb[k]) for k in ka if k in kb and ka[k].tag != kb[k].tag]
|
|
67
|
+
for key in sorted((set(fa[p].frontmatter) | set(fb[p].frontmatter)) - _IGNORED):
|
|
68
|
+
va, vb = fa[p].frontmatter.get(key), fb[p].frontmatter.get(key)
|
|
69
|
+
if isinstance(va, list) and isinstance(vb, list) and set(va) == set(vb):
|
|
70
|
+
continue
|
|
71
|
+
if va != vb:
|
|
72
|
+
fd.frontmatter[key] = (va, vb)
|
|
73
|
+
if not fd.empty:
|
|
74
|
+
changed.append(fd)
|
|
75
|
+
return VaultDiff(
|
|
76
|
+
only_a=sorted(fa.keys() - fb.keys()), only_b=sorted(fb.keys() - fa.keys()), changed=changed
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def diff_dirs(a: Path, b: Path) -> VaultDiff:
|
|
81
|
+
return diff_vaults(load_vault(a), load_vault(b))
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Detectors for categories the spec excludes from a passport by default.
|
|
2
|
+
|
|
3
|
+
Detection is deliberately conservative and heuristic: it exists to stop the
|
|
4
|
+
obvious cases (a card number pasted into a fact line) rather than to be a
|
|
5
|
+
complete data-loss-prevention system. See SPEC.md §7 for the rationale.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Literal
|
|
13
|
+
|
|
14
|
+
Category = Literal["card-number", "bank-account", "government-id", "secret", "health"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class Hit:
|
|
19
|
+
category: Category
|
|
20
|
+
detail: str
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _luhn_ok(digits: str) -> bool:
|
|
24
|
+
total, parity = 0, len(digits) % 2
|
|
25
|
+
for i, ch in enumerate(digits):
|
|
26
|
+
d = int(ch)
|
|
27
|
+
if i % 2 == parity:
|
|
28
|
+
d *= 2
|
|
29
|
+
if d > 9:
|
|
30
|
+
d -= 9
|
|
31
|
+
total += d
|
|
32
|
+
return total % 10 == 0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _iban_ok(s: str) -> bool:
|
|
36
|
+
s = s.upper()
|
|
37
|
+
rearranged = s[4:] + s[:4]
|
|
38
|
+
numeric = "".join(str(ord(c) - 55) if c.isalpha() else c for c in rearranged)
|
|
39
|
+
return int(numeric) % 97 == 1
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
_CARD_RE = re.compile(r"(?<![\d-])(?:\d[ -]?){12,18}\d(?![\d-])")
|
|
43
|
+
_IBAN_RE = re.compile(r"\b[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]{4}){2,7}[ ]?[A-Z0-9]{1,4}\b")
|
|
44
|
+
_UK_SORT_ACCT_RE = re.compile(r"\b\d{2}-\d{2}-\d{2}\b[^\n]{0,20}\b\d{8}\b")
|
|
45
|
+
_US_SSN_RE = re.compile(r"\b(?!000|666|9\d\d)\d{3}-(?!00)\d{2}-(?!0000)\d{4}\b")
|
|
46
|
+
_UK_NINO_RE = re.compile(
|
|
47
|
+
r"\b(?!BG|GB|NK|KN|TN|NT|ZZ)[A-CEGHJ-PR-TW-Z][A-CEGHJ-NPR-TW-Z]\s?\d{2}\s?\d{2}\s?\d{2}\s?[A-D]\b"
|
|
48
|
+
)
|
|
49
|
+
_ID_KEYWORD_RE = re.compile(
|
|
50
|
+
r"\b(passport|national insurance|social security|driving licence|driver'?s licen[cs]e|"
|
|
51
|
+
r"nhs|aadhaar|pan card|tax id|ssn|nino)\b\s*(number|no\.?|#|:)?\s*(is\s+)?[:#]?\s*"
|
|
52
|
+
r"((?=[A-Z -]*\d)[A-Z0-9][A-Z0-9 -]{5,})",
|
|
53
|
+
re.IGNORECASE,
|
|
54
|
+
)
|
|
55
|
+
_SECRET_RE = re.compile(
|
|
56
|
+
r"(sk-(?:proj-|ant-)?[A-Za-z0-9_-]{20,}|AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{30,}|"
|
|
57
|
+
r"xox[baprs]-[A-Za-z0-9-]{10,}|AIza[0-9A-Za-z_-]{35}|-----BEGIN [A-Z ]*PRIVATE KEY-----)"
|
|
58
|
+
)
|
|
59
|
+
_PASSWORD_RE = re.compile(
|
|
60
|
+
r"\b(password|passcode|pin)\b\s*(is|:|=)\s*(?!\[redacted)\S+", re.IGNORECASE
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
_HEALTH_CONDITIONS = (
|
|
64
|
+
"adhd|autism|autistic|anxiety disorder|depression|depressive|bipolar|schizophreni|ptsd|ocd|"
|
|
65
|
+
"eating disorder|anorexi|bulimi|diabet|cancer|tumour|tumor|hiv|aids|hepatitis|epilep|"
|
|
66
|
+
"asthma|crohn|coeliac|celiac|arthritis|fibromyalgia|dementia|alzheimer|parkinson|"
|
|
67
|
+
"multiple sclerosis|chronic fatigue|long covid|hypertension|heart disease|stroke|"
|
|
68
|
+
"pregnan|miscarriage|infertil|erectile|std|sti|herpes|chlamydia"
|
|
69
|
+
)
|
|
70
|
+
_HEALTH_RE = re.compile(
|
|
71
|
+
r"\b(diagnos(?:ed|is)|prescri(?:bed|ption)|medication|antidepressant|therapist|psychiatrist|"
|
|
72
|
+
r"\d+\s?mg\b|" + _HEALTH_CONDITIONS + r")",
|
|
73
|
+
re.IGNORECASE,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
REDACTABLE: tuple[Category, ...] = ("card-number", "bank-account", "government-id", "secret")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def redact(text: str, *, allow_health: bool = False) -> tuple[str, list[Hit]]:
|
|
81
|
+
"""Replace every redactable span with ``[redacted <category>]``.
|
|
82
|
+
|
|
83
|
+
Returns the new text and the hits that could *not* be redacted (currently only
|
|
84
|
+
``health``, which is a topic rather than a token and so must be dropped instead).
|
|
85
|
+
"""
|
|
86
|
+
out = text
|
|
87
|
+
for m in list(_CARD_RE.finditer(out))[::-1]:
|
|
88
|
+
digits = re.sub(r"\D", "", m.group())
|
|
89
|
+
if 13 <= len(digits) <= 19 and _luhn_ok(digits):
|
|
90
|
+
out = out[: m.start()] + "[redacted card-number]" + out[m.end() :]
|
|
91
|
+
for m in list(_IBAN_RE.finditer(out))[::-1]:
|
|
92
|
+
compact = m.group().replace(" ", "")
|
|
93
|
+
if 15 <= len(compact) <= 34 and _iban_ok(compact):
|
|
94
|
+
out = out[: m.start()] + "[redacted bank-account]" + out[m.end() :]
|
|
95
|
+
out = _UK_SORT_ACCT_RE.sub("[redacted bank-account]", out)
|
|
96
|
+
out = _US_SSN_RE.sub("[redacted government-id]", out)
|
|
97
|
+
out = _UK_NINO_RE.sub("[redacted government-id]", out)
|
|
98
|
+
out = _ID_KEYWORD_RE.sub(
|
|
99
|
+
lambda m: m.group(0)[: m.start(4) - m.start(0)] + "[redacted government-id]", out
|
|
100
|
+
)
|
|
101
|
+
out = _SECRET_RE.sub("[redacted secret]", out)
|
|
102
|
+
out = _PASSWORD_RE.sub(lambda m: f"{m.group(1)} {m.group(2)} [redacted secret]", out)
|
|
103
|
+
return out, scan(out, allow_health=allow_health)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def scan(text: str, *, allow_health: bool = False) -> list[Hit]:
|
|
107
|
+
"""Return every exclusion hit in ``text``. Empty list means clean."""
|
|
108
|
+
hits: list[Hit] = []
|
|
109
|
+
|
|
110
|
+
for m in _CARD_RE.finditer(text):
|
|
111
|
+
digits = re.sub(r"\D", "", m.group())
|
|
112
|
+
if 13 <= len(digits) <= 19 and _luhn_ok(digits):
|
|
113
|
+
hits.append(Hit("card-number", f"Luhn-valid {len(digits)}-digit number"))
|
|
114
|
+
|
|
115
|
+
for m in _IBAN_RE.finditer(text):
|
|
116
|
+
compact = m.group().replace(" ", "")
|
|
117
|
+
if 15 <= len(compact) <= 34 and _iban_ok(compact):
|
|
118
|
+
hits.append(Hit("bank-account", f"IBAN starting {compact[:4]}"))
|
|
119
|
+
if _UK_SORT_ACCT_RE.search(text):
|
|
120
|
+
hits.append(Hit("bank-account", "UK sort code with account number"))
|
|
121
|
+
|
|
122
|
+
if _US_SSN_RE.search(text):
|
|
123
|
+
hits.append(Hit("government-id", "US social security number pattern"))
|
|
124
|
+
if _UK_NINO_RE.search(text):
|
|
125
|
+
hits.append(Hit("government-id", "UK national insurance number pattern"))
|
|
126
|
+
for m in _ID_KEYWORD_RE.finditer(text):
|
|
127
|
+
hits.append(Hit("government-id", f"'{m.group(1)}' followed by an identifier"))
|
|
128
|
+
|
|
129
|
+
if _SECRET_RE.search(text):
|
|
130
|
+
hits.append(Hit("secret", "API key or private key pattern"))
|
|
131
|
+
if m := _PASSWORD_RE.search(text):
|
|
132
|
+
hits.append(Hit("secret", f"'{m.group(1)}' with a value"))
|
|
133
|
+
|
|
134
|
+
if not allow_health and (m := _HEALTH_RE.search(text)):
|
|
135
|
+
hits.append(Hit("health", f"health term '{m.group(1)}'"))
|
|
136
|
+
|
|
137
|
+
return hits
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""Exporters render a vault into what a product will accept."""
|
|
2
|
+
|
|
3
|
+
from memory_passport.exporters.base import Exporter, ExportResult
|
|
4
|
+
from memory_passport.exporters.registry import get_exporter, list_exporters
|
|
5
|
+
|
|
6
|
+
__all__ = ["ExportResult", "Exporter", "get_exporter", "list_exporters"]
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import ClassVar
|
|
6
|
+
|
|
7
|
+
from memory_passport.model import Fact, MemoryFile, Vault
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class ExportResult:
|
|
12
|
+
files: dict[str, str]
|
|
13
|
+
"""Relative path -> content. A single entry named ``-`` means "print to stdout"."""
|
|
14
|
+
notes: list[str] = field(default_factory=list)
|
|
15
|
+
|
|
16
|
+
@property
|
|
17
|
+
def single(self) -> str | None:
|
|
18
|
+
return next(iter(self.files.values())) if len(self.files) == 1 else None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Exporter(ABC):
|
|
22
|
+
name: ClassVar[str]
|
|
23
|
+
help: ClassVar[str] = ""
|
|
24
|
+
default_out: ClassVar[str] = "-"
|
|
25
|
+
|
|
26
|
+
@abstractmethod
|
|
27
|
+
def render(self, vault: Vault) -> ExportResult: ...
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def fact_sentence(fact: Fact, *, hedge: bool = True) -> str:
|
|
31
|
+
"""A fact as plain prose for products with no provenance. Inferred facts are hedged."""
|
|
32
|
+
if hedge and fact.tag == "inferred":
|
|
33
|
+
return f"Possibly: {fact.text}"
|
|
34
|
+
if hedge and fact.tag == "observed":
|
|
35
|
+
return f"Observed: {fact.text}"
|
|
36
|
+
return fact.text
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def ordered(vault: Vault) -> list[MemoryFile]:
|
|
40
|
+
order = {"profile": 0, "preferences": 1, "person": 2, "area": 3, "topic": 4}
|
|
41
|
+
return sorted(vault.files, key=lambda f: (order.get(f.kind or "", 9), str(f.path)))
|