memoryintelligence-mcp 0.2.7__tar.gz → 0.2.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of memoryintelligence-mcp might be problematic. Click here for more details.
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/CHANGELOG.md +73 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/PKG-INFO +14 -2
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/README.md +13 -1
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/pyproject.toml +1 -1
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/__init__.py +1 -1
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/cli.py +91 -0
- memoryintelligence_mcp-0.2.9/src/mi_mcp/scrub.py +251 -0
- memoryintelligence_mcp-0.2.7/src/mi_mcp/scrub.py +0 -135
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/.gitignore +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/CONTRIBUTING.md +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/LICENSE +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/__main__.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/client.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/config.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/embedder.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/indexer.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/keys.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/local_index.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/localreads.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/paths.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/recall_guard.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/server.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/umo_format.py +0 -0
- {memoryintelligence_mcp-0.2.7 → memoryintelligence_mcp-0.2.9}/src/mi_mcp/vault.py +0 -0
|
@@ -3,6 +3,79 @@
|
|
|
3
3
|
All notable changes to `memoryintelligence-mcp` are documented here.
|
|
4
4
|
The format follows [Keep a Changelog](https://keepachangelog.com/); this project uses [Semantic Versioning](https://semver.org/).
|
|
5
5
|
|
|
6
|
+
## [0.2.9] — 2026-08-09
|
|
7
|
+
|
|
8
|
+
### Fixed — the thin-install PII floor had drifted behind core (#1464)
|
|
9
|
+
|
|
10
|
+
`scrub.py` carries its own hard-PII regex floor. On a thin PyPI install it is the
|
|
11
|
+
**only** protection: this package depends on `mcp` / `httpx` / `pydantic` and NOT on
|
|
12
|
+
`memory-intelligence`, so the "strong path" through the core detector is absent for
|
|
13
|
+
every ordinary `pip install memoryintelligence-mcp` user.
|
|
14
|
+
|
|
15
|
+
That floor fell behind the core detector without anything noticing. **Measured: 7 of
|
|
16
|
+
14 hard-PII cases leaked** to the model context.
|
|
17
|
+
|
|
18
|
+
Brought to parity for everything a regex can do:
|
|
19
|
+
|
|
20
|
+
- **SSN** — separator-agnostic. Was hyphen-only, so `412 55 9832` and `412.55.9832`
|
|
21
|
+
walked straight through.
|
|
22
|
+
- **Phone** — added the `+CC` short form and a **context-anchored** bare 7-digit.
|
|
23
|
+
`555-0142` and `+1-555-0142` both leaked before, including an exemplar fixture.
|
|
24
|
+
The bare form is anchored to a telephony keyword rather than matched outright,
|
|
25
|
+
because a naked `NNN-NNNN` also matches file:line ranges and UUID fragments.
|
|
26
|
+
- **Address** — there was **no address detector at all**. Now covers English,
|
|
27
|
+
international and compound/glued street forms.
|
|
28
|
+
- **Obfuscation** — NFKC normalisation, zero-width strip and homoglyph folding, so a
|
|
29
|
+
zero-width-split or Cyrillic-lookalike value is caught.
|
|
30
|
+
- **Email** — tolerates the whitespace-split form.
|
|
31
|
+
|
|
32
|
+
Result: **15 of 16 cases redact**, with no over-redaction of file:line ranges,
|
|
33
|
+
numeric ranges, migration numbers or numbered headings.
|
|
34
|
+
|
|
35
|
+
### Known limitation — PERSON
|
|
36
|
+
|
|
37
|
+
`PERSON` / `ORG` / `LOCATION` need the corpus NER, which a thin install does not
|
|
38
|
+
have. Names are still covered *conditionally*, by the stored-entity belt, when
|
|
39
|
+
capture extracted them. This is asserted in the test suite as a known state rather
|
|
40
|
+
than left implicit.
|
|
41
|
+
|
|
42
|
+
## [0.2.8] — 2026-08-04
|
|
43
|
+
|
|
44
|
+
### Added — `mi-mcp doctor` tells you when you're behind
|
|
45
|
+
Doctor reported the binary, PATH, wrapper, key, allowlist, vault and wiring — and
|
|
46
|
+
not one word about **which version it was running**. So the most common way to be
|
|
47
|
+
broken was invisible: a user on 0.2.5 asking why the workspace tools "don't exist",
|
|
48
|
+
when they exist fine and the install is two releases old.
|
|
49
|
+
|
|
50
|
+
Doctor now checks PyPI and reports as its **first** line:
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
[✗] version 0.2.5 installed, 0.2.8 available — `uv tool upgrade memoryintelligence-mcp && mi-mcp wire`
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Details that matter:
|
|
57
|
+
|
|
58
|
+
- **The upgrade command matches how you actually installed it** — `uv tool` /
|
|
59
|
+
`pipx` / `pip`, detected from the interpreter's location. Suggesting `pip` to a
|
|
60
|
+
uv-tool user is a dead end; on a Homebrew Python, `pip` may not exist at all
|
|
61
|
+
and is PEP-668 externally-managed regardless.
|
|
62
|
+
- **`mi-mcp wire` is in the suggestion on purpose.** Upgrading across 0.2.6 without
|
|
63
|
+
re-wiring leaves your config pointing at a server id that no longer announces
|
|
64
|
+
itself, and the tools silently disappear.
|
|
65
|
+
- **Out of date never fails doctor's exit code** — behind is not broken, and scripts
|
|
66
|
+
that gate on `mi-mcp doctor` must not start failing the day a release lands.
|
|
67
|
+
- **Unreachable PyPI reports "latest unknown", never "up to date".** Offline is not
|
|
68
|
+
evidence of being current.
|
|
69
|
+
- **Opt out** with `--no-version-check` or `MI_MCP_NO_VERSION_CHECK=1`. Then **no
|
|
70
|
+
request is made at all** — not a request whose answer is ignored. The check is an
|
|
71
|
+
anonymous PyPI index fetch, the same one any install performs.
|
|
72
|
+
|
|
73
|
+
It reads the **simple index** (PEP 691), not `/pypi/<project>/json`. The aggregate
|
|
74
|
+
endpoint is CDN-cached and was observed serving a stale version for minutes after
|
|
75
|
+
0.2.7 published — a doctor reading it would confidently tell a user they were
|
|
76
|
+
current when they were two releases behind, which is precisely the bug this check
|
|
77
|
+
exists to prevent.
|
|
78
|
+
|
|
6
79
|
## [0.2.7] — 2026-08-03
|
|
7
80
|
|
|
8
81
|
### Added — `mi_ask` / `mi_list` take a `workspace_id` (#385 UC2)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: memoryintelligence-mcp
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.9
|
|
4
4
|
Summary: MCP server for MemoryIntelligence — give Claude a memory you own, set up in one command
|
|
5
5
|
Project-URL: Homepage, https://memoryintelligence.io
|
|
6
6
|
Project-URL: Repository, https://github.com/somewhere11/memoryintelligence-mcp
|
|
@@ -222,11 +222,23 @@ Never paste the key into an MCP client config.
|
|
|
222
222
|
|
|
223
223
|
**Repair / inspect without re-running setup:**
|
|
224
224
|
```bash
|
|
225
|
-
mi-mcp doctor # checks binary, PATH, key, wiring, opt-in, vault path
|
|
225
|
+
mi-mcp doctor # checks version, binary, PATH, key, wiring, opt-in, vault path
|
|
226
226
|
mi-mcp status # wired surfaces + opt-in allowlist
|
|
227
227
|
mi-mcp wire --dry-run # preview wiring changes
|
|
228
228
|
```
|
|
229
229
|
|
|
230
|
+
**Staying current.** `doctor` compares your installed version against PyPI and
|
|
231
|
+
prints the upgrade command for how you actually installed it:
|
|
232
|
+
|
|
233
|
+
```
|
|
234
|
+
[✗] version 0.2.5 installed, 0.2.8 available — `uv tool upgrade memoryintelligence-mcp && mi-mcp wire`
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
Run `mi-mcp wire` after any upgrade — 0.2.6 renamed the server, and a config left
|
|
238
|
+
pointing at the old id loses its tools silently. The check is an anonymous PyPI
|
|
239
|
+
index fetch; skip it entirely with `--no-version-check` or
|
|
240
|
+
`MI_MCP_NO_VERSION_CHECK=1`, and it fails quietly when you're offline.
|
|
241
|
+
|
|
230
242
|
**Which server am I talking to?** The local server announces `mi-local`; the
|
|
231
243
|
remote MCP surface announces `memoryintelligence-remote`. The local default
|
|
232
244
|
surface is 7 tools; the remote is 4 — if the tool list doesn't match what you
|
|
@@ -183,11 +183,23 @@ Never paste the key into an MCP client config.
|
|
|
183
183
|
|
|
184
184
|
**Repair / inspect without re-running setup:**
|
|
185
185
|
```bash
|
|
186
|
-
mi-mcp doctor # checks binary, PATH, key, wiring, opt-in, vault path
|
|
186
|
+
mi-mcp doctor # checks version, binary, PATH, key, wiring, opt-in, vault path
|
|
187
187
|
mi-mcp status # wired surfaces + opt-in allowlist
|
|
188
188
|
mi-mcp wire --dry-run # preview wiring changes
|
|
189
189
|
```
|
|
190
190
|
|
|
191
|
+
**Staying current.** `doctor` compares your installed version against PyPI and
|
|
192
|
+
prints the upgrade command for how you actually installed it:
|
|
193
|
+
|
|
194
|
+
```
|
|
195
|
+
[✗] version 0.2.5 installed, 0.2.8 available — `uv tool upgrade memoryintelligence-mcp && mi-mcp wire`
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Run `mi-mcp wire` after any upgrade — 0.2.6 renamed the server, and a config left
|
|
199
|
+
pointing at the old id loses its tools silently. The check is an anonymous PyPI
|
|
200
|
+
index fetch; skip it entirely with `--no-version-check` or
|
|
201
|
+
`MI_MCP_NO_VERSION_CHECK=1`, and it fails quietly when you're offline.
|
|
202
|
+
|
|
191
203
|
**Which server am I talking to?** The local server announces `mi-local`; the
|
|
192
204
|
remote MCP surface announces `memoryintelligence-remote`. The local default
|
|
193
205
|
surface is 7 tools; the remote is 4 — if the tool list doesn't match what you
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "memoryintelligence-mcp"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.9"
|
|
8
8
|
description = "MCP server for MemoryIntelligence — give Claude a memory you own, set up in one command"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -452,9 +452,75 @@ def cmd_wire(argv: list[str]) -> int:
|
|
|
452
452
|
return 0
|
|
453
453
|
|
|
454
454
|
|
|
455
|
+
#: PyPI project name — the package this CLI ships as.
|
|
456
|
+
PYPI_PROJECT = "memoryintelligence-mcp"
|
|
457
|
+
|
|
458
|
+
#: Set truthy to skip doctor's release check entirely (no network call at all).
|
|
459
|
+
NO_VERSION_CHECK_ENV = "MI_MCP_NO_VERSION_CHECK"
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _version_tuple(v: str) -> tuple:
|
|
463
|
+
"""`"0.2.10"` → `(0, 2, 10)` for ordering. Non-numeric parts sort as -1.
|
|
464
|
+
|
|
465
|
+
Deliberately not `packaging.version` — it is not a declared dependency, and a
|
|
466
|
+
doctor check must never be the thing that fails to import.
|
|
467
|
+
"""
|
|
468
|
+
parts: list = []
|
|
469
|
+
for chunk in (v or "").strip().split("."):
|
|
470
|
+
digits = "".join(c for c in chunk if c.isdigit())
|
|
471
|
+
parts.append(int(digits) if digits else -1)
|
|
472
|
+
return tuple(parts)
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _latest_pypi_version(project: str = PYPI_PROJECT, timeout: float = 2.0) -> str | None:
|
|
476
|
+
"""Newest version on PyPI, or None if it cannot be determined. Never raises.
|
|
477
|
+
|
|
478
|
+
Uses the **simple index** (PEP 691), NOT ``/pypi/<project>/json``. Learned the
|
|
479
|
+
hard way on 2026-08-04: minutes after 0.2.7 published, the aggregate JSON
|
|
480
|
+
endpoint was still serving a cached ``0.2.5`` while the simple index was already
|
|
481
|
+
correct. A doctor that reads the stale endpoint would tell a user they are
|
|
482
|
+
up to date when they are not — the exact failure this check exists to prevent.
|
|
483
|
+
"""
|
|
484
|
+
import urllib.request
|
|
485
|
+
|
|
486
|
+
url = f"https://pypi.org/simple/{project}/"
|
|
487
|
+
req = urllib.request.Request(
|
|
488
|
+
url,
|
|
489
|
+
headers={
|
|
490
|
+
"Accept": "application/vnd.pypi.simple.v1+json",
|
|
491
|
+
"User-Agent": f"mi-mcp/{__version__}",
|
|
492
|
+
},
|
|
493
|
+
)
|
|
494
|
+
try:
|
|
495
|
+
with urllib.request.urlopen(req, timeout=timeout) as resp: # noqa: S310 (fixed https URL)
|
|
496
|
+
versions = json.loads(resp.read().decode("utf-8")).get("versions") or []
|
|
497
|
+
except Exception:
|
|
498
|
+
return None # offline, DNS, timeout, 5xx, schema change — all "unknown"
|
|
499
|
+
real = [v for v in versions if _version_tuple(v) != (-1,)]
|
|
500
|
+
return max(real, key=_version_tuple) if real else None
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _upgrade_command() -> str:
|
|
504
|
+
"""The upgrade command for THIS install, not a generic guess.
|
|
505
|
+
|
|
506
|
+
`pip` is wrong for most installs of this package and is not even present on
|
|
507
|
+
some machines (Homebrew python is PEP-668 externally-managed), so guessing
|
|
508
|
+
costs the user a failed command and a detour. Detect from the interpreter's
|
|
509
|
+
location instead.
|
|
510
|
+
"""
|
|
511
|
+
prefix = str(Path(sys.prefix).resolve())
|
|
512
|
+
if f"{os.sep}uv{os.sep}tools{os.sep}" in prefix + os.sep:
|
|
513
|
+
return f"uv tool upgrade {PYPI_PROJECT}"
|
|
514
|
+
if f"{os.sep}pipx{os.sep}venvs{os.sep}" in prefix + os.sep or os.environ.get("PIPX_HOME"):
|
|
515
|
+
return f"pipx upgrade {PYPI_PROJECT}"
|
|
516
|
+
return f"{Path(sys.executable).name} -m pip install --upgrade {PYPI_PROJECT}"
|
|
517
|
+
|
|
518
|
+
|
|
455
519
|
def cmd_doctor(argv: list[str]) -> int:
|
|
456
520
|
ap = argparse.ArgumentParser(prog="mi-mcp doctor")
|
|
457
521
|
ap.add_argument("--home", default=os.environ.get("HOME"))
|
|
522
|
+
ap.add_argument("--no-version-check", action="store_true",
|
|
523
|
+
help="skip the PyPI release check (no network call)")
|
|
458
524
|
args = ap.parse_args(argv)
|
|
459
525
|
home = Path(args.home)
|
|
460
526
|
|
|
@@ -466,6 +532,31 @@ def cmd_doctor(argv: list[str]) -> int:
|
|
|
466
532
|
ok = False
|
|
467
533
|
print(f" [{'✓' if good else '✗'}] {label}" + (f" {detail}" if detail else ""))
|
|
468
534
|
|
|
535
|
+
# Release check FIRST — being two versions behind explains most "the tool
|
|
536
|
+
# doesn't have X" reports, and it is the one thing nothing else surfaces.
|
|
537
|
+
# Non-critical: out of date is not broken, so it never fails doctor's exit code.
|
|
538
|
+
# Opt out with --no-version-check or MI_MCP_NO_VERSION_CHECK=1; the call is an
|
|
539
|
+
# anonymous PyPI index fetch (same as any install) and is skipped entirely then.
|
|
540
|
+
skip_vc = args.no_version_check or os.environ.get(
|
|
541
|
+
NO_VERSION_CHECK_ENV, ""
|
|
542
|
+
).strip().lower() in ("1", "true", "yes", "on")
|
|
543
|
+
if skip_vc:
|
|
544
|
+
check("version", True, f"{__version__} (release check skipped)", critical=False)
|
|
545
|
+
else:
|
|
546
|
+
latest = _latest_pypi_version()
|
|
547
|
+
if latest is None:
|
|
548
|
+
# Unknown is NOT "up to date" — say so rather than imply green.
|
|
549
|
+
check("version", True,
|
|
550
|
+
f"{__version__} installed — could not reach PyPI, latest unknown",
|
|
551
|
+
critical=False)
|
|
552
|
+
elif _version_tuple(latest) > _version_tuple(__version__):
|
|
553
|
+
check("version", False,
|
|
554
|
+
f"{__version__} installed, {latest} available — "
|
|
555
|
+
f"`{_upgrade_command()} && mi-mcp wire`",
|
|
556
|
+
critical=False)
|
|
557
|
+
else:
|
|
558
|
+
check("version", True, f"{__version__} (latest)", critical=False)
|
|
559
|
+
|
|
469
560
|
bin_path = _mi_mcp_bin()
|
|
470
561
|
check("mi-mcp binary", Path(bin_path).exists(), bin_path)
|
|
471
562
|
|
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""Egress redaction for the LOCAL agent surface (#433 — the R11 gate on local reads).
|
|
2
|
+
|
|
3
|
+
Everything a local read hands the model — ``mi_ask``/``mi_list`` summaries AND
|
|
4
|
+
``topics`` — passes through :func:`scrub_text` before it leaves the process. The
|
|
5
|
+
cloud enforces this for ``X-MI-Source`` agent surfaces via
|
|
6
|
+
``core.security.export_scrub`` (the R11 read-time gate); the local path must match
|
|
7
|
+
or it is a privacy DOWNGRADE (review finding CR1, issue #433).
|
|
8
|
+
|
|
9
|
+
Three layers, strongest available first:
|
|
10
|
+
|
|
11
|
+
1. **Core detector (strong path).** When the ``memory-intelligence`` core is
|
|
12
|
+
importable (monorepo checkouts, school-tier installs that add the core), the
|
|
13
|
+
text runs through ``core.security.export_scrub.scrub_for_export`` — the SAME
|
|
14
|
+
machinery the cloud read endpoints use: the full hard-PII pattern set
|
|
15
|
+
(email / SSN / card+Luhn / phone / IP / IBAN / DOB / license / passport) plus
|
|
16
|
+
fresh corpus-NER detection of PERSON / ORG / LOCATION. Because detection runs
|
|
17
|
+
at READ time, it catches names the capture-time NER missed — the class of leak
|
|
18
|
+
the substring-only scrubber could not see. Non-PII labels (e.g. TECHNOLOGY)
|
|
19
|
+
are not in the detector's PII map and render through.
|
|
20
|
+
2. **Hard-PII regex floor.** The in-package email/SSN/card/phone patterns always
|
|
21
|
+
run — on a thin PyPI install (no core) they are the hard-PII guarantee, and on
|
|
22
|
+
the strong path they backstop ``scrub_for_export``'s fail-open detector.
|
|
23
|
+
3. **Stored-entity belt.** Entity names the engine extracted at capture (carried
|
|
24
|
+
in the index) are redacted deterministically — including per-token variants
|
|
25
|
+
and possessives ("Maria", "Gonzalez's" for a stored "Maria Gonzalez") — so a
|
|
26
|
+
name the read-time NER misses still cannot leak if capture caught it.
|
|
27
|
+
|
|
28
|
+
The agent surface is NON-OVERRIDABLE: there is no raw switch on this path
|
|
29
|
+
(#433 acceptance criteria). Owner-raw reads live on the human surfaces (desktop
|
|
30
|
+
app, ``mi-mcp memory open``), never here — per the #500 decision the owner reads
|
|
31
|
+
their own vault unredacted, but an LLM context never receives raw PII.
|
|
32
|
+
|
|
33
|
+
This is still the fail-safe floor, not the end state. The fuller, reversible form
|
|
34
|
+
— a pseudonym ledger that maps names↔tokens and re-identifies for the human
|
|
35
|
+
surface — is Phase 2 (#433 follow-on). Until then, fail CLOSED: any error
|
|
36
|
+
redacts the whole field.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import logging
|
|
42
|
+
import re
|
|
43
|
+
import unicodedata
|
|
44
|
+
from typing import Iterable, Optional
|
|
45
|
+
|
|
46
|
+
logger = logging.getLogger("mi_mcp.scrub")
|
|
47
|
+
|
|
48
|
+
# =============================================================================
|
|
49
|
+
# HARD-PII FLOOR (#1464)
|
|
50
|
+
# =============================================================================
|
|
51
|
+
#
|
|
52
|
+
# ⚠️ THIS IS THE ONLY PII PROTECTION ON A THIN PyPI INSTALL. `pyproject.toml`
|
|
53
|
+
# depends on mcp / httpx / pydantic — NOT on `memory-intelligence` — so
|
|
54
|
+
# `_scrub_with_core` returns None for every ordinary
|
|
55
|
+
# `pip install memoryintelligence-mcp` user and layer 1 never runs.
|
|
56
|
+
#
|
|
57
|
+
# It had drifted badly behind `core/security/pii_detector.py`: measured, 7 of 14
|
|
58
|
+
# hard-PII cases leaked straight to the model context. Core gained an ADDRESS
|
|
59
|
+
# detector (#1371), separator-agnostic SSN (#1446), a 7-digit phone form and
|
|
60
|
+
# unicode-obfuscation handling (#846/#847) — and none of it reached here, because
|
|
61
|
+
# nothing tied the two floors together.
|
|
62
|
+
#
|
|
63
|
+
# `tests/test_mcp_scrub_contract.py` is now that tie: one shared fixture set
|
|
64
|
+
# asserted against BOTH surfaces, living in `tests/` so it runs under `test-core`
|
|
65
|
+
# and therefore GATES merges. (`mcp-tests.yml` is standalone and advisory — a
|
|
66
|
+
# contract test living there would not have stopped this.)
|
|
67
|
+
#
|
|
68
|
+
# Patterns are literal copies, not imports: the entire point is that `core` is
|
|
69
|
+
# absent here. Keep them in step with the named constants in
|
|
70
|
+
# `core/security/pii_detector.py`; the contract test tells you when you haven't.
|
|
71
|
+
|
|
72
|
+
#: Zero-width / invisible splitters (#846) and cross-script confusables (#847).
|
|
73
|
+
#: Both are length-preserving so a match on the healed shadow maps back cleanly.
|
|
74
|
+
_INVISIBLE = {ord(c): None for c in ""}
|
|
75
|
+
_HOMOGLYPHS = str.maketrans({
|
|
76
|
+
"а": "a", "е": "e", "о": "o", "р": "p", "с": "c", "х": "x", "у": "y",
|
|
77
|
+
"і": "i", "ѕ": "s", "ԁ": "d", "ј": "j", "һ": "h",
|
|
78
|
+
"А": "A", "В": "B", "Е": "E", "К": "K", "М": "M", "Н": "H", "О": "O",
|
|
79
|
+
"Р": "P", "С": "C", "Т": "T", "Х": "X",
|
|
80
|
+
"ο": "o", "α": "a", "ρ": "p", "ε": "e", "ι": "i", "ν": "v", "τ": "t",
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
#: SSN — SEPARATOR-AGNOSTIC (#1446). The old pattern required a literal hyphen,
|
|
84
|
+
#: so "412 55 9832" and "412.55.9832" walked straight through.
|
|
85
|
+
_SSN_RE = re.compile(r"\b(?!000|666|9\d\d)\d{3}[\s.\-]+(?!00)\d{2}[\s.\-]+(?!0000)\d{4}\b")
|
|
86
|
+
|
|
87
|
+
#: Phone. `_PHONE_LONG` is the pre-existing 10–15 digit run. The short forms are
|
|
88
|
+
#: the #1446/A2 gap: every alternative needed THREE digit groups, so "555-0142"
|
|
89
|
+
#: and "+1-555-0142" leaked — including `pii-phone-001`, an EXEMPLAR fixture.
|
|
90
|
+
_PHONE_LONG_RE = re.compile(r"(?<![\d.])(?:\+?\d[\s().-]?){10,15}(?![\d.])")
|
|
91
|
+
_PHONE_INTL_SHORT_RE = re.compile(r"(?<![\d])\+\d{1,3}[\s.\-]?\d{3}[\s.\-]\d{4}(?![\d])")
|
|
92
|
+
#: A bare 7-digit run is NOT distinctive — core measured it hitting file:line
|
|
93
|
+
#: ranges, ULID fragments and coordinates — so it is CONTEXT-ANCHORED, as there.
|
|
94
|
+
_PHONE_BARE7_RE = re.compile(r"(?<![\d.\-:])[2-9]\d{2}[\s.\-]\d{4}(?![\d\-])")
|
|
95
|
+
_PHONE_CTX_RE = re.compile(
|
|
96
|
+
r"\b(?:call|called|calling|phone|telephone|tel|mobile|cell|fax|pager"
|
|
97
|
+
r"|hotline|extension|ext|dial|voicemail|landline|backup\s+line|number\s+is)\b",
|
|
98
|
+
re.IGNORECASE,
|
|
99
|
+
)
|
|
100
|
+
_PHONE_WINDOW = 40
|
|
101
|
+
|
|
102
|
+
#: Street addresses (#1371 + A3 + #1467 tiering). ADDRESS is a HARD type in core
|
|
103
|
+
#: — stripped on every surface — and this floor had NO address detector at all.
|
|
104
|
+
#: Ambiguous common-noun suffixes (Loop/Path/Way/...) are deliberately EXCLUDED
|
|
105
|
+
#: rather than corroborated: without core's context machinery the safe subset is
|
|
106
|
+
#: the unambiguous one, and over-redacting here would be a silent content loss.
|
|
107
|
+
_ST_UNAMBIG = (r"Street|St|Avenue|Ave|Boulevard|Blvd|Road|Rd|Lane|Ln|Drive|Dr"
|
|
108
|
+
r"|Court|Ct|Terrace|Ter|Parkway|Pkwy|Highway|Hwy|Crescent|Cres|Alley|Pike")
|
|
109
|
+
_ST_INTL = (r"Rue|Boulevard|Chemin|Via|Viale|Piazza|Corso|Calle|Avenida"
|
|
110
|
+
r"|Paseo|Carrera|Rua|Ulica|Aleja|Laan|Plein")
|
|
111
|
+
_ST_GLUED = (r"stra(?:ss|ß)e|gasse|platz|weg|allee|straat|gracht"
|
|
112
|
+
r"|gata|gatan|gade|vei(?:en)?|katu|tie")
|
|
113
|
+
_ADDRESS_RE = re.compile(
|
|
114
|
+
r"\b\d{1,6}[A-Za-z]?\s+(?:[A-Z][A-Za-z0-9.'\-]*\s+){1,4}(?:" + _ST_UNAMBIG + r")\b\.?"
|
|
115
|
+
r"|\b\d{1,5}[a-zA-Z]?,?\s+(?:" + _ST_INTL + r")\b"
|
|
116
|
+
r"(?:\s+(?:[A-ZÀ-Þ][\wÀ-ÿ'’\-]*|de|del|della|di|la|le|les|des|du|van|von)){1,4}"
|
|
117
|
+
r"|\b[A-ZÀ-Þ][\wÀ-ÿ'’\-]*(?:" + _ST_GLUED + r")\.?\s+\d{1,5}[a-zA-Z]?\b"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# Order matters: SSN / card / address before the looser phone matcher so digit
|
|
121
|
+
# runs are not mis-tagged.
|
|
122
|
+
_HARD = [
|
|
123
|
+
# `\s*@\s*` tolerates the whitespace-split form (#846).
|
|
124
|
+
(re.compile(r"[A-Za-z0-9._%+-]+\s*@\s*[A-Za-z0-9.-]+\.[A-Za-z]{2,}"), "<EMAIL>"),
|
|
125
|
+
(_SSN_RE, "<SSN>"),
|
|
126
|
+
(re.compile(r"\b(?:\d[ -]?){13,19}\b"), "<CARD>"),
|
|
127
|
+
(_ADDRESS_RE, "<ADDRESS>"),
|
|
128
|
+
(_PHONE_INTL_SHORT_RE, "<PHONE>"),
|
|
129
|
+
(_PHONE_LONG_RE, "<PHONE>"),
|
|
130
|
+
]
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _healed(text: str) -> str:
|
|
134
|
+
"""NFKC + zero-width strip + homoglyph fold — the #846/#847 shadow.
|
|
135
|
+
|
|
136
|
+
Length-preserving per character, so running the same patterns over the healed
|
|
137
|
+
copy and substituting on it cannot corrupt offsets.
|
|
138
|
+
"""
|
|
139
|
+
out = unicodedata.normalize("NFKC", text).translate(_INVISIBLE)
|
|
140
|
+
return out.translate(_HOMOGLYPHS)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _redact_anchored_phone(text: str) -> str:
|
|
144
|
+
"""7-digit local numbers, only near a telephony keyword (#1446/A2)."""
|
|
145
|
+
anchors = [(m.start(), m.end()) for m in _PHONE_CTX_RE.finditer(text)]
|
|
146
|
+
if not anchors:
|
|
147
|
+
return text
|
|
148
|
+
out, shift = text, 0
|
|
149
|
+
for m in _PHONE_BARE7_RE.finditer(text):
|
|
150
|
+
s, e = m.start(), m.end()
|
|
151
|
+
if not any(a_s <= e + _PHONE_WINDOW and s <= a_e + _PHONE_WINDOW
|
|
152
|
+
for a_s, a_e in anchors):
|
|
153
|
+
continue
|
|
154
|
+
out = out[: s + shift] + "<PHONE>" + out[e + shift:]
|
|
155
|
+
shift += len("<PHONE>") - (e - s)
|
|
156
|
+
return out
|
|
157
|
+
|
|
158
|
+
# Sticky flag: once the core import fails we don't retry it on every call.
|
|
159
|
+
_CORE_UNAVAILABLE = False
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _scrub_with_core(text: str) -> Optional[str]:
|
|
163
|
+
"""Run the cloud's R11 egress detector when the core package is installed.
|
|
164
|
+
|
|
165
|
+
Returns ``None`` when the core isn't importable (thin PyPI install) so the
|
|
166
|
+
caller falls back to the in-package floor. ``skip_types=None`` redacts soft
|
|
167
|
+
PII (PERSON / ORG / LOCATION) too — stricter than the cloud agent gate's
|
|
168
|
+
``skip_types=SOFT_PII_TYPES``, which is deliberate: the local surface has no
|
|
169
|
+
Phase-2 pseudonym ledger yet, so blanket-redacting detected names is the
|
|
170
|
+
conservative side of #433.
|
|
171
|
+
"""
|
|
172
|
+
global _CORE_UNAVAILABLE
|
|
173
|
+
if _CORE_UNAVAILABLE:
|
|
174
|
+
return None
|
|
175
|
+
try:
|
|
176
|
+
from core.security.export_scrub import scrub_for_export
|
|
177
|
+
except Exception:
|
|
178
|
+
_CORE_UNAVAILABLE = True
|
|
179
|
+
logger.info(
|
|
180
|
+
"core export_scrub unavailable — local egress uses the in-package floor"
|
|
181
|
+
)
|
|
182
|
+
return None
|
|
183
|
+
return scrub_for_export(text, mode="redact", skip_types=None).scrubbed
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _redact_entity_names(text: str, entities: Iterable) -> str:
|
|
187
|
+
"""Redact every stored entity name — full form, per-token, and possessive.
|
|
188
|
+
|
|
189
|
+
"Maria Gonzalez" in the index redacts "Maria Gonzalez", "Maria", "Gonzalez",
|
|
190
|
+
and "Gonzalez's" (the variant leak from the #506 hold assessment). Tokens
|
|
191
|
+
shorter than 3 chars are skipped ("Al", initials) — they over-match common
|
|
192
|
+
words more than they protect.
|
|
193
|
+
"""
|
|
194
|
+
for name in entities or ():
|
|
195
|
+
n = str(name).strip()
|
|
196
|
+
if len(n) < 3:
|
|
197
|
+
continue
|
|
198
|
+
variants = {n}
|
|
199
|
+
variants.update(p for p in re.split(r"\s+", n) if len(p) >= 3)
|
|
200
|
+
for v in sorted(variants, key=len, reverse=True):
|
|
201
|
+
text = re.sub(
|
|
202
|
+
r"(?<!\w)" + re.escape(v) + r"(?:[’']s)?(?!\w)",
|
|
203
|
+
"<ENTITY>",
|
|
204
|
+
text,
|
|
205
|
+
flags=re.IGNORECASE,
|
|
206
|
+
)
|
|
207
|
+
return text
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def scrub_text(text: Optional[str], entities: Iterable = ()) -> Optional[str]:
|
|
211
|
+
"""Redact PII from text destined for an agent surface (all three layers).
|
|
212
|
+
|
|
213
|
+
Returns the input unchanged when empty/None; fails CLOSED (``<REDACTED>``) on
|
|
214
|
+
any error so a scrubber bug can never leak.
|
|
215
|
+
"""
|
|
216
|
+
if not text:
|
|
217
|
+
return text
|
|
218
|
+
try:
|
|
219
|
+
out = str(text)
|
|
220
|
+
strong = _scrub_with_core(out)
|
|
221
|
+
if strong is not None:
|
|
222
|
+
out = strong
|
|
223
|
+
|
|
224
|
+
# #846/#847: run the floor over an obfuscation-HEALED copy first, so a
|
|
225
|
+
# zero-width-split or homoglyph-substituted value is caught. The healed
|
|
226
|
+
# copy is what we keep when it differs — the raw form only reaches here
|
|
227
|
+
# if it was never obfuscated, and returning healed text is strictly safer
|
|
228
|
+
# than returning text whose PII we could not see.
|
|
229
|
+
healed = _healed(out)
|
|
230
|
+
if healed != out:
|
|
231
|
+
for pattern, repl in _HARD:
|
|
232
|
+
healed = pattern.sub(repl, healed)
|
|
233
|
+
out = healed
|
|
234
|
+
|
|
235
|
+
for pattern, repl in _HARD:
|
|
236
|
+
out = pattern.sub(repl, out)
|
|
237
|
+
out = _redact_anchored_phone(out)
|
|
238
|
+
return _redact_entity_names(out, entities)
|
|
239
|
+
except Exception:
|
|
240
|
+
return "<REDACTED>"
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def scrub_topics(topics, entities: Iterable = ()) -> list:
|
|
244
|
+
"""Scrub a topics list for the agent surface (each label through scrub_text).
|
|
245
|
+
|
|
246
|
+
``topics`` was the ungated field in the #506 hold assessment — a name that
|
|
247
|
+
became a topic label at capture leaked raw through ``mi_list``.
|
|
248
|
+
"""
|
|
249
|
+
if not topics:
|
|
250
|
+
return []
|
|
251
|
+
return [scrub_text(str(t), entities) for t in topics]
|
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
"""Egress redaction for the LOCAL agent surface (#433 — the R11 gate on local reads).
|
|
2
|
-
|
|
3
|
-
Everything a local read hands the model — ``mi_ask``/``mi_list`` summaries AND
|
|
4
|
-
``topics`` — passes through :func:`scrub_text` before it leaves the process. The
|
|
5
|
-
cloud enforces this for ``X-MI-Source`` agent surfaces via
|
|
6
|
-
``core.security.export_scrub`` (the R11 read-time gate); the local path must match
|
|
7
|
-
or it is a privacy DOWNGRADE (review finding CR1, issue #433).
|
|
8
|
-
|
|
9
|
-
Three layers, strongest available first:
|
|
10
|
-
|
|
11
|
-
1. **Core detector (strong path).** When the ``memory-intelligence`` core is
|
|
12
|
-
importable (monorepo checkouts, school-tier installs that add the core), the
|
|
13
|
-
text runs through ``core.security.export_scrub.scrub_for_export`` — the SAME
|
|
14
|
-
machinery the cloud read endpoints use: the full hard-PII pattern set
|
|
15
|
-
(email / SSN / card+Luhn / phone / IP / IBAN / DOB / license / passport) plus
|
|
16
|
-
fresh corpus-NER detection of PERSON / ORG / LOCATION. Because detection runs
|
|
17
|
-
at READ time, it catches names the capture-time NER missed — the class of leak
|
|
18
|
-
the substring-only scrubber could not see. Non-PII labels (e.g. TECHNOLOGY)
|
|
19
|
-
are not in the detector's PII map and render through.
|
|
20
|
-
2. **Hard-PII regex floor.** The in-package email/SSN/card/phone patterns always
|
|
21
|
-
run — on a thin PyPI install (no core) they are the hard-PII guarantee, and on
|
|
22
|
-
the strong path they backstop ``scrub_for_export``'s fail-open detector.
|
|
23
|
-
3. **Stored-entity belt.** Entity names the engine extracted at capture (carried
|
|
24
|
-
in the index) are redacted deterministically — including per-token variants
|
|
25
|
-
and possessives ("Maria", "Gonzalez's" for a stored "Maria Gonzalez") — so a
|
|
26
|
-
name the read-time NER misses still cannot leak if capture caught it.
|
|
27
|
-
|
|
28
|
-
The agent surface is NON-OVERRIDABLE: there is no raw switch on this path
|
|
29
|
-
(#433 acceptance criteria). Owner-raw reads live on the human surfaces (desktop
|
|
30
|
-
app, ``mi-mcp memory open``), never here — per the #500 decision the owner reads
|
|
31
|
-
their own vault unredacted, but an LLM context never receives raw PII.
|
|
32
|
-
|
|
33
|
-
This is still the fail-safe floor, not the end state. The fuller, reversible form
|
|
34
|
-
— a pseudonym ledger that maps names↔tokens and re-identifies for the human
|
|
35
|
-
surface — is Phase 2 (#433 follow-on). Until then, fail CLOSED: any error
|
|
36
|
-
redacts the whole field.
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
from __future__ import annotations
|
|
40
|
-
|
|
41
|
-
import logging
|
|
42
|
-
import re
|
|
43
|
-
from typing import Iterable, Optional
|
|
44
|
-
|
|
45
|
-
logger = logging.getLogger("mi_mcp.scrub")
|
|
46
|
-
|
|
47
|
-
# Order matters: SSN/card before the looser phone matcher so digits aren't mis-tagged.
|
|
48
|
-
_HARD = [
|
|
49
|
-
(re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"), "<EMAIL>"),
|
|
50
|
-
(re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), "<SSN>"),
|
|
51
|
-
(re.compile(r"\b(?:\d[ -]?){13,19}\b"), "<CARD>"),
|
|
52
|
-
(re.compile(r"(?<![\d.])(?:\+?\d[\s().-]?){10,15}(?![\d.])"), "<PHONE>"),
|
|
53
|
-
]
|
|
54
|
-
|
|
55
|
-
# Sticky flag: once the core import fails we don't retry it on every call.
|
|
56
|
-
_CORE_UNAVAILABLE = False
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def _scrub_with_core(text: str) -> Optional[str]:
|
|
60
|
-
"""Run the cloud's R11 egress detector when the core package is installed.
|
|
61
|
-
|
|
62
|
-
Returns ``None`` when the core isn't importable (thin PyPI install) so the
|
|
63
|
-
caller falls back to the in-package floor. ``skip_types=None`` redacts soft
|
|
64
|
-
PII (PERSON / ORG / LOCATION) too — stricter than the cloud agent gate's
|
|
65
|
-
``skip_types=SOFT_PII_TYPES``, which is deliberate: the local surface has no
|
|
66
|
-
Phase-2 pseudonym ledger yet, so blanket-redacting detected names is the
|
|
67
|
-
conservative side of #433.
|
|
68
|
-
"""
|
|
69
|
-
global _CORE_UNAVAILABLE
|
|
70
|
-
if _CORE_UNAVAILABLE:
|
|
71
|
-
return None
|
|
72
|
-
try:
|
|
73
|
-
from core.security.export_scrub import scrub_for_export
|
|
74
|
-
except Exception:
|
|
75
|
-
_CORE_UNAVAILABLE = True
|
|
76
|
-
logger.info(
|
|
77
|
-
"core export_scrub unavailable — local egress uses the in-package floor"
|
|
78
|
-
)
|
|
79
|
-
return None
|
|
80
|
-
return scrub_for_export(text, mode="redact", skip_types=None).scrubbed
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def _redact_entity_names(text: str, entities: Iterable) -> str:
|
|
84
|
-
"""Redact every stored entity name — full form, per-token, and possessive.
|
|
85
|
-
|
|
86
|
-
"Maria Gonzalez" in the index redacts "Maria Gonzalez", "Maria", "Gonzalez",
|
|
87
|
-
and "Gonzalez's" (the variant leak from the #506 hold assessment). Tokens
|
|
88
|
-
shorter than 3 chars are skipped ("Al", initials) — they over-match common
|
|
89
|
-
words more than they protect.
|
|
90
|
-
"""
|
|
91
|
-
for name in entities or ():
|
|
92
|
-
n = str(name).strip()
|
|
93
|
-
if len(n) < 3:
|
|
94
|
-
continue
|
|
95
|
-
variants = {n}
|
|
96
|
-
variants.update(p for p in re.split(r"\s+", n) if len(p) >= 3)
|
|
97
|
-
for v in sorted(variants, key=len, reverse=True):
|
|
98
|
-
text = re.sub(
|
|
99
|
-
r"(?<!\w)" + re.escape(v) + r"(?:[’']s)?(?!\w)",
|
|
100
|
-
"<ENTITY>",
|
|
101
|
-
text,
|
|
102
|
-
flags=re.IGNORECASE,
|
|
103
|
-
)
|
|
104
|
-
return text
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def scrub_text(text: Optional[str], entities: Iterable = ()) -> Optional[str]:
|
|
108
|
-
"""Redact PII from text destined for an agent surface (all three layers).
|
|
109
|
-
|
|
110
|
-
Returns the input unchanged when empty/None; fails CLOSED (``<REDACTED>``) on
|
|
111
|
-
any error so a scrubber bug can never leak.
|
|
112
|
-
"""
|
|
113
|
-
if not text:
|
|
114
|
-
return text
|
|
115
|
-
try:
|
|
116
|
-
out = str(text)
|
|
117
|
-
strong = _scrub_with_core(out)
|
|
118
|
-
if strong is not None:
|
|
119
|
-
out = strong
|
|
120
|
-
for pattern, repl in _HARD:
|
|
121
|
-
out = pattern.sub(repl, out)
|
|
122
|
-
return _redact_entity_names(out, entities)
|
|
123
|
-
except Exception:
|
|
124
|
-
return "<REDACTED>"
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def scrub_topics(topics, entities: Iterable = ()) -> list:
|
|
128
|
-
"""Scrub a topics list for the agent surface (each label through scrub_text).
|
|
129
|
-
|
|
130
|
-
``topics`` was the ungated field in the #506 hold assessment — a name that
|
|
131
|
-
became a topic label at capture leaked raw through ``mi_list``.
|
|
132
|
-
"""
|
|
133
|
-
if not topics:
|
|
134
|
-
return []
|
|
135
|
-
return [scrub_text(str(t), entities) for t in topics]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|