memoryintelligence-mcp 0.2.8__tar.gz → 0.2.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of memoryintelligence-mcp might be problematic. Click here for more details.
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/CHANGELOG.md +36 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/PKG-INFO +2 -2
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/pyproject.toml +1 -1
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/__init__.py +1 -1
- memoryintelligence_mcp-0.2.10/src/mi_mcp/scrub.py +288 -0
- memoryintelligence_mcp-0.2.8/src/mi_mcp/scrub.py +0 -135
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/.gitignore +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/CONTRIBUTING.md +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/LICENSE +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/README.md +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/__main__.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/cli.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/client.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/config.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/embedder.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/indexer.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/keys.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/local_index.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/localreads.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/paths.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/recall_guard.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/server.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/umo_format.py +0 -0
- {memoryintelligence_mcp-0.2.8 → memoryintelligence_mcp-0.2.10}/src/mi_mcp/vault.py +0 -0
|
@@ -3,6 +3,42 @@
|
|
|
3
3
|
All notable changes to `memoryintelligence-mcp` are documented here.
|
|
4
4
|
The format follows [Keep a Changelog](https://keepachangelog.com/); this project uses [Semantic Versioning](https://semver.org/).
|
|
5
5
|
|
|
6
|
+
## [0.2.9] — 2026-08-09
|
|
7
|
+
|
|
8
|
+
### Fixed — the thin-install PII floor had drifted behind core (#1464)
|
|
9
|
+
|
|
10
|
+
`scrub.py` carries its own hard-PII regex floor. On a thin PyPI install it is the
|
|
11
|
+
**only** protection: this package depends on `mcp` / `httpx` / `pydantic` and NOT on
|
|
12
|
+
`memory-intelligence`, so the "strong path" through the core detector is absent for
|
|
13
|
+
every ordinary `pip install memoryintelligence-mcp` user.
|
|
14
|
+
|
|
15
|
+
That floor fell behind the core detector without anything noticing. **Measured: 7 of
|
|
16
|
+
14 hard-PII cases leaked** to the model context.
|
|
17
|
+
|
|
18
|
+
Brought to parity for everything a regex can do:
|
|
19
|
+
|
|
20
|
+
- **SSN** — separator-agnostic. Was hyphen-only, so `412 55 9832` and `412.55.9832`
|
|
21
|
+
walked straight through.
|
|
22
|
+
- **Phone** — added the `+CC` short form and a **context-anchored** bare 7-digit.
|
|
23
|
+
`555-0142` and `+1-555-0142` both leaked before, including an exemplar fixture.
|
|
24
|
+
The bare form is anchored to a telephony keyword rather than matched outright,
|
|
25
|
+
because a naked `NNN-NNNN` also matches file:line ranges and UUID fragments.
|
|
26
|
+
- **Address** — there was **no address detector at all**. Now covers English,
|
|
27
|
+
international and compound/glued street forms.
|
|
28
|
+
- **Obfuscation** — NFKC normalisation, zero-width strip and homoglyph folding, so a
|
|
29
|
+
zero-width-split or Cyrillic-lookalike value is caught.
|
|
30
|
+
- **Email** — tolerates the whitespace-split form.
|
|
31
|
+
|
|
32
|
+
Result: **15 of 16 cases redact**, with no over-redaction of file:line ranges,
|
|
33
|
+
numeric ranges, migration numbers or numbered headings.
|
|
34
|
+
|
|
35
|
+
### Known limitation — PERSON
|
|
36
|
+
|
|
37
|
+
`PERSON` / `ORG` / `LOCATION` need the corpus NER, which a thin install does not
|
|
38
|
+
have. Names are still covered *conditionally*, by the stored-entity belt, when
|
|
39
|
+
capture extracted them. This is asserted in the test suite as a known state rather
|
|
40
|
+
than left implicit.
|
|
41
|
+
|
|
6
42
|
## [0.2.8] — 2026-08-04
|
|
7
43
|
|
|
8
44
|
### Added — `mi-mcp doctor` tells you when you're behind
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: memoryintelligence-mcp
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.10
|
|
4
4
|
Summary: MCP server for MemoryIntelligence — give Claude a memory you own, set up in one command
|
|
5
5
|
Project-URL: Homepage, https://memoryintelligence.io
|
|
6
6
|
Project-URL: Repository, https://github.com/somewhere11/memoryintelligence-mcp
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "memoryintelligence-mcp"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.10"
|
|
8
8
|
description = "MCP server for MemoryIntelligence — give Claude a memory you own, set up in one command"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
"""Egress redaction for the LOCAL agent surface (#433 — the R11 gate on local reads).
|
|
2
|
+
|
|
3
|
+
Everything a local read hands the model — ``mi_ask``/``mi_list`` summaries AND
|
|
4
|
+
``topics`` — passes through :func:`scrub_text` before it leaves the process. The
|
|
5
|
+
cloud enforces this for ``X-MI-Source`` agent surfaces via
|
|
6
|
+
``core.security.export_scrub`` (the R11 read-time gate); the local path must match
|
|
7
|
+
or it is a privacy DOWNGRADE (review finding CR1, issue #433).
|
|
8
|
+
|
|
9
|
+
Three layers, strongest available first:
|
|
10
|
+
|
|
11
|
+
1. **Core detector (strong path).** When the ``memory-intelligence`` core is
|
|
12
|
+
importable (monorepo checkouts, school-tier installs that add the core), the
|
|
13
|
+
text runs through ``core.security.export_scrub.scrub_for_export`` — the SAME
|
|
14
|
+
machinery the cloud read endpoints use: the full hard-PII pattern set
|
|
15
|
+
(email / SSN / card+Luhn / phone / IP / IBAN / DOB / license / passport) plus
|
|
16
|
+
fresh corpus-NER detection of PERSON / ORG / LOCATION. Because detection runs
|
|
17
|
+
at READ time, it catches names the capture-time NER missed — the class of leak
|
|
18
|
+
the substring-only scrubber could not see. Non-PII labels (e.g. TECHNOLOGY)
|
|
19
|
+
are not in the detector's PII map and render through.
|
|
20
|
+
2. **Hard-PII regex floor.** The in-package email/SSN/card/phone patterns always
|
|
21
|
+
run — on a thin PyPI install (no core) they are the hard-PII guarantee, and on
|
|
22
|
+
the strong path they backstop ``scrub_for_export``'s fail-open detector.
|
|
23
|
+
3. **Stored-entity belt.** Entity names the engine extracted at capture (carried
|
|
24
|
+
in the index) are redacted deterministically — including per-token variants
|
|
25
|
+
and possessives ("Maria", "Gonzalez's" for a stored "Maria Gonzalez") — so a
|
|
26
|
+
name the read-time NER misses still cannot leak if capture caught it.
|
|
27
|
+
|
|
28
|
+
The agent surface is NON-OVERRIDABLE: there is no raw switch on this path
|
|
29
|
+
(#433 acceptance criteria). Owner-raw reads live on the human surfaces (desktop
|
|
30
|
+
app, ``mi-mcp memory open``), never here — per the #500 decision the owner reads
|
|
31
|
+
their own vault unredacted, but an LLM context never receives raw PII.
|
|
32
|
+
|
|
33
|
+
This is still the fail-safe floor, not the end state. The fuller, reversible form
|
|
34
|
+
— a pseudonym ledger that maps names↔tokens and re-identifies for the human
|
|
35
|
+
surface — is Phase 2 (#433 follow-on). Until then, fail CLOSED: any error
|
|
36
|
+
redacts the whole field.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
from __future__ import annotations
|
|
40
|
+
|
|
41
|
+
import logging
|
|
42
|
+
import re
|
|
43
|
+
import unicodedata
|
|
44
|
+
from typing import Iterable, Optional
|
|
45
|
+
|
|
46
|
+
logger = logging.getLogger("mi_mcp.scrub")
|
|
47
|
+
|
|
48
|
+
# =============================================================================
|
|
49
|
+
# HARD-PII FLOOR (#1464)
|
|
50
|
+
# =============================================================================
|
|
51
|
+
#
|
|
52
|
+
# ⚠️ THIS IS THE ONLY PII PROTECTION ON A THIN PyPI INSTALL. `pyproject.toml`
|
|
53
|
+
# depends on mcp / httpx / pydantic — NOT on `memory-intelligence` — so
|
|
54
|
+
# `_scrub_with_core` returns None for every ordinary
|
|
55
|
+
# `pip install memoryintelligence-mcp` user and layer 1 never runs.
|
|
56
|
+
#
|
|
57
|
+
# It had drifted badly behind `core/security/pii_detector.py`: measured, 7 of 14
|
|
58
|
+
# hard-PII cases leaked straight to the model context. Core gained an ADDRESS
|
|
59
|
+
# detector (#1371), separator-agnostic SSN (#1446), a 7-digit phone form and
|
|
60
|
+
# unicode-obfuscation handling (#846/#847) — and none of it reached here, because
|
|
61
|
+
# nothing tied the two floors together.
|
|
62
|
+
#
|
|
63
|
+
# `tests/test_mcp_scrub_contract.py` is now that tie: one shared fixture set
|
|
64
|
+
# asserted against BOTH surfaces, living in `tests/` so it runs under `test-core`
|
|
65
|
+
# and therefore GATES merges. (`mcp-tests.yml` is standalone and advisory — a
|
|
66
|
+
# contract test living there would not have stopped this.)
|
|
67
|
+
#
|
|
68
|
+
# Patterns are literal copies, not imports: the entire point is that `core` is
|
|
69
|
+
# absent here. Keep them in step with the named constants in
|
|
70
|
+
# `core/security/pii_detector.py`; the contract test tells you when you haven't.
|
|
71
|
+
|
|
72
|
+
#: Zero-width / invisible splitters (#846) and cross-script confusables (#847).
|
|
73
|
+
#: Both are length-preserving so a match on the healed shadow maps back cleanly.
|
|
74
|
+
_INVISIBLE = {ord(c): None for c in ""}
|
|
75
|
+
_HOMOGLYPHS = str.maketrans({
|
|
76
|
+
"а": "a", "е": "e", "о": "o", "р": "p", "с": "c", "х": "x", "у": "y",
|
|
77
|
+
"і": "i", "ѕ": "s", "ԁ": "d", "ј": "j", "һ": "h",
|
|
78
|
+
"А": "A", "В": "B", "Е": "E", "К": "K", "М": "M", "Н": "H", "О": "O",
|
|
79
|
+
"Р": "P", "С": "C", "Т": "T", "Х": "X",
|
|
80
|
+
"ο": "o", "α": "a", "ρ": "p", "ε": "e", "ι": "i", "ν": "v", "τ": "t",
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
#: SSN — SEPARATOR-AGNOSTIC (#1446). The old pattern required a literal hyphen,
|
|
84
|
+
#: so "412 55 9832" and "412.55.9832" walked straight through.
|
|
85
|
+
_SSN_RE = re.compile(r"\b(?!000|666|9\d\d)\d{3}[\s.\-]+(?!00)\d{2}[\s.\-]+(?!0000)\d{4}\b")
|
|
86
|
+
|
|
87
|
+
#: Phone. `_PHONE_LONG` is the pre-existing 10–15 digit run. The short forms are
|
|
88
|
+
#: the #1446/A2 gap: every alternative needed THREE digit groups, so "555-0142"
|
|
89
|
+
#: and "+1-555-0142" leaked — including `pii-phone-001`, an EXEMPLAR fixture.
|
|
90
|
+
# ⚠️ #1586 — `_PHONE_LONG_RE` was `(?:\+?\d[\s().-]?){10,15}`, i.e. ANY 10–15
|
|
91
|
+
# digit run, with each digit allowed a trailing separator. Two defects fell out
|
|
92
|
+
# of that, and they compounded:
|
|
93
|
+
#
|
|
94
|
+
# 1. it matched UNIX EPOCH SECONDS — 10 digits, and the most common 10-digit
|
|
95
|
+
# run in MI's own corpus — so `the watermark is 1737654321 seconds` became
|
|
96
|
+
# `<PHONE>`;
|
|
97
|
+
# 2. the final iteration's `[\s().-]?` SWALLOWED THE FOLLOWING SEPARATOR, so
|
|
98
|
+
# the next word was glued to the marker: `the watermark is <PHONE>seconds`.
|
|
99
|
+
#
|
|
100
|
+
# This is not merely a thin-install concern. `scrub_text` applies `_HARD` on top
|
|
101
|
+
# of core's result unconditionally, so a floor looser than core *overwrites a
|
|
102
|
+
# correct answer with a wrong one* on every MCP read.
|
|
103
|
+
#
|
|
104
|
+
# Split into three precise patterns, mirroring `core/security/pii_detector.py`:
|
|
105
|
+
#
|
|
106
|
+
# SEPARATED — the separators themselves are the signal, so no digit-value
|
|
107
|
+
# constraint is needed. Ends on a DIGIT, which is defect 2's fix.
|
|
108
|
+
_PHONE_SEP_RE = re.compile(
|
|
109
|
+
r"(?<![\d.])\+?\d{1,4}[\s().\-]+\d(?:[\s().\-]?\d){6,12}(?![\d.])"
|
|
110
|
+
)
|
|
111
|
+
# E.164 — the leading `+` is the signal. Measured in core: 0 matches across
|
|
112
|
+
# 2,618 of MI's own files, so it needs no further constraint.
|
|
113
|
+
_PHONE_E164_RE = re.compile(r"(?<![\d+])\+[1-9]\d{9,14}(?![\d])")
|
|
114
|
+
# BARE 10-DIGIT — the dangerous one, and the direct fix for defect 1. Same two
|
|
115
|
+
# constraints core measured its way to: a NANP shape (area code and exchange
|
|
116
|
+
# both start 2-9), which excludes epoch seconds structurally because they begin
|
|
117
|
+
# with `1` until 2033; and an ALNUM-AWARE boundary, because the digits of a hex
|
|
118
|
+
# hash are still digits (`"svo_hash": "914eb6579098371c"` contains a 10-digit
|
|
119
|
+
# run). Core measured 360 → 48 → 1 matches as those are applied.
|
|
120
|
+
_PHONE_BARE10_RE = re.compile(
|
|
121
|
+
r"(?<![0-9A-Za-z._\-:+])[2-9]\d{2}[2-9]\d{6}(?![0-9A-Za-z\-])"
|
|
122
|
+
)
|
|
123
|
+
_PHONE_INTL_SHORT_RE = re.compile(r"(?<![\d])\+\d{1,3}[\s.\-]?\d{3}[\s.\-]\d{4}(?![\d])")
|
|
124
|
+
#: A bare 7-digit run is NOT distinctive — core measured it hitting file:line
|
|
125
|
+
#: ranges, ULID fragments and coordinates — so it is CONTEXT-ANCHORED, as there.
|
|
126
|
+
_PHONE_BARE7_RE = re.compile(r"(?<![\d.\-:])[2-9]\d{2}[\s.\-]\d{4}(?![\d\-])")
|
|
127
|
+
_PHONE_CTX_RE = re.compile(
|
|
128
|
+
r"\b(?:call|called|calling|phone|telephone|tel|mobile|cell|fax|pager"
|
|
129
|
+
r"|hotline|extension|ext|dial|voicemail|landline|backup\s+line|number\s+is)\b",
|
|
130
|
+
re.IGNORECASE,
|
|
131
|
+
)
|
|
132
|
+
_PHONE_WINDOW = 40
|
|
133
|
+
|
|
134
|
+
#: Street addresses (#1371 + A3 + #1467 tiering). ADDRESS is a HARD type in core
|
|
135
|
+
#: — stripped on every surface — and this floor had NO address detector at all.
|
|
136
|
+
#: Ambiguous common-noun suffixes (Loop/Path/Way/...) are deliberately EXCLUDED
|
|
137
|
+
#: rather than corroborated: without core's context machinery the safe subset is
|
|
138
|
+
#: the unambiguous one, and over-redacting here would be a silent content loss.
|
|
139
|
+
_ST_UNAMBIG = (r"Street|St|Avenue|Ave|Boulevard|Blvd|Road|Rd|Lane|Ln|Drive|Dr"
|
|
140
|
+
r"|Court|Ct|Terrace|Ter|Parkway|Pkwy|Highway|Hwy|Crescent|Cres|Alley|Pike")
|
|
141
|
+
_ST_INTL = (r"Rue|Boulevard|Chemin|Via|Viale|Piazza|Corso|Calle|Avenida"
|
|
142
|
+
r"|Paseo|Carrera|Rua|Ulica|Aleja|Laan|Plein")
|
|
143
|
+
_ST_GLUED = (r"stra(?:ss|ß)e|gasse|platz|weg|allee|straat|gracht"
|
|
144
|
+
r"|gata|gatan|gade|vei(?:en)?|katu|tie")
|
|
145
|
+
_ADDRESS_RE = re.compile(
|
|
146
|
+
r"\b\d{1,6}[A-Za-z]?\s+(?:[A-Z][A-Za-z0-9.'\-]*\s+){1,4}(?:" + _ST_UNAMBIG + r")\b\.?"
|
|
147
|
+
r"|\b\d{1,5}[a-zA-Z]?,?\s+(?:" + _ST_INTL + r")\b"
|
|
148
|
+
r"(?:\s+(?:[A-ZÀ-Þ][\wÀ-ÿ'’\-]*|de|del|della|di|la|le|les|des|du|van|von)){1,4}"
|
|
149
|
+
r"|\b[A-ZÀ-Þ][\wÀ-ÿ'’\-]*(?:" + _ST_GLUED + r")\.?\s+\d{1,5}[a-zA-Z]?\b"
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
# Order matters: SSN / card / address before the looser phone matcher so digit
|
|
153
|
+
# runs are not mis-tagged.
|
|
154
|
+
_HARD = [
|
|
155
|
+
# `\s*@\s*` tolerates the whitespace-split form (#846).
|
|
156
|
+
(re.compile(r"[A-Za-z0-9._%+-]+\s*@\s*[A-Za-z0-9.-]+\.[A-Za-z]{2,}"), "<EMAIL>"),
|
|
157
|
+
(_SSN_RE, "<SSN>"),
|
|
158
|
+
(re.compile(r"\b(?:\d[ -]?){13,19}\b"), "<CARD>"),
|
|
159
|
+
(_ADDRESS_RE, "<ADDRESS>"),
|
|
160
|
+
(_PHONE_INTL_SHORT_RE, "<PHONE>"),
|
|
161
|
+
# #1586 — order matters within the phone group: E.164 and the separated form
|
|
162
|
+
# first (both carry their own signal), bare-10 last so it only ever sees runs
|
|
163
|
+
# the more specific patterns did not already claim.
|
|
164
|
+
(_PHONE_E164_RE, "<PHONE>"),
|
|
165
|
+
(_PHONE_SEP_RE, "<PHONE>"),
|
|
166
|
+
(_PHONE_BARE10_RE, "<PHONE>"),
|
|
167
|
+
]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _healed(text: str) -> str:
|
|
171
|
+
"""NFKC + zero-width strip + homoglyph fold — the #846/#847 shadow.
|
|
172
|
+
|
|
173
|
+
Length-preserving per character, so running the same patterns over the healed
|
|
174
|
+
copy and substituting on it cannot corrupt offsets.
|
|
175
|
+
"""
|
|
176
|
+
out = unicodedata.normalize("NFKC", text).translate(_INVISIBLE)
|
|
177
|
+
return out.translate(_HOMOGLYPHS)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _redact_anchored_phone(text: str) -> str:
|
|
181
|
+
"""7-digit local numbers, only near a telephony keyword (#1446/A2)."""
|
|
182
|
+
anchors = [(m.start(), m.end()) for m in _PHONE_CTX_RE.finditer(text)]
|
|
183
|
+
if not anchors:
|
|
184
|
+
return text
|
|
185
|
+
out, shift = text, 0
|
|
186
|
+
for m in _PHONE_BARE7_RE.finditer(text):
|
|
187
|
+
s, e = m.start(), m.end()
|
|
188
|
+
if not any(a_s <= e + _PHONE_WINDOW and s <= a_e + _PHONE_WINDOW
|
|
189
|
+
for a_s, a_e in anchors):
|
|
190
|
+
continue
|
|
191
|
+
out = out[: s + shift] + "<PHONE>" + out[e + shift:]
|
|
192
|
+
shift += len("<PHONE>") - (e - s)
|
|
193
|
+
return out
|
|
194
|
+
|
|
195
|
+
# Sticky flag: once the core import fails we don't retry it on every call.
|
|
196
|
+
_CORE_UNAVAILABLE = False
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _scrub_with_core(text: str) -> Optional[str]:
|
|
200
|
+
"""Run the cloud's R11 egress detector when the core package is installed.
|
|
201
|
+
|
|
202
|
+
Returns ``None`` when the core isn't importable (thin PyPI install) so the
|
|
203
|
+
caller falls back to the in-package floor. ``skip_types=None`` redacts soft
|
|
204
|
+
PII (PERSON / ORG / LOCATION) too — stricter than the cloud agent gate's
|
|
205
|
+
``skip_types=SOFT_PII_TYPES``, which is deliberate: the local surface has no
|
|
206
|
+
Phase-2 pseudonym ledger yet, so blanket-redacting detected names is the
|
|
207
|
+
conservative side of #433.
|
|
208
|
+
"""
|
|
209
|
+
global _CORE_UNAVAILABLE
|
|
210
|
+
if _CORE_UNAVAILABLE:
|
|
211
|
+
return None
|
|
212
|
+
try:
|
|
213
|
+
from core.security.export_scrub import scrub_for_export
|
|
214
|
+
except Exception:
|
|
215
|
+
_CORE_UNAVAILABLE = True
|
|
216
|
+
logger.info(
|
|
217
|
+
"core export_scrub unavailable — local egress uses the in-package floor"
|
|
218
|
+
)
|
|
219
|
+
return None
|
|
220
|
+
return scrub_for_export(text, mode="redact", skip_types=None).scrubbed
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _redact_entity_names(text: str, entities: Iterable) -> str:
|
|
224
|
+
"""Redact every stored entity name — full form, per-token, and possessive.
|
|
225
|
+
|
|
226
|
+
"Maria Gonzalez" in the index redacts "Maria Gonzalez", "Maria", "Gonzalez",
|
|
227
|
+
and "Gonzalez's" (the variant leak from the #506 hold assessment). Tokens
|
|
228
|
+
shorter than 3 chars are skipped ("Al", initials) — they over-match common
|
|
229
|
+
words more than they protect.
|
|
230
|
+
"""
|
|
231
|
+
for name in entities or ():
|
|
232
|
+
n = str(name).strip()
|
|
233
|
+
if len(n) < 3:
|
|
234
|
+
continue
|
|
235
|
+
variants = {n}
|
|
236
|
+
variants.update(p for p in re.split(r"\s+", n) if len(p) >= 3)
|
|
237
|
+
for v in sorted(variants, key=len, reverse=True):
|
|
238
|
+
text = re.sub(
|
|
239
|
+
r"(?<!\w)" + re.escape(v) + r"(?:[’']s)?(?!\w)",
|
|
240
|
+
"<ENTITY>",
|
|
241
|
+
text,
|
|
242
|
+
flags=re.IGNORECASE,
|
|
243
|
+
)
|
|
244
|
+
return text
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def scrub_text(text: Optional[str], entities: Iterable = ()) -> Optional[str]:
|
|
248
|
+
"""Redact PII from text destined for an agent surface (all three layers).
|
|
249
|
+
|
|
250
|
+
Returns the input unchanged when empty/None; fails CLOSED (``<REDACTED>``) on
|
|
251
|
+
any error so a scrubber bug can never leak.
|
|
252
|
+
"""
|
|
253
|
+
if not text:
|
|
254
|
+
return text
|
|
255
|
+
try:
|
|
256
|
+
out = str(text)
|
|
257
|
+
strong = _scrub_with_core(out)
|
|
258
|
+
if strong is not None:
|
|
259
|
+
out = strong
|
|
260
|
+
|
|
261
|
+
# #846/#847: run the floor over an obfuscation-HEALED copy first, so a
|
|
262
|
+
# zero-width-split or homoglyph-substituted value is caught. The healed
|
|
263
|
+
# copy is what we keep when it differs — the raw form only reaches here
|
|
264
|
+
# if it was never obfuscated, and returning healed text is strictly safer
|
|
265
|
+
# than returning text whose PII we could not see.
|
|
266
|
+
healed = _healed(out)
|
|
267
|
+
if healed != out:
|
|
268
|
+
for pattern, repl in _HARD:
|
|
269
|
+
healed = pattern.sub(repl, healed)
|
|
270
|
+
out = healed
|
|
271
|
+
|
|
272
|
+
for pattern, repl in _HARD:
|
|
273
|
+
out = pattern.sub(repl, out)
|
|
274
|
+
out = _redact_anchored_phone(out)
|
|
275
|
+
return _redact_entity_names(out, entities)
|
|
276
|
+
except Exception:
|
|
277
|
+
return "<REDACTED>"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def scrub_topics(topics, entities: Iterable = ()) -> list:
|
|
281
|
+
"""Scrub a topics list for the agent surface (each label through scrub_text).
|
|
282
|
+
|
|
283
|
+
``topics`` was the ungated field in the #506 hold assessment — a name that
|
|
284
|
+
became a topic label at capture leaked raw through ``mi_list``.
|
|
285
|
+
"""
|
|
286
|
+
if not topics:
|
|
287
|
+
return []
|
|
288
|
+
return [scrub_text(str(t), entities) for t in topics]
|
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
"""Egress redaction for the LOCAL agent surface (#433 — the R11 gate on local reads).
|
|
2
|
-
|
|
3
|
-
Everything a local read hands the model — ``mi_ask``/``mi_list`` summaries AND
|
|
4
|
-
``topics`` — passes through :func:`scrub_text` before it leaves the process. The
|
|
5
|
-
cloud enforces this for ``X-MI-Source`` agent surfaces via
|
|
6
|
-
``core.security.export_scrub`` (the R11 read-time gate); the local path must match
|
|
7
|
-
or it is a privacy DOWNGRADE (review finding CR1, issue #433).
|
|
8
|
-
|
|
9
|
-
Three layers, strongest available first:
|
|
10
|
-
|
|
11
|
-
1. **Core detector (strong path).** When the ``memory-intelligence`` core is
|
|
12
|
-
importable (monorepo checkouts, school-tier installs that add the core), the
|
|
13
|
-
text runs through ``core.security.export_scrub.scrub_for_export`` — the SAME
|
|
14
|
-
machinery the cloud read endpoints use: the full hard-PII pattern set
|
|
15
|
-
(email / SSN / card+Luhn / phone / IP / IBAN / DOB / license / passport) plus
|
|
16
|
-
fresh corpus-NER detection of PERSON / ORG / LOCATION. Because detection runs
|
|
17
|
-
at READ time, it catches names the capture-time NER missed — the class of leak
|
|
18
|
-
the substring-only scrubber could not see. Non-PII labels (e.g. TECHNOLOGY)
|
|
19
|
-
are not in the detector's PII map and render through.
|
|
20
|
-
2. **Hard-PII regex floor.** The in-package email/SSN/card/phone patterns always
|
|
21
|
-
run — on a thin PyPI install (no core) they are the hard-PII guarantee, and on
|
|
22
|
-
the strong path they backstop ``scrub_for_export``'s fail-open detector.
|
|
23
|
-
3. **Stored-entity belt.** Entity names the engine extracted at capture (carried
|
|
24
|
-
in the index) are redacted deterministically — including per-token variants
|
|
25
|
-
and possessives ("Maria", "Gonzalez's" for a stored "Maria Gonzalez") — so a
|
|
26
|
-
name the read-time NER misses still cannot leak if capture caught it.
|
|
27
|
-
|
|
28
|
-
The agent surface is NON-OVERRIDABLE: there is no raw switch on this path
|
|
29
|
-
(#433 acceptance criteria). Owner-raw reads live on the human surfaces (desktop
|
|
30
|
-
app, ``mi-mcp memory open``), never here — per the #500 decision the owner reads
|
|
31
|
-
their own vault unredacted, but an LLM context never receives raw PII.
|
|
32
|
-
|
|
33
|
-
This is still the fail-safe floor, not the end state. The fuller, reversible form
|
|
34
|
-
— a pseudonym ledger that maps names↔tokens and re-identifies for the human
|
|
35
|
-
surface — is Phase 2 (#433 follow-on). Until then, fail CLOSED: any error
|
|
36
|
-
redacts the whole field.
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
from __future__ import annotations
|
|
40
|
-
|
|
41
|
-
import logging
|
|
42
|
-
import re
|
|
43
|
-
from typing import Iterable, Optional
|
|
44
|
-
|
|
45
|
-
logger = logging.getLogger("mi_mcp.scrub")
|
|
46
|
-
|
|
47
|
-
# Order matters: SSN/card before the looser phone matcher so digits aren't mis-tagged.
|
|
48
|
-
_HARD = [
|
|
49
|
-
(re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"), "<EMAIL>"),
|
|
50
|
-
(re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), "<SSN>"),
|
|
51
|
-
(re.compile(r"\b(?:\d[ -]?){13,19}\b"), "<CARD>"),
|
|
52
|
-
(re.compile(r"(?<![\d.])(?:\+?\d[\s().-]?){10,15}(?![\d.])"), "<PHONE>"),
|
|
53
|
-
]
|
|
54
|
-
|
|
55
|
-
# Sticky flag: once the core import fails we don't retry it on every call.
|
|
56
|
-
_CORE_UNAVAILABLE = False
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def _scrub_with_core(text: str) -> Optional[str]:
|
|
60
|
-
"""Run the cloud's R11 egress detector when the core package is installed.
|
|
61
|
-
|
|
62
|
-
Returns ``None`` when the core isn't importable (thin PyPI install) so the
|
|
63
|
-
caller falls back to the in-package floor. ``skip_types=None`` redacts soft
|
|
64
|
-
PII (PERSON / ORG / LOCATION) too — stricter than the cloud agent gate's
|
|
65
|
-
``skip_types=SOFT_PII_TYPES``, which is deliberate: the local surface has no
|
|
66
|
-
Phase-2 pseudonym ledger yet, so blanket-redacting detected names is the
|
|
67
|
-
conservative side of #433.
|
|
68
|
-
"""
|
|
69
|
-
global _CORE_UNAVAILABLE
|
|
70
|
-
if _CORE_UNAVAILABLE:
|
|
71
|
-
return None
|
|
72
|
-
try:
|
|
73
|
-
from core.security.export_scrub import scrub_for_export
|
|
74
|
-
except Exception:
|
|
75
|
-
_CORE_UNAVAILABLE = True
|
|
76
|
-
logger.info(
|
|
77
|
-
"core export_scrub unavailable — local egress uses the in-package floor"
|
|
78
|
-
)
|
|
79
|
-
return None
|
|
80
|
-
return scrub_for_export(text, mode="redact", skip_types=None).scrubbed
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def _redact_entity_names(text: str, entities: Iterable) -> str:
|
|
84
|
-
"""Redact every stored entity name — full form, per-token, and possessive.
|
|
85
|
-
|
|
86
|
-
"Maria Gonzalez" in the index redacts "Maria Gonzalez", "Maria", "Gonzalez",
|
|
87
|
-
and "Gonzalez's" (the variant leak from the #506 hold assessment). Tokens
|
|
88
|
-
shorter than 3 chars are skipped ("Al", initials) — they over-match common
|
|
89
|
-
words more than they protect.
|
|
90
|
-
"""
|
|
91
|
-
for name in entities or ():
|
|
92
|
-
n = str(name).strip()
|
|
93
|
-
if len(n) < 3:
|
|
94
|
-
continue
|
|
95
|
-
variants = {n}
|
|
96
|
-
variants.update(p for p in re.split(r"\s+", n) if len(p) >= 3)
|
|
97
|
-
for v in sorted(variants, key=len, reverse=True):
|
|
98
|
-
text = re.sub(
|
|
99
|
-
r"(?<!\w)" + re.escape(v) + r"(?:[’']s)?(?!\w)",
|
|
100
|
-
"<ENTITY>",
|
|
101
|
-
text,
|
|
102
|
-
flags=re.IGNORECASE,
|
|
103
|
-
)
|
|
104
|
-
return text
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
def scrub_text(text: Optional[str], entities: Iterable = ()) -> Optional[str]:
|
|
108
|
-
"""Redact PII from text destined for an agent surface (all three layers).
|
|
109
|
-
|
|
110
|
-
Returns the input unchanged when empty/None; fails CLOSED (``<REDACTED>``) on
|
|
111
|
-
any error so a scrubber bug can never leak.
|
|
112
|
-
"""
|
|
113
|
-
if not text:
|
|
114
|
-
return text
|
|
115
|
-
try:
|
|
116
|
-
out = str(text)
|
|
117
|
-
strong = _scrub_with_core(out)
|
|
118
|
-
if strong is not None:
|
|
119
|
-
out = strong
|
|
120
|
-
for pattern, repl in _HARD:
|
|
121
|
-
out = pattern.sub(repl, out)
|
|
122
|
-
return _redact_entity_names(out, entities)
|
|
123
|
-
except Exception:
|
|
124
|
-
return "<REDACTED>"
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def scrub_topics(topics, entities: Iterable = ()) -> list:
|
|
128
|
-
"""Scrub a topics list for the agent surface (each label through scrub_text).
|
|
129
|
-
|
|
130
|
-
``topics`` was the ungated field in the #506 hold assessment — a name that
|
|
131
|
-
became a topic label at capture leaked raw through ``mi_list``.
|
|
132
|
-
"""
|
|
133
|
-
if not topics:
|
|
134
|
-
return []
|
|
135
|
-
return [scrub_text(str(t), entities) for t in topics]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|