phiproxy 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
phiproxy/__init__.py ADDED
@@ -0,0 +1,30 @@
1
+ """phiproxy: reversible PHI masking for LLM traffic.
2
+
3
+ Mask patient identifiers before a prompt leaves your network, restore them in the
4
+ reply. phiproxy reduces how much PHI reaches an LLM provider; it does not make an
5
+ unapproved provider compliant. See README for the full disclaimer.
6
+ """
7
+
8
+ from .detectors import KnownEntities, KnownEntityDetector, RegexDetector
9
+ from .entities import Entity, EntityType
10
+ from .masker import Masker, MaskResult, PhiproxyError, Policy, StreamUnmasker, UnsupportedContentError
11
+ from .vault import InMemoryVault, RedisVault, Vault
12
+
13
+ __version__ = "0.1.0a1"
14
+
15
+ __all__ = [
16
+ "Entity",
17
+ "EntityType",
18
+ "InMemoryVault",
19
+ "KnownEntities",
20
+ "KnownEntityDetector",
21
+ "MaskResult",
22
+ "Masker",
23
+ "PhiproxyError",
24
+ "Policy",
25
+ "RedisVault",
26
+ "RegexDetector",
27
+ "StreamUnmasker",
28
+ "UnsupportedContentError",
29
+ "Vault",
30
+ ]
phiproxy/dates.py ADDED
@@ -0,0 +1,128 @@
1
+ """Date shifting that keeps the original writing style.
2
+
3
+ Shifting every date in a session by the same random offset hides the real dates
4
+ but keeps intervals intact (cycle day 3 -> retrieval -> transfer 5 days later),
5
+ which clinical reasoning depends on. The reply is shifted back by the same offset.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+ from datetime import date, timedelta
12
+ from typing import Optional
13
+
14
+ _MONTHS_FULL = [
15
+ "january", "february", "march", "april", "may", "june", "july", "august",
16
+ "september", "october", "november", "december",
17
+ ]
18
+ _MONTH_LOOKUP = {}
19
+ for _i, _m in enumerate(_MONTHS_FULL, start=1):
20
+ _MONTH_LOOKUP[_m] = _i
21
+ _MONTH_LOOKUP[_m[:3]] = _i
22
+ _MONTH_LOOKUP["sept"] = 9
23
+
24
+ _NO_YEAR_REF = 2024 # leap year, so "Feb 29" without a year still parses
25
+
26
+ _ISO = re.compile(r"^(?P<y>\d{4})-(?P<m>\d{1,2})-(?P<d>\d{1,2})$")
27
+ _NUMERIC = re.compile(r"^(?P<a>\d{1,2})(?P<sep>[/\-.])(?P<b>\d{1,2})(?:(?P=sep)(?P<y>\d{4}|\d{2}))?$")
28
+ _MONTH_FIRST = re.compile(
29
+ r"^(?P<mon>[A-Za-z]+)(?P<dot>\.?)(?P<sp>\s+)(?P<d>\d{1,2})(?P<suf>st|nd|rd|th)?"
30
+ r"(?:(?P<comma>,?)(?P<sp2>\s+)(?P<y>\d{4}))?$",
31
+ re.IGNORECASE,
32
+ )
33
+ _DAY_FIRST = re.compile(
34
+ r"^(?P<d>\d{1,2})(?P<suf>st|nd|rd|th)?(?P<sp>\s+)(?P<of>of\s+)?(?P<mon>[A-Za-z]+)(?P<dot>\.?)"
35
+ r"(?:(?P<comma>,?)(?P<sp2>\s+)(?P<y>\d{4}))?$",
36
+ re.IGNORECASE,
37
+ )
38
+
39
+
40
+ def _suffix(day: int) -> str:
41
+ if 11 <= day % 100 <= 13:
42
+ return "th"
43
+ return {1: "st", 2: "nd", 3: "rd"}.get(day % 10, "th")
44
+
45
+
46
+ def _month_word(month: int, like: str) -> str:
47
+ full = _MONTHS_FULL[month - 1]
48
+ if len(like.rstrip(".")) <= 3 or (len(like) == 4 and like.lower() == "sept"):
49
+ word = full[:3]
50
+ else:
51
+ word = full
52
+ if like.isupper():
53
+ return word.upper()
54
+ if like[0].isupper():
55
+ return word.capitalize()
56
+ return word
57
+
58
+
59
+ def _pad(n: int, like: str, other: str = "") -> str:
60
+ """Zero-pad when the original date shows a padded style ("03/14" or "11/03")."""
61
+ padded = (len(like) == 2 and like.startswith("0")) or (len(other) == 2 and other.startswith("0"))
62
+ return f"{n:02d}" if padded else str(n)
63
+
64
+
65
+ def _year(y: int, like: str) -> str:
66
+ return f"{y % 100:02d}" if len(like) == 2 else str(y)
67
+
68
+
69
+ def _full_year(two_or_four: str) -> int:
70
+ y = int(two_or_four)
71
+ if len(two_or_four) == 2:
72
+ y += 2000 if y < 70 else 1900
73
+ return y
74
+
75
+
76
+ def shift_date_text(text: str, days: int, day_first: bool = False) -> Optional[str]:
77
+ """Shift a date written as text by ``days``, keeping its format.
78
+
79
+ Returns None if the text is not a date this module understands; callers
80
+ should then fall back to replacing it with a token.
81
+ """
82
+ s = text.strip()
83
+ try:
84
+ m = _ISO.match(s)
85
+ if m:
86
+ d = date(int(m["y"]), int(m["m"]), int(m["d"])) + timedelta(days=days)
87
+ return f"{d.year:04d}-{d.month:02d}-{d.day:02d}"
88
+
89
+ m = _NUMERIC.match(s)
90
+ if m:
91
+ a, b = int(m["a"]), int(m["b"])
92
+ month_first = not day_first
93
+ if a > 12 and b <= 12:
94
+ month_first = False
95
+ elif b > 12 and a <= 12:
96
+ month_first = True
97
+ mo, dy = (a, b) if month_first else (b, a)
98
+ mo_like, dy_like = (m["a"], m["b"]) if month_first else (m["b"], m["a"])
99
+ year = _full_year(m["y"]) if m["y"] else _NO_YEAR_REF
100
+ d = date(year, mo, dy) + timedelta(days=days)
101
+ mo_s, dy_s = _pad(d.month, mo_like, dy_like), _pad(d.day, dy_like, mo_like)
102
+ first, second = (mo_s, dy_s) if month_first else (dy_s, mo_s)
103
+ out = f"{first}{m['sep']}{second}"
104
+ if m["y"]:
105
+ out += f"{m['sep']}{_year(d.year, m['y'])}"
106
+ return out
107
+
108
+ for pat, month_first in ((_MONTH_FIRST, True), (_DAY_FIRST, False)):
109
+ m = pat.match(s)
110
+ if not m:
111
+ continue
112
+ month = _MONTH_LOOKUP.get(m["mon"].lower())
113
+ if month is None:
114
+ return None
115
+ year = int(m["y"]) if m["y"] else _NO_YEAR_REF
116
+ d = date(year, month, int(m["d"])) + timedelta(days=days)
117
+ mon = _month_word(d.month, m["mon"]) + m["dot"]
118
+ day = str(d.day) + (_suffix(d.day) if m["suf"] else "")
119
+ if month_first:
120
+ out = f"{mon}{m['sp']}{day}"
121
+ else:
122
+ out = f"{day}{m['sp']}{m['of'] or ''}{mon}"
123
+ if m["y"]:
124
+ out += f"{m['comma']}{m['sp2']}{d.year}"
125
+ return out
126
+ except ValueError: # e.g. 2/30
127
+ return None
128
+ return None
@@ -0,0 +1,5 @@
1
+ from .base import Detector, merge_entities
2
+ from .known import KnownEntities, KnownEntityDetector
3
+ from .regex import RegexDetector
4
+
5
+ __all__ = ["Detector", "merge_entities", "KnownEntities", "KnownEntityDetector", "RegexDetector"]
@@ -0,0 +1,73 @@
1
+ """Detector interface and span merging."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Iterable, Protocol, Sequence
6
+
7
+ from ..entities import Entity
8
+
9
+ # When two detections overlap, the one from the higher-priority source wins.
10
+ # Known-entity matches come from the clinic's own records, so they are the most
11
+ # trustworthy; NER guesses are the least.
12
+ SOURCE_PRIORITY = {"known": 3, "regex": 2, "presidio": 1}
13
+
14
+
15
+ class Detector(Protocol):
16
+ """Finds identifier spans in text. Must not call any external service."""
17
+
18
+ name: str
19
+
20
+ def detect(self, text: str) -> list[Entity]: ...
21
+
22
+
23
+ def merge_entities(entities: Iterable[Entity]) -> list[Entity]:
24
+ """Resolve overlapping spans into a non-overlapping, start-ordered list.
25
+
26
+ Overlaps are resolved by (span length, source priority, score): the longer
27
+ span wins (an email beats a first name found inside it); on equal length the
28
+ more trustworthy source wins. If a loser extends past the winner, the winner
29
+ is widened to the union so no character of the loser leaks.
30
+ """
31
+ ordered = sorted(
32
+ entities,
33
+ key=lambda e: (e.length, SOURCE_PRIORITY.get(e.source, 0), e.score),
34
+ reverse=True,
35
+ )
36
+ kept: list[Entity] = []
37
+ for cand in ordered:
38
+ clash = [k for k in kept if k.overlaps(cand)]
39
+ if not clash:
40
+ kept.append(cand)
41
+ continue
42
+ # Widen the strongest overlapping winner to the union so no character
43
+ # of the loser (or of other winners it bridges) leaks.
44
+ winner = clash[0] # kept is in priority order, so the first clash is strongest
45
+ start = min([cand.start] + [k.start for k in clash])
46
+ end = max([cand.end] + [k.end for k in clash])
47
+ if len(clash) == 1 and (start, end) == (winner.start, winner.end):
48
+ continue
49
+ for k in clash:
50
+ kept.remove(k)
51
+ kept.insert(
52
+ 0,
53
+ Entity(
54
+ type=winner.type,
55
+ start=start,
56
+ end=end,
57
+ text="?", # placeholder; fill_text() re-attaches real text
58
+ source=winner.source,
59
+ score=winner.score,
60
+ ),
61
+ )
62
+ return sorted(kept, key=lambda e: e.start)
63
+
64
+
65
+ def fill_text(text: str, entities: Sequence[Entity]) -> list[Entity]:
66
+ """Re-attach span text after merging (merge may widen spans)."""
67
+ out = []
68
+ for e in entities:
69
+ span = text[e.start : e.end]
70
+ # A widened span no longer matches its canonical form; fall back to the text.
71
+ canonical = e.canonical if span == e.text else None
72
+ out.append(Entity(e.type, e.start, e.end, span, e.source, e.score, canonical))
73
+ return out
@@ -0,0 +1,235 @@
1
+ """Known-entity detection: match the identifiers you already hold for this patient.
2
+
3
+ The application usually knows who the conversation is about (the patient's name,
4
+ date of birth, phone, health card number...). Matching those exact values is the
5
+ highest-recall, lowest-cost layer: it catches casual first names, lowercase
6
+ spellings and one-letter typos that generic NER misses.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from dataclasses import dataclass, field
13
+ from datetime import date
14
+ from typing import Iterable, Mapping, Optional, Sequence, Union
15
+
16
+ from ..entities import Entity, EntityType
17
+
18
+ # First names that are also common English words. A lowercase match of these
19
+ # ("I have hope") is not treated as the name; a capitalized one is.
20
+ COMMON_WORD_NAMES = {
21
+ "hope", "faith", "grace", "joy", "will", "may", "june", "april", "rose", "mark", "bill",
22
+ "art", "page", "sky", "summer", "autumn", "dawn", "eve", "ray", "rich", "max", "pat",
23
+ "sue", "hunter", "chase", "lane", "brook", "river", "star", "sunny", "honey", "ginger",
24
+ "iris", "lily", "daisy", "ivy", "ruby", "amber", "crystal", "angel", "major", "young",
25
+ "white", "black", "brown", "green", "long", "short", "little", "king", "price", "cash",
26
+ "love", "christian", "miles", "frank", "jean", "gay", "deb", "dick", "guy", "an", "bo",
27
+ }
28
+
29
+ _MONTH_NAMES = [
30
+ "January", "February", "March", "April", "May", "June", "July", "August",
31
+ "September", "October", "November", "December",
32
+ ]
33
+
34
+
35
+ def _damerau_leq1(a: str, b: str) -> bool:
36
+ """True if a and b are within one edit (insert, delete, substitute, adjacent swap)."""
37
+ if a == b:
38
+ return True
39
+ la, lb = len(a), len(b)
40
+ if abs(la - lb) > 1:
41
+ return False
42
+ if la == lb:
43
+ diff = [i for i in range(la) if a[i] != b[i]]
44
+ if len(diff) == 1:
45
+ return True
46
+ if len(diff) == 2 and diff[1] == diff[0] + 1:
47
+ i, j = diff
48
+ return a[i] == b[j] and a[j] == b[i]
49
+ return False
50
+ if la > lb:
51
+ a, b = b, a
52
+ # b is longer by one: try deleting each char of b
53
+ i = j = 0
54
+ skipped = False
55
+ while i < len(a) and j < len(b):
56
+ if a[i] == b[j]:
57
+ i += 1
58
+ j += 1
59
+ elif skipped:
60
+ return False
61
+ else:
62
+ skipped = True
63
+ j += 1
64
+ return True
65
+
66
+
67
+ def _date_variants(d: date) -> list[str]:
68
+ m = _MONTH_NAMES[d.month - 1]
69
+ mon = m[:3]
70
+ return [
71
+ d.isoformat(),
72
+ f"{d.month}/{d.day}/{d.year}",
73
+ f"{d.month:02d}/{d.day:02d}/{d.year}",
74
+ f"{d.day}/{d.month}/{d.year}",
75
+ f"{d.day:02d}/{d.month:02d}/{d.year}",
76
+ f"{d.month}/{d.day}/{d.year % 100:02d}",
77
+ f"{d.month:02d}/{d.day:02d}/{d.year % 100:02d}",
78
+ f"{d.month}-{d.day}-{d.year}",
79
+ f"{d.day}.{d.month}.{d.year}",
80
+ f"{m} {d.day}, {d.year}",
81
+ f"{m} {d.day} {d.year}",
82
+ f"{mon} {d.day}, {d.year}",
83
+ f"{mon} {d.day} {d.year}",
84
+ f"{mon}. {d.day}, {d.year}",
85
+ f"{d.day} {m} {d.year}",
86
+ f"{d.day} {mon} {d.year}",
87
+ f"{m} {d.day}",
88
+ f"{mon} {d.day}",
89
+ ]
90
+
91
+
92
+ DateLike = Union[date, str]
93
+
94
+
95
+ @dataclass
96
+ class KnownEntities:
97
+ """Identifiers the application already holds for the people in this conversation.
98
+
99
+ All fields are optional. Pass whatever you have; more is better.
100
+ """
101
+
102
+ names: Sequence[str] = () # full names of the patient, partner, donor, ...
103
+ phones: Sequence[str] = ()
104
+ emails: Sequence[str] = ()
105
+ dates_of_birth: Sequence[DateLike] = ()
106
+ identifiers: Sequence[str] = () # health card, MRN, chart #, donor ID, account #
107
+ addresses: Sequence[str] = ()
108
+ other: Mapping[str, Sequence[str]] = field(default_factory=dict) # EntityType name -> values
109
+
110
+ @classmethod
111
+ def from_dict(cls, data: Mapping[str, object]) -> "KnownEntities":
112
+ def seq(key: str) -> list:
113
+ v = data.get(key) or []
114
+ return [v] if isinstance(v, (str, date)) else list(v) # type: ignore[arg-type]
115
+
116
+ return cls(
117
+ names=seq("names"),
118
+ phones=seq("phones"),
119
+ emails=seq("emails"),
120
+ dates_of_birth=seq("dates_of_birth"),
121
+ identifiers=seq("identifiers"),
122
+ addresses=seq("addresses"),
123
+ other=dict(data.get("other") or {}), # type: ignore[arg-type]
124
+ )
125
+
126
+ def is_empty(self) -> bool:
127
+ return not any(
128
+ [self.names, self.phones, self.emails, self.dates_of_birth,
129
+ self.identifiers, self.addresses, self.other]
130
+ )
131
+
132
+
133
+ def _flex_pattern(value: str) -> str:
134
+ """Regex for a value where spaces, dashes and dots between characters are optional."""
135
+ chars = [re.escape(c) for c in re.sub(r"[\s\-.]", "", value)]
136
+ return r"[\s\-.]?".join(chars)
137
+
138
+
139
+ @dataclass
140
+ class KnownEntityDetector:
141
+ known: KnownEntities
142
+ name: str = "known"
143
+ fuzzy_names: bool = True # allow one typo against record names of 5+ letters
144
+
145
+ def detect(self, text: str) -> list[Entity]:
146
+ out: list[Entity] = []
147
+ k = self.known
148
+ out += self._names(text, k.names)
149
+ for p in k.phones:
150
+ digits = re.sub(r"\D", "", p)
151
+ if len(digits) >= 10:
152
+ digits = digits[-10:]
153
+ pat = (
154
+ r"(?<!\d)(?:\+?1[\s.\-]?)?\(?" + digits[:3] + r"\)?[\s.\-]?"
155
+ + digits[3:6] + r"[\s.\-]?" + digits[6:] + r"(?!\d)"
156
+ )
157
+ out += self._find(text, pat, EntityType.PHONE)
158
+ elif len(digits) >= 7:
159
+ out += self._find(text, r"(?<!\d)" + _flex_pattern(digits) + r"(?!\d)", EntityType.PHONE)
160
+ for e in k.emails:
161
+ out += self._find(text, re.escape(e), EntityType.EMAIL, re.IGNORECASE)
162
+ for d in k.dates_of_birth:
163
+ out += self._dates(text, d)
164
+ for ident in k.identifiers:
165
+ if len(re.sub(r"[\s\-.]", "", ident)) >= 4:
166
+ out += self._find(text, r"(?<![A-Za-z0-9])" + _flex_pattern(ident) + r"(?![A-Za-z0-9])",
167
+ EntityType.ID, re.IGNORECASE)
168
+ for a in k.addresses:
169
+ pat = r"\s+".join(re.escape(w) for w in a.split())
170
+ out += self._find(text, pat, EntityType.ADDRESS, re.IGNORECASE)
171
+ for type_name, values in k.other.items():
172
+ t = EntityType(type_name)
173
+ for v in values:
174
+ out += self._find(text, re.escape(v), t, re.IGNORECASE)
175
+ return out
176
+
177
+ def _find(self, text: str, pattern: str, t: EntityType, flags: int = 0,
178
+ canonical: Optional[str] = None) -> list[Entity]:
179
+ return [
180
+ Entity(t, m.start(), m.end(), m.group(0), self.name, canonical=canonical)
181
+ for m in re.finditer(pattern, text, flags)
182
+ if m.end() > m.start()
183
+ ]
184
+
185
+ def _dates(self, text: str, d: DateLike) -> list[Entity]:
186
+ if isinstance(d, str):
187
+ try:
188
+ d = date.fromisoformat(d)
189
+ except ValueError:
190
+ return self._find(text, re.escape(d), EntityType.DATE, re.IGNORECASE)
191
+ out: list[Entity] = []
192
+ for v in _date_variants(d):
193
+ out += self._find(text, r"(?<![\d/\-.])" + re.escape(v) + r"(?![\d])", EntityType.DATE, re.IGNORECASE)
194
+ return out
195
+
196
+ def _names(self, text: str, names: Iterable[str]) -> list[Entity]:
197
+ out: list[Entity] = []
198
+ tokens: dict[str, str] = {} # lowercase -> spelling from the record
199
+ for full in names:
200
+ full = full.strip()
201
+ if not full:
202
+ continue
203
+ # The full name first (also catches it across line breaks / extra spaces).
204
+ parts = full.split()
205
+ pat = r"\b" + r"\s+".join(re.escape(p) for p in parts) + r"\b"
206
+ out += self._find(text, pat, EntityType.NAME, re.IGNORECASE, canonical=" ".join(parts))
207
+ for p in re.split(r"[\s\-]+", full):
208
+ p = p.strip(".,'")
209
+ if len(p) >= 2:
210
+ tokens.setdefault(p.lower(), p)
211
+ if not tokens:
212
+ return out
213
+ for m in re.finditer(r"[A-Za-zÀ-ÖØ-öø-ÿ'\-]+", text):
214
+ raw = m.group(0)
215
+ lead = len(raw) - len(raw.lstrip("'-"))
216
+ word = raw.strip("'-")
217
+ if word.lower().endswith("'s"): # possessive: keep the 's outside the mask
218
+ word = word[:-2]
219
+ if len(word) < 2:
220
+ continue
221
+ lw = word.lower()
222
+ match = tokens.get(lw)
223
+ if match is None and self.fuzzy_names and len(lw) >= 4:
224
+ match = next(
225
+ (orig for t, orig in tokens.items()
226
+ if len(t) >= 5 and t[0] == lw[0] and _damerau_leq1(lw, t)),
227
+ None,
228
+ )
229
+ if match is None:
230
+ continue
231
+ if lw in COMMON_WORD_NAMES and not word[0].isupper():
232
+ continue
233
+ start = m.start() + lead
234
+ out.append(Entity(EntityType.NAME, start, start + len(word), word, self.name, canonical=match))
235
+ return out
@@ -0,0 +1,64 @@
1
+ """Optional NER layer using Microsoft Presidio, run in-process (nothing leaves).
2
+
3
+ Install with ``pip install phiproxy[presidio]`` plus a spaCy model, e.g.
4
+ ``python -m spacy download en_core_web_lg``.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from dataclasses import dataclass, field
10
+ from typing import Any, Optional
11
+
12
+ from ..entities import Entity, EntityType
13
+
14
+ # Presidio entity -> phiproxy type. Unlisted Presidio entities are ignored.
15
+ DEFAULT_MAP: dict[str, EntityType] = {
16
+ "PERSON": EntityType.NAME,
17
+ "PHONE_NUMBER": EntityType.PHONE,
18
+ "EMAIL_ADDRESS": EntityType.EMAIL,
19
+ "DATE_TIME": EntityType.DATE,
20
+ "LOCATION": EntityType.ADDRESS,
21
+ "US_SSN": EntityType.SSN,
22
+ "IP_ADDRESS": EntityType.IP_ADDRESS,
23
+ "URL": EntityType.URL,
24
+ "MEDICAL_LICENSE": EntityType.ID,
25
+ "US_DRIVER_LICENSE": EntityType.ID,
26
+ "US_PASSPORT": EntityType.ID,
27
+ "US_BANK_NUMBER": EntityType.ACCOUNT,
28
+ "CREDIT_CARD": EntityType.ACCOUNT,
29
+ "IBAN_CODE": EntityType.ACCOUNT,
30
+ }
31
+
32
+
33
+ @dataclass
34
+ class PresidioDetector:
35
+ name: str = "presidio"
36
+ language: str = "en"
37
+ score_threshold: float = 0.35 # low on purpose: recall over precision
38
+ entity_map: dict[str, EntityType] = field(default_factory=lambda: dict(DEFAULT_MAP))
39
+ analyzer: Optional[Any] = None
40
+
41
+ def __post_init__(self) -> None:
42
+ if self.analyzer is None:
43
+ from presidio_analyzer import AnalyzerEngine # type: ignore[import-not-found]
44
+
45
+ self.analyzer = AnalyzerEngine()
46
+
47
+ def detect(self, text: str) -> list[Entity]:
48
+ results = self.analyzer.analyze(
49
+ text=text,
50
+ language=self.language,
51
+ entities=list(self.entity_map.keys()),
52
+ score_threshold=self.score_threshold,
53
+ )
54
+ out = []
55
+ for r in results:
56
+ t = self.entity_map.get(r.entity_type)
57
+ if t is None or r.end <= r.start:
58
+ continue
59
+ # Presidio tags times ("3pm") and durations ("5 days") as DATE_TIME too;
60
+ # those are not identifiers. Calendar dates are left to the regex layer.
61
+ if t is EntityType.DATE:
62
+ continue
63
+ out.append(Entity(t, r.start, r.end, text[r.start : r.end], self.name, float(r.score)))
64
+ return out