phiproxy 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phiproxy/__init__.py +30 -0
- phiproxy/dates.py +128 -0
- phiproxy/detectors/__init__.py +5 -0
- phiproxy/detectors/base.py +73 -0
- phiproxy/detectors/known.py +235 -0
- phiproxy/detectors/presidio.py +64 -0
- phiproxy/detectors/regex.py +276 -0
- phiproxy/entities.py +65 -0
- phiproxy/integrations/__init__.py +1 -0
- phiproxy/integrations/litellm.py +304 -0
- phiproxy/masker.py +282 -0
- phiproxy/vault.py +182 -0
- phiproxy-0.1.0a1.dist-info/METADATA +208 -0
- phiproxy-0.1.0a1.dist-info/RECORD +17 -0
- phiproxy-0.1.0a1.dist-info/WHEEL +4 -0
- phiproxy-0.1.0a1.dist-info/licenses/LICENSE +202 -0
- phiproxy-0.1.0a1.dist-info/licenses/NOTICE +5 -0
phiproxy/__init__.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""phiproxy: reversible PHI masking for LLM traffic.
|
|
2
|
+
|
|
3
|
+
Mask patient identifiers before a prompt leaves your network, restore them in the
|
|
4
|
+
reply. phiproxy reduces how much PHI reaches an LLM provider; it does not make an
|
|
5
|
+
unapproved provider compliant. See README for the full disclaimer.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .detectors import KnownEntities, KnownEntityDetector, RegexDetector
|
|
9
|
+
from .entities import Entity, EntityType
|
|
10
|
+
from .masker import Masker, MaskResult, PhiproxyError, Policy, StreamUnmasker, UnsupportedContentError
|
|
11
|
+
from .vault import InMemoryVault, RedisVault, Vault
|
|
12
|
+
|
|
13
|
+
__version__ = "0.1.0a1"
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"Entity",
|
|
17
|
+
"EntityType",
|
|
18
|
+
"InMemoryVault",
|
|
19
|
+
"KnownEntities",
|
|
20
|
+
"KnownEntityDetector",
|
|
21
|
+
"MaskResult",
|
|
22
|
+
"Masker",
|
|
23
|
+
"PhiproxyError",
|
|
24
|
+
"Policy",
|
|
25
|
+
"RedisVault",
|
|
26
|
+
"RegexDetector",
|
|
27
|
+
"StreamUnmasker",
|
|
28
|
+
"UnsupportedContentError",
|
|
29
|
+
"Vault",
|
|
30
|
+
]
|
phiproxy/dates.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Date shifting that keeps the original writing style.
|
|
2
|
+
|
|
3
|
+
Shifting every date in a session by the same random offset hides the real dates
|
|
4
|
+
but keeps intervals intact (cycle day 3 -> retrieval -> transfer 5 days later),
|
|
5
|
+
which clinical reasoning depends on. The reply is shifted back by the same offset.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from datetime import date, timedelta
|
|
12
|
+
from typing import Optional
|
|
13
|
+
|
|
14
|
+
_MONTHS_FULL = [
|
|
15
|
+
"january", "february", "march", "april", "may", "june", "july", "august",
|
|
16
|
+
"september", "october", "november", "december",
|
|
17
|
+
]
|
|
18
|
+
_MONTH_LOOKUP = {}
|
|
19
|
+
for _i, _m in enumerate(_MONTHS_FULL, start=1):
|
|
20
|
+
_MONTH_LOOKUP[_m] = _i
|
|
21
|
+
_MONTH_LOOKUP[_m[:3]] = _i
|
|
22
|
+
_MONTH_LOOKUP["sept"] = 9
|
|
23
|
+
|
|
24
|
+
_NO_YEAR_REF = 2024 # leap year, so "Feb 29" without a year still parses
|
|
25
|
+
|
|
26
|
+
_ISO = re.compile(r"^(?P<y>\d{4})-(?P<m>\d{1,2})-(?P<d>\d{1,2})$")
|
|
27
|
+
_NUMERIC = re.compile(r"^(?P<a>\d{1,2})(?P<sep>[/\-.])(?P<b>\d{1,2})(?:(?P=sep)(?P<y>\d{4}|\d{2}))?$")
|
|
28
|
+
_MONTH_FIRST = re.compile(
|
|
29
|
+
r"^(?P<mon>[A-Za-z]+)(?P<dot>\.?)(?P<sp>\s+)(?P<d>\d{1,2})(?P<suf>st|nd|rd|th)?"
|
|
30
|
+
r"(?:(?P<comma>,?)(?P<sp2>\s+)(?P<y>\d{4}))?$",
|
|
31
|
+
re.IGNORECASE,
|
|
32
|
+
)
|
|
33
|
+
_DAY_FIRST = re.compile(
|
|
34
|
+
r"^(?P<d>\d{1,2})(?P<suf>st|nd|rd|th)?(?P<sp>\s+)(?P<of>of\s+)?(?P<mon>[A-Za-z]+)(?P<dot>\.?)"
|
|
35
|
+
r"(?:(?P<comma>,?)(?P<sp2>\s+)(?P<y>\d{4}))?$",
|
|
36
|
+
re.IGNORECASE,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _suffix(day: int) -> str:
|
|
41
|
+
if 11 <= day % 100 <= 13:
|
|
42
|
+
return "th"
|
|
43
|
+
return {1: "st", 2: "nd", 3: "rd"}.get(day % 10, "th")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _month_word(month: int, like: str) -> str:
|
|
47
|
+
full = _MONTHS_FULL[month - 1]
|
|
48
|
+
if len(like.rstrip(".")) <= 3 or (len(like) == 4 and like.lower() == "sept"):
|
|
49
|
+
word = full[:3]
|
|
50
|
+
else:
|
|
51
|
+
word = full
|
|
52
|
+
if like.isupper():
|
|
53
|
+
return word.upper()
|
|
54
|
+
if like[0].isupper():
|
|
55
|
+
return word.capitalize()
|
|
56
|
+
return word
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _pad(n: int, like: str, other: str = "") -> str:
|
|
60
|
+
"""Zero-pad when the original date shows a padded style ("03/14" or "11/03")."""
|
|
61
|
+
padded = (len(like) == 2 and like.startswith("0")) or (len(other) == 2 and other.startswith("0"))
|
|
62
|
+
return f"{n:02d}" if padded else str(n)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _year(y: int, like: str) -> str:
|
|
66
|
+
return f"{y % 100:02d}" if len(like) == 2 else str(y)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _full_year(two_or_four: str) -> int:
|
|
70
|
+
y = int(two_or_four)
|
|
71
|
+
if len(two_or_four) == 2:
|
|
72
|
+
y += 2000 if y < 70 else 1900
|
|
73
|
+
return y
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def shift_date_text(text: str, days: int, day_first: bool = False) -> Optional[str]:
|
|
77
|
+
"""Shift a date written as text by ``days``, keeping its format.
|
|
78
|
+
|
|
79
|
+
Returns None if the text is not a date this module understands; callers
|
|
80
|
+
should then fall back to replacing it with a token.
|
|
81
|
+
"""
|
|
82
|
+
s = text.strip()
|
|
83
|
+
try:
|
|
84
|
+
m = _ISO.match(s)
|
|
85
|
+
if m:
|
|
86
|
+
d = date(int(m["y"]), int(m["m"]), int(m["d"])) + timedelta(days=days)
|
|
87
|
+
return f"{d.year:04d}-{d.month:02d}-{d.day:02d}"
|
|
88
|
+
|
|
89
|
+
m = _NUMERIC.match(s)
|
|
90
|
+
if m:
|
|
91
|
+
a, b = int(m["a"]), int(m["b"])
|
|
92
|
+
month_first = not day_first
|
|
93
|
+
if a > 12 and b <= 12:
|
|
94
|
+
month_first = False
|
|
95
|
+
elif b > 12 and a <= 12:
|
|
96
|
+
month_first = True
|
|
97
|
+
mo, dy = (a, b) if month_first else (b, a)
|
|
98
|
+
mo_like, dy_like = (m["a"], m["b"]) if month_first else (m["b"], m["a"])
|
|
99
|
+
year = _full_year(m["y"]) if m["y"] else _NO_YEAR_REF
|
|
100
|
+
d = date(year, mo, dy) + timedelta(days=days)
|
|
101
|
+
mo_s, dy_s = _pad(d.month, mo_like, dy_like), _pad(d.day, dy_like, mo_like)
|
|
102
|
+
first, second = (mo_s, dy_s) if month_first else (dy_s, mo_s)
|
|
103
|
+
out = f"{first}{m['sep']}{second}"
|
|
104
|
+
if m["y"]:
|
|
105
|
+
out += f"{m['sep']}{_year(d.year, m['y'])}"
|
|
106
|
+
return out
|
|
107
|
+
|
|
108
|
+
for pat, month_first in ((_MONTH_FIRST, True), (_DAY_FIRST, False)):
|
|
109
|
+
m = pat.match(s)
|
|
110
|
+
if not m:
|
|
111
|
+
continue
|
|
112
|
+
month = _MONTH_LOOKUP.get(m["mon"].lower())
|
|
113
|
+
if month is None:
|
|
114
|
+
return None
|
|
115
|
+
year = int(m["y"]) if m["y"] else _NO_YEAR_REF
|
|
116
|
+
d = date(year, month, int(m["d"])) + timedelta(days=days)
|
|
117
|
+
mon = _month_word(d.month, m["mon"]) + m["dot"]
|
|
118
|
+
day = str(d.day) + (_suffix(d.day) if m["suf"] else "")
|
|
119
|
+
if month_first:
|
|
120
|
+
out = f"{mon}{m['sp']}{day}"
|
|
121
|
+
else:
|
|
122
|
+
out = f"{day}{m['sp']}{m['of'] or ''}{mon}"
|
|
123
|
+
if m["y"]:
|
|
124
|
+
out += f"{m['comma']}{m['sp2']}{d.year}"
|
|
125
|
+
return out
|
|
126
|
+
except ValueError: # e.g. 2/30
|
|
127
|
+
return None
|
|
128
|
+
return None
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Detector interface and span merging."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Iterable, Protocol, Sequence
|
|
6
|
+
|
|
7
|
+
from ..entities import Entity
|
|
8
|
+
|
|
9
|
+
# When two detections overlap, the one from the higher-priority source wins.
|
|
10
|
+
# Known-entity matches come from the clinic's own records, so they are the most
|
|
11
|
+
# trustworthy; NER guesses are the least.
|
|
12
|
+
SOURCE_PRIORITY = {"known": 3, "regex": 2, "presidio": 1}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Detector(Protocol):
|
|
16
|
+
"""Finds identifier spans in text. Must not call any external service."""
|
|
17
|
+
|
|
18
|
+
name: str
|
|
19
|
+
|
|
20
|
+
def detect(self, text: str) -> list[Entity]: ...
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def merge_entities(entities: Iterable[Entity]) -> list[Entity]:
|
|
24
|
+
"""Resolve overlapping spans into a non-overlapping, start-ordered list.
|
|
25
|
+
|
|
26
|
+
Overlaps are resolved by (span length, source priority, score): the longer
|
|
27
|
+
span wins (an email beats a first name found inside it); on equal length the
|
|
28
|
+
more trustworthy source wins. If a loser extends past the winner, the winner
|
|
29
|
+
is widened to the union so no character of the loser leaks.
|
|
30
|
+
"""
|
|
31
|
+
ordered = sorted(
|
|
32
|
+
entities,
|
|
33
|
+
key=lambda e: (e.length, SOURCE_PRIORITY.get(e.source, 0), e.score),
|
|
34
|
+
reverse=True,
|
|
35
|
+
)
|
|
36
|
+
kept: list[Entity] = []
|
|
37
|
+
for cand in ordered:
|
|
38
|
+
clash = [k for k in kept if k.overlaps(cand)]
|
|
39
|
+
if not clash:
|
|
40
|
+
kept.append(cand)
|
|
41
|
+
continue
|
|
42
|
+
# Widen the strongest overlapping winner to the union so no character
|
|
43
|
+
# of the loser (or of other winners it bridges) leaks.
|
|
44
|
+
winner = clash[0] # kept is in priority order, so the first clash is strongest
|
|
45
|
+
start = min([cand.start] + [k.start for k in clash])
|
|
46
|
+
end = max([cand.end] + [k.end for k in clash])
|
|
47
|
+
if len(clash) == 1 and (start, end) == (winner.start, winner.end):
|
|
48
|
+
continue
|
|
49
|
+
for k in clash:
|
|
50
|
+
kept.remove(k)
|
|
51
|
+
kept.insert(
|
|
52
|
+
0,
|
|
53
|
+
Entity(
|
|
54
|
+
type=winner.type,
|
|
55
|
+
start=start,
|
|
56
|
+
end=end,
|
|
57
|
+
text="?", # placeholder; fill_text() re-attaches real text
|
|
58
|
+
source=winner.source,
|
|
59
|
+
score=winner.score,
|
|
60
|
+
),
|
|
61
|
+
)
|
|
62
|
+
return sorted(kept, key=lambda e: e.start)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def fill_text(text: str, entities: Sequence[Entity]) -> list[Entity]:
|
|
66
|
+
"""Re-attach span text after merging (merge may widen spans)."""
|
|
67
|
+
out = []
|
|
68
|
+
for e in entities:
|
|
69
|
+
span = text[e.start : e.end]
|
|
70
|
+
# A widened span no longer matches its canonical form; fall back to the text.
|
|
71
|
+
canonical = e.canonical if span == e.text else None
|
|
72
|
+
out.append(Entity(e.type, e.start, e.end, span, e.source, e.score, canonical))
|
|
73
|
+
return out
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""Known-entity detection: match the identifiers you already hold for this patient.
|
|
2
|
+
|
|
3
|
+
The application usually knows who the conversation is about (the patient's name,
|
|
4
|
+
date of birth, phone, health card number...). Matching those exact values is the
|
|
5
|
+
highest-recall, lowest-cost layer: it catches casual first names, lowercase
|
|
6
|
+
spellings and one-letter typos that generic NER misses.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from datetime import date
|
|
14
|
+
from typing import Iterable, Mapping, Optional, Sequence, Union
|
|
15
|
+
|
|
16
|
+
from ..entities import Entity, EntityType
|
|
17
|
+
|
|
18
|
+
# First names that are also common English words. A lowercase match of these
|
|
19
|
+
# ("I have hope") is not treated as the name; a capitalized one is.
|
|
20
|
+
COMMON_WORD_NAMES = {
|
|
21
|
+
"hope", "faith", "grace", "joy", "will", "may", "june", "april", "rose", "mark", "bill",
|
|
22
|
+
"art", "page", "sky", "summer", "autumn", "dawn", "eve", "ray", "rich", "max", "pat",
|
|
23
|
+
"sue", "hunter", "chase", "lane", "brook", "river", "star", "sunny", "honey", "ginger",
|
|
24
|
+
"iris", "lily", "daisy", "ivy", "ruby", "amber", "crystal", "angel", "major", "young",
|
|
25
|
+
"white", "black", "brown", "green", "long", "short", "little", "king", "price", "cash",
|
|
26
|
+
"love", "christian", "miles", "frank", "jean", "gay", "deb", "dick", "guy", "an", "bo",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
_MONTH_NAMES = [
|
|
30
|
+
"January", "February", "March", "April", "May", "June", "July", "August",
|
|
31
|
+
"September", "October", "November", "December",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _damerau_leq1(a: str, b: str) -> bool:
|
|
36
|
+
"""True if a and b are within one edit (insert, delete, substitute, adjacent swap)."""
|
|
37
|
+
if a == b:
|
|
38
|
+
return True
|
|
39
|
+
la, lb = len(a), len(b)
|
|
40
|
+
if abs(la - lb) > 1:
|
|
41
|
+
return False
|
|
42
|
+
if la == lb:
|
|
43
|
+
diff = [i for i in range(la) if a[i] != b[i]]
|
|
44
|
+
if len(diff) == 1:
|
|
45
|
+
return True
|
|
46
|
+
if len(diff) == 2 and diff[1] == diff[0] + 1:
|
|
47
|
+
i, j = diff
|
|
48
|
+
return a[i] == b[j] and a[j] == b[i]
|
|
49
|
+
return False
|
|
50
|
+
if la > lb:
|
|
51
|
+
a, b = b, a
|
|
52
|
+
# b is longer by one: try deleting each char of b
|
|
53
|
+
i = j = 0
|
|
54
|
+
skipped = False
|
|
55
|
+
while i < len(a) and j < len(b):
|
|
56
|
+
if a[i] == b[j]:
|
|
57
|
+
i += 1
|
|
58
|
+
j += 1
|
|
59
|
+
elif skipped:
|
|
60
|
+
return False
|
|
61
|
+
else:
|
|
62
|
+
skipped = True
|
|
63
|
+
j += 1
|
|
64
|
+
return True
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _date_variants(d: date) -> list[str]:
|
|
68
|
+
m = _MONTH_NAMES[d.month - 1]
|
|
69
|
+
mon = m[:3]
|
|
70
|
+
return [
|
|
71
|
+
d.isoformat(),
|
|
72
|
+
f"{d.month}/{d.day}/{d.year}",
|
|
73
|
+
f"{d.month:02d}/{d.day:02d}/{d.year}",
|
|
74
|
+
f"{d.day}/{d.month}/{d.year}",
|
|
75
|
+
f"{d.day:02d}/{d.month:02d}/{d.year}",
|
|
76
|
+
f"{d.month}/{d.day}/{d.year % 100:02d}",
|
|
77
|
+
f"{d.month:02d}/{d.day:02d}/{d.year % 100:02d}",
|
|
78
|
+
f"{d.month}-{d.day}-{d.year}",
|
|
79
|
+
f"{d.day}.{d.month}.{d.year}",
|
|
80
|
+
f"{m} {d.day}, {d.year}",
|
|
81
|
+
f"{m} {d.day} {d.year}",
|
|
82
|
+
f"{mon} {d.day}, {d.year}",
|
|
83
|
+
f"{mon} {d.day} {d.year}",
|
|
84
|
+
f"{mon}. {d.day}, {d.year}",
|
|
85
|
+
f"{d.day} {m} {d.year}",
|
|
86
|
+
f"{d.day} {mon} {d.year}",
|
|
87
|
+
f"{m} {d.day}",
|
|
88
|
+
f"{mon} {d.day}",
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
DateLike = Union[date, str]
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@dataclass
|
|
96
|
+
class KnownEntities:
|
|
97
|
+
"""Identifiers the application already holds for the people in this conversation.
|
|
98
|
+
|
|
99
|
+
All fields are optional. Pass whatever you have; more is better.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
names: Sequence[str] = () # full names of the patient, partner, donor, ...
|
|
103
|
+
phones: Sequence[str] = ()
|
|
104
|
+
emails: Sequence[str] = ()
|
|
105
|
+
dates_of_birth: Sequence[DateLike] = ()
|
|
106
|
+
identifiers: Sequence[str] = () # health card, MRN, chart #, donor ID, account #
|
|
107
|
+
addresses: Sequence[str] = ()
|
|
108
|
+
other: Mapping[str, Sequence[str]] = field(default_factory=dict) # EntityType name -> values
|
|
109
|
+
|
|
110
|
+
@classmethod
|
|
111
|
+
def from_dict(cls, data: Mapping[str, object]) -> "KnownEntities":
|
|
112
|
+
def seq(key: str) -> list:
|
|
113
|
+
v = data.get(key) or []
|
|
114
|
+
return [v] if isinstance(v, (str, date)) else list(v) # type: ignore[arg-type]
|
|
115
|
+
|
|
116
|
+
return cls(
|
|
117
|
+
names=seq("names"),
|
|
118
|
+
phones=seq("phones"),
|
|
119
|
+
emails=seq("emails"),
|
|
120
|
+
dates_of_birth=seq("dates_of_birth"),
|
|
121
|
+
identifiers=seq("identifiers"),
|
|
122
|
+
addresses=seq("addresses"),
|
|
123
|
+
other=dict(data.get("other") or {}), # type: ignore[arg-type]
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
def is_empty(self) -> bool:
|
|
127
|
+
return not any(
|
|
128
|
+
[self.names, self.phones, self.emails, self.dates_of_birth,
|
|
129
|
+
self.identifiers, self.addresses, self.other]
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _flex_pattern(value: str) -> str:
|
|
134
|
+
"""Regex for a value where spaces, dashes and dots between characters are optional."""
|
|
135
|
+
chars = [re.escape(c) for c in re.sub(r"[\s\-.]", "", value)]
|
|
136
|
+
return r"[\s\-.]?".join(chars)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@dataclass
|
|
140
|
+
class KnownEntityDetector:
|
|
141
|
+
known: KnownEntities
|
|
142
|
+
name: str = "known"
|
|
143
|
+
fuzzy_names: bool = True # allow one typo against record names of 5+ letters
|
|
144
|
+
|
|
145
|
+
def detect(self, text: str) -> list[Entity]:
|
|
146
|
+
out: list[Entity] = []
|
|
147
|
+
k = self.known
|
|
148
|
+
out += self._names(text, k.names)
|
|
149
|
+
for p in k.phones:
|
|
150
|
+
digits = re.sub(r"\D", "", p)
|
|
151
|
+
if len(digits) >= 10:
|
|
152
|
+
digits = digits[-10:]
|
|
153
|
+
pat = (
|
|
154
|
+
r"(?<!\d)(?:\+?1[\s.\-]?)?\(?" + digits[:3] + r"\)?[\s.\-]?"
|
|
155
|
+
+ digits[3:6] + r"[\s.\-]?" + digits[6:] + r"(?!\d)"
|
|
156
|
+
)
|
|
157
|
+
out += self._find(text, pat, EntityType.PHONE)
|
|
158
|
+
elif len(digits) >= 7:
|
|
159
|
+
out += self._find(text, r"(?<!\d)" + _flex_pattern(digits) + r"(?!\d)", EntityType.PHONE)
|
|
160
|
+
for e in k.emails:
|
|
161
|
+
out += self._find(text, re.escape(e), EntityType.EMAIL, re.IGNORECASE)
|
|
162
|
+
for d in k.dates_of_birth:
|
|
163
|
+
out += self._dates(text, d)
|
|
164
|
+
for ident in k.identifiers:
|
|
165
|
+
if len(re.sub(r"[\s\-.]", "", ident)) >= 4:
|
|
166
|
+
out += self._find(text, r"(?<![A-Za-z0-9])" + _flex_pattern(ident) + r"(?![A-Za-z0-9])",
|
|
167
|
+
EntityType.ID, re.IGNORECASE)
|
|
168
|
+
for a in k.addresses:
|
|
169
|
+
pat = r"\s+".join(re.escape(w) for w in a.split())
|
|
170
|
+
out += self._find(text, pat, EntityType.ADDRESS, re.IGNORECASE)
|
|
171
|
+
for type_name, values in k.other.items():
|
|
172
|
+
t = EntityType(type_name)
|
|
173
|
+
for v in values:
|
|
174
|
+
out += self._find(text, re.escape(v), t, re.IGNORECASE)
|
|
175
|
+
return out
|
|
176
|
+
|
|
177
|
+
def _find(self, text: str, pattern: str, t: EntityType, flags: int = 0,
|
|
178
|
+
canonical: Optional[str] = None) -> list[Entity]:
|
|
179
|
+
return [
|
|
180
|
+
Entity(t, m.start(), m.end(), m.group(0), self.name, canonical=canonical)
|
|
181
|
+
for m in re.finditer(pattern, text, flags)
|
|
182
|
+
if m.end() > m.start()
|
|
183
|
+
]
|
|
184
|
+
|
|
185
|
+
def _dates(self, text: str, d: DateLike) -> list[Entity]:
|
|
186
|
+
if isinstance(d, str):
|
|
187
|
+
try:
|
|
188
|
+
d = date.fromisoformat(d)
|
|
189
|
+
except ValueError:
|
|
190
|
+
return self._find(text, re.escape(d), EntityType.DATE, re.IGNORECASE)
|
|
191
|
+
out: list[Entity] = []
|
|
192
|
+
for v in _date_variants(d):
|
|
193
|
+
out += self._find(text, r"(?<![\d/\-.])" + re.escape(v) + r"(?![\d])", EntityType.DATE, re.IGNORECASE)
|
|
194
|
+
return out
|
|
195
|
+
|
|
196
|
+
def _names(self, text: str, names: Iterable[str]) -> list[Entity]:
|
|
197
|
+
out: list[Entity] = []
|
|
198
|
+
tokens: dict[str, str] = {} # lowercase -> spelling from the record
|
|
199
|
+
for full in names:
|
|
200
|
+
full = full.strip()
|
|
201
|
+
if not full:
|
|
202
|
+
continue
|
|
203
|
+
# The full name first (also catches it across line breaks / extra spaces).
|
|
204
|
+
parts = full.split()
|
|
205
|
+
pat = r"\b" + r"\s+".join(re.escape(p) for p in parts) + r"\b"
|
|
206
|
+
out += self._find(text, pat, EntityType.NAME, re.IGNORECASE, canonical=" ".join(parts))
|
|
207
|
+
for p in re.split(r"[\s\-]+", full):
|
|
208
|
+
p = p.strip(".,'")
|
|
209
|
+
if len(p) >= 2:
|
|
210
|
+
tokens.setdefault(p.lower(), p)
|
|
211
|
+
if not tokens:
|
|
212
|
+
return out
|
|
213
|
+
for m in re.finditer(r"[A-Za-zÀ-ÖØ-öø-ÿ'\-]+", text):
|
|
214
|
+
raw = m.group(0)
|
|
215
|
+
lead = len(raw) - len(raw.lstrip("'-"))
|
|
216
|
+
word = raw.strip("'-")
|
|
217
|
+
if word.lower().endswith("'s"): # possessive: keep the 's outside the mask
|
|
218
|
+
word = word[:-2]
|
|
219
|
+
if len(word) < 2:
|
|
220
|
+
continue
|
|
221
|
+
lw = word.lower()
|
|
222
|
+
match = tokens.get(lw)
|
|
223
|
+
if match is None and self.fuzzy_names and len(lw) >= 4:
|
|
224
|
+
match = next(
|
|
225
|
+
(orig for t, orig in tokens.items()
|
|
226
|
+
if len(t) >= 5 and t[0] == lw[0] and _damerau_leq1(lw, t)),
|
|
227
|
+
None,
|
|
228
|
+
)
|
|
229
|
+
if match is None:
|
|
230
|
+
continue
|
|
231
|
+
if lw in COMMON_WORD_NAMES and not word[0].isupper():
|
|
232
|
+
continue
|
|
233
|
+
start = m.start() + lead
|
|
234
|
+
out.append(Entity(EntityType.NAME, start, start + len(word), word, self.name, canonical=match))
|
|
235
|
+
return out
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Optional NER layer using Microsoft Presidio, run in-process (nothing leaves).
|
|
2
|
+
|
|
3
|
+
Install with ``pip install phiproxy[presidio]`` plus a spaCy model, e.g.
|
|
4
|
+
``python -m spacy download en_core_web_lg``.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from typing import Any, Optional
|
|
11
|
+
|
|
12
|
+
from ..entities import Entity, EntityType
|
|
13
|
+
|
|
14
|
+
# Presidio entity -> phiproxy type. Unlisted Presidio entities are ignored.
|
|
15
|
+
DEFAULT_MAP: dict[str, EntityType] = {
|
|
16
|
+
"PERSON": EntityType.NAME,
|
|
17
|
+
"PHONE_NUMBER": EntityType.PHONE,
|
|
18
|
+
"EMAIL_ADDRESS": EntityType.EMAIL,
|
|
19
|
+
"DATE_TIME": EntityType.DATE,
|
|
20
|
+
"LOCATION": EntityType.ADDRESS,
|
|
21
|
+
"US_SSN": EntityType.SSN,
|
|
22
|
+
"IP_ADDRESS": EntityType.IP_ADDRESS,
|
|
23
|
+
"URL": EntityType.URL,
|
|
24
|
+
"MEDICAL_LICENSE": EntityType.ID,
|
|
25
|
+
"US_DRIVER_LICENSE": EntityType.ID,
|
|
26
|
+
"US_PASSPORT": EntityType.ID,
|
|
27
|
+
"US_BANK_NUMBER": EntityType.ACCOUNT,
|
|
28
|
+
"CREDIT_CARD": EntityType.ACCOUNT,
|
|
29
|
+
"IBAN_CODE": EntityType.ACCOUNT,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class PresidioDetector:
|
|
35
|
+
name: str = "presidio"
|
|
36
|
+
language: str = "en"
|
|
37
|
+
score_threshold: float = 0.35 # low on purpose: recall over precision
|
|
38
|
+
entity_map: dict[str, EntityType] = field(default_factory=lambda: dict(DEFAULT_MAP))
|
|
39
|
+
analyzer: Optional[Any] = None
|
|
40
|
+
|
|
41
|
+
def __post_init__(self) -> None:
|
|
42
|
+
if self.analyzer is None:
|
|
43
|
+
from presidio_analyzer import AnalyzerEngine # type: ignore[import-not-found]
|
|
44
|
+
|
|
45
|
+
self.analyzer = AnalyzerEngine()
|
|
46
|
+
|
|
47
|
+
def detect(self, text: str) -> list[Entity]:
|
|
48
|
+
results = self.analyzer.analyze(
|
|
49
|
+
text=text,
|
|
50
|
+
language=self.language,
|
|
51
|
+
entities=list(self.entity_map.keys()),
|
|
52
|
+
score_threshold=self.score_threshold,
|
|
53
|
+
)
|
|
54
|
+
out = []
|
|
55
|
+
for r in results:
|
|
56
|
+
t = self.entity_map.get(r.entity_type)
|
|
57
|
+
if t is None or r.end <= r.start:
|
|
58
|
+
continue
|
|
59
|
+
# Presidio tags times ("3pm") and durations ("5 days") as DATE_TIME too;
|
|
60
|
+
# those are not identifiers. Calendar dates are left to the regex layer.
|
|
61
|
+
if t is EntityType.DATE:
|
|
62
|
+
continue
|
|
63
|
+
out.append(Entity(t, r.start, r.end, text[r.start : r.end], self.name, float(r.score)))
|
|
64
|
+
return out
|