phiproxy 0.1.0a2__tar.gz → 0.1.0a4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/.github/workflows/ci.yml +4 -0
  2. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/.gitignore +1 -0
  3. phiproxy-0.1.0a4/NOTICE +11 -0
  4. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/PKG-INFO +56 -25
  5. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/README.md +51 -24
  6. phiproxy-0.1.0a4/benchmark/README.md +78 -0
  7. phiproxy-0.1.0a4/benchmark/blind/audit_v1.jsonl +45 -0
  8. phiproxy-0.1.0a4/benchmark/blind/audit_v2.jsonl +149 -0
  9. phiproxy-0.1.0a4/benchmark/blind/blind_v1.source.jsonl +420 -0
  10. phiproxy-0.1.0a4/benchmark/blind/blind_v2.source.jsonl +400 -0
  11. phiproxy-0.1.0a4/benchmark/blind/clinic_v2.json +1 -0
  12. phiproxy-0.1.0a4/benchmark/blind/convert.py +108 -0
  13. phiproxy-0.1.0a4/benchmark/data/patient_comms_blind_v1.jsonl +420 -0
  14. phiproxy-0.1.0a4/benchmark/data/patient_comms_blind_v2.jsonl +400 -0
  15. phiproxy-0.1.0a4/benchmark/results_blind_v1.json +182 -0
  16. phiproxy-0.1.0a4/benchmark/results_blind_v1_frozen.json +182 -0
  17. phiproxy-0.1.0a4/benchmark/results_blind_v2.json +174 -0
  18. phiproxy-0.1.0a4/benchmark/results_blind_v2_frozen.json +174 -0
  19. phiproxy-0.1.0a4/benchmark/results_v0.json +130 -0
  20. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/benchmark/score.py +53 -11
  21. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/pyproject.toml +1 -0
  22. phiproxy-0.1.0a4/scripts/build_places.py +58 -0
  23. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/__init__.py +1 -1
  24. phiproxy-0.1.0a4/src/phiproxy/data/__init__.py +0 -0
  25. phiproxy-0.1.0a4/src/phiproxy/data/places_ca_us.tsv +3460 -0
  26. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/detectors/base.py +1 -1
  27. phiproxy-0.1.0a4/src/phiproxy/detectors/gliner.py +163 -0
  28. phiproxy-0.1.0a4/src/phiproxy/detectors/regex.py +457 -0
  29. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/entities.py +1 -0
  30. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/masker.py +45 -1
  31. phiproxy-0.1.0a4/tests/test_gliner.py +75 -0
  32. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/tests/test_masker.py +19 -0
  33. phiproxy-0.1.0a4/tests/test_regex.py +172 -0
  34. phiproxy-0.1.0a2/NOTICE +0 -5
  35. phiproxy-0.1.0a2/benchmark/README.md +0 -38
  36. phiproxy-0.1.0a2/benchmark/results_v0.json +0 -56
  37. phiproxy-0.1.0a2/src/phiproxy/detectors/regex.py +0 -276
  38. phiproxy-0.1.0a2/tests/test_regex.py +0 -82
  39. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/.github/workflows/release.yml +0 -0
  40. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/CONTRIBUTING.md +0 -0
  41. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/LICENSE +0 -0
  42. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/SECURITY.md +0 -0
  43. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/benchmark/data/patient_comms_v0.jsonl +0 -0
  44. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/benchmark/generate.py +0 -0
  45. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/deploy/.env.example +0 -0
  46. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/deploy/Dockerfile +0 -0
  47. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/deploy/docker-compose.yml +0 -0
  48. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/deploy/litellm.config.yaml +0 -0
  49. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/examples/e2e/config.yaml +0 -0
  50. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/examples/e2e/fake_llm.py +0 -0
  51. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/examples/e2e/run_e2e.py +0 -0
  52. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/dates.py +0 -0
  53. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/detectors/__init__.py +0 -0
  54. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/detectors/known.py +0 -0
  55. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/detectors/presidio.py +0 -0
  56. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/integrations/__init__.py +0 -0
  57. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/integrations/litellm.py +0 -0
  58. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/src/phiproxy/vault.py +0 -0
  59. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/tests/test_dates.py +0 -0
  60. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/tests/test_known.py +0 -0
  61. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/tests/test_litellm.py +0 -0
  62. {phiproxy-0.1.0a2 → phiproxy-0.1.0a4}/tests/test_vault.py +0 -0
@@ -23,6 +23,10 @@ jobs:
23
23
  - run: pytest -q
24
24
  - name: Benchmark (regex+known must stay at or above 85% recall)
25
25
  run: python benchmark/score.py --config regex+known --min-recall 0.85
26
+ - name: Blind benchmark v1 (regex+known must stay at or above 70% recall)
27
+ run: python benchmark/score.py --data benchmark/data/patient_comms_blind_v1.jsonl --config regex+known --min-recall 0.70
28
+ - name: Blind benchmark v2 (regex+known must stay at or above 78% recall)
29
+ run: python benchmark/score.py --data benchmark/data/patient_comms_blind_v2.jsonl --config regex+known --min-recall 0.78
26
30
 
27
31
  e2e:
28
32
  runs-on: ubuntu-latest
@@ -8,3 +8,4 @@ build/
8
8
  deploy/.env
9
9
  received.jsonl
10
10
  .DS_Store
11
+ .venv-ml/
@@ -0,0 +1,11 @@
1
+ phiproxy
2
+ Copyright 2026 Decision Tree Technology
3
+
4
+ This product includes software developed by Decision Tree Technology
5
+ (https://decisiontree.tech/).
6
+
7
+ This product includes place names from GeoNames (https://www.geonames.org/),
8
+ licensed under the Creative Commons Attribution 4.0 License
9
+ (https://creativecommons.org/licenses/by/4.0/). The file
10
+ src/phiproxy/data/places_ca_us.tsv is derived from cities15000.txt
11
+ (Canadian and US places, name and ASCII name only).
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: phiproxy
3
- Version: 0.1.0a2
3
+ Version: 0.1.0a4
4
4
  Summary: Your patients' stand-in on the way to any LLM, and back: reversible PHI masking proxy (LiteLLM guardrail + Python library).
5
5
  Project-URL: Homepage, https://decisiontree.tech/open-source/phiproxy
6
6
  Project-URL: Source, https://github.com/DecisionTreeTechnology/phiproxy
@@ -22,6 +22,10 @@ Requires-Python: >=3.10
22
22
  Provides-Extra: dev
23
23
  Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
24
24
  Requires-Dist: pytest>=8; extra == 'dev'
25
+ Provides-Extra: gliner
26
+ Requires-Dist: gliner>=0.2.20; extra == 'gliner'
27
+ Requires-Dist: protobuf; extra == 'gliner'
28
+ Requires-Dist: sentencepiece; extra == 'gliner'
25
29
  Provides-Extra: litellm
26
30
  Requires-Dist: litellm>=1.70; extra == 'litellm'
27
31
  Provides-Extra: presidio
@@ -51,7 +55,7 @@ When the reply comes back, phiproxy swaps the stand-ins for the real values, so
51
55
  "Thanks Sarah Chen, we'll call 416-555-0142 today." ◄──
52
56
  ```
53
57
 
54
- > **Status: pre-alpha (0.1.0a2).** APIs will change. Do not rely on it as your only control.
58
+ > **Status: pre-alpha (0.1.0a4).** APIs will change. Do not rely on it as your only control.
55
59
 
56
60
  ## Read this first
57
61
 
@@ -61,7 +65,9 @@ When the reply comes back, phiproxy swaps the stand-ins for the real values, so
61
65
 
62
66
  ## What makes it different
63
67
 
64
- - **Your own records as a detector.** Pass the identifiers you already hold for the patient (name, DOB, phone, health card). Exact and one-typo matching against them catches lowercase names, nicknames in context and misspellings that pattern matching misses. On the benchmark below this lifts recall from 61% to 86%.
68
+ - **Your own records as a detector.** Pass the identifiers you already hold for the patient (name, DOB, phone, health card). Exact and one-typo matching against them catches lowercase names and misspellings that pattern matching misses. On the blind benchmark below this lifts recall from 57% to 80%.
69
+ - **Optional local model.** `phiproxy[gliner]` adds a small named-entity model that runs on your own machine and finds names, cities and workplaces that no rule or record covers ("my husband ravi", "I live in Sudbury"). With it, recall on a blind set written after the fixes was 92%, at about 50 ms per message on CPU.
70
+ - **Your clinic's own details pass.** `Policy(allow=["Harborview Reproductive Centre", "613-555-0100", "@clinic.example.org"])` lets the clinic's name, phone numbers, address, email domain and staff names reach the model unmasked, so answers keep them. Anything that matches the patient's own record is still masked.
65
71
  - **Stable tokens per conversation.** The same value gets the same token on every turn of a session, so the model can follow who is who and provider prompt caching keeps working. Tokens are sequential, not derived from the value (see 45 CFR 164.514(c)).
66
72
  - **Fail closed.** If masking errors, or a request holds media the policy does not allow, the request is rejected and nothing is sent. Clients cannot switch the guardrail off through request metadata.
67
73
  - **Clinic-specific identifiers.** Canadian health card numbers (OHIP, RAMQ), SIN, donor and embryo IDs, spoken numbers and dates from call transcripts.
@@ -149,40 +155,65 @@ docker compose -f deploy/docker-compose.yml up --build
149
155
 
150
156
  | Type | Examples | Layer |
151
157
  |---|---|---|
152
- | `NAME` | "this is Maria Lopez", "my husband Dave", any name from your records (any case, one typo) | regex context + known |
158
+ | `NAME` | "this is Maria Lopez", "my husband Dave", any name from your records (any case, one typo), "my husband ravi" | regex context + known + local model |
153
159
  | `PHONE` | 416-555-0142, (416) 555 0142, +44 20 7946 0958, "four one six five five five…" | regex + known |
154
160
  | `EMAIL`, `URL`, `IP_ADDRESS` | | regex |
155
161
  | `DATE` | 2026-11-03, 11/3, March 5, 1990, Nov 8th, "march fifteenth 1988" (year alone is not masked) | regex + known DOB |
156
162
  | `AGE` | ages over 89 only (Safe Harbor) | regex |
157
- | `ADDRESS`, `POSTAL_CODE`, `ZIP` | 22 Elm Street apt 4, PO Box 12, M5V 2T6, TX 78701 | regex + known |
163
+ | `ADDRESS`, `POSTAL_CODE`, `ZIP` | 22 Elm Street apt 4, PO Box 12, M5V 2T6, TX 78701, "I live in Sudbury" | regex + known + local model |
158
164
  | `HEALTH_CARD`, `SIN`, `SSN` | OHIP 1234-567-890-AB, RAMQ, keyword + number, Luhn-checked SIN | regex + known |
159
165
  | `MRN`, `ACCOUNT`, `DONOR_ID`, `ID` | "MRN: 00482913", "policy number: XK-99812", "donor D-4521" | regex + known |
160
- | `PROVIDER` | "Dr. Patel" (passes by default, as clinician names are not patient identifiers; enable with `Policy(disabled_types=set())`) | regex |
166
+ | `PROVIDER` | "Dr. Patel" (passes by default, as clinician names are not patient identifiers; enable with `Policy(disabled_types=set())`) | regex + local model |
167
+ | `EMPLOYER` | "I work at the Tim Hortons on Barton" | local model |
161
168
 
162
- Optional NER layer: `pip install phiproxy[presidio]` and add `PresidioDetector()` to the detector list. It runs in-process; nothing is sent anywhere to detect PHI.
169
+ Dates are also matched in French, Spanish, Polish and Hebrew, written or spoken ("le 12 mai 1990", "el catorce de febrero de mil novecientos noventa y uno", "trzeciego lipca", "5 בנובמבר"), and spoken digits in those languages. Canadian and US cities and towns over 15,000 people (from [GeoNames](https://www.geonames.org/), CC BY 4.0) are masked after a location cue ("I live in Sudbury", "j'habite à Laval", "mieszkam w Toronto"); names that are also common words, such as Normal or Mobile, must be capitalized. Code-shaped identifiers without a keyword ("SL-4471920-03", "W123456789", any bare number of 7+ digits) are masked as `ID`.
170
+
171
+ ### Optional local model
172
+
173
+ ```bash
174
+ pip install "phiproxy[gliner]"
175
+ ```
176
+
177
+ ```python
178
+ from phiproxy import Masker, RegexDetector
179
+ from phiproxy.detectors.gliner import GlinerDetector
180
+
181
+ masker = Masker(detectors=[RegexDetector(), GlinerDetector()])
182
+ ```
183
+
184
+ `GlinerDetector` uses [GLiNER](https://github.com/urchade/GLiNER) with the Apache-2.0 model `urchade/gliner_multi_pii-v1` (about 1.2 GB). It runs in-process: nothing is sent anywhere to detect PHI. The model is downloaded from Hugging Face on first use, so in production download it once and pass its local path as `model=`, or set `HF_HUB_OFFLINE=1`. Model guesses never override the policy: a guess on "Dr. Haddad" does not mask a clinician name that the policy lets through.
185
+
186
+ Presidio is also supported: `pip install phiproxy[presidio]` and add `PresidioDetector()`.
163
187
 
164
188
  ## Benchmark
165
189
 
166
- `benchmark/` holds a synthetic **patient communications** set: SMS, web chat, call transcripts and referral notes, the kind of text clinics actually send to LLMs. Most existing de-identification work targets clinical notes instead. All people and numbers are invented. Run it with `python benchmark/score.py`.
190
+ `benchmark/` holds three synthetic **patient communications** sets: SMS, web chat, call transcripts and voicemails, emails, referral letters and intake text, the kind of text clinics actually send to LLMs. Most existing de-identification work targets clinical notes instead. All people and numbers are invented.
167
191
 
168
- v0 results (288 messages, 560 labelled spans):
192
+ - **v0** (288 messages, English): template-based, written by the detector authors.
193
+ - **blind v1** (420 messages) and **blind v2** (400 messages): English, French, Spanish, Polish, Hebrew and mixed, each written by a separate AI agent that never saw the detector code, each with about 30% of messages containing no identifiers but full of lookalikes (drug doses, embryo grades, billing codes). A second annotator, also blind to the code, corrected the labels.
169
194
 
170
- | config | recall | missed |
171
- |---|---|---|
172
- | regex | 61.4% | 216 |
173
- | regex + known entities | **85.7%** | 80 |
195
+ Each blind set was scored once with the detectors frozen. v2 was written after the fixes made on v1, so its first score is the re-test:
174
196
 
175
- | recall by channel | regex | regex + known |
176
- |---|---|---|
177
- | SMS | 55% | 95% |
178
- | web chat | 74% | 84% |
179
- | call transcripts | 50% | 80% |
180
- | referral notes | 63% | 79% |
197
+ | recall | v0 | v1 first score | v2 first score |
198
+ |---|---|---|---|
199
+ | regex | 64.3% | 36.0% | 57.3% |
200
+ | regex + known entities | 88.6% | 54.1% | 80.2% |
201
+ | regex + local model | 98.2% | 77.3% | 87.9% |
202
+ | regex + known + local model | 99.1% | 79.2% | **92.3%** |
203
+
204
+ | v2, regex + known + local model | |
205
+ |---|---|
206
+ | by channel | SMS 97%, chat 97%, email 98%, notes 95%, call transcripts 83% |
207
+ | by language | en 92%, fr 97%, es 95%, pl 90%, he 85%, mixed 94% |
208
+ | precision | 76.5% (80.6% with the clinic's own details allowed) |
209
+ | messages without identifiers where something was masked | 51% (35% with the clinic's own details allowed) |
181
210
 
182
211
  **How to read this honestly:**
183
- - v0 was written by the same team that wrote the detectors. Expect real-world recall to be **lower**. An independent, larger set is the next milestone, and contributions are welcome.
184
- - Precision is reported as 100% on v0, but v0 has almost no non-PHI distractor text, so that number means little yet.
185
- - Known misses today: **cities and workplaces** ("I live in Sudbury", "I work at the Tim Hortons on King") and **names not in your records written in lowercase** ("my husband ravi"). These need the NER layer or a local model, which is on the roadmap.
212
+ - v0 overstated recall by about 30 points. Expect real traffic to score lower than the blind sets too: they are AI-written, not written by clinic staff.
213
+ - On v2 the "known entities" rows are optimistic: for some messages the patient record was filled in from the message itself.
214
+ - Over-masking is real. Most of it is the clinic's own contact details (pass them in `Policy(allow=...)`), holiday dates, and staff first names that are also words.
215
+ - Call transcripts are the weakest channel. Without the local model, names not in your records are mostly missed.
216
+ - Details, label rules, the corrections and full tables are in [`benchmark/README.md`](https://github.com/DecisionTreeTechnology/phiproxy/blob/main/benchmark/README.md).
186
217
 
187
218
  ## Deployment shapes
188
219
 
@@ -194,9 +225,9 @@ The vault holds real values. Protect Redis like the PHI store it is: private net
194
225
 
195
226
  ## Roadmap
196
227
 
197
- - Larger independent benchmark with distractor text, plus multilingual sets (French, Polish, Hebrew, Spanish)
198
- - Local small-model detector for names, places and workplaces
199
- - Gazetteer for cities and regions
228
+ - A benchmark set written by people who work in clinics
229
+ - Better call-transcript handling (fragments, voice-to-text errors)
230
+ - Smaller and faster local models (distilled or ONNX)
200
231
  - HTTP sidecar; Anthropic `/v1/messages` streaming
201
232
  - Masked logging hook for LiteLLM
202
233
 
@@ -19,7 +19,7 @@ When the reply comes back, phiproxy swaps the stand-ins for the real values, so
19
19
  "Thanks Sarah Chen, we'll call 416-555-0142 today." ◄──
20
20
  ```
21
21
 
22
- > **Status: pre-alpha (0.1.0a2).** APIs will change. Do not rely on it as your only control.
22
+ > **Status: pre-alpha (0.1.0a4).** APIs will change. Do not rely on it as your only control.
23
23
 
24
24
  ## Read this first
25
25
 
@@ -29,7 +29,9 @@ When the reply comes back, phiproxy swaps the stand-ins for the real values, so
29
29
 
30
30
  ## What makes it different
31
31
 
32
- - **Your own records as a detector.** Pass the identifiers you already hold for the patient (name, DOB, phone, health card). Exact and one-typo matching against them catches lowercase names, nicknames in context and misspellings that pattern matching misses. On the benchmark below this lifts recall from 61% to 86%.
32
+ - **Your own records as a detector.** Pass the identifiers you already hold for the patient (name, DOB, phone, health card). Exact and one-typo matching against them catches lowercase names and misspellings that pattern matching misses. On the blind benchmark below this lifts recall from 57% to 80%.
33
+ - **Optional local model.** `phiproxy[gliner]` adds a small named-entity model that runs on your own machine and finds names, cities and workplaces that no rule or record covers ("my husband ravi", "I live in Sudbury"). With it, recall on a blind set written after the fixes was 92%, at about 50 ms per message on CPU.
34
+ - **Your clinic's own details pass.** `Policy(allow=["Harborview Reproductive Centre", "613-555-0100", "@clinic.example.org"])` lets the clinic's name, phone numbers, address, email domain and staff names reach the model unmasked, so answers keep them. Anything that matches the patient's own record is still masked.
33
35
  - **Stable tokens per conversation.** The same value gets the same token on every turn of a session, so the model can follow who is who and provider prompt caching keeps working. Tokens are sequential, not derived from the value (see 45 CFR 164.514(c)).
34
36
  - **Fail closed.** If masking errors, or a request holds media the policy does not allow, the request is rejected and nothing is sent. Clients cannot switch the guardrail off through request metadata.
35
37
  - **Clinic-specific identifiers.** Canadian health card numbers (OHIP, RAMQ), SIN, donor and embryo IDs, spoken numbers and dates from call transcripts.
@@ -117,40 +119,65 @@ docker compose -f deploy/docker-compose.yml up --build
117
119
 
118
120
  | Type | Examples | Layer |
119
121
  |---|---|---|
120
- | `NAME` | "this is Maria Lopez", "my husband Dave", any name from your records (any case, one typo) | regex context + known |
122
+ | `NAME` | "this is Maria Lopez", "my husband Dave", any name from your records (any case, one typo), "my husband ravi" | regex context + known + local model |
121
123
  | `PHONE` | 416-555-0142, (416) 555 0142, +44 20 7946 0958, "four one six five five five…" | regex + known |
122
124
  | `EMAIL`, `URL`, `IP_ADDRESS` | | regex |
123
125
  | `DATE` | 2026-11-03, 11/3, March 5, 1990, Nov 8th, "march fifteenth 1988" (year alone is not masked) | regex + known DOB |
124
126
  | `AGE` | ages over 89 only (Safe Harbor) | regex |
125
- | `ADDRESS`, `POSTAL_CODE`, `ZIP` | 22 Elm Street apt 4, PO Box 12, M5V 2T6, TX 78701 | regex + known |
127
+ | `ADDRESS`, `POSTAL_CODE`, `ZIP` | 22 Elm Street apt 4, PO Box 12, M5V 2T6, TX 78701, "I live in Sudbury" | regex + known + local model |
126
128
  | `HEALTH_CARD`, `SIN`, `SSN` | OHIP 1234-567-890-AB, RAMQ, keyword + number, Luhn-checked SIN | regex + known |
127
129
  | `MRN`, `ACCOUNT`, `DONOR_ID`, `ID` | "MRN: 00482913", "policy number: XK-99812", "donor D-4521" | regex + known |
128
- | `PROVIDER` | "Dr. Patel" (passes by default, as clinician names are not patient identifiers; enable with `Policy(disabled_types=set())`) | regex |
130
+ | `PROVIDER` | "Dr. Patel" (passes by default, as clinician names are not patient identifiers; enable with `Policy(disabled_types=set())`) | regex + local model |
131
+ | `EMPLOYER` | "I work at the Tim Hortons on Barton" | local model |
129
132
 
130
- Optional NER layer: `pip install phiproxy[presidio]` and add `PresidioDetector()` to the detector list. It runs in-process; nothing is sent anywhere to detect PHI.
133
+ Dates are also matched in French, Spanish, Polish and Hebrew, written or spoken ("le 12 mai 1990", "el catorce de febrero de mil novecientos noventa y uno", "trzeciego lipca", "5 בנובמבר"), and spoken digits in those languages. Canadian and US cities and towns over 15,000 people (from [GeoNames](https://www.geonames.org/), CC BY 4.0) are masked after a location cue ("I live in Sudbury", "j'habite à Laval", "mieszkam w Toronto"); names that are also common words, such as Normal or Mobile, must be capitalized. Code-shaped identifiers without a keyword ("SL-4471920-03", "W123456789", any bare number of 7+ digits) are masked as `ID`.
134
+
135
+ ### Optional local model
136
+
137
+ ```bash
138
+ pip install "phiproxy[gliner]"
139
+ ```
140
+
141
+ ```python
142
+ from phiproxy import Masker, RegexDetector
143
+ from phiproxy.detectors.gliner import GlinerDetector
144
+
145
+ masker = Masker(detectors=[RegexDetector(), GlinerDetector()])
146
+ ```
147
+
148
+ `GlinerDetector` uses [GLiNER](https://github.com/urchade/GLiNER) with the Apache-2.0 model `urchade/gliner_multi_pii-v1` (about 1.2 GB). It runs in-process: nothing is sent anywhere to detect PHI. The model is downloaded from Hugging Face on first use, so in production download it once and pass its local path as `model=`, or set `HF_HUB_OFFLINE=1`. Model guesses never override the policy: a guess on "Dr. Haddad" does not mask a clinician name that the policy lets through.
149
+
150
+ Presidio is also supported: `pip install phiproxy[presidio]` and add `PresidioDetector()`.
131
151
 
132
152
  ## Benchmark
133
153
 
134
- `benchmark/` holds a synthetic **patient communications** set: SMS, web chat, call transcripts and referral notes, the kind of text clinics actually send to LLMs. Most existing de-identification work targets clinical notes instead. All people and numbers are invented. Run it with `python benchmark/score.py`.
154
+ `benchmark/` holds three synthetic **patient communications** sets: SMS, web chat, call transcripts and voicemails, emails, referral letters and intake text, the kind of text clinics actually send to LLMs. Most existing de-identification work targets clinical notes instead. All people and numbers are invented.
135
155
 
136
- v0 results (288 messages, 560 labelled spans):
156
+ - **v0** (288 messages, English): template-based, written by the detector authors.
157
+ - **blind v1** (420 messages) and **blind v2** (400 messages): English, French, Spanish, Polish, Hebrew and mixed, each written by a separate AI agent that never saw the detector code, each with about 30% of messages containing no identifiers but full of lookalikes (drug doses, embryo grades, billing codes). A second annotator, also blind to the code, corrected the labels.
137
158
 
138
- | config | recall | missed |
139
- |---|---|---|
140
- | regex | 61.4% | 216 |
141
- | regex + known entities | **85.7%** | 80 |
159
+ Each blind set was scored once with the detectors frozen. v2 was written after the fixes made on v1, so its first score is the re-test:
142
160
 
143
- | recall by channel | regex | regex + known |
144
- |---|---|---|
145
- | SMS | 55% | 95% |
146
- | web chat | 74% | 84% |
147
- | call transcripts | 50% | 80% |
148
- | referral notes | 63% | 79% |
161
+ | recall | v0 | v1 first score | v2 first score |
162
+ |---|---|---|---|
163
+ | regex | 64.3% | 36.0% | 57.3% |
164
+ | regex + known entities | 88.6% | 54.1% | 80.2% |
165
+ | regex + local model | 98.2% | 77.3% | 87.9% |
166
+ | regex + known + local model | 99.1% | 79.2% | **92.3%** |
167
+
168
+ | v2, regex + known + local model | |
169
+ |---|---|
170
+ | by channel | SMS 97%, chat 97%, email 98%, notes 95%, call transcripts 83% |
171
+ | by language | en 92%, fr 97%, es 95%, pl 90%, he 85%, mixed 94% |
172
+ | precision | 76.5% (80.6% with the clinic's own details allowed) |
173
+ | messages without identifiers where something was masked | 51% (35% with the clinic's own details allowed) |
149
174
 
150
175
  **How to read this honestly:**
151
- - v0 was written by the same team that wrote the detectors. Expect real-world recall to be **lower**. An independent, larger set is the next milestone, and contributions are welcome.
152
- - Precision is reported as 100% on v0, but v0 has almost no non-PHI distractor text, so that number means little yet.
153
- - Known misses today: **cities and workplaces** ("I live in Sudbury", "I work at the Tim Hortons on King") and **names not in your records written in lowercase** ("my husband ravi"). These need the NER layer or a local model, which is on the roadmap.
176
+ - v0 overstated recall by about 30 points. Expect real traffic to score lower than the blind sets too: they are AI-written, not written by clinic staff.
177
+ - On v2 the "known entities" rows are optimistic: for some messages the patient record was filled in from the message itself.
178
+ - Over-masking is real. Most of it is the clinic's own contact details (pass them in `Policy(allow=...)`), holiday dates, and staff first names that are also words.
179
+ - Call transcripts are the weakest channel. Without the local model, names not in your records are mostly missed.
180
+ - Details, label rules, the corrections and full tables are in [`benchmark/README.md`](https://github.com/DecisionTreeTechnology/phiproxy/blob/main/benchmark/README.md).
154
181
 
155
182
  ## Deployment shapes
156
183
 
@@ -162,9 +189,9 @@ The vault holds real values. Protect Redis like the PHI store it is: private net
162
189
 
163
190
  ## Roadmap
164
191
 
165
- - Larger independent benchmark with distractor text, plus multilingual sets (French, Polish, Hebrew, Spanish)
166
- - Local small-model detector for names, places and workplaces
167
- - Gazetteer for cities and regions
192
+ - A benchmark set written by people who work in clinics
193
+ - Better call-transcript handling (fragments, voice-to-text errors)
194
+ - Smaller and faster local models (distilled or ONNX)
168
195
  - HTTP sidecar; Anthropic `/v1/messages` streaming
169
196
  - Masked logging hook for LiteLLM
170
197
 
@@ -0,0 +1,78 @@
1
+ # Patient-communications PHI benchmark
2
+
3
+ Labelled sets of the kind of text clinics send to LLMs: patient SMS, web chat, phone-call transcripts and voicemails, emails, referral letters, fax and intake text. Most open de-identification datasets cover clinical notes; these cover patient communications. Every person and number in them is invented.
4
+
5
+ ## The sets
6
+
7
+ | set | messages | labelled identifiers | languages | written by |
8
+ |---|---|---|---|---|
9
+ | `patient_comms_v0.jsonl` | 288 | 560 | English | generated from templates by the detector authors |
10
+ | `patient_comms_blind_v1.jsonl` | 420 (128 with no identifiers) | 684 + 63 clinician names | en, fr, es, pl, he, mixed | blind AI author (Claude Opus) |
11
+ | `patient_comms_blind_v2.jsonl` | 400 (122 with no identifiers) | 778 + 106 clinician names | en, fr, es, pl, he, mixed | blind AI author (Claude Sonnet), different brief: more fax, intake, voicemail and email text |
12
+
13
+ **How the blind sets were made.** Each was written by a separate AI agent that had only a brief: the label rules, channels, languages, realistic mess, and a list of things that look like identifiers but are not. It was told not to read the detector code, the tests or the other sets, and it wrote every message from scratch. The sets are independent of the detector code, but they are AI-written, not written by clinic staff.
14
+
15
+ **Second annotator.** A third agent, also without access to code or detector output, reviewed each blind set against the written label rules and listed missing and wrong labels (`blind/audit_v1.jsonl`, `blind/audit_v2.jsonl`; 45 and 149 corrections, mostly unlabelled repeat mentions of names, staff names and spoken dates). `blind/convert.py` applies them. All scores below use the corrected labels.
16
+
17
+ **Frozen scoring.** Before each blind set was scored, the detectors were committed and not changed: v1 against commit 082c2e1, v2 against ab99dc9. Those first scores are the fair ones (`results_blind_*_frozen.json`). After a set was scored it was used to find and fix gaps, so its later scores are optimistic. v2 was the re-test of the fixes made on v1.
18
+
19
+ ## Results (recall)
20
+
21
+ | configuration | v0 | v1 first score | v1 now | v2 first score | v2 now |
22
+ |---|---|---|---|---|---|
23
+ | regex | 64.3% | 36.0% | 54.4% | 57.3% | 57.3% |
24
+ | regex + known entities | 88.6% | 54.1% | 72.1% | 80.2% | 80.2% |
25
+ | regex + local model | 98.2% | 77.3% | 90.2% | **87.9%** | 87.9% |
26
+ | regex + known + local model | 99.1% | 79.2% | 92.4% | **92.3%** | 92.3% |
27
+
28
+ - The fixes made on v1 held up on v2: recall with the local model moved from 79% on v1's first score to 92% on v2's first score.
29
+ - v0 overstates recall by about 30 points, because it was written by the people who wrote the detectors.
30
+ - On v2 the "known entities" rows are optimistic: for some messages the v2 author filled the patient record from the message itself.
31
+ - v2 recall with the local model, by channel: SMS 97%, chat 97%, email 98%, notes 95%, calls 83%. By language: en 92%, fr 97%, es 95%, pl 90%, he 85%, mixed 94%.
32
+
33
+ ## Results (over-masking)
34
+
35
+ | regex + known + local model | precision | messages without identifiers where something was masked |
36
+ |---|---|---|
37
+ | v1 now | 89.7% | 16% |
38
+ | v2 now, default policy | 76.5% | 51% |
39
+ | v2 now, clinic details allowed | 80.6% | 35% |
40
+
41
+ v2 contains many email footers and letterheads with the fictional clinic's own name, phone, fax, address and email. Those are not patient data, so they are not labelled, and phiproxy masks them unless the deployment lists them in `Policy(allow=[...])`. The last row passes the six clinic details in `blind/clinic_v2.json` with `--allow`. Most of the rest is holiday and clinic-closure dates (masked on purpose: a date is never exempted, because it could be a birthday), staff first names that are also words ("Will", "Grace", "Hope"), and staff names without a title.
42
+
43
+ ## Files
44
+
45
+ - `generate.py`: deterministic generator (fixed seed) for `data/patient_comms_v0.jsonl`.
46
+ - `blind/blind_v*.source.jsonl`: the blind sets as written, with inline labels (`[[NAME|jess]]`). `blind/audit_v*.jsonl`: second-annotator corrections. `blind/convert.py v1|v2` builds `data/patient_comms_blind_v*.jsonl`.
47
+ - `score.py`: scores every available configuration and prints recall by type, channel and language, precision, and clean-message masking. `--show-misses N`, `--show-false-positives N` and `--allow clinic.json` help with error analysis. The local-model configurations appear when `phiproxy[gliner]` is installed.
48
+ - `results_*.json`: scores for the built-in configurations.
49
+
50
+ ## Record format
51
+
52
+ ```json
53
+ {
54
+ "id": "sms-00-3",
55
+ "channel": "sms",
56
+ "text": "Hi it's Priya, can I move my bloodwork to Oct 3? My cell is 647-555-0112",
57
+ "spans": [{"start": 8, "end": 13, "type": "NAME"}, ...],
58
+ "known": {"names": ["Priya Natarajan"], "phones": ["647-555-0112"], ...}
59
+ }
60
+ ```
61
+
62
+ `known` is what the clinic system would already hold about the patient. It lets the benchmark measure the known-entity layer.
63
+
64
+ ## Scoring rules
65
+
66
+ - A labelled span is **caught** when detected spans cover at least 80% of its characters. The detected type does not have to match, because catching a phone number as an `ID` still protects it.
67
+ - Scoring uses the default policy, under which clinician names pass through. Clinician names (`PROVIDER`) are therefore not counted in recall, and masking one counts as a false positive.
68
+ - A detected span is **correct** when it overlaps a labelled identifier. **Clean messages masked** is the share of messages without identifiers in which something was masked anyway.
69
+ - What counts as an identifier follows HIPAA Safe Harbor (45 CFR 164.514(b)(2)): names of patients and relatives, all date elements except year, ages over 89, geographic units smaller than a state or province (street, city, postal code), contact details, and record, account, plan and other identifying numbers. Workplaces are labelled too because they can identify a person.
70
+
71
+ ## Known limits
72
+
73
+ - The blind sets are AI-written. Real messages are messier, and a set written by people who work in clinics is still the goal.
74
+ - Call transcripts are the weakest channel (83% on v2): names and numbers spoken in fragments, voice-to-text errors ("for one six" for 416), and repeated spellings.
75
+ - Still missed: some lowercase names that are also words ("joy", "will"), workplaces described in a phrase, Hebrew abbreviations of months, numeric spoken dates ("oh four oh nine eighty two").
76
+ - Over-masking costs answer quality, not privacy, but it is real: see the table above.
77
+
78
+ Contributions are welcome, especially messages written by people who work in clinics (see ../CONTRIBUTING.md). All data must be synthetic.
@@ -0,0 +1,45 @@
1
+ {"id": "en-sms-056", "action": "add", "type": "NAME", "surface": "Faith", "occurrence": 1, "reason": "friend first name"}
2
+ {"id": "en-sms-057", "action": "add", "type": "PROVIDER", "surface": "Holly", "occurrence": 1, "reason": "staff first name"}
3
+ {"id": "en-chat-107", "action": "add", "type": "PHONE", "surface": "647 555 0113", "occurrence": 1, "reason": "unmarked first copy of phone"}
4
+ {"id": "en-chat-125", "action": "add", "type": "DATE", "surface": "12th", "occurrence": 1, "reason": "appointment date"}
5
+ {"id": "en-chat-129", "action": "add", "type": "AGE", "surface": "92", "occurrence": 1, "reason": "age over 89"}
6
+ {"id": "en-call-150", "action": "add", "type": "HEALTH_CARD", "surface": "four seven two nine dash one one eight dash three oh five and then k t", "occurrence": 1, "reason": "spoken health card"}
7
+ {"id": "en-call-153", "action": "add", "type": "NAME", "surface": "mark", "occurrence": 1, "reason": "caller first name"}
8
+ {"id": "en-call-153", "action": "add", "type": "NAME", "surface": "hope", "occurrence": 1, "reason": "patient first name"}
9
+ {"id": "en-call-155", "action": "add", "type": "POSTAL_CODE", "surface": "m five v two t six", "occurrence": 1, "reason": "spoken postal"}
10
+ {"id": "en-call-155", "action": "add", "type": "POSTAL_CODE", "surface": "m5v 2t6", "occurrence": 1, "reason": "postal"}
11
+ {"id": "en-call-157", "action": "add", "type": "DATE", "surface": "the twentieth", "occurrence": 1, "reason": "spoken date"}
12
+ {"id": "en-call-160", "action": "add", "type": "DONOR_ID", "surface": "c b two zero four one eight s three", "occurrence": 1, "reason": "repeated donor id"}
13
+ {"id": "en-call-161", "action": "add", "type": "DATE", "surface": "the eighth", "occurrence": 1, "reason": "spoken date"}
14
+ {"id": "en-call-161", "action": "add", "type": "DATE", "surface": "the eighteenth", "occurrence": 1, "reason": "spoken date"}
15
+ {"id": "en-call-176", "action": "add", "type": "MRN", "surface": "zero zero four eight two one three", "occurrence": 1, "reason": "spoken MRN"}
16
+ {"id": "en-call-176", "action": "add", "type": "NAME", "surface": "whitney", "occurrence": 1, "reason": "patient first name"}
17
+ {"id": "en-call-185", "action": "add", "type": "DATE", "surface": "the eighteenth", "occurrence": 1, "reason": "spoken appt date"}
18
+ {"id": "en-call-196", "action": "add", "type": "NAME", "surface": "beth", "occurrence": 2, "reason": "name in agent turn"}
19
+ {"id": "en-email-197", "action": "add", "type": "NAME", "surface": "Jess", "occurrence": 2, "reason": "signature"}
20
+ {"id": "en-email-203", "action": "add", "type": "NAME", "surface": "Lauren", "occurrence": 1, "reason": "signature"}
21
+ {"id": "en-email-207", "action": "add", "type": "DATE", "surface": "Dec 1", "occurrence": 1, "reason": "shipping date tied to patient"}
22
+ {"id": "en-email-208", "action": "add", "type": "NAME", "surface": "Carmen", "occurrence": 1, "reason": "signature"}
23
+ {"id": "en-email-215", "action": "add", "type": "NAME", "surface": "Mireille", "occurrence": 1, "reason": "signature"}
24
+ {"id": "en-email-220", "action": "add", "type": "NAME", "surface": "Bill", "occurrence": 1, "reason": "nickname of patient husband"}
25
+ {"id": "en-note-246", "action": "add", "type": "NAME", "surface": "kay", "occurrence": 1, "reason": "nickname"}
26
+ {"id": "en-note-254", "action": "add", "type": "AGE", "surface": "91", "occurrence": 1, "reason": "age over 89"}
27
+ {"id": "en-note-255", "action": "add", "type": "PROVIDER", "surface": "Holly", "occurrence": 2, "reason": "staff name"}
28
+ {"id": "en-note-255", "action": "add", "type": "PROVIDER", "surface": "Hope Bergman", "occurrence": 1, "reason": "staff name"}
29
+ {"id": "en-note-260", "action": "add", "type": "NAME", "surface": "Cassandra Boyle", "occurrence": 1, "reason": "patient full name"}
30
+ {"id": "fr-chat-023", "action": "add", "type": "ADDRESS", "surface": "Ottawa", "occurrence": 1, "reason": "partner location"}
31
+ {"id": "fr-call-029", "action": "add", "type": "HEALTH_CARD", "surface": "t r e m neuf zéro un deux trois quatre cinq six", "occurrence": 1, "reason": "spoken card number"}
32
+ {"id": "es-call-025", "action": "add", "type": "NAME", "surface": "hernández", "occurrence": 2, "reason": "repeated surname"}
33
+ {"id": "es-call-025", "action": "add", "type": "NAME", "surface": "h e r n á n d e z", "occurrence": 1, "reason": "spelled surname"}
34
+ {"id": "es-email-034", "action": "add", "type": "NAME", "surface": "Laura", "occurrence": 1, "reason": "signature"}
35
+ {"id": "en-sms-008", "action": "remove", "type": "ADDRESS", "surface": "Dundas St", "occurrence": 1, "reason": "clinic parking lot"}
36
+ {"id": "en-sms-022", "action": "remove", "type": "ADDRESS", "surface": "downtown location", "occurrence": 1, "reason": "clinic location"}
37
+ {"id": "en-sms-026", "action": "remove", "type": "EMPLOYER", "surface": "Mount Sinai Hospital lab", "occurrence": 1, "reason": "lab, not workplace"}
38
+ {"id": "en-chat-111", "action": "remove", "type": "DATE", "surface": "Sunday", "occurrence": 1, "reason": "bare weekday"}
39
+ {"id": "en-chat-114", "action": "remove", "type": "DATE", "surface": "yesterday", "occurrence": 1, "reason": "relative word"}
40
+ {"id": "en-email-212", "action": "remove", "type": "DATE", "surface": "Saturday, November 22", "occurrence": 1, "reason": "clinic event"}
41
+ {"id": "en-email-215", "action": "remove", "type": "EMPLOYER", "surface": "Health Sciences North", "occurrence": 1, "reason": "lab, not workplace"}
42
+ {"id": "en-email-218", "action": "remove", "type": "EMAIL", "surface": "billing@example.com", "occurrence": 1, "reason": "clinic email"}
43
+ {"id": "en-email-218", "action": "retype", "from": "NAME", "type": "PROVIDER", "surface": "Ellen Brandt", "occurrence": 1, "reason": "billing staff"}
44
+ {"id": "en-email-219", "action": "retype", "from": "NAME", "type": "PROVIDER", "surface": "Dr. Alan Brandt", "occurrence": 1, "reason": "clinician"}
45
+ {"id": "es-call-030", "action": "retype", "from": "NAME", "type": "PROVIDER", "surface": "doctora ramos", "occurrence": 1, "reason": "clinician"}