phiproxy 0.1.0a3__tar.gz → 0.1.0a4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/.github/workflows/ci.yml +4 -2
  2. phiproxy-0.1.0a4/NOTICE +11 -0
  3. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/PKG-INFO +27 -22
  4. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/README.md +26 -21
  5. phiproxy-0.1.0a4/benchmark/README.md +78 -0
  6. phiproxy-0.1.0a4/benchmark/blind/audit_v1.jsonl +45 -0
  7. phiproxy-0.1.0a4/benchmark/blind/audit_v2.jsonl +149 -0
  8. phiproxy-0.1.0a4/benchmark/blind/blind_v2.source.jsonl +400 -0
  9. phiproxy-0.1.0a4/benchmark/blind/clinic_v2.json +1 -0
  10. phiproxy-0.1.0a4/benchmark/blind/convert.py +108 -0
  11. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/benchmark/data/patient_comms_blind_v1.jsonl +37 -37
  12. phiproxy-0.1.0a4/benchmark/data/patient_comms_blind_v2.jsonl +400 -0
  13. phiproxy-0.1.0a4/benchmark/results_blind_v1.json +182 -0
  14. phiproxy-0.1.0a4/benchmark/results_blind_v1_frozen.json +182 -0
  15. phiproxy-0.1.0a4/benchmark/results_blind_v2.json +174 -0
  16. phiproxy-0.1.0a4/benchmark/results_blind_v2_frozen.json +174 -0
  17. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/benchmark/results_v0.json +14 -14
  18. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/benchmark/score.py +18 -5
  19. phiproxy-0.1.0a4/scripts/build_places.py +58 -0
  20. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/__init__.py +1 -1
  21. phiproxy-0.1.0a4/src/phiproxy/data/__init__.py +0 -0
  22. phiproxy-0.1.0a4/src/phiproxy/data/places_ca_us.tsv +3460 -0
  23. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/detectors/gliner.py +10 -3
  24. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/detectors/regex.py +95 -1
  25. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/masker.py +30 -0
  26. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/tests/test_masker.py +19 -0
  27. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/tests/test_regex.py +33 -0
  28. phiproxy-0.1.0a3/NOTICE +0 -5
  29. phiproxy-0.1.0a3/benchmark/README.md +0 -62
  30. phiproxy-0.1.0a3/benchmark/blind/convert.py +0 -56
  31. phiproxy-0.1.0a3/benchmark/results_blind_v1.json +0 -182
  32. phiproxy-0.1.0a3/benchmark/results_blind_v1_frozen.json +0 -182
  33. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/.github/workflows/release.yml +0 -0
  34. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/.gitignore +0 -0
  35. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/CONTRIBUTING.md +0 -0
  36. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/LICENSE +0 -0
  37. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/SECURITY.md +0 -0
  38. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/benchmark/blind/blind_v1.source.jsonl +0 -0
  39. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/benchmark/data/patient_comms_v0.jsonl +0 -0
  40. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/benchmark/generate.py +0 -0
  41. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/deploy/.env.example +0 -0
  42. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/deploy/Dockerfile +0 -0
  43. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/deploy/docker-compose.yml +0 -0
  44. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/deploy/litellm.config.yaml +0 -0
  45. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/examples/e2e/config.yaml +0 -0
  46. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/examples/e2e/fake_llm.py +0 -0
  47. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/examples/e2e/run_e2e.py +0 -0
  48. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/pyproject.toml +0 -0
  49. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/dates.py +0 -0
  50. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/detectors/__init__.py +0 -0
  51. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/detectors/base.py +0 -0
  52. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/detectors/known.py +0 -0
  53. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/detectors/presidio.py +0 -0
  54. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/entities.py +0 -0
  55. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/integrations/__init__.py +0 -0
  56. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/integrations/litellm.py +0 -0
  57. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/src/phiproxy/vault.py +0 -0
  58. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/tests/test_dates.py +0 -0
  59. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/tests/test_gliner.py +0 -0
  60. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/tests/test_known.py +0 -0
  61. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/tests/test_litellm.py +0 -0
  62. {phiproxy-0.1.0a3 → phiproxy-0.1.0a4}/tests/test_vault.py +0 -0
@@ -23,8 +23,10 @@ jobs:
23
23
  - run: pytest -q
24
24
  - name: Benchmark (regex+known must stay at or above 85% recall)
25
25
  run: python benchmark/score.py --config regex+known --min-recall 0.85
26
- - name: Blind benchmark (regex+known must stay at or above 65% recall)
27
- run: python benchmark/score.py --data benchmark/data/patient_comms_blind_v1.jsonl --config regex+known --min-recall 0.65
26
+ - name: Blind benchmark v1 (regex+known must stay at or above 70% recall)
27
+ run: python benchmark/score.py --data benchmark/data/patient_comms_blind_v1.jsonl --config regex+known --min-recall 0.70
28
+ - name: Blind benchmark v2 (regex+known must stay at or above 78% recall)
29
+ run: python benchmark/score.py --data benchmark/data/patient_comms_blind_v2.jsonl --config regex+known --min-recall 0.78
28
30
 
29
31
  e2e:
30
32
  runs-on: ubuntu-latest
@@ -0,0 +1,11 @@
1
+ phiproxy
2
+ Copyright 2026 Decision Tree Technology
3
+
4
+ This product includes software developed by Decision Tree Technology
5
+ (https://decisiontree.tech/).
6
+
7
+ This product includes place names from GeoNames (https://www.geonames.org/),
8
+ licensed under the Creative Commons Attribution 4.0 License
9
+ (https://creativecommons.org/licenses/by/4.0/). The file
10
+ src/phiproxy/data/places_ca_us.tsv is derived from cities15000.txt
11
+ (Canadian and US places, name and ASCII name only).
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: phiproxy
3
- Version: 0.1.0a3
3
+ Version: 0.1.0a4
4
4
  Summary: Your patients' stand-in on the way to any LLM, and back: reversible PHI masking proxy (LiteLLM guardrail + Python library).
5
5
  Project-URL: Homepage, https://decisiontree.tech/open-source/phiproxy
6
6
  Project-URL: Source, https://github.com/DecisionTreeTechnology/phiproxy
@@ -55,7 +55,7 @@ When the reply comes back, phiproxy swaps the stand-ins for the real values, so
55
55
  "Thanks Sarah Chen, we'll call 416-555-0142 today." ◄──
56
56
  ```
57
57
 
58
- > **Status: pre-alpha (0.1.0a3).** APIs will change. Do not rely on it as your only control.
58
+ > **Status: pre-alpha (0.1.0a4).** APIs will change. Do not rely on it as your only control.
59
59
 
60
60
  ## Read this first
61
61
 
@@ -65,8 +65,9 @@ When the reply comes back, phiproxy swaps the stand-ins for the real values, so
65
65
 
66
66
  ## What makes it different
67
67
 
68
- - **Your own records as a detector.** Pass the identifiers you already hold for the patient (name, DOB, phone, health card). Exact and one-typo matching against them catches lowercase names and misspellings that pattern matching misses. On the blind benchmark below this lifts recall from 51% to 68%.
69
- - **Optional local model.** `phiproxy[gliner]` adds a small named-entity model that runs on your own machine and finds names, cities and workplaces that no rule or record covers ("my husband ravi", "I live in Sudbury"). It lifts blind-benchmark recall to 92%, at about 50 ms per message on CPU.
68
+ - **Your own records as a detector.** Pass the identifiers you already hold for the patient (name, DOB, phone, health card). Exact and one-typo matching against them catches lowercase names and misspellings that pattern matching misses. On the blind benchmark below this lifts recall from 57% to 80%.
69
+ - **Optional local model.** `phiproxy[gliner]` adds a small named-entity model that runs on your own machine and finds names, cities and workplaces that no rule or record covers ("my husband ravi", "I live in Sudbury"). With it, recall on a blind set written after the fixes was 92%, at about 50 ms per message on CPU.
70
+ - **Your clinic's own details pass.** `Policy(allow=["Harborview Reproductive Centre", "613-555-0100", "@clinic.example.org"])` lets the clinic's name, phone numbers, address, email domain and staff names reach the model unmasked, so answers keep them. Anything that matches the patient's own record is still masked.
70
71
  - **Stable tokens per conversation.** The same value gets the same token on every turn of a session, so the model can follow who is who and provider prompt caching keeps working. Tokens are sequential, not derived from the value (see 45 CFR 164.514(c)).
71
72
  - **Fail closed.** If masking errors, or a request holds media the policy does not allow, the request is rejected and nothing is sent. Clients cannot switch the guardrail off through request metadata.
72
73
  - **Clinic-specific identifiers.** Canadian health card numbers (OHIP, RAMQ), SIN, donor and embryo IDs, spoken numbers and dates from call transcripts.
@@ -165,7 +166,7 @@ docker compose -f deploy/docker-compose.yml up --build
165
166
  | `PROVIDER` | "Dr. Patel" (passes by default, as clinician names are not patient identifiers; enable with `Policy(disabled_types=set())`) | regex + local model |
166
167
  | `EMPLOYER` | "I work at the Tim Hortons on Barton" | local model |
167
168
 
168
- Dates are also matched in French, Spanish, Polish and Hebrew ("le 12 mai 1990", "jueves 6 de noviembre", "5 בנובמבר"), and spoken digits in those languages. Code-shaped identifiers without a keyword ("SL-4471920-03", "W123456789", any bare number of 7+ digits) are masked as `ID`.
169
+ Dates are also matched in French, Spanish, Polish and Hebrew, written or spoken ("le 12 mai 1990", "el catorce de febrero de mil novecientos noventa y uno", "trzeciego lipca", "5 בנובמבר"), and spoken digits in those languages. Canadian and US cities and towns over 15,000 people (from [GeoNames](https://www.geonames.org/), CC BY 4.0) are masked after a location cue ("I live in Sudbury", "j'habite à Laval", "mieszkam w Toronto"); names that are also common words, such as Normal or Mobile, must be capitalized. Code-shaped identifiers without a keyword ("SL-4471920-03", "W123456789", any bare number of 7+ digits) are masked as `ID`.
169
170
 
170
171
  ### Optional local model
171
172
 
@@ -186,29 +187,33 @@ Presidio is also supported: `pip install phiproxy[presidio]` and add `PresidioDe
186
187
 
187
188
  ## Benchmark
188
189
 
189
- `benchmark/` holds two synthetic **patient communications** sets: SMS, web chat, call transcripts, emails and referral notes, the kind of text clinics actually send to LLMs. Most existing de-identification work targets clinical notes instead. All people and numbers are invented.
190
+ `benchmark/` holds three synthetic **patient communications** sets: SMS, web chat, call transcripts and voicemails, emails, referral letters and intake text, the kind of text clinics actually send to LLMs. Most existing de-identification work targets clinical notes instead. All people and numbers are invented.
190
191
 
191
192
  - **v0** (288 messages, English): template-based, written by the detector authors.
192
- - **blind v1** (420 messages; English, French, Spanish, Polish, Hebrew and mixed; 129 messages with no identifiers but full of lookalikes such as drug doses and embryo grades): written by a separate AI agent that never saw the detector code.
193
+ - **blind v1** (420 messages) and **blind v2** (400 messages): English, French, Spanish, Polish, Hebrew and mixed, each written by a separate AI agent that never saw the detector code, each with about 30% of messages containing no identifiers but full of lookalikes (drug doses, embryo grades, billing codes). A second annotator, also blind to the code, corrected the labels.
193
194
 
194
- | recall | v0 | blind v1, first score | blind v1, now |
195
+ Each blind set was scored once with the detectors frozen. v2 was written after the fixes made on v1, so its first score is the re-test:
196
+
197
+ | recall | v0 | v1 first score | v2 first score |
195
198
  |---|---|---|---|
196
- | regex | 61.4% | 36.6% | 50.5% |
197
- | regex + known entities | 85.7% | 54.4% | 67.8% |
198
- | regex + known + local model | 99.1% | 80.1% | **92.2%** |
199
+ | regex | 64.3% | 36.0% | 57.3% |
200
+ | regex + known entities | 88.6% | 54.1% | 80.2% |
201
+ | regex + local model | 98.2% | 77.3% | 87.9% |
202
+ | regex + known + local model | 99.1% | 79.2% | **92.3%** |
199
203
 
200
- | blind v1, regex + known + local model | |
204
+ | v2, regex + known + local model | |
201
205
  |---|---|
202
- | by channel | SMS 94%, chat 91%, email 96%, notes 94%, call transcripts 86% |
203
- | by language | en 91%, fr 94%, es 93%, pl 95%, he 92%, mixed 97% |
204
- | precision | 86.9% |
205
- | messages without identifiers where something was masked | 19% |
206
+ | by channel | SMS 97%, chat 97%, email 98%, notes 95%, call transcripts 83% |
207
+ | by language | en 92%, fr 97%, es 95%, pl 90%, he 85%, mixed 94% |
208
+ | precision | 76.5% (80.6% with the clinic's own details allowed) |
209
+ | messages without identifiers where something was masked | 51% (35% with the clinic's own details allowed) |
206
210
 
207
211
  **How to read this honestly:**
208
- - The first blind score is the fair one: the detectors were frozen before v1 was scored. "Now" includes fixes made after looking at v1's misses, so it is optimistic. A new blind set will re-test it.
209
- - v0 overstated recall by about 30 points. Expect real traffic to score lower than v1 too.
210
- - Known misses: dates spoken in words outside English, some cities, workplaces described in a phrase, and lowercase names that are also words ("joy", "will"). Without the local model, cities and names not in your records are mostly missed.
211
- - Details, label rules and the full tables are in [`benchmark/README.md`](https://github.com/DecisionTreeTechnology/phiproxy/blob/main/benchmark/README.md).
212
+ - v0 overstated recall by about 30 points. Expect real traffic to score lower than the blind sets too: they are AI-written, not written by clinic staff.
213
+ - On v2 the "known entities" rows are optimistic: for some messages the patient record was filled in from the message itself.
214
+ - Over-masking is real. Most of it is the clinic's own contact details (pass them in `Policy(allow=...)`), holiday dates, and staff first names that are also words.
215
+ - Call transcripts are the weakest channel. Without the local model, names not in your records are mostly missed.
216
+ - Details, label rules, the corrections and full tables are in [`benchmark/README.md`](https://github.com/DecisionTreeTechnology/phiproxy/blob/main/benchmark/README.md).
212
217
 
213
218
  ## Deployment shapes
214
219
 
@@ -220,8 +225,8 @@ The vault holds real values. Protect Redis like the PHI store it is: private net
220
225
 
221
226
  ## Roadmap
222
227
 
223
- - A benchmark set written by clinic staff, and a second blind AI-written set to re-test the current numbers
224
- - Gazetteer for cities the model misses; spoken dates in French, Spanish, Polish and Hebrew
228
+ - A benchmark set written by people who work in clinics
229
+ - Better call-transcript handling (fragments, voice-to-text errors)
225
230
  - Smaller and faster local models (distilled or ONNX)
226
231
  - HTTP sidecar; Anthropic `/v1/messages` streaming
227
232
  - Masked logging hook for LiteLLM
@@ -19,7 +19,7 @@ When the reply comes back, phiproxy swaps the stand-ins for the real values, so
19
19
  "Thanks Sarah Chen, we'll call 416-555-0142 today." ◄──
20
20
  ```
21
21
 
22
- > **Status: pre-alpha (0.1.0a3).** APIs will change. Do not rely on it as your only control.
22
+ > **Status: pre-alpha (0.1.0a4).** APIs will change. Do not rely on it as your only control.
23
23
 
24
24
  ## Read this first
25
25
 
@@ -29,8 +29,9 @@ When the reply comes back, phiproxy swaps the stand-ins for the real values, so
29
29
 
30
30
  ## What makes it different
31
31
 
32
- - **Your own records as a detector.** Pass the identifiers you already hold for the patient (name, DOB, phone, health card). Exact and one-typo matching against them catches lowercase names and misspellings that pattern matching misses. On the blind benchmark below this lifts recall from 51% to 68%.
33
- - **Optional local model.** `phiproxy[gliner]` adds a small named-entity model that runs on your own machine and finds names, cities and workplaces that no rule or record covers ("my husband ravi", "I live in Sudbury"). It lifts blind-benchmark recall to 92%, at about 50 ms per message on CPU.
32
+ - **Your own records as a detector.** Pass the identifiers you already hold for the patient (name, DOB, phone, health card). Exact and one-typo matching against them catches lowercase names and misspellings that pattern matching misses. On the blind benchmark below this lifts recall from 57% to 80%.
33
+ - **Optional local model.** `phiproxy[gliner]` adds a small named-entity model that runs on your own machine and finds names, cities and workplaces that no rule or record covers ("my husband ravi", "I live in Sudbury"). With it, recall on a blind set written after the fixes was 92%, at about 50 ms per message on CPU.
34
+ - **Your clinic's own details pass.** `Policy(allow=["Harborview Reproductive Centre", "613-555-0100", "@clinic.example.org"])` lets the clinic's name, phone numbers, address, email domain and staff names reach the model unmasked, so answers keep them. Anything that matches the patient's own record is still masked.
34
35
  - **Stable tokens per conversation.** The same value gets the same token on every turn of a session, so the model can follow who is who and provider prompt caching keeps working. Tokens are sequential, not derived from the value (see 45 CFR 164.514(c)).
35
36
  - **Fail closed.** If masking errors, or a request holds media the policy does not allow, the request is rejected and nothing is sent. Clients cannot switch the guardrail off through request metadata.
36
37
  - **Clinic-specific identifiers.** Canadian health card numbers (OHIP, RAMQ), SIN, donor and embryo IDs, spoken numbers and dates from call transcripts.
@@ -129,7 +130,7 @@ docker compose -f deploy/docker-compose.yml up --build
129
130
  | `PROVIDER` | "Dr. Patel" (passes by default, as clinician names are not patient identifiers; enable with `Policy(disabled_types=set())`) | regex + local model |
130
131
  | `EMPLOYER` | "I work at the Tim Hortons on Barton" | local model |
131
132
 
132
- Dates are also matched in French, Spanish, Polish and Hebrew ("le 12 mai 1990", "jueves 6 de noviembre", "5 בנובמבר"), and spoken digits in those languages. Code-shaped identifiers without a keyword ("SL-4471920-03", "W123456789", any bare number of 7+ digits) are masked as `ID`.
133
+ Dates are also matched in French, Spanish, Polish and Hebrew, written or spoken ("le 12 mai 1990", "el catorce de febrero de mil novecientos noventa y uno", "trzeciego lipca", "5 בנובמבר"), and spoken digits in those languages. Canadian and US cities and towns over 15,000 people (from [GeoNames](https://www.geonames.org/), CC BY 4.0) are masked after a location cue ("I live in Sudbury", "j'habite à Laval", "mieszkam w Toronto"); names that are also common words, such as Normal or Mobile, must be capitalized. Code-shaped identifiers without a keyword ("SL-4471920-03", "W123456789", any bare number of 7+ digits) are masked as `ID`.
133
134
 
134
135
  ### Optional local model
135
136
 
@@ -150,29 +151,33 @@ Presidio is also supported: `pip install phiproxy[presidio]` and add `PresidioDe
150
151
 
151
152
  ## Benchmark
152
153
 
153
- `benchmark/` holds two synthetic **patient communications** sets: SMS, web chat, call transcripts, emails and referral notes, the kind of text clinics actually send to LLMs. Most existing de-identification work targets clinical notes instead. All people and numbers are invented.
154
+ `benchmark/` holds three synthetic **patient communications** sets: SMS, web chat, call transcripts and voicemails, emails, referral letters and intake text, the kind of text clinics actually send to LLMs. Most existing de-identification work targets clinical notes instead. All people and numbers are invented.
154
155
 
155
156
  - **v0** (288 messages, English): template-based, written by the detector authors.
156
- - **blind v1** (420 messages; English, French, Spanish, Polish, Hebrew and mixed; 129 messages with no identifiers but full of lookalikes such as drug doses and embryo grades): written by a separate AI agent that never saw the detector code.
157
+ - **blind v1** (420 messages) and **blind v2** (400 messages): English, French, Spanish, Polish, Hebrew and mixed, each written by a separate AI agent that never saw the detector code, each with about 30% of messages containing no identifiers but full of lookalikes (drug doses, embryo grades, billing codes). A second annotator, also blind to the code, corrected the labels.
157
158
 
158
- | recall | v0 | blind v1, first score | blind v1, now |
159
+ Each blind set was scored once with the detectors frozen. v2 was written after the fixes made on v1, so its first score is the re-test:
160
+
161
+ | recall | v0 | v1 first score | v2 first score |
159
162
  |---|---|---|---|
160
- | regex | 61.4% | 36.6% | 50.5% |
161
- | regex + known entities | 85.7% | 54.4% | 67.8% |
162
- | regex + known + local model | 99.1% | 80.1% | **92.2%** |
163
+ | regex | 64.3% | 36.0% | 57.3% |
164
+ | regex + known entities | 88.6% | 54.1% | 80.2% |
165
+ | regex + local model | 98.2% | 77.3% | 87.9% |
166
+ | regex + known + local model | 99.1% | 79.2% | **92.3%** |
163
167
 
164
- | blind v1, regex + known + local model | |
168
+ | v2, regex + known + local model | |
165
169
  |---|---|
166
- | by channel | SMS 94%, chat 91%, email 96%, notes 94%, call transcripts 86% |
167
- | by language | en 91%, fr 94%, es 93%, pl 95%, he 92%, mixed 97% |
168
- | precision | 86.9% |
169
- | messages without identifiers where something was masked | 19% |
170
+ | by channel | SMS 97%, chat 97%, email 98%, notes 95%, call transcripts 83% |
171
+ | by language | en 92%, fr 97%, es 95%, pl 90%, he 85%, mixed 94% |
172
+ | precision | 76.5% (80.6% with the clinic's own details allowed) |
173
+ | messages without identifiers where something was masked | 51% (35% with the clinic's own details allowed) |
170
174
 
171
175
  **How to read this honestly:**
172
- - The first blind score is the fair one: the detectors were frozen before v1 was scored. "Now" includes fixes made after looking at v1's misses, so it is optimistic. A new blind set will re-test it.
173
- - v0 overstated recall by about 30 points. Expect real traffic to score lower than v1 too.
174
- - Known misses: dates spoken in words outside English, some cities, workplaces described in a phrase, and lowercase names that are also words ("joy", "will"). Without the local model, cities and names not in your records are mostly missed.
175
- - Details, label rules and the full tables are in [`benchmark/README.md`](https://github.com/DecisionTreeTechnology/phiproxy/blob/main/benchmark/README.md).
176
+ - v0 overstated recall by about 30 points. Expect real traffic to score lower than the blind sets too: they are AI-written, not written by clinic staff.
177
+ - On v2 the "known entities" rows are optimistic: for some messages the patient record was filled in from the message itself.
178
+ - Over-masking is real. Most of it is the clinic's own contact details (pass them in `Policy(allow=...)`), holiday dates, and staff first names that are also words.
179
+ - Call transcripts are the weakest channel. Without the local model, names not in your records are mostly missed.
180
+ - Details, label rules, the corrections and full tables are in [`benchmark/README.md`](https://github.com/DecisionTreeTechnology/phiproxy/blob/main/benchmark/README.md).
176
181
 
177
182
  ## Deployment shapes
178
183
 
@@ -184,8 +189,8 @@ The vault holds real values. Protect Redis like the PHI store it is: private net
184
189
 
185
190
  ## Roadmap
186
191
 
187
- - A benchmark set written by clinic staff, and a second blind AI-written set to re-test the current numbers
188
- - Gazetteer for cities the model misses; spoken dates in French, Spanish, Polish and Hebrew
192
+ - A benchmark set written by people who work in clinics
193
+ - Better call-transcript handling (fragments, voice-to-text errors)
189
194
  - Smaller and faster local models (distilled or ONNX)
190
195
  - HTTP sidecar; Anthropic `/v1/messages` streaming
191
196
  - Masked logging hook for LiteLLM
@@ -0,0 +1,78 @@
1
+ # Patient-communications PHI benchmark
2
+
3
+ Labelled sets of the kind of text clinics send to LLMs: patient SMS, web chat, phone-call transcripts and voicemails, emails, referral letters, fax and intake text. Most open de-identification datasets cover clinical notes; these cover patient communications. Every person and number in them is invented.
4
+
5
+ ## The sets
6
+
7
+ | set | messages | labelled identifiers | languages | written by |
8
+ |---|---|---|---|---|
9
+ | `patient_comms_v0.jsonl` | 288 | 560 | English | generated from templates by the detector authors |
10
+ | `patient_comms_blind_v1.jsonl` | 420 (128 with no identifiers) | 684 + 63 clinician names | en, fr, es, pl, he, mixed | blind AI author (Claude Opus) |
11
+ | `patient_comms_blind_v2.jsonl` | 400 (122 with no identifiers) | 778 + 106 clinician names | en, fr, es, pl, he, mixed | blind AI author (Claude Sonnet), different brief: more fax, intake, voicemail and email text |
12
+
13
+ **How the blind sets were made.** Each was written by a separate AI agent that had only a brief: the label rules, channels, languages, realistic mess, and a list of things that look like identifiers but are not. It was told not to read the detector code, the tests or the other sets, and it wrote every message from scratch. The sets are independent of the detector code, but they are AI-written, not written by clinic staff.
14
+
15
+ **Second annotator.** A third agent, also without access to code or detector output, reviewed each blind set against the written label rules and listed missing and wrong labels (`blind/audit_v1.jsonl`, `blind/audit_v2.jsonl`; 45 and 149 corrections, mostly unlabelled repeat mentions of names, staff names and spoken dates). `blind/convert.py` applies them. All scores below use the corrected labels.
16
+
17
+ **Frozen scoring.** Before each blind set was scored, the detectors were committed and not changed: v1 against commit 082c2e1, v2 against ab99dc9. Those first scores are the fair ones (`results_blind_*_frozen.json`). After a set was scored it was used to find and fix gaps, so its later scores are optimistic. v2 was the re-test of the fixes made on v1.
18
+
19
+ ## Results (recall)
20
+
21
+ | configuration | v0 | v1 first score | v1 now | v2 first score | v2 now |
22
+ |---|---|---|---|---|---|
23
+ | regex | 64.3% | 36.0% | 54.4% | 57.3% | 57.3% |
24
+ | regex + known entities | 88.6% | 54.1% | 72.1% | 80.2% | 80.2% |
25
+ | regex + local model | 98.2% | 77.3% | 90.2% | **87.9%** | 87.9% |
26
+ | regex + known + local model | 99.1% | 79.2% | 92.4% | **92.3%** | 92.3% |
27
+
28
+ - The fixes made on v1 held up on v2: recall with the local model moved from 79% on v1's first score to 92% on v2's first score.
29
+ - v0 overstates recall by about 30 points, because it was written by the people who wrote the detectors.
30
+ - On v2 the "known entities" rows are optimistic: for some messages the v2 author filled the patient record from the message itself.
31
+ - v2 recall with the local model, by channel: SMS 97%, chat 97%, email 98%, notes 95%, calls 83%. By language: en 92%, fr 97%, es 95%, pl 90%, he 85%, mixed 94%.
32
+
33
+ ## Results (over-masking)
34
+
35
+ | regex + known + local model | precision | messages without identifiers where something was masked |
36
+ |---|---|---|
37
+ | v1 now | 89.7% | 16% |
38
+ | v2 now, default policy | 76.5% | 51% |
39
+ | v2 now, clinic details allowed | 80.6% | 35% |
40
+
41
+ v2 contains many email footers and letterheads with the fictional clinic's own name, phone, fax, address and email. Those are not patient data, so they are not labelled, and phiproxy masks them unless the deployment lists them in `Policy(allow=[...])`. The last row passes the six clinic details in `blind/clinic_v2.json` with `--allow`. Most of the rest is holiday and clinic-closure dates (masked on purpose: a date is never exempted, because it could be a birthday), staff first names that are also words ("Will", "Grace", "Hope"), and staff names without a title.
42
+
43
+ ## Files
44
+
45
+ - `generate.py`: deterministic generator (fixed seed) for `data/patient_comms_v0.jsonl`.
46
+ - `blind/blind_v*.source.jsonl`: the blind sets as written, with inline labels (`[[NAME|jess]]`). `blind/audit_v*.jsonl`: second-annotator corrections. `blind/convert.py v1|v2` builds `data/patient_comms_blind_v*.jsonl`.
47
+ - `score.py`: scores every available configuration and prints recall by type, channel and language, precision, and clean-message masking. `--show-misses N`, `--show-false-positives N` and `--allow clinic.json` help with error analysis. The local-model configurations appear when `phiproxy[gliner]` is installed.
48
+ - `results_*.json`: scores for the built-in configurations.
49
+
50
+ ## Record format
51
+
52
+ ```json
53
+ {
54
+ "id": "sms-00-3",
55
+ "channel": "sms",
56
+ "text": "Hi it's Priya, can I move my bloodwork to Oct 3? My cell is 647-555-0112",
57
+ "spans": [{"start": 8, "end": 13, "type": "NAME"}, ...],
58
+ "known": {"names": ["Priya Natarajan"], "phones": ["647-555-0112"], ...}
59
+ }
60
+ ```
61
+
62
+ `known` is what the clinic system would already hold about the patient. It lets the benchmark measure the known-entity layer.
63
+
64
+ ## Scoring rules
65
+
66
+ - A labelled span is **caught** when detected spans cover at least 80% of its characters. The detected type does not have to match, because catching a phone number as an `ID` still protects it.
67
+ - Scoring uses the default policy, under which clinician names pass through. Clinician names (`PROVIDER`) are therefore not counted in recall, and masking one counts as a false positive.
68
+ - A detected span is **correct** when it overlaps a labelled identifier. **Clean messages masked** is the share of messages without identifiers in which something was masked anyway.
69
+ - What counts as an identifier follows HIPAA Safe Harbor (45 CFR 164.514(b)(2)): names of patients and relatives, all date elements except year, ages over 89, geographic units smaller than a state or province (street, city, postal code), contact details, and record, account, plan and other identifying numbers. Workplaces are labelled too because they can identify a person.
70
+
71
+ ## Known limits
72
+
73
+ - The blind sets are AI-written. Real messages are messier, and a set written by people who work in clinics is still the goal.
74
+ - Call transcripts are the weakest channel (83% on v2): names and numbers spoken in fragments, voice-to-text errors ("for one six" for 416), and repeated spellings.
75
+ - Still missed: some lowercase names that are also words ("joy", "will"), workplaces described in a phrase, Hebrew abbreviations of months, numeric spoken dates ("oh four oh nine eighty two").
76
+ - Over-masking costs answer quality, not privacy, but it is real: see the table above.
77
+
78
+ Contributions are welcome, especially messages written by people who work in clinics (see ../CONTRIBUTING.md). All data must be synthetic.
@@ -0,0 +1,45 @@
1
+ {"id": "en-sms-056", "action": "add", "type": "NAME", "surface": "Faith", "occurrence": 1, "reason": "friend first name"}
2
+ {"id": "en-sms-057", "action": "add", "type": "PROVIDER", "surface": "Holly", "occurrence": 1, "reason": "staff first name"}
3
+ {"id": "en-chat-107", "action": "add", "type": "PHONE", "surface": "647 555 0113", "occurrence": 1, "reason": "unmarked first copy of phone"}
4
+ {"id": "en-chat-125", "action": "add", "type": "DATE", "surface": "12th", "occurrence": 1, "reason": "appointment date"}
5
+ {"id": "en-chat-129", "action": "add", "type": "AGE", "surface": "92", "occurrence": 1, "reason": "age over 89"}
6
+ {"id": "en-call-150", "action": "add", "type": "HEALTH_CARD", "surface": "four seven two nine dash one one eight dash three oh five and then k t", "occurrence": 1, "reason": "spoken health card"}
7
+ {"id": "en-call-153", "action": "add", "type": "NAME", "surface": "mark", "occurrence": 1, "reason": "caller first name"}
8
+ {"id": "en-call-153", "action": "add", "type": "NAME", "surface": "hope", "occurrence": 1, "reason": "patient first name"}
9
+ {"id": "en-call-155", "action": "add", "type": "POSTAL_CODE", "surface": "m five v two t six", "occurrence": 1, "reason": "spoken postal"}
10
+ {"id": "en-call-155", "action": "add", "type": "POSTAL_CODE", "surface": "m5v 2t6", "occurrence": 1, "reason": "postal"}
11
+ {"id": "en-call-157", "action": "add", "type": "DATE", "surface": "the twentieth", "occurrence": 1, "reason": "spoken date"}
12
+ {"id": "en-call-160", "action": "add", "type": "DONOR_ID", "surface": "c b two zero four one eight s three", "occurrence": 1, "reason": "repeated donor id"}
13
+ {"id": "en-call-161", "action": "add", "type": "DATE", "surface": "the eighth", "occurrence": 1, "reason": "spoken date"}
14
+ {"id": "en-call-161", "action": "add", "type": "DATE", "surface": "the eighteenth", "occurrence": 1, "reason": "spoken date"}
15
+ {"id": "en-call-176", "action": "add", "type": "MRN", "surface": "zero zero four eight two one three", "occurrence": 1, "reason": "spoken MRN"}
16
+ {"id": "en-call-176", "action": "add", "type": "NAME", "surface": "whitney", "occurrence": 1, "reason": "patient first name"}
17
+ {"id": "en-call-185", "action": "add", "type": "DATE", "surface": "the eighteenth", "occurrence": 1, "reason": "spoken appt date"}
18
+ {"id": "en-call-196", "action": "add", "type": "NAME", "surface": "beth", "occurrence": 2, "reason": "name in agent turn"}
19
+ {"id": "en-email-197", "action": "add", "type": "NAME", "surface": "Jess", "occurrence": 2, "reason": "signature"}
20
+ {"id": "en-email-203", "action": "add", "type": "NAME", "surface": "Lauren", "occurrence": 1, "reason": "signature"}
21
+ {"id": "en-email-207", "action": "add", "type": "DATE", "surface": "Dec 1", "occurrence": 1, "reason": "shipping date tied to patient"}
22
+ {"id": "en-email-208", "action": "add", "type": "NAME", "surface": "Carmen", "occurrence": 1, "reason": "signature"}
23
+ {"id": "en-email-215", "action": "add", "type": "NAME", "surface": "Mireille", "occurrence": 1, "reason": "signature"}
24
+ {"id": "en-email-220", "action": "add", "type": "NAME", "surface": "Bill", "occurrence": 1, "reason": "nickname of patient husband"}
25
+ {"id": "en-note-246", "action": "add", "type": "NAME", "surface": "kay", "occurrence": 1, "reason": "nickname"}
26
+ {"id": "en-note-254", "action": "add", "type": "AGE", "surface": "91", "occurrence": 1, "reason": "age over 89"}
27
+ {"id": "en-note-255", "action": "add", "type": "PROVIDER", "surface": "Holly", "occurrence": 2, "reason": "staff name"}
28
+ {"id": "en-note-255", "action": "add", "type": "PROVIDER", "surface": "Hope Bergman", "occurrence": 1, "reason": "staff name"}
29
+ {"id": "en-note-260", "action": "add", "type": "NAME", "surface": "Cassandra Boyle", "occurrence": 1, "reason": "patient full name"}
30
+ {"id": "fr-chat-023", "action": "add", "type": "ADDRESS", "surface": "Ottawa", "occurrence": 1, "reason": "partner location"}
31
+ {"id": "fr-call-029", "action": "add", "type": "HEALTH_CARD", "surface": "t r e m neuf zéro un deux trois quatre cinq six", "occurrence": 1, "reason": "spoken card number"}
32
+ {"id": "es-call-025", "action": "add", "type": "NAME", "surface": "hernández", "occurrence": 2, "reason": "repeated surname"}
33
+ {"id": "es-call-025", "action": "add", "type": "NAME", "surface": "h e r n á n d e z", "occurrence": 1, "reason": "spelled surname"}
34
+ {"id": "es-email-034", "action": "add", "type": "NAME", "surface": "Laura", "occurrence": 1, "reason": "signature"}
35
+ {"id": "en-sms-008", "action": "remove", "type": "ADDRESS", "surface": "Dundas St", "occurrence": 1, "reason": "clinic parking lot"}
36
+ {"id": "en-sms-022", "action": "remove", "type": "ADDRESS", "surface": "downtown location", "occurrence": 1, "reason": "clinic location"}
37
+ {"id": "en-sms-026", "action": "remove", "type": "EMPLOYER", "surface": "Mount Sinai Hospital lab", "occurrence": 1, "reason": "lab, not workplace"}
38
+ {"id": "en-chat-111", "action": "remove", "type": "DATE", "surface": "Sunday", "occurrence": 1, "reason": "bare weekday"}
39
+ {"id": "en-chat-114", "action": "remove", "type": "DATE", "surface": "yesterday", "occurrence": 1, "reason": "relative word"}
40
+ {"id": "en-email-212", "action": "remove", "type": "DATE", "surface": "Saturday, November 22", "occurrence": 1, "reason": "clinic event"}
41
+ {"id": "en-email-215", "action": "remove", "type": "EMPLOYER", "surface": "Health Sciences North", "occurrence": 1, "reason": "lab, not workplace"}
42
+ {"id": "en-email-218", "action": "remove", "type": "EMAIL", "surface": "billing@example.com", "occurrence": 1, "reason": "clinic email"}
43
+ {"id": "en-email-218", "action": "retype", "from": "NAME", "type": "PROVIDER", "surface": "Ellen Brandt", "occurrence": 1, "reason": "billing staff"}
44
+ {"id": "en-email-219", "action": "retype", "from": "NAME", "type": "PROVIDER", "surface": "Dr. Alan Brandt", "occurrence": 1, "reason": "clinician"}
45
+ {"id": "es-call-030", "action": "retype", "from": "NAME", "type": "PROVIDER", "surface": "doctora ramos", "occurrence": 1, "reason": "clinician"}
@@ -0,0 +1,149 @@
1
+ {"id":"v2-en-note-001","action":"add","type":"DATE","surface":"Mar 03 2026","occurrence":1,"reason":"fax header date tied to patient referral"}
2
+ {"id":"v2-en-note-004","action":"add","type":"EMPLOYER","surface":"Northern Tool and Die","occurrence":1,"reason":"employer group of member"}
3
+ {"id":"v2-en-note-009","action":"add","type":"NAME","surface":"Dev","occurrence":2,"reason":"partner first name"}
4
+ {"id":"v2-en-note-011","action":"add","type":"NAME","surface":"SM1TH, JENN1FER L.","occurrence":1,"reason":"OCR patient name"}
5
+ {"id":"v2-en-note-013","action":"add","type":"PROVIDER","surface":"Vasquez","occurrence":2,"reason":"Dr. Vasquez salutation"}
6
+ {"id":"v2-en-note-013","action":"add","type":"NAME","surface":"Reyes","occurrence":2,"reason":"patient surname Ms. Reyes"}
7
+ {"id":"v2-en-note-018","action":"remove","type":"PROVIDER","surface":"Hirsch & Dale","occurrence":1,"reason":"law firm, not clinician/staff"}
8
+ {"id":"v2-en-note-019","action":"add","type":"PROVIDER","surface":"Grace","occurrence":1,"reason":"clinic staff first name"}
9
+ {"id":"v2-en-note-019","action":"add","type":"PROVIDER","surface":"Will","occurrence":1,"reason":"clinic staff first name"}
10
+ {"id":"v2-en-note-019","action":"add","type":"PROVIDER","surface":"Hope","occurrence":1,"reason":"clinic staff first name"}
11
+ {"id":"v2-en-note-019","action":"add","type":"PROVIDER","surface":"Bill","occurrence":1,"reason":"clinic staff first name"}
12
+ {"id":"v2-en-note-025","action":"add","type":"PROVIDER","surface":"Achebe","occurrence":1,"reason":"Dr. Achebe"}
13
+ {"id":"v2-en-note-027","action":"add","type":"DATE","surface":"1989-03-12","occurrence":1,"reason":"chart DOB"}
14
+ {"id":"v2-en-note-033","action":"add","type":"ADDRESS","surface":"Kyiv","occurrence":1,"reason":"place of birth, city person comes from"}
15
+ {"id":"v2-en-note-040","action":"retype","from":"NAME","type":"PROVIDER","surface":"Raymond Sutherland","occurrence":1,"reason":"billing contact staff"}
16
+ {"id":"v2-en-em-001","action":"add","type":"PROVIDER","surface":"Maria","occurrence":1,"reason":"nurse addressed in greeting"}
17
+ {"id":"v2-en-em-001","action":"add","type":"PROVIDER","surface":"Maria Santos","occurrence":1,"reason":"nurse in quoted header"}
18
+ {"id":"v2-en-em-001","action":"add","type":"DATE","surface":"Oct 12, 2026","occurrence":1,"reason":"quote header date"}
19
+ {"id":"v2-en-em-001","action":"add","type":"NAME","surface":"Alicia","occurrence":2,"reason":"patient signature"}
20
+ {"id":"v2-en-em-004","action":"add","type":"NAME","surface":"Ianelli","occurrence":2,"reason":"Ms. Ianelli patient"}
21
+ {"id":"v2-en-em-005","action":"add","type":"PROVIDER","surface":"Faith","occurrence":1,"reason":"staff first name"}
22
+ {"id":"v2-en-em-005","action":"add","type":"PROVIDER","surface":"Hope","occurrence":1,"reason":"staff first name"}
23
+ {"id":"v2-en-em-005","action":"add","type":"PROVIDER","surface":"Joy","occurrence":1,"reason":"staff first name"}
24
+ {"id":"v2-en-em-005","action":"add","type":"PROVIDER","surface":"Grace","occurrence":1,"reason":"staff first name"}
25
+ {"id":"v2-en-em-005","action":"add","type":"PROVIDER","surface":"Will","occurrence":1,"reason":"staff first name"}
26
+ {"id":"v2-en-em-005","action":"add","type":"PROVIDER","surface":"Bill","occurrence":1,"reason":"staff signature"}
27
+ {"id":"v2-en-em-006","action":"add","type":"NAME","surface":"Karen","occurrence":2,"reason":"patient signature"}
28
+ {"id":"v2-en-em-006","action":"add","type":"DATE","surface":"Nov 2, 2026","occurrence":1,"reason":"quoted message date"}
29
+ {"id":"v2-en-em-006","action":"add","type":"DATE","surface":"Nov 2, 2026","occurrence":2,"reason":"quoted message date"}
30
+ {"id":"v2-en-em-009","action":"add","type":"PROVIDER","surface":"Hartwell","occurrence":1,"reason":"Dr. Hartwell"}
31
+ {"id":"v2-en-em-011","action":"add","type":"NAME","surface":"Fiona","occurrence":2,"reason":"signature"}
32
+ {"id":"v2-en-em-012","action":"add","type":"ID","surface":"jsingh1988","occurrence":1,"reason":"personal portal username"}
33
+ {"id":"v2-en-em-015","action":"add","type":"PROVIDER","surface":"Lal","occurrence":1,"reason":"Dr. Lal"}
34
+ {"id":"v2-en-em-017","action":"add","type":"NAME","surface":"Gabriel Santos-Pereira","occurrence":2,"reason":"signature"}
35
+ {"id":"v2-en-em-018","action":"add","type":"NAME","surface":"Rebecca Mullins","occurrence":1,"reason":"patient full name"}
36
+ {"id":"v2-en-em-020","action":"add","type":"NAME","surface":"Wanda","occurrence":2,"reason":"signature"}
37
+ {"id":"v2-en-em-022","action":"add","type":"DATE","surface":"March 6","occurrence":1,"reason":"return-to-work date"}
38
+ {"id":"v2-en-em-031","action":"add","type":"NAME","surface":"Jonathan Lee","occurrence":1,"reason":"patient name"}
39
+ {"id":"v2-en-em-031","action":"add","type":"NAME","surface":"John Lee","occurrence":1,"reason":"patient name"}
40
+ {"id":"v2-en-call-002","action":"add","type":"DATE","surface":"september thirtieth nineteen eighty eight","occurrence":1,"reason":"spoken DOB by caller"}
41
+ {"id":"v2-en-call-002","action":"add","type":"MRN","surface":"zero zero seven one two four five","occurrence":1,"reason":"spoken file number by caller"}
42
+ {"id":"v2-en-call-003","action":"add","type":"PHONE","surface":"four one six, five five five, oh one three seven","occurrence":1,"reason":"spoken repeat of phone"}
43
+ {"id":"v2-en-call-004","action":"add","type":"NAME","surface":"R-A-C-H-E-L","occurrence":1,"reason":"spelled first name"}
44
+ {"id":"v2-en-call-004","action":"add","type":"NAME","surface":"O-Y-E-L-A-R-A-N","occurrence":1,"reason":"spelled surname"}
45
+ {"id":"v2-en-call-006","action":"add","type":"NAME","surface":"E Z E hyphen W I L L I A M S","occurrence":1,"reason":"spelled surname"}
46
+ {"id":"v2-en-call-006","action":"add","type":"NAME","surface":"Eze-Williams","occurrence":2,"reason":"Ms. Eze-Williams"}
47
+ {"id":"v2-en-call-006","action":"add","type":"PROVIDER","surface":"Lal","occurrence":1,"reason":"Dr. Lal"}
48
+ {"id":"v2-en-call-008","action":"add","type":"PROVIDER","surface":"Jon","occurrence":1,"reason":"billing staff self-intro"}
49
+ {"id":"v2-en-call-008","action":"add","type":"ACCOUNT","surface":"four four zero two one nine","occurrence":1,"reason":"invoice number tied to person"}
50
+ {"id":"v2-en-call-009","action":"add","type":"PROVIDER","surface":"Faith","occurrence":1,"reason":"lab staff"}
51
+ {"id":"v2-en-call-009","action":"add","type":"PROVIDER","surface":"Faith","occurrence":2,"reason":"lab staff"}
52
+ {"id":"v2-en-call-013","action":"add","type":"ADDRESS","surface":"fifty-five, that's five five, Riverbend Drive Northwest","occurrence":1,"reason":"spoken street address"}
53
+ {"id":"v2-en-call-013","action":"add","type":"ADDRESS","surface":"Calgary","occurrence":1,"reason":"city of person"}
54
+ {"id":"v2-en-call-013","action":"add","type":"ADDRESS","surface":"Calgary","occurrence":2,"reason":"city of person"}
55
+ {"id":"v2-en-call-013","action":"add","type":"POSTAL_CODE","surface":"T three A, four B seven","occurrence":1,"reason":"spoken postal code"}
56
+ {"id":"v2-en-call-013","action":"add","type":"POSTAL_CODE","surface":"T3A 4B7","occurrence":1,"reason":"postal code"}
57
+ {"id":"v2-en-call-014","action":"add","type":"ID","surface":"C F seven seven eight one","occurrence":1,"reason":"spoken file number"}
58
+ {"id":"v2-en-call-016","action":"remove","type":"ADDRESS","surface":"Lawrence Avenue","occurrence":1,"reason":"pharmacy location, not where person lives"}
59
+ {"id":"v2-en-call-018","action":"add","type":"HEALTH_CARD","surface":"four seven two nine one one zero six three K T","occurrence":1,"reason":"spoken health card"}
60
+ {"id":"v2-en-call-018","action":"add","type":"HEALTH_CARD","surface":"Four seven two nine, one one zero, three six","occurrence":1,"reason":"corrected spoken health card"}
61
+ {"id":"v2-en-call-019","action":"add","type":"NAME","surface":"T Y R O N E","occurrence":1,"reason":"spelled name"}
62
+ {"id":"v2-en-call-019","action":"add","type":"NAME","surface":"Bishop","occurrence":2,"reason":"surname repeated"}
63
+ {"id":"v2-en-call-019","action":"add","type":"DATE","surface":"the second","occurrence":1,"reason":"specific day of month for chemo start"}
64
+ {"id":"v2-en-call-020","action":"remove","type":"EMPLOYER","surface":"Northgate Medical","occurrence":1,"reason":"referring clinician's practice, not a patient employer"}
65
+ {"id":"v2-en-call-023","action":"add","type":"DONOR_ID","surface":"D twenty thousand seven seventy-one","occurrence":1,"reason":"spoken donor straw id"}
66
+ {"id":"v2-en-call-025","action":"add","type":"NAME","surface":"Dorothea Kapoor","occurrence":1,"reason":"full name"}
67
+ {"id":"v2-en-call-025","action":"add","type":"PHONE","surface":"four one six... five five five... one one seven","occurrence":1,"reason":"spoken phone"}
68
+ {"id":"v2-en-call-029","action":"remove","type":"EMPLOYER","surface":"Shoppers-Rx Dundas","occurrence":1,"reason":"pharmacy, not patient employer"}
69
+ {"id":"v2-en-call-030","action":"add","type":"PROVIDER","surface":"will","occurrence":1,"reason":"lab staff first name"}
70
+ {"id":"v2-en-call-030","action":"add","type":"PROVIDER","surface":"Will","occurrence":1,"reason":"lab staff"}
71
+ {"id":"v2-en-call-030","action":"add","type":"PROVIDER","surface":"Will","occurrence":2,"reason":"billing staff"}
72
+ {"id":"v2-en-call-030","action":"add","type":"PROVIDER","surface":"Will","occurrence":3,"reason":"lab staff"}
73
+ {"id":"v2-en-call-032","action":"add","type":"NAME","surface":"Nguyen","occurrence":1,"reason":"spoken patient name"}
74
+ {"id":"v2-en-call-032","action":"add","type":"NAME","surface":"N G U Y E N","occurrence":1,"reason":"spelled surname"}
75
+ {"id":"v2-en-call-032","action":"add","type":"NAME","surface":"Thi Lan","occurrence":1,"reason":"given name"}
76
+ {"id":"v2-en-call-032","action":"add","type":"DATE","surface":"twenty-two eleven nineteen ninety","occurrence":1,"reason":"spoken DOB"}
77
+ {"id":"v2-en-call-032","action":"add","type":"DATE","surface":"twenty-second of November nineteen ninety","occurrence":1,"reason":"spoken DOB"}
78
+ {"id":"v2-en-call-039","action":"add","type":"DATE","surface":"one four, oh seven, nineteen eighty-nine","occurrence":1,"reason":"spoken DOB"}
79
+ {"id":"v2-en-call-039","action":"add","type":"DATE","surface":"fourteenth of July","occurrence":1,"reason":"spoken DOB"}
80
+ {"id":"v2-en-sms-002","action":"add","type":"NAME","surface":"marcus","occurrence":1,"reason":"sender first name"}
81
+ {"id":"v2-en-sms-004","action":"add","type":"NAME","surface":"Jess Tavares","occurrence":1,"reason":"patient name"}
82
+ {"id":"v2-en-sms-008","action":"add","type":"PROVIDER","surface":"Hartwell","occurrence":1,"reason":"Dr. Hartwell"}
83
+ {"id":"v2-en-sms-025","action":"add","type":"NAME","surface":"Lena Fischer","occurrence":1,"reason":"signature name"}
84
+ {"id":"v2-en-em-006","action":"add","type":"DATE","surface":"the 20th","occurrence":1,"reason":"day of month for embryo move"}
85
+ {"id":"v2-en-em-018","action":"add","type":"DATE","surface":"the 18th","occurrence":1,"reason":"transfer date"}
86
+ {"id":"v2-en-em-018","action":"add","type":"DATE","surface":"the 20th","occurrence":1,"reason":"requested transfer date"}
87
+ {"id":"v2-en-sms-017","action":"add","type":"DATE","surface":"the 20th","occurrence":1,"reason":"appointment day, consistent with 'the 12th'"}
88
+ {"id":"v2-en-sms-029","action":"add","type":"DATE","surface":"20th","occurrence":1,"reason":"appointment day of month"}
89
+ {"id":"v2-en-chat-001","action":"add","type":"NAME","surface":"Ruiz","occurrence":1,"reason":"surname of Dana Ruiz"}
90
+ {"id":"v2-en-chat-001","action":"add","type":"MRN","surface":"0071245","occurrence":1,"reason":"file number given by visitor"}
91
+ {"id":"v2-en-chat-004","action":"add","type":"NAME","surface":"Tamara","occurrence":2,"reason":"patient first name in agent reply"}
92
+ {"id":"v2-en-chat-009","action":"add","type":"DATE","surface":"Nov 24","occurrence":1,"reason":"patient appointment date"}
93
+ {"id":"v2-en-chat-009","action":"add","type":"DATE","surface":"Nov 24","occurrence":2,"reason":"patient appointment date"}
94
+ {"id":"v2-en-chat-016","action":"remove","type":"ID","surface":"E12345678","occurrence":1,"reason":"generic example format, not a person's number"}
95
+ {"id":"v2-en-call-042","action":"add","type":"PROVIDER","surface":"Grace","occurrence":2,"reason":"staff first name in greeting"}
96
+ {"id":"v2-en-call-042","action":"add","type":"DATE","surface":"fourth of April, sixty-eight","occurrence":1,"reason":"spoken DOB"}
97
+ {"id":"v2-en-call-050","action":"add","type":"ADDRESS","surface":"two twenty-one Pine Street, Apartment six","occurrence":1,"reason":"person's old street address"}
98
+ {"id":"v2-en-call-050","action":"add","type":"ADDRESS","surface":"Thunder Bay","occurrence":1,"reason":"person's city"}
99
+ {"id":"v2-en-call-050","action":"add","type":"POSTAL_CODE","surface":"P seven B, three C two","occurrence":1,"reason":"spoken postal code"}
100
+ {"id":"v2-en-call-050","action":"add","type":"POSTAL_CODE","surface":"P zero T one A zero","occurrence":1,"reason":"spoken postal code"}
101
+ {"id":"v2-en-call-053","action":"add","type":"NAME","surface":"K O W","occurrence":1,"reason":"spelled surname letters"}
102
+ {"id":"v2-en-call-053","action":"add","type":"NAME","surface":"K O W","occurrence":2,"reason":"spelled surname letters"}
103
+ {"id":"v2-en-call-053","action":"add","type":"DATE","surface":"seven four eighty-seven","occurrence":1,"reason":"spoken DOB"}
104
+ {"id":"v2-en-call-053","action":"add","type":"DATE","surface":"April seventh","occurrence":1,"reason":"corrected spoken DOB"}
105
+ {"id":"v2-en-call-054","action":"add","type":"PHONE","surface":"six oh four, five five five, oh one three four","occurrence":1,"reason":"spoken patient phone"}
106
+ {"id":"v2-en-call-056","action":"add","type":"NAME","surface":"B-R-E-N-D-A K-O-W-A-L-S-K-I","occurrence":1,"reason":"spelled name"}
107
+ {"id":"v2-en-call-059","action":"add","type":"NAME","surface":"J-A-M-A-L","occurrence":1,"reason":"spelled name"}
108
+ {"id":"v2-en-note-044","action":"add","type":"NAME","surface":"Tremblay","occurrence":1,"reason":"Ms. Tremblay patient"}
109
+ {"id":"v2-en-note-044","action":"add","type":"PROVIDER","surface":"Lal","occurrence":1,"reason":"Dear Dr. Lal"}
110
+ {"id":"v2-en-note-057","action":"add","type":"NAME","surface":"Samir","occurrence":2,"reason":"patient first name repeated"}
111
+ {"id":"v2-en-em-035","action":"add","type":"PROVIDER","surface":"Will","occurrence":1,"reason":"lab staff signature"}
112
+ {"id":"v2-en-em-037","action":"add","type":"DATE","surface":"2027-02-20","occurrence":1,"reason":"travel end date"}
113
+ {"id":"v2-en-sms-039","action":"add","type":"PROVIDER","surface":"Lal","occurrence":1,"reason":"Dr Lal"}
114
+ {"id":"v2-en-note-060","action":"add","type":"PROVIDER","surface":"Hope","occurrence":1,"reason":"staff first name"}
115
+ {"id":"v2-en-note-060","action":"add","type":"PROVIDER","surface":"Faith","occurrence":1,"reason":"staff first name"}
116
+ {"id":"v2-en-note-060","action":"add","type":"PROVIDER","surface":"Grace","occurrence":1,"reason":"staff first name"}
117
+ {"id":"v2-en-sms-041","action":"add","type":"PROVIDER","surface":"Will","occurrence":1,"reason":"staff covering"}
118
+ {"id":"v2-en-sms-041","action":"add","type":"PROVIDER","surface":"Grace","occurrence":1,"reason":"staff first name"}
119
+ {"id":"v2-fr-call-002","action":"add","type":"DATE","surface":"le vingt-deux novembre mille neuf cent quatre-vingt-dix","occurrence":1,"reason":"spoken DOB by caller"}
120
+ {"id":"v2-fr-call-005","action":"add","type":"NAME","surface":"N-G-U-Y-E-N","occurrence":1,"reason":"spelled surname"}
121
+ {"id":"v2-fr-call-005","action":"add","type":"NAME","surface":"Thi Lan","occurrence":1,"reason":"given name"}
122
+ {"id":"v2-fr-call-006","action":"add","type":"PHONE","surface":"quatre un six, cinq cinq cinq, zéro un un sept","occurrence":1,"reason":"spoken phone"}
123
+ {"id":"v2-fr-chat-002","action":"add","type":"NAME","surface":"Hélène","occurrence":2,"reason":"patient first name by agent"}
124
+ {"id":"v2-fr-em-001","action":"add","type":"NAME","surface":"Bouchard","occurrence":2,"reason":"Mme Bouchard patient"}
125
+ {"id":"v2-fr-em-004","action":"add","type":"DATE","surface":"3 novembre 2026","occurrence":1,"reason":"quoted message date"}
126
+ {"id":"v2-fr-em-007","action":"add","type":"DATE","surface":"24 novembre","occurrence":1,"reason":"patient appointment date"}
127
+ {"id":"v2-fr-call-009","action":"remove","type":"EMPLOYER","surface":"Jean Coutu","occurrence":1,"reason":"pharmacy, not a patient employer"}
128
+ {"id":"v2-fr-em-010","action":"retype","from":"PROVIDER","type":"NAME","surface":"Hélène Marcoux","occurrence":1,"reason":"lawyer, not clinician or clinic staff"}
129
+ {"id":"v2-es-call-002","action":"add","type":"DATE","surface":"el quince de agosto de mil novecientos ochenta y nueve","occurrence":1,"reason":"spoken DOB by caller"}
130
+ {"id":"v2-es-call-004","action":"add","type":"NAME","surface":"G-U-T-I-E-R-R-E-Z","occurrence":1,"reason":"spelled surname"}
131
+ {"id":"v2-es-call-007","action":"add","type":"PHONE","surface":"cuatro uno seis... cinco cinco cinco... uno uno siete","occurrence":1,"reason":"spoken phone"}
132
+ {"id":"v2-es-chat-002","action":"add","type":"NAME","surface":"Yesenia","occurrence":2,"reason":"patient first name by agent"}
133
+ {"id":"v2-es-em-001","action":"add","type":"NAME","surface":"Beltrán","occurrence":2,"reason":"Sra. Beltrán patient"}
134
+ {"id":"v2-es-call-010","action":"remove","type":"EMPLOYER","surface":"Farmacia La Esperanza","occurrence":1,"reason":"pharmacy, not a patient employer"}
135
+ {"id":"v2-es-em-007","action":"add","type":"DATE","surface":"8 de septiembre","occurrence":1,"reason":"restated DOB"}
136
+ {"id":"v2-pl-call-002","action":"add","type":"DATE","surface":"czwartego kwietnia tysiąc dziewięćset osiemdziesiątego siódmego","occurrence":1,"reason":"spoken DOB by caller"}
137
+ {"id":"v2-pl-call-005","action":"add","type":"NAME","surface":"S-Z-C-Z-E-P-A-Ń-S-K-A","occurrence":1,"reason":"spelled surname"}
138
+ {"id":"v2-pl-call-007","action":"add","type":"PHONE","surface":"cztery jeden sześć... pięć pięć pięć... zero jeden jeden siedem","occurrence":1,"reason":"spoken phone"}
139
+ {"id":"v2-pl-chat-002","action":"add","type":"NAME","surface":"Justyno","occurrence":1,"reason":"patient first name (vocative) by agent"}
140
+ {"id":"v2-pl-em-001","action":"add","type":"NAME","surface":"Bielecką","occurrence":1,"reason":"patient surname, declined form"}
141
+ {"id":"v2-pl-call-009","action":"remove","type":"EMPLOYER","surface":"Cefarm","occurrence":1,"reason":"pharmacy, not a patient employer"}
142
+ {"id":"v2-pl-em-006","action":"add","type":"DATE","surface":"8 września","occurrence":1,"reason":"restated DOB"}
143
+ {"id":"v2-he-call-005","action":"add","type":"NAME","surface":"ג-ו-ל-ד-ב-ר-ג","occurrence":1,"reason":"spelled surname"}
144
+ {"id":"v2-he-chat-002","action":"add","type":"NAME","surface":"מורן","occurrence":2,"reason":"patient first name by agent"}
145
+ {"id":"v2-he-em-001","action":"add","type":"NAME","surface":"ברק","occurrence":2,"reason":"patient surname in body"}
146
+ {"id":"v2-mixed-chat-002","action":"add","type":"NAME","surface":"Marisol","occurrence":2,"reason":"patient first name by agent"}
147
+ {"id":"v2-mixed-call-003","action":"add","type":"NAME","surface":"H-É-L-È-N-E","occurrence":1,"reason":"spelled first name"}
148
+ {"id":"v2-mixed-note-002","action":"add","type":"DATE","surface":"3 April","occurrence":1,"reason":"clarified DOB"}
149
+ {"id":"v2-mixed-note-002","action":"add","type":"DATE","surface":"March 4","occurrence":1,"reason":"erroneous DOB entry"}