polanon 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. polanon-0.1.0/CHANGELOG.md +56 -0
  2. polanon-0.1.0/CONTRIBUTING.md +47 -0
  3. polanon-0.1.0/LICENSE +21 -0
  4. polanon-0.1.0/MANIFEST.in +8 -0
  5. polanon-0.1.0/PKG-INFO +376 -0
  6. polanon-0.1.0/README.md +336 -0
  7. polanon-0.1.0/SECURITY.md +18 -0
  8. polanon-0.1.0/benchmarks/README.md +93 -0
  9. polanon-0.1.0/benchmarks/compare_presidio.py +208 -0
  10. polanon-0.1.0/benchmarks/demo.py +45 -0
  11. polanon-0.1.0/benchmarks/gold/cv.json +65 -0
  12. polanon-0.1.0/benchmarks/gold/czat.json +41 -0
  13. polanon-0.1.0/benchmarks/gold/faktura.json +69 -0
  14. polanon-0.1.0/benchmarks/gold/invoice_se.json +21 -0
  15. polanon-0.1.0/benchmarks/gold/invoice_uk.json +25 -0
  16. polanon-0.1.0/benchmarks/gold/mail_klienta.json +49 -0
  17. polanon-0.1.0/benchmarks/gold/notatka_wewnetrzna.json +73 -0
  18. polanon-0.1.0/benchmarks/gold/rechnung_ch.json +20 -0
  19. polanon-0.1.0/benchmarks/gold/umowa.json +77 -0
  20. polanon-0.1.0/benchmarks/gold/zgloszenie_it.json +77 -0
  21. polanon-0.1.0/benchmarks/run.py +253 -0
  22. polanon-0.1.0/benchmarks/samples/README.md +15 -0
  23. polanon-0.1.0/benchmarks/samples/cv.txt +18 -0
  24. polanon-0.1.0/benchmarks/samples/czat.txt +16 -0
  25. polanon-0.1.0/benchmarks/samples/faktura.txt +22 -0
  26. polanon-0.1.0/benchmarks/samples/invoice_se.txt +27 -0
  27. polanon-0.1.0/benchmarks/samples/invoice_uk.txt +38 -0
  28. polanon-0.1.0/benchmarks/samples/mail_klienta.txt +19 -0
  29. polanon-0.1.0/benchmarks/samples/notatka_wewnetrzna.txt +18 -0
  30. polanon-0.1.0/benchmarks/samples/rechnung_ch.txt +32 -0
  31. polanon-0.1.0/benchmarks/samples/umowa.txt +19 -0
  32. polanon-0.1.0/benchmarks/samples/zgloszenie_it.txt +19 -0
  33. polanon-0.1.0/docs/THREAT_MODEL.md +98 -0
  34. polanon-0.1.0/examples/anthropic_chat.py +107 -0
  35. polanon-0.1.0/examples/openai_chat.py +78 -0
  36. polanon-0.1.0/polanon/__init__.py +255 -0
  37. polanon-0.1.0/polanon/__main__.py +90 -0
  38. polanon-0.1.0/polanon/_detection.py +79 -0
  39. polanon-0.1.0/polanon/_matching.py +624 -0
  40. polanon-0.1.0/polanon/_view.py +102 -0
  41. polanon-0.1.0/polanon/allowlist.py +246 -0
  42. polanon-0.1.0/polanon/guards/__init__.py +0 -0
  43. polanon-0.1.0/polanon/guards/base.py +22 -0
  44. polanon-0.1.0/polanon/guards/company_pl.py +214 -0
  45. polanon-0.1.0/polanon/guards/denylist.py +71 -0
  46. polanon-0.1.0/polanon/guards/international.py +771 -0
  47. polanon-0.1.0/polanon/guards/names_pl.py +823 -0
  48. polanon-0.1.0/polanon/guards/ner.py +427 -0
  49. polanon-0.1.0/polanon/guards/recognizers_pl.py +562 -0
  50. polanon-0.1.0/polanon/guards/secrets.py +162 -0
  51. polanon-0.1.0/polanon/judge/__init__.py +30 -0
  52. polanon-0.1.0/polanon/judge/base.py +39 -0
  53. polanon-0.1.0/polanon/judge/mock.py +27 -0
  54. polanon-0.1.0/polanon/judge/ollama.py +67 -0
  55. polanon-0.1.0/polanon/judge/parsing.py +121 -0
  56. polanon-0.1.0/polanon/judge/prompt.py +48 -0
  57. polanon-0.1.0/polanon/judge/remote.py +76 -0
  58. polanon-0.1.0/polanon/output_sweep.py +79 -0
  59. polanon-0.1.0/polanon/pipeline.py +425 -0
  60. polanon-0.1.0/polanon/policy.py +67 -0
  61. polanon-0.1.0/polanon/py.typed +0 -0
  62. polanon-0.1.0/polanon/stream_restore.py +46 -0
  63. polanon-0.1.0/polanon/types.py +179 -0
  64. polanon-0.1.0/polanon.egg-info/PKG-INFO +376 -0
  65. polanon-0.1.0/polanon.egg-info/SOURCES.txt +90 -0
  66. polanon-0.1.0/polanon.egg-info/dependency_links.txt +1 -0
  67. polanon-0.1.0/polanon.egg-info/entry_points.txt +2 -0
  68. polanon-0.1.0/polanon.egg-info/requires.txt +15 -0
  69. polanon-0.1.0/polanon.egg-info/top_level.txt +1 -0
  70. polanon-0.1.0/pyproject.toml +127 -0
  71. polanon-0.1.0/setup.cfg +4 -0
  72. polanon-0.1.0/tests/__init__.py +0 -0
  73. polanon-0.1.0/tests/test_allowlist.py +245 -0
  74. polanon-0.1.0/tests/test_benchmark.py +123 -0
  75. polanon-0.1.0/tests/test_cli.py +104 -0
  76. polanon-0.1.0/tests/test_company_guard.py +202 -0
  77. polanon-0.1.0/tests/test_denylist_guard.py +60 -0
  78. polanon-0.1.0/tests/test_examples.py +190 -0
  79. polanon-0.1.0/tests/test_international_guard.py +454 -0
  80. polanon-0.1.0/tests/test_judges.py +310 -0
  81. polanon-0.1.0/tests/test_layering.py +106 -0
  82. polanon-0.1.0/tests/test_mock_judge.py +15 -0
  83. polanon-0.1.0/tests/test_names_guard.py +334 -0
  84. polanon-0.1.0/tests/test_ner_guard.py +222 -0
  85. polanon-0.1.0/tests/test_output_sweep.py +70 -0
  86. polanon-0.1.0/tests/test_pipeline.py +422 -0
  87. polanon-0.1.0/tests/test_pipeline_invariants.py +808 -0
  88. polanon-0.1.0/tests/test_public_api.py +179 -0
  89. polanon-0.1.0/tests/test_recognizers_pl.py +699 -0
  90. polanon-0.1.0/tests/test_secrets_guard.py +177 -0
  91. polanon-0.1.0/tests/test_stream_restore.py +63 -0
  92. polanon-0.1.0/tests/test_types.py +134 -0
@@ -0,0 +1,56 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are recorded here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project follows
5
+ [semantic versioning](https://semver.org/spec/v2.0.0.html).
6
+
7
+ ## [0.1.0] - 2026-11-03
8
+
9
+ The first release.
10
+
11
+ ### Added
12
+
13
+ - **Reversible masking.** `Anonymizer` replaces personal data with tokens such as `<OSOBA_1>` and
14
+ `<PESEL_1>`, `Anonymizer.restore` puts the originals back, and `StreamRestorer` does it while an
15
+ answer streams, never splitting a token across chunks. Tokens are stable for as long as the
16
+ mapping is passed along, and a token is never reused for another value. `anonymize_sync` serves
17
+ scripts, web views and notebooks; `await anonymize` does the work in a worker thread.
18
+ - **Polish identifiers with checksums**: PESEL, NIP, REGON, ID card, IBAN and payment cards, plus
19
+ KRS numbers and passports next to their label. `is_valid_pesel`, `is_valid_iban` and the
20
+ per-country `IBAN_LENGTHS` can be imported from `polanon.guards.recognizers_pl`.
21
+ - **Contacts, devices and secrets**: e-mail addresses, phone numbers, street addresses and postcodes,
22
+ IPv4 and IPv6, MAC addresses, VINs, vehicle registrations, birth dates (`BIRTH_DATE`), API keys,
23
+ JWTs, private keys and password assignments. A secret is masked whole, with whatever was found
24
+ inside it.
25
+ - **`PersonNameGuard`.** Polish first names in all seven cases, in lowercase, in capitals and without
26
+ Polish letters, with the surname that agrees with them or after "pan", "pani", "dr". No language
27
+ model needed.
28
+ - **`CompanyGuard`.** Companies by their legal form (`Sp. z o.o.`, `S.A.`, `sp. j.`, `s.c.`,
29
+ `Fundacja`, `Kancelaria`), not by guessing a name word by word.
30
+ - **`InternationalGuard`.** Foreign invoices: EU, UK, Swiss and Norwegian VAT numbers, company
31
+ register numbers, Bankgiro, Plusgiro, sort codes and account numbers, international phone numbers,
32
+ foreign addresses and postcodes, companies by foreign legal form (Ltd, Inc, GmbH, AB), and names
33
+ after a form of address, a label or a closing formula.
34
+ - **An optional language model.** `NerGuard` reads the NKJP labels Polish spaCy models emit
35
+ (`persName`, `placeName`, `orgName`), which a pipeline configured for English labels never finds.
36
+ - **Every occurrence of a known value**, in every spelling a person reads as the same: capitals,
37
+ line breaks and runs of spaces, no-break spaces, invisible characters, decomposed accents, and an
38
+ identifier's groups written apart or together. `mask_known_values` applies a mapping to text
39
+ masked earlier.
40
+ - **An optional second stage.** A judge reviews the *masked* text for what still identifies someone
41
+ by context: `OllamaJudge` locally, `RemoteJudge` for testing, `MockJudge` in tests, and
42
+ `judge_fallback` to decide what happens when it is down.
43
+ - **`Allowlist`** with `WELL_KNOWN_NAMES`: platforms and tools a model takes for organisations stay
44
+ readable, in every Polish case form, without ever releasing an identifier.
45
+ - **`sweep`**, which reports tokens a model invented and personal data its answer introduced.
46
+ - **Choosing what to mask.** `DEFAULT_ENABLED` (every type but `DATE`), `ENTITY_TYPES`, and types of
47
+ your own declared in `Guard.entity_types`. A name that is not a type raises instead of quietly
48
+ switching masking off.
49
+ - **`polanon` on the command line**: masked text to stdout, the mapping to stderr.
50
+ - **A benchmark.** Ten synthetic documents, seven Polish and three foreign, with 164 annotated values;
51
+ `benchmarks/run.py` scores full, partial and missed coverage, and Microsoft Presidio is measured on
52
+ the same documents in `benchmarks/compare_presidio.py`. The tests turn the current numbers into
53
+ floors.
54
+ - **Recipes for the Anthropic and OpenAI SDKs** in `examples/`, tested offline against the real SDKs.
55
+ - Python 3.11 to 3.13, no dependencies in the core, `ner` and `judge` extras, and type information
56
+ (`py.typed`).
@@ -0,0 +1,47 @@
1
+ # Contributing
2
+
3
+ Bug reports, missed values, new detectors and new languages are all welcome.
4
+
5
+ ## Setup
6
+
7
+ ```bash
8
+ python -m venv .venv && source .venv/bin/activate
9
+ pip install -e ".[dev,ner,judge]"
10
+ python -m spacy download pl_core_news_md
11
+ pre-commit install
12
+ ```
13
+
14
+ Add `pip install anthropic openai` if you touch `examples/`: their tests run against the real SDKs,
15
+ offline, and are skipped without them.
16
+
17
+ ## Quality bar
18
+
19
+ - `ruff check .`, `ruff format --check .`, `mypy polanon` and `pytest --cov` pass locally. CI runs
20
+ them, then runs the tests again against the built wheel on Python 3.11, 3.12 and 3.13, and masks a
21
+ document with a wheel installed on its own.
22
+ - The core imports nothing but the standard library at module level; spaCy and httpx are loaded
23
+ inside the functions that need them. `tests/test_layering.py` enforces it.
24
+ - Type hints on every signature, Google-style docstrings on the public API.
25
+ - Tests for every behaviour change, and no network in them: the judges are tested with
26
+ `httpx.MockTransport`. Coverage stays at 90% or above.
27
+ - A change to detection is measured, not asserted: run `python benchmarks/run.py --ner` before and
28
+ after, and add a case worth keeping to `benchmarks/samples/` and `benchmarks/gold/`.
29
+ `tests/test_benchmark.py` holds the current numbers as floors.
30
+ - Conventional Commits in the imperative mood, one logical change each:
31
+ `feat(guards): recognise Czech VAT numbers`, `fix(pipeline): ...`, `docs: ...`, `test: ...`.
32
+ - Synthetic data only. No real person's name, identifier, document or log belongs in an issue, a
33
+ test or a commit.
34
+
35
+ ## A value that was not masked
36
+
37
+ Open a *Something was not masked* issue with invented text of the same shape: a made-up name, a
38
+ PESEL with a valid checksum (90010112349), a fictional company. The shortest text that still shows
39
+ the miss is the most useful. A way to get a value that already has a token past the masking is a
40
+ security report instead: see `SECURITY.md`.
41
+
42
+ ## A new detector
43
+
44
+ Subclass `Guard`, return `Span`s in original-text coordinates and respect the `enabled` set; the
45
+ README's *Adding a detector* section has a complete example. Score 1.0 for what a checksum, a label
46
+ or an official format vouches for and 0.85 for a statistical guess, and add the type to
47
+ `polanon/policy.py` if it should be built in.
polanon-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 BoroMat
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,8 @@
1
+ # The sdist carries what its tests read, so the suite runs from it: the annotated benchmark the
2
+ # floors in tests/test_benchmark.py are measured on, and the recipes tests/test_examples.py checks.
3
+ graft tests
4
+ graft benchmarks
5
+ graft examples
6
+ graft docs
7
+ include CHANGELOG.md CONTRIBUTING.md SECURITY.md
8
+ global-exclude __pycache__ *.py[cod]
polanon-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,376 @@
1
+ Metadata-Version: 2.4
2
+ Name: polanon
3
+ Version: 0.1.0
4
+ Summary: Reversible PII masking for LLM calls, Polish-first: checksum-validated PESEL, NIP, REGON and IBAN, Polish names in every case, stable tokens and streaming restore.
5
+ Author: BoroMat
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/boromat/polanon
8
+ Project-URL: Source, https://github.com/boromat/polanon
9
+ Project-URL: Issues, https://github.com/boromat/polanon/issues
10
+ Project-URL: Changelog, https://github.com/boromat/polanon/blob/main/CHANGELOG.md
11
+ Keywords: pii,anonymization,pseudonymization,gdpr,rodo,llm,privacy,polish,pesel,nip
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Natural Language :: Polish
15
+ Classifier: Natural Language :: English
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Security
22
+ Classifier: Topic :: Text Processing :: Linguistic
23
+ Classifier: Typing :: Typed
24
+ Requires-Python: >=3.11
25
+ Description-Content-Type: text/markdown
26
+ License-File: LICENSE
27
+ Provides-Extra: ner
28
+ Requires-Dist: spacy>=3.7; extra == "ner"
29
+ Provides-Extra: judge
30
+ Requires-Dist: httpx>=0.27; extra == "judge"
31
+ Provides-Extra: dev
32
+ Requires-Dist: pytest>=8; extra == "dev"
33
+ Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
34
+ Requires-Dist: pytest-cov>=5; extra == "dev"
35
+ Requires-Dist: ruff>=0.6; extra == "dev"
36
+ Requires-Dist: mypy>=1.11; extra == "dev"
37
+ Requires-Dist: pre-commit>=3.8; extra == "dev"
38
+ Requires-Dist: httpx>=0.27; extra == "dev"
39
+ Dynamic: license-file
40
+
41
+ # polanon
42
+
43
+ [![CI](https://github.com/boromat/polanon/actions/workflows/ci.yml/badge.svg)](https://github.com/boromat/polanon/actions/workflows/ci.yml)
44
+ [![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://github.com/boromat/polanon/blob/main/pyproject.toml)
45
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/boromat/polanon/blob/main/LICENSE)
46
+
47
+ **Reversible PII masking for LLM calls, Polish-first.** Mask personal data before a prompt leaves
48
+ your network, send only tokens to the model, and put the originals back into its answer — even
49
+ while it streams. Pure Python: the standard library is the whole core.
50
+
51
+ ```bash
52
+ pip install polanon
53
+ ```
54
+
55
+ ```python
56
+ from polanon import Anonymizer, default_guards
57
+
58
+ anonymizer = Anonymizer(default_guards(with_ner=False))
59
+
60
+ result = anonymizer.anonymize_sync(
61
+ "Proszę o korektę faktury dla Jana Kowalskiego, PESEL 90010112349, "
62
+ "konto PL61109010140000071219812874, mail jan.kowalski@firma.pl."
63
+ )
64
+ print(result.anonymized)
65
+ # Proszę o korektę faktury dla <OSOBA_1>, PESEL <PESEL_1>, konto <IBAN_1>, mail <EMAIL_1>.
66
+
67
+ answer = "Korekta dla <OSOBA_1> (PESEL <PESEL_1>) jest gotowa." # what the model replied
68
+ print(Anonymizer.restore(answer, result.mapping))
69
+ # Korekta dla Jana Kowalskiego (PESEL 90010112349) jest gotowa.
70
+ ```
71
+
72
+ The model only ever sees `<OSOBA_1>`. The mapping that turns it back into a name never leaves your
73
+ process. `anonymize_sync` works in scripts, Django and Flask views, Celery tasks and notebooks; in
74
+ async code, `await anonymizer.anonymize(...)` does the same work in a worker thread.
75
+
76
+ ## Why another PII library
77
+
78
+ General-purpose tools are built around English. Polish data needs three things they do not do out
79
+ of the box:
80
+
81
+ - **Checksums.** A PESEL, NIP, REGON, ID card number, IBAN or payment card is validated, not just
82
+ pattern-matched: the checksum rejects most random digit strings — for a PESEL, together with the
83
+ birth date, about 98% of them — while a valid number always passes. KRS numbers, which have no
84
+ public checksum, and passports are recognised next to their label as well.
85
+ - **Seven grammatical cases.** "Jan Kowalski", "Jana Kowalskiego" and "z Janem Kowalskim" are the
86
+ same person. Polish first names are recognised in every case, in lowercase and typed without
87
+ Polish letters, together with the surname that agrees with them, without any language model.
88
+ - **Polish model labels.** Polish spaCy models tag people as `persName`, not `PERSON`. A pipeline
89
+ configured for English labels finds no names at all, and says nothing about it.
90
+
91
+ It also reads foreign invoices: EU, UK and Swiss VAT numbers, company register numbers, Bankgiro,
92
+ Plusgiro and sort codes, international phone numbers, street addresses and postcodes.
93
+
94
+ ## Install
95
+
96
+ | You want | Install |
97
+ |---|---|
98
+ | Identifiers, contacts, secrets, Polish names, companies by legal form | `pip install polanon` |
99
+ | Also places, and names and organisations no rule can see | `pip install "polanon[ner]"` then `python -m spacy download pl_core_news_md` |
100
+ | The stage-2 judges (a local Ollama model, or a remote one) | `pip install "polanon[judge]"` |
101
+
102
+ Then use `default_guards()` with the model, or `default_guards(with_ner=False)` without it.
103
+ Another spaCy model is one argument away: `default_guards(spacy_model="pl_core_news_lg")`.
104
+
105
+ ## From the command line
106
+
107
+ Text comes from the arguments or, without any, from standard input. Masked text goes to stdout
108
+ and the mapping to stderr, so a pipeline can keep the two apart:
109
+
110
+ ```bash
111
+ polanon "Jan Kowalski, PESEL 90010112349"
112
+ cat umowa.txt | polanon --ner > masked.txt 2> mapping.json
113
+ ```
114
+
115
+ A non-zero exit status means stderr holds an error rather than the mapping. In Windows
116
+ PowerShell 5.1, read the file with `Get-Content -Encoding UTF8 umowa.txt | polanon`: a plain
117
+ `cat` there re-encodes Polish letters before they reach the program, and a garbled name is not
118
+ recognised.
119
+
120
+ ## Streaming
121
+
122
+ A provider streams arbitrary fragments, so `<OSOBA_1>` can arrive as `<OSO` and `BA_1>`. The
123
+ restorer holds back anything that could still grow into a token:
124
+
125
+ ```python
126
+ restorer = Anonymizer.stream_restorer(result.mapping)
127
+ for chunk in stream: # "Dzień dobry <OSO", "BA_1>, PESEL <PESEL", "_1> zweryfikowany."
128
+ print(restorer.feed(chunk), end="")
129
+ print(restorer.flush())
130
+ # Dzień dobry Jana Kowalskiego, PESEL 90010112349 zweryfikowany.
131
+ ```
132
+
133
+ ## With Claude, GPT or a local model
134
+
135
+ Mask before the call, restore while the answer streams. With the Anthropic SDK:
136
+
137
+ ```python
138
+ import anthropic
139
+ from polanon import Anonymizer, default_guards
140
+
141
+ anonymizer = Anonymizer(default_guards(with_ner=False))
142
+ masked = anonymizer.anonymize_sync(question)
143
+ restorer = Anonymizer.stream_restorer(masked.mapping)
144
+
145
+ with anthropic.Anthropic().messages.stream(
146
+ model="claude-opus-5-5",
147
+ max_tokens=64000,
148
+ system="Personal data is replaced by tokens such as <OSOBA_1>. Copy a token exactly as written.",
149
+ messages=[{"role": "user", "content": masked.anonymized}],
150
+ ) as stream:
151
+ for text in stream.text_stream:
152
+ print(restorer.feed(text), end="")
153
+ print(restorer.flush())
154
+ ```
155
+
156
+ Telling the model what the tokens are keeps it from rewriting them. The
157
+ [examples](https://github.com/boromat/polanon/tree/main/examples) go further:
158
+ `openai_chat.py` does the same for any OpenAI-compatible endpoint (OpenAI, OpenRouter, Azure, vLLM,
159
+ a local Ollama), and `anthropic_chat.py` holds a conversation whose history keeps masked text only,
160
+ so an answer restored for the reader is never sent back in the clear. Both are tested in CI against
161
+ the real SDKs, offline.
162
+
163
+ ## What it detects
164
+
165
+ Checksum-validated Polish identifiers (PESEL, NIP, REGON, ID card, IBAN, payment cards), KRS numbers
166
+ next to their label, e-mail addresses and phone numbers, street addresses and postcodes, IPv4 and
167
+ IPv6, MAC addresses, vehicle registrations after their label ("nr rej."), VINs, passport numbers,
168
+ birth dates, API keys, tokens and passwords, Polish first names with their surnames, and company
169
+ names anchored on a legal form ("Sp. z o.o.", "S.A.", "Ltd", "GmbH"). With the `ner` extra, a spaCy
170
+ model adds places and the names nothing else marks. An optional second stage lets a small local
171
+ model review the *masked* text for what still identifies someone by context. Other dates are
172
+ recognised too, but left readable unless you ask for them.
173
+
174
+ Text copied from PDFs, Word or macOS is read as a person would read it: letters written as a base
175
+ letter plus an accent, no-break spaces and en dashes inside numbers, and invisible zero-width
176
+ characters do not hide a value. The mapping keeps each value as it was first written.
177
+
178
+ On the annotated sample documents in the repository, the rules alone fully mask 85% of the
179
+ values and the rules with `pl_core_news_md` mask 98%, with nothing left half-masked. Ten short
180
+ documents are a regression check, not a research benchmark: see the
181
+ [benchmark notes](https://github.com/boromat/polanon/blob/main/benchmarks/README.md) for what is
182
+ missed and why, and for Microsoft Presidio on the same documents with the same model: 65% fully
183
+ masked, 74% at its most lenient.
184
+
185
+ ## Worth knowing
186
+
187
+ **Tokens are stable per conversation, not global.** Pass the mapping from the latest result to
188
+ the next call: the same value keeps the same token, and a token is never reused for another
189
+ value. The mapping you pass in is read, not changed, so keep the one each result returns:
190
+
191
+ ```python
192
+ mapping = result.mapping
193
+ second = anonymizer.anonymize_sync("Ten sam PESEL 90010112349", mapping)
194
+ second.anonymized # "Ten sam PESEL <PESEL_1>"
195
+ mapping = second.mapping # everything so far, for the next turn
196
+ ```
197
+
198
+ **A token restores the form it replaced.** Masking happens on the text as written, so a name masked
199
+ in the genitive comes back in the genitive. Ask the model to keep tokens where the grammar allows it
200
+ ("dla <OSOBA_1>"), or restore into a template you control. Spellings that differ only in letter
201
+ case, in the kind of whitespace (a no-break space, a line break, a run of spaces) or in invisible
202
+ characters are the same value, and so are an identifier's groups written apart or together: they
203
+ share its token, and restore writes the spelling seen first. Passwords and custom types are the
204
+ exception, since two passwords that differ in case are two passwords, and a password never shares
205
+ a token with a name that reads alike. A one-word name found in all lowercase keeps a token of its
206
+ own too, for the reason below.
207
+
208
+ **Every occurrence of a masked value is masked**, also where no detector would recognise it on its
209
+ own: as written, in capitals, across a line break, with a zero-width character in it, with its
210
+ groups spaced differently, and around whatever a detector found inside it. Two exceptions keep ordinary words readable. A capitalised
211
+ one-word name is not matched in all lowercase, because Zima, Kot and Wilk are surnames and "zima"
212
+ is winter; identifiers such as a VIN or an e-mail address are matched in any case. A number shorter
213
+ than six digits, such as a PIN, is not matched inside a longer number, a date, a time or an
214
+ amount: "15.04.2025", "2024–2025", "2025,50" and "20250" stay readable when the PIN is 2025, while
215
+ "w 2025 roku" and "PIN/PUK 2025/36912874" do not.
216
+
217
+ **A token you write stays yours.** A placeholder such as `<PESEL_1>` in your own text is never handed
218
+ to real data. It joins the mapping as standing for itself, so the reservation holds in later turns
219
+ too.
220
+
221
+ **The mapping is the sensitive half.** `result.anonymized` holds a token for everything the
222
+ detectors found, and only what they missed in the clear; `result.mapping` is the key that undoes
223
+ it. Store the mapping the way you would store the original document.
224
+
225
+ **Replaying a conversation?** `mask_known_values(text, mapping)` masks every value a mapping already
226
+ holds in text that was masked earlier, such as history sent again after a new value got its token.
227
+ The same call checks a payload before it leaves: `mask_known_values(payload, mapping) == payload`
228
+ means no value the mapping holds is left where the library would mask it.
229
+
230
+ **Words of your own.** In-house names no rule can know, such as a project codename or a client,
231
+ go on a denylist and are masked wherever they appear, in any case and across line breaks:
232
+
233
+ ```python
234
+ guards = default_guards(denylist=["Projekt Wichura", "Zakład Nowak"], with_ner=False)
235
+ ```
236
+
237
+ **Checking the answer.** After restoring, `anonymizer.sweep(restored, mapping)` reports tokens the
238
+ model invented and personal data its answer added: `report.is_clean`, or `report.unknown_tokens`
239
+ and `report.new_spans` to look at.
240
+
241
+ ## Choosing what to mask
242
+
243
+ `Anonymizer(guards)` masks `DEFAULT_ENABLED`: every type in `ENTITY_TYPES` except `DATE`, because a
244
+ date rarely identifies anyone and masking every deadline leaves the model unable to answer. A birth
245
+ date does identify someone, so a date after "ur.", "data urodzenia", "DOB" or "geb." is its own
246
+ type, `BIRTH_DATE`, and stays masked. Pass `enabled` to choose for yourself:
247
+
248
+ ```python
249
+ from polanon import DEFAULT_ENABLED, ENTITY_TYPES, Anonymizer, default_guards
250
+
251
+ guards = default_guards(with_ner=False)
252
+ Anonymizer(guards, enabled=DEFAULT_ENABLED | {"DATE"}) # dates too
253
+ Anonymizer(guards, enabled={"PESEL", "NIP", "IBAN"}) # identifiers only
254
+ list(ENTITY_TYPES) # every name you can use
255
+ ```
256
+
257
+ A name that is neither built in nor declared in a guard's `entity_types` raises `ValueError`, so a
258
+ typo such as `"PERSONS"` cannot quietly switch masking off.
259
+
260
+ ## A second opinion: the judge
261
+
262
+ Rules and a language model miss what identifies someone only by context: "the finance director of
263
+ the only steelworks in town". A judge is a small model that reads the *masked* text after stage 1
264
+ and flags such fragments. Run it locally with Ollama, so the review never leaves your network:
265
+
266
+ ```python
267
+ import asyncio
268
+
269
+ from polanon import Anonymizer, JudgeError, default_guards
270
+ from polanon.judge import OllamaJudge
271
+
272
+ anonymizer = Anonymizer(
273
+ default_guards(),
274
+ judge=OllamaJudge(model="qwen2.5:1.5b"),
275
+ judge_fallback="block", # refuse to go on if the judge is down
276
+ )
277
+
278
+
279
+ async def main(text: str) -> None:
280
+ try:
281
+ result = await anonymizer.anonymize(text)
282
+ except JudgeError:
283
+ ... # the judge was down, and "block" means nothing goes out
284
+
285
+
286
+ asyncio.run(main("..."))
287
+ ```
288
+
289
+ It needs `pip install "polanon[ner,judge]"`, the spaCy model, a running Ollama and
290
+ `ollama pull qwen2.5:1.5b`. With `judge_fallback="send_stage1"` the call goes on when the judge
291
+ is down, and `result.judge_failed` says that stage 1 went out alone.
292
+
293
+ Judges are asynchronous, so `anonymize_sync` refuses an anonymizer that has one. `RemoteJudge` asks an
294
+ OpenAI-compatible endpoint instead; the masked text then leaves your network, which makes it a
295
+ testing aid rather than a privacy control.
296
+
297
+ ## Names that should stay readable
298
+
299
+ A language model flags "Facebook" or "Google Ads" as an organisation, which masks the very thing a
300
+ question is about. Pass an allowlist and those detections are dropped before tokens are assigned:
301
+
302
+ ```python
303
+ from polanon import WELL_KNOWN_NAMES, Allowlist, Anonymizer, default_guards
304
+
305
+ anonymizer = Anonymizer(default_guards(), allowlist=Allowlist(WELL_KNOWN_NAMES))
306
+ ```
307
+
308
+ A one-word name also matches its Polish case forms ("Facebooku", "Canvie"). Only detections of
309
+ names are dropped, and only when every word in them is on the list; identifiers, the denylist and
310
+ values a conversation has already masked stay masked.
311
+
312
+ ## The pieces
313
+
314
+ | Module | What it is for |
315
+ |---|---|
316
+ | `guards/` | deterministic detectors. Each returns `Span`s in original-text coordinates |
317
+ | `pipeline.py` | runs the guards in a thread, resolves overlaps, assigns stable tokens |
318
+ | `policy.py` | the entity taxonomy and the token naming (`<OSOBA_1>`, `<PESEL_1>`) |
319
+ | `judge/` | the optional second stage: a model reviews the masked text for what still identifies |
320
+ | `stream_restore.py` | puts originals back into a streamed reply without splitting a token |
321
+ | `output_sweep.py` | flags invented tokens and personal data the answer introduced |
322
+
323
+ ## Adding a detector
324
+
325
+ Subclass `Guard`, return spans in original-text coordinates and respect the `enabled` set. A type of
326
+ your own goes in `entity_types`, which turns it on by default; it becomes the label of its tokens,
327
+ so it is written in uppercase letters and underscores:
328
+
329
+ ```python
330
+ import re
331
+
332
+ from polanon import Anonymizer, Guard, Span, default_guards
333
+
334
+
335
+ class EmployeeIdGuard(Guard):
336
+ name = "employee_id"
337
+ entity_types = frozenset({"EMPLOYEE_ID"})
338
+
339
+ def detect(self, text: str, enabled: set[str]) -> list[Span]:
340
+ if "EMPLOYEE_ID" not in enabled:
341
+ return []
342
+ return [
343
+ Span(m.start(), m.end(), "EMPLOYEE_ID", 1.0, self.name)
344
+ for m in re.finditer(r"\bEMP-\d{5}\b", text)
345
+ ]
346
+
347
+
348
+ anonymizer = Anonymizer([*default_guards(with_ner=False), EmployeeIdGuard()])
349
+ anonymizer.anonymize_sync("Wniosek EMP-11111").anonymized # "Wniosek <EMPLOYEE_ID_1>"
350
+ ```
351
+
352
+ The score decides who wins where detections overlap: 1.0 for a value a checksum, a label or an
353
+ official format vouches for, 0.85 for a statistical guess; the judge's findings score 0.5. A secret
354
+ is masked whole, with whatever was detected inside it. A span outside the text, a type spelled any
355
+ other way, or one spelled like a built-in token label (`OSOBA`, `DATA`) raises `ValueError` naming
356
+ your guard.
357
+
358
+ ## Before you rely on it
359
+
360
+ Masking is pseudonymisation, not anonymisation in the legal sense, and no detector catches
361
+ everything. The [threat model](https://github.com/boromat/polanon/blob/main/docs/THREAT_MODEL.md) says what is covered, what is
362
+ not, and what stays your application's job, starting with keeping the mapping safe.
363
+
364
+ polanon was built as the masking engine of piigate, a self-hosted hub that lets a team use Claude
365
+ or GPT on real documents without sending personal data to the provider; the hub is not published
366
+ yet. The library knows nothing about it — no HTTP, no database, no settings — and a test fails if
367
+ it ever starts to.
368
+
369
+ ## Contributing
370
+
371
+ Bug reports, missed values and new detectors are welcome: see
372
+ [CONTRIBUTING.md](https://github.com/boromat/polanon/blob/main/CONTRIBUTING.md). A way to get an original value past the
373
+ masking is a security report, made privately as [SECURITY.md](https://github.com/boromat/polanon/blob/main/SECURITY.md)
374
+ describes.
375
+
376
+ MIT licensed.