polanon 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- polanon-0.1.0/CHANGELOG.md +56 -0
- polanon-0.1.0/CONTRIBUTING.md +47 -0
- polanon-0.1.0/LICENSE +21 -0
- polanon-0.1.0/MANIFEST.in +8 -0
- polanon-0.1.0/PKG-INFO +376 -0
- polanon-0.1.0/README.md +336 -0
- polanon-0.1.0/SECURITY.md +18 -0
- polanon-0.1.0/benchmarks/README.md +93 -0
- polanon-0.1.0/benchmarks/compare_presidio.py +208 -0
- polanon-0.1.0/benchmarks/demo.py +45 -0
- polanon-0.1.0/benchmarks/gold/cv.json +65 -0
- polanon-0.1.0/benchmarks/gold/czat.json +41 -0
- polanon-0.1.0/benchmarks/gold/faktura.json +69 -0
- polanon-0.1.0/benchmarks/gold/invoice_se.json +21 -0
- polanon-0.1.0/benchmarks/gold/invoice_uk.json +25 -0
- polanon-0.1.0/benchmarks/gold/mail_klienta.json +49 -0
- polanon-0.1.0/benchmarks/gold/notatka_wewnetrzna.json +73 -0
- polanon-0.1.0/benchmarks/gold/rechnung_ch.json +20 -0
- polanon-0.1.0/benchmarks/gold/umowa.json +77 -0
- polanon-0.1.0/benchmarks/gold/zgloszenie_it.json +77 -0
- polanon-0.1.0/benchmarks/run.py +253 -0
- polanon-0.1.0/benchmarks/samples/README.md +15 -0
- polanon-0.1.0/benchmarks/samples/cv.txt +18 -0
- polanon-0.1.0/benchmarks/samples/czat.txt +16 -0
- polanon-0.1.0/benchmarks/samples/faktura.txt +22 -0
- polanon-0.1.0/benchmarks/samples/invoice_se.txt +27 -0
- polanon-0.1.0/benchmarks/samples/invoice_uk.txt +38 -0
- polanon-0.1.0/benchmarks/samples/mail_klienta.txt +19 -0
- polanon-0.1.0/benchmarks/samples/notatka_wewnetrzna.txt +18 -0
- polanon-0.1.0/benchmarks/samples/rechnung_ch.txt +32 -0
- polanon-0.1.0/benchmarks/samples/umowa.txt +19 -0
- polanon-0.1.0/benchmarks/samples/zgloszenie_it.txt +19 -0
- polanon-0.1.0/docs/THREAT_MODEL.md +98 -0
- polanon-0.1.0/examples/anthropic_chat.py +107 -0
- polanon-0.1.0/examples/openai_chat.py +78 -0
- polanon-0.1.0/polanon/__init__.py +255 -0
- polanon-0.1.0/polanon/__main__.py +90 -0
- polanon-0.1.0/polanon/_detection.py +79 -0
- polanon-0.1.0/polanon/_matching.py +624 -0
- polanon-0.1.0/polanon/_view.py +102 -0
- polanon-0.1.0/polanon/allowlist.py +246 -0
- polanon-0.1.0/polanon/guards/__init__.py +0 -0
- polanon-0.1.0/polanon/guards/base.py +22 -0
- polanon-0.1.0/polanon/guards/company_pl.py +214 -0
- polanon-0.1.0/polanon/guards/denylist.py +71 -0
- polanon-0.1.0/polanon/guards/international.py +771 -0
- polanon-0.1.0/polanon/guards/names_pl.py +823 -0
- polanon-0.1.0/polanon/guards/ner.py +427 -0
- polanon-0.1.0/polanon/guards/recognizers_pl.py +562 -0
- polanon-0.1.0/polanon/guards/secrets.py +162 -0
- polanon-0.1.0/polanon/judge/__init__.py +30 -0
- polanon-0.1.0/polanon/judge/base.py +39 -0
- polanon-0.1.0/polanon/judge/mock.py +27 -0
- polanon-0.1.0/polanon/judge/ollama.py +67 -0
- polanon-0.1.0/polanon/judge/parsing.py +121 -0
- polanon-0.1.0/polanon/judge/prompt.py +48 -0
- polanon-0.1.0/polanon/judge/remote.py +76 -0
- polanon-0.1.0/polanon/output_sweep.py +79 -0
- polanon-0.1.0/polanon/pipeline.py +425 -0
- polanon-0.1.0/polanon/policy.py +67 -0
- polanon-0.1.0/polanon/py.typed +0 -0
- polanon-0.1.0/polanon/stream_restore.py +46 -0
- polanon-0.1.0/polanon/types.py +179 -0
- polanon-0.1.0/polanon.egg-info/PKG-INFO +376 -0
- polanon-0.1.0/polanon.egg-info/SOURCES.txt +90 -0
- polanon-0.1.0/polanon.egg-info/dependency_links.txt +1 -0
- polanon-0.1.0/polanon.egg-info/entry_points.txt +2 -0
- polanon-0.1.0/polanon.egg-info/requires.txt +15 -0
- polanon-0.1.0/polanon.egg-info/top_level.txt +1 -0
- polanon-0.1.0/pyproject.toml +127 -0
- polanon-0.1.0/setup.cfg +4 -0
- polanon-0.1.0/tests/__init__.py +0 -0
- polanon-0.1.0/tests/test_allowlist.py +245 -0
- polanon-0.1.0/tests/test_benchmark.py +123 -0
- polanon-0.1.0/tests/test_cli.py +104 -0
- polanon-0.1.0/tests/test_company_guard.py +202 -0
- polanon-0.1.0/tests/test_denylist_guard.py +60 -0
- polanon-0.1.0/tests/test_examples.py +190 -0
- polanon-0.1.0/tests/test_international_guard.py +454 -0
- polanon-0.1.0/tests/test_judges.py +310 -0
- polanon-0.1.0/tests/test_layering.py +106 -0
- polanon-0.1.0/tests/test_mock_judge.py +15 -0
- polanon-0.1.0/tests/test_names_guard.py +334 -0
- polanon-0.1.0/tests/test_ner_guard.py +222 -0
- polanon-0.1.0/tests/test_output_sweep.py +70 -0
- polanon-0.1.0/tests/test_pipeline.py +422 -0
- polanon-0.1.0/tests/test_pipeline_invariants.py +808 -0
- polanon-0.1.0/tests/test_public_api.py +179 -0
- polanon-0.1.0/tests/test_recognizers_pl.py +699 -0
- polanon-0.1.0/tests/test_secrets_guard.py +177 -0
- polanon-0.1.0/tests/test_stream_restore.py +63 -0
- polanon-0.1.0/tests/test_types.py +134 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are recorded here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project follows
|
|
5
|
+
[semantic versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - 2026-11-03
|
|
8
|
+
|
|
9
|
+
The first release.
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Reversible masking.** `Anonymizer` replaces personal data with tokens such as `<OSOBA_1>` and
|
|
14
|
+
`<PESEL_1>`, `Anonymizer.restore` puts the originals back, and `StreamRestorer` does it while an
|
|
15
|
+
answer streams, never splitting a token across chunks. Tokens are stable for as long as the
|
|
16
|
+
mapping is passed along, and a token is never reused for another value. `anonymize_sync` serves
|
|
17
|
+
scripts, web views and notebooks; `await anonymize` does the work in a worker thread.
|
|
18
|
+
- **Polish identifiers with checksums**: PESEL, NIP, REGON, ID card, IBAN and payment cards, plus
|
|
19
|
+
KRS numbers and passports next to their label. `is_valid_pesel`, `is_valid_iban` and the
|
|
20
|
+
per-country `IBAN_LENGTHS` can be imported from `polanon.guards.recognizers_pl`.
|
|
21
|
+
- **Contacts, devices and secrets**: e-mail addresses, phone numbers, street addresses and postcodes,
|
|
22
|
+
IPv4 and IPv6, MAC addresses, VINs, vehicle registrations, birth dates (`BIRTH_DATE`), API keys,
|
|
23
|
+
JWTs, private keys and password assignments. A secret is masked whole, with whatever was found
|
|
24
|
+
inside it.
|
|
25
|
+
- **`PersonNameGuard`.** Polish first names in all seven cases, in lowercase, in capitals and without
|
|
26
|
+
Polish letters, with the surname that agrees with them or after "pan", "pani", "dr". No language
|
|
27
|
+
model needed.
|
|
28
|
+
- **`CompanyGuard`.** Companies by their legal form (`Sp. z o.o.`, `S.A.`, `sp. j.`, `s.c.`,
|
|
29
|
+
`Fundacja`, `Kancelaria`), not by guessing a name word by word.
|
|
30
|
+
- **`InternationalGuard`.** Foreign invoices: EU, UK, Swiss and Norwegian VAT numbers, company
|
|
31
|
+
register numbers, Bankgiro, Plusgiro, sort codes and account numbers, international phone numbers,
|
|
32
|
+
foreign addresses and postcodes, companies by foreign legal form (Ltd, Inc, GmbH, AB), and names
|
|
33
|
+
after a form of address, a label or a closing formula.
|
|
34
|
+
- **An optional language model.** `NerGuard` reads the NKJP labels Polish spaCy models emit
|
|
35
|
+
(`persName`, `placeName`, `orgName`), which a pipeline configured for English labels never finds.
|
|
36
|
+
- **Every occurrence of a known value**, in every spelling a person reads as the same: capitals,
|
|
37
|
+
line breaks and runs of spaces, no-break spaces, invisible characters, decomposed accents, and an
|
|
38
|
+
identifier's groups written apart or together. `mask_known_values` applies a mapping to text
|
|
39
|
+
masked earlier.
|
|
40
|
+
- **An optional second stage.** A judge reviews the *masked* text for what still identifies someone
|
|
41
|
+
by context: `OllamaJudge` locally, `RemoteJudge` for testing, `MockJudge` in tests, and
|
|
42
|
+
`judge_fallback` to decide what happens when it is down.
|
|
43
|
+
- **`Allowlist`** with `WELL_KNOWN_NAMES`: platforms and tools a model takes for organisations stay
|
|
44
|
+
readable, in every Polish case form, without ever releasing an identifier.
|
|
45
|
+
- **`sweep`**, which reports tokens a model invented and personal data its answer introduced.
|
|
46
|
+
- **Choosing what to mask.** `DEFAULT_ENABLED` (every type but `DATE`), `ENTITY_TYPES`, and types of
|
|
47
|
+
your own declared in `Guard.entity_types`. A name that is not a type raises instead of quietly
|
|
48
|
+
switching masking off.
|
|
49
|
+
- **`polanon` on the command line**: masked text to stdout, the mapping to stderr.
|
|
50
|
+
- **A benchmark.** Ten synthetic documents, seven Polish and three foreign, with 164 annotated values;
|
|
51
|
+
`benchmarks/run.py` scores full, partial and missed coverage, and Microsoft Presidio is measured on
|
|
52
|
+
the same documents in `benchmarks/compare_presidio.py`. The tests turn the current numbers into
|
|
53
|
+
floors.
|
|
54
|
+
- **Recipes for the Anthropic and OpenAI SDKs** in `examples/`, tested offline against the real SDKs.
|
|
55
|
+
- Python 3.11 to 3.13, no dependencies in the core, `ner` and `judge` extras, and type information
|
|
56
|
+
(`py.typed`).
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Bug reports, missed values, new detectors and new languages are all welcome.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
python -m venv .venv && source .venv/bin/activate
|
|
9
|
+
pip install -e ".[dev,ner,judge]"
|
|
10
|
+
python -m spacy download pl_core_news_md
|
|
11
|
+
pre-commit install
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
Add `pip install anthropic openai` if you touch `examples/`: their tests run against the real SDKs,
|
|
15
|
+
offline, and are skipped without them.
|
|
16
|
+
|
|
17
|
+
## Quality bar
|
|
18
|
+
|
|
19
|
+
- `ruff check .`, `ruff format --check .`, `mypy polanon` and `pytest --cov` pass locally. CI runs
|
|
20
|
+
them, then runs the tests again against the built wheel on Python 3.11, 3.12 and 3.13, and masks a
|
|
21
|
+
document with a wheel installed on its own.
|
|
22
|
+
- The core imports nothing but the standard library at module level; spaCy and httpx are loaded
|
|
23
|
+
inside the functions that need them. `tests/test_layering.py` enforces it.
|
|
24
|
+
- Type hints on every signature, Google-style docstrings on the public API.
|
|
25
|
+
- Tests for every behaviour change, and no network in them: the judges are tested with
|
|
26
|
+
`httpx.MockTransport`. Coverage stays at 90% or above.
|
|
27
|
+
- A change to detection is measured, not asserted: run `python benchmarks/run.py --ner` before and
|
|
28
|
+
after, and add a case worth keeping to `benchmarks/samples/` and `benchmarks/gold/`.
|
|
29
|
+
`tests/test_benchmark.py` holds the current numbers as floors.
|
|
30
|
+
- Conventional Commits in the imperative mood, one logical change each:
|
|
31
|
+
`feat(guards): recognise Czech VAT numbers`, `fix(pipeline): ...`, `docs: ...`, `test: ...`.
|
|
32
|
+
- Synthetic data only. No real person's name, identifier, document or log belongs in an issue, a
|
|
33
|
+
test or a commit.
|
|
34
|
+
|
|
35
|
+
## A value that was not masked
|
|
36
|
+
|
|
37
|
+
Open a *Something was not masked* issue with invented text of the same shape: a made-up name, a
|
|
38
|
+
PESEL with a valid checksum (90010112349), a fictional company. The shortest text that still shows
|
|
39
|
+
the miss is the most useful. A way to get a value that already has a token past the masking is a
|
|
40
|
+
security report instead: see `SECURITY.md`.
|
|
41
|
+
|
|
42
|
+
## A new detector
|
|
43
|
+
|
|
44
|
+
Subclass `Guard`, return `Span`s in original-text coordinates and respect the `enabled` set; the
|
|
45
|
+
README's *Adding a detector* section has a complete example. Score 1.0 for what a checksum, a label
|
|
46
|
+
or an official format vouches for and 0.85 for a statistical guess, and add the type to
|
|
47
|
+
`polanon/policy.py` if it should be built in.
|
polanon-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 BoroMat
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# The sdist carries what its tests read, so the suite runs from it: the annotated benchmark the
|
|
2
|
+
# floors in tests/test_benchmark.py are measured on, and the recipes tests/test_examples.py checks.
|
|
3
|
+
graft tests
|
|
4
|
+
graft benchmarks
|
|
5
|
+
graft examples
|
|
6
|
+
graft docs
|
|
7
|
+
include CHANGELOG.md CONTRIBUTING.md SECURITY.md
|
|
8
|
+
global-exclude __pycache__ *.py[cod]
|
polanon-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: polanon
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reversible PII masking for LLM calls, Polish-first: checksum-validated PESEL, NIP, REGON and IBAN, Polish names in every case, stable tokens and streaming restore.
|
|
5
|
+
Author: BoroMat
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/boromat/polanon
|
|
8
|
+
Project-URL: Source, https://github.com/boromat/polanon
|
|
9
|
+
Project-URL: Issues, https://github.com/boromat/polanon/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/boromat/polanon/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: pii,anonymization,pseudonymization,gdpr,rodo,llm,privacy,polish,pesel,nip
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Natural Language :: Polish
|
|
15
|
+
Classifier: Natural Language :: English
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Security
|
|
22
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Provides-Extra: ner
|
|
28
|
+
Requires-Dist: spacy>=3.7; extra == "ner"
|
|
29
|
+
Provides-Extra: judge
|
|
30
|
+
Requires-Dist: httpx>=0.27; extra == "judge"
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
33
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
34
|
+
Requires-Dist: pytest-cov>=5; extra == "dev"
|
|
35
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
36
|
+
Requires-Dist: mypy>=1.11; extra == "dev"
|
|
37
|
+
Requires-Dist: pre-commit>=3.8; extra == "dev"
|
|
38
|
+
Requires-Dist: httpx>=0.27; extra == "dev"
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# polanon
|
|
42
|
+
|
|
43
|
+
[](https://github.com/boromat/polanon/actions/workflows/ci.yml)
|
|
44
|
+
[](https://github.com/boromat/polanon/blob/main/pyproject.toml)
|
|
45
|
+
[](https://github.com/boromat/polanon/blob/main/LICENSE)
|
|
46
|
+
|
|
47
|
+
**Reversible PII masking for LLM calls, Polish-first.** Mask personal data before a prompt leaves
|
|
48
|
+
your network, send only tokens to the model, and put the originals back into its answer — even
|
|
49
|
+
while it streams. Pure Python: the standard library is the whole core.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install polanon
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
```python
|
|
56
|
+
from polanon import Anonymizer, default_guards
|
|
57
|
+
|
|
58
|
+
anonymizer = Anonymizer(default_guards(with_ner=False))
|
|
59
|
+
|
|
60
|
+
result = anonymizer.anonymize_sync(
|
|
61
|
+
"Proszę o korektę faktury dla Jana Kowalskiego, PESEL 90010112349, "
|
|
62
|
+
"konto PL61109010140000071219812874, mail jan.kowalski@firma.pl."
|
|
63
|
+
)
|
|
64
|
+
print(result.anonymized)
|
|
65
|
+
# Proszę o korektę faktury dla <OSOBA_1>, PESEL <PESEL_1>, konto <IBAN_1>, mail <EMAIL_1>.
|
|
66
|
+
|
|
67
|
+
answer = "Korekta dla <OSOBA_1> (PESEL <PESEL_1>) jest gotowa." # what the model replied
|
|
68
|
+
print(Anonymizer.restore(answer, result.mapping))
|
|
69
|
+
# Korekta dla Jana Kowalskiego (PESEL 90010112349) jest gotowa.
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The model only ever sees `<OSOBA_1>`. The mapping that turns it back into a name never leaves your
|
|
73
|
+
process. `anonymize_sync` works in scripts, Django and Flask views, Celery tasks and notebooks; in
|
|
74
|
+
async code, `await anonymizer.anonymize(...)` does the same work in a worker thread.
|
|
75
|
+
|
|
76
|
+
## Why another PII library
|
|
77
|
+
|
|
78
|
+
General-purpose tools are built around English. Polish data needs three things they do not do out
|
|
79
|
+
of the box:
|
|
80
|
+
|
|
81
|
+
- **Checksums.** A PESEL, NIP, REGON, ID card number, IBAN or payment card is validated, not just
|
|
82
|
+
pattern-matched: the checksum rejects most random digit strings — for a PESEL, together with the
|
|
83
|
+
birth date, about 98% of them — while a valid number always passes. KRS numbers, which have no
|
|
84
|
+
public checksum, and passports are recognised next to their label as well.
|
|
85
|
+
- **Seven grammatical cases.** "Jan Kowalski", "Jana Kowalskiego" and "z Janem Kowalskim" are the
|
|
86
|
+
same person. Polish first names are recognised in every case, in lowercase and typed without
|
|
87
|
+
Polish letters, together with the surname that agrees with them, without any language model.
|
|
88
|
+
- **Polish model labels.** Polish spaCy models tag people as `persName`, not `PERSON`. A pipeline
|
|
89
|
+
configured for English labels finds no names at all, and says nothing about it.
|
|
90
|
+
|
|
91
|
+
It also reads foreign invoices: EU, UK and Swiss VAT numbers, company register numbers, Bankgiro,
|
|
92
|
+
Plusgiro and sort codes, international phone numbers, street addresses and postcodes.
|
|
93
|
+
|
|
94
|
+
## Install
|
|
95
|
+
|
|
96
|
+
| You want | Install |
|
|
97
|
+
|---|---|
|
|
98
|
+
| Identifiers, contacts, secrets, Polish names, companies by legal form | `pip install polanon` |
|
|
99
|
+
| Also places, and names and organisations no rule can see | `pip install "polanon[ner]"` then `python -m spacy download pl_core_news_md` |
|
|
100
|
+
| The stage-2 judges (a local Ollama model, or a remote one) | `pip install "polanon[judge]"` |
|
|
101
|
+
|
|
102
|
+
Then use `default_guards()` with the model, or `default_guards(with_ner=False)` without it.
|
|
103
|
+
Another spaCy model is one argument away: `default_guards(spacy_model="pl_core_news_lg")`.
|
|
104
|
+
|
|
105
|
+
## From the command line
|
|
106
|
+
|
|
107
|
+
Text comes from the arguments or, without any, from standard input. Masked text goes to stdout
|
|
108
|
+
and the mapping to stderr, so a pipeline can keep the two apart:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
polanon "Jan Kowalski, PESEL 90010112349"
|
|
112
|
+
cat umowa.txt | polanon --ner > masked.txt 2> mapping.json
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
A non-zero exit status means stderr holds an error rather than the mapping. In Windows
|
|
116
|
+
PowerShell 5.1, read the file with `Get-Content -Encoding UTF8 umowa.txt | polanon`: a plain
|
|
117
|
+
`cat` there re-encodes Polish letters before they reach the program, and a garbled name is not
|
|
118
|
+
recognised.
|
|
119
|
+
|
|
120
|
+
## Streaming
|
|
121
|
+
|
|
122
|
+
A provider streams arbitrary fragments, so `<OSOBA_1>` can arrive as `<OSO` and `BA_1>`. The
|
|
123
|
+
restorer holds back anything that could still grow into a token:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
restorer = Anonymizer.stream_restorer(result.mapping)
|
|
127
|
+
for chunk in stream: # "Dzień dobry <OSO", "BA_1>, PESEL <PESEL", "_1> zweryfikowany."
|
|
128
|
+
print(restorer.feed(chunk), end="")
|
|
129
|
+
print(restorer.flush())
|
|
130
|
+
# Dzień dobry Jana Kowalskiego, PESEL 90010112349 zweryfikowany.
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## With Claude, GPT or a local model
|
|
134
|
+
|
|
135
|
+
Mask before the call, restore while the answer streams. With the Anthropic SDK:
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
import anthropic
|
|
139
|
+
from polanon import Anonymizer, default_guards
|
|
140
|
+
|
|
141
|
+
anonymizer = Anonymizer(default_guards(with_ner=False))
|
|
142
|
+
masked = anonymizer.anonymize_sync(question)
|
|
143
|
+
restorer = Anonymizer.stream_restorer(masked.mapping)
|
|
144
|
+
|
|
145
|
+
with anthropic.Anthropic().messages.stream(
|
|
146
|
+
model="claude-opus-5-5",
|
|
147
|
+
max_tokens=64000,
|
|
148
|
+
system="Personal data is replaced by tokens such as <OSOBA_1>. Copy a token exactly as written.",
|
|
149
|
+
messages=[{"role": "user", "content": masked.anonymized}],
|
|
150
|
+
) as stream:
|
|
151
|
+
for text in stream.text_stream:
|
|
152
|
+
print(restorer.feed(text), end="")
|
|
153
|
+
print(restorer.flush())
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Telling the model what the tokens are keeps it from rewriting them. The
|
|
157
|
+
[examples](https://github.com/boromat/polanon/tree/main/examples) go further:
|
|
158
|
+
`openai_chat.py` does the same for any OpenAI-compatible endpoint (OpenAI, OpenRouter, Azure, vLLM,
|
|
159
|
+
a local Ollama), and `anthropic_chat.py` holds a conversation whose history keeps masked text only,
|
|
160
|
+
so an answer restored for the reader is never sent back in the clear. Both are tested in CI against
|
|
161
|
+
the real SDKs, offline.
|
|
162
|
+
|
|
163
|
+
## What it detects
|
|
164
|
+
|
|
165
|
+
Checksum-validated Polish identifiers (PESEL, NIP, REGON, ID card, IBAN, payment cards), KRS numbers
|
|
166
|
+
next to their label, e-mail addresses and phone numbers, street addresses and postcodes, IPv4 and
|
|
167
|
+
IPv6, MAC addresses, vehicle registrations after their label ("nr rej."), VINs, passport numbers,
|
|
168
|
+
birth dates, API keys, tokens and passwords, Polish first names with their surnames, and company
|
|
169
|
+
names anchored on a legal form ("Sp. z o.o.", "S.A.", "Ltd", "GmbH"). With the `ner` extra, a spaCy
|
|
170
|
+
model adds places and the names nothing else marks. An optional second stage lets a small local
|
|
171
|
+
model review the *masked* text for what still identifies someone by context. Other dates are
|
|
172
|
+
recognised too, but left readable unless you ask for them.
|
|
173
|
+
|
|
174
|
+
Text copied from PDFs, Word or macOS is read as a person would read it: letters written as a base
|
|
175
|
+
letter plus an accent, no-break spaces and en dashes inside numbers, and invisible zero-width
|
|
176
|
+
characters do not hide a value. The mapping keeps each value as it was first written.
|
|
177
|
+
|
|
178
|
+
On the annotated sample documents in the repository, the rules alone fully mask 85% of the
|
|
179
|
+
values and the rules with `pl_core_news_md` mask 98%, with nothing left half-masked. Ten short
|
|
180
|
+
documents are a regression check, not a research benchmark: see the
|
|
181
|
+
[benchmark notes](https://github.com/boromat/polanon/blob/main/benchmarks/README.md) for what is
|
|
182
|
+
missed and why, and for Microsoft Presidio on the same documents with the same model: 65% fully
|
|
183
|
+
masked, 74% at its most lenient.
|
|
184
|
+
|
|
185
|
+
## Worth knowing
|
|
186
|
+
|
|
187
|
+
**Tokens are stable per conversation, not global.** Pass the mapping from the latest result to
|
|
188
|
+
the next call: the same value keeps the same token, and a token is never reused for another
|
|
189
|
+
value. The mapping you pass in is read, not changed, so keep the one each result returns:
|
|
190
|
+
|
|
191
|
+
```python
|
|
192
|
+
mapping = result.mapping
|
|
193
|
+
second = anonymizer.anonymize_sync("Ten sam PESEL 90010112349", mapping)
|
|
194
|
+
second.anonymized # "Ten sam PESEL <PESEL_1>"
|
|
195
|
+
mapping = second.mapping # everything so far, for the next turn
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
**A token restores the form it replaced.** Masking happens on the text as written, so a name masked
|
|
199
|
+
in the genitive comes back in the genitive. Ask the model to keep tokens where the grammar allows it
|
|
200
|
+
("dla <OSOBA_1>"), or restore into a template you control. Spellings that differ only in letter
|
|
201
|
+
case, in the kind of whitespace (a no-break space, a line break, a run of spaces) or in invisible
|
|
202
|
+
characters are the same value, and so are an identifier's groups written apart or together: they
|
|
203
|
+
share its token, and restore writes the spelling seen first. Passwords and custom types are the
|
|
204
|
+
exception, since two passwords that differ in case are two passwords, and a password never shares
|
|
205
|
+
a token with a name that reads alike. A one-word name found in all lowercase keeps a token of its
|
|
206
|
+
own too, for the reason below.
|
|
207
|
+
|
|
208
|
+
**Every occurrence of a masked value is masked**, also where no detector would recognise it on its
|
|
209
|
+
own: as written, in capitals, across a line break, with a zero-width character in it, with its
|
|
210
|
+
groups spaced differently, and around whatever a detector found inside it. Two exceptions keep ordinary words readable. A capitalised
|
|
211
|
+
one-word name is not matched in all lowercase, because Zima, Kot and Wilk are surnames and "zima"
|
|
212
|
+
is winter; identifiers such as a VIN or an e-mail address are matched in any case. A number shorter
|
|
213
|
+
than six digits, such as a PIN, is not matched inside a longer number, a date, a time or an
|
|
214
|
+
amount: "15.04.2025", "2024–2025", "2025,50" and "20250" stay readable when the PIN is 2025, while
|
|
215
|
+
"w 2025 roku" and "PIN/PUK 2025/36912874" do not.
|
|
216
|
+
|
|
217
|
+
**A token you write stays yours.** A placeholder such as `<PESEL_1>` in your own text is never handed
|
|
218
|
+
to real data. It joins the mapping as standing for itself, so the reservation holds in later turns
|
|
219
|
+
too.
|
|
220
|
+
|
|
221
|
+
**The mapping is the sensitive half.** `result.anonymized` holds a token for everything the
|
|
222
|
+
detectors found, and only what they missed in the clear; `result.mapping` is the key that undoes
|
|
223
|
+
it. Store the mapping the way you would store the original document.
|
|
224
|
+
|
|
225
|
+
**Replaying a conversation?** `mask_known_values(text, mapping)` masks every value a mapping already
|
|
226
|
+
holds in text that was masked earlier, such as history sent again after a new value got its token.
|
|
227
|
+
The same call checks a payload before it leaves: `mask_known_values(payload, mapping) == payload`
|
|
228
|
+
means no value the mapping holds is left where the library would mask it.
|
|
229
|
+
|
|
230
|
+
**Words of your own.** In-house names no rule can know, such as a project codename or a client,
|
|
231
|
+
go on a denylist and are masked wherever they appear, in any case and across line breaks:
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
guards = default_guards(denylist=["Projekt Wichura", "Zakład Nowak"], with_ner=False)
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
**Checking the answer.** After restoring, `anonymizer.sweep(restored, mapping)` reports tokens the
|
|
238
|
+
model invented and personal data its answer added: `report.is_clean`, or `report.unknown_tokens`
|
|
239
|
+
and `report.new_spans` to look at.
|
|
240
|
+
|
|
241
|
+
## Choosing what to mask
|
|
242
|
+
|
|
243
|
+
`Anonymizer(guards)` masks `DEFAULT_ENABLED`: every type in `ENTITY_TYPES` except `DATE`, because a
|
|
244
|
+
date rarely identifies anyone and masking every deadline leaves the model unable to answer. A birth
|
|
245
|
+
date does identify someone, so a date after "ur.", "data urodzenia", "DOB" or "geb." is its own
|
|
246
|
+
type, `BIRTH_DATE`, and stays masked. Pass `enabled` to choose for yourself:
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
from polanon import DEFAULT_ENABLED, ENTITY_TYPES, Anonymizer, default_guards
|
|
250
|
+
|
|
251
|
+
guards = default_guards(with_ner=False)
|
|
252
|
+
Anonymizer(guards, enabled=DEFAULT_ENABLED | {"DATE"}) # dates too
|
|
253
|
+
Anonymizer(guards, enabled={"PESEL", "NIP", "IBAN"}) # identifiers only
|
|
254
|
+
list(ENTITY_TYPES) # every name you can use
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
A name that is neither built in nor declared in a guard's `entity_types` raises `ValueError`, so a
|
|
258
|
+
typo such as `"PERSONS"` cannot quietly switch masking off.
|
|
259
|
+
|
|
260
|
+
## A second opinion: the judge
|
|
261
|
+
|
|
262
|
+
Rules and a language model miss what identifies someone only by context: "the finance director of
|
|
263
|
+
the only steelworks in town". A judge is a small model that reads the *masked* text after stage 1
|
|
264
|
+
and flags such fragments. Run it locally with Ollama, so the review never leaves your network:
|
|
265
|
+
|
|
266
|
+
```python
|
|
267
|
+
import asyncio
|
|
268
|
+
|
|
269
|
+
from polanon import Anonymizer, JudgeError, default_guards
|
|
270
|
+
from polanon.judge import OllamaJudge
|
|
271
|
+
|
|
272
|
+
anonymizer = Anonymizer(
|
|
273
|
+
default_guards(),
|
|
274
|
+
judge=OllamaJudge(model="qwen2.5:1.5b"),
|
|
275
|
+
judge_fallback="block", # refuse to go on if the judge is down
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
async def main(text: str) -> None:
|
|
280
|
+
try:
|
|
281
|
+
result = await anonymizer.anonymize(text)
|
|
282
|
+
except JudgeError:
|
|
283
|
+
... # the judge was down, and "block" means nothing goes out
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
asyncio.run(main("..."))
|
|
287
|
+
```
|
|
288
|
+
|
|
289
|
+
It needs `pip install "polanon[ner,judge]"`, the spaCy model, a running Ollama and
|
|
290
|
+
`ollama pull qwen2.5:1.5b`. With `judge_fallback="send_stage1"` the call goes on when the judge
|
|
291
|
+
is down, and `result.judge_failed` says that stage 1 went out alone.
|
|
292
|
+
|
|
293
|
+
Judges are asynchronous, so `anonymize_sync` refuses an anonymizer that has one. `RemoteJudge` asks an
|
|
294
|
+
OpenAI-compatible endpoint instead; the masked text then leaves your network, which makes it a
|
|
295
|
+
testing aid rather than a privacy control.
|
|
296
|
+
|
|
297
|
+
## Names that should stay readable
|
|
298
|
+
|
|
299
|
+
A language model flags "Facebook" or "Google Ads" as an organisation, which masks the very thing a
|
|
300
|
+
question is about. Pass an allowlist and those detections are dropped before tokens are assigned:
|
|
301
|
+
|
|
302
|
+
```python
|
|
303
|
+
from polanon import WELL_KNOWN_NAMES, Allowlist, Anonymizer, default_guards
|
|
304
|
+
|
|
305
|
+
anonymizer = Anonymizer(default_guards(), allowlist=Allowlist(WELL_KNOWN_NAMES))
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
A one-word name also matches its Polish case forms ("Facebooku", "Canvie"). Only detections of
|
|
309
|
+
names are dropped, and only when every word in them is on the list; identifiers, the denylist and
|
|
310
|
+
values a conversation has already masked stay masked.
|
|
311
|
+
|
|
312
|
+
## The pieces
|
|
313
|
+
|
|
314
|
+
| Module | What it is for |
|
|
315
|
+
|---|---|
|
|
316
|
+
| `guards/` | deterministic detectors. Each returns `Span`s in original-text coordinates |
|
|
317
|
+
| `pipeline.py` | runs the guards in a thread, resolves overlaps, assigns stable tokens |
|
|
318
|
+
| `policy.py` | the entity taxonomy and the token naming (`<OSOBA_1>`, `<PESEL_1>`) |
|
|
319
|
+
| `judge/` | the optional second stage: a model reviews the masked text for what still identifies |
|
|
320
|
+
| `stream_restore.py` | puts originals back into a streamed reply without splitting a token |
|
|
321
|
+
| `output_sweep.py` | flags invented tokens and personal data the answer introduced |
|
|
322
|
+
|
|
323
|
+
## Adding a detector
|
|
324
|
+
|
|
325
|
+
Subclass `Guard`, return spans in original-text coordinates and respect the `enabled` set. A type of
|
|
326
|
+
your own goes in `entity_types`, which turns it on by default; it becomes the label of its tokens,
|
|
327
|
+
so it is written in uppercase letters and underscores:
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
import re
|
|
331
|
+
|
|
332
|
+
from polanon import Anonymizer, Guard, Span, default_guards
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class EmployeeIdGuard(Guard):
|
|
336
|
+
name = "employee_id"
|
|
337
|
+
entity_types = frozenset({"EMPLOYEE_ID"})
|
|
338
|
+
|
|
339
|
+
def detect(self, text: str, enabled: set[str]) -> list[Span]:
|
|
340
|
+
if "EMPLOYEE_ID" not in enabled:
|
|
341
|
+
return []
|
|
342
|
+
return [
|
|
343
|
+
Span(m.start(), m.end(), "EMPLOYEE_ID", 1.0, self.name)
|
|
344
|
+
for m in re.finditer(r"\bEMP-\d{5}\b", text)
|
|
345
|
+
]
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
anonymizer = Anonymizer([*default_guards(with_ner=False), EmployeeIdGuard()])
|
|
349
|
+
anonymizer.anonymize_sync("Wniosek EMP-11111").anonymized # "Wniosek <EMPLOYEE_ID_1>"
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
The score decides who wins where detections overlap: 1.0 for a value a checksum, a label or an
|
|
353
|
+
official format vouches for, 0.85 for a statistical guess; the judge's findings score 0.5. A secret
|
|
354
|
+
is masked whole, with whatever was detected inside it. A span outside the text, a type spelled any
|
|
355
|
+
other way, or one spelled like a built-in token label (`OSOBA`, `DATA`) raises `ValueError` naming
|
|
356
|
+
your guard.
|
|
357
|
+
|
|
358
|
+
## Before you rely on it
|
|
359
|
+
|
|
360
|
+
Masking is pseudonymisation, not anonymisation in the legal sense, and no detector catches
|
|
361
|
+
everything. The [threat model](https://github.com/boromat/polanon/blob/main/docs/THREAT_MODEL.md) says what is covered, what is
|
|
362
|
+
not, and what stays your application's job, starting with keeping the mapping safe.
|
|
363
|
+
|
|
364
|
+
polanon was built as the masking engine of piigate, a self-hosted hub that lets a team use Claude
|
|
365
|
+
or GPT on real documents without sending personal data to the provider; the hub is not published
|
|
366
|
+
yet. The library knows nothing about it — no HTTP, no database, no settings — and a test fails if
|
|
367
|
+
it ever starts to.
|
|
368
|
+
|
|
369
|
+
## Contributing
|
|
370
|
+
|
|
371
|
+
Bug reports, missed values and new detectors are welcome: see
|
|
372
|
+
[CONTRIBUTING.md](https://github.com/boromat/polanon/blob/main/CONTRIBUTING.md). A way to get an original value past the
|
|
373
|
+
masking is a security report, made privately as [SECURITY.md](https://github.com/boromat/polanon/blob/main/SECURITY.md)
|
|
374
|
+
describes.
|
|
375
|
+
|
|
376
|
+
MIT licensed.
|