isomorphism 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- isomorphism-0.1.0/.github/workflows/ci.yml +25 -0
- isomorphism-0.1.0/.github/workflows/release.yml +38 -0
- isomorphism-0.1.0/.gitignore +9 -0
- isomorphism-0.1.0/CHANGELOG.md +10 -0
- isomorphism-0.1.0/LICENSE +21 -0
- isomorphism-0.1.0/PKG-INFO +94 -0
- isomorphism-0.1.0/README.md +71 -0
- isomorphism-0.1.0/RELEASE_CHECKLIST.md +22 -0
- isomorphism-0.1.0/docs/getting-started.md +40 -0
- isomorphism-0.1.0/docs/introduction.md +21 -0
- isomorphism-0.1.0/docs/legal.md +12 -0
- isomorphism-0.1.0/docs/nav.json +7 -0
- isomorphism-0.1.0/docs/reference/api.md +38 -0
- isomorphism-0.1.0/docs/reference/recognisers.md +18 -0
- isomorphism-0.1.0/pyproject.toml +55 -0
- isomorphism-0.1.0/src/isomorphism/__init__.py +8 -0
- isomorphism-0.1.0/src/isomorphism/_checks.py +29 -0
- isomorphism-0.1.0/src/isomorphism/core.py +115 -0
- isomorphism-0.1.0/src/isomorphism/detectors.py +85 -0
- isomorphism-0.1.0/src/isomorphism/py.typed +0 -0
- isomorphism-0.1.0/tests/__init__.py +0 -0
- isomorphism-0.1.0/tests/cases.py +140 -0
- isomorphism-0.1.0/tests/measure.py +38 -0
- isomorphism-0.1.0/tests/test_checks.py +22 -0
- isomorphism-0.1.0/tests/test_core.py +109 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
check:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
strategy:
|
|
15
|
+
matrix:
|
|
16
|
+
python: ["3.12", "3.13"]
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: astral-sh/setup-uv@v5
|
|
20
|
+
- run: uv venv --python ${{ matrix.python }}
|
|
21
|
+
- run: uv pip install -e . pytest ruff mypy
|
|
22
|
+
- run: .venv/bin/ruff check .
|
|
23
|
+
- run: .venv/bin/ruff format --check .
|
|
24
|
+
- run: .venv/bin/mypy
|
|
25
|
+
- run: .venv/bin/pytest -q
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Tag v<version> -> tests, build, publish to PyPI (trusted publishing, no token in the repo).
|
|
4
|
+
on:
|
|
5
|
+
push:
|
|
6
|
+
tags: ["v*"]
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: astral-sh/setup-uv@v5
|
|
17
|
+
- name: Tag matches pyproject version
|
|
18
|
+
run: test "v$(grep -m1 '^version' pyproject.toml | cut -d'"' -f2)" = "${GITHUB_REF_NAME}"
|
|
19
|
+
- run: uv venv && uv pip install -e . pytest ruff mypy
|
|
20
|
+
- run: .venv/bin/ruff check . && .venv/bin/mypy && .venv/bin/pytest -q
|
|
21
|
+
- run: uv build
|
|
22
|
+
- uses: actions/upload-artifact@v4
|
|
23
|
+
with:
|
|
24
|
+
name: dist
|
|
25
|
+
path: dist/
|
|
26
|
+
|
|
27
|
+
publish:
|
|
28
|
+
needs: build
|
|
29
|
+
runs-on: ubuntu-latest
|
|
30
|
+
environment: pypi
|
|
31
|
+
permissions:
|
|
32
|
+
id-token: write
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/download-artifact@v4
|
|
35
|
+
with:
|
|
36
|
+
name: dist
|
|
37
|
+
path: dist/
|
|
38
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0
|
|
4
|
+
|
|
5
|
+
First release.
|
|
6
|
+
|
|
7
|
+
- Recognisers for AHV numbers (EAN-13), IBANs of Switzerland and Liechtenstein (mod 97), UID numbers (mod 11), Swiss phone numbers and email addresses.
|
|
8
|
+
- `anonymize(text, lang=None, keep=())` returns the text with placeholders and a mapping; `restore(text, mapping)` reverses it; `detect(text)` returns the hits only.
|
|
9
|
+
- Same value, same placeholder. Findings carry kind, position, confidence, recogniser and checksum result, never the value.
|
|
10
|
+
- Invented test texts in German, French, Italian and English with hit rate and false alarms per kind.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Seya
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: isomorphism
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reversible anonymisation of Swiss personal data (AHV, IBAN, UID, phone, email) before text goes to a language model. Fully local, no dependencies.
|
|
5
|
+
Project-URL: Homepage, https://packages.sweber.dev/isomorphism
|
|
6
|
+
Project-URL: Documentation, https://packages.sweber.dev/isomorphism/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/Weber-Development/isomorphism
|
|
8
|
+
Project-URL: Changelog, https://github.com/Weber-Development/isomorphism/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Seya Weber
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: ahv,anonymization,dsg,iban,llm,pii,privacy,pseudonymization,switzerland
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Security
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# isomorphism
|
|
25
|
+
|
|
26
|
+
Reversible anonymisation of Swiss personal data for Python. It replaces AHV numbers, IBANs, UID numbers, phone numbers and email addresses with placeholders before a text goes to a language model, and puts the real values back into the answer.
|
|
27
|
+
|
|
28
|
+
- **Local only.** No network calls, no telemetry, no dependencies, nothing downloaded at runtime.
|
|
29
|
+
- **Reversible.** `anonymize(text)` returns the text with placeholders and a mapping, `restore(text, mapping)` undoes it.
|
|
30
|
+
- **Same value, same placeholder.** The model can follow that `[AHV_1]` appears three times.
|
|
31
|
+
- **Checksums.** AHV (EAN-13), IBAN (mod 97) and UID (mod 11) are verified. Numbers that look right but fail the check are reported anyway, with lower confidence.
|
|
32
|
+
- **Audit-friendly.** Every hit comes back as kind, position, confidence and recogniser, without the value.
|
|
33
|
+
|
|
34
|
+
Python 3.12 or newer. MIT license.
|
|
35
|
+
|
|
36
|
+
```sh
|
|
37
|
+
pip install isomorphism
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from isomorphism import anonymize, restore
|
|
42
|
+
|
|
43
|
+
text = "Anna Muster, AHV 756.1234.5678.97, IBAN CH93 0076 2011 6238 5295 7, Tel. 071 123 45 67"
|
|
44
|
+
safe, mapping = anonymize(text, lang="de")
|
|
45
|
+
# 'Anna Muster, AHV [AHV_1], IBAN [IBAN_1], Tel. [PHONE_1]'
|
|
46
|
+
|
|
47
|
+
answer = call_your_model(safe) # the model only sees placeholders
|
|
48
|
+
print(restore(answer, mapping)) # real values are back
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
The mapping holds the real values. This package never logs or stores it; keeping it (in memory, per request) is up to you. `repr(mapping)` shows only the number of placeholders.
|
|
52
|
+
|
|
53
|
+
## What it finds
|
|
54
|
+
|
|
55
|
+
| Kind | Placeholder | Formats | Check |
|
|
56
|
+
|---|---|---|---|
|
|
57
|
+
| AHV number | `[AHV_n]` | `756.1234.5678.97`, with spaces or without separators | prefix 756, EAN-13 |
|
|
58
|
+
| IBAN (CH, LI) | `[IBAN_n]` | with or without spaces, upper or lower case | mod 97 |
|
|
59
|
+
| UID number | `[UID_n]` | `CHE-123.456.788`, `CHE123456788`; `MWST`/`TVA`/`IVA`/`HR` stay in the text | mod 11 |
|
|
60
|
+
| Phone (Switzerland) | `[PHONE_n]` | `+41 71 123 45 67`, `0041 …`, `+41 (0)71 …`, `071 123 45 67`, `071/123 45 67`, `(071) …`, `0711234567` | format |
|
|
61
|
+
| Email | `[EMAIL_n]` | | format |
|
|
62
|
+
|
|
63
|
+
`detect(text)` returns the hits without changing the text. Passing `keep=["info@gemeinde.example"]` leaves exact values untouched (case-insensitive), for example the address of an authority.
|
|
64
|
+
|
|
65
|
+
## Measured on the test cases
|
|
66
|
+
|
|
67
|
+
`tests/cases.py` holds invented Swiss texts (payslip extract, client email, receipt, tax query, French and Italian letters, written-out variants, numbers with wrong check digits, and a text full of look-alikes that must not match). The numbers below come from `python -m tests.measure`, and a test fails if they are out of date or if a listed value slips through.
|
|
68
|
+
|
|
69
|
+
<!-- measure:start -->
|
|
70
|
+
| Kind | Expected | Found | Missed | False alarms |
|
|
71
|
+
|---|---|---|---|---|
|
|
72
|
+
| AHV | 10 | 10 | 0 | 0 |
|
|
73
|
+
| IBAN | 10 | 10 | 0 | 0 |
|
|
74
|
+
| UID | 9 | 9 | 0 | 0 |
|
|
75
|
+
| PHONE | 15 | 15 | 0 | 0 |
|
|
76
|
+
| EMAIL | 6 | 6 | 0 | 0 |
|
|
77
|
+
<!-- measure:end -->
|
|
78
|
+
|
|
79
|
+
This is a small invented set, not a benchmark on real documents. Read it as a regression guard, and test with your own (invented) samples before you rely on it.
|
|
80
|
+
|
|
81
|
+
## Not in this version
|
|
82
|
+
|
|
83
|
+
Personal names, postal addresses, company names, dates of birth, account and card numbers, and restoring a streamed answer arrive in the next version. Until then names and addresses stay in the text. `lang` ("de", "fr", "it", "en") is accepted and checked already but has no effect yet. Foreign IBANs and the old 11-digit AHV format are not recognised.
|
|
84
|
+
|
|
85
|
+
No tool finds all personal data. Treat the result as one protective layer, not as a guarantee of anonymity.
|
|
86
|
+
|
|
87
|
+
## Development
|
|
88
|
+
|
|
89
|
+
```sh
|
|
90
|
+
uv venv && uv pip install -e . pytest ruff mypy
|
|
91
|
+
ruff check . && ruff format --check . && mypy && pytest
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Docs: <https://packages.sweber.dev/isomorphism/docs>
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# isomorphism
|
|
2
|
+
|
|
3
|
+
Reversible anonymisation of Swiss personal data for Python. It replaces AHV numbers, IBANs, UID numbers, phone numbers and email addresses with placeholders before a text goes to a language model, and puts the real values back into the answer.
|
|
4
|
+
|
|
5
|
+
- **Local only.** No network calls, no telemetry, no dependencies, nothing downloaded at runtime.
|
|
6
|
+
- **Reversible.** `anonymize(text)` returns the text with placeholders and a mapping, `restore(text, mapping)` undoes it.
|
|
7
|
+
- **Same value, same placeholder.** The model can follow that `[AHV_1]` appears three times.
|
|
8
|
+
- **Checksums.** AHV (EAN-13), IBAN (mod 97) and UID (mod 11) are verified. Numbers that look right but fail the check are reported anyway, with lower confidence.
|
|
9
|
+
- **Audit-friendly.** Every hit comes back as kind, position, confidence and recogniser, without the value.
|
|
10
|
+
|
|
11
|
+
Python 3.12 or newer. MIT license.
|
|
12
|
+
|
|
13
|
+
```sh
|
|
14
|
+
pip install isomorphism
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
from isomorphism import anonymize, restore
|
|
19
|
+
|
|
20
|
+
text = "Anna Muster, AHV 756.1234.5678.97, IBAN CH93 0076 2011 6238 5295 7, Tel. 071 123 45 67"
|
|
21
|
+
safe, mapping = anonymize(text, lang="de")
|
|
22
|
+
# 'Anna Muster, AHV [AHV_1], IBAN [IBAN_1], Tel. [PHONE_1]'
|
|
23
|
+
|
|
24
|
+
answer = call_your_model(safe) # the model only sees placeholders
|
|
25
|
+
print(restore(answer, mapping)) # real values are back
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The mapping holds the real values. This package never logs or stores it; keeping it (in memory, per request) is up to you. `repr(mapping)` shows only the number of placeholders.
|
|
29
|
+
|
|
30
|
+
## What it finds
|
|
31
|
+
|
|
32
|
+
| Kind | Placeholder | Formats | Check |
|
|
33
|
+
|---|---|---|---|
|
|
34
|
+
| AHV number | `[AHV_n]` | `756.1234.5678.97`, with spaces or without separators | prefix 756, EAN-13 |
|
|
35
|
+
| IBAN (CH, LI) | `[IBAN_n]` | with or without spaces, upper or lower case | mod 97 |
|
|
36
|
+
| UID number | `[UID_n]` | `CHE-123.456.788`, `CHE123456788`; `MWST`/`TVA`/`IVA`/`HR` stay in the text | mod 11 |
|
|
37
|
+
| Phone (Switzerland) | `[PHONE_n]` | `+41 71 123 45 67`, `0041 …`, `+41 (0)71 …`, `071 123 45 67`, `071/123 45 67`, `(071) …`, `0711234567` | format |
|
|
38
|
+
| Email | `[EMAIL_n]` | | format |
|
|
39
|
+
|
|
40
|
+
`detect(text)` returns the hits without changing the text. Passing `keep=["info@gemeinde.example"]` leaves exact values untouched (case-insensitive), for example the address of an authority.
|
|
41
|
+
|
|
42
|
+
## Measured on the test cases
|
|
43
|
+
|
|
44
|
+
`tests/cases.py` holds invented Swiss texts (payslip extract, client email, receipt, tax query, French and Italian letters, written-out variants, numbers with wrong check digits, and a text full of look-alikes that must not match). The numbers below come from `python -m tests.measure`, and a test fails if they are out of date or if a listed value slips through.
|
|
45
|
+
|
|
46
|
+
<!-- measure:start -->
|
|
47
|
+
| Kind | Expected | Found | Missed | False alarms |
|
|
48
|
+
|---|---|---|---|---|
|
|
49
|
+
| AHV | 10 | 10 | 0 | 0 |
|
|
50
|
+
| IBAN | 10 | 10 | 0 | 0 |
|
|
51
|
+
| UID | 9 | 9 | 0 | 0 |
|
|
52
|
+
| PHONE | 15 | 15 | 0 | 0 |
|
|
53
|
+
| EMAIL | 6 | 6 | 0 | 0 |
|
|
54
|
+
<!-- measure:end -->
|
|
55
|
+
|
|
56
|
+
This is a small invented set, not a benchmark on real documents. Read it as a regression guard, and test with your own (invented) samples before you rely on it.
|
|
57
|
+
|
|
58
|
+
## Not in this version
|
|
59
|
+
|
|
60
|
+
Personal names, postal addresses, company names, dates of birth, account and card numbers, and restoring a streamed answer arrive in the next version. Until then names and addresses stay in the text. `lang` ("de", "fr", "it", "en") is accepted and checked already but has no effect yet. Foreign IBANs and the old 11-digit AHV format are not recognised.
|
|
61
|
+
|
|
62
|
+
No tool finds all personal data. Treat the result as one protective layer, not as a guarantee of anonymity.
|
|
63
|
+
|
|
64
|
+
## Development
|
|
65
|
+
|
|
66
|
+
```sh
|
|
67
|
+
uv venv && uv pip install -e . pytest ruff mypy
|
|
68
|
+
ruff check . && ruff format --check . && mypy && pytest
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Docs: <https://packages.sweber.dev/isomorphism/docs>
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Release checklist (package-launch)
|
|
2
|
+
|
|
3
|
+
| Item | Status |
|
|
4
|
+
|---|---|
|
|
5
|
+
| Repo `Weber-Development/isomorphism` | private; make it public before docs can render (read from GitHub) |
|
|
6
|
+
| PyPI `isomorphism` 0.1.0 | waits for the PyPI account and a trusted publisher (see below) |
|
|
7
|
+
| packages.sweber.dev | entry, docs config, demo, blog post: sxwxbxr/portfoliov3 PR |
|
|
8
|
+
| Pro | not planned for 0.1; core is MIT |
|
|
9
|
+
| Polar | not needed (free only) |
|
|
10
|
+
|
|
11
|
+
## Publishing (Seya)
|
|
12
|
+
|
|
13
|
+
1. Create the PyPI account (2FA on) if there is none.
|
|
14
|
+
2. On pypi.org: Your projects, Publishing, **Add a new pending publisher**: project `isomorphism`,
|
|
15
|
+
owner `Weber-Development`, repository `isomorphism`, workflow `release.yml`, environment `pypi`.
|
|
16
|
+
3. In GitHub, repo Settings, Environments: create `pypi`.
|
|
17
|
+
4. Tag `v0.1.0` on `main` (the Release workflow builds, tests and publishes). No token is stored anywhere.
|
|
18
|
+
|
|
19
|
+
## Next version (0.2.0)
|
|
20
|
+
|
|
21
|
+
Names, addresses, company names, dates of birth, account and card numbers, streaming `restore`,
|
|
22
|
+
optional Presidio backend for names in running text.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Getting started
|
|
3
|
+
description: Install Isomorphism, anonymise a text, call a model and restore the answer.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
```sh
|
|
7
|
+
pip install isomorphism
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
Python 3.12 or newer.
|
|
11
|
+
|
|
12
|
+
## Anonymise and restore
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from isomorphism import anonymize, restore
|
|
16
|
+
|
|
17
|
+
text = "AHV 756.1234.5678.97, IBAN CH93 0076 2011 6238 5295 7, UID CHE-123.456.788 MWST"
|
|
18
|
+
safe, mapping = anonymize(text, lang="de")
|
|
19
|
+
print(safe) # AHV [AHV_1], IBAN [IBAN_1], UID [UID_1] MWST
|
|
20
|
+
|
|
21
|
+
answer = call_your_model(safe)
|
|
22
|
+
print(restore(answer, mapping))
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Keep the mapping for as long as the request runs, for example in a variable or your request context. If you need to keep it longer, `dict(mapping)` gives you a plain dictionary to store. The package never writes it anywhere.
|
|
26
|
+
|
|
27
|
+
## Keep some values
|
|
28
|
+
|
|
29
|
+
```python
|
|
30
|
+
safe, mapping = anonymize(text, keep=["info@gemeinde.example"])
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Audit log
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
for f in mapping.findings:
|
|
37
|
+
print(f.kind, f.start, f.end, f.confidence, f.detector, f.checksum_ok)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Positions refer to the original text. The findings contain no values, so they are safe to log.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Why Isomorphism
|
|
3
|
+
description: Replace Swiss personal data with placeholders before a model call and restore it afterwards, fully local.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
A language model does not need to know a client's AHV number, IBAN or phone number to summarise a letter or answer a question about it. Isomorphism replaces such values with placeholders like `[AHV_1]` before the text leaves your system, and puts the real values back into the answer.
|
|
7
|
+
|
|
8
|
+
- **Swiss formats first.** AHV numbers, IBANs of Switzerland and Liechtenstein, UID numbers, Swiss phone numbers and email addresses, with the check digits that Swiss documents carry.
|
|
9
|
+
- **Reversible.** The same value always gets the same placeholder, so the model can follow who is meant. `restore` swaps the real values back in.
|
|
10
|
+
- **Local.** Pure Python, no dependencies, no network calls, no telemetry. The mapping with the real values never leaves your process unless you send it somewhere.
|
|
11
|
+
- **Audit-friendly.** Each hit reports kind, position, confidence and recogniser, never the value.
|
|
12
|
+
|
|
13
|
+
## Limits
|
|
14
|
+
|
|
15
|
+
Isomorphism recognises patterns. It does not understand context, and it cannot promise that a text is free of personal data. Names and postal addresses are not recognised in this version. See [Legal notes](legal.md).
|
|
16
|
+
|
|
17
|
+
## Packages
|
|
18
|
+
|
|
19
|
+
| Package | License | Content |
|
|
20
|
+
|---|---|---|
|
|
21
|
+
| `isomorphism` (PyPI) | MIT | Recognisers, `anonymize`, `restore`, `detect` |
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Legal notes
|
|
3
|
+
description: What Isomorphism does and does not promise.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Isomorphism is a technical tool. It does not provide legal advice, and using it does not by itself make a processing operation compliant with the Swiss Federal Act on Data Protection (FADP/DSG), the GDPR or professional secrecy rules.
|
|
7
|
+
|
|
8
|
+
- Recognition is pattern based. A value written in an unusual way, or a kind of data the version does not cover yet, stays in the text.
|
|
9
|
+
- Placeholders replace values, but the surrounding text can still identify a person.
|
|
10
|
+
- The mapping contains the real values. Treat it as personal data.
|
|
11
|
+
|
|
12
|
+
You remain responsible for what you send to a model.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: API
|
|
3
|
+
description: anonymize, restore, detect, Finding and Mapping.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
## `anonymize(text, lang=None, *, keep=())`
|
|
7
|
+
|
|
8
|
+
Returns `(text_with_placeholders, mapping)`.
|
|
9
|
+
|
|
10
|
+
- `lang`: `"de"`, `"fr"`, `"it"`, `"en"` or `None` (a region such as `"de-CH"` is accepted). Anything else raises `ValueError`. It has no effect yet; it will select context words for names and addresses.
|
|
11
|
+
- `keep`: values that stay in the text. Matching is exact and case-insensitive.
|
|
12
|
+
|
|
13
|
+
Identical values get the same placeholder inside one call. A placeholder that already stands in the input text, like a literal `[AHV_1]`, is never reused.
|
|
14
|
+
|
|
15
|
+
## `restore(text, mapping)`
|
|
16
|
+
|
|
17
|
+
Replaces placeholders with the real values in one pass. Models sometimes change the case or add spaces inside the brackets (`[ahv_1]`, `[ AHV_1 ]`); both still match. Placeholders that are not in the mapping stay unchanged. `mapping` can be any `dict[str, str]`, for example one you loaded from your own storage.
|
|
18
|
+
|
|
19
|
+
## `detect(text, lang=None, *, keep=())`
|
|
20
|
+
|
|
21
|
+
Returns `list[Finding]` without changing the text.
|
|
22
|
+
|
|
23
|
+
## `Finding`
|
|
24
|
+
|
|
25
|
+
| Field | Meaning |
|
|
26
|
+
|---|---|
|
|
27
|
+
| `kind` | `AHV`, `IBAN`, `UID`, `PHONE`, `EMAIL` |
|
|
28
|
+
| `start`, `end` | Position in the original text |
|
|
29
|
+
| `confidence` | 0 to 1 |
|
|
30
|
+
| `detector` | Name of the recogniser, for example `ahv-ean13` |
|
|
31
|
+
| `checksum_ok` | `True` or `False` for AHV, IBAN and UID, `None` for the others |
|
|
32
|
+
| `placeholder` | The placeholder used (empty from `detect`) |
|
|
33
|
+
|
|
34
|
+
A `Finding` holds no value.
|
|
35
|
+
|
|
36
|
+
## `Mapping`
|
|
37
|
+
|
|
38
|
+
A `dict` from placeholder to real value with a `findings` list. `repr()` and `str()` show only the number of placeholders, so the values do not end up in logs by accident. The package does not log anything.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Recognisers
|
|
3
|
+
description: Formats, checks and confidence per kind.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
| Kind | Confidence | Notes |
|
|
7
|
+
|---|---|---|
|
|
8
|
+
| `AHV` | 0.99 valid, 0.60 wrong check digit | Dotted form. Without dots: 0.90 and 0.40 |
|
|
9
|
+
| `IBAN` | 0.99 valid, 0.60 wrong check digit | `CH` and `LI`, 21 characters |
|
|
10
|
+
| `UID` | 0.98 valid, 0.60 wrong check digit | `CHE` and nine digits. `MWST`, `TVA`, `IVA`, `HR` are not part of the match |
|
|
11
|
+
| `PHONE` | 0.95 with `+41` or `0041`, 0.85 with separators, 0.60 for ten bare digits | Area code 2 to 9 |
|
|
12
|
+
| `EMAIL` | 0.95 | |
|
|
13
|
+
|
|
14
|
+
Wrong check digits are reported on purpose: a typo in a document is still personal data.
|
|
15
|
+
|
|
16
|
+
When two hits overlap, the higher confidence wins, then the longer match. This is why the digits inside an IBAN do not also come out as a phone number.
|
|
17
|
+
|
|
18
|
+
Not recognised: foreign IBANs, the old 11-digit AHV format, service numbers such as `0800 …`.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "isomorphism"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Reversible anonymisation of Swiss personal data (AHV, IBAN, UID, phone, email) before text goes to a language model. Fully local, no dependencies."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.12"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Seya Weber" }]
|
|
14
|
+
keywords = ["anonymization", "pseudonymization", "pii", "switzerland", "ahv", "iban", "llm", "privacy", "dsg"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"License :: OSI Approved :: MIT License",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Topic :: Security",
|
|
23
|
+
"Typing :: Typed",
|
|
24
|
+
]
|
|
25
|
+
dependencies = []
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://packages.sweber.dev/isomorphism"
|
|
29
|
+
Documentation = "https://packages.sweber.dev/isomorphism/docs"
|
|
30
|
+
Repository = "https://github.com/Weber-Development/isomorphism"
|
|
31
|
+
Changelog = "https://github.com/Weber-Development/isomorphism/blob/main/CHANGELOG.md"
|
|
32
|
+
|
|
33
|
+
[dependency-groups]
|
|
34
|
+
dev = ["pytest>=8", "ruff>=0.8", "mypy>=1.13"]
|
|
35
|
+
|
|
36
|
+
[tool.hatch.build.targets.wheel]
|
|
37
|
+
packages = ["src/isomorphism"]
|
|
38
|
+
|
|
39
|
+
[tool.pytest.ini_options]
|
|
40
|
+
testpaths = ["tests"]
|
|
41
|
+
pythonpath = ["."]
|
|
42
|
+
|
|
43
|
+
[tool.ruff]
|
|
44
|
+
line-length = 100
|
|
45
|
+
target-version = "py312"
|
|
46
|
+
|
|
47
|
+
[tool.ruff.lint]
|
|
48
|
+
select = ["E", "F", "I", "UP", "B", "SIM"]
|
|
49
|
+
|
|
50
|
+
[tool.ruff.lint.per-file-ignores]
|
|
51
|
+
"tests/cases.py" = ["E501"] # fixture lines are long on purpose
|
|
52
|
+
|
|
53
|
+
[tool.mypy]
|
|
54
|
+
strict = true
|
|
55
|
+
files = ["src", "tests"]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Reversible anonymisation of Swiss personal data. Local only: no network, no telemetry."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
|
|
5
|
+
from .core import Finding, Mapping, anonymize, detect, restore
|
|
6
|
+
|
|
7
|
+
__version__ = version("isomorphism")
|
|
8
|
+
__all__ = ["Finding", "Mapping", "__version__", "anonymize", "detect", "restore"]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Check-digit algorithms. Pure functions on digit strings, no I/O."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def ahv_ok(digits: str) -> bool:
|
|
7
|
+
"""AHV number: 13 digits, prefix 756, EAN-13 check digit."""
|
|
8
|
+
if len(digits) != 13 or not digits.startswith("756"):
|
|
9
|
+
return False
|
|
10
|
+
total = sum(int(c) * (3 if i % 2 else 1) for i, c in enumerate(digits[:12]))
|
|
11
|
+
return (10 - total % 10) % 10 == int(digits[12])
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def iban_ok(compact: str) -> bool:
|
|
15
|
+
"""ISO 13616 mod-97 on an IBAN without spaces."""
|
|
16
|
+
if len(compact) < 5 or not compact.isalnum():
|
|
17
|
+
return False
|
|
18
|
+
rearranged = compact[4:] + compact[:4]
|
|
19
|
+
return int("".join(str(int(c, 36)) for c in rearranged)) % 97 == 1
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def uid_ok(digits: str) -> bool:
|
|
23
|
+
"""UID (CHE) number: 9 digits, mod-11 check digit with weights 5,4,3,2,7,6,5,4."""
|
|
24
|
+
if len(digits) != 9:
|
|
25
|
+
return False
|
|
26
|
+
check = (
|
|
27
|
+
11 - sum(int(c) * w for c, w in zip(digits[:8], (5, 4, 3, 2, 7, 6, 5, 4), strict=True)) % 11
|
|
28
|
+
)
|
|
29
|
+
return check != 10 and check % 11 == int(digits[8])
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""anonymize / restore. The mapping holds real values: this module never logs or stores it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
from collections.abc import Mapping as AbstractMapping
|
|
8
|
+
from dataclasses import dataclass, replace
|
|
9
|
+
|
|
10
|
+
from .detectors import DETECTORS
|
|
11
|
+
|
|
12
|
+
LANGS = frozenset({"de", "fr", "it", "en"})
|
|
13
|
+
|
|
14
|
+
# Tolerant on restore: models sometimes change case or add spaces inside the brackets.
|
|
15
|
+
_TOKEN = re.compile(r"\[\s*([A-Za-z]+)\s*_\s*(\d+)\s*\]")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True, slots=True)
|
|
19
|
+
class Finding:
|
|
20
|
+
"""One hit for the audit log. Positions refer to the original text; the value is not kept."""
|
|
21
|
+
|
|
22
|
+
kind: str
|
|
23
|
+
start: int
|
|
24
|
+
end: int
|
|
25
|
+
confidence: float
|
|
26
|
+
detector: str
|
|
27
|
+
checksum_ok: bool | None = None
|
|
28
|
+
placeholder: str = ""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Mapping(dict[str, str]):
|
|
32
|
+
"""placeholder -> real value, plus the findings. repr() hides the values."""
|
|
33
|
+
|
|
34
|
+
findings: list[Finding]
|
|
35
|
+
|
|
36
|
+
def __init__(self) -> None:
|
|
37
|
+
super().__init__()
|
|
38
|
+
self.findings = []
|
|
39
|
+
|
|
40
|
+
def __repr__(self) -> str:
|
|
41
|
+
return f"Mapping(<{len(self)} placeholders, values hidden>)"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _check_lang(lang: str | None) -> None:
|
|
45
|
+
# shortcut: lang has no effect yet (the checksum recognisers are language-neutral);
|
|
46
|
+
# it selects context words once names and addresses arrive.
|
|
47
|
+
if lang is not None and lang.lower().split("-")[0] not in LANGS:
|
|
48
|
+
raise ValueError(f"unsupported lang {lang!r}, expected one of {sorted(LANGS)}")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def detect(text: str, lang: str | None = None, *, keep: Iterable[str] = ()) -> list[Finding]:
|
|
52
|
+
"""Find personal data. Overlaps go to the higher confidence, then the longer match.
|
|
53
|
+
|
|
54
|
+
``keep``: exact values (case-insensitive) that must stay in the text.
|
|
55
|
+
"""
|
|
56
|
+
_check_lang(lang)
|
|
57
|
+
kept = {k.casefold() for k in keep}
|
|
58
|
+
found = sorted(
|
|
59
|
+
(c for d in DETECTORS for c in d(text) if text[c.start : c.end].casefold() not in kept),
|
|
60
|
+
key=lambda c: (-c.confidence, c.start - c.end, c.start),
|
|
61
|
+
)
|
|
62
|
+
taken: list[tuple[int, int]] = []
|
|
63
|
+
out: list[Finding] = []
|
|
64
|
+
for c in found:
|
|
65
|
+
if all(c.end <= s or c.start >= e for s, e in taken):
|
|
66
|
+
taken.append((c.start, c.end))
|
|
67
|
+
out.append(Finding(c.kind, c.start, c.end, c.confidence, c.detector, c.checksum_ok))
|
|
68
|
+
return sorted(out, key=lambda f: f.start)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def anonymize(
|
|
72
|
+
text: str, lang: str | None = None, *, keep: Iterable[str] = ()
|
|
73
|
+
) -> tuple[str, Mapping]:
|
|
74
|
+
"""Replace personal data by placeholders like ``[AHV_1]``.
|
|
75
|
+
|
|
76
|
+
Identical values get the same placeholder. Returns ``(text, mapping)``; the mapping has
|
|
77
|
+
the real values and a ``findings`` list. Storing the mapping is up to the caller.
|
|
78
|
+
"""
|
|
79
|
+
# Never reuse a placeholder that already stands in the text.
|
|
80
|
+
in_text = {_name(m) for m in _TOKEN.finditer(text)}
|
|
81
|
+
mapping = Mapping()
|
|
82
|
+
by_value: dict[tuple[str, str], str] = {}
|
|
83
|
+
counters: dict[str, int] = {}
|
|
84
|
+
parts: list[str] = []
|
|
85
|
+
pos = 0
|
|
86
|
+
for f in detect(text, lang, keep=keep):
|
|
87
|
+
value = text[f.start : f.end]
|
|
88
|
+
ph = by_value.get((f.kind, value))
|
|
89
|
+
if ph is None:
|
|
90
|
+
n = counters.get(f.kind, 0)
|
|
91
|
+
while True:
|
|
92
|
+
n += 1
|
|
93
|
+
ph = f"[{f.kind}_{n}]"
|
|
94
|
+
if ph not in in_text:
|
|
95
|
+
break
|
|
96
|
+
counters[f.kind] = n
|
|
97
|
+
by_value[(f.kind, value)] = ph
|
|
98
|
+
mapping[ph] = value
|
|
99
|
+
mapping.findings.append(replace(f, placeholder=ph))
|
|
100
|
+
parts += (text[pos : f.start], ph)
|
|
101
|
+
pos = f.end
|
|
102
|
+
parts.append(text[pos:])
|
|
103
|
+
return "".join(parts), mapping
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _name(m: re.Match[str]) -> str:
|
|
107
|
+
return f"[{m[1].upper()}_{int(m[2])}]"
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def restore(text: str, mapping: AbstractMapping[str, str]) -> str:
|
|
111
|
+
"""Put the real values back. Single pass, so restored values are never rescanned.
|
|
112
|
+
|
|
113
|
+
Unknown placeholders stay unchanged.
|
|
114
|
+
"""
|
|
115
|
+
return _TOKEN.sub(lambda m: mapping.get(_name(m), m[0]), text)
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Swiss recognisers. Each yields candidates; overlaps are resolved in core."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Callable, Iterator
|
|
7
|
+
from dataclasses import dataclass, replace
|
|
8
|
+
|
|
9
|
+
from ._checks import ahv_ok, iban_ok, uid_ok
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(frozen=True, slots=True)
|
|
13
|
+
class Candidate:
|
|
14
|
+
kind: str
|
|
15
|
+
start: int
|
|
16
|
+
end: int
|
|
17
|
+
confidence: float
|
|
18
|
+
detector: str
|
|
19
|
+
checksum_ok: bool | None = None
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _digits(s: str) -> str:
|
|
23
|
+
return "".join(c for c in s if c.isdigit())
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _checked(
|
|
27
|
+
kind: str,
|
|
28
|
+
detector: str,
|
|
29
|
+
pattern: re.Pattern[str],
|
|
30
|
+
ok: Callable[[str], bool],
|
|
31
|
+
clean: Callable[[str], str],
|
|
32
|
+
text: str,
|
|
33
|
+
valid: float,
|
|
34
|
+
invalid: float,
|
|
35
|
+
) -> Iterator[Candidate]:
|
|
36
|
+
# Invalid check digits are still reported, with lower confidence (better too many than too few).
|
|
37
|
+
for m in pattern.finditer(text):
|
|
38
|
+
good = ok(clean(m[0]))
|
|
39
|
+
yield Candidate(kind, m.start(), m.end(), valid if good else invalid, detector, good)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
_AHV = re.compile(r"(?<!\d)756[ .\-]?\d{4}[ .\-]?\d{4}[ .\-]?\d{2}(?!\d)")
|
|
43
|
+
_IBAN = re.compile(r"(?<![A-Za-z0-9])(?:CH|LI)\d{2}(?: ?[0-9A-Z]){17}(?![A-Za-z0-9])", re.I)
|
|
44
|
+
_UID = re.compile(r"(?<![A-Za-z0-9])CHE[- ]?\d{3}[. ]?\d{3}[. ]?\d{3}(?!\d)", re.I)
|
|
45
|
+
_EMAIL = re.compile(r"(?<![\w.+\-])[\w.+\-]+@[\w\-]+(?:\.[\w\-]+)*\.[^\W\d_]{2,}")
|
|
46
|
+
_PHONE = re.compile(
|
|
47
|
+
r"""(?<![\w+])(?<!\d[ .\-/])
|
|
48
|
+
(?:(?:\+|00)41[ .\-]?(?:\(0\)[ .\-]?)?|\(?0)
|
|
49
|
+
[1-9]\d\)?
|
|
50
|
+
[ .\-/]?\d{3}[ .\-]?\d{2}[ .\-]?\d{2}(?!\d)""",
|
|
51
|
+
re.X,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def ahv(text: str) -> Iterator[Candidate]:
|
|
56
|
+
for c in _checked("AHV", "ahv-ean13", _AHV, ahv_ok, _digits, text, 0.99, 0.6):
|
|
57
|
+
if "." not in text[c.start : c.end]: # bare digits are weaker evidence than the dotted form
|
|
58
|
+
c = replace(c, confidence=0.9 if c.checksum_ok else 0.4)
|
|
59
|
+
yield c
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def iban(text: str) -> Iterator[Candidate]:
|
|
63
|
+
yield from _checked(
|
|
64
|
+
"IBAN", "iban-mod97", _IBAN, iban_ok, lambda s: s.replace(" ", "").upper(), text, 0.99, 0.6
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def uid(text: str) -> Iterator[Candidate]:
|
|
69
|
+
yield from _checked("UID", "uid-mod11", _UID, uid_ok, _digits, text, 0.98, 0.6)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def email(text: str) -> Iterator[Candidate]:
|
|
73
|
+
for m in _EMAIL.finditer(text):
|
|
74
|
+
yield Candidate("EMAIL", m.start(), m.end(), 0.95, "email-regex")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def phone(text: str) -> Iterator[Candidate]:
|
|
78
|
+
for m in _PHONE.finditer(text):
|
|
79
|
+
s = m[0]
|
|
80
|
+
international = s.startswith(("+", "00"))
|
|
81
|
+
confidence = 0.95 if international else 0.85 if not s.isdigit() else 0.6
|
|
82
|
+
yield Candidate("PHONE", m.start(), m.end(), confidence, "phone-ch")
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
DETECTORS: tuple[Callable[[str], Iterator[Candidate]], ...] = (ahv, iban, uid, email, phone)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Invented Swiss texts with the expected hits marked inline as ⟦KIND|value⟧.
|
|
2
|
+
|
|
3
|
+
Everything here is made up. Names, street addresses and companies appear in the texts but are not
|
|
4
|
+
marked yet: those kinds arrive with the next version. Numbers with a wrong check digit are marked
|
|
5
|
+
too, because they must be found as well.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
_MARK = re.compile(r"⟦([A-Z]+)\|([^⟧]*)⟧")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class Case:
|
|
18
|
+
name: str
|
|
19
|
+
lang: str
|
|
20
|
+
text: str # without markup
|
|
21
|
+
expected: frozenset[tuple[str, int, int]] # kind, start, end
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _parse(name: str, lang: str, marked: str) -> Case:
|
|
25
|
+
out, expected, pos = [], set(), 0
|
|
26
|
+
for m in _MARK.finditer(marked):
|
|
27
|
+
out.append(marked[pos : m.start()])
|
|
28
|
+
start = sum(len(p) for p in out)
|
|
29
|
+
out.append(m[2])
|
|
30
|
+
expected.add((m[1], start, start + len(m[2])))
|
|
31
|
+
pos = m.end()
|
|
32
|
+
out.append(marked[pos:])
|
|
33
|
+
return Case(name, lang, "".join(out), frozenset(expected))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
_RAW = [
|
|
37
|
+
(
|
|
38
|
+
"lohnausweis",
|
|
39
|
+
"de",
|
|
40
|
+
"""\
|
|
41
|
+
Lohnausweis 2025 (Auszug)
|
|
42
|
+
Arbeitnehmerin: Anna Muster, Musterstrasse 12, 9000 St. Gallen
|
|
43
|
+
AHV-Nr.: ⟦AHV|756.5260.1815.91⟧
|
|
44
|
+
Arbeitgeberin: Beispiel Treuhand AG, ⟦UID|CHE-937.865.795⟧ MWST
|
|
45
|
+
Bruttolohn 78'450.00, Nettolohn 66'918.40, Auszahlung auf ⟦IBAN|CH10 7971 1471 0497 4650 7⟧.
|
|
46
|
+
Rückfragen: Lohnbuchhaltung, ⟦PHONE|071 123 45 67⟧, ⟦EMAIL|lohn@beispiel-treuhand.example⟧
|
|
47
|
+
""",
|
|
48
|
+
),
|
|
49
|
+
(
|
|
50
|
+
"mandanten-mail",
|
|
51
|
+
"de",
|
|
52
|
+
"""\
|
|
53
|
+
Grüezi Frau Keller
|
|
54
|
+
|
|
55
|
+
Wir haben die Unterlagen für die Steuererklärung erhalten. Meine Mobilnummer ist
|
|
56
|
+
⟦PHONE|+41 79 123 45 67⟧, erreichbar bin ich auch unter ⟦PHONE|079 123 45 67⟧.
|
|
57
|
+
Die Rückzahlung bitte auf ⟦IBAN|CH21 5291 7034 2366 7127 6⟧ überweisen.
|
|
58
|
+
Meine AHV-Nummer: ⟦AHV|756.0830.1661.38⟧ (die Nummer meines Mannes: ⟦AHV|756.1860.9139.07⟧).
|
|
59
|
+
Freundliche Grüsse
|
|
60
|
+
Hans Muster, ⟦EMAIL|hans.muster@mail.example⟧
|
|
61
|
+
""",
|
|
62
|
+
),
|
|
63
|
+
(
|
|
64
|
+
"beleg",
|
|
65
|
+
"de",
|
|
66
|
+
"""\
|
|
67
|
+
Beleg 2025-0417, Rechnung Nr. 20250417, Referenz 21 00000 00003 13947 14300 09017
|
|
68
|
+
Lieferant: Muster Garage GmbH, ⟦UID|CHE-754.323.198⟧ HR, ⟦UID|CHE-487.574.914⟧ MWST
|
|
69
|
+
Betrag CHF 1'234.50 inkl. 8.1% MWST, fällig am 30.11.2026
|
|
70
|
+
Zahlbar an ⟦IBAN|CH19 8426 8465 6321 2233 0⟧ (kein Barzahlung), Tel. ⟦PHONE|+41 (0)71 987 65 43⟧
|
|
71
|
+
""",
|
|
72
|
+
),
|
|
73
|
+
(
|
|
74
|
+
"steuerrueckfrage",
|
|
75
|
+
"de",
|
|
76
|
+
"""\
|
|
77
|
+
Betreff: Steuerrückfrage Veranlagung 2025
|
|
78
|
+
Das Steueramt St. Gallen fragt nach dem Beleg zu Kontonummer ⟦IBAN|CH67 7924 4026 8599 5289 0⟧.
|
|
79
|
+
Bitte melden Sie sich unter ⟦PHONE|058 123 45 67⟧ oder per Mail an ⟦EMAIL|steueramt.anfrage@kanton.example⟧.
|
|
80
|
+
Die AHV-Nummer ⟦AHV|756.9960.3082.41⟧ stimmt nicht mit den Unterlagen überein.
|
|
81
|
+
""",
|
|
82
|
+
),
|
|
83
|
+
(
|
|
84
|
+
"courrier",
|
|
85
|
+
"fr",
|
|
86
|
+
"""\
|
|
87
|
+
Madame, Monsieur,
|
|
88
|
+
Veuillez trouver ci-joint le décompte pour M. Jean Dupont, n° AVS ⟦AHV|756.6281.9482.17⟧.
|
|
89
|
+
Le montant est à verser sur ⟦IBAN|CH35 7866 6617 6031 3721 5⟧. Numéro IDE: ⟦UID|CHE-186.252.760⟧ TVA.
|
|
90
|
+
Contact: ⟦PHONE|+41 21 123 45 67⟧ ou ⟦PHONE|021 123 45 67⟧, ⟦EMAIL|contact@fiduciaire.example⟧
|
|
91
|
+
""",
|
|
92
|
+
),
|
|
93
|
+
(
|
|
94
|
+
"lettera",
|
|
95
|
+
"it",
|
|
96
|
+
"""\
|
|
97
|
+
Gentile signora Bianchi,
|
|
98
|
+
il suo numero AVS è ⟦AHV|756.9935.1819.09⟧. Il pagamento va fatto sul conto ⟦IBAN|LI27 9010 9281 5901 3962 4⟧.
|
|
99
|
+
Numero IDI: ⟦UID|CHE-018.955.594⟧ IVA. Può chiamarci al ⟦PHONE|0041 91 123 45 67⟧ oppure scrivere a ⟦EMAIL|info@studio-fiduciario.example⟧.
|
|
100
|
+
""",
|
|
101
|
+
),
|
|
102
|
+
(
|
|
103
|
+
"mail-en",
|
|
104
|
+
"en",
|
|
105
|
+
"""\
|
|
106
|
+
Hi Anna,
|
|
107
|
+
please transfer the refund to ⟦IBAN|CH93 0076 2011 6238 5295 7⟧ (I cannot find my ⟦AHV|756.1234.5678.97⟧ card).
|
|
108
|
+
Our company is ⟦UID|CHE-123.456.788⟧; reach me on ⟦PHONE|+41 71 123 45 67⟧ or ⟦EMAIL|anna.muster@example.ch⟧.
|
|
109
|
+
""",
|
|
110
|
+
),
|
|
111
|
+
(
|
|
112
|
+
"formate",
|
|
113
|
+
"de",
|
|
114
|
+
"""\
|
|
115
|
+
Schreibweisen: ⟦AHV|7561234567897⟧, ⟦AHV|756 1234 5678 97⟧, ⟦IBAN|CH9300762011623852957⟧, ⟦IBAN|ch93 0076 2011 6238 5295 7⟧,
|
|
116
|
+
⟦UID|CHE123456788⟧, ⟦UID|CHE-123.456.788⟧, ⟦PHONE|0711234567⟧, ⟦PHONE|071/123 45 67⟧, ⟦PHONE|(071) 123 45 67⟧,
|
|
117
|
+
⟦PHONE|+41711234567⟧, ⟦PHONE|0041 71 123 45 67⟧, ⟦PHONE|071.123.45.67⟧
|
|
118
|
+
""",
|
|
119
|
+
),
|
|
120
|
+
(
|
|
121
|
+
"falsche-pruefziffer",
|
|
122
|
+
"de",
|
|
123
|
+
"""\
|
|
124
|
+
Tippfehler müssen trotzdem gefunden werden:
|
|
125
|
+
AHV ⟦AHV|756.1234.5678.98⟧, IBAN ⟦IBAN|CH93 0076 2011 6238 5295 8⟧, UID ⟦UID|CHE-123.456.789⟧.
|
|
126
|
+
""",
|
|
127
|
+
),
|
|
128
|
+
(
|
|
129
|
+
"keine-treffer",
|
|
130
|
+
"de",
|
|
131
|
+
"""\
|
|
132
|
+
Rechnung 2026-1102, Kunde 756-20, Artikel 0711234, Menge 12 Stk. zu CHF 1'290.00.
|
|
133
|
+
Datum 03.04.1981, Zeit 08.30 Uhr, PLZ 9000 St. Gallen, Seite 4 von 12, Version 3.13.1.
|
|
134
|
+
Tarif 5.2.0, Konto 1020, Debitoren 1100, Kreditoren 2000, Anzahl 12345678901.
|
|
135
|
+
Referenz 21 00000 00003 13947 14300 09017, Kontakt: siehe Impressum, Kapital 100'000.
|
|
136
|
+
""",
|
|
137
|
+
),
|
|
138
|
+
]
|
|
139
|
+
|
|
140
|
+
CASES = [_parse(*row) for row in _RAW]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Hit rate and false alarms per kind over tests/cases.py.
|
|
2
|
+
|
|
3
|
+
`python -m tests.measure` prints the README table.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections import Counter
|
|
9
|
+
|
|
10
|
+
from isomorphism import detect
|
|
11
|
+
|
|
12
|
+
from .cases import CASES
|
|
13
|
+
|
|
14
|
+
KINDS = ("AHV", "IBAN", "UID", "PHONE", "EMAIL")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def measure() -> dict[str, Counter[str]]:
|
|
18
|
+
stats: dict[str, Counter[str]] = {k: Counter() for k in KINDS}
|
|
19
|
+
for case in CASES:
|
|
20
|
+
found = {(f.kind, f.start, f.end) for f in detect(case.text, case.lang)}
|
|
21
|
+
for hit in case.expected & found:
|
|
22
|
+
stats[hit[0]]["tp"] += 1
|
|
23
|
+
for miss in case.expected - found:
|
|
24
|
+
stats[miss[0]]["fn"] += 1
|
|
25
|
+
for extra in found - case.expected:
|
|
26
|
+
stats[extra[0]]["fp"] += 1
|
|
27
|
+
return stats
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def table() -> str:
|
|
31
|
+
rows = ["| Kind | Expected | Found | Missed | False alarms |", "|---|---|---|---|---|"]
|
|
32
|
+
for kind, c in measure().items():
|
|
33
|
+
rows.append(f"| {kind} | {c['tp'] + c['fn']} | {c['tp']} | {c['fn']} | {c['fp']} |")
|
|
34
|
+
return "\n".join(rows)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
if __name__ == "__main__":
|
|
38
|
+
print(table())
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from isomorphism._checks import ahv_ok, iban_ok, uid_ok
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_ahv() -> None:
|
|
5
|
+
assert ahv_ok("7561234567897")
|
|
6
|
+
assert not ahv_ok("7561234567898") # wrong check digit
|
|
7
|
+
assert not ahv_ok("7551234567897") # wrong prefix
|
|
8
|
+
assert not ahv_ok("756123456789") # too short
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_iban() -> None:
|
|
12
|
+
assert iban_ok("CH9300762011623852957")
|
|
13
|
+
assert iban_ok("LI21088100002324013AA")
|
|
14
|
+
assert not iban_ok("CH9300762011623852958")
|
|
15
|
+
assert not iban_ok("CH93")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_uid() -> None:
|
|
19
|
+
assert uid_ok("123456788")
|
|
20
|
+
assert not uid_ok("123456789")
|
|
21
|
+
assert not uid_ok("12345678")
|
|
22
|
+
assert not uid_ok("100000007") # mod-11 check would be 10, never valid
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import pickle
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from isomorphism import anonymize, detect, restore
|
|
7
|
+
from tests.cases import CASES
|
|
8
|
+
from tests.measure import KINDS, measure
|
|
9
|
+
|
|
10
|
+
AHV = "756.1234.5678.97"
|
|
11
|
+
IBAN = "CH93 0076 2011 6238 5295 7"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_roundtrip_all_cases() -> None:
|
|
15
|
+
for case in CASES:
|
|
16
|
+
anon, mapping = anonymize(case.text, case.lang)
|
|
17
|
+
assert restore(anon, mapping) == case.text, case.name
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_same_value_same_placeholder() -> None:
|
|
21
|
+
anon, mapping = anonymize(f"{AHV} und {AHV} und 756.9960.3082.41")
|
|
22
|
+
assert anon == "[AHV_1] und [AHV_1] und [AHV_2]"
|
|
23
|
+
assert mapping["[AHV_1]"] == AHV
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_no_original_value_left() -> None:
|
|
27
|
+
for case in CASES:
|
|
28
|
+
anon, mapping = anonymize(case.text, case.lang)
|
|
29
|
+
for value in mapping.values():
|
|
30
|
+
assert value not in anon, case.name
|
|
31
|
+
assert len(mapping.findings) == len(case.expected), case.name
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_findings_carry_no_value() -> None:
|
|
35
|
+
_, mapping = anonymize(f"{AHV}, {IBAN}")
|
|
36
|
+
f = mapping.findings[0]
|
|
37
|
+
assert (f.kind, f.start, f.end, f.detector, f.checksum_ok) == ("AHV", 0, 16, "ahv-ean13", True)
|
|
38
|
+
assert 0 < f.confidence <= 1
|
|
39
|
+
for secret in (AHV, IBAN, "7561234567897"):
|
|
40
|
+
assert secret not in repr(mapping) + str(mapping) + repr(mapping.findings)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_invalid_checksum_reported_with_lower_confidence() -> None:
|
|
44
|
+
good, bad = detect("756.1234.5678.97 756.1234.5678.98")
|
|
45
|
+
assert good.checksum_ok and not bad.checksum_ok
|
|
46
|
+
assert bad.confidence < good.confidence
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_keep_exceptions() -> None:
|
|
50
|
+
text = "Mail an Steueramt.SG@Kanton.example oder anna@muster.example"
|
|
51
|
+
anon, _ = anonymize(text, keep=["steueramt.sg@kanton.example"])
|
|
52
|
+
assert anon == "Mail an Steueramt.SG@Kanton.example oder [EMAIL_1]"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_literal_placeholder_in_text_is_not_reused() -> None:
|
|
56
|
+
anon, mapping = anonymize(f"Vorlage [AHV_1]: {AHV}")
|
|
57
|
+
assert anon == "Vorlage [AHV_1]: [AHV_2]"
|
|
58
|
+
assert restore(anon, mapping) == f"Vorlage [AHV_1]: {AHV}"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_restore_tolerates_model_changes_and_keeps_unknown() -> None:
|
|
62
|
+
_, mapping = anonymize(AHV)
|
|
63
|
+
assert restore("[ahv_1] / [ AHV_01 ] / [AHV_2] / [OTHER_1]", mapping) == (
|
|
64
|
+
f"{AHV} / {AHV} / [AHV_2] / [OTHER_1]"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_restore_is_single_pass() -> None:
|
|
69
|
+
assert restore("[A_1] [B_1]", {"[A_1]": "[B_1]", "[B_1]": "x"}) == "[B_1] x"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_overlap_prefers_checksum_match() -> None:
|
|
73
|
+
# The digits inside an IBAN must not also come out as a phone number.
|
|
74
|
+
anon, _ = anonymize("CH93 0076 2011 6238 5295 7")
|
|
75
|
+
assert anon == "[IBAN_1]"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_lang() -> None:
|
|
79
|
+
anonymize("x", "de-CH")
|
|
80
|
+
with pytest.raises(ValueError, match="unsupported lang"):
|
|
81
|
+
anonymize("x", "xx")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_nothing_is_logged(caplog: pytest.LogCaptureFixture) -> None:
|
|
85
|
+
with caplog.at_level(logging.DEBUG):
|
|
86
|
+
anon, mapping = anonymize(f"{AHV} {IBAN}")
|
|
87
|
+
restore(anon, mapping)
|
|
88
|
+
assert caplog.records == []
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_mapping_survives_pickle() -> None:
|
|
92
|
+
_, mapping = anonymize(AHV)
|
|
93
|
+
assert dict(pickle.loads(pickle.dumps(mapping))) == {"[AHV_1]": AHV} # noqa: S301
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_acceptance_numbers() -> None:
|
|
97
|
+
stats = measure()
|
|
98
|
+
for kind in KINDS:
|
|
99
|
+
assert stats[kind]["fn"] == 0, f"{kind}: a listed value slipped through"
|
|
100
|
+
assert stats[kind]["fp"] == 0, f"{kind}: false alarm"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_readme_table_is_current() -> None:
|
|
104
|
+
from pathlib import Path
|
|
105
|
+
|
|
106
|
+
from tests.measure import table
|
|
107
|
+
|
|
108
|
+
readme = Path(__file__).parent.parent / "README.md"
|
|
109
|
+
assert f"<!-- measure:start -->\n{table()}\n<!-- measure:end -->" in readme.read_text()
|