hushmask 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hushmask-0.1.0/LICENSE +21 -0
- hushmask-0.1.0/PKG-INFO +176 -0
- hushmask-0.1.0/README.md +136 -0
- hushmask-0.1.0/pyproject.toml +94 -0
- hushmask-0.1.0/pyproject.toml.orig +75 -0
- hushmask-0.1.0/src/hushmask/__init__.py +24 -0
- hushmask-0.1.0/src/hushmask/addresses.py +105 -0
- hushmask-0.1.0/src/hushmask/cli.py +292 -0
- hushmask-0.1.0/src/hushmask/detectors.py +228 -0
- hushmask-0.1.0/src/hushmask/entities.py +40 -0
- hushmask-0.1.0/src/hushmask/evaluation/__init__.py +1 -0
- hushmask-0.1.0/src/hushmask/evaluation/ai4privacy.py +56 -0
- hushmask-0.1.0/src/hushmask/evaluation/detection.py +60 -0
- hushmask-0.1.0/src/hushmask/evaluation/metrics.py +151 -0
- hushmask-0.1.0/src/hushmask/evaluation/report.py +90 -0
- hushmask-0.1.0/src/hushmask/evaluation/synthetic.py +260 -0
- hushmask-0.1.0/src/hushmask/evaluation/task.py +141 -0
- hushmask-0.1.0/src/hushmask/masker.py +270 -0
- hushmask-0.1.0/src/hushmask/ner.py +174 -0
- hushmask-0.1.0/src/hushmask/onnx_ner.py +124 -0
- hushmask-0.1.0/src/hushmask/policy.py +36 -0
- hushmask-0.1.0/src/hushmask/proxy/__init__.py +16 -0
- hushmask-0.1.0/src/hushmask/proxy/admin.py +252 -0
- hushmask-0.1.0/src/hushmask/proxy/config.py +154 -0
- hushmask-0.1.0/src/hushmask/proxy/openai.py +143 -0
- hushmask-0.1.0/src/hushmask/proxy/server.py +280 -0
- hushmask-0.1.0/src/hushmask/proxy/store.py +190 -0
- hushmask-0.1.0/src/hushmask/py.typed +0 -0
- hushmask-0.1.0/src/hushmask/validators.py +104 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-cyrillic-400-normal-BSMlKf0J.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-cyrillic-400-normal-CEL4l2ZJ.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-cyrillic-500-normal-Ael50iVv.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-cyrillic-500-normal-Bq9vWWag.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-cyrillic-ext-400-normal-DMdlQ8Kv.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-cyrillic-ext-400-normal-xuaO2J-f.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-cyrillic-ext-500-normal-BIfNGwUT.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-cyrillic-ext-500-normal-BqneJy0T.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-latin-400-normal-CvHOgSBP.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-latin-400-normal-DMJ8VG8y.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-latin-500-normal-CB9ihrfo.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-latin-500-normal-DSY6xOcd.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-latin-ext-400-normal-BmRBH3aV.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-latin-ext-400-normal-D3D2R8hC.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-latin-ext-500-normal-CAhNIIs5.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-latin-ext-500-normal-CZ70TYgx.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-vietnamese-400-normal-BulugwFq.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-vietnamese-400-normal-DDuiU_S-.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-vietnamese-500-normal-C8zxqsMH.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/ibm-plex-mono-vietnamese-500-normal-DZ4AoWbu.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/index-Ci9I59Ct.js +17 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/index-apC-urZ9.css +2 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-400-normal-BkiRe4WW.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-400-normal-DPhJBilQ.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-500-normal-Ba39e-CX.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-500-normal-rf9C4Thp.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-700-normal-BkH0uJ1o.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-700-normal-Dz-okVa0.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-ext-400-normal-DHVTfbSM.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-ext-400-normal-DUPvg9bQ.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-ext-500-normal-BjmYCtCC.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-ext-500-normal-Ch1izu81.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-ext-700-normal-DDW2aNyx.woff +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/assets/schibsted-grotesk-latin-ext-700-normal-o210KhU4.woff2 +0 -0
- hushmask-0.1.0/src/hushmask/web_dist/favicon.svg +6 -0
- hushmask-0.1.0/src/hushmask/web_dist/index.html +15 -0
hushmask-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Bilal
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
hushmask-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: hushmask
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Hide names, emails, IBANs and addresses before text reaches an AI, and put them back in the answer.
|
|
5
|
+
Keywords: privacy,pii,llm,openai,proxy,anonymization,pseudonymization,gdpr,redaction
|
|
6
|
+
Author: Bilal
|
|
7
|
+
Author-email: Bilal <bilalrizvi2110@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Web Environment
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Security
|
|
19
|
+
Classifier: Topic :: Text Processing
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Dist: hushmask[proxy,ner] ; extra == 'all'
|
|
22
|
+
Requires-Dist: numpy>=2.5.5 ; extra == 'ner'
|
|
23
|
+
Requires-Dist: onnxruntime>=1.20 ; extra == 'ner'
|
|
24
|
+
Requires-Dist: tokenizers>=0.20 ; extra == 'ner'
|
|
25
|
+
Requires-Dist: gliner>=0.2.30 ; extra == 'ner-torch'
|
|
26
|
+
Requires-Dist: protobuf>=7.36.2 ; extra == 'ner-torch'
|
|
27
|
+
Requires-Dist: fastapi>=0.165.0 ; extra == 'proxy'
|
|
28
|
+
Requires-Dist: httpx>=0.28.1 ; extra == 'proxy'
|
|
29
|
+
Requires-Dist: uvicorn[standard]>=0.54.0 ; extra == 'proxy'
|
|
30
|
+
Requires-Python: >=3.12
|
|
31
|
+
Project-URL: Homepage, https://bilal-py.github.io/hushmask/
|
|
32
|
+
Project-URL: Source, https://github.com/bilal-py/hushmask
|
|
33
|
+
Project-URL: Issues, https://github.com/bilal-py/hushmask/issues
|
|
34
|
+
Project-URL: Evaluation, https://github.com/bilal-py/hushmask/blob/main/docs/eval/RESULTS.md
|
|
35
|
+
Provides-Extra: all
|
|
36
|
+
Provides-Extra: ner
|
|
37
|
+
Provides-Extra: ner-torch
|
|
38
|
+
Provides-Extra: proxy
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# hushmask
|
|
42
|
+
|
|
43
|
+
[](https://github.com/bilal-py/hushmask/actions/workflows/ci.yml)
|
|
44
|
+
· [Project page](https://bilal-py.github.io/hushmask/)
|
|
45
|
+
|
|
46
|
+
Hide personal data before text goes to an AI, and put the real values back in the answer. Runs on your machine.
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
please send the contract to Lena Hoffmann please send the contract to PERSON_0f24a5
|
|
50
|
+
(lena.hoffmann@example.de, +49 151 23456789) ─▶ (EMAIL_75333f, PHONE_311bb1)
|
|
51
|
+
The deposit goes to DE89 3704 0044 0532 0130 00 The deposit goes to IBAN_b55883
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The AI still understands the request; it just never learns who is involved. Any app that talks to an
|
|
55
|
+
OpenAI-style API can use it by changing one URL.
|
|
56
|
+
|
|
57
|
+
> **Status: early.** Text only. See [docs/PLAN.md](https://github.com/bilal-py/hushmask/blob/main/docs/PLAN.md)
|
|
58
|
+
> for the roadmap, known gaps and ideas.
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
uv tool install "hushmask[all]" # or: pipx install "hushmask[all]"
|
|
64
|
+
hushmask serve --open # dashboard at http://127.0.0.1:8700/dashboard/
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`[all]` = proxy + dashboard + the names model (downloaded on first use, ~0.6 GB, runs on CPU). For a lighter
|
|
68
|
+
install: `hushmask[proxy]` (no names model) or plain `hushmask` (library and CLI only).
|
|
69
|
+
|
|
70
|
+
## From source
|
|
71
|
+
|
|
72
|
+
```powershell
|
|
73
|
+
git clone https://github.com/bilal-py/hushmask && cd hushmask
|
|
74
|
+
.\setup.ps1 # macOS/Linux: ./setup.sh (add -Names / --names for the names model)
|
|
75
|
+
uv run hushmask serve --open # opens the dashboard at http://127.0.0.1:8700/dashboard/
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The setup script installs [uv](https://docs.astral.sh/uv/) if needed, the Python package, and builds the
|
|
79
|
+
dashboard (needs Node 20+; without Node you still get the proxy and CLI). Then, in the dashboard:
|
|
80
|
+
|
|
81
|
+
1. **Playground:** paste any text and see what an AI would receive.
|
|
82
|
+
2. **Apps → Add app:** pick a provider (Ollama for fully local, or OpenAI / Gemini / OpenRouter / Groq), then
|
|
83
|
+
copy the base URL and key it shows into your app.
|
|
84
|
+
3. **Playground → Send it to an AI:** run a real request through that app and see each stage.
|
|
85
|
+
|
|
86
|
+
Just want to try it on a file, no server:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
uv run hushmask mask examples/email.txt --known "PERSON=Lena Hoffmann"
|
|
90
|
+
uv run hushmask mask examples/email.txt --known "PERSON=Lena Hoffmann" --ner # also names nobody listed
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## How well it works
|
|
94
|
+
|
|
95
|
+
Share of personal values **fully** hidden on labelled data ([tables](https://github.com/bilal-py/hushmask/blob/main/docs/eval/RESULTS.md),
|
|
96
|
+
[method](https://github.com/bilal-py/hushmask/blob/main/docs/eval/README.md)):
|
|
97
|
+
|
|
98
|
+
| Data | Patterns | + your list | + names model | Over-masking (best setup) |
|
|
99
|
+
|---|---:|---:|---:|---|
|
|
100
|
+
| ai4privacy English (500 everyday texts) | 26.5% | n/a | **87.0%** | 200 ordinary words |
|
|
101
|
+
| ai4privacy German (300 everyday texts) | 25.6% | n/a | **90.0%** | 106 ordinary words |
|
|
102
|
+
| Synthetic bank statements (300) | 57.6% | 72.9% | **99.5%** | 1 of 1,340 merchants ("OLA CABS") |
|
|
103
|
+
|
|
104
|
+
**Does the AI still do the job?** A local `qwen2.5:7b` categorized 52 transactions: 94.2% correct seeing real
|
|
105
|
+
data, 92.3% seeing only tokens (one transaction apart), and every person's name came back correctly.
|
|
106
|
+
|
|
107
|
+
ai4privacy is the honest number; the synthetic statements were written by this project and flatter it. Most
|
|
108
|
+
leaks are random-digit "card numbers" that fail the card checksum (left alone on purpose) and street addresses.
|
|
109
|
+
Usernames, passwords, dates of birth and ID numbers aren't detected; they're reported separately.
|
|
110
|
+
|
|
111
|
+
## What it hides
|
|
112
|
+
|
|
113
|
+
| Kind | How |
|
|
114
|
+
|---|---|
|
|
115
|
+
| Your own values (name, address, IDs) | your "always hide" list: whole words, any case |
|
|
116
|
+
| Street addresses (DE/NL/FR/IT/ES, US/UK/AU formats) | patterns: street word + house number (+ postcode, city) |
|
|
117
|
+
| Other people's names, free-form addresses, cities | local names model (GLiNER via ONNX, optional, ~0.6 GB) |
|
|
118
|
+
| Email, phone (international and national formats) | patterns |
|
|
119
|
+
| Card numbers, IBAN | checksums (Luhn, mod 97) |
|
|
120
|
+
| Account numbers | next to "Account No." and similar |
|
|
121
|
+
| Indian IDs: Aadhaar, PAN, IFSC, UPI | checksums and formats |
|
|
122
|
+
|
|
123
|
+
Presets: **strict** hides everything above; **finance** keeps cities and bank codes so an AI can still
|
|
124
|
+
categorize spending. Same value, same token, every time; "Mr Becker" links to "Jonas Becker" as
|
|
125
|
+
`PERSON_<id>_1` so the AI knows it's the same person.
|
|
126
|
+
|
|
127
|
+
## Use it as a proxy
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
uv run hushmask serve # apps managed in the dashboard (~/.hushmask/apps.json)
|
|
131
|
+
uv run hushmask serve --upstream http://localhost:11434/v1 --known "PERSON=Lena Hoffmann" # one provider, no setup
|
|
132
|
+
uv run hushmask serve --config hushmask.toml # apps from a file, see hushmask.example.toml
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
In your app: base URL `http://127.0.0.1:8700/v1`, API key = the app's hushmask key (or, with `--upstream`,
|
|
136
|
+
your provider key, which is forwarded).
|
|
137
|
+
|
|
138
|
+
- Masks every message (system, user, history, tool calls and results) and restores the answer, including JSON.
|
|
139
|
+
- Refuses images, audio and files unless an app allows them (they can't be masked).
|
|
140
|
+
- Waits for the whole answer before replying (a token split across stream chunks can't be restored); clients
|
|
141
|
+
asking for a stream still work.
|
|
142
|
+
- Counts only, never values, in `GET /hushmask/stats` and the dashboard.
|
|
143
|
+
|
|
144
|
+
## Use it from Python
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from hushmask import EntityType, GlinerDetector, KnownValue, Masker, Policy
|
|
148
|
+
|
|
149
|
+
masker = Masker(
|
|
150
|
+
policy=Policy.preset("strict"),
|
|
151
|
+
known=[KnownValue("Lena Hoffmann", EntityType.PERSON)],
|
|
152
|
+
extra_detectors=[GlinerDetector()], # optional, needs the 'ner' extra
|
|
153
|
+
)
|
|
154
|
+
masked = masker.mask(text) # send masked.text to the AI
|
|
155
|
+
print(masker.restore(ai_answer).text) # real values back
|
|
156
|
+
data, info = masker.restore_json(parsed_json) # structured output
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## Safety rules
|
|
160
|
+
|
|
161
|
+
- **Fails closed:** masked text is scanned again; if anything detectable survived, it's refused, not sent.
|
|
162
|
+
- **Real values stay in memory** for the request. The CLI writes a vault file only if you pass `--vault`.
|
|
163
|
+
- **Dashboard is local only:** answers on localhost with a header other websites can't add.
|
|
164
|
+
- **Not anonymization:** what's left (dates, amounts, places) can still say a lot in combination.
|
|
165
|
+
|
|
166
|
+
## Development
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
uv sync --extra proxy --extra ner
|
|
170
|
+
uv run pytest && uv run ruff check . && uv run ruff format --check .
|
|
171
|
+
cd web && npm run dev # dashboard with hot reload, talks to a running `hushmask serve`
|
|
172
|
+
uv run hushmask eval detect # see docs/eval/README.md
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
CI runs tests, lint and the dashboard build on every push; pushes to `main` publish `site/` to GitHub Pages;
|
|
176
|
+
tags `v*` attach a ready-to-install wheel (dashboard included) to a GitHub release.
|
hushmask-0.1.0/README.md
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# hushmask
|
|
2
|
+
|
|
3
|
+
[](https://github.com/bilal-py/hushmask/actions/workflows/ci.yml)
|
|
4
|
+
· [Project page](https://bilal-py.github.io/hushmask/)
|
|
5
|
+
|
|
6
|
+
Hide personal data before text goes to an AI, and put the real values back in the answer. Runs on your machine.
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
please send the contract to Lena Hoffmann please send the contract to PERSON_0f24a5
|
|
10
|
+
(lena.hoffmann@example.de, +49 151 23456789) ─▶ (EMAIL_75333f, PHONE_311bb1)
|
|
11
|
+
The deposit goes to DE89 3704 0044 0532 0130 00 The deposit goes to IBAN_b55883
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
The AI still understands the request; it just never learns who is involved. Any app that talks to an
|
|
15
|
+
OpenAI-style API can use it by changing one URL.
|
|
16
|
+
|
|
17
|
+
> **Status: early.** Text only. See [docs/PLAN.md](https://github.com/bilal-py/hushmask/blob/main/docs/PLAN.md)
|
|
18
|
+
> for the roadmap, known gaps and ideas.
|
|
19
|
+
|
|
20
|
+
## Install
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
uv tool install "hushmask[all]" # or: pipx install "hushmask[all]"
|
|
24
|
+
hushmask serve --open # dashboard at http://127.0.0.1:8700/dashboard/
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
`[all]` = proxy + dashboard + the names model (downloaded on first use, ~0.6 GB, runs on CPU). For a lighter
|
|
28
|
+
install: `hushmask[proxy]` (no names model) or plain `hushmask` (library and CLI only).
|
|
29
|
+
|
|
30
|
+
## From source
|
|
31
|
+
|
|
32
|
+
```powershell
|
|
33
|
+
git clone https://github.com/bilal-py/hushmask && cd hushmask
|
|
34
|
+
.\setup.ps1 # macOS/Linux: ./setup.sh (add -Names / --names for the names model)
|
|
35
|
+
uv run hushmask serve --open # opens the dashboard at http://127.0.0.1:8700/dashboard/
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The setup script installs [uv](https://docs.astral.sh/uv/) if needed, the Python package, and builds the
|
|
39
|
+
dashboard (needs Node 20+; without Node you still get the proxy and CLI). Then, in the dashboard:
|
|
40
|
+
|
|
41
|
+
1. **Playground:** paste any text and see what an AI would receive.
|
|
42
|
+
2. **Apps → Add app:** pick a provider (Ollama for fully local, or OpenAI / Gemini / OpenRouter / Groq), then
|
|
43
|
+
copy the base URL and key it shows into your app.
|
|
44
|
+
3. **Playground → Send it to an AI:** run a real request through that app and see each stage.
|
|
45
|
+
|
|
46
|
+
Just want to try it on a file, no server:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
uv run hushmask mask examples/email.txt --known "PERSON=Lena Hoffmann"
|
|
50
|
+
uv run hushmask mask examples/email.txt --known "PERSON=Lena Hoffmann" --ner # also names nobody listed
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## How well it works
|
|
54
|
+
|
|
55
|
+
Share of personal values **fully** hidden on labelled data ([tables](https://github.com/bilal-py/hushmask/blob/main/docs/eval/RESULTS.md),
|
|
56
|
+
[method](https://github.com/bilal-py/hushmask/blob/main/docs/eval/README.md)):
|
|
57
|
+
|
|
58
|
+
| Data | Patterns | + your list | + names model | Over-masking (best setup) |
|
|
59
|
+
|---|---:|---:|---:|---|
|
|
60
|
+
| ai4privacy English (500 everyday texts) | 26.5% | n/a | **87.0%** | 200 ordinary words |
|
|
61
|
+
| ai4privacy German (300 everyday texts) | 25.6% | n/a | **90.0%** | 106 ordinary words |
|
|
62
|
+
| Synthetic bank statements (300) | 57.6% | 72.9% | **99.5%** | 1 of 1,340 merchants ("OLA CABS") |
|
|
63
|
+
|
|
64
|
+
**Does the AI still do the job?** A local `qwen2.5:7b` categorized 52 transactions: 94.2% correct seeing real
|
|
65
|
+
data, 92.3% seeing only tokens (one transaction apart), and every person's name came back correctly.
|
|
66
|
+
|
|
67
|
+
ai4privacy is the honest number; the synthetic statements were written by this project and flatter it. Most
|
|
68
|
+
leaks are random-digit "card numbers" that fail the card checksum (left alone on purpose) and street addresses.
|
|
69
|
+
Usernames, passwords, dates of birth and ID numbers aren't detected; they're reported separately.
|
|
70
|
+
|
|
71
|
+
## What it hides
|
|
72
|
+
|
|
73
|
+
| Kind | How |
|
|
74
|
+
|---|---|
|
|
75
|
+
| Your own values (name, address, IDs) | your "always hide" list: whole words, any case |
|
|
76
|
+
| Street addresses (DE/NL/FR/IT/ES, US/UK/AU formats) | patterns: street word + house number (+ postcode, city) |
|
|
77
|
+
| Other people's names, free-form addresses, cities | local names model (GLiNER via ONNX, optional, ~0.6 GB) |
|
|
78
|
+
| Email, phone (international and national formats) | patterns |
|
|
79
|
+
| Card numbers, IBAN | checksums (Luhn, mod 97) |
|
|
80
|
+
| Account numbers | next to "Account No." and similar |
|
|
81
|
+
| Indian IDs: Aadhaar, PAN, IFSC, UPI | checksums and formats |
|
|
82
|
+
|
|
83
|
+
Presets: **strict** hides everything above; **finance** keeps cities and bank codes so an AI can still
|
|
84
|
+
categorize spending. Same value, same token, every time; "Mr Becker" links to "Jonas Becker" as
|
|
85
|
+
`PERSON_<id>_1` so the AI knows it's the same person.
|
|
86
|
+
|
|
87
|
+
## Use it as a proxy
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
uv run hushmask serve # apps managed in the dashboard (~/.hushmask/apps.json)
|
|
91
|
+
uv run hushmask serve --upstream http://localhost:11434/v1 --known "PERSON=Lena Hoffmann" # one provider, no setup
|
|
92
|
+
uv run hushmask serve --config hushmask.toml # apps from a file, see hushmask.example.toml
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
In your app: base URL `http://127.0.0.1:8700/v1`, API key = the app's hushmask key (or, with `--upstream`,
|
|
96
|
+
your provider key, which is forwarded).
|
|
97
|
+
|
|
98
|
+
- Masks every message (system, user, history, tool calls and results) and restores the answer, including JSON.
|
|
99
|
+
- Refuses images, audio and files unless an app allows them (they can't be masked).
|
|
100
|
+
- Waits for the whole answer before replying (a token split across stream chunks can't be restored); clients
|
|
101
|
+
asking for a stream still work.
|
|
102
|
+
- Counts only, never values, in `GET /hushmask/stats` and the dashboard.
|
|
103
|
+
|
|
104
|
+
## Use it from Python
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from hushmask import EntityType, GlinerDetector, KnownValue, Masker, Policy
|
|
108
|
+
|
|
109
|
+
masker = Masker(
|
|
110
|
+
policy=Policy.preset("strict"),
|
|
111
|
+
known=[KnownValue("Lena Hoffmann", EntityType.PERSON)],
|
|
112
|
+
extra_detectors=[GlinerDetector()], # optional, needs the 'ner' extra
|
|
113
|
+
)
|
|
114
|
+
masked = masker.mask(text) # send masked.text to the AI
|
|
115
|
+
print(masker.restore(ai_answer).text) # real values back
|
|
116
|
+
data, info = masker.restore_json(parsed_json) # structured output
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Safety rules
|
|
120
|
+
|
|
121
|
+
- **Fails closed:** masked text is scanned again; if anything detectable survived, it's refused, not sent.
|
|
122
|
+
- **Real values stay in memory** for the request. The CLI writes a vault file only if you pass `--vault`.
|
|
123
|
+
- **Dashboard is local only:** answers on localhost with a header other websites can't add.
|
|
124
|
+
- **Not anonymization:** what's left (dates, amounts, places) can still say a lot in combination.
|
|
125
|
+
|
|
126
|
+
## Development
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
uv sync --extra proxy --extra ner
|
|
130
|
+
uv run pytest && uv run ruff check . && uv run ruff format --check .
|
|
131
|
+
cd web && npm run dev # dashboard with hot reload, talks to a running `hushmask serve`
|
|
132
|
+
uv run hushmask eval detect # see docs/eval/README.md
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
CI runs tests, lint and the dashboard build on every push; pushes to `main` publish `site/` to GitHub Pages;
|
|
136
|
+
tags `v*` attach a ready-to-install wheel (dashboard included) to a GitHub release.
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hushmask"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Hide names, emails, IBANs and addresses before text reaches an AI, and put them back in the answer."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.12"
|
|
9
|
+
dependencies = []
|
|
10
|
+
keywords = [
|
|
11
|
+
"privacy",
|
|
12
|
+
"pii",
|
|
13
|
+
"llm",
|
|
14
|
+
"openai",
|
|
15
|
+
"proxy",
|
|
16
|
+
"anonymization",
|
|
17
|
+
"pseudonymization",
|
|
18
|
+
"gdpr",
|
|
19
|
+
"redaction",
|
|
20
|
+
]
|
|
21
|
+
classifiers = [
|
|
22
|
+
"Development Status :: 3 - Alpha",
|
|
23
|
+
"Environment :: Web Environment",
|
|
24
|
+
"Intended Audience :: Developers",
|
|
25
|
+
"Intended Audience :: End Users/Desktop",
|
|
26
|
+
"Operating System :: OS Independent",
|
|
27
|
+
"Programming Language :: Python :: 3",
|
|
28
|
+
"Programming Language :: Python :: 3.12",
|
|
29
|
+
"Programming Language :: Python :: 3.13",
|
|
30
|
+
"Topic :: Security",
|
|
31
|
+
"Topic :: Text Processing",
|
|
32
|
+
"Typing :: Typed",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
[[project.authors]]
|
|
36
|
+
name = "Bilal"
|
|
37
|
+
email = "bilalrizvi2110@gmail.com"
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://bilal-py.github.io/hushmask/"
|
|
41
|
+
Source = "https://github.com/bilal-py/hushmask"
|
|
42
|
+
Issues = "https://github.com/bilal-py/hushmask/issues"
|
|
43
|
+
Evaluation = "https://github.com/bilal-py/hushmask/blob/main/docs/eval/RESULTS.md"
|
|
44
|
+
|
|
45
|
+
[project.scripts]
|
|
46
|
+
hushmask = "hushmask.cli:main"
|
|
47
|
+
|
|
48
|
+
[project.optional-dependencies]
|
|
49
|
+
all = ["hushmask[proxy,ner]"]
|
|
50
|
+
ner = [
|
|
51
|
+
"numpy>=2.5.5",
|
|
52
|
+
"onnxruntime>=1.20",
|
|
53
|
+
"tokenizers>=0.20",
|
|
54
|
+
]
|
|
55
|
+
proxy = [
|
|
56
|
+
"fastapi>=0.165.0",
|
|
57
|
+
"httpx>=0.28.1",
|
|
58
|
+
"uvicorn[standard]>=0.54.0",
|
|
59
|
+
]
|
|
60
|
+
ner-torch = [
|
|
61
|
+
"gliner>=0.2.30",
|
|
62
|
+
"protobuf>=7.36.2",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
[build-system]
|
|
66
|
+
requires = ["uv_build>=0.11.17,<0.12.0"]
|
|
67
|
+
build-backend = "uv_build"
|
|
68
|
+
|
|
69
|
+
[dependency-groups]
|
|
70
|
+
dev = [
|
|
71
|
+
"fastapi>=0.165.0",
|
|
72
|
+
"httpx>=0.28.1",
|
|
73
|
+
"pytest>=9.2.0",
|
|
74
|
+
"ruff>=0.17.6",
|
|
75
|
+
]
|
|
76
|
+
|
|
77
|
+
[tool.pytest.ini_options]
|
|
78
|
+
testpaths = ["tests"]
|
|
79
|
+
|
|
80
|
+
[tool.ruff]
|
|
81
|
+
line-length = 120
|
|
82
|
+
target-version = "py312"
|
|
83
|
+
|
|
84
|
+
[tool.ruff.lint]
|
|
85
|
+
select = [
|
|
86
|
+
"E",
|
|
87
|
+
"F",
|
|
88
|
+
"W",
|
|
89
|
+
"I",
|
|
90
|
+
"B",
|
|
91
|
+
"UP",
|
|
92
|
+
"SIM",
|
|
93
|
+
"RUF",
|
|
94
|
+
]
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "hushmask"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Hide names, emails, IBANs and addresses before text reaches an AI, and put them back in the answer."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
authors = [
|
|
7
|
+
{ name = "Bilal", email = "bilalrizvi2110@gmail.com" }
|
|
8
|
+
]
|
|
9
|
+
license = "MIT"
|
|
10
|
+
license-files = ["LICENSE"]
|
|
11
|
+
requires-python = ">=3.12"
|
|
12
|
+
dependencies = []
|
|
13
|
+
keywords = ["privacy", "pii", "llm", "openai", "proxy", "anonymization", "pseudonymization", "gdpr", "redaction"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Environment :: Web Environment",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Intended Audience :: End Users/Desktop",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Security",
|
|
24
|
+
"Topic :: Text Processing",
|
|
25
|
+
"Typing :: Typed",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://bilal-py.github.io/hushmask/"
|
|
30
|
+
Source = "https://github.com/bilal-py/hushmask"
|
|
31
|
+
Issues = "https://github.com/bilal-py/hushmask/issues"
|
|
32
|
+
Evaluation = "https://github.com/bilal-py/hushmask/blob/main/docs/eval/RESULTS.md"
|
|
33
|
+
|
|
34
|
+
[project.scripts]
|
|
35
|
+
hushmask = "hushmask.cli:main"
|
|
36
|
+
|
|
37
|
+
[project.optional-dependencies]
|
|
38
|
+
# everything most people want: the proxy + dashboard and the names model (ONNX, ~0.6 GB on first use)
|
|
39
|
+
all = ["hushmask[proxy,ner]"]
|
|
40
|
+
ner = [
|
|
41
|
+
"numpy>=2.5.5",
|
|
42
|
+
"onnxruntime>=1.20",
|
|
43
|
+
"tokenizers>=0.20",
|
|
44
|
+
]
|
|
45
|
+
proxy = [
|
|
46
|
+
"fastapi>=0.165.0",
|
|
47
|
+
"httpx>=0.28.1",
|
|
48
|
+
"uvicorn[standard]>=0.54.0",
|
|
49
|
+
]
|
|
50
|
+
ner-torch = [
|
|
51
|
+
"gliner>=0.2.30",
|
|
52
|
+
"protobuf>=7.36.2",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
[build-system]
|
|
56
|
+
requires = ["uv_build>=0.11.17,<0.12.0"]
|
|
57
|
+
build-backend = "uv_build"
|
|
58
|
+
|
|
59
|
+
[dependency-groups]
|
|
60
|
+
dev = [
|
|
61
|
+
"fastapi>=0.165.0",
|
|
62
|
+
"httpx>=0.28.1",
|
|
63
|
+
"pytest>=9.2.0",
|
|
64
|
+
"ruff>=0.17.6",
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
[tool.pytest.ini_options]
|
|
68
|
+
testpaths = ["tests"]
|
|
69
|
+
|
|
70
|
+
[tool.ruff]
|
|
71
|
+
line-length = 120
|
|
72
|
+
target-version = "py312"
|
|
73
|
+
|
|
74
|
+
[tool.ruff.lint]
|
|
75
|
+
select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""hushmask: hide PII before text goes to an AI, put it back after."""
|
|
2
|
+
|
|
3
|
+
from .detectors import Detector, KnownValue, KnownValuesDetector, PatternDetector, default_detectors
|
|
4
|
+
from .entities import Entity, EntityType
|
|
5
|
+
from .masker import LeakError, Masker, MaskResult, RestoreResult
|
|
6
|
+
from .ner import GlinerDetector
|
|
7
|
+
from .policy import PRESETS, Policy
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"PRESETS",
|
|
11
|
+
"Detector",
|
|
12
|
+
"Entity",
|
|
13
|
+
"EntityType",
|
|
14
|
+
"GlinerDetector",
|
|
15
|
+
"KnownValue",
|
|
16
|
+
"KnownValuesDetector",
|
|
17
|
+
"LeakError",
|
|
18
|
+
"MaskResult",
|
|
19
|
+
"Masker",
|
|
20
|
+
"PatternDetector",
|
|
21
|
+
"Policy",
|
|
22
|
+
"RestoreResult",
|
|
23
|
+
"default_detectors",
|
|
24
|
+
]
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Street addresses by pattern: German/Dutch, US/UK/Australian, and French/Italian/Spanish styles.
|
|
2
|
+
|
|
3
|
+
These are deliberately conservative. A street needs a street word ("straße", "Street", "rue", "Via"…) and a
|
|
4
|
+
house number; ambiguous German suffixes ("ring", "weg") also need a postcode and city, so "Spring 2026" or
|
|
5
|
+
"Monitoring 2" never match. House numbers can't be part of an amount ("10.99"), "Dr" isn't a street suffix
|
|
6
|
+
(statements use it for debit), and Romance keywords must be capitalized ("Via Roma 10", not "via PayPal 10").
|
|
7
|
+
The names model still catches the free-form rest.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
|
|
14
|
+
from .detectors import PRIORITY_FORMAT
|
|
15
|
+
from .entities import Entity, EntityType
|
|
16
|
+
|
|
17
|
+
_U = "A-ZÄÖÜÀ-ÖØ-Þ" # upper-case letters, incl. accented
|
|
18
|
+
_WORD = rf"[{_U}][\w'\u2019.-]*"
|
|
19
|
+
_HOUSE = r"\d{1,4}\s?[a-zA-Z]?(?:\s?[-/]\s?\d{1,4}[a-zA-Z]?)?(?![.,]\d)(?!\w)"
|
|
20
|
+
_CITY = rf"[{_U}][\w'\u2019-]+(?:[ -][{_U}][\w'\u2019-]+){{0,2}}"
|
|
21
|
+
|
|
22
|
+
# German / Dutch / Austrian / Swiss: "Bergstraße 12", "Lange Straße 5a", "Keizersgracht 123, 1015 CJ Amsterdam"
|
|
23
|
+
_SAFE = ("straße", "strasse", "str.", "allee", "gasse", "platz", "damm", "chaussee", "gracht", "straat", "laan",
|
|
24
|
+
"plein", "kade", "steeg", "promenade") # fmt: skip
|
|
25
|
+
_AMBIGUOUS = ("weg", "ring", "ufer", "steig", "pfad", "dijk", "markt", "berg", "hof")
|
|
26
|
+
_STANDALONE = {"straße", "strasse", "str.", "allee", "gasse", "platz", "weg", "ring", "markt", "damm", "ufer"}
|
|
27
|
+
_ADJECTIVES = {"alte", "alter", "altes", "neue", "neuer", "neues", "lange", "langer", "große", "großer", "kleine",
|
|
28
|
+
"kleiner", "hohe", "hoher", "obere", "untere", "am", "an", "auf", "im", "in", "zum", "zur"} # fmt: skip
|
|
29
|
+
_DE = re.compile(
|
|
30
|
+
rf"(?<![\w])(?P<street>(?:{_WORD}\s+){{0,2}}{_WORD})\s+(?P<house>{_HOUSE})"
|
|
31
|
+
rf"(?P<post>,?\s+\d{{4,5}}(?:\s?[A-Z]{{2}})?\s+{_CITY})?"
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
# US / UK / Australia: "1600 Pennsylvania Ave NW", "Unit 3/42 Smith St, Sydney NSW 2000", "10 Downing Street"
|
|
35
|
+
_EN_SUFFIX = (
|
|
36
|
+
"Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Lane|Ln|Drive|Court|Ct|Way|Place|Pl|Terrace|Tce|Crescent|Cres|"
|
|
37
|
+
"Parade|Pde|Highway|Hwy|Close|Circuit|Cct|Square|Sq|Grove|Gardens|Mews|Row|Walk|Parkway|Pkwy"
|
|
38
|
+
)
|
|
39
|
+
_EN = re.compile(
|
|
40
|
+
rf"(?<![\w/.,])(?:(?:Unit|Apt|Suite|Flat)\s*\w+\s*[,/]?\s*)?\d{{1,6}}[A-Za-z]?(?:/\d{{1,6}})?"
|
|
41
|
+
rf"\s+(?:{_WORD}\s+){{1,4}}?(?i:{_EN_SUFFIX})\.?(?![\w])"
|
|
42
|
+
r"(?:\s+(?:N|S|E|W|NE|NW|SE|SW)\b)?"
|
|
43
|
+
r"(?:,?\s+(?:Apt|Suite|Unit|Ste|#)\.?\s*[\w-]+)?"
|
|
44
|
+
# locality: "Washington, DC 20500", "Sydney NSW 2000", "London SW1A 2AA", "Pune 411001"
|
|
45
|
+
rf"(?:,\s*(?:{_WORD}\s?){{1,3}},?\s+"
|
|
46
|
+
r"(?:(?:[A-Z]{2,3}\s+)?\d{4,6}(?:-\d{4})?|[A-Z]{1,2}\d[A-Z\d]?\s?\d[A-Z]{2}))?"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
# French: "12 rue de Rivoli, 75001 Paris"
|
|
50
|
+
_FR = re.compile(
|
|
51
|
+
r"(?<![\w.,])\d{1,4}(?:\s?(?:bis|ter))?,?\s+"
|
|
52
|
+
r"(?i:rue|avenue|av\.|boulevard|bd|place|chemin|allée|impasse|quai|route|cours)\s+"
|
|
53
|
+
rf"(?:(?:de\s+la|de\s+l'|du|des|de|d')\s*)?{_WORD}(?:\s+{_WORD}){{0,3}}"
|
|
54
|
+
rf"(?:,?\s+\d{{5}}\s+{_CITY})?"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
# Italian / Spanish / Portuguese / Catalan: "Via Roma 10", "Calle Mayor 5, 28013 Madrid"
|
|
58
|
+
_ROMANCE = re.compile(
|
|
59
|
+
r"(?<![\w])(?:Via|Viale|Piazza|Corso|Largo|Calle|Avenida|Avda\.|Plaza|Paseo|Carrer|Rua|Travessa)\s+"
|
|
60
|
+
rf"(?:(?:del|della|delle|dei|de|di|la|los|las|dos|das|do|da)\s+)?{_WORD}(?:\s+{_WORD}){{0,3}},?\s+"
|
|
61
|
+
rf"(?:n\.?\s?)?{_HOUSE}"
|
|
62
|
+
rf"(?:,?\s+\d{{5}}\s+{_CITY})?"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _german_street(street: str, has_post: bool) -> int | None:
|
|
67
|
+
"""Return how many leading words to drop, or None if this isn't a street."""
|
|
68
|
+
words = street.split()
|
|
69
|
+
last = words[-1].casefold()
|
|
70
|
+
if last in _STANDALONE: # "Lange Straße": keep adjectives before it, drop anything else
|
|
71
|
+
if len(words) == 1 and not has_post:
|
|
72
|
+
return None # a bare "Platz 2" is a ranking, not an address
|
|
73
|
+
keep = 1
|
|
74
|
+
while keep < len(words) and words[-keep - 1].casefold() in _ADJECTIVES | {"der", "den", "dem"}:
|
|
75
|
+
keep += 1
|
|
76
|
+
if keep == 1 and len(words) > 1:
|
|
77
|
+
keep = 2 # "Kurfürsten Straße"
|
|
78
|
+
return len(words) - keep
|
|
79
|
+
if last.endswith(_SAFE) or (has_post and last.endswith(_AMBIGUOUS)):
|
|
80
|
+
return len(words) - 1 # compound: "Bergstraße"; earlier capitalized words are not part of it
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class AddressDetector:
|
|
85
|
+
name = "address"
|
|
86
|
+
|
|
87
|
+
def detect(self, text: str) -> list[Entity]:
|
|
88
|
+
found: list[Entity] = []
|
|
89
|
+
for match in _DE.finditer(text):
|
|
90
|
+
drop = _german_street(match.group("street"), bool(match.group("post")))
|
|
91
|
+
if drop is None:
|
|
92
|
+
continue
|
|
93
|
+
start = match.start("street")
|
|
94
|
+
for _ in range(drop):
|
|
95
|
+
start = text.index(" ", start) + 1
|
|
96
|
+
while text[start].isspace():
|
|
97
|
+
start += 1
|
|
98
|
+
found.append(self._entity(text, start, match.end()))
|
|
99
|
+
for pattern in (_EN, _FR, _ROMANCE):
|
|
100
|
+
for match in pattern.finditer(text):
|
|
101
|
+
found.append(self._entity(text, match.start(), match.end()))
|
|
102
|
+
return found
|
|
103
|
+
|
|
104
|
+
def _entity(self, text: str, start: int, end: int) -> Entity:
|
|
105
|
+
return Entity(EntityType.ADDRESS, start, end, text[start:end], self.name, PRIORITY_FORMAT)
|