maskvault 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. maskvault-0.1.0/LICENSE +21 -0
  2. maskvault-0.1.0/PKG-INFO +651 -0
  3. maskvault-0.1.0/README.md +618 -0
  4. maskvault-0.1.0/pyproject.toml +154 -0
  5. maskvault-0.1.0/pyproject.toml.orig +126 -0
  6. maskvault-0.1.0/src/maskvault/__init__.py +121 -0
  7. maskvault-0.1.0/src/maskvault/__main__.py +11 -0
  8. maskvault-0.1.0/src/maskvault/anonymizer.py +495 -0
  9. maskvault-0.1.0/src/maskvault/ciphers/__init__.py +25 -0
  10. maskvault-0.1.0/src/maskvault/ciphers/base.py +51 -0
  11. maskvault-0.1.0/src/maskvault/ciphers/fernet.py +133 -0
  12. maskvault-0.1.0/src/maskvault/ciphers/plaintext.py +34 -0
  13. maskvault-0.1.0/src/maskvault/cli/__init__.py +187 -0
  14. maskvault-0.1.0/src/maskvault/cli/anonymize_command.py +77 -0
  15. maskvault-0.1.0/src/maskvault/cli/command.py +66 -0
  16. maskvault-0.1.0/src/maskvault/cli/exit_codes.py +29 -0
  17. maskvault-0.1.0/src/maskvault/cli/explain_command.py +113 -0
  18. maskvault-0.1.0/src/maskvault/cli/policy_command.py +52 -0
  19. maskvault-0.1.0/src/maskvault/cli/reader.py +33 -0
  20. maskvault-0.1.0/src/maskvault/cli/restore_command.py +85 -0
  21. maskvault-0.1.0/src/maskvault/constants.py +188 -0
  22. maskvault-0.1.0/src/maskvault/detectors/__init__.py +41 -0
  23. maskvault-0.1.0/src/maskvault/detectors/adjustments.py +88 -0
  24. maskvault-0.1.0/src/maskvault/detectors/entropy.py +58 -0
  25. maskvault-0.1.0/src/maskvault/detectors/general.py +77 -0
  26. maskvault-0.1.0/src/maskvault/detectors/identifiers.py +68 -0
  27. maskvault-0.1.0/src/maskvault/detectors/registry.py +54 -0
  28. maskvault-0.1.0/src/maskvault/detectors/shapes.py +119 -0
  29. maskvault-0.1.0/src/maskvault/detectors/vendor.py +124 -0
  30. maskvault-0.1.0/src/maskvault/entropy.py +176 -0
  31. maskvault-0.1.0/src/maskvault/exceptions/__init__.py +32 -0
  32. maskvault-0.1.0/src/maskvault/exceptions/base.py +11 -0
  33. maskvault-0.1.0/src/maskvault/exceptions/detector.py +16 -0
  34. maskvault-0.1.0/src/maskvault/exceptions/structure.py +21 -0
  35. maskvault-0.1.0/src/maskvault/exceptions/token_format.py +19 -0
  36. maskvault-0.1.0/src/maskvault/exceptions/vault.py +16 -0
  37. maskvault-0.1.0/src/maskvault/exceptions/whitelist.py +14 -0
  38. maskvault-0.1.0/src/maskvault/formats.py +160 -0
  39. maskvault-0.1.0/src/maskvault/levels.py +145 -0
  40. maskvault-0.1.0/src/maskvault/models/__init__.py +31 -0
  41. maskvault-0.1.0/src/maskvault/models/action.py +12 -0
  42. maskvault-0.1.0/src/maskvault/models/detector.py +67 -0
  43. maskvault-0.1.0/src/maskvault/models/entropy_thresholds.py +93 -0
  44. maskvault-0.1.0/src/maskvault/models/entropy_verdict.py +17 -0
  45. maskvault-0.1.0/src/maskvault/models/finding.py +29 -0
  46. maskvault-0.1.0/src/maskvault/models/level.py +19 -0
  47. maskvault-0.1.0/src/maskvault/models/substitution.py +24 -0
  48. maskvault-0.1.0/src/maskvault/models/token_format.py +207 -0
  49. maskvault-0.1.0/src/maskvault/models/vault.py +475 -0
  50. maskvault-0.1.0/src/maskvault/models/whitelist.py +282 -0
  51. maskvault-0.1.0/src/maskvault/patterns.py +252 -0
  52. maskvault-0.1.0/src/maskvault/personal.py +244 -0
  53. maskvault-0.1.0/src/maskvault/py.typed +0 -0
  54. maskvault-0.1.0/src/maskvault/scopes.py +108 -0
  55. maskvault-0.1.0/src/maskvault/stores/__init__.py +53 -0
  56. maskvault-0.1.0/src/maskvault/stores/backend.py +51 -0
  57. maskvault-0.1.0/src/maskvault/stores/base.py +64 -0
  58. maskvault-0.1.0/src/maskvault/stores/database.py +249 -0
  59. maskvault-0.1.0/src/maskvault/stores/expiry.py +73 -0
  60. maskvault-0.1.0/src/maskvault/stores/memory.py +103 -0
  61. maskvault-0.1.0/src/maskvault/stores/serialized.py +120 -0
  62. maskvault-0.1.0/src/maskvault/structured/__init__.py +108 -0
  63. maskvault-0.1.0/src/maskvault/structured/restorer.py +92 -0
  64. maskvault-0.1.0/src/maskvault/structured/sampler.py +86 -0
  65. maskvault-0.1.0/src/maskvault/structured/text_bytes.py +162 -0
  66. maskvault-0.1.0/src/maskvault/structured/walker.py +165 -0
  67. maskvault-0.1.0/src/maskvault/utils/__init__.py +16 -0
  68. maskvault-0.1.0/src/maskvault/utils/letters.py +40 -0
  69. maskvault-0.1.0/src/maskvault/utils/text.py +62 -0
  70. maskvault-0.1.0/src/maskvault/vocabularies/__init__.py +64 -0
  71. maskvault-0.1.0/src/maskvault/vocabularies/criminal.py +50 -0
  72. maskvault-0.1.0/src/maskvault/vocabularies/health.py +123 -0
  73. maskvault-0.1.0/src/maskvault/vocabularies/homonyms.py +38 -0
  74. maskvault-0.1.0/src/maskvault/vocabularies/identity.py +56 -0
  75. maskvault-0.1.0/src/maskvault/vocabularies/origin.py +58 -0
  76. maskvault-0.1.0/src/maskvault/vocabularies/politics.py +61 -0
  77. maskvault-0.1.0/src/maskvault/vocabularies/religion.py +84 -0
  78. maskvault-0.1.0/src/maskvault/vocabularies/safe_values.py +80 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 janbalitaan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,651 @@
1
+ Metadata-Version: 2.4
2
+ Name: maskvault
3
+ Version: 0.1.0
4
+ Summary: Reversible anonymization with a vault: replace confidential values with tokens, pass them through anything, restore them afterwards
5
+ Keywords: anonymization,pseudonymization,pii,privacy,redaction,secrets,gdpr,masking,tokenization,data-protection,llm
6
+ Author: janbalitaan
7
+ License-Expression: MIT
8
+ License-File: LICENSE
9
+ Classifier: Development Status :: 3 - Alpha
10
+ Classifier: Intended Audience :: System Administrators
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Topic :: Security
20
+ Classifier: Topic :: Database
21
+ Classifier: Topic :: System :: Logging
22
+ Classifier: Topic :: Text Processing :: Filters
23
+ Classifier: Typing :: Typed
24
+ Requires-Dist: cryptography>=43.0 ; extra == 'encryption'
25
+ Requires-Python: >=3.10
26
+ Project-URL: Homepage, https://github.com/janbalitaan/maskvault
27
+ Project-URL: Documentation, https://github.com/janbalitaan/maskvault#readme
28
+ Project-URL: Source, https://github.com/janbalitaan/maskvault
29
+ Project-URL: Changelog, https://github.com/janbalitaan/maskvault/blob/main/CHANGELOG.md
30
+ Project-URL: Issues, https://github.com/janbalitaan/maskvault/issues
31
+ Provides-Extra: encryption
32
+ Description-Content-Type: text/markdown
33
+
34
+ # maskvault
35
+
36
+ Reversible anonymization with a vault. Replace confidential values with
37
+ tokens, pass the text through anything, restore the originals afterwards.
38
+
39
+ [![PyPI](https://img.shields.io/pypi/v/maskvault)](https://pypi.org/project/maskvault/)
40
+ [![Python](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)](https://github.com/janbalitaan/maskvault/blob/main/pyproject.toml)
41
+ [![CI](https://github.com/janbalitaan/maskvault/actions/workflows/ci.yml/badge.svg)](https://github.com/janbalitaan/maskvault/actions/workflows/ci.yml)
42
+ [![License](https://img.shields.io/badge/license-MIT-green)](https://github.com/janbalitaan/maskvault/blob/main/LICENSE)
43
+
44
+ ```python
45
+ from maskvault import Anonymizer, Level
46
+
47
+ anonymizer = Anonymizer(level=Level.STRICT, terms={"ORG": ["Acme Corp"]})
48
+
49
+ safe, vault = anonymizer.anonymize("I work at Acme Corp, email me@acme.com")
50
+ # safe -> "I work at !|ORG_1|!, email !|EMAIL_1|!"
51
+
52
+ result = send_somewhere(safe) # a model, a log, a vendor, a dataset
53
+ final = anonymizer.restore(result, vault) # the real values come back
54
+ ```
55
+
56
+ It is not tied to any one destination. The same mechanism serves a prompt sent
57
+ to a provider, a log line shipped to a vendor, a dataset handed to a
58
+ contractor, a bug report pasted into a tracker, or a support transcript
59
+ reviewed by a third party.
60
+
61
+ ## Examples
62
+
63
+ Runnable files in [`examples/`](https://github.com/janbalitaan/maskvault/blob/main/examples/README.md),
64
+ each self-contained and printing what it did:
65
+
66
+ ```bash
67
+ python examples/01_round_trip.py
68
+ ```
69
+
70
+ Every one is run by the test suite, so an example that stops working fails CI.
71
+
72
+ ## Documentation
73
+
74
+ Full documentation is in [`docs/`](https://github.com/janbalitaan/maskvault/blob/main/docs/README.md):
75
+
76
+ | Page | |
77
+ | --- | --- |
78
+ | [Getting started](https://github.com/janbalitaan/maskvault/blob/main/docs/getting-started.md) | Install, a first round trip, wiring it in |
79
+ | [Levels](https://github.com/janbalitaan/maskvault/blob/main/docs/levels.md) | The four strictness levels |
80
+ | [Detection](https://github.com/janbalitaan/maskvault/blob/main/docs/detection.md) | What is found, and how |
81
+ | [Whitelisting](https://github.com/janbalitaan/maskvault/blob/main/docs/whitelisting.md) | Nine mechanisms for taking control back |
82
+ | [Token formats](https://github.com/janbalitaan/maskvault/blob/main/docs/token-formats.md) | Delimiters, collisions, the default |
83
+ | [Structured data](https://github.com/janbalitaan/maskvault/blob/main/docs/structured-data.md) | Dicts, lists and JSON |
84
+ | [Stores](https://github.com/janbalitaan/maskvault/blob/main/docs/stores.md) | Moving a vault between processes and nodes |
85
+ | [Encryption](https://github.com/janbalitaan/maskvault/blob/main/docs/encryption.md) | Who encrypts, who holds the key |
86
+ | [Concurrency](https://github.com/janbalitaan/maskvault/blob/main/docs/concurrency.md) | One anonymizer, many callers |
87
+ | [Command line](https://github.com/janbalitaan/maskvault/blob/main/docs/cli.md) | Cleaning a file or a pipe |
88
+ | [API reference](https://github.com/janbalitaan/maskvault/blob/main/docs/api.md) | Every public name |
89
+ | [Security](https://github.com/janbalitaan/maskvault/blob/main/docs/security.md) | What this protects, and what it does not |
90
+ | [Limitations](https://github.com/janbalitaan/maskvault/blob/main/docs/limitations.md) | Stated plainly |
91
+
92
+ Security reports go through
93
+ [SECURITY.md](https://github.com/janbalitaan/maskvault/blob/main/SECURITY.md) rather than an issue.
94
+
95
+ ## Install
96
+
97
+ ```bash
98
+ pip install maskvault
99
+ ```
100
+
101
+ Python 3.10+. **No dependencies**, so it runs in a serverless function, in CI,
102
+ in a build step, or on a Raspberry Pi with no model download and no install
103
+ step.
104
+
105
+ ---
106
+
107
+ ## The two-way distinction
108
+
109
+ Most tools in this space do one thing: they remove. maskvault separates two
110
+ cases that need opposite handling.
111
+
112
+ **Reversible** — identifying but legitimate. An employer, a phone number, a
113
+ salary, a health condition. Replaced with a token, restored afterwards. The
114
+ third party never sees it; your user still gets a correct answer.
115
+
116
+ **Destroyed** — must never arrive and must never come back. An API key, a
117
+ private key, a card number, a connection string. Replaced with a notice and
118
+ *not* restored, because if one of these appeared, something went wrong
119
+ upstream and quietly putting it back would hide the mistake.
120
+
121
+ ```python
122
+ safe, vault = anonymizer.anonymize("key sk-abc123… and I work at Acme Corp")
123
+ # "key [an API key was removed before sending] and I work at !|ORG_1|!"
124
+ vault.destroyed # ["api_key"]
125
+ ```
126
+
127
+ So an exact round trip holds **only when nothing was destroyed**. That is the
128
+ intended behavior, not a limitation to work around.
129
+
130
+ ## Strictness levels
131
+
132
+ ```python
133
+ Anonymizer(level=Level.MINIMAL) # credentials only
134
+ Anonymizer(level=Level.STANDARD) # + contact details, identifiers (default)
135
+ Anonymizer(level=Level.STRICT) # + GDPR Article 9 special categories
136
+ Anonymizer(level=Level.PARANOID) # + a lower entropy bar
137
+ ```
138
+
139
+ `STRICT` adds vocabularies for health, religion, politics, identity, origin
140
+ and criminal or financial history — the categories European law treats as
141
+ needing explicit consent, which has held up well as a description of what
142
+ people do not want shared. It is the right level when the subject is a person
143
+ and the wrong level for a codebase, because "black" is in the origin
144
+ vocabulary and also half the CSS ever written.
145
+
146
+ ## Control
147
+
148
+ Nothing is allowed through by default. Every mechanism for allowing something:
149
+
150
+ ```python
151
+ Anonymizer(
152
+ level=Level.STRICT,
153
+ whitelist={
154
+ "labels": ["url", "age"], # turn a detector off
155
+ "categories": ["identity"], # turn a category off
156
+ "values": ["Python", "FastAPI"], # exact strings
157
+ "words": ["black", "native"], # word-bounded
158
+ "contains": ["github.com"], # substring
159
+ "prefixes": ["https://mysite.dev/"],
160
+ "suffixes": ["@mycompany.com"],
161
+ "domains": ["mycompany.com"], # emails and URLs, subdomains too
162
+ "patterns": [r"^EMP-\d{4}$"], # regex
163
+ },
164
+ )
165
+ ```
166
+
167
+ Plus `predicate=` for arbitrary logic, which has no JSON form by design — a
168
+ config file should not be able to introduce executable code.
169
+
170
+ Two curated starting lists are included: `COMMON_SAFE_VALUES` (public
171
+ technology names) and `TECHNICAL_HOMONYMS` (words in the sensitive
172
+ vocabularies that are also ordinary technical vocabulary).
173
+
174
+ ## Configure from what you actually see
175
+
176
+ Guessing a whitelist up front is the wrong way round. Run real input through
177
+ it and read the findings:
178
+
179
+ ```python
180
+ print(anonymizer.preview(text))
181
+ ```
182
+
183
+ ```
184
+ 9 findings at level strict:
185
+ reversible AGE I am 34
186
+ reversible STREET_ADDRESS 42 Rizal Street
187
+ reversible SALARY earn $95,000 a year
188
+ reversible HEALTH diabetes
189
+ reversible RELIGION Catholic
190
+ allowed URL https://github.com/me
191
+ destroy AWS_ACCESS_KEY AKIAIOSFODNN7EXAMPLE
192
+ 8 would be replaced, 1 allowed through
193
+ ```
194
+
195
+ `explain()` returns the same as structured `Finding` objects.
196
+ `suggest_whitelist()` turns it into a config draft to edit down.
197
+
198
+ `preview()` prints real values, so it is for a terminal, not a log. For a log
199
+ there is `vault.summary()`, which counts by kind and contains no values:
200
+
201
+ ```
202
+ "1 age, 2 health, 1 salary, 1 street_address, 1 destroyed"
203
+ ```
204
+
205
+ ## Moving a vault between processes
206
+
207
+ The common case needs nothing. Anonymize, call the provider and restore in
208
+ one request handler, and the vault is a local variable that never leaves it:
209
+
210
+ ```python
211
+ safe, vault = anonymizer.anonymize(question)
212
+ reply = provider.complete(safe)
213
+ answer = anonymizer.restore(reply, vault)
214
+ ```
215
+
216
+ Scale that to any number of nodes and nothing crosses, because nothing is
217
+ shared. **Adding a store here makes it less private, not more.**
218
+
219
+ A store is for when the vault has to outlive the request — a multi-turn
220
+ conversation where one person should keep one token, anonymizing on one node
221
+ and restoring on another, or a queue where the worker is a different process.
222
+
223
+ ```python
224
+ from maskvault.ciphers import FernetCipher
225
+ from maskvault.stores import MemoryVaultStore, SerializedVaultStore
226
+
227
+ MemoryVaultStore() # one process
228
+ SerializedVaultStore(backend, FernetCipher.from_env()) # Redis, Valkey, ...
229
+ DatabaseVaultStore(connection, FernetCipher.from_env()) # PostgreSQL, MySQL
230
+ ```
231
+
232
+ A key-value backend is about ten lines, which is why none of them is a
233
+ dependency here:
234
+
235
+ ```python
236
+ class RedisBackend:
237
+ def __init__(self, client):
238
+ self._client = client
239
+
240
+ def get(self, key):
241
+ return self._client.get(key)
242
+
243
+ def set(self, key, value, ttl_seconds):
244
+ self._client.set(key, value, ex=int(ttl_seconds))
245
+
246
+ def delete(self, key):
247
+ self._client.delete(key)
248
+ ```
249
+
250
+ ### Encryption is not optional, and not defaulted either
251
+
252
+ A stored vault holds every original value. It usually goes somewhere with
253
+ wider access than the application — a shared cache, a table, a backup of
254
+ either — while the key stays in the application. Those are different trust
255
+ domains, which is what makes encrypting here worth doing.
256
+
257
+ So a store refuses to be built without a cipher. There is no secure default
258
+ to fall back to: a key cannot be invented, because a random one per process
259
+ would leave the node that restores unable to decrypt what the node that
260
+ anonymized wrote. And defaulting to plaintext would mean a store that looks
261
+ encrypted to anyone who did not read the signature.
262
+
263
+ ```bash
264
+ pip install 'maskvault[encryption]'
265
+ ```
266
+
267
+ ```python
268
+ key = FernetCipher.generate_key() # once, at deployment, kept somewhere safe
269
+ cipher = FernetCipher(key) # or FernetCipher.from_env()
270
+ ```
271
+
272
+ Opting out is a thing you have to type, so that it shows in review:
273
+
274
+ ```python
275
+ SerializedVaultStore(backend, PlaintextCipher()) # faster, and visible
276
+ ```
277
+
278
+ `store.describe()` returns a line safe to log at startup. "plaintext" in a
279
+ production log is the kind of thing someone notices; a silent default never
280
+ would be.
281
+
282
+ ### Who does the encrypting
283
+
284
+ maskvault performs it and implements none of it. The only cipher here
285
+ delegates to `cryptography`'s Fernet — an audited implementation — and this
286
+ package contains no cryptographic primitive of its own.
287
+
288
+ The key is yours throughout. Where it lives, how it rotates, and whether it
289
+ ever reaches the process at all are your decisions, and the key never appears
290
+ in anything this library writes: not a repr, not `describe()`, not the vault,
291
+ not the stored bytes.
292
+
293
+ If you would rather own the whole operation, implement `VaultCipher`. A KMS,
294
+ an HSM or a Vault transit backend all fit, and none of them require importing
295
+ anything from here:
296
+
297
+ ```python
298
+ class KmsCipher:
299
+ def __init__(self, client, key_id):
300
+ self._client, self._key_id = client, key_id
301
+
302
+ def encrypt(self, plaintext):
303
+ return self._client.encrypt(
304
+ KeyId=self._key_id, Plaintext=plaintext
305
+ )["CiphertextBlob"]
306
+
307
+ def decrypt(self, ciphertext):
308
+ return self._client.decrypt(CiphertextBlob=ciphertext)["Plaintext"]
309
+
310
+ @property
311
+ def describes(self):
312
+ return f"aws kms, key {self._key_id}"
313
+ ```
314
+
315
+ That key never reaches your process, and `store.describe()` reports
316
+ `aws kms, key alias/prod` so the choice is visible in a startup log.
317
+
318
+ ### The lifetime is yours, with a default if you want one
319
+
320
+ Entries expire after fifteen minutes unless you say otherwise, because a
321
+ vault's lifetime is the window in which its values are exposed.
322
+
323
+ **`ttl_seconds=0` means no expiry**, which hands retention to you. That is
324
+ the point of using a database rather than a cache: retention there is already
325
+ somebody's job — a nightly job, a partition drop, a policy the DBA owns — and
326
+ a library second-guessing it is a library fighting the deployment.
327
+
328
+ ```python
329
+ DatabaseVaultStore(connection, cipher, ttl_seconds=0) # yours to manage
330
+ DatabaseVaultStore(connection, cipher, ttl_seconds=3600) # an hour
331
+ ```
332
+
333
+ A negative value raises rather than being read as zero. A TTL is usually
334
+ computed, and one that came out negative is a bug worth hearing about — not a
335
+ silent switch to keeping plaintext forever.
336
+
337
+ A database has no expiry of its own, so `purge()` has to be called by
338
+ something when a TTL is set. With `ttl_seconds=0` there is nothing for it to
339
+ do, and `describe()` stops asking for it.
340
+
341
+ And key by something two callers cannot share. A conversation id or a request
342
+ id is right; a user id usually is not, because one user has more than one
343
+ conversation, and a shared key hands one caller's values to another.
344
+
345
+ ## The token format is yours
346
+
347
+ The default is `!|LABEL_1|!`. Two collisions can break a token and only one
348
+ is detectable.
349
+
350
+ **With the input.** The text already contains the delimiter, so a replacement
351
+ nests inside it or a real occurrence is mistaken for a token. This one is
352
+ handled automatically: the format is chosen per operation by trying each
353
+ candidate until one is absent from the text, and the choice is recorded in
354
+ the vault so restoring needs no bookkeeping from you.
355
+
356
+ ```python
357
+ safe, vault = anonymizer.anonymize("a note mentioning !|EXISTING|!")
358
+ vault.token_format.name # "pipe_brackets", since the default was taken
359
+ ```
360
+
361
+ **With wherever the output goes.** The input is clean, but the *result* is
362
+ pasted into a wiki, rendered as Markdown, or fed to a template engine — where
363
+ the token means something. Selection cannot help, because it sees the text it
364
+ is given and knows nothing about the destination. Only the default can
365
+ protect against this, by being a pair nothing claims.
366
+
367
+ So the default was picked by testing each candidate against what people
368
+ actually paste into a model:
369
+
370
+ | | claimed by |
371
+ | --- | --- |
372
+ | `!|X|!` | **nothing** — the default |
373
+ | `[\|X\|]` | Haskell and Idris quasi-quotation. Rare, more legible, so it is second |
374
+ | `@@X@@` | unified diff hunk headers, `@@ -1,5 +1,5 @@` |
375
+ | `[[X]]` | wikilinks — Obsidian, Roam, Logseq, MediaWiki, Dendron, Foam |
376
+ | `[!X!]` | GitHub alerts, `> [!NOTE]`. And `[![` is a linked image — the badge syntax at the top of most READMEs |
377
+ | `{{X}}` | templates — Jinja, Handlebars, Mustache, Vue, Angular, Liquid |
378
+ | `{\|X\|}` | MediaWiki tables, and LaTeX |
379
+ | `<<X>>` | AsciiDoc cross-references |
380
+ | `%%X%%` | Obsidian comments, SQL `LIKE`, `printf` |
381
+ | `((X))` | Roam block references |
382
+ | `(\|X\|)` | Haskell idiom brackets, the Elixir and OCaml pipe operator |
383
+ | `<\|X\|>` | a model's own special tokens — the one pair to never use |
384
+
385
+ What the default does **not** fix: a model that reformats a token breaks
386
+ restoring whatever the delimiters are, and `!|EMAIL_1|` is rejected exactly as
387
+ `@@EMAIL_1@` was. [`leaked()`](#detecting-a-mangled-token) is what reports
388
+ that, and no choice of delimiter removes the need for it.
389
+
390
+ Ask for brackets by name when you control the whole pipeline and want the
391
+ most model-legible form:
392
+
393
+ ```python
394
+ Anonymizer(token_format="brackets") # -> [[ORG_1]]
395
+ ```
396
+
397
+ Pin one when you need it stable, by name or outright:
398
+
399
+ ```python
400
+ Anonymizer(token_format="pipe_bang")
401
+
402
+ Anonymizer(token_format=TokenFormat(
403
+ name="mine",
404
+ opening="<%|",
405
+ closing="|%>",
406
+ label_pattern=r"[a-z_]+",
407
+ label_case="lower",
408
+ )) # -> <%|org_1|%>
409
+ ```
410
+
411
+ Or supply your own candidate list, and auto-selection chooses from yours
412
+ rather than the built-ins:
413
+
414
+ ```python
415
+ Anonymizer(format_candidates=(my_first_choice, my_fallback))
416
+ ```
417
+
418
+ `random_format()` generates unguessable delimiters per call. That removes any
419
+ possibility of collision, and raises the cost of text forging a token that
420
+ `restore()` would expand — unpredictable, which is weaker than secret, but
421
+ useful when the text passes through something that processes untrusted input.
422
+
423
+ A format is validated at construction: it must match its own output. A format
424
+ that cannot would lose every value at restore time with no error, so it is
425
+ refused immediately instead:
426
+
427
+ ```python
428
+ TokenFormat(name="bad", opening="[[", closing="]]", label_pattern=r"\d+")
429
+ # ValueError: token format 'bad' cannot match its own output ('[[LABEL_1]]')
430
+ ```
431
+
432
+ Formats round-trip through `format_from_dict` / `as_dict`, so one can live in
433
+ a config file.
434
+
435
+ ## When a token comes back damaged
436
+
437
+ A model may mangle a token. `leaked()` reports any that survive restoration,
438
+ so you can fall back rather than show `[[ORG_2]]` to a user:
439
+
440
+ ```python
441
+ if anonymizer.leaked(reply, vault):
442
+ return deterministic_answer()
443
+ ```
444
+
445
+ An invented token such as `[[ORG_9]]` is deliberately left visible rather than
446
+ resolved to something unrelated — a visible oddity beats a silent wrong answer
447
+ about someone's employer.
448
+
449
+ ---
450
+
451
+ ## Limitations
452
+
453
+ Read this section before trusting the library with anything that matters. A
454
+ privacy tool that oversells itself is worse than no privacy tool, because it
455
+ replaces caution with false confidence.
456
+
457
+ ### It cannot find a name it was not told about
458
+
459
+ There is no NLP here. maskvault will not find "Maria Santos" in free text
460
+ unless you list it in `terms`. Microsoft Presidio will, using a named-entity
461
+ model, and for unstructured text about unknown people that is a genuine and
462
+ large advantage over this library.
463
+
464
+ Use `terms` for the names you know. If you need to anonymize text about people
465
+ whose names you do not know in advance, use Presidio, or run both.
466
+
467
+ ### The sensitive vocabularies are word lists, not understanding
468
+
469
+ `STRICT` matches terms from curated lists. That means:
470
+
471
+ - **It misses what is not listed.** A condition, faith or affiliation nobody
472
+ added will pass straight through. The health list has ~107 entries; medicine
473
+ has tens of thousands of terms.
474
+ - **It matches innocent uses.** "black", "native", "union", "strike",
475
+ "terminal", "partner" and "default" all appear in these lists for good
476
+ reasons and all have ordinary technical meanings. `TECHNICAL_HOMONYMS`
477
+ exists because of this, and it is a mitigation, not a fix.
478
+ - **It cannot read negation or context.** "I do not have diabetes" and "my
479
+ work on diabetes research" are both matched.
480
+
481
+ ### It is English-centric and Western-format-centric
482
+
483
+ The vocabularies are English. The national-ID pattern is a US SSN shape.
484
+ Street suffixes, postcodes and phone conventions lean US, UK and Philippine.
485
+ Text in another language, or identifiers in another country's format, are
486
+ largely unprotected unless you supply your own detectors.
487
+
488
+ This is the most valuable place to contribute, and the easiest.
489
+
490
+ ### Entropy detection has a floor and a ceiling
491
+
492
+ The entropy net catches secrets whose format nobody anticipated, but:
493
+
494
+ - A secret from a **narrow alphabet** scores low. An AWS access key id is
495
+ uppercase and digits only, measures 3.68 bits, and falls under the base64
496
+ threshold — which is why there is also an explicit AWS pattern. An unknown
497
+ vendor with a similarly narrow format will be missed.
498
+ - A **short** secret cannot be judged. Below the minimum length entropy is
499
+ too noisy to act on, so a six-character password is invisible to this layer.
500
+ `Level.PARANOID` lowers the bar and widens the alphabet to catch punctuated
501
+ passwords, at the cost of more false positives.
502
+ - A **passphrase** is not caught. Entropy is measured per character, so
503
+ "correct horse battery staple" scores like the ordinary words it is made of
504
+ even though it is a strong secret. Nothing here will find it.
505
+ - Entropy is only one of four conditions, and not the strongest. Per-character
506
+ entropy cannot tell a short varied English phrase from a generated string —
507
+ "the quick brown fox jumps" scores above 4 bits. The absence of whitespace
508
+ does more separating work than the entropy figure does, which also means a
509
+ secret containing a space is invisible to this layer.
510
+ - It will **flag things that are not secrets** — a long opaque identifier, a
511
+ hash in a URL path. Known-safe shapes (UUIDs, git SHAs, version strings) are
512
+ excluded, but that list is not complete.
513
+
514
+ ### Regexes over-match and under-match
515
+
516
+ The failure mode to watch for is a **partial** match: a pattern that claims
517
+ part of a value and leaves the rest in the output. Half a phone number is
518
+ still a phone number, so this is worse than no match at all.
519
+
520
+ It happens when a pattern tries to encode a value's grammar. A phone number
521
+ expressed as groups of digits backtracks — the greedy middle consumes the
522
+ trailing digits, the engine retries from a later position, and the result
523
+ covers only the middle of the number. The phone detector therefore finds
524
+ candidates with a loose pattern and validates them in code, which is the
525
+ approach to follow for anything whose shape is not genuinely regular.
526
+
527
+ ### Tokens can degrade the result
528
+
529
+ A model reading `[[ORG_1]]` may write worse prose, ask what the token means,
530
+ or refuse. Tokens are chosen to survive quoting and reformatting, and
531
+ `leaked()` tells you when one did not, but the trade is real: more privacy,
532
+ somewhat less fluency.
533
+
534
+ ### The vault is in-process memory
535
+
536
+ It is not encrypted, not persisted, and not shared between processes. Restore
537
+ has to happen where anonymization happened, or you have to pass the vault
538
+ yourself — and if you pass it over a network you have moved the problem rather
539
+ than solved it.
540
+
541
+ ### Structured data
542
+
543
+ Dicts, lists and JSON are walked rather than flattened, so the structure
544
+ survives:
545
+
546
+ ```python
547
+ from maskvault import anonymize_data, restore_data
548
+
549
+ safe, vault = anonymize_data({"user": {"email": "me@acme.com"}}, anonymizer)
550
+ # {"user": {"email": "!|EMAIL_1|!"}}
551
+
552
+ restore_data(safe, anonymizer, vault) # the original, exactly
553
+ ```
554
+
555
+ One vault covers the whole structure, so a value appearing in two places gets
556
+ one token. Keys are scanned too: a schema key like `email` matches no detector
557
+ and passes through, while `{"alice@acme.com": {...}}` — a map of users by
558
+ email address — has its key replaced, because that key is the data. Pass
559
+ `include_keys=False` where something downstream matches on the literal key.
560
+ Numbers are left alone, with `stringify_numbers=True` to opt in. `skip_keys`
561
+ passes named fields through untouched, for an id or timestamp something
562
+ downstream matches on.
563
+
564
+ ### No format preservation
565
+
566
+ `[[PHONE_1]]` does not look like a phone number. Anything downstream that
567
+ validates the shape of its input will reject it. Format-preserving
568
+ tokenization is a reasonable feature request and is not implemented.
569
+
570
+ ### It is not compliance
571
+
572
+ GDPR Article 9 is used here as an organizing principle for *what to look
573
+ for*. Using this library does not make a system GDPR, HIPAA or PCI compliant.
574
+ Compliance is about lawful basis, retention, contracts, access control and
575
+ disclosure — most of which are organizational, and none of which a regex can
576
+ establish.
577
+
578
+ ### First match wins
579
+
580
+ Detectors run in a fixed order and the first to match a span claims it.
581
+ Specific formats are ordered before generic ones for this reason, but nested
582
+ or overlapping sensitive values may not be handled the way you expect. Check
583
+ with `explain()`.
584
+
585
+ ---
586
+
587
+ ## How this compares
588
+
589
+ | | maskvault | Presidio | scrubadub | detect-secrets |
590
+ | --- | --- | --- | --- | --- |
591
+ | Dependencies | **0** | 31 + NLP models | 8 | 4 |
592
+ | Reversible | **first-class vault** | via encryption | no | no |
593
+ | Credentials destroyed, not tokenized | **yes** | no | no | n/a (one-way) |
594
+ | Secret scanning | entropy + ~20 formats | limited | no | **its whole focus** |
595
+ | Finds unknown person names | **no** | **yes (NER)** | partial | no |
596
+ | Special-category vocabularies | **yes** | no | no | no |
597
+ | Allowlist mechanisms | **9 + predicate** | per-recognizer | limited | regex allowlist |
598
+ | Inspect before trusting | **preview/explain** | analyzer results | no | audit mode |
599
+ | Non-English | **weak** | **strong** | weak | n/a |
600
+
601
+ **Pick Presidio** if you need to find names and entities in text about people
602
+ you do not know in advance, or need strong non-English support, and can carry
603
+ the install weight.
604
+
605
+ **Pick detect-secrets** if you only want to stop credentials reaching a repo.
606
+
607
+ **Pick maskvault** if you need the values *back*, want no dependencies, and
608
+ want credentials and personal data handled differently from one another.
609
+
610
+ They compose: Presidio for entity discovery, maskvault for the reversible
611
+ vault and the credential rules.
612
+
613
+ ---
614
+
615
+ ## Contributing
616
+
617
+ The limitations above are a roadmap, and most of them are fixed by adding
618
+ data rather than changing design. Contributions are very welcome, and these
619
+ are the ones that help most:
620
+
621
+ **New locales.** A national ID format, phone convention, postcode shape or
622
+ address pattern for a country that is not covered. This is the highest-value
623
+ contribution and needs no knowledge of the internals.
624
+
625
+ **Vocabulary for another language.** The special categories are English-only.
626
+ Any language added makes the `STRICT` level meaningfully more useful.
627
+
628
+ **A missed credential format.** A vendor whose token shape is not in
629
+ `VENDOR_CREDENTIALS`. These are quick to add and immediately valuable.
630
+
631
+ **A false positive you hit.** A real one, with the input that caused it, is
632
+ worth more than a hypothetical. Add it as a test.
633
+
634
+ Each detector needs a test in **both** directions — something it must catch
635
+ and something it must leave alone. A detector with only positive tests tends
636
+ to be the one that over-matches in someone else's data.
637
+
638
+ ```bash
639
+ git clone https://github.com/janbalitaan/maskvault
640
+ cd maskvault
641
+ uv sync
642
+ uv run pytest
643
+ uv run ruff check src tests
644
+ uv run ty check src
645
+ ```
646
+
647
+ See `CONTRIBUTING.md` for the detector contract and how to add one.
648
+
649
+ ## License
650
+
651
+ MIT