aa-toolkit-redact 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aa_toolkit_redact-0.1.0a1/LICENSE +21 -0
- aa_toolkit_redact-0.1.0a1/PKG-INFO +303 -0
- aa_toolkit_redact-0.1.0a1/README.md +294 -0
- aa_toolkit_redact-0.1.0a1/pyproject.toml +16 -0
- aa_toolkit_redact-0.1.0a1/pyproject.toml.orig +16 -0
- aa_toolkit_redact-0.1.0a1/src/aa_redact/__init__.py +7 -0
- aa_toolkit_redact-0.1.0a1/src/aa_redact/_detectors.py +316 -0
- aa_toolkit_redact-0.1.0a1/src/aa_redact/_finding.py +20 -0
- aa_toolkit_redact-0.1.0a1/src/aa_redact/_names.py +127 -0
- aa_toolkit_redact-0.1.0a1/src/aa_redact/_redactor.py +169 -0
- aa_toolkit_redact-0.1.0a1/src/aa_redact/py.typed +0 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Aaron Aco
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: aa-toolkit-redact
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Takes sensitive values out of text, with Python's standard library only.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: >=3.14
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
|
|
10
|
+
# aa-toolkit-redact
|
|
11
|
+
|
|
12
|
+
Takes sensitive values out of text and data before they leave an app. It uses only Python's standard library, so it has no dependencies.
|
|
13
|
+
|
|
14
|
+
A `Redactor` finds sensitive values with Detectors and leaves a Placeholder such as `[EMAIL]` where each one was.
|
|
15
|
+
|
|
16
|
+
## Install
|
|
17
|
+
|
|
18
|
+
Install it from the toolkit's GitHub repo, pinned to a version tag:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
uv add "aa-toolkit-redact @ git+https://github.com/aaronaco/aa-toolkit@redact-v0.1.0#subdirectory=packages/aa-toolkit-redact"
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Use
|
|
25
|
+
|
|
26
|
+
Create a `Redactor` once and call it with any value:
|
|
27
|
+
|
|
28
|
+
```python
|
|
29
|
+
from aa_redact import Redactor
|
|
30
|
+
|
|
31
|
+
redact = Redactor()
|
|
32
|
+
|
|
33
|
+
print(redact("order 4521 for bob.smith@example.com"))
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
That prints:
|
|
37
|
+
|
|
38
|
+
```text
|
|
39
|
+
order 4521 for [EMAIL]
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
A `Redactor` doesn't change after it's created, so every thread can share one.
|
|
43
|
+
|
|
44
|
+
## What's found
|
|
45
|
+
|
|
46
|
+
These built-in Detectors are on without any setup:
|
|
47
|
+
|
|
48
|
+
| Kind | What it finds |
|
|
49
|
+
| --- | --- |
|
|
50
|
+
| `AWS_ACCESS_KEY` | AWS access key IDs: `AKIA`, `ASIA`, `ABIA` or `ACCA` and at least 16 more capitals or digits. |
|
|
51
|
+
| `GITHUB_TOKEN` | GitHub tokens: `ghp_`, `gho_`, `ghu_`, `ghs_` or `ghr_` and at least 36 more characters, or `github_pat_` and at least 22. |
|
|
52
|
+
| `API_KEY` | OpenAI-style keys: `sk-` and at least 20 more characters, with a digit or both capitals and small letters, so `sk-learn-…` isn't one. |
|
|
53
|
+
| `SLACK_TOKEN` | Slack tokens: `xoxb-`, `xoxp-`, `xoxd-`, `xapp-` and the like, with at least 10 more characters. |
|
|
54
|
+
| `JWT` | JSON Web Tokens: three dot-separated parts, the first two starting with `eyJ`. |
|
|
55
|
+
| `BEARER_TOKEN` | `Bearer` and the token after it, when the token has at least 8 characters and isn't a plain word. |
|
|
56
|
+
| `PRIVATE_KEY` | A private key from its begin line through its end line. A key cut off before its end line is hidden up to where its lines stop. |
|
|
57
|
+
| `URL_PASSWORD` | The password in a URL such as `postgres://user:password@host`. The rest of the URL stays. |
|
|
58
|
+
| `EMAIL` | Email addresses. |
|
|
59
|
+
| `CARD` | Card numbers that start like a real card and pass the Luhn check. They can be one run of digits, or groups of four (or 4, 6 and 5 for American Express) split by spaces, dashes or dots. |
|
|
60
|
+
|
|
61
|
+
Tokens match on their prefix and a minimum length, so they keep working when a service makes its tokens longer. A long number that starts like a card and passes the check still becomes `[CARD]`. Turn `CARD` off with `without=` if your app's numbers do that.
|
|
62
|
+
|
|
63
|
+
The `EMAIL` Detector reads at most 64 characters before the `@`, so a longer address keeps its first characters. An address that uses a symbol such as `!`, `&` or `/` before the `@` keeps what comes before the symbol.
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from aa_redact import Redactor
|
|
67
|
+
|
|
68
|
+
redact = Redactor()
|
|
69
|
+
|
|
70
|
+
print(redact("connect postgres://app:hunter2@db.internal:5432/orders failed"))
|
|
71
|
+
print(redact("charged 4111 1111 1111 1111 for order 4521"))
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
That prints:
|
|
75
|
+
|
|
76
|
+
```text
|
|
77
|
+
connect postgres://app:[URL_PASSWORD]@db.internal:5432/orders failed
|
|
78
|
+
charged [CARD] for order 4521
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### IP addresses and phone numbers
|
|
82
|
+
|
|
83
|
+
`ip_address` and `phone_number` are built in but off, because whether they count as personal data depends on the app. Pass them in `detectors=` to turn them on:
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from aa_redact import Redactor, ip_address, phone_number
|
|
87
|
+
|
|
88
|
+
redact = Redactor(detectors=[ip_address, phone_number])
|
|
89
|
+
|
|
90
|
+
print(redact("login from 203.0.113.7, callback +1 555 123 4567"))
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
That prints:
|
|
94
|
+
|
|
95
|
+
```text
|
|
96
|
+
login from [IP_ADDRESS], callback [PHONE]
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`ip_address` finds IPv4 and IPv6 addresses. `phone_number` finds numbers that start with `+` and a country code, numbers with an area code in brackets, like `(555) 123-4567`, and numbers written in groups, like `555-123-4567` or `020 7946 0958`. A run of digits with no `+`, brackets or groups isn't found, because it can't be told apart from an order number.
|
|
100
|
+
|
|
101
|
+
## Clean nested values
|
|
102
|
+
|
|
103
|
+
A `Redactor` gives back the same shape it was given:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from aa_redact import Redactor
|
|
107
|
+
|
|
108
|
+
redact = Redactor()
|
|
109
|
+
|
|
110
|
+
print(redact({"to": ["bob@example.com"], "qty": 2}))
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
That prints:
|
|
114
|
+
|
|
115
|
+
```text
|
|
116
|
+
{'to': ['[EMAIL]'], 'qty': 2}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
| Value | What comes back |
|
|
120
|
+
| --- | --- |
|
|
121
|
+
| Text | Text with Placeholders in it. |
|
|
122
|
+
| Dictionary, list, tuple or set | The same shape with every value cleaned, at any depth. Dictionary keys are cleaned too, and two keys that clean to the same text become one. A named tuple stays that named tuple, with its field names checked. Other subclasses come back as the plain type. |
|
|
123
|
+
| Number | The same number, or a Placeholder when a Detector matches it. |
|
|
124
|
+
| `None`, `True` or `False` | The same value. |
|
|
125
|
+
| Bytes | Text, decoded as UTF-8 and cleaned. |
|
|
126
|
+
| Any other object | The text it would be shown as, cleaned. An object that can't be turned into text becomes a note such as `[UNPRINTABLE Customer]`. |
|
|
127
|
+
| A structure that contains itself, or is nested more than 32 levels deep | `[REDACTED]` in place of that part. |
|
|
128
|
+
|
|
129
|
+
## Hide values by field name
|
|
130
|
+
|
|
131
|
+
A value under a Sensitive name is hidden whatever it looks like. The same rule applies to `name=value` pairs inside text:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from aa_redact import Redactor
|
|
135
|
+
|
|
136
|
+
redact = Redactor()
|
|
137
|
+
|
|
138
|
+
print(redact({"access_token": "abc123", "token_count": 5}))
|
|
139
|
+
print(redact("login password=abc123 page=2"))
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
That prints:
|
|
143
|
+
|
|
144
|
+
```text
|
|
145
|
+
{'access_token': '[REDACTED]', 'token_count': 5}
|
|
146
|
+
login password=[REDACTED] page=2
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Names are compared after lower-casing them and treating dashes, underscores and camelCase as word breaks, so `X-Api-Key`, `x_api_key` and `xApiKey` are the same name.
|
|
150
|
+
|
|
151
|
+
- `password`, `passwd`, `secret` and `credential` match anywhere in the name, so `passwords` and `secret_key` are hidden, and so are harmless names like `secretary`.
|
|
152
|
+
- `token`, `api_key`, `apikey`, `access_key`, `private_key`, `authorization`, `cookie`, `pwd` and `passphrase` match when they are the last words of the name, so `access_token` is hidden and `token_count` isn't.
|
|
153
|
+
|
|
154
|
+
A dictionary or list under a Sensitive name is hidden as a whole. `None` and booleans stay, because they hold no secret.
|
|
155
|
+
|
|
156
|
+
Inside text, the pair can be written `name=value`, `name: value`, with quotes, as Python prints a dictionary, or as escaped JSON. A name in quotes can hold spaces, like `"API Key"`. The name stays and the value becomes `[REDACTED]`. A value in brackets, or written as a call like `Secret('abc')`, is hidden up to its closing bracket. A value without quotes runs to the next space, `"` or `&`, and a value after `authorization` or `cookie` runs to the end of the line. So `Server=db;Password=abc;Port=1` hides the port as well.
|
|
157
|
+
|
|
158
|
+
Pass `names=` to add words, which match as the last words of a name. Pass `allow_names=` to keep exact field names visible, both as fields and inside text. It never changes the words, so allowing `token` leaves `access_token` hidden. A value under an allowed name still goes through the Detectors.
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
from aa_redact import Redactor
|
|
162
|
+
|
|
163
|
+
redact = Redactor(names=["ssn"], allow_names=["password_length"])
|
|
164
|
+
|
|
165
|
+
print(redact({"customer_ssn": "123", "password_length": 12, "password": "abc"}))
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
That prints:
|
|
169
|
+
|
|
170
|
+
```text
|
|
171
|
+
{'customer_ssn': '[REDACTED]', 'password_length': 12, 'password': '[REDACTED]'}
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
## Add your own Detector
|
|
175
|
+
|
|
176
|
+
`pattern(kind, regex)` makes a Detector from a regular expression. Pass it in `detectors=`. Your Detectors are added to the built-in ones, so adding one never turns another off.
|
|
177
|
+
|
|
178
|
+
```python
|
|
179
|
+
from aa_redact import Redactor, pattern
|
|
180
|
+
|
|
181
|
+
redact = Redactor(detectors=[pattern("TAX_ID", r"\b\d{3}-\d{3}-\d{3}\b")])
|
|
182
|
+
|
|
183
|
+
print(redact("tax id 123-456-789 for bob@example.com"))
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
That prints:
|
|
187
|
+
|
|
188
|
+
```text
|
|
189
|
+
tax id [TAX_ID] for [EMAIL]
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
A regex that isn't valid raises when `pattern()` is called, not later when text is checked.
|
|
193
|
+
|
|
194
|
+
A Detector can also be a plain function. It takes the text and returns a `Finding(kind, start, end)` for each value it found, where `text[start:end]` is the value:
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
from aa_redact import Finding, Redactor
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def order_numbers(text: str) -> list[Finding]:
|
|
201
|
+
start = text.find("A-")
|
|
202
|
+
return [Finding("ORDER", start, start + 6)] if start >= 0 else []
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
redact = Redactor(detectors=[order_numbers])
|
|
206
|
+
|
|
207
|
+
print(redact("order A-4521 shipped"))
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
That prints:
|
|
211
|
+
|
|
212
|
+
```text
|
|
213
|
+
order [ORDER] shipped
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
When a Detector raises an error, the error comes out of the `Redactor`. So does a `ValueError` when a `Finding` doesn't fit the text.
|
|
217
|
+
|
|
218
|
+
When Findings overlap, they become one Placeholder. The longest Finding gives the kind. When two are the same length, the earlier Detector wins: the built-in ones come first, then yours in the order you passed them.
|
|
219
|
+
|
|
220
|
+
## Hide Known values
|
|
221
|
+
|
|
222
|
+
`known_values(kind, values)` makes a Detector for the exact secrets your app already holds, such as a database password or a key with no prefix. Each Known value is hidden when it appears as written, URL-encoded or JSON-escaped. Empty values are ignored, so an unset setting can be passed safely, and when one Known value is contained in a longer one, the longer one is matched first.
|
|
223
|
+
|
|
224
|
+
```python
|
|
225
|
+
from aa_redact import Redactor, known_values
|
|
226
|
+
|
|
227
|
+
redact = Redactor(detectors=[known_values("DB_PASSWORD", ["p@ss/w0rd", ""])])
|
|
228
|
+
|
|
229
|
+
print(redact("dsn=https://db.internal/?pw=p%40ss%2Fw0rd"))
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
That prints:
|
|
233
|
+
|
|
234
|
+
```text
|
|
235
|
+
dsn=https://db.internal/?pw=[DB_PASSWORD]
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
## Turn off a built-in Detector
|
|
239
|
+
|
|
240
|
+
Pass a list of kinds in `without=`. Every other Detector still runs, and so do the ones you added.
|
|
241
|
+
|
|
242
|
+
```python
|
|
243
|
+
from aa_redact import Redactor
|
|
244
|
+
|
|
245
|
+
redact = Redactor(without=["EMAIL"])
|
|
246
|
+
|
|
247
|
+
print(redact("order 4521 for bob.smith@example.com"))
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
That prints:
|
|
251
|
+
|
|
252
|
+
```text
|
|
253
|
+
order 4521 for bob.smith@example.com
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
## Find without replacing
|
|
257
|
+
|
|
258
|
+
Call `find()` with text to get the Findings the `Redactor` would replace, in text order, without changing the text. Each `Finding` gives its `kind`, `start` and `end`, and never the value itself:
|
|
259
|
+
|
|
260
|
+
```python
|
|
261
|
+
from aa_redact import Redactor
|
|
262
|
+
|
|
263
|
+
redact = Redactor()
|
|
264
|
+
|
|
265
|
+
print(redact.find("mail jane@corp.com password=hunter2"))
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
That prints:
|
|
269
|
+
|
|
270
|
+
```text
|
|
271
|
+
[Finding(kind='EMAIL', start=5, end=18), Finding(kind='REDACTED', start=28, end=35)]
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
`find()` takes text only and follows the same Detectors, Sensitive names, Known values, `without=`, `allow_names=` and `max_length=` rules as replacing.
|
|
275
|
+
|
|
276
|
+
## Size limit
|
|
277
|
+
|
|
278
|
+
Text longer than `max_length`, 64 KB (`65536` characters) by default per value, is cut at the limit and ends with `[TRUNCATED]`. Detectors run on the limit plus a 64 KB margin (`LONGEST_VALUE`) so a value that sits across the cut is still found and no part of it comes back. Text past the cut is never returned.
|
|
279
|
+
|
|
280
|
+
Pass `max_length=` to set a different limit:
|
|
281
|
+
|
|
282
|
+
```python
|
|
283
|
+
from aa_redact import Redactor
|
|
284
|
+
|
|
285
|
+
redact = Redactor(max_length=12)
|
|
286
|
+
|
|
287
|
+
print(redact("user: bob.smith@example.com and more"))
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
That prints:
|
|
291
|
+
|
|
292
|
+
```text
|
|
293
|
+
user: [EMAIL][TRUNCATED]
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
## What this library can't do
|
|
297
|
+
|
|
298
|
+
Redaction is a safety net, not a guarantee. The better fix is not logging secrets at all.
|
|
299
|
+
|
|
300
|
+
- **Values with no recognizable pattern aren't found.** Patterns can't find a person's name, a street address, or an AWS secret access key, which is 40 random base64 characters with no prefix. Pass secrets your app already holds to `known_values()`, or keep them under Sensitive names.
|
|
301
|
+
- **Base64-encoded forms of Known values aren't found.** `known_values()` checks each value as written, URL-encoded and JSON-escaped.
|
|
302
|
+
- **A very slow custom pattern can stall logging.** Every built-in pattern has bounded repeats so hostile input stays fast, and `max_length` caps how much text is checked, but Python's `re` has no timeout. Write custom patterns in `pattern()` with bounded repeats and no nested `+` or `*`.
|
|
303
|
+
- **Log handlers an app adds itself aren't covered.** When used with [`aa-toolkit-log`](../aa-toolkit-log/README.md), `configure_logging(redact=...)` cleans only the handlers it installs.
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
# aa-toolkit-redact
|
|
2
|
+
|
|
3
|
+
Takes sensitive values out of text and data before they leave an app. It uses only Python's standard library, so it has no dependencies.
|
|
4
|
+
|
|
5
|
+
A `Redactor` finds sensitive values with Detectors and leaves a Placeholder such as `[EMAIL]` where each one was.
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
Install it from the toolkit's GitHub repo, pinned to a version tag:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
uv add "aa-toolkit-redact @ git+https://github.com/aaronaco/aa-toolkit@redact-v0.1.0#subdirectory=packages/aa-toolkit-redact"
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Use
|
|
16
|
+
|
|
17
|
+
Create a `Redactor` once and call it with any value:
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from aa_redact import Redactor
|
|
21
|
+
|
|
22
|
+
redact = Redactor()
|
|
23
|
+
|
|
24
|
+
print(redact("order 4521 for bob.smith@example.com"))
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
That prints:
|
|
28
|
+
|
|
29
|
+
```text
|
|
30
|
+
order 4521 for [EMAIL]
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
A `Redactor` doesn't change after it's created, so every thread can share one.
|
|
34
|
+
|
|
35
|
+
## What's found
|
|
36
|
+
|
|
37
|
+
These built-in Detectors are on without any setup:
|
|
38
|
+
|
|
39
|
+
| Kind | What it finds |
|
|
40
|
+
| --- | --- |
|
|
41
|
+
| `AWS_ACCESS_KEY` | AWS access key IDs: `AKIA`, `ASIA`, `ABIA` or `ACCA` and at least 16 more capitals or digits. |
|
|
42
|
+
| `GITHUB_TOKEN` | GitHub tokens: `ghp_`, `gho_`, `ghu_`, `ghs_` or `ghr_` and at least 36 more characters, or `github_pat_` and at least 22. |
|
|
43
|
+
| `API_KEY` | OpenAI-style keys: `sk-` and at least 20 more characters, with a digit or both capitals and small letters, so `sk-learn-…` isn't one. |
|
|
44
|
+
| `SLACK_TOKEN` | Slack tokens: `xoxb-`, `xoxp-`, `xoxd-`, `xapp-` and the like, with at least 10 more characters. |
|
|
45
|
+
| `JWT` | JSON Web Tokens: three dot-separated parts, the first two starting with `eyJ`. |
|
|
46
|
+
| `BEARER_TOKEN` | `Bearer` and the token after it, when the token has at least 8 characters and isn't a plain word. |
|
|
47
|
+
| `PRIVATE_KEY` | A private key from its begin line through its end line. A key cut off before its end line is hidden up to where its lines stop. |
|
|
48
|
+
| `URL_PASSWORD` | The password in a URL such as `postgres://user:password@host`. The rest of the URL stays. |
|
|
49
|
+
| `EMAIL` | Email addresses. |
|
|
50
|
+
| `CARD` | Card numbers that start like a real card and pass the Luhn check. They can be one run of digits, or groups of four (or 4, 6 and 5 for American Express) split by spaces, dashes or dots. |
|
|
51
|
+
|
|
52
|
+
Tokens match on their prefix and a minimum length, so they keep working when a service makes its tokens longer. A long number that starts like a card and passes the check still becomes `[CARD]`. Turn `CARD` off with `without=` if your app's numbers do that.
|
|
53
|
+
|
|
54
|
+
The `EMAIL` Detector reads at most 64 characters before the `@`, so a longer address keeps its first characters. An address that uses a symbol such as `!`, `&` or `/` before the `@` keeps what comes before the symbol.
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from aa_redact import Redactor
|
|
58
|
+
|
|
59
|
+
redact = Redactor()
|
|
60
|
+
|
|
61
|
+
print(redact("connect postgres://app:hunter2@db.internal:5432/orders failed"))
|
|
62
|
+
print(redact("charged 4111 1111 1111 1111 for order 4521"))
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
That prints:
|
|
66
|
+
|
|
67
|
+
```text
|
|
68
|
+
connect postgres://app:[URL_PASSWORD]@db.internal:5432/orders failed
|
|
69
|
+
charged [CARD] for order 4521
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### IP addresses and phone numbers
|
|
73
|
+
|
|
74
|
+
`ip_address` and `phone_number` are built in but off, because whether they count as personal data depends on the app. Pass them in `detectors=` to turn them on:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from aa_redact import Redactor, ip_address, phone_number
|
|
78
|
+
|
|
79
|
+
redact = Redactor(detectors=[ip_address, phone_number])
|
|
80
|
+
|
|
81
|
+
print(redact("login from 203.0.113.7, callback +1 555 123 4567"))
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
That prints:
|
|
85
|
+
|
|
86
|
+
```text
|
|
87
|
+
login from [IP_ADDRESS], callback [PHONE]
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
`ip_address` finds IPv4 and IPv6 addresses. `phone_number` finds numbers that start with `+` and a country code, numbers with an area code in brackets, like `(555) 123-4567`, and numbers written in groups, like `555-123-4567` or `020 7946 0958`. A run of digits with no `+`, brackets or groups isn't found, because it can't be told apart from an order number.
|
|
91
|
+
|
|
92
|
+
## Clean nested values
|
|
93
|
+
|
|
94
|
+
A `Redactor` gives back the same shape it was given:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
from aa_redact import Redactor
|
|
98
|
+
|
|
99
|
+
redact = Redactor()
|
|
100
|
+
|
|
101
|
+
print(redact({"to": ["bob@example.com"], "qty": 2}))
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
That prints:
|
|
105
|
+
|
|
106
|
+
```text
|
|
107
|
+
{'to': ['[EMAIL]'], 'qty': 2}
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
| Value | What comes back |
|
|
111
|
+
| --- | --- |
|
|
112
|
+
| Text | Text with Placeholders in it. |
|
|
113
|
+
| Dictionary, list, tuple or set | The same shape with every value cleaned, at any depth. Dictionary keys are cleaned too, and two keys that clean to the same text become one. A named tuple stays that named tuple, with its field names checked. Other subclasses come back as the plain type. |
|
|
114
|
+
| Number | The same number, or a Placeholder when a Detector matches it. |
|
|
115
|
+
| `None`, `True` or `False` | The same value. |
|
|
116
|
+
| Bytes | Text, decoded as UTF-8 and cleaned. |
|
|
117
|
+
| Any other object | The text it would be shown as, cleaned. An object that can't be turned into text becomes a note such as `[UNPRINTABLE Customer]`. |
|
|
118
|
+
| A structure that contains itself, or is nested more than 32 levels deep | `[REDACTED]` in place of that part. |
|
|
119
|
+
|
|
120
|
+
## Hide values by field name
|
|
121
|
+
|
|
122
|
+
A value under a Sensitive name is hidden whatever it looks like. The same rule applies to `name=value` pairs inside text:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
from aa_redact import Redactor
|
|
126
|
+
|
|
127
|
+
redact = Redactor()
|
|
128
|
+
|
|
129
|
+
print(redact({"access_token": "abc123", "token_count": 5}))
|
|
130
|
+
print(redact("login password=abc123 page=2"))
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
That prints:
|
|
134
|
+
|
|
135
|
+
```text
|
|
136
|
+
{'access_token': '[REDACTED]', 'token_count': 5}
|
|
137
|
+
login password=[REDACTED] page=2
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Names are compared after lower-casing them and treating dashes, underscores and camelCase as word breaks, so `X-Api-Key`, `x_api_key` and `xApiKey` are the same name.
|
|
141
|
+
|
|
142
|
+
- `password`, `passwd`, `secret` and `credential` match anywhere in the name, so `passwords` and `secret_key` are hidden, and so are harmless names like `secretary`.
|
|
143
|
+
- `token`, `api_key`, `apikey`, `access_key`, `private_key`, `authorization`, `cookie`, `pwd` and `passphrase` match when they are the last words of the name, so `access_token` is hidden and `token_count` isn't.
|
|
144
|
+
|
|
145
|
+
A dictionary or list under a Sensitive name is hidden as a whole. `None` and booleans stay, because they hold no secret.
|
|
146
|
+
|
|
147
|
+
Inside text, the pair can be written `name=value`, `name: value`, with quotes, as Python prints a dictionary, or as escaped JSON. A name in quotes can hold spaces, like `"API Key"`. The name stays and the value becomes `[REDACTED]`. A value in brackets, or written as a call like `Secret('abc')`, is hidden up to its closing bracket. A value without quotes runs to the next space, `"` or `&`, and a value after `authorization` or `cookie` runs to the end of the line. So `Server=db;Password=abc;Port=1` hides the port as well.
|
|
148
|
+
|
|
149
|
+
Pass `names=` to add words, which match as the last words of a name. Pass `allow_names=` to keep exact field names visible, both as fields and inside text. It never changes the words, so allowing `token` leaves `access_token` hidden. A value under an allowed name still goes through the Detectors.
|
|
150
|
+
|
|
151
|
+
```python
|
|
152
|
+
from aa_redact import Redactor
|
|
153
|
+
|
|
154
|
+
redact = Redactor(names=["ssn"], allow_names=["password_length"])
|
|
155
|
+
|
|
156
|
+
print(redact({"customer_ssn": "123", "password_length": 12, "password": "abc"}))
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
That prints:
|
|
160
|
+
|
|
161
|
+
```text
|
|
162
|
+
{'customer_ssn': '[REDACTED]', 'password_length': 12, 'password': '[REDACTED]'}
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## Add your own Detector
|
|
166
|
+
|
|
167
|
+
`pattern(kind, regex)` makes a Detector from a regular expression. Pass it in `detectors=`. Your Detectors are added to the built-in ones, so adding one never turns another off.
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
from aa_redact import Redactor, pattern
|
|
171
|
+
|
|
172
|
+
redact = Redactor(detectors=[pattern("TAX_ID", r"\b\d{3}-\d{3}-\d{3}\b")])
|
|
173
|
+
|
|
174
|
+
print(redact("tax id 123-456-789 for bob@example.com"))
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
That prints:
|
|
178
|
+
|
|
179
|
+
```text
|
|
180
|
+
tax id [TAX_ID] for [EMAIL]
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
A regex that isn't valid raises when `pattern()` is called, not later when text is checked.
|
|
184
|
+
|
|
185
|
+
A Detector can also be a plain function. It takes the text and returns a `Finding(kind, start, end)` for each value it found, where `text[start:end]` is the value:
|
|
186
|
+
|
|
187
|
+
```python
|
|
188
|
+
from aa_redact import Finding, Redactor
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def order_numbers(text: str) -> list[Finding]:
|
|
192
|
+
start = text.find("A-")
|
|
193
|
+
return [Finding("ORDER", start, start + 6)] if start >= 0 else []
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
redact = Redactor(detectors=[order_numbers])
|
|
197
|
+
|
|
198
|
+
print(redact("order A-4521 shipped"))
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
That prints:
|
|
202
|
+
|
|
203
|
+
```text
|
|
204
|
+
order [ORDER] shipped
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
When a Detector raises an error, the error comes out of the `Redactor`. So does a `ValueError` when a `Finding` doesn't fit the text.
|
|
208
|
+
|
|
209
|
+
When Findings overlap, they become one Placeholder. The longest Finding gives the kind. When two are the same length, the earlier Detector wins: the built-in ones come first, then yours in the order you passed them.
|
|
210
|
+
|
|
211
|
+
## Hide Known values
|
|
212
|
+
|
|
213
|
+
`known_values(kind, values)` makes a Detector for the exact secrets your app already holds, such as a database password or a key with no prefix. Each Known value is hidden when it appears as written, URL-encoded or JSON-escaped. Empty values are ignored, so an unset setting can be passed safely, and when one Known value is contained in a longer one, the longer one is matched first.
|
|
214
|
+
|
|
215
|
+
```python
|
|
216
|
+
from aa_redact import Redactor, known_values
|
|
217
|
+
|
|
218
|
+
redact = Redactor(detectors=[known_values("DB_PASSWORD", ["p@ss/w0rd", ""])])
|
|
219
|
+
|
|
220
|
+
print(redact("dsn=https://db.internal/?pw=p%40ss%2Fw0rd"))
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
That prints:
|
|
224
|
+
|
|
225
|
+
```text
|
|
226
|
+
dsn=https://db.internal/?pw=[DB_PASSWORD]
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
## Turn off a built-in Detector
|
|
230
|
+
|
|
231
|
+
Pass a list of kinds in `without=`. Every other Detector still runs, and so do the ones you added.
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
from aa_redact import Redactor
|
|
235
|
+
|
|
236
|
+
redact = Redactor(without=["EMAIL"])
|
|
237
|
+
|
|
238
|
+
print(redact("order 4521 for bob.smith@example.com"))
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
That prints:
|
|
242
|
+
|
|
243
|
+
```text
|
|
244
|
+
order 4521 for bob.smith@example.com
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
## Find without replacing
|
|
248
|
+
|
|
249
|
+
Call `find()` with text to get the Findings the `Redactor` would replace, in text order, without changing the text. Each `Finding` gives its `kind`, `start` and `end`, and never the value itself:
|
|
250
|
+
|
|
251
|
+
```python
|
|
252
|
+
from aa_redact import Redactor
|
|
253
|
+
|
|
254
|
+
redact = Redactor()
|
|
255
|
+
|
|
256
|
+
print(redact.find("mail jane@corp.com password=hunter2"))
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
That prints:
|
|
260
|
+
|
|
261
|
+
```text
|
|
262
|
+
[Finding(kind='EMAIL', start=5, end=18), Finding(kind='REDACTED', start=28, end=35)]
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
`find()` takes text only and follows the same Detectors, Sensitive names, Known values, `without=`, `allow_names=` and `max_length=` rules as replacing.
|
|
266
|
+
|
|
267
|
+
## Size limit
|
|
268
|
+
|
|
269
|
+
Text longer than `max_length`, 64 KB (`65536` characters) by default per value, is cut at the limit and ends with `[TRUNCATED]`. Detectors run on the limit plus a 64 KB margin (`LONGEST_VALUE`) so a value that sits across the cut is still found and no part of it comes back. Text past the cut is never returned.
|
|
270
|
+
|
|
271
|
+
Pass `max_length=` to set a different limit:
|
|
272
|
+
|
|
273
|
+
```python
|
|
274
|
+
from aa_redact import Redactor
|
|
275
|
+
|
|
276
|
+
redact = Redactor(max_length=12)
|
|
277
|
+
|
|
278
|
+
print(redact("user: bob.smith@example.com and more"))
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
That prints:
|
|
282
|
+
|
|
283
|
+
```text
|
|
284
|
+
user: [EMAIL][TRUNCATED]
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
## What this library can't do
|
|
288
|
+
|
|
289
|
+
Redaction is a safety net, not a guarantee. The better fix is not logging secrets at all.
|
|
290
|
+
|
|
291
|
+
- **Values with no recognizable pattern aren't found.** Patterns can't find a person's name, a street address, or an AWS secret access key, which is 40 random base64 characters with no prefix. Pass secrets your app already holds to `known_values()`, or keep them under Sensitive names.
|
|
292
|
+
- **Base64-encoded forms of Known values aren't found.** `known_values()` checks each value as written, URL-encoded and JSON-escaped.
|
|
293
|
+
- **A very slow custom pattern can stall logging.** Every built-in pattern has bounded repeats so hostile input stays fast, and `max_length` caps how much text is checked, but Python's `re` has no timeout. Write custom patterns in `pattern()` with bounded repeats and no nested `+` or `*`.
|
|
294
|
+
- **Log handlers an app adds itself aren't covered.** When used with [`aa-toolkit-log`](../aa-toolkit-log/README.md), `configure_logging(redact=...)` cleans only the handlers it installs.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "aa-toolkit-redact"
|
|
3
|
+
version = "0.1.0a1"
|
|
4
|
+
description = "Takes sensitive values out of text, with Python's standard library only."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.14"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
dependencies = []
|
|
10
|
+
|
|
11
|
+
[build-system]
|
|
12
|
+
requires = ["uv_build>=0.13.7,<0.14"]
|
|
13
|
+
build-backend = "uv_build"
|
|
14
|
+
|
|
15
|
+
[tool.uv.build-backend]
|
|
16
|
+
module-name = "aa_redact"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "aa-toolkit-redact"
|
|
3
|
+
version = "0.1.0a1"
|
|
4
|
+
description = "Takes sensitive values out of text, with Python's standard library only."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.14"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
|
+
dependencies = []
|
|
10
|
+
|
|
11
|
+
[build-system]
|
|
12
|
+
requires = ["uv_build>=0.13.7,<0.14"]
|
|
13
|
+
build-backend = "uv_build"
|
|
14
|
+
|
|
15
|
+
[tool.uv.build-backend]
|
|
16
|
+
module-name = "aa_redact"
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Takes sensitive values out of text and data before they leave an app."""
|
|
2
|
+
|
|
3
|
+
from aa_redact._detectors import ip_address, known_values, pattern, phone_number
|
|
4
|
+
from aa_redact._finding import Finding
|
|
5
|
+
from aa_redact._redactor import Redactor
|
|
6
|
+
|
|
7
|
+
__all__ = ["Finding", "Redactor", "ip_address", "known_values", "pattern", "phone_number"]
|
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"""Making Detectors, and the ones that come built in."""
|
|
2
|
+
|
|
3
|
+
import bisect
|
|
4
|
+
import ipaddress
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import unicodedata
|
|
8
|
+
import urllib.parse
|
|
9
|
+
from collections.abc import Callable, Iterable, Iterator
|
|
10
|
+
|
|
11
|
+
from aa_redact._finding import LONGEST_VALUE, Detector, Finding
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def pattern(kind: str, regex: str) -> Detector:
|
|
15
|
+
"""Make a Detector that finds every match of a regular expression."""
|
|
16
|
+
compiled = re.compile(regex)
|
|
17
|
+
|
|
18
|
+
def detect(text: str) -> list[Finding]:
|
|
19
|
+
return [Finding(kind, *match.span()) for match in compiled.finditer(text) if match[0]]
|
|
20
|
+
|
|
21
|
+
return detect
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
_PERCENT_ESCAPE = re.compile(r"%[0-9A-Fa-f]{2}")
|
|
25
|
+
_UNICODE_ESCAPE = re.compile(r"\\u([0-9a-f]{4})")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _known_regex(part: str) -> str:
|
|
29
|
+
"""Escape one encoded character for regex, matching its hex escapes in either case."""
|
|
30
|
+
escaped = _PERCENT_ESCAPE.sub(lambda match: f"(?i:{match[0]})", re.escape(part))
|
|
31
|
+
return _UNICODE_ESCAPE.sub(r"\\u(?i:\1)", escaped)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _json_escape(parts: Iterable[str], *, ensure_ascii: bool) -> list[str]:
|
|
35
|
+
"""Escape each character part the way `json.dumps` writes it inside quotes."""
|
|
36
|
+
return [json.dumps(part, ensure_ascii=ensure_ascii)[1:-1] for part in parts]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _known_forms(value: str) -> Iterator[tuple[int, str]]:
|
|
40
|
+
"""Yield `(length, regex)` for a Known value as written, URL-encoded and JSON-escaped."""
|
|
41
|
+
for url_parts in (
|
|
42
|
+
list(value),
|
|
43
|
+
[urllib.parse.quote(character, safe="") for character in value],
|
|
44
|
+
[urllib.parse.quote(character) for character in value],
|
|
45
|
+
[urllib.parse.quote_plus(character) for character in value],
|
|
46
|
+
):
|
|
47
|
+
for ensure_ascii in (True, False):
|
|
48
|
+
escaped = _json_escape(url_parts, ensure_ascii=ensure_ascii)
|
|
49
|
+
for parts in (
|
|
50
|
+
url_parts,
|
|
51
|
+
escaped,
|
|
52
|
+
[part.replace("/", r"\/") for part in escaped],
|
|
53
|
+
_json_escape(escaped, ensure_ascii=ensure_ascii),
|
|
54
|
+
):
|
|
55
|
+
yield (
|
|
56
|
+
sum(len(part) for part in parts),
|
|
57
|
+
"".join(_known_regex(part) for part in parts),
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def known_values(kind: str, values: Iterable[str]) -> Detector:
|
|
62
|
+
"""Make a Detector that finds the exact values an app already holds."""
|
|
63
|
+
if isinstance(values, str):
|
|
64
|
+
raise TypeError(f"values takes a list, like [{values!r}]")
|
|
65
|
+
forms = [
|
|
66
|
+
regex
|
|
67
|
+
for _, regex in sorted(
|
|
68
|
+
{form for value in values if value for form in _known_forms(value)},
|
|
69
|
+
reverse=True,
|
|
70
|
+
)
|
|
71
|
+
]
|
|
72
|
+
if not forms:
|
|
73
|
+
return lambda text: ()
|
|
74
|
+
compiled = re.compile("|".join(forms))
|
|
75
|
+
|
|
76
|
+
def detect(text: str) -> Iterator[Finding]:
|
|
77
|
+
position = 0
|
|
78
|
+
end = 0
|
|
79
|
+
while match := compiled.search(text, position):
|
|
80
|
+
if match.end() > end:
|
|
81
|
+
end = match.end()
|
|
82
|
+
yield Finding(kind, *match.span())
|
|
83
|
+
position = match.start() + 1
|
|
84
|
+
|
|
85
|
+
return detect
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _kept(kind: str, regex: str, keep: Callable[[str], bool]) -> Detector:
|
|
89
|
+
"""Make a Detector for the matches whose `value` group `keep` agrees with."""
|
|
90
|
+
compiled = re.compile(regex)
|
|
91
|
+
|
|
92
|
+
def detect(text: str) -> list[Finding]:
|
|
93
|
+
return [
|
|
94
|
+
Finding(kind, *match.span())
|
|
95
|
+
for match in compiled.finditer(text)
|
|
96
|
+
if keep(match["value"])
|
|
97
|
+
]
|
|
98
|
+
|
|
99
|
+
return detect
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _mark_ranges() -> str:
|
|
103
|
+
"""Return the combining marks, such as accents and vowel signs, as regex character ranges.
|
|
104
|
+
|
|
105
|
+
A regex's word class leaves them out, so a name written with one would stop an email match.
|
|
106
|
+
"""
|
|
107
|
+
ranges: list[list[int]] = []
|
|
108
|
+
for code in range(0x300, 0x20000):
|
|
109
|
+
if unicodedata.category(chr(code))[0] == "M":
|
|
110
|
+
if ranges and ranges[-1][1] == code - 1:
|
|
111
|
+
ranges[-1][1] = code
|
|
112
|
+
else:
|
|
113
|
+
ranges.append([code, code])
|
|
114
|
+
return "".join(f"{chr(first)}-{chr(last)}" for first, last in ranges)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# Every repeat is bounded, so text written to be slow can't stall the app. A token starts only
|
|
118
|
+
# where a longer token can't have started, so a long run of token characters is tried once.
|
|
119
|
+
|
|
120
|
+
_MARK = _mark_ranges()
|
|
121
|
+
_LOCAL = rf"[\w{_MARK}.%+-]{{1,64}}"
|
|
122
|
+
_EMAIL = re.compile(
|
|
123
|
+
# An apostrophe counts only inside a name, so the quotes around an email stay.
|
|
124
|
+
rf"{_LOCAL}(?:(?<=\w)['’](?=\w){_LOCAL}){{0,3}}"
|
|
125
|
+
rf"@(?:[\w{_MARK}-]{{1,63}}\.){{1,126}}"
|
|
126
|
+
rf"(?:[xX][nN]--[A-Za-z0-9-]{{1,59}}|(?:[^\W\d_][{_MARK}]{{0,4}}){{2,24}})"
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# Tokens match on their prefix and a minimum length, so a longer format still matches.
|
|
130
|
+
_AWS = rf"(?<![A-Za-z0-9])(?:AKIA|ASIA|ABIA|ACCA)[A-Z0-9]{{16,{LONGEST_VALUE}}}+"
|
|
131
|
+
_GITHUB = (
|
|
132
|
+
rf"(?<![A-Za-z0-9])(?:gh[pousr]_[A-Za-z0-9]{{36,{LONGEST_VALUE}}}+"
|
|
133
|
+
rf"|github_pat_[A-Za-z0-9_]{{22,{LONGEST_VALUE}}}+)"
|
|
134
|
+
)
|
|
135
|
+
_OPENAI_STYLE = rf"(?<![A-Za-z0-9_-])sk-(?P<value>[A-Za-z0-9_-]{{20,{LONGEST_VALUE}}}+)"
|
|
136
|
+
_SLACK = (
|
|
137
|
+
r"(?<![A-Za-z0-9_.-])(?:xox[abcdeprs](?:\.xox[abcdeprs])?|xapp)-"
|
|
138
|
+
rf"[A-Za-z0-9%-]{{10,{LONGEST_VALUE}}}+"
|
|
139
|
+
)
|
|
140
|
+
_JWT_PART = rf"[A-Za-z0-9_-]{{5,{LONGEST_VALUE}}}+"
|
|
141
|
+
_JWT = rf"(?<![A-Za-z0-9_-])eyJ{_JWT_PART}\.eyJ{_JWT_PART}\.[A-Za-z0-9_-]{{0,{LONGEST_VALUE}}}+"
|
|
142
|
+
_BEARER = (
|
|
143
|
+
r"(?<![A-Za-z0-9_-])(?i:bearer)[ \t]{1,16}+"
|
|
144
|
+
rf"(?P<value>[A-Za-z0-9._~+/-]{{8,{LONGEST_VALUE}}}+={{0,8}}+)"
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
_KEY_LABEL = r"(?:[A-Z0-9]{1,20} ){0,4}PRIVATE KEY(?: BLOCK)?-----"
|
|
148
|
+
# A real line break, one escaped as \n, or none, as when a key's line breaks were taken out.
|
|
149
|
+
_KEY_LINE_BREAK = r"(?:\s|\\{1,4}[rn]){0,8}+"
|
|
150
|
+
_KEY_HEADER = r"[A-Za-z-]{1,32}+:[ \t]?[\w,.]{1,64}+(?:[ -][\w,.]{1,64}+){0,8}+"
|
|
151
|
+
_KEY_LINE = rf"(?:{_KEY_HEADER}|[A-Za-z0-9+/=.…]{{1,256}}+)" # with ... where it was shortened
|
|
152
|
+
# A key cut off before its end line is hidden up to where its lines stop. A long line is read
|
|
153
|
+
# in pieces, so no part of it is left.
|
|
154
|
+
_PEM = (
|
|
155
|
+
rf"-----BEGIN {_KEY_LABEL}(?:{_KEY_LINE_BREAK}{_KEY_LINE}){{0,{LONGEST_VALUE}}}+"
|
|
156
|
+
rf"(?:{_KEY_LINE_BREAK}-----END {_KEY_LABEL})?"
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
# A password may hold an @ that wasn't encoded, so it runs to the last @ before the host.
|
|
160
|
+
_URL_PASSWORD = re.compile(
|
|
161
|
+
r"(?<![A-Za-z0-9+.-])[A-Za-z][A-Za-z0-9+.-]{0,31}+:\\?/\\?/" # :// or JSON's :\/\/
|
|
162
|
+
rf"[^\s:/?#@\[\]\"'<>\\]{{0,{LONGEST_VALUE}}}+"
|
|
163
|
+
rf":(?P<password>[^\s/?#\"'<>\\]{{1,{LONGEST_VALUE}}})@"
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
# A card is written as one run of digits, or in the groups printed on it.
|
|
167
|
+
_CARD_SEPARATOR = "[ .-]"
|
|
168
|
+
_CARD = re.compile(
|
|
169
|
+
r"(?<![0-9])(?:"
|
|
170
|
+
r"[0-9]{13,19}+"
|
|
171
|
+
rf"|(?P<main>[0-9]{{4}}{_CARD_SEPARATOR}[0-9]{{4}}{_CARD_SEPARATOR}[0-9]{{4}}"
|
|
172
|
+
rf"{_CARD_SEPARATOR}[0-9]{{3,4}})(?P<tail>{_CARD_SEPARATOR}[0-9]{{1,3}})?"
|
|
173
|
+
rf"|[0-9]{{4}}{_CARD_SEPARATOR}[0-9]{{6}}{_CARD_SEPARATOR}[0-9]{{4,5}}"
|
|
174
|
+
r")(?![0-9])"
|
|
175
|
+
)
|
|
176
|
+
# Each issuer's first digits and the lengths its cards come in.
|
|
177
|
+
_CARD_ISSUER = re.compile(
|
|
178
|
+
r"4(?:[0-9]{12}|[0-9]{15}|[0-9]{18})" # Visa
|
|
179
|
+
r"|(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}"
|
|
180
|
+
r"|3[47][0-9]{13}" # American Express
|
|
181
|
+
r"|(?:6011|64[4-9][0-9]|65[0-9]{2})[0-9]{12,15}" # Discover
|
|
182
|
+
r"|62[0-9]{14,17}" # UnionPay
|
|
183
|
+
r"|35(?:2[89]|[3-8][0-9])[0-9]{12,15}" # JCB
|
|
184
|
+
r"|3(?:0[0-5][0-9]|095|[689][0-9]{2})[0-9]{10,15}" # Diners Club
|
|
185
|
+
r"|(?:5018|5020|5038|5893|6304|6759|676[1-3])[0-9]{9,15}" # Maestro
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
_IPV4 = re.compile(r"(?<![\d.])\d{1,3}(?:\.\d{1,3}){3}(?!\.?\d)", re.ASCII)
|
|
189
|
+
_IPV6 = re.compile(r"(?<![\w:.])[0-9A-Fa-f:.]{2,45}+(?![\w:])")
|
|
190
|
+
_HEX_DIGIT = re.compile(r"[0-9A-Fa-f]")
|
|
191
|
+
|
|
192
|
+
_PHONE = re.compile(
|
|
193
|
+
r"(?<![\w+.-])(?:"
|
|
194
|
+
r"\+[0-9]{1,3}[ -]?(?:\([0-9]{1,4}\)[ -]?)?(?:[0-9][ -]?){5,13}[0-9]" # +44 20 7946 0958
|
|
195
|
+
r"|\([0-9]{2,5}\)[ -]?(?:[0-9][ -]?){5,11}[0-9]" # (555) 123-4567
|
|
196
|
+
r"|[0-9]{3}(?P<sep>[ .-])[0-9]{3}(?P=sep)[0-9]{4}" # 555-123-4567
|
|
197
|
+
r"|0[0-9]{1,4}[ -][0-9]{3,4}[ -][0-9]{3,4}" # 020 7946 0958
|
|
198
|
+
r")(?!\w)"
|
|
199
|
+
)
|
|
200
|
+
_NOT_A_DIGIT = re.compile(r"\D", re.ASCII)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _looks_random(value: str) -> bool:
|
|
204
|
+
"""Say whether a key has a digit or both cases, as a random one does and `sk-learn` doesn't."""
|
|
205
|
+
return any(character.isdigit() for character in value) or (
|
|
206
|
+
any(character.isupper() for character in value)
|
|
207
|
+
and any(character.islower() for character in value)
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
_LONGEST_WORD = 24 # past any English word, so a longer run of letters is a token
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _is_plain_word(word: str) -> bool:
|
|
215
|
+
return (
|
|
216
|
+
len(word) <= _LONGEST_WORD
|
|
217
|
+
and word.isalpha()
|
|
218
|
+
and (word.islower() or word.isupper() or word.istitle())
|
|
219
|
+
)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _is_not_plain_words(value: str) -> bool:
|
|
223
|
+
"""Say whether a value isn't just words, like `authentication.` or `token-based`."""
|
|
224
|
+
return not all(_is_plain_word(word) for word in value.replace("-", ".").split(".") if word)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _url_password(text: str) -> list[Finding]:
|
|
228
|
+
return [
|
|
229
|
+
Finding("URL_PASSWORD", *match.span("password")) for match in _URL_PASSWORD.finditer(text)
|
|
230
|
+
]
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _email(text: str) -> Iterator[Finding]:
|
|
234
|
+
"""Find emails, leaving out a URL's `password@host`, which only looks like one."""
|
|
235
|
+
passwords = [match.span("password") for match in _URL_PASSWORD.finditer(text)]
|
|
236
|
+
starts = [start for start, _ in passwords]
|
|
237
|
+
for match in _EMAIL.finditer(text):
|
|
238
|
+
before = bisect.bisect_right(starts, match.start()) - 1
|
|
239
|
+
if before < 0 or not match.start() < passwords[before][1] < match.end():
|
|
240
|
+
yield Finding("EMAIL", *match.span())
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _is_card(digits: str) -> bool:
|
|
244
|
+
"""Say whether the digits start like a real card and pass the Luhn check."""
|
|
245
|
+
if not _CARD_ISSUER.fullmatch(digits):
|
|
246
|
+
return False
|
|
247
|
+
total = 0
|
|
248
|
+
for position, digit in enumerate(reversed(digits)):
|
|
249
|
+
value = int(digit) * (2 if position % 2 else 1)
|
|
250
|
+
total += value - 9 if value > 9 else value
|
|
251
|
+
return total % 10 == 0
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _card(text: str) -> Iterator[Finding]:
|
|
255
|
+
position = 0
|
|
256
|
+
while match := _CARD.search(text, position):
|
|
257
|
+
start = match.start()
|
|
258
|
+
# Digits after a grouped card, like an expiry date, may not be part of it.
|
|
259
|
+
ends = [match.end(), match.end("main")] if match["tail"] else [match.end()]
|
|
260
|
+
end = next((end for end in ends if _is_card(_NOT_A_DIGIT.sub("", text[start:end]))), None)
|
|
261
|
+
if end is None:
|
|
262
|
+
position = start + 1 # a card can still start at a later group of digits
|
|
263
|
+
else:
|
|
264
|
+
yield Finding("CARD", start, end)
|
|
265
|
+
position = end
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def ip_address(text: str) -> list[Finding]:
|
|
269
|
+
"""Find IPv4 and IPv6 addresses. Built in, and off until an app passes it in `detectors=`."""
|
|
270
|
+
found: list[Finding] = []
|
|
271
|
+
for match in _IPV4.finditer(text):
|
|
272
|
+
try:
|
|
273
|
+
ipaddress.IPv4Address(match[0])
|
|
274
|
+
except ValueError:
|
|
275
|
+
continue
|
|
276
|
+
found.append(Finding("IP_ADDRESS", *match.span()))
|
|
277
|
+
for match in _IPV6.finditer(text):
|
|
278
|
+
address = match[0].rstrip(".")
|
|
279
|
+
if not _is_ipv6(address) and address.endswith(":"):
|
|
280
|
+
address = address[:-1] # a colon after the address, as in `peer ::1: down`
|
|
281
|
+
if _is_ipv6(address):
|
|
282
|
+
found.append(Finding("IP_ADDRESS", match.start(), match.start() + len(address)))
|
|
283
|
+
return found
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _is_ipv6(address: str) -> bool:
|
|
287
|
+
if ":" not in address or not _HEX_DIGIT.search(address):
|
|
288
|
+
return False
|
|
289
|
+
try:
|
|
290
|
+
ipaddress.IPv6Address(address)
|
|
291
|
+
except ValueError:
|
|
292
|
+
return False
|
|
293
|
+
return True
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def phone_number(text: str) -> list[Finding]:
|
|
297
|
+
"""Find phone numbers. Built in, and off until an app passes it in `detectors=`."""
|
|
298
|
+
return [
|
|
299
|
+
Finding("PHONE", *match.span())
|
|
300
|
+
for match in _PHONE.finditer(text)
|
|
301
|
+
if 7 <= len(_NOT_A_DIGIT.sub("", match[0])) <= 15
|
|
302
|
+
]
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
BUILT_IN: dict[str, Detector] = {
|
|
306
|
+
"AWS_ACCESS_KEY": pattern("AWS_ACCESS_KEY", _AWS),
|
|
307
|
+
"GITHUB_TOKEN": pattern("GITHUB_TOKEN", _GITHUB),
|
|
308
|
+
"API_KEY": _kept("API_KEY", _OPENAI_STYLE, _looks_random),
|
|
309
|
+
"SLACK_TOKEN": pattern("SLACK_TOKEN", _SLACK),
|
|
310
|
+
"JWT": pattern("JWT", _JWT),
|
|
311
|
+
"BEARER_TOKEN": _kept("BEARER_TOKEN", _BEARER, _is_not_plain_words),
|
|
312
|
+
"PRIVATE_KEY": pattern("PRIVATE_KEY", _PEM),
|
|
313
|
+
"URL_PASSWORD": _url_password,
|
|
314
|
+
"EMAIL": _email,
|
|
315
|
+
"CARD": _card,
|
|
316
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""What a Detector reports back."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable, Iterable
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True, slots=True)
|
|
8
|
+
class Finding:
|
|
9
|
+
"""One place a Detector found a value: text[start:end], and what kind of value it is."""
|
|
10
|
+
|
|
11
|
+
kind: str
|
|
12
|
+
start: int
|
|
13
|
+
end: int
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
type Detector = Callable[[str], Iterable[Finding]]
|
|
17
|
+
|
|
18
|
+
LONGEST_VALUE = 65536
|
|
19
|
+
"""The longest value a built-in Detector reads, as long as the text a Redactor reads, so a
|
|
20
|
+
long value is never cut short and its end left visible."""
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Deciding which field names are Sensitive names."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from collections.abc import Iterable
|
|
5
|
+
|
|
6
|
+
from aa_redact._finding import LONGEST_VALUE, Finding
|
|
7
|
+
|
|
8
|
+
_ANYWHERE = ("password", "passwd", "secret", "credential")
|
|
9
|
+
_LAST_WORDS = (
|
|
10
|
+
"token",
|
|
11
|
+
"api_key",
|
|
12
|
+
"apikey",
|
|
13
|
+
"access_key",
|
|
14
|
+
"private_key",
|
|
15
|
+
"authorization",
|
|
16
|
+
"cookie",
|
|
17
|
+
"pwd",
|
|
18
|
+
"passphrase",
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
# These headers can hold several secrets separated by spaces or semicolons.
|
|
22
|
+
_WHOLE_LINE_NAMES = ("authorization", "cookie")
|
|
23
|
+
|
|
24
|
+
HIDDEN_KIND = "REDACTED"
|
|
25
|
+
|
|
26
|
+
_CAMEL_CASE_BREAK = re.compile(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])")
|
|
27
|
+
_WORD = re.compile(r"[^\W_]+")
|
|
28
|
+
|
|
29
|
+
# A name can start only where a longer name can't have started, so a long run of name
|
|
30
|
+
# characters is tried once. A value is read in steps, so a name inside it isn't tried again.
|
|
31
|
+
# The user name in a URL isn't one, so `https://x-access-token:pw@host` keeps its host.
|
|
32
|
+
_NAME = (
|
|
33
|
+
r"""(?<![\w.-])(?<!://)(?<!:\\/\\/)"""
|
|
34
|
+
rf"""(?P<name>[\w.-]{{1,{LONGEST_VALUE}}})(?:\\{{0,32}}["'])?""" # with a closing quote
|
|
35
|
+
)
|
|
36
|
+
_SPACED_NAME_CHARACTER = r"(?:\\[^\r\n]|(?!(?P=quote))[^\\\r\n])"
|
|
37
|
+
_SPACED_NAME = (
|
|
38
|
+
rf"""(?P<quote>\\{{0,32}}["'])(?P<spaced_name>{_SPACED_NAME_CHARACTER}{{1,256}}?)(?P=quote)"""
|
|
39
|
+
)
|
|
40
|
+
_SEPARATOR = r"[ \t]*[=:]{1,2}>?[ \t]*" # =, :, :=, == or =>
|
|
41
|
+
_NAME_AND_SEPARATOR = re.compile(rf"(?:{_NAME}|{_SPACED_NAME}){_SEPARATOR}")
|
|
42
|
+
_QUOTE = r"""(?P<slashes>\\{0,32})(?P<quote>["'])"""
|
|
43
|
+
_OPENING_QUOTE = re.compile(rf"b?{_QUOTE}")
|
|
44
|
+
_OPENING_BRACKET = re.compile(r"[\w.]{0,256}[\[({]")
|
|
45
|
+
_OPENING_BRACKETS = ("[", "(", "{")
|
|
46
|
+
_BRACKET_OR_QUOTE = re.compile(rf"[\[\](){{}}]|{_QUOTE}")
|
|
47
|
+
_WHOLE_LINE = re.compile(r"[^\r\n]+")
|
|
48
|
+
_UNQUOTED = re.compile(r'[^\s"&]+')
|
|
49
|
+
_AROUND_A_VALUE = ",;:.)}]'\"\\ \t" # punctuation after a value belongs to the text around it
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _words(name: str) -> tuple[str, ...]:
|
|
53
|
+
words: list[str] = _WORD.findall(_CAMEL_CASE_BREAK.sub(" ", name).lower())
|
|
54
|
+
return tuple(words)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _closing_quote(text: str, opening: re.Match[str]) -> int:
|
|
58
|
+
"""Find where a quoted value ends. Quotes inside it have more backslashes than its own."""
|
|
59
|
+
mark = opening["slashes"] + opening["quote"]
|
|
60
|
+
escaped_backslash = 2 * (len(opening["slashes"]) + 1)
|
|
61
|
+
closing = rf"(?<!\\)(?:\\{{{escaped_backslash}}})*{re.escape(mark)}"
|
|
62
|
+
found = re.compile(closing).search(text, opening.end())
|
|
63
|
+
return found.end() - len(mark) if found else len(text)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _closing_bracket(text: str, start: int) -> int:
|
|
67
|
+
depth = 0
|
|
68
|
+
position = start
|
|
69
|
+
while token := _BRACKET_OR_QUOTE.search(text, position):
|
|
70
|
+
position = token.end()
|
|
71
|
+
if token["quote"]:
|
|
72
|
+
position = _closing_quote(text, token) + len(token[0])
|
|
73
|
+
else:
|
|
74
|
+
depth += 1 if token[0] in _OPENING_BRACKETS else -1
|
|
75
|
+
if depth == 0:
|
|
76
|
+
return position
|
|
77
|
+
return len(text)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _value_span(text: str, start: int, whole_line: bool) -> tuple[int, int]:
|
|
81
|
+
"""Return where the value starting at `start` begins and ends, without its quotes."""
|
|
82
|
+
if opening := _OPENING_QUOTE.match(text, start):
|
|
83
|
+
return opening.end(), _closing_quote(text, opening)
|
|
84
|
+
end = start
|
|
85
|
+
while bracket := _OPENING_BRACKET.match(text, end):
|
|
86
|
+
end = _closing_bracket(text, bracket.end() - 1)
|
|
87
|
+
rest = (_WHOLE_LINE if whole_line else _UNQUOTED).match(text, end)
|
|
88
|
+
return start, end + len(rest[0].rstrip(_AROUND_A_VALUE)) if rest else end
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class SensitiveNames:
|
|
92
|
+
"""The words an app's Redactor hides values for, and the exact names it leaves visible."""
|
|
93
|
+
|
|
94
|
+
__slots__ = ("_allowed", "_endings")
|
|
95
|
+
|
|
96
|
+
def __init__(self, names: Iterable[str], allow_names: Iterable[str]) -> None:
|
|
97
|
+
added = [_words(name) for name in names]
|
|
98
|
+
for name, words in zip(names, added, strict=True):
|
|
99
|
+
if not words:
|
|
100
|
+
raise ValueError(f"names= takes words with letters or digits, not {name!r}")
|
|
101
|
+
self._endings = (*(_words(word) for word in _LAST_WORDS), *added)
|
|
102
|
+
self._allowed = {_words(name) for name in allow_names}
|
|
103
|
+
|
|
104
|
+
def is_sensitive(self, name: str) -> bool:
|
|
105
|
+
"""Say whether the value under this field name must be hidden."""
|
|
106
|
+
words = _words(name)
|
|
107
|
+
if words in self._allowed:
|
|
108
|
+
return False
|
|
109
|
+
joined = "".join(words)
|
|
110
|
+
return any(word in joined for word in _ANYWHERE) or any(
|
|
111
|
+
words[-len(ending) :] == ending for ending in self._endings
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
def find_values(self, text: str) -> list[Finding]:
|
|
115
|
+
"""Find the value of every `name=value` pair in text whose name must be hidden."""
|
|
116
|
+
found: list[Finding] = []
|
|
117
|
+
position = 0
|
|
118
|
+
while pair := _NAME_AND_SEPARATOR.search(text, position):
|
|
119
|
+
position = pair.end()
|
|
120
|
+
name = pair["name"] or pair["spaced_name"]
|
|
121
|
+
if self.is_sensitive(name):
|
|
122
|
+
whole_line = _words(name)[-1] in _WHOLE_LINE_NAMES
|
|
123
|
+
start, end = _value_span(text, pair.end(), whole_line)
|
|
124
|
+
position = max(position, end)
|
|
125
|
+
if start < end:
|
|
126
|
+
found.append(Finding(HIDDEN_KIND, start, end))
|
|
127
|
+
return found
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Walking any value, finding what's sensitive in it and replacing that with Placeholders."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
from numbers import Number
|
|
5
|
+
from typing import overload
|
|
6
|
+
|
|
7
|
+
from aa_redact._detectors import BUILT_IN
|
|
8
|
+
from aa_redact._finding import LONGEST_VALUE, Detector, Finding
|
|
9
|
+
from aa_redact._names import HIDDEN_KIND, SensitiveNames
|
|
10
|
+
|
|
11
|
+
_HIDDEN = f"[{HIDDEN_KIND}]"
|
|
12
|
+
_TRUNCATED = "[TRUNCATED]"
|
|
13
|
+
_MAX_DEPTH = 32 # far past real data, and well inside Python's recursion limit
|
|
14
|
+
_SHAPES = (list, tuple, set, frozenset)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _find(text: str, detectors: Iterable[Detector]) -> list[Finding]:
|
|
18
|
+
"""Return what the Detectors found, in text order, with overlaps merged into one Finding."""
|
|
19
|
+
found: list[tuple[Finding, int]] = []
|
|
20
|
+
for detector_order, detector in enumerate(detectors):
|
|
21
|
+
for finding in detector(text):
|
|
22
|
+
if not 0 <= finding.start < finding.end <= len(text):
|
|
23
|
+
raise ValueError(
|
|
24
|
+
f"a Detector returned a Finding that doesn't fit the text: {finding}"
|
|
25
|
+
)
|
|
26
|
+
found.append((finding, detector_order))
|
|
27
|
+
return _merge(found)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _merge(found: list[tuple[Finding, int]]) -> list[Finding]:
|
|
31
|
+
"""Merge overlapping Findings. The longest names the kind, then the earlier Detector."""
|
|
32
|
+
merged: list[Finding] = []
|
|
33
|
+
naming_rank = (0, 0) # the rank of the Finding that gave the last merged one its kind
|
|
34
|
+
for finding, detector_order in sorted(found, key=lambda pair: pair[0].start):
|
|
35
|
+
rank = (finding.end - finding.start, -detector_order)
|
|
36
|
+
if merged and finding.start < merged[-1].end:
|
|
37
|
+
last = merged[-1]
|
|
38
|
+
kind = finding.kind if rank > naming_rank else last.kind
|
|
39
|
+
merged[-1] = Finding(kind, last.start, max(last.end, finding.end))
|
|
40
|
+
naming_rank = max(naming_rank, rank)
|
|
41
|
+
else:
|
|
42
|
+
merged.append(finding)
|
|
43
|
+
naming_rank = rank
|
|
44
|
+
return merged
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _replace(text: str, findings: list[Finding]) -> str:
|
|
48
|
+
parts: list[str] = []
|
|
49
|
+
done = 0
|
|
50
|
+
for finding in findings:
|
|
51
|
+
parts += [text[done : finding.start], f"[{finding.kind}]"]
|
|
52
|
+
done = finding.end
|
|
53
|
+
return "".join([*parts, text[done:]])
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _listed(option: str, values: Iterable[str]) -> list[str]:
|
|
57
|
+
if isinstance(values, str):
|
|
58
|
+
raise TypeError(f"{option}= takes a list, like {option}=[{values!r}]")
|
|
59
|
+
return list(values)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _holds_no_secret(value: object) -> bool:
|
|
63
|
+
return value is None or isinstance(value, bool)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _shown_as(value: object) -> str | None:
|
|
67
|
+
"""Return the text a value would be shown as, or None when it can't be turned into text."""
|
|
68
|
+
try:
|
|
69
|
+
if isinstance(value, bytes | bytearray | memoryview):
|
|
70
|
+
return bytes(value).decode(errors="replace")
|
|
71
|
+
return str(value)
|
|
72
|
+
except Exception:
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class Redactor:
|
|
77
|
+
"""Cleans any value, leaving a Placeholder such as [EMAIL] where each sensitive one was."""
|
|
78
|
+
|
|
79
|
+
__slots__ = ("_detectors", "_max_length", "_names")
|
|
80
|
+
|
|
81
|
+
def __init__(
|
|
82
|
+
self,
|
|
83
|
+
*,
|
|
84
|
+
detectors: Iterable[Detector] = (),
|
|
85
|
+
names: Iterable[str] = (),
|
|
86
|
+
without: Iterable[str] = (),
|
|
87
|
+
allow_names: Iterable[str] = (),
|
|
88
|
+
max_length: int = LONGEST_VALUE,
|
|
89
|
+
) -> None:
|
|
90
|
+
"""Set up the Detectors, Sensitive names and size limit this Redactor uses."""
|
|
91
|
+
if isinstance(max_length, bool) or not isinstance(max_length, int) or max_length < 1:
|
|
92
|
+
raise ValueError(f"max_length= must be at least 1, not {max_length!r}")
|
|
93
|
+
off = set(_listed("without", without))
|
|
94
|
+
built_in = (detector for kind, detector in BUILT_IN.items() if kind not in off)
|
|
95
|
+
self._names = SensitiveNames(_listed("names", names), _listed("allow_names", allow_names))
|
|
96
|
+
# Sensitive names come first, so they win a tie with an equally long Finding.
|
|
97
|
+
self._detectors = (self._names.find_values, *built_in, *detectors)
|
|
98
|
+
self._max_length = max_length
|
|
99
|
+
|
|
100
|
+
@overload
|
|
101
|
+
def __call__(self, value: str) -> str: ...
|
|
102
|
+
|
|
103
|
+
@overload
|
|
104
|
+
def __call__(self, value: object) -> object: ...
|
|
105
|
+
|
|
106
|
+
def __call__(self, value: object) -> object:
|
|
107
|
+
return self._clean(value, set())
|
|
108
|
+
|
|
109
|
+
def find(self, text: str) -> list[Finding]:
|
|
110
|
+
"""Return every Finding this Redactor would replace in `text`, in text order."""
|
|
111
|
+
if not isinstance(text, str):
|
|
112
|
+
raise TypeError(f"find() takes text (str), not {type(text).__name__}")
|
|
113
|
+
limit = self._max_length
|
|
114
|
+
if len(text) <= limit:
|
|
115
|
+
return _find(text, self._detectors)
|
|
116
|
+
return [
|
|
117
|
+
finding
|
|
118
|
+
for finding in _find(text[: limit + LONGEST_VALUE], self._detectors)
|
|
119
|
+
if finding.start < limit
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
def _clean(self, value: object, parents: set[int]) -> object:
|
|
123
|
+
"""Clean one value. `parents` holds the structures this value sits inside."""
|
|
124
|
+
if _holds_no_secret(value):
|
|
125
|
+
return value
|
|
126
|
+
if isinstance(value, (dict, *_SHAPES)):
|
|
127
|
+
return self._clean_structure(value, parents)
|
|
128
|
+
text = _shown_as(value)
|
|
129
|
+
if text is None:
|
|
130
|
+
return f"[UNPRINTABLE {type(value).__name__}]"
|
|
131
|
+
cleaned = self._clean_text(text)
|
|
132
|
+
return value if isinstance(value, Number) and cleaned == text else cleaned
|
|
133
|
+
|
|
134
|
+
def _clean_structure(self, value: Iterable[object], parents: set[int]) -> object:
|
|
135
|
+
if id(value) in parents or len(parents) >= _MAX_DEPTH:
|
|
136
|
+
return _HIDDEN
|
|
137
|
+
parents.add(id(value))
|
|
138
|
+
cleaned: object
|
|
139
|
+
if isinstance(value, dict):
|
|
140
|
+
cleaned = {
|
|
141
|
+
self._clean(key, parents): self._clean_field(key, item, parents)
|
|
142
|
+
for key, item in value.items()
|
|
143
|
+
}
|
|
144
|
+
elif isinstance(value, tuple) and isinstance(
|
|
145
|
+
fields := getattr(value, "_fields", None), tuple
|
|
146
|
+
):
|
|
147
|
+
pairs = zip(fields, value, strict=True)
|
|
148
|
+
items = [self._clean_field(field, item, parents) for field, item in pairs]
|
|
149
|
+
cleaned = tuple.__new__(type(value), items)
|
|
150
|
+
else:
|
|
151
|
+
shape = next(kind for kind in _SHAPES if isinstance(value, kind))
|
|
152
|
+
cleaned = shape([self._clean(item, parents) for item in value])
|
|
153
|
+
parents.remove(id(value))
|
|
154
|
+
return cleaned
|
|
155
|
+
|
|
156
|
+
def _clean_field(self, key: object, item: object, parents: set[int]) -> object:
|
|
157
|
+
name = None if isinstance(key, Number | tuple | frozenset) else _shown_as(key)
|
|
158
|
+
if name is not None and not _holds_no_secret(item) and self._names.is_sensitive(name):
|
|
159
|
+
return _HIDDEN
|
|
160
|
+
return self._clean(item, parents)
|
|
161
|
+
|
|
162
|
+
def _clean_text(self, text: str) -> str:
|
|
163
|
+
"""Replace every found value in text, cutting text over the limit at `max_length`."""
|
|
164
|
+
findings = self.find(text)
|
|
165
|
+
limit = self._max_length
|
|
166
|
+
if len(text) <= limit:
|
|
167
|
+
return _replace(text, findings)
|
|
168
|
+
# A Finding that runs past `limit` leaves `done >= limit`, so the tail after it is empty.
|
|
169
|
+
return _replace(text[:limit], findings) + _TRUNCATED
|
|
File without changes
|