flare-redact 1.10.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flare_redact-1.10.1/.gitignore +22 -0
- flare_redact-1.10.1/LICENSE +21 -0
- flare_redact-1.10.1/PKG-INFO +329 -0
- flare_redact-1.10.1/README.md +296 -0
- flare_redact-1.10.1/pyproject.toml +56 -0
- flare_redact-1.10.1/src/flare_redact/__init__.py +120 -0
- flare_redact-1.10.1/src/flare_redact/__main__.py +10 -0
- flare_redact-1.10.1/src/flare_redact/checksums.py +532 -0
- flare_redact-1.10.1/src/flare_redact/cli.py +277 -0
- flare_redact-1.10.1/src/flare_redact/client.py +252 -0
- flare_redact-1.10.1/src/flare_redact/conformance.py +133 -0
- flare_redact-1.10.1/src/flare_redact/crypto.py +54 -0
- flare_redact-1.10.1/src/flare_redact/detectors.json +2711 -0
- flare_redact-1.10.1/src/flare_redact/engine.py +710 -0
- flare_redact-1.10.1/src/flare_redact/integrations.py +226 -0
- flare_redact-1.10.1/src/flare_redact/keywords.py +83 -0
- flare_redact-1.10.1/src/flare_redact/log.py +117 -0
- flare_redact-1.10.1/src/flare_redact/ml.py +139 -0
- flare_redact-1.10.1/src/flare_redact/pack.py +443 -0
- flare_redact-1.10.1/src/flare_redact/py.typed +0 -0
- flare_redact-1.10.1/src/flare_redact/transforms.py +115 -0
- flare_redact-1.10.1/src/flare_redact/vault.py +351 -0
- flare_redact-1.10.1/src/flare_redact/wire.py +144 -0
- flare_redact-1.10.1/tests/__init__.py +1 -0
- flare_redact-1.10.1/tests/test_checksums.py +45 -0
- flare_redact-1.10.1/tests/test_cli.py +102 -0
- flare_redact-1.10.1/tests/test_client.py +172 -0
- flare_redact-1.10.1/tests/test_conformance.py +56 -0
- flare_redact-1.10.1/tests/test_engine.py +147 -0
- flare_redact-1.10.1/tests/test_json_restore.py +283 -0
- flare_redact-1.10.1/tests/test_log.py +92 -0
- flare_redact-1.10.1/tests/test_pack.py +129 -0
- flare_redact-1.10.1/tests/test_vault.py +106 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
node_modules/
|
|
2
|
+
dist/
|
|
3
|
+
*.tgz
|
|
4
|
+
.DS_Store
|
|
5
|
+
npm-debug.log*
|
|
6
|
+
.env
|
|
7
|
+
.env.*
|
|
8
|
+
|
|
9
|
+
# SDK build output
|
|
10
|
+
sdk/rust/target/
|
|
11
|
+
sdk/python/dist/
|
|
12
|
+
sdk/python/build/
|
|
13
|
+
__pycache__/
|
|
14
|
+
*.py[cod]
|
|
15
|
+
*.egg-info/
|
|
16
|
+
|
|
17
|
+
# Local research and paper artifacts
|
|
18
|
+
research/
|
|
19
|
+
bench/adapters/
|
|
20
|
+
bench/evaluate.mjs
|
|
21
|
+
output/pdf/
|
|
22
|
+
tmp/pdfs/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Umud Hasanli
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: flare-redact
|
|
3
|
+
Version: 1.10.1
|
|
4
|
+
Summary: Zero-dependency secret and PII redaction for Python — logs, prompts, HTTP, and any data structure.
|
|
5
|
+
Project-URL: Homepage, https://flare-collection.github.io/flare-redact/
|
|
6
|
+
Project-URL: Documentation, https://flare-collection.github.io/flare-redact/guides/
|
|
7
|
+
Project-URL: Source, https://github.com/flare-collection/flare-redact
|
|
8
|
+
Project-URL: Issues, https://github.com/flare-collection/flare-redact/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/flare-collection/flare-redact/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Specification, https://github.com/flare-collection/flare-redact/blob/main/spec/SPEC.md
|
|
11
|
+
Author-email: Umud Hasanli <uh.hesenli81015@gmail.com>
|
|
12
|
+
License: MIT
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Keywords: ai-privacy,anonymization,compliance,dlp,gdpr,llm,logging,masking,pci-dss,pii,pii-redaction,prompt,pseudonymization,redact,redaction,secret-detection,secrets,security
|
|
15
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
26
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
27
|
+
Classifier: Topic :: Security
|
|
28
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
29
|
+
Classifier: Topic :: System :: Logging
|
|
30
|
+
Classifier: Typing :: Typed
|
|
31
|
+
Requires-Python: >=3.9
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# flare-redact for Python
|
|
35
|
+
|
|
36
|
+
Hide secrets and PII before they reach a log, a model, or a vendor.
|
|
37
|
+
|
|
38
|
+
Zero dependencies, one file to read, and the same detector spec as the
|
|
39
|
+
JavaScript, Go and Rust engines — so a policy written once behaves identically
|
|
40
|
+
in every service you own. Every claim below is covered by the
|
|
41
|
+
[shared conformance corpus](https://github.com/flare-collection/flare-redact/tree/main/spec/conformance/).
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install flare-redact
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
PyPI releases start with the first release after 1.9.1. Until it lands, or to
|
|
48
|
+
pin a tag, install from the repository:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install "flare-redact @ git+https://github.com/flare-collection/flare-redact@v1.10.1#subdirectory=sdk/python"
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from flare_redact import redact
|
|
56
|
+
|
|
57
|
+
redact("contact ada@example.com with key AKIAIOSFODNN7EXAMPLE")
|
|
58
|
+
# 'contact a***@*** with key AKIA***'
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Contents
|
|
62
|
+
|
|
63
|
+
- [Redact anything](#redact-anything)
|
|
64
|
+
- [Protect your logs in one line](#protect-your-logs-in-one-line)
|
|
65
|
+
- [Send a prompt without sending the data](#send-a-prompt-without-sending-the-data)
|
|
66
|
+
- [Wrap the client you already use](#wrap-the-client-you-already-use)
|
|
67
|
+
- [One policy, reused](#one-policy-reused)
|
|
68
|
+
- [See what leaks, and why](#see-what-leaks-and-why)
|
|
69
|
+
- [Ways to hide a value](#ways-to-hide-a-value)
|
|
70
|
+
- [Your own words and detector packs](#your-own-words-and-detector-packs)
|
|
71
|
+
- [Talk to the gateway instead](#talk-to-the-gateway-instead)
|
|
72
|
+
- [Command line](#command-line)
|
|
73
|
+
- [API](#api)
|
|
74
|
+
- [What this does not do](#what-this-does-not-do)
|
|
75
|
+
|
|
76
|
+
## Redact anything
|
|
77
|
+
|
|
78
|
+
Strings, dicts, lists, tuples, sets, exceptions — anything with strings in it.
|
|
79
|
+
Shape, types and shared references survive; only the strings change.
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from flare_redact import redact
|
|
83
|
+
|
|
84
|
+
redact({
|
|
85
|
+
"user": {"email": "ada@example.com", "id": 42},
|
|
86
|
+
"db": "postgres://app:s3cr3t@db.internal/main",
|
|
87
|
+
"password": "hunter2",
|
|
88
|
+
})
|
|
89
|
+
# {'user': {'email': 'a***@***', 'id': 42},
|
|
90
|
+
# 'db': 'postgres://app:***@db.internal/main',
|
|
91
|
+
# 'password': '***'}
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Numbers stay numbers, `None` stays `None`, and a structure that points at itself
|
|
95
|
+
terminates instead of recursing forever.
|
|
96
|
+
|
|
97
|
+
## Protect your logs in one line
|
|
98
|
+
|
|
99
|
+
```python
|
|
100
|
+
import logging
|
|
101
|
+
import flare_redact.log as flare_log
|
|
102
|
+
|
|
103
|
+
flare_log.install(enable=["pii"])
|
|
104
|
+
|
|
105
|
+
logging.getLogger(__name__).info("charge failed for %s", "ada@example.com")
|
|
106
|
+
# charge failed for a***@***
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`install()` adds a filter to the logger *and* wraps each handler's formatter.
|
|
110
|
+
Both are needed: the filter sees `record.msg` and `record.args` cheaply and keeps
|
|
111
|
+
structured fields structured, while the formatter is the only thing that sees a
|
|
112
|
+
rendered traceback — and a secret passed to the function that raised is *in* that
|
|
113
|
+
traceback.
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
from flare_redact.log import RedactingFilter, RedactingFormatter
|
|
117
|
+
|
|
118
|
+
logger.addFilter(RedactingFilter(enable=["pii"], extra_attributes=["context"]))
|
|
119
|
+
handler.setFormatter(RedactingFormatter(logging.Formatter("%(message)s")))
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## Send a prompt without sending the data
|
|
123
|
+
|
|
124
|
+
A session mints a stable placeholder per value and remembers it, so the model
|
|
125
|
+
sees `[FR_EMAIL_…]` and your user still gets the right answer.
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
from flare_redact import create_session
|
|
129
|
+
|
|
130
|
+
session = create_session(enable=["pii"])
|
|
131
|
+
|
|
132
|
+
safe = session.redact("Email the invoice to ada@example.com and copy grace@example.org")
|
|
133
|
+
reply = call_your_model(safe)
|
|
134
|
+
print(session.restore(reply))
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The same value maps to the same placeholder for the life of the session, so a
|
|
138
|
+
follow-up turn like *"resend it to the first address"* still resolves. Streaming
|
|
139
|
+
works too, including when a placeholder is split across chunks:
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
restorer = session.stream()
|
|
143
|
+
for chunk in model_stream:
|
|
144
|
+
print(restorer.push(chunk), end="")
|
|
145
|
+
print(restorer.flush(), end="")
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
When the streamed text is JSON, such as tool-call arguments, use
|
|
149
|
+
`session.stream(json=True)`: the originals go in JSON-escaped, so the arguments
|
|
150
|
+
stay valid JSON with the same structure. `restore()` does this on its own for any
|
|
151
|
+
string that is a JSON object or array.
|
|
152
|
+
|
|
153
|
+
## Wrap the client you already use
|
|
154
|
+
|
|
155
|
+
```python
|
|
156
|
+
from openai import OpenAI
|
|
157
|
+
from flare_redact.integrations import wrap_openai
|
|
158
|
+
|
|
159
|
+
client = wrap_openai(OpenAI(), enable=["pii"])
|
|
160
|
+
|
|
161
|
+
client.chat.completions.create(
|
|
162
|
+
model="gpt-4o",
|
|
163
|
+
messages=[{"role": "user", "content": "Draft a reply to ada@example.com"}],
|
|
164
|
+
)
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Prompts are redacted on the way out and replies restored on the way back. The
|
|
168
|
+
wrapper is duck-typed and never imports the vendor SDK, so it keeps working
|
|
169
|
+
across client versions and adds nothing to your dependency tree. Wrapping the
|
|
170
|
+
same client twice is a no-op. `wrap_anthropic` works the same way. Tool-call
|
|
171
|
+
arguments, partial tool input and JSON-mode output are restored JSON-escaped,
|
|
172
|
+
streamed or not, so they still parse.
|
|
173
|
+
|
|
174
|
+
## One policy, reused
|
|
175
|
+
|
|
176
|
+
Compiling once resolves the detector list, the mask function and the key tests
|
|
177
|
+
a single time — and, more importantly, guarantees your logs, your HTTP layer and
|
|
178
|
+
your prompts all agree on what "sensitive" means.
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
from flare_redact import compile_policy
|
|
182
|
+
|
|
183
|
+
policy = compile_policy(enable=["pii", "tr"], disable=["ipv4"], min_confidence=0.7)
|
|
184
|
+
|
|
185
|
+
policy.redact(payload)
|
|
186
|
+
policy.redact_str(line)
|
|
187
|
+
policy.scan(payload)
|
|
188
|
+
policy.summary(payload)
|
|
189
|
+
policy.vault()
|
|
190
|
+
policy.session()
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
## See what leaks, and why
|
|
194
|
+
|
|
195
|
+
```python
|
|
196
|
+
from flare_redact import scan
|
|
197
|
+
|
|
198
|
+
for finding in scan({"note": "card 4111 1111 1111 1111"}):
|
|
199
|
+
print(finding.detector, finding.risk, finding.path, finding.line, finding.why)
|
|
200
|
+
# credit_card high note 1 A card number in logs is a PCI-DSS violation.
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
Findings never carry the matched value unless you ask for it with
|
|
204
|
+
`include_values=True`, because findings are the thing people paste into tickets.
|
|
205
|
+
|
|
206
|
+
## Ways to hide a value
|
|
207
|
+
|
|
208
|
+
| `mode` | `ada@example.com` becomes | Use it for |
|
|
209
|
+
|---|---|---|
|
|
210
|
+
| `mask` (default) | `a***@***` | logs, anything human-read |
|
|
211
|
+
| `label` | `[REDACTED:email]` | when you want to see *what* was removed |
|
|
212
|
+
| `hash` | `email_12016e848bcb…` | correlating without storing |
|
|
213
|
+
| `pseudonym` | `nlp@kmxjgxr.hjh` | keeping shape for parsers |
|
|
214
|
+
| `surrogate` | `user_7560a7763fe8@example.invalid` | test data that still validates |
|
|
215
|
+
|
|
216
|
+
`hash`, `pseudonym` and `surrogate` are keyed and deterministic — pass
|
|
217
|
+
`transform_secret` (from your secret manager, never a literal) and the same input
|
|
218
|
+
always maps to the same output, so joins survive anonymisation. A missing secret
|
|
219
|
+
raises rather than silently producing unkeyed output.
|
|
220
|
+
|
|
221
|
+
Surrogates are type-aware: a card number becomes another card number with a
|
|
222
|
+
valid Luhn check digit, so downstream validation still passes.
|
|
223
|
+
|
|
224
|
+
## Your own words and detector packs
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
redact("project Bluebird ships in Q3", terms={"Bluebird": "[PROJECT]"})
|
|
228
|
+
# 'project [PROJECT] ships in Q3'
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Terms match longest-first on Unicode word boundaries, so `Bluebirds` is left
|
|
232
|
+
alone. For a whole detector set, write an [FRS-1 pack](https://github.com/flare-collection/flare-redact/blob/main/spec/SPEC.md) and
|
|
233
|
+
every language loads the same file:
|
|
234
|
+
|
|
235
|
+
```python
|
|
236
|
+
from flare_redact import load_pack, redact
|
|
237
|
+
|
|
238
|
+
pack = load_pack("detectors/acme.json")
|
|
239
|
+
redact(text, pack=pack)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
A pack that uses a construct this engine cannot execute exactly — a lookahead, a
|
|
243
|
+
`\d` whose meaning differs between languages, an unknown checksum — fails to
|
|
244
|
+
load. Failing open is how a redactor leaks.
|
|
245
|
+
|
|
246
|
+
## Talk to the gateway instead
|
|
247
|
+
|
|
248
|
+
The local engine implements the portable FRS-1 profile. The
|
|
249
|
+
[gateway](https://github.com/flare-collection/flare-redact/blob/main/docker/README.md) runs the full detector set and holds
|
|
250
|
+
server-side sessions; point a client at your sidecar when you want the policy
|
|
251
|
+
configured in one place for every language in the estate.
|
|
252
|
+
|
|
253
|
+
```python
|
|
254
|
+
from flare_redact.client import GatewayClient
|
|
255
|
+
|
|
256
|
+
gateway = GatewayClient("http://127.0.0.1:8080", token=os.environ["FLARE_GATEWAY_TOKEN"])
|
|
257
|
+
|
|
258
|
+
gateway.redact({"email": "ada@example.com"})
|
|
259
|
+
|
|
260
|
+
with gateway.session(enable=["pii"]) as session:
|
|
261
|
+
safe = session.redact(user_message)
|
|
262
|
+
reply = session.restore(model_reply)
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
For an agent, the gateway can enforce its operator-owned FlareFlow policy. The
|
|
266
|
+
client names the trusted source and runtime-resolved sink; it cannot submit or
|
|
267
|
+
widen the capabilities:
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
with gateway.agent_flow() as flow:
|
|
271
|
+
safe = flow.protect_for_model("postgres.result", tool_result)
|
|
272
|
+
decision = flow.inspect_for_sink("postgres.execute", model_call)
|
|
273
|
+
local_call = flow.authorize_for_sink("postgres.execute", model_call)
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
A denial raises `GatewayError` with status `403`, code
|
|
277
|
+
`ERR_AGENT_FLOW_BLOCKED`, and a value-free decision in `error.details`.
|
|
278
|
+
|
|
279
|
+
Standard library only — no `requests`, no `httpx`. `transform_secret` is never
|
|
280
|
+
serialised: configure keyed transforms on the gateway, not in every caller.
|
|
281
|
+
|
|
282
|
+
## Command line
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
tail -f app.log | flare-redact
|
|
286
|
+
flare-redact --scan --format json config.env
|
|
287
|
+
flare-redact --sarif . > flare-redact.sarif
|
|
288
|
+
flare-redact --json --mode label < event.json
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
Exit codes match the JavaScript CLI — `0` clean, `1` findings, `2` error — so the
|
|
292
|
+
two are interchangeable in a pipeline or a pre-commit hook. `python -m flare_redact`
|
|
293
|
+
works without installing a script.
|
|
294
|
+
|
|
295
|
+
## API
|
|
296
|
+
|
|
297
|
+
| | |
|
|
298
|
+
|---|---|
|
|
299
|
+
| `redact(value, **options)` | redact every string reachable from `value` |
|
|
300
|
+
| `scan(value, **options)` | list `Finding`s with paths and line/column |
|
|
301
|
+
| `is_clean(value, **options)` | `True` when nothing would be redacted |
|
|
302
|
+
| `summary(value, **options)` | counts by detector and by risk |
|
|
303
|
+
| `compile_policy(**options)` | a reusable `Policy` |
|
|
304
|
+
| `create_vault(**options)` | reversible redaction |
|
|
305
|
+
| `create_session(**options)` | a conversation-scoped vault |
|
|
306
|
+
| `restore(value, source)` | put originals back from a vault or mapping |
|
|
307
|
+
| `load_pack(source)` / `core_pack()` | FRS-1 detector packs |
|
|
308
|
+
|
|
309
|
+
Options: `only`, `enable`, `disable`, `mode`, `mask`, `transform_secret`,
|
|
310
|
+
`redact_keys`, `allow`, `terms`, `terms_case_sensitive`, `min_confidence`,
|
|
311
|
+
`refine_confidence`, `include_values`, `max_input_length`, `max_findings`, `pack`.
|
|
312
|
+
|
|
313
|
+
Requires Python 3.9+. Ships type hints (`py.typed`).
|
|
314
|
+
|
|
315
|
+
## What this does not do
|
|
316
|
+
|
|
317
|
+
- **It is not a replacement for not collecting the data.** Redaction is a
|
|
318
|
+
backstop for the places data ends up anyway: logs, prompts, crash reports.
|
|
319
|
+
- **`pseudonym` is not encryption.** It is a keyed, shape-preserving pseudonym.
|
|
320
|
+
It is not reversible and does not implement NIST FF1.
|
|
321
|
+
- **Detection is not proof.** A detector that has never seen your internal token
|
|
322
|
+
format will not find it — add it with `terms`, a custom pack, or by opening an
|
|
323
|
+
issue.
|
|
324
|
+
- **Limits fail closed.** Input over `max_input_length` raises instead of
|
|
325
|
+
returning partially redacted text, because a caller cannot tell partially
|
|
326
|
+
redacted output from clean output.
|
|
327
|
+
|
|
328
|
+
MIT © Umud Hasanli · [Specification](https://github.com/flare-collection/flare-redact/blob/main/spec/SPEC.md) ·
|
|
329
|
+
[Source](https://github.com/flare-collection/flare-redact)
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
# flare-redact for Python
|
|
2
|
+
|
|
3
|
+
Hide secrets and PII before they reach a log, a model, or a vendor.
|
|
4
|
+
|
|
5
|
+
Zero dependencies, one file to read, and the same detector spec as the
|
|
6
|
+
JavaScript, Go and Rust engines — so a policy written once behaves identically
|
|
7
|
+
in every service you own. Every claim below is covered by the
|
|
8
|
+
[shared conformance corpus](https://github.com/flare-collection/flare-redact/tree/main/spec/conformance/).
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
pip install flare-redact
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
PyPI releases start with the first release after 1.9.1. Until it lands, or to
|
|
15
|
+
pin a tag, install from the repository:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pip install "flare-redact @ git+https://github.com/flare-collection/flare-redact@v1.10.1#subdirectory=sdk/python"
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
from flare_redact import redact
|
|
23
|
+
|
|
24
|
+
redact("contact ada@example.com with key AKIAIOSFODNN7EXAMPLE")
|
|
25
|
+
# 'contact a***@*** with key AKIA***'
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Contents
|
|
29
|
+
|
|
30
|
+
- [Redact anything](#redact-anything)
|
|
31
|
+
- [Protect your logs in one line](#protect-your-logs-in-one-line)
|
|
32
|
+
- [Send a prompt without sending the data](#send-a-prompt-without-sending-the-data)
|
|
33
|
+
- [Wrap the client you already use](#wrap-the-client-you-already-use)
|
|
34
|
+
- [One policy, reused](#one-policy-reused)
|
|
35
|
+
- [See what leaks, and why](#see-what-leaks-and-why)
|
|
36
|
+
- [Ways to hide a value](#ways-to-hide-a-value)
|
|
37
|
+
- [Your own words and detector packs](#your-own-words-and-detector-packs)
|
|
38
|
+
- [Talk to the gateway instead](#talk-to-the-gateway-instead)
|
|
39
|
+
- [Command line](#command-line)
|
|
40
|
+
- [API](#api)
|
|
41
|
+
- [What this does not do](#what-this-does-not-do)
|
|
42
|
+
|
|
43
|
+
## Redact anything
|
|
44
|
+
|
|
45
|
+
Strings, dicts, lists, tuples, sets, exceptions — anything with strings in it.
|
|
46
|
+
Shape, types and shared references survive; only the strings change.
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from flare_redact import redact
|
|
50
|
+
|
|
51
|
+
redact({
|
|
52
|
+
"user": {"email": "ada@example.com", "id": 42},
|
|
53
|
+
"db": "postgres://app:s3cr3t@db.internal/main",
|
|
54
|
+
"password": "hunter2",
|
|
55
|
+
})
|
|
56
|
+
# {'user': {'email': 'a***@***', 'id': 42},
|
|
57
|
+
# 'db': 'postgres://app:***@db.internal/main',
|
|
58
|
+
# 'password': '***'}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Numbers stay numbers, `None` stays `None`, and a structure that points at itself
|
|
62
|
+
terminates instead of recursing forever.
|
|
63
|
+
|
|
64
|
+
## Protect your logs in one line
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
import logging
|
|
68
|
+
import flare_redact.log as flare_log
|
|
69
|
+
|
|
70
|
+
flare_log.install(enable=["pii"])
|
|
71
|
+
|
|
72
|
+
logging.getLogger(__name__).info("charge failed for %s", "ada@example.com")
|
|
73
|
+
# charge failed for a***@***
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
`install()` adds a filter to the logger *and* wraps each handler's formatter.
|
|
77
|
+
Both are needed: the filter sees `record.msg` and `record.args` cheaply and keeps
|
|
78
|
+
structured fields structured, while the formatter is the only thing that sees a
|
|
79
|
+
rendered traceback — and a secret passed to the function that raised is *in* that
|
|
80
|
+
traceback.
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from flare_redact.log import RedactingFilter, RedactingFormatter
|
|
84
|
+
|
|
85
|
+
logger.addFilter(RedactingFilter(enable=["pii"], extra_attributes=["context"]))
|
|
86
|
+
handler.setFormatter(RedactingFormatter(logging.Formatter("%(message)s")))
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Send a prompt without sending the data
|
|
90
|
+
|
|
91
|
+
A session mints a stable placeholder per value and remembers it, so the model
|
|
92
|
+
sees `[FR_EMAIL_…]` and your user still gets the right answer.
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from flare_redact import create_session
|
|
96
|
+
|
|
97
|
+
session = create_session(enable=["pii"])
|
|
98
|
+
|
|
99
|
+
safe = session.redact("Email the invoice to ada@example.com and copy grace@example.org")
|
|
100
|
+
reply = call_your_model(safe)
|
|
101
|
+
print(session.restore(reply))
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
The same value maps to the same placeholder for the life of the session, so a
|
|
105
|
+
follow-up turn like *"resend it to the first address"* still resolves. Streaming
|
|
106
|
+
works too, including when a placeholder is split across chunks:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
restorer = session.stream()
|
|
110
|
+
for chunk in model_stream:
|
|
111
|
+
print(restorer.push(chunk), end="")
|
|
112
|
+
print(restorer.flush(), end="")
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
When the streamed text is JSON, such as tool-call arguments, use
|
|
116
|
+
`session.stream(json=True)`: the originals go in JSON-escaped, so the arguments
|
|
117
|
+
stay valid JSON with the same structure. `restore()` does this on its own for any
|
|
118
|
+
string that is a JSON object or array.
|
|
119
|
+
|
|
120
|
+
## Wrap the client you already use
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
from openai import OpenAI
|
|
124
|
+
from flare_redact.integrations import wrap_openai
|
|
125
|
+
|
|
126
|
+
client = wrap_openai(OpenAI(), enable=["pii"])
|
|
127
|
+
|
|
128
|
+
client.chat.completions.create(
|
|
129
|
+
model="gpt-4o",
|
|
130
|
+
messages=[{"role": "user", "content": "Draft a reply to ada@example.com"}],
|
|
131
|
+
)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Prompts are redacted on the way out and replies restored on the way back. The
|
|
135
|
+
wrapper is duck-typed and never imports the vendor SDK, so it keeps working
|
|
136
|
+
across client versions and adds nothing to your dependency tree. Wrapping the
|
|
137
|
+
same client twice is a no-op. `wrap_anthropic` works the same way. Tool-call
|
|
138
|
+
arguments, partial tool input and JSON-mode output are restored JSON-escaped,
|
|
139
|
+
streamed or not, so they still parse.
|
|
140
|
+
|
|
141
|
+
## One policy, reused
|
|
142
|
+
|
|
143
|
+
Compiling once resolves the detector list, the mask function and the key tests
|
|
144
|
+
a single time — and, more importantly, guarantees your logs, your HTTP layer and
|
|
145
|
+
your prompts all agree on what "sensitive" means.
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from flare_redact import compile_policy
|
|
149
|
+
|
|
150
|
+
policy = compile_policy(enable=["pii", "tr"], disable=["ipv4"], min_confidence=0.7)
|
|
151
|
+
|
|
152
|
+
policy.redact(payload)
|
|
153
|
+
policy.redact_str(line)
|
|
154
|
+
policy.scan(payload)
|
|
155
|
+
policy.summary(payload)
|
|
156
|
+
policy.vault()
|
|
157
|
+
policy.session()
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
## See what leaks, and why
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from flare_redact import scan
|
|
164
|
+
|
|
165
|
+
for finding in scan({"note": "card 4111 1111 1111 1111"}):
|
|
166
|
+
print(finding.detector, finding.risk, finding.path, finding.line, finding.why)
|
|
167
|
+
# credit_card high note 1 A card number in logs is a PCI-DSS violation.
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Findings never carry the matched value unless you ask for it with
|
|
171
|
+
`include_values=True`, because findings are the thing people paste into tickets.
|
|
172
|
+
|
|
173
|
+
## Ways to hide a value
|
|
174
|
+
|
|
175
|
+
| `mode` | `ada@example.com` becomes | Use it for |
|
|
176
|
+
|---|---|---|
|
|
177
|
+
| `mask` (default) | `a***@***` | logs, anything human-read |
|
|
178
|
+
| `label` | `[REDACTED:email]` | when you want to see *what* was removed |
|
|
179
|
+
| `hash` | `email_12016e848bcb…` | correlating without storing |
|
|
180
|
+
| `pseudonym` | `nlp@kmxjgxr.hjh` | keeping shape for parsers |
|
|
181
|
+
| `surrogate` | `user_7560a7763fe8@example.invalid` | test data that still validates |
|
|
182
|
+
|
|
183
|
+
`hash`, `pseudonym` and `surrogate` are keyed and deterministic — pass
|
|
184
|
+
`transform_secret` (from your secret manager, never a literal) and the same input
|
|
185
|
+
always maps to the same output, so joins survive anonymisation. A missing secret
|
|
186
|
+
raises rather than silently producing unkeyed output.
|
|
187
|
+
|
|
188
|
+
Surrogates are type-aware: a card number becomes another card number with a
|
|
189
|
+
valid Luhn check digit, so downstream validation still passes.
|
|
190
|
+
|
|
191
|
+
## Your own words and detector packs
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
redact("project Bluebird ships in Q3", terms={"Bluebird": "[PROJECT]"})
|
|
195
|
+
# 'project [PROJECT] ships in Q3'
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
Terms match longest-first on Unicode word boundaries, so `Bluebirds` is left
|
|
199
|
+
alone. For a whole detector set, write an [FRS-1 pack](https://github.com/flare-collection/flare-redact/blob/main/spec/SPEC.md) and
|
|
200
|
+
every language loads the same file:
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
from flare_redact import load_pack, redact
|
|
204
|
+
|
|
205
|
+
pack = load_pack("detectors/acme.json")
|
|
206
|
+
redact(text, pack=pack)
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
A pack that uses a construct this engine cannot execute exactly — a lookahead, a
|
|
210
|
+
`\d` whose meaning differs between languages, an unknown checksum — fails to
|
|
211
|
+
load. Failing open is how a redactor leaks.
|
|
212
|
+
|
|
213
|
+
## Talk to the gateway instead
|
|
214
|
+
|
|
215
|
+
The local engine implements the portable FRS-1 profile. The
|
|
216
|
+
[gateway](https://github.com/flare-collection/flare-redact/blob/main/docker/README.md) runs the full detector set and holds
|
|
217
|
+
server-side sessions; point a client at your sidecar when you want the policy
|
|
218
|
+
configured in one place for every language in the estate.
|
|
219
|
+
|
|
220
|
+
```python
|
|
221
|
+
from flare_redact.client import GatewayClient
|
|
222
|
+
|
|
223
|
+
gateway = GatewayClient("http://127.0.0.1:8080", token=os.environ["FLARE_GATEWAY_TOKEN"])
|
|
224
|
+
|
|
225
|
+
gateway.redact({"email": "ada@example.com"})
|
|
226
|
+
|
|
227
|
+
with gateway.session(enable=["pii"]) as session:
|
|
228
|
+
safe = session.redact(user_message)
|
|
229
|
+
reply = session.restore(model_reply)
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
For an agent, the gateway can enforce its operator-owned FlareFlow policy. The
|
|
233
|
+
client names the trusted source and runtime-resolved sink; it cannot submit or
|
|
234
|
+
widen the capabilities:
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
with gateway.agent_flow() as flow:
|
|
238
|
+
safe = flow.protect_for_model("postgres.result", tool_result)
|
|
239
|
+
decision = flow.inspect_for_sink("postgres.execute", model_call)
|
|
240
|
+
local_call = flow.authorize_for_sink("postgres.execute", model_call)
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
A denial raises `GatewayError` with status `403`, code
|
|
244
|
+
`ERR_AGENT_FLOW_BLOCKED`, and a value-free decision in `error.details`.
|
|
245
|
+
|
|
246
|
+
Standard library only — no `requests`, no `httpx`. `transform_secret` is never
|
|
247
|
+
serialised: configure keyed transforms on the gateway, not in every caller.
|
|
248
|
+
|
|
249
|
+
## Command line
|
|
250
|
+
|
|
251
|
+
```bash
|
|
252
|
+
tail -f app.log | flare-redact
|
|
253
|
+
flare-redact --scan --format json config.env
|
|
254
|
+
flare-redact --sarif . > flare-redact.sarif
|
|
255
|
+
flare-redact --json --mode label < event.json
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
Exit codes match the JavaScript CLI — `0` clean, `1` findings, `2` error — so the
|
|
259
|
+
two are interchangeable in a pipeline or a pre-commit hook. `python -m flare_redact`
|
|
260
|
+
works without installing a script.
|
|
261
|
+
|
|
262
|
+
## API
|
|
263
|
+
|
|
264
|
+
| | |
|
|
265
|
+
|---|---|
|
|
266
|
+
| `redact(value, **options)` | redact every string reachable from `value` |
|
|
267
|
+
| `scan(value, **options)` | list `Finding`s with paths and line/column |
|
|
268
|
+
| `is_clean(value, **options)` | `True` when nothing would be redacted |
|
|
269
|
+
| `summary(value, **options)` | counts by detector and by risk |
|
|
270
|
+
| `compile_policy(**options)` | a reusable `Policy` |
|
|
271
|
+
| `create_vault(**options)` | reversible redaction |
|
|
272
|
+
| `create_session(**options)` | a conversation-scoped vault |
|
|
273
|
+
| `restore(value, source)` | put originals back from a vault or mapping |
|
|
274
|
+
| `load_pack(source)` / `core_pack()` | FRS-1 detector packs |
|
|
275
|
+
|
|
276
|
+
Options: `only`, `enable`, `disable`, `mode`, `mask`, `transform_secret`,
|
|
277
|
+
`redact_keys`, `allow`, `terms`, `terms_case_sensitive`, `min_confidence`,
|
|
278
|
+
`refine_confidence`, `include_values`, `max_input_length`, `max_findings`, `pack`.
|
|
279
|
+
|
|
280
|
+
Requires Python 3.9+. Ships type hints (`py.typed`).
|
|
281
|
+
|
|
282
|
+
## What this does not do
|
|
283
|
+
|
|
284
|
+
- **It is not a replacement for not collecting the data.** Redaction is a
|
|
285
|
+
backstop for the places data ends up anyway: logs, prompts, crash reports.
|
|
286
|
+
- **`pseudonym` is not encryption.** It is a keyed, shape-preserving pseudonym.
|
|
287
|
+
It is not reversible and does not implement NIST FF1.
|
|
288
|
+
- **Detection is not proof.** A detector that has never seen your internal token
|
|
289
|
+
format will not find it — add it with `terms`, a custom pack, or by opening an
|
|
290
|
+
issue.
|
|
291
|
+
- **Limits fail closed.** Input over `max_input_length` raises instead of
|
|
292
|
+
returning partially redacted text, because a caller cannot tell partially
|
|
293
|
+
redacted output from clean output.
|
|
294
|
+
|
|
295
|
+
MIT © Umud Hasanli · [Specification](https://github.com/flare-collection/flare-redact/blob/main/spec/SPEC.md) ·
|
|
296
|
+
[Source](https://github.com/flare-collection/flare-redact)
|