agent-redact 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_redact-0.2.0/.gitignore +51 -0
- agent_redact-0.2.0/PKG-INFO +55 -0
- agent_redact-0.2.0/README.md +39 -0
- agent_redact-0.2.0/pyproject.toml +31 -0
- agent_redact-0.2.0/src/agent_redact/__init__.py +363 -0
- agent_redact-0.2.0/src/agent_redact/py.typed +0 -0
- agent_redact-0.2.0/tests/test_agent_redact.py +114 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Dependencies
|
|
2
|
+
node_modules/
|
|
3
|
+
.pnp
|
|
4
|
+
.pnp.js
|
|
5
|
+
|
|
6
|
+
# Build outputs
|
|
7
|
+
build/
|
|
8
|
+
*.tsbuildinfo
|
|
9
|
+
|
|
10
|
+
# Coverage & Tests
|
|
11
|
+
coverage/
|
|
12
|
+
.nyc_output/
|
|
13
|
+
|
|
14
|
+
# Logs
|
|
15
|
+
npm-debug.log*
|
|
16
|
+
yarn-debug.log*
|
|
17
|
+
yarn-error.log*
|
|
18
|
+
pnpm-debug.log*
|
|
19
|
+
*.log
|
|
20
|
+
|
|
21
|
+
# Environment & Secrets (ironic if leaked here!)
|
|
22
|
+
.env
|
|
23
|
+
.env.*
|
|
24
|
+
!.env.example
|
|
25
|
+
*.pem
|
|
26
|
+
*.key
|
|
27
|
+
|
|
28
|
+
# IDE & OS
|
|
29
|
+
.vscode/
|
|
30
|
+
.idea/
|
|
31
|
+
*.suo
|
|
32
|
+
*.ntvs*
|
|
33
|
+
*.njsproj
|
|
34
|
+
*.sln
|
|
35
|
+
*.sw?
|
|
36
|
+
.DS_Store
|
|
37
|
+
Thumbs.db
|
|
38
|
+
|
|
39
|
+
# Cache
|
|
40
|
+
.eslintcache
|
|
41
|
+
.vitest/
|
|
42
|
+
.turbo/
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# Python
|
|
46
|
+
__pycache__/
|
|
47
|
+
*.py[cod]
|
|
48
|
+
.venv/
|
|
49
|
+
.pytest_cache/
|
|
50
|
+
python/dist/
|
|
51
|
+
*.egg-info/
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: agent-redact
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Secret & PII redaction for Python AI agents (wrapper around the agent-redact CLI)
|
|
5
|
+
Project-URL: Homepage, https://github.com/TonyDuongg/agent-redact
|
|
6
|
+
Project-URL: Issues, https://github.com/TonyDuongg/agent-redact/issues
|
|
7
|
+
Author: Tony Duong
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Keywords: ai-agents,llm,pii,redaction,secrets,security
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Security
|
|
13
|
+
Classifier: Typing :: Typed
|
|
14
|
+
Requires-Python: >=3.10
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# agent-redact (Python)
|
|
18
|
+
|
|
19
|
+
Python bindings for [agent-redact](https://github.com/TonyDuongg/agent-redact). This is a thin, dependency-free wrapper around the Node.js CLI, so Python agents get exactly the same detection engine as the npm package.
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pip install agent-redact
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
> **Node.js 20+ is still required.** This package contains only the Python wrapper; the detection engine is the Node.js CLI (see below for how it is found).
|
|
26
|
+
|
|
27
|
+
**Requirements:**
|
|
28
|
+
- Python 3.10+
|
|
29
|
+
- Node.js 20+
|
|
30
|
+
- the CLI, from one of these (checked in order):
|
|
31
|
+
1. `AGENT_REDACT_CLI` set to the **absolute** path of `dist/cli/index.js`;
|
|
32
|
+
2. a global `npm install -g agent-redact`;
|
|
33
|
+
3. npm next to `node`: the matching version is installed once into a private user cache (`--ignore-scripts`) and reused.
|
|
34
|
+
|
|
35
|
+
The CLI is always started as `node <absolute path>`. It is never run through `.cmd`/`.bat` shims, through `npx`, or from a binary found in the current directory, so scanning an untrusted repository cannot hijack it.
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from agent_redact import redact, scan, Session
|
|
39
|
+
|
|
40
|
+
# One-way redaction
|
|
41
|
+
result = redact("deploy with AKIA...") # result.text, result.matches (no secret values)
|
|
42
|
+
redact(text, presets=["vn-pii"]) # opt-in presets
|
|
43
|
+
|
|
44
|
+
# Reversible masking across a whole conversation
|
|
45
|
+
with Session() as s:
|
|
46
|
+
safe_prompt = s.redact(user_prompt) # [AWS_ACCESS_KEY_ID_1] …
|
|
47
|
+
answer = llm(safe_prompt)
|
|
48
|
+
print(s.unmask(answer)) # placeholders restored locally
|
|
49
|
+
|
|
50
|
+
# Repository / staged-changes audit
|
|
51
|
+
findings = scan(["."]) # honours .gitignore in git repos
|
|
52
|
+
findings = scan(staged=True) # what is about to be committed
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`Session` keeps its vault in a private temporary directory, **encrypted** with a random per-session passphrase that is never written to disk, and deletes it on `close()` or when the `with` block ends.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# agent-redact (Python)
|
|
2
|
+
|
|
3
|
+
Python bindings for [agent-redact](https://github.com/TonyDuongg/agent-redact). This is a thin, dependency-free wrapper around the Node.js CLI, so Python agents get exactly the same detection engine as the npm package.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
pip install agent-redact
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
> **Node.js 20+ is still required.** This package contains only the Python wrapper; the detection engine is the Node.js CLI (see below for how it is found).
|
|
10
|
+
|
|
11
|
+
**Requirements:**
|
|
12
|
+
- Python 3.10+
|
|
13
|
+
- Node.js 20+
|
|
14
|
+
- the CLI, from one of these (checked in order):
|
|
15
|
+
1. `AGENT_REDACT_CLI` set to the **absolute** path of `dist/cli/index.js`;
|
|
16
|
+
2. a global `npm install -g agent-redact`;
|
|
17
|
+
3. npm next to `node`: the matching version is installed once into a private user cache (`--ignore-scripts`) and reused.
|
|
18
|
+
|
|
19
|
+
The CLI is always started as `node <absolute path>`. It is never run through `.cmd`/`.bat` shims, through `npx`, or from a binary found in the current directory, so scanning an untrusted repository cannot hijack it.
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
from agent_redact import redact, scan, Session
|
|
23
|
+
|
|
24
|
+
# One-way redaction
|
|
25
|
+
result = redact("deploy with AKIA...") # result.text, result.matches (no secret values)
|
|
26
|
+
redact(text, presets=["vn-pii"]) # opt-in presets
|
|
27
|
+
|
|
28
|
+
# Reversible masking across a whole conversation
|
|
29
|
+
with Session() as s:
|
|
30
|
+
safe_prompt = s.redact(user_prompt) # [AWS_ACCESS_KEY_ID_1] …
|
|
31
|
+
answer = llm(safe_prompt)
|
|
32
|
+
print(s.unmask(answer)) # placeholders restored locally
|
|
33
|
+
|
|
34
|
+
# Repository / staged-changes audit
|
|
35
|
+
findings = scan(["."]) # honours .gitignore in git repos
|
|
36
|
+
findings = scan(staged=True) # what is about to be committed
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
`Session` keeps its vault in a private temporary directory, **encrypted** with a random per-session passphrase that is never written to disk, and deletes it on `close()` or when the `with` block ends.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "agent-redact"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Secret & PII redaction for Python AI agents (wrapper around the agent-redact CLI)"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [{ name = "Tony Duong" }]
|
|
13
|
+
keywords = ["security", "redaction", "secrets", "pii", "llm", "ai-agents"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
"Topic :: Security",
|
|
18
|
+
"Typing :: Typed",
|
|
19
|
+
]
|
|
20
|
+
dependencies = []
|
|
21
|
+
|
|
22
|
+
[project.urls]
|
|
23
|
+
Homepage = "https://github.com/TonyDuongg/agent-redact"
|
|
24
|
+
Issues = "https://github.com/TonyDuongg/agent-redact/issues"
|
|
25
|
+
|
|
26
|
+
[tool.hatch.build.targets.wheel]
|
|
27
|
+
packages = ["src/agent_redact"]
|
|
28
|
+
|
|
29
|
+
[tool.pytest.ini_options]
|
|
30
|
+
pythonpath = ["src"]
|
|
31
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
"""Python bindings for agent-redact.
|
|
2
|
+
|
|
3
|
+
This package is a thin, dependency-free wrapper around the agent-redact Node.js CLI, so
|
|
4
|
+
Python agents (LangChain, CrewAI, AutoGen, plain SDK code) get exactly the same detection
|
|
5
|
+
engine as the npm package. It needs Node.js 20+ and one of:
|
|
6
|
+
|
|
7
|
+
* ``AGENT_REDACT_CLI`` set to the absolute path of ``dist/cli/index.js`` (or an executable),
|
|
8
|
+
* a global ``npm install -g agent-redact``, or
|
|
9
|
+
* npm next to ``node``: the matching version is installed once into a private user cache
|
|
10
|
+
(with ``--ignore-scripts``) and reused.
|
|
11
|
+
|
|
12
|
+
The CLI is always started as ``node <absolute path>``: never through ``.cmd``/``.bat`` shims,
|
|
13
|
+
``npx``, or a binary found in the current directory.
|
|
14
|
+
|
|
15
|
+
Secret values are never written to disk in plaintext: :class:`Session` keeps its vault
|
|
16
|
+
encrypted with a random per-session passphrase that only lives in this process.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import os
|
|
23
|
+
import secrets
|
|
24
|
+
import shutil
|
|
25
|
+
import subprocess
|
|
26
|
+
import tempfile
|
|
27
|
+
from dataclasses import dataclass, field
|
|
28
|
+
from typing import Iterable, Mapping, Sequence
|
|
29
|
+
|
|
30
|
+
__version__ = "0.2.0"
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"AgentRedactError",
|
|
34
|
+
"Finding",
|
|
35
|
+
"Match",
|
|
36
|
+
"RedactResult",
|
|
37
|
+
"Session",
|
|
38
|
+
"redact",
|
|
39
|
+
"scan",
|
|
40
|
+
"__version__",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class AgentRedactError(RuntimeError):
|
|
45
|
+
"""Raised when the CLI cannot be found or exits with an error."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class Match:
|
|
50
|
+
"""One redacted span. The secret value itself is never included."""
|
|
51
|
+
|
|
52
|
+
pattern_id: str
|
|
53
|
+
pattern_name: str
|
|
54
|
+
category: str
|
|
55
|
+
severity: str
|
|
56
|
+
start: int
|
|
57
|
+
end: int
|
|
58
|
+
replacement: str
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class RedactResult:
|
|
63
|
+
text: str
|
|
64
|
+
matches: list[Match] = field(default_factory=list)
|
|
65
|
+
stats: dict[str, int] = field(default_factory=dict)
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def total_leaks(self) -> int:
|
|
69
|
+
return len(self.matches)
|
|
70
|
+
|
|
71
|
+
@property
|
|
72
|
+
def has_leaks(self) -> bool:
|
|
73
|
+
return bool(self.matches)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class Finding:
|
|
78
|
+
"""A finding from :func:`scan`, without the secret value."""
|
|
79
|
+
|
|
80
|
+
file: str
|
|
81
|
+
line: int
|
|
82
|
+
column: int
|
|
83
|
+
pattern_id: str
|
|
84
|
+
pattern_name: str
|
|
85
|
+
severity: str
|
|
86
|
+
category: str
|
|
87
|
+
fingerprint: str
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
_NOT_FOUND = (
|
|
91
|
+
"agent-redact CLI not found. Install Node.js 20+ and run `npm install -g agent-redact`, "
|
|
92
|
+
"or set AGENT_REDACT_CLI to the absolute path of dist/cli/index.js."
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _safe_which(name: str) -> str | None:
|
|
97
|
+
"""Like shutil.which, but only searches absolute PATH entries and never the current
|
|
98
|
+
directory (Windows searches cwd first by default; a scanned repo could plant a binary)."""
|
|
99
|
+
cwd = os.path.realpath(os.getcwd())
|
|
100
|
+
exts = [""]
|
|
101
|
+
if os.name == "nt":
|
|
102
|
+
exts = [e.lower() for e in os.environ.get("PATHEXT", ".EXE").split(os.pathsep) if e] + [""]
|
|
103
|
+
for entry in os.environ.get("PATH", "").split(os.pathsep):
|
|
104
|
+
if not entry or not os.path.isabs(entry) or os.path.realpath(entry) == cwd:
|
|
105
|
+
continue
|
|
106
|
+
for ext in exts:
|
|
107
|
+
candidate = os.path.join(entry, name + ext)
|
|
108
|
+
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
|
109
|
+
return candidate
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _node() -> str:
|
|
114
|
+
node = _safe_which("node")
|
|
115
|
+
if not node:
|
|
116
|
+
raise AgentRedactError("`node` (Node.js 20+) was not found on PATH. " + _NOT_FOUND)
|
|
117
|
+
return node
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _global_cli_js() -> str | None:
|
|
121
|
+
"""Resolve the JS entry of a global `npm i -g agent-redact` without running npm's shims."""
|
|
122
|
+
shim = _safe_which("agent-redact")
|
|
123
|
+
if not shim:
|
|
124
|
+
return None
|
|
125
|
+
real = os.path.realpath(shim) # POSIX: the bin is a symlink to dist/cli/index.js
|
|
126
|
+
if real.endswith(".js") and os.path.isfile(real):
|
|
127
|
+
return real
|
|
128
|
+
# Windows: <prefix>\agent-redact.cmd sits next to <prefix>\node_modules\agent-redact
|
|
129
|
+
candidate = os.path.join(os.path.dirname(shim), "node_modules", "agent-redact", "dist", "cli", "index.js")
|
|
130
|
+
return candidate if os.path.isfile(candidate) else None
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _cache_root() -> str:
|
|
134
|
+
if os.name == "nt":
|
|
135
|
+
base = os.environ.get("LOCALAPPDATA") or os.path.expanduser("~\\AppData\\Local")
|
|
136
|
+
else:
|
|
137
|
+
base = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")
|
|
138
|
+
return os.path.join(base, "agent-redact", "cli")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _cached_cli_js() -> str:
|
|
142
|
+
"""Install the matching CLI once into a private cache (no install scripts) and reuse it.
|
|
143
|
+
|
|
144
|
+
npm is run through its own JS entry with the cache directory as cwd, never inside a
|
|
145
|
+
caller-controlled directory, so a project's .npmrc cannot redirect the install.
|
|
146
|
+
"""
|
|
147
|
+
target = os.path.join(_cache_root(), __version__)
|
|
148
|
+
entry = os.path.join(target, "node_modules", "agent-redact", "dist", "cli", "index.js")
|
|
149
|
+
if os.path.isfile(entry):
|
|
150
|
+
return entry
|
|
151
|
+
node = _node()
|
|
152
|
+
node_dir = os.path.dirname(node)
|
|
153
|
+
npm_cli = next(
|
|
154
|
+
(
|
|
155
|
+
p
|
|
156
|
+
for p in (
|
|
157
|
+
os.path.join(node_dir, "node_modules", "npm", "bin", "npm-cli.js"), # Windows layout
|
|
158
|
+
os.path.join(node_dir, "..", "lib", "node_modules", "npm", "bin", "npm-cli.js"), # POSIX
|
|
159
|
+
)
|
|
160
|
+
if os.path.isfile(p)
|
|
161
|
+
),
|
|
162
|
+
None,
|
|
163
|
+
)
|
|
164
|
+
if not npm_cli:
|
|
165
|
+
raise AgentRedactError(_NOT_FOUND)
|
|
166
|
+
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
167
|
+
staging = tempfile.mkdtemp(prefix=f"{__version__}-", dir=os.path.dirname(target))
|
|
168
|
+
proc = subprocess.run(
|
|
169
|
+
[node, npm_cli, "install", "--prefix", staging, "--no-audit", "--no-fund", "--ignore-scripts",
|
|
170
|
+
"--userconfig", os.devnull, f"agent-redact@{__version__}"],
|
|
171
|
+
cwd=staging,
|
|
172
|
+
capture_output=True,
|
|
173
|
+
text=True,
|
|
174
|
+
encoding="utf-8",
|
|
175
|
+
check=False,
|
|
176
|
+
)
|
|
177
|
+
if proc.returncode != 0:
|
|
178
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
179
|
+
raise AgentRedactError(f"could not install agent-redact@{__version__}: {proc.stderr.strip()}")
|
|
180
|
+
try:
|
|
181
|
+
os.replace(staging, target) # atomic publish; a concurrent installer may have won
|
|
182
|
+
except OSError:
|
|
183
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
184
|
+
if not os.path.isfile(entry):
|
|
185
|
+
raise AgentRedactError(_NOT_FOUND)
|
|
186
|
+
return entry
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _cli_command() -> list[str]:
|
|
190
|
+
"""Always `node <absolute path to dist/cli/index.js>`: never a .cmd/.bat shim, never npx,
|
|
191
|
+
never a relative path that could resolve inside an untrusted working directory."""
|
|
192
|
+
explicit = os.environ.get("AGENT_REDACT_CLI")
|
|
193
|
+
if explicit:
|
|
194
|
+
if not os.path.isabs(explicit):
|
|
195
|
+
raise AgentRedactError("AGENT_REDACT_CLI must be an absolute path")
|
|
196
|
+
if explicit.endswith((".js", ".mjs", ".cjs")):
|
|
197
|
+
return [_node(), explicit]
|
|
198
|
+
return [explicit]
|
|
199
|
+
js = _global_cli_js() or _cached_cli_js()
|
|
200
|
+
return [_node(), js]
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _run(
|
|
204
|
+
args: Sequence[str],
|
|
205
|
+
*,
|
|
206
|
+
stdin: str | None = None,
|
|
207
|
+
env: Mapping[str, str] | None = None,
|
|
208
|
+
ok_codes: Iterable[int] = (0,),
|
|
209
|
+
cwd: str | None = None,
|
|
210
|
+
) -> subprocess.CompletedProcess[str]:
|
|
211
|
+
command = [*_cli_command(), *args]
|
|
212
|
+
try:
|
|
213
|
+
proc = subprocess.run(
|
|
214
|
+
command,
|
|
215
|
+
input=stdin if stdin is not None else "",
|
|
216
|
+
capture_output=True,
|
|
217
|
+
text=True,
|
|
218
|
+
encoding="utf-8",
|
|
219
|
+
env=dict(env) if env is not None else None,
|
|
220
|
+
cwd=cwd,
|
|
221
|
+
check=False,
|
|
222
|
+
)
|
|
223
|
+
except OSError as exc: # e.g. the executable disappeared
|
|
224
|
+
raise AgentRedactError(f"could not start agent-redact: {exc}") from exc
|
|
225
|
+
if proc.returncode not in tuple(ok_codes):
|
|
226
|
+
message = (proc.stderr or proc.stdout or "").strip() or f"exit code {proc.returncode}"
|
|
227
|
+
raise AgentRedactError(message)
|
|
228
|
+
return proc
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _common_flags(presets: Sequence[str], entropy: bool, config: str | None) -> list[str]:
|
|
232
|
+
flags: list[str] = []
|
|
233
|
+
if presets:
|
|
234
|
+
flags += ["--preset", ",".join(presets)]
|
|
235
|
+
if entropy:
|
|
236
|
+
flags.append("--entropy")
|
|
237
|
+
if config:
|
|
238
|
+
flags += ["--config", config]
|
|
239
|
+
return flags
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def redact(
|
|
243
|
+
text: str,
|
|
244
|
+
*,
|
|
245
|
+
presets: Sequence[str] = (),
|
|
246
|
+
entropy: bool = False,
|
|
247
|
+
config: str | None = None,
|
|
248
|
+
) -> RedactResult:
|
|
249
|
+
"""One-way redaction: secrets become ``[REDACTED_<TYPE>]`` placeholders."""
|
|
250
|
+
proc = _run(["--json", *_common_flags(presets, entropy, config)], stdin=text)
|
|
251
|
+
data = json.loads(proc.stdout)
|
|
252
|
+
matches = [
|
|
253
|
+
Match(
|
|
254
|
+
pattern_id=m["patternId"],
|
|
255
|
+
pattern_name=m["patternName"],
|
|
256
|
+
category=m["category"],
|
|
257
|
+
severity=m["severity"],
|
|
258
|
+
start=m["startIndex"],
|
|
259
|
+
end=m["endIndex"],
|
|
260
|
+
replacement=m["replacement"],
|
|
261
|
+
)
|
|
262
|
+
for m in data.get("matches", [])
|
|
263
|
+
]
|
|
264
|
+
return RedactResult(text=data["text"], matches=matches, stats=data.get("stats", {}))
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def scan(
|
|
268
|
+
paths: Sequence[str] = (),
|
|
269
|
+
*,
|
|
270
|
+
staged: bool = False,
|
|
271
|
+
presets: Sequence[str] = (),
|
|
272
|
+
entropy: bool = False,
|
|
273
|
+
config: str | None = None,
|
|
274
|
+
baseline: str | None = None,
|
|
275
|
+
cwd: str | None = None,
|
|
276
|
+
) -> list[Finding]:
|
|
277
|
+
"""Scan files/directories (honouring .gitignore inside git repos) or the git index."""
|
|
278
|
+
if not paths and not staged:
|
|
279
|
+
raise ValueError("pass at least one path, or staged=True")
|
|
280
|
+
args = [*paths, "--format", "json", *_common_flags(presets, entropy, config)]
|
|
281
|
+
if staged:
|
|
282
|
+
args.append("--staged")
|
|
283
|
+
if baseline:
|
|
284
|
+
args += ["--baseline", baseline]
|
|
285
|
+
proc = _run(args, ok_codes=(0, 1), cwd=cwd) # exit 1 means "findings present"
|
|
286
|
+
data = json.loads(proc.stdout)
|
|
287
|
+
return [
|
|
288
|
+
Finding(
|
|
289
|
+
file=f["file"],
|
|
290
|
+
line=f["line"],
|
|
291
|
+
column=f["column"],
|
|
292
|
+
pattern_id=f["patternId"],
|
|
293
|
+
pattern_name=f["patternName"],
|
|
294
|
+
severity=f["severity"],
|
|
295
|
+
category=f["category"],
|
|
296
|
+
fingerprint=f["fingerprint"],
|
|
297
|
+
)
|
|
298
|
+
for f in data.get("findings", [])
|
|
299
|
+
]
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
class Session:
|
|
303
|
+
"""Reversible masking across several calls, e.g. one LLM conversation.
|
|
304
|
+
|
|
305
|
+
Placeholders are numbered consistently across calls (``[AWS_ACCESS_KEY_ID_1]``,
|
|
306
|
+
``[AWS_ACCESS_KEY_ID_2]`` …) and :meth:`unmask` restores them. The vault lives in a
|
|
307
|
+
private temporary directory, encrypted with a random passphrase that is passed to the
|
|
308
|
+
CLI through an environment variable and never written anywhere. Use as a context
|
|
309
|
+
manager (or call :meth:`close`) to delete it.
|
|
310
|
+
|
|
311
|
+
>>> with Session() as s: # doctest: +SKIP
|
|
312
|
+
... safe = s.redact(user_prompt)
|
|
313
|
+
... answer = call_llm(safe)
|
|
314
|
+
... print(s.unmask(answer))
|
|
315
|
+
"""
|
|
316
|
+
|
|
317
|
+
_KEY_ENV = "AGENT_REDACT_PY_SESSION_KEY"
|
|
318
|
+
|
|
319
|
+
def __init__(self, *, presets: Sequence[str] = (), entropy: bool = False, config: str | None = None) -> None:
|
|
320
|
+
self._dir = tempfile.mkdtemp(prefix="agent-redact-")
|
|
321
|
+
self._vault = os.path.join(self._dir, "vault.json")
|
|
322
|
+
self._env = {**os.environ, self._KEY_ENV: secrets.token_urlsafe(32)}
|
|
323
|
+
self._flags = _common_flags(presets, entropy, config)
|
|
324
|
+
self._closed = False
|
|
325
|
+
|
|
326
|
+
def _vault_flags(self) -> list[str]:
|
|
327
|
+
return ["--vault-key-env", self._KEY_ENV]
|
|
328
|
+
|
|
329
|
+
def redact(self, text: str) -> str:
|
|
330
|
+
"""Mask secrets with reversible, session-wide numbered placeholders."""
|
|
331
|
+
self._ensure_open()
|
|
332
|
+
args = ["-r", "--vault-out", self._vault, *self._vault_flags(), *self._flags]
|
|
333
|
+
if os.path.exists(self._vault):
|
|
334
|
+
args += ["--vault-in", self._vault]
|
|
335
|
+
return _run(args, stdin=text, env=self._env).stdout
|
|
336
|
+
|
|
337
|
+
def unmask(self, text: str) -> str:
|
|
338
|
+
"""Restore placeholders produced by this session (unknown ones are left as is)."""
|
|
339
|
+
self._ensure_open()
|
|
340
|
+
if not os.path.exists(self._vault):
|
|
341
|
+
return text
|
|
342
|
+
return _run(["--unmask", self._vault, *self._vault_flags()], stdin=text, env=self._env).stdout
|
|
343
|
+
|
|
344
|
+
def close(self) -> None:
|
|
345
|
+
if not self._closed:
|
|
346
|
+
shutil.rmtree(self._dir, ignore_errors=True)
|
|
347
|
+
self._closed = True
|
|
348
|
+
|
|
349
|
+
def _ensure_open(self) -> None:
|
|
350
|
+
if self._closed:
|
|
351
|
+
raise AgentRedactError("this Session is closed")
|
|
352
|
+
|
|
353
|
+
def __enter__(self) -> "Session":
|
|
354
|
+
return self
|
|
355
|
+
|
|
356
|
+
def __exit__(self, *exc: object) -> None:
|
|
357
|
+
self.close()
|
|
358
|
+
|
|
359
|
+
def __del__(self) -> None: # best effort cleanup
|
|
360
|
+
try:
|
|
361
|
+
self.close()
|
|
362
|
+
except Exception:
|
|
363
|
+
pass
|
|
File without changes
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Tests run against the repository's built CLI (dist/cli/index.js) unless
|
|
2
|
+
AGENT_REDACT_CLI is already set. Build it first with `npm run build`."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
REPO = Path(__file__).resolve().parents[2]
|
|
13
|
+
os.environ.setdefault("AGENT_REDACT_CLI", str(REPO / "dist" / "cli" / "index.js"))
|
|
14
|
+
|
|
15
|
+
import agent_redact # noqa: E402
|
|
16
|
+
from agent_redact import AgentRedactError, Session, redact, scan # noqa: E402
|
|
17
|
+
|
|
18
|
+
# Token-shaped fixtures are assembled at runtime so they never appear verbatim in the repo
|
|
19
|
+
AWS = "AKIA" + "IOSFODNN7EXAMPLE"
|
|
20
|
+
AWS2 = "AKIA" + "ABCDEFGHIJKLMNOP"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_version_matches_npm_package() -> None:
|
|
24
|
+
pkg = json.loads((REPO / "package.json").read_text(encoding="utf-8"))
|
|
25
|
+
assert agent_redact.__version__ == pkg["version"]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_redact_returns_text_and_matches_without_values() -> None:
|
|
29
|
+
result = redact(f"key {AWS} end")
|
|
30
|
+
assert result.text == "key [REDACTED_AWS_ACCESS_KEY_ID] end"
|
|
31
|
+
assert result.total_leaks == 1 and result.has_leaks
|
|
32
|
+
assert result.matches[0].pattern_id == "aws-access-key-id"
|
|
33
|
+
assert AWS not in repr(result)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_redact_handles_unicode_and_presets() -> None:
|
|
37
|
+
cccd = "079" + "2" + "03" + "012345"
|
|
38
|
+
assert redact(f"Xin chào {AWS} ổn").text == "Xin chào [REDACTED_AWS_ACCESS_KEY_ID] ổn"
|
|
39
|
+
assert cccd in redact(f"CCCD {cccd}").text # opt-in only
|
|
40
|
+
assert redact(f"CCCD {cccd}", presets=["vn-pii"]).text == "CCCD [REDACTED_VN_CITIZEN_ID]"
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_unknown_preset_raises() -> None:
|
|
44
|
+
with pytest.raises(AgentRedactError, match="Unknown preset"):
|
|
45
|
+
redact("x", presets=["nope"])
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_session_numbers_consistently_and_unmasks(tmp_path: Path) -> None:
|
|
49
|
+
with Session() as s:
|
|
50
|
+
first = s.redact(f"first {AWS}")
|
|
51
|
+
second = s.redact(f"second {AWS2} and again {AWS}")
|
|
52
|
+
assert first == "first [AWS_ACCESS_KEY_ID_1]"
|
|
53
|
+
assert second == "second [AWS_ACCESS_KEY_ID_2] and again [AWS_ACCESS_KEY_ID_1]"
|
|
54
|
+
assert s.unmask("use [AWS_ACCESS_KEY_ID_2] then [AWS_ACCESS_KEY_ID_1]") == f"use {AWS2} then {AWS}"
|
|
55
|
+
|
|
56
|
+
vault = Path(s._vault) # noqa: SLF001 - verify the on-disk format
|
|
57
|
+
content = vault.read_text(encoding="utf-8")
|
|
58
|
+
assert json.loads(content)["format"] == "agent-redact-vault" # encrypted
|
|
59
|
+
assert AWS not in content and AWS2 not in content
|
|
60
|
+
vault_dir = vault.parent
|
|
61
|
+
assert not vault_dir.exists() # deleted on exit
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_session_unmask_before_redact_is_identity() -> None:
|
|
65
|
+
with Session() as s:
|
|
66
|
+
assert s.unmask("nothing [AWS_ACCESS_KEY_ID_1]") == "nothing [AWS_ACCESS_KEY_ID_1]"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_closed_session_raises() -> None:
|
|
70
|
+
s = Session()
|
|
71
|
+
s.close()
|
|
72
|
+
with pytest.raises(AgentRedactError, match="closed"):
|
|
73
|
+
s.redact("x")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def test_scan_reports_findings_without_values(tmp_path: Path) -> None:
|
|
77
|
+
(tmp_path / "app.env").write_text(f"\nKEY={AWS}\n", encoding="utf-8")
|
|
78
|
+
(tmp_path / "clean.txt").write_text("nothing here", encoding="utf-8")
|
|
79
|
+
findings = scan(["."], cwd=str(tmp_path))
|
|
80
|
+
assert [(f.file, f.line, f.pattern_id) for f in findings] == [("app.env", 2, "aws-access-key-id")]
|
|
81
|
+
assert AWS not in repr(findings)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_scan_requires_a_target() -> None:
|
|
85
|
+
with pytest.raises(ValueError):
|
|
86
|
+
scan()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_missing_cli_raises_helpful_error(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
90
|
+
monkeypatch.delenv("AGENT_REDACT_CLI", raising=False)
|
|
91
|
+
monkeypatch.setattr(agent_redact, "_safe_which", lambda _name: None)
|
|
92
|
+
with pytest.raises(AgentRedactError, match="not found"):
|
|
93
|
+
redact("x")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_relative_cli_override_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
97
|
+
monkeypatch.setenv("AGENT_REDACT_CLI", "dist/cli/index.js")
|
|
98
|
+
with pytest.raises(AgentRedactError, match="absolute path"):
|
|
99
|
+
redact("x")
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_safe_which_ignores_cwd_and_relative_path_entries(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
103
|
+
planted = tmp_path / ("agent-redact.exe" if os.name == "nt" else "agent-redact")
|
|
104
|
+
planted.write_text("#!/bin/sh\necho pwned\n", encoding="utf-8")
|
|
105
|
+
planted.chmod(0o755)
|
|
106
|
+
monkeypatch.chdir(tmp_path)
|
|
107
|
+
monkeypatch.setenv("PATH", os.pathsep.join([".", "", str(tmp_path)]))
|
|
108
|
+
assert agent_redact._safe_which("agent-redact") is None # noqa: SLF001
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_cli_command_never_uses_shims_or_npx() -> None:
|
|
112
|
+
cmd = agent_redact._cli_command() # noqa: SLF001
|
|
113
|
+
assert os.path.basename(cmd[0]).lower().startswith("node")
|
|
114
|
+
assert cmd[1].endswith("index.js") and os.path.isabs(cmd[1])
|