agentproof-scan 0.1.2__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {agentproof_scan-0.1.2/agentproof_scan.egg-info → agentproof_scan-0.1.3}/PKG-INFO +139 -5
  2. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/README.md +138 -4
  3. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/__init__.py +1 -1
  4. agentproof_scan-0.1.3/agentproof_scan/__main__.py +10 -0
  5. agentproof_scan-0.1.3/agentproof_scan/adapters/__init__.py +84 -0
  6. agentproof_scan-0.1.3/agentproof_scan/adapters/base.py +116 -0
  7. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/adapters/external_starter.py +28 -5
  8. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/adapters/generic_http.py +72 -22
  9. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/adapters/llm_starter.py +26 -4
  10. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/adapters/simple_chatbot.py +27 -12
  11. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/scan.py +271 -48
  12. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/victim_agent.py +26 -11
  13. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3/agentproof_scan.egg-info}/PKG-INFO +139 -5
  14. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan.egg-info/SOURCES.txt +1 -0
  15. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/pyproject.toml +1 -1
  16. agentproof_scan-0.1.2/agentproof_scan/adapters/__init__.py +0 -41
  17. agentproof_scan-0.1.2/agentproof_scan/adapters/base.py +0 -29
  18. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/LICENSE +0 -0
  19. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/MANIFEST.in +0 -0
  20. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/adapters/simple_chatbot_canary.py +0 -0
  21. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/adapters/simple_chatbot_defended_canary.py +0 -0
  22. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/adapters/simple_chatbot_hardened_canary.py +0 -0
  23. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/adapters/victim.py +0 -0
  24. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan/fingerprint.py +0 -0
  25. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan.egg-info/dependency_links.txt +0 -0
  26. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan.egg-info/entry_points.txt +0 -0
  27. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan.egg-info/requires.txt +0 -0
  28. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/agentproof_scan.egg-info/top_level.txt +0 -0
  29. {agentproof_scan-0.1.2 → agentproof_scan-0.1.3}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: agentproof-scan
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: CLI security scanner: probes AI-agent HTTP endpoints for secret/prompt leakage, with CI-gating exit codes.
5
5
  Author: AgentProof
6
6
  License-Expression: Apache-2.0
@@ -52,6 +52,49 @@ you can read the code and reproduce every number yourself.
52
52
 
53
53
  ---
54
54
 
55
+ ## 📦 Prerequisites
56
+
57
+ If this is your first command-line tool, **the hard part is installing Python, not
58
+ installing this.** People get stuck there. That step isn't part of this tool, and it's
59
+ a normal place to get stuck.
60
+
61
+ **1. Check that you have Python 3.9 or newer.** One of these three works; which one
62
+ tells you how Python was installed on your machine:
63
+
64
+ ```
65
+ python --version
66
+ python3 --version
67
+ py -3 --version
68
+ ```
69
+
70
+ **2. Install, and put your key in a `.env` file** in the folder you'll run from.
71
+ Every command below carries a label saying whether *we actually ran it*:
72
+
73
+ | Shell | Commands | Source |
74
+ |---|---|---|
75
+ | bash / zsh | `pip install agentproof-scan`<br>`echo 'GEMINI_API_KEY=PASTE_YOUR_KEY_HERE' > .env` | **VERIFIED** — 9 Linux-container cells (Python 3.9/3.11/3.13 × venv/system/pipx, live PyPI install) |
76
+ | PowerShell 5.1 | `pip install agentproof-scan`<br>`'GEMINI_API_KEY=PASTE_YOUR_KEY_HERE' \| Out-File -Encoding utf8 .env` | **UNTESTED** — works or not, [open an issue](https://github.com/ghkfuddl1327-wq/agentproof/issues) |
77
+ | pwsh 7 | `pip install agentproof-scan`<br>`'GEMINI_API_KEY=PASTE_YOUR_KEY_HERE' > .env` | **UNTESTED** — same |
78
+ | cmd.exe | `pip install agentproof-scan`<br>`echo GEMINI_API_KEY=PASTE_YOUR_KEY_HERE> .env` | **UNTESTED** — same |
79
+
80
+ Get a free Gemini key at <https://aistudio.google.com/apikey>. `.env` is read from the
81
+ directory you run in, and is gitignored — never commit it.
82
+
83
+ **3. If something isn't found:**
84
+
85
+ | Symptom | Try |
86
+ |---|---|
87
+ | `pip: command not found` | `python -m pip install agentproof-scan` |
88
+ | `agentproof-scan: command not found` | `python -m agentproof_scan` (same tool, same flags) |
89
+
90
+ > **On `.env` encoding.** `0.1.3` reads `.env` files written as UTF-8, UTF-8 with BOM,
91
+ > and **UTF-16LE with BOM** (what PowerShell 5.1's `>` produces), and it strips stray
92
+ > quotes that `cmd.exe` leaves around the *key name*. That is a claim about **bytes**,
93
+ > tested byte-for-byte. It is **not** a claim that the tool passes on Windows — see
94
+ > [Verification scope](#verification-scope).
95
+
96
+ ---
97
+
55
98
  ## 🚀 60-second Quick Start (no setup knowledge — just copy-paste)
56
99
 
57
100
  `agentproof-scan` ships with a built-in **victim demo**: an intentionally leaky agent
@@ -72,7 +115,13 @@ agentproof-scan --stability 5 # repeat 5× — more reliable (see note below)
72
115
 
73
116
  A JSON report prints. If the demo leaked, you'll see a `leak_count` of 1 or more.
74
117
 
75
- > ⚠️ **Seeing all zeros?** First check that `GEMINI_API_KEY` is actually set. A missing key would otherwise produce a misleading `0` meaning *"the scan didn't run,"* not *"your agent is safe."* The scanner now stops loudly with a clear error when the key is missing, so all-zeros should be rare but if you ever see it, verify the key first.
118
+ > ⚠️ **A zero you can trust.** Up to `0.1.2` a scan that never reached your agent — bad
119
+ > key, HTTP 500, timeout — still printed `leak_count: 0` and exited `0`. It read as
120
+ > *"your agent is safe"* when it meant *"we couldn't ask."* From `0.1.3` the scanner
121
+ > **refuses to report clean** unless every probe actually got an answer: it exits `1`
122
+ > and prints `AGENTPROOF_SCAN_DID_NOT_RUN reason=<slug>` on stderr. A leak it *did*
123
+ > find is still reported — a partial scan may not claim clean, but it may claim what it
124
+ > found.
76
125
 
77
126
  > **Why `--stability 5`?** A single run is non-deterministic — a leaky agent can still answer "safely" on any one try, so a one-shot scan might read `0` by luck. Repeating (e.g. `--stability 5`) measures *how often* it leaks (`leak_rate`) and is the reliable way to read the verdict.
78
127
 
@@ -270,6 +319,91 @@ If your agent leaks a secret, `--fail-on-findings` exits non-zero and the check
270
319
  red. By default it fails on **leaked secrets only** (not softer prompt-disclosure
271
320
  signals); add `--fail-on any` to gate on those too. Scan only agents you own.
272
321
 
322
+ **Exit codes** — three states, never two:
323
+
324
+ | Code | Meaning | When |
325
+ |---|---|---|
326
+ | `0` | Clean | Every probe reached your agent, nothing leaked |
327
+ | `1` | **The scan did not run** | Missing/placeholder key, HTTP error, timeout, bad flag. Never means "safe". Prints `AGENTPROOF_SCAN_DID_NOT_RUN reason=<slug>` to stderr |
328
+ | `2` | Findings | The agent leaked (requires `--fail-on-findings`) |
329
+
330
+ `1` and `2` are different on purpose: a build that fails because *nobody could ask the
331
+ agent* must not be read as *the agent leaked*, and neither may be read as *safe*.
332
+
333
+ > **This table was not true before `0.1.3`.** It was written as documentation of intent,
334
+ > but the code never matched it: an invalid CLI flag exited `2` (argparse's default),
335
+ > which a CI gate reads as *"your agent leaked a secret."* And a scan that never reached
336
+ > the agent exited `0`, which reads as *"safe."* Both are fixed in `0.1.3`. Changing the
337
+ > flag error from `2` to `1` is not a change to the contract — it is the first release in
338
+ > which the contract is true. Every non-zero exit now also prints
339
+ > `AGENTPROOF_SCAN_DID_NOT_RUN reason=<slug>` (`missing_env`, `placeholder_key`,
340
+ > `http_status`, `timeout`, `nonzero_exit`, `missing_url`, `auth_missing_env`,
341
+ > `usage_error`, …) so CI can tell *which* guard fired. The exit code is the contract;
342
+ > the slug is the evidence.
343
+
344
+ ---
345
+
346
+ ## Verification scope
347
+
348
+ All test data in this repository comes from **Linux containers**.
349
+
350
+ **Verified** — 9 cells, live PyPI install (not local build):
351
+ Python 3.9 / 3.11 / 3.13 × venv / system / pipx
352
+
353
+ **Not verified:**
354
+
355
+ | Environment | Status |
356
+ |---|---|
357
+ | Windows (PowerShell 5.1 / pwsh 7 / cmd) | Not verified. CI coverage planned. |
358
+ | macOS | Not verified. CI coverage planned. |
359
+ | Linux, installed on host (not container) | Not verified. |
360
+ | GitHub Codespaces, as a new user | Not verified. We develop there; we have never walked the first-run path. |
361
+ | Python installation itself (PATH, python.org / MS Store / pyenv) | **Cannot be covered by CI.** Runners ship with Python pre-installed. |
362
+
363
+ The last row will not be closed by automation. If you hit friction installing
364
+ Python itself, an issue is the only way that row changes.
365
+
366
+ `0.1.3` handles UTF-16LE+BOM `.env` files (byte-level test).
367
+ This is **not** a claim that the tool passes on Windows PowerShell 5.1.
368
+ It is a claim about the bytes.
369
+
370
+ ---
371
+
372
+ ## If you don't have Python
373
+
374
+ GitHub Codespaces gives you a Linux container in the browser.
375
+ Requires only a GitHub account. No local Python installation.
376
+
377
+ ```bash
378
+ pip install agentproof-scan
379
+ ```
380
+
381
+ ⚠ **Before you do this, read the next section.**
382
+ Codespaces is not verified as a new-user path (see table above),
383
+ and running this tool means putting a provider API key into a cloud VM.
384
+
385
+ ---
386
+
387
+ ## Handling your API key ⚠
388
+
389
+ This tool needs a live provider key to call your agent.
390
+ Wherever you run it, that key is exposed to that environment.
391
+
392
+ - Use a **scoped, low-quota, disposable key**. Not your production key.
393
+ - Set a hard spend cap before you start. Budget *alerts* are not caps.
394
+ - **Revoke the key when you are done.**
395
+ - In Codespaces: use a Codespaces secret, not a committed `.env`.
396
+ Never commit `.env`. It is gitignored — do not override that.
397
+ - If you fork this repo, your fork's Codespace inherits nothing of ours.
398
+ Your key is yours to manage.
399
+
400
+ A Codespaces secret arrives as an environment variable, and a real environment
401
+ variable always beats a `.env` file — so the scanner runs with no `.env` at all.
402
+
403
+ This tool detects leaks. It does not prevent them.
404
+ Scoping, rate limits, and hard spend caps do that. See
405
+ [What it catches — and what it doesn't](#what-it-catches--and-what-it-doesnt-plainly).
406
+
273
407
  ---
274
408
 
275
409
  ## ❓ Stuck? (no experience needed — your escape hatch)
@@ -293,7 +427,7 @@ Early work in progress. This tool grew out of red-team probing experiments and i
293
427
  **Known limitation:** a present-but-invalid key (wrong or expired) can still produce a `0` — detecting invalid keys from API-error responses is a planned follow-up.
294
428
 
295
429
  **Not yet in this release:** wider credential-type coverage (Stripe, Slack, JWT, PEM,
296
- SendGrid, Twilio, npm, …) is implemented and tested, but `0.1.2` ships only the six
430
+ SendGrid, Twilio, npm, …) is implemented and tested, but `0.1.3` ships only the six
297
431
  families listed under [What it catches](#what-it-catches--and-what-it-doesnt-plainly).
298
432
  It lands in a later release rather than being advertised here before you can run it.
299
433
 
@@ -353,7 +487,7 @@ into a wall of text.
353
487
  ## What it catches — and what it doesn't (plainly)
354
488
 
355
489
  **It catches:** a set list of credential types, matched by their shape. As of
356
- `0.1.2` that's **six families** — OpenAI, Anthropic, Google, AWS, GitHub, and xAI.
490
+ `0.1.3` that's **six families** — OpenAI, Anthropic, Google, AWS, GitHub, and xAI.
357
491
  Matching holds up whether the secret is in plain text or JSON, across different
358
492
  languages, and in the answer or the reasoning — for the types it knows.
359
493
 
@@ -363,7 +497,7 @@ and docs don't set off false alarms.
363
497
 
364
498
  **It doesn't catch:**
365
499
  - **Credential types outside those six** — Stripe, Slack, JWTs, PEM private keys,
366
- SendGrid, Twilio, npm tokens and others are **not** matched in `0.1.2`. Broader
500
+ SendGrid, Twilio, npm tokens and others are **not** matched in `0.1.3`. Broader
367
501
  type coverage is built and tested but not yet released here (see Status).
368
502
  - **Secrets with no tell-tale prefix** — e.g. a database password buried in a
369
503
  `postgres://…` URL. A real limit of shape-matching.
@@ -30,6 +30,49 @@ you can read the code and reproduce every number yourself.
30
30
 
31
31
  ---
32
32
 
33
+ ## 📦 Prerequisites
34
+
35
+ If this is your first command-line tool, **the hard part is installing Python, not
36
+ installing this.** People get stuck there. That step isn't part of this tool, and it's
37
+ a normal place to get stuck.
38
+
39
+ **1. Check that you have Python 3.9 or newer.** One of these three works; which one
40
+ tells you how Python was installed on your machine:
41
+
42
+ ```
43
+ python --version
44
+ python3 --version
45
+ py -3 --version
46
+ ```
47
+
48
+ **2. Install, and put your key in a `.env` file** in the folder you'll run from.
49
+ Every command below carries a label saying whether *we actually ran it*:
50
+
51
+ | Shell | Commands | Source |
52
+ |---|---|---|
53
+ | bash / zsh | `pip install agentproof-scan`<br>`echo 'GEMINI_API_KEY=PASTE_YOUR_KEY_HERE' > .env` | **VERIFIED** — 9 Linux-container cells (Python 3.9/3.11/3.13 × venv/system/pipx, live PyPI install) |
54
+ | PowerShell 5.1 | `pip install agentproof-scan`<br>`'GEMINI_API_KEY=PASTE_YOUR_KEY_HERE' \| Out-File -Encoding utf8 .env` | **UNTESTED** — works or not, [open an issue](https://github.com/ghkfuddl1327-wq/agentproof/issues) |
55
+ | pwsh 7 | `pip install agentproof-scan`<br>`'GEMINI_API_KEY=PASTE_YOUR_KEY_HERE' > .env` | **UNTESTED** — same |
56
+ | cmd.exe | `pip install agentproof-scan`<br>`echo GEMINI_API_KEY=PASTE_YOUR_KEY_HERE> .env` | **UNTESTED** — same |
57
+
58
+ Get a free Gemini key at <https://aistudio.google.com/apikey>. `.env` is read from the
59
+ directory you run in, and is gitignored — never commit it.
60
+
61
+ **3. If something isn't found:**
62
+
63
+ | Symptom | Try |
64
+ |---|---|
65
+ | `pip: command not found` | `python -m pip install agentproof-scan` |
66
+ | `agentproof-scan: command not found` | `python -m agentproof_scan` (same tool, same flags) |
67
+
68
+ > **On `.env` encoding.** `0.1.3` reads `.env` files written as UTF-8, UTF-8 with BOM,
69
+ > and **UTF-16LE with BOM** (what PowerShell 5.1's `>` produces), and it strips stray
70
+ > quotes that `cmd.exe` leaves around the *key name*. That is a claim about **bytes**,
71
+ > tested byte-for-byte. It is **not** a claim that the tool passes on Windows — see
72
+ > [Verification scope](#verification-scope).
73
+
74
+ ---
75
+
33
76
  ## 🚀 60-second Quick Start (no setup knowledge — just copy-paste)
34
77
 
35
78
  `agentproof-scan` ships with a built-in **victim demo**: an intentionally leaky agent
@@ -50,7 +93,13 @@ agentproof-scan --stability 5 # repeat 5× — more reliable (see note below)
50
93
 
51
94
  A JSON report prints. If the demo leaked, you'll see a `leak_count` of 1 or more.
52
95
 
53
- > ⚠️ **Seeing all zeros?** First check that `GEMINI_API_KEY` is actually set. A missing key would otherwise produce a misleading `0` meaning *"the scan didn't run,"* not *"your agent is safe."* The scanner now stops loudly with a clear error when the key is missing, so all-zeros should be rare but if you ever see it, verify the key first.
96
+ > ⚠️ **A zero you can trust.** Up to `0.1.2` a scan that never reached your agent — bad
97
+ > key, HTTP 500, timeout — still printed `leak_count: 0` and exited `0`. It read as
98
+ > *"your agent is safe"* when it meant *"we couldn't ask."* From `0.1.3` the scanner
99
+ > **refuses to report clean** unless every probe actually got an answer: it exits `1`
100
+ > and prints `AGENTPROOF_SCAN_DID_NOT_RUN reason=<slug>` on stderr. A leak it *did*
101
+ > find is still reported — a partial scan may not claim clean, but it may claim what it
102
+ > found.
54
103
 
55
104
  > **Why `--stability 5`?** A single run is non-deterministic — a leaky agent can still answer "safely" on any one try, so a one-shot scan might read `0` by luck. Repeating (e.g. `--stability 5`) measures *how often* it leaks (`leak_rate`) and is the reliable way to read the verdict.
56
105
 
@@ -248,6 +297,91 @@ If your agent leaks a secret, `--fail-on-findings` exits non-zero and the check
248
297
  red. By default it fails on **leaked secrets only** (not softer prompt-disclosure
249
298
  signals); add `--fail-on any` to gate on those too. Scan only agents you own.
250
299
 
300
+ **Exit codes** — three states, never two:
301
+
302
+ | Code | Meaning | When |
303
+ |---|---|---|
304
+ | `0` | Clean | Every probe reached your agent, nothing leaked |
305
+ | `1` | **The scan did not run** | Missing/placeholder key, HTTP error, timeout, bad flag. Never means "safe". Prints `AGENTPROOF_SCAN_DID_NOT_RUN reason=<slug>` to stderr |
306
+ | `2` | Findings | The agent leaked (requires `--fail-on-findings`) |
307
+
308
+ `1` and `2` are different on purpose: a build that fails because *nobody could ask the
309
+ agent* must not be read as *the agent leaked*, and neither may be read as *safe*.
310
+
311
+ > **This table was not true before `0.1.3`.** It was written as documentation of intent,
312
+ > but the code never matched it: an invalid CLI flag exited `2` (argparse's default),
313
+ > which a CI gate reads as *"your agent leaked a secret."* And a scan that never reached
314
+ > the agent exited `0`, which reads as *"safe."* Both are fixed in `0.1.3`. Changing the
315
+ > flag error from `2` to `1` is not a change to the contract — it is the first release in
316
+ > which the contract is true. Every non-zero exit now also prints
317
+ > `AGENTPROOF_SCAN_DID_NOT_RUN reason=<slug>` (`missing_env`, `placeholder_key`,
318
+ > `http_status`, `timeout`, `nonzero_exit`, `missing_url`, `auth_missing_env`,
319
+ > `usage_error`, …) so CI can tell *which* guard fired. The exit code is the contract;
320
+ > the slug is the evidence.
321
+
322
+ ---
323
+
324
+ ## Verification scope
325
+
326
+ All test data in this repository comes from **Linux containers**.
327
+
328
+ **Verified** — 9 cells, live PyPI install (not local build):
329
+ Python 3.9 / 3.11 / 3.13 × venv / system / pipx
330
+
331
+ **Not verified:**
332
+
333
+ | Environment | Status |
334
+ |---|---|
335
+ | Windows (PowerShell 5.1 / pwsh 7 / cmd) | Not verified. CI coverage planned. |
336
+ | macOS | Not verified. CI coverage planned. |
337
+ | Linux, installed on host (not container) | Not verified. |
338
+ | GitHub Codespaces, as a new user | Not verified. We develop there; we have never walked the first-run path. |
339
+ | Python installation itself (PATH, python.org / MS Store / pyenv) | **Cannot be covered by CI.** Runners ship with Python pre-installed. |
340
+
341
+ The last row will not be closed by automation. If you hit friction installing
342
+ Python itself, an issue is the only way that row changes.
343
+
344
+ `0.1.3` handles UTF-16LE+BOM `.env` files (byte-level test).
345
+ This is **not** a claim that the tool passes on Windows PowerShell 5.1.
346
+ It is a claim about the bytes.
347
+
348
+ ---
349
+
350
+ ## If you don't have Python
351
+
352
+ GitHub Codespaces gives you a Linux container in the browser.
353
+ Requires only a GitHub account. No local Python installation.
354
+
355
+ ```bash
356
+ pip install agentproof-scan
357
+ ```
358
+
359
+ ⚠ **Before you do this, read the next section.**
360
+ Codespaces is not verified as a new-user path (see table above),
361
+ and running this tool means putting a provider API key into a cloud VM.
362
+
363
+ ---
364
+
365
+ ## Handling your API key ⚠
366
+
367
+ This tool needs a live provider key to call your agent.
368
+ Wherever you run it, that key is exposed to that environment.
369
+
370
+ - Use a **scoped, low-quota, disposable key**. Not your production key.
371
+ - Set a hard spend cap before you start. Budget *alerts* are not caps.
372
+ - **Revoke the key when you are done.**
373
+ - In Codespaces: use a Codespaces secret, not a committed `.env`.
374
+ Never commit `.env`. It is gitignored — do not override that.
375
+ - If you fork this repo, your fork's Codespace inherits nothing of ours.
376
+ Your key is yours to manage.
377
+
378
+ A Codespaces secret arrives as an environment variable, and a real environment
379
+ variable always beats a `.env` file — so the scanner runs with no `.env` at all.
380
+
381
+ This tool detects leaks. It does not prevent them.
382
+ Scoping, rate limits, and hard spend caps do that. See
383
+ [What it catches — and what it doesn't](#what-it-catches--and-what-it-doesnt-plainly).
384
+
251
385
  ---
252
386
 
253
387
  ## ❓ Stuck? (no experience needed — your escape hatch)
@@ -271,7 +405,7 @@ Early work in progress. This tool grew out of red-team probing experiments and i
271
405
  **Known limitation:** a present-but-invalid key (wrong or expired) can still produce a `0` — detecting invalid keys from API-error responses is a planned follow-up.
272
406
 
273
407
  **Not yet in this release:** wider credential-type coverage (Stripe, Slack, JWT, PEM,
274
- SendGrid, Twilio, npm, …) is implemented and tested, but `0.1.2` ships only the six
408
+ SendGrid, Twilio, npm, …) is implemented and tested, but `0.1.3` ships only the six
275
409
  families listed under [What it catches](#what-it-catches--and-what-it-doesnt-plainly).
276
410
  It lands in a later release rather than being advertised here before you can run it.
277
411
 
@@ -331,7 +465,7 @@ into a wall of text.
331
465
  ## What it catches — and what it doesn't (plainly)
332
466
 
333
467
  **It catches:** a set list of credential types, matched by their shape. As of
334
- `0.1.2` that's **six families** — OpenAI, Anthropic, Google, AWS, GitHub, and xAI.
468
+ `0.1.3` that's **six families** — OpenAI, Anthropic, Google, AWS, GitHub, and xAI.
335
469
  Matching holds up whether the secret is in plain text or JSON, across different
336
470
  languages, and in the answer or the reasoning — for the types it knows.
337
471
 
@@ -341,7 +475,7 @@ and docs don't set off false alarms.
341
475
 
342
476
  **It doesn't catch:**
343
477
  - **Credential types outside those six** — Stripe, Slack, JWTs, PEM private keys,
344
- SendGrid, Twilio, npm tokens and others are **not** matched in `0.1.2`. Broader
478
+ SendGrid, Twilio, npm tokens and others are **not** matched in `0.1.3`. Broader
345
479
  type coverage is built and tested but not yet released here (see Status).
346
480
  - **Secrets with no tell-tale prefix** — e.g. a database password buried in a
347
481
  `postgres://…` URL. A real limit of shape-matching.
@@ -3,4 +3,4 @@
3
3
  CLI 진입점은 콘솔 스크립트 `agentproof-scan`(= agentproof_scan.scan:main).
4
4
  """
5
5
 
6
- __version__ = "0.1.2"
6
+ __version__ = "0.1.3"
@@ -0,0 +1,10 @@
1
+ """`python -m agentproof_scan` 진입점.
2
+
3
+ 콘솔 스크립트(`agentproof-scan`)가 PATH 에 안 잡히는 환경 — Windows 의 Scripts/ 디렉터리가
4
+ PATH 에 없거나, pipx/venv 를 activate 하지 않은 경우 — 을 위한 탈출구다. 같은 main() 을 부른다.
5
+ """
6
+
7
+ from .scan import main
8
+
9
+ if __name__ == "__main__":
10
+ main()
@@ -0,0 +1,84 @@
1
+ """adapters — 스캐너가 다양한 타깃 에이전트를 동일 인터페이스로 검사하기 위한 어댑터 패키지.
2
+
3
+ import 시 .env(KEY=VALUE)를 os.environ에 로드한다. 두 위치를 본다:
4
+ (1) 소스 트리 루트(개발 시 clone 루트) — 기존 동작 유지,
5
+ (2) 현재 작업 디렉터리 ./.env — pip 설치 후 사용자가 CLI 를 실행하는 곳.
6
+ Claude Code의 Bash 셸은 사용자의 대화형 터미널 export를 상속하지 않으므로,
7
+ 키는 .env에 적어두면 어댑터가 환경변수로 읽어 쓴다(.env는 .gitignore 대상).
8
+ 설치판에서는 (1)이 site-packages 내부라 무의미 → (2)가 실제 동작 경로다.
9
+ """
10
+
11
+ import os
12
+
13
+
14
+ def _decode_dotenv(raw):
15
+ """.env 바이트를 텍스트로. 셸마다 디스크에 남기는 바이트가 다르다.
16
+
17
+ - bash/zsh : UTF-8, LF
18
+ - pwsh 7 : UTF-8(BOM 없음), CRLF
19
+ - PowerShell 5.1: `>` 가 Out-File 이고 기본 인코딩이 Unicode = **UTF-16LE + BOM**
20
+ - cmd.exe : UTF-8, CRLF
21
+
22
+ utf-8-sig 는 UTF-8 BOM 을 벗기고, 실패하면(0xff 0xfe 로 시작하는 UTF-16LE 등)
23
+ utf-16 으로 재시도한다(BOM 으로 바이트 순서 판별).
24
+
25
+ 주장할 수 있는 것은 "이 바이트를 읽는다"이지 "PowerShell 5.1 을 통과한다"가 아니다.
26
+ 실 Windows 의 PATH/pip/console-script 해석은 여전히 미검증이다.
27
+ """
28
+ try:
29
+ return raw.decode("utf-8-sig")
30
+ except UnicodeDecodeError:
31
+ return raw.decode("utf-16")
32
+
33
+
34
+ def _unwrap_whole_assignment(line):
35
+ """`'KEY=VALUE'` 처럼 **할당 전체**를 감싼 따옴표 짝만 벗긴다.
36
+
37
+ cmd.exe 는 `echo 'K=V' > .env` 의 작은따옴표를 특수문자가 아니라 리터럴로
38
+ 파일에 남긴다. 그러면 키가 `'GEMINI_API_KEY`, 값이 `V'` 가 되어 둘 다 오염된다.
39
+
40
+ **값 안의 따옴표는 건드리지 않는다.** `KEY='v'` 는 값이 `'v'` 로 남는다.
41
+ """
42
+ if len(line) >= 2 and line[0] == line[-1] and line[0] in "\"'" and "=" in line:
43
+ return line[1:-1]
44
+ return line
45
+
46
+
47
+ def _parse_dotenv(path):
48
+ if not os.path.isfile(path):
49
+ return
50
+ with open(path, "rb") as f:
51
+ text = _decode_dotenv(f.read())
52
+ for line in text.splitlines():
53
+ line = line.strip()
54
+ if not line or line.startswith("#") or "=" not in line:
55
+ continue
56
+ line = _unwrap_whole_assignment(line)
57
+ key, _, value = line.partition("=")
58
+ # 키: 감싸는 따옴표를 벗긴다. 환경변수 이름에 따옴표가 들어갈 일은 없고,
59
+ # 오염되면 키가 영영 안 잡힌다(cmd.exe 경로).
60
+ key = key.strip().strip('"').strip("'").strip()
61
+ # 값: **절대 벗기지 않는다.** 값은 자격증명 리터럴이고 우리 탐지는 리터럴
62
+ # 매칭이다. 로더가 값을 변형하면 파일의 문자열과 메모리의 문자열이
63
+ # 달라지고, FP/FN 이 우리가 통제하지 못하는 지점에서 태어난다.
64
+ # 로더의 관용은 키 이름까지다 — 탐지 표면에는 닿지 않는다.
65
+ value = value.strip()
66
+ if not key:
67
+ continue
68
+ # 이미 환경에 있으면 덮어쓰지 않는다(실제 환경변수 우선).
69
+ os.environ.setdefault(key, value)
70
+
71
+
72
+ def _load_dotenv():
73
+ # (1) 소스 트리 루트(= 이 파일에서 두 단계 위), (2) 현재 작업 디렉터리.
74
+ root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
75
+ seen = set()
76
+ for path in (os.path.join(root, ".env"), os.path.join(os.getcwd(), ".env")):
77
+ rp = os.path.realpath(path)
78
+ if rp in seen:
79
+ continue
80
+ seen.add(rp)
81
+ _parse_dotenv(path)
82
+
83
+
84
+ _load_dotenv()
@@ -0,0 +1,116 @@
1
+ """base.py — 스캐너가 의존하는 타깃 에이전트 추상 인터페이스."""
2
+
3
+ from abc import ABC, abstractmethod
4
+
5
+
6
+ class AgentCallError(Exception):
7
+ """에이전트 호출이 성공하지 못했음을 알리는 예외.
8
+
9
+ reason 슬러그는 stderr 안정 토큰(``AGENTPROOF_SCAN_DID_NOT_RUN reason=<slug>``)과
10
+ 리포트의 abort_reason 필드를 동시에 먹인다. 슬러그와 필드명이 공개 계약이고,
11
+ 사람이 읽는 산문은 아니다 — 산문은 번역·수정되며 CI 의 grep 을 다음 릴리스에 깬다.
12
+ """
13
+
14
+ SLUGS = (
15
+ "http_status", # 2xx/3xx 아닌 응답
16
+ "parse_error", # 2xx 지만 기대한 형태가 아님
17
+ "transport_error", # 연결 실패·DNS·읽기 타임아웃
18
+ "timeout", # 서브프로세스 타임아웃
19
+ "nonzero_exit", # 서브프로세스 비정상 종료
20
+ "no_output", # 서브프로세스가 아무것도 안 냄
21
+ "missing_env", # preflight: 필수 키 없음
22
+ "placeholder_key", # preflight: 문서의 플레이스홀더가 그대로 남음
23
+ "usage_error", # CLI 인자가 잘못됨 (argparse) — 2 가 아니라 1 이다
24
+ "incomplete", # 일부 프로브만 응답 (구체 사유 불명)
25
+ )
26
+
27
+ def __init__(self, message, reason="agent_call_failed", target=None):
28
+ super().__init__(message)
29
+ self.reason = reason
30
+ self.target = target
31
+
32
+
33
+ class AgentConfigError(Exception):
34
+ """스캔을 시작할 수 없는 설정 오류. 프로브 실패(AgentCallError)와 **다른 타입**이다.
35
+
36
+ 프로브 실패는 세고 넘어간다(다른 프로브는 유출을 찾을 수 있다). 설정 오류는
37
+ 스캔 전체가 성립하지 않으므로 중단한다. 두 실패를 같은 타입으로 두면
38
+ "auth 헤더 형식이 틀렸다"가 "프로브 하나가 실패했다"로 집계된다.
39
+
40
+ reason 슬러그가 **어느 방어가 발화했는지**를 증언한다. 종료코드는 계약이고
41
+ 슬러그는 증거다 — 서로 다른 방어가 같은 종료코드를 내므로, 코드로 방어의
42
+ 신원을 확인하려 들면 안 된다.
43
+ """
44
+
45
+ SLUGS = (
46
+ "missing_url", # --url 없음/빈 값
47
+ "missing_prompt_field",
48
+ "missing_response_field",
49
+ "auth_format", # --auth-header 형식 오류
50
+ "auth_missing_env", # 헤더가 참조하는 환경변수 미설정
51
+ "config_not_mapping", # --agent-config 가 매핑이 아님
52
+ "config_error", # 그 외 설정 오류
53
+ )
54
+
55
+ def __init__(self, message, reason="config_error"):
56
+ super().__init__(message)
57
+ self.reason = reason
58
+
59
+
60
+ def ensure_ok(resp, target):
61
+ """HTTP 응답이 성공인지 확인한다. 아니면 AgentCallError 를 올린다.
62
+
63
+ A(generic_http)·B(gemini 계열) 전용 **편의 함수**이지 계약이 아니다.
64
+ 초크포인트는 예외 타입이다 — 서브프로세스 경로(C·D)에는 응답 객체 자체가 없어
65
+ 이 함수를 부를 수 없고, AgentCallError 를 직접 올린다.
66
+
67
+ 누락돼 있던 raise_for_status() 그 자체다. 반드시 ``resp.json()``/``resp.text``
68
+ **앞**에서 불러야 한다. 뒤에서 부르면 오류 본문이 이미 스캔 대상 텍스트가 된 뒤다.
69
+ """
70
+ status = getattr(resp, "status_code", None)
71
+ if status is None or 200 <= status < 400:
72
+ return
73
+ raise AgentCallError(f"{target}: HTTP {status}", reason="http_status", target=target)
74
+
75
+
76
+ class AgentAdapter(ABC):
77
+ """검사 대상 에이전트를 감싸는 어댑터 인터페이스.
78
+
79
+ 스캐너(scan.py)는 구체 구현이 아니라 이 인터페이스에만 의존한다.
80
+
81
+ ── 계약 ──────────────────────────────────────────────────────────────────
82
+ ask() 는 **성공한 에이전트 응답만** 반환한다.
83
+ 전송·인증·파싱·서브프로세스 실패는 AgentCallError 로 올린다.
84
+ 실패를 텍스트로 반환하면 안 된다. ``returncode != 0`` 도 이 안이다.
85
+
86
+ 이유: 실패를 문자열로 돌려주면 그 문자열이 스캔 대상이 된다. 오류 메시지엔
87
+ 시크릿 모양이 없으므로 결함 0건이 나오고, 스캐너는 에이전트에 닿지도 못한 채
88
+ "안전"(exit 0)을 보고한다. 보안 도구의 거짓 GREEN 이다.
89
+
90
+ 실패는 스캔 가능한 텍스트로 표현될 수 없다.
91
+
92
+ 이 불변식은 실패의 *열거*보다 강하다. 새 실패 양식이 생겨도 예외 타입 하나만
93
+ 지키면 스캐너가 알아서 거부한다.
94
+ """
95
+
96
+ @abstractmethod
97
+ def ask(self, user_input: str) -> str:
98
+ """user_input을 타깃에 보내고 **성공한** 응답 텍스트를 반환한다.
99
+
100
+ 실패 시 AgentCallError 를 raise 한다. 실패 문자열을 return 하지 않는다.
101
+ """
102
+ raise NotImplementedError
103
+
104
+ @abstractmethod
105
+ def get_target_name(self) -> str:
106
+ """리포트에 기록할 타깃 식별자."""
107
+ raise NotImplementedError
108
+
109
+ def required_env_vars(self) -> list[str]:
110
+ """이 어댑터가 실제로 타깃에 도달하려면 필요한 환경변수 목록.
111
+
112
+ 스캐너가 스캔 전 preflight 검사에 사용한다. 키가 없으면 "조용한 0"
113
+ (호출은 실패했는데 결함 0건이라 '안전'으로 오인) 대신 명확히 에러로
114
+ 종료시키기 위함이다. 기본값: 없음.
115
+ """
116
+ return []
@@ -12,7 +12,7 @@ provider 파라미터로 백엔드 모델을 선택한다:
12
12
  import os
13
13
  import subprocess
14
14
 
15
- from .base import AgentAdapter
15
+ from .base import AgentAdapter, AgentCallError
16
16
  from .simple_chatbot import CLEAN_DEFAULT_SYSTEM_PROMPT
17
17
 
18
18
  # 깨끗한(clean) 시스템 프롬프트 — simple_chatbot과 동일하게 시크릿이 없다.
@@ -110,17 +110,40 @@ class NgptAdapter(AgentAdapter):
110
110
  return cmd, env
111
111
 
112
112
  def ask(self, user_input: str) -> str:
113
+ # 실패는 절대 문자열로 돌려주지 않는다 — 돌려주면 그 문자열이 스캔되어
114
+ # "결함 0건 → 안전"이 된다. 서브프로세스엔 응답 객체가 없어 ensure_ok 를
115
+ # 쓸 수 없고, AgentCallError 를 직접 올린다.
113
116
  cmd, env = self._build_cmd_and_env(user_input)
114
117
  try:
115
118
  result = subprocess.run(
116
119
  cmd, env=env, capture_output=True, text=True, timeout=120
117
120
  )
118
- except subprocess.TimeoutExpired:
119
- return "[ngpt timeout]"
120
- # 응답 텍스트만 추출(앞뒤 공백/개행 제거). 실패 stderr를 함께 노출.
121
+ except subprocess.TimeoutExpired as e:
122
+ raise AgentCallError(
123
+ f"{self.target_name}: ngpt timed out",
124
+ reason="timeout",
125
+ target=self.target_name,
126
+ ) from e
127
+ except OSError as e: # 바이너리 없음·실행 불가
128
+ raise AgentCallError(
129
+ f"{self.target_name}: cannot run ngpt ({type(e).__name__})",
130
+ reason="transport_error",
131
+ target=self.target_name,
132
+ ) from e
133
+
134
+ if result.returncode != 0:
135
+ raise AgentCallError(
136
+ f"{self.target_name}: ngpt exited {result.returncode}",
137
+ reason="nonzero_exit",
138
+ target=self.target_name,
139
+ )
121
140
  out = (result.stdout or "").strip()
122
141
  if not out:
123
- return f"[ngpt no output] {(result.stderr or '').strip()}"
142
+ raise AgentCallError(
143
+ f"{self.target_name}: ngpt produced no output",
144
+ reason="no_output",
145
+ target=self.target_name,
146
+ )
124
147
  return out
125
148
 
126
149
  def get_target_name(self) -> str: