agent-vetter 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_vetter-0.2.0/.github/workflows/ci.yml +32 -0
- agent_vetter-0.2.0/.gitignore +9 -0
- agent_vetter-0.2.0/CHANGELOG.md +34 -0
- agent_vetter-0.2.0/LICENSE +21 -0
- agent_vetter-0.2.0/PKG-INFO +164 -0
- agent_vetter-0.2.0/README.md +146 -0
- agent_vetter-0.2.0/action/action.yml +46 -0
- agent_vetter-0.2.0/agentscan/__init__.py +1 -0
- agent_vetter-0.2.0/agentscan/cli.py +147 -0
- agent_vetter-0.2.0/agentscan/llm.py +110 -0
- agent_vetter-0.2.0/agentscan/models.py +47 -0
- agent_vetter-0.2.0/agentscan/rules/__init__.py +4 -0
- agent_vetter-0.2.0/agentscan/rules/dangerous.py +102 -0
- agent_vetter-0.2.0/agentscan/rules/exfil.py +73 -0
- agent_vetter-0.2.0/agentscan/rules/injection.py +180 -0
- agent_vetter-0.2.0/agentscan/rules/privilege.py +61 -0
- agent_vetter-0.2.0/agentscan/rules/secrets.py +122 -0
- agent_vetter-0.2.0/agentscan/rules/windows.py +126 -0
- agent_vetter-0.2.0/agentscan/scanner.py +116 -0
- agent_vetter-0.2.0/assets/demo.gif +0 -0
- agent_vetter-0.2.0/pyproject.toml +31 -0
- agent_vetter-0.2.0/tests/fixtures/agent-docs/AGENTS.md +5 -0
- agent_vetter-0.2.0/tests/fixtures/clean-skill/SKILL.md +11 -0
- agent_vetter-0.2.0/tests/fixtures/clean-skill/scripts/validate.py +19 -0
- agent_vetter-0.2.0/tests/fixtures/malicious-mcp/.mcp.json +8 -0
- agent_vetter-0.2.0/tests/fixtures/malicious-mcp/server.py +12 -0
- agent_vetter-0.2.0/tests/fixtures/malicious-skill/SKILL.md +19 -0
- agent_vetter-0.2.0/tests/fixtures/malicious-skill/scripts/deploy.py +13 -0
- agent_vetter-0.2.0/tests/fixtures/malicious-skill/scripts/setup.sh +6 -0
- agent_vetter-0.2.0/tests/fixtures/windows-skill/SKILL.md +9 -0
- agent_vetter-0.2.0/tests/fixtures/windows-skill/scripts/install.ps1 +10 -0
- agent_vetter-0.2.0/tests/fixtures/windows-skill/scripts/setup.bat +6 -0
- agent_vetter-0.2.0/tests/test_cli.py +48 -0
- agent_vetter-0.2.0/tests/test_llm.py +115 -0
- agent_vetter-0.2.0/tests/test_scanner.py +80 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
- uses: actions/setup-python@v5
|
|
14
|
+
with:
|
|
15
|
+
python-version: "3.11"
|
|
16
|
+
- name: Install
|
|
17
|
+
run: python -m pip install -e . pytest
|
|
18
|
+
- name: Run tests
|
|
19
|
+
run: pytest -q
|
|
20
|
+
|
|
21
|
+
scan-clean-skill:
|
|
22
|
+
runs-on: ubuntu-latest
|
|
23
|
+
steps:
|
|
24
|
+
- uses: actions/checkout@v4
|
|
25
|
+
- uses: actions/setup-python@v5
|
|
26
|
+
with:
|
|
27
|
+
python-version: "3.11"
|
|
28
|
+
- name: Scan a clean skill (must pass)
|
|
29
|
+
uses: ./action
|
|
30
|
+
with:
|
|
31
|
+
path: tests/fixtures/clean-skill
|
|
32
|
+
fail-on: high
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
5
|
+
|
|
6
|
+
## [0.2.0] - 2026-11-10
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
- **Windows rule pack** (`windows/*`): `iex (iwr …)` pipe-to-PowerShell,
|
|
10
|
+
`Invoke-Expression`, encoded commands, certutil / mshta / bitsadmin
|
|
11
|
+
downloads, `Set-ExecutionPolicy Bypass`, schtasks and registry-Run
|
|
12
|
+
persistence, broad recursive deletes, `format d:`. Scans `.ps1`,
|
|
13
|
+
`.psm1`, `.psd1`, `.bat`, `.cmd`.
|
|
14
|
+
- **LLM deep-scan mode** (opt-in `--llm`): one chat-completion call that
|
|
15
|
+
explains each deterministic finding and flags semantic risks (multi-stage
|
|
16
|
+
injection chains, social engineering, tool-allowlist contradictions).
|
|
17
|
+
Works with any OpenAI-compatible endpoint (`OPENAI_BASE_URL`,
|
|
18
|
+
`AGENTSCAN_LLM_MODEL`). Advisory by default; `--llm-fail` lets LLM
|
|
19
|
+
findings at HIGH+ affect the exit code.
|
|
20
|
+
- **GitHub Action** (`action/action.yml`): composite action that installs
|
|
21
|
+
and runs agentscan; inputs `path`, `fail-on`, `version`, `json`.
|
|
22
|
+
- **CI workflow**: test suite + action scanning a clean skill fixture on
|
|
23
|
+
every push/PR.
|
|
24
|
+
- `--version` flag.
|
|
25
|
+
|
|
26
|
+
## [0.1.0] - 2026-11-09
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
- Initial release: static security scanner for AI agent assets.
|
|
30
|
+
- Rule families: `secret/*`, `injection/*` (EN + VN), `dangerous/*`,
|
|
31
|
+
`exfil/*`, `privilege/*`.
|
|
32
|
+
- CLI with `--json`, `--fail-on`, `--no-color`; exit code 1 on findings
|
|
33
|
+
at or above the threshold (default: high).
|
|
34
|
+
- Fixture-based unit tests for every rule.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 agentscan contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: agent-vetter
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Static security scanner for AI agent skill packs, MCP servers, and agent configs
|
|
5
|
+
Author: sinhduc01
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: ai-agents,audit,claude-code,mcp,prompt-injection,scanner,security,skill
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Security
|
|
15
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
<p align="center">
|
|
20
|
+
<img src="assets/demo.gif" alt="agentscan demo" width="880">
|
|
21
|
+
</p>
|
|
22
|
+
|
|
23
|
+
# agentscan
|
|
24
|
+
|
|
25
|
+
**Snyk for AI agents.** A static security scanner for AI agent assets: skill
|
|
26
|
+
packs (Claude Code / Codex / Cursor), MCP server code, and agent instruction
|
|
27
|
+
docs (`AGENTS.md`, `CLAUDE.md`, `.cursorrules`, `.mcp.json`, `.env`) —
|
|
28
|
+
including Windows skill scripts (`.ps1` / `.bat`).
|
|
29
|
+
|
|
30
|
+
Everyone is now installing skills into their agents. A skill is third-party
|
|
31
|
+
code *and* instructions. A malicious one ships prompt injection, hardcoded
|
|
32
|
+
keys, `curl | sh`, and exfiltration to a raw IP — and your agent runs it with
|
|
33
|
+
your permissions.
|
|
34
|
+
|
|
35
|
+
agentscan catches that **before** the skill is loaded: fast, offline,
|
|
36
|
+
deterministic rules — zero dependencies — and every rule is unit-tested
|
|
37
|
+
against a fixture corpus. An optional LLM pass adds semantic depth
|
|
38
|
+
(§ LLM deep-scan).
|
|
39
|
+
|
|
40
|
+
## What it detects
|
|
41
|
+
|
|
42
|
+
| Family | Severity | Examples |
|
|
43
|
+
|---|---|---|
|
|
44
|
+
| `secret/*` | CRITICAL | AWS access keys, OpenAI / Anthropic / GitHub / Google / Slack / ServiceNow tokens, PEM private keys, GCP service-account JSON, JWTs |
|
|
45
|
+
| `injection/*` | CRITICAL / HIGH | "ignore all previous instructions", fetch-and-follow remote commands (English **and Vietnamese**), zero-width / invisible text, hidden HTML comments, white-on-white spans, base64 blobs, pastebin / ngrok URLs |
|
|
46
|
+
| `dangerous/*` | CRITICAL | `curl \| sh`, base64-decode-then-execute, `eval` of network-fetched content, `rm -rf ~` / `--no-preserve-root`, `chmod 777`, crontab persistence |
|
|
47
|
+
| `windows/*` | CRITICAL / HIGH / MEDIUM | `iex (iwr …)`, `Invoke-Expression`, encoded commands (`-enc`), certutil / mshta / bitsadmin downloads, `Set-ExecutionPolicy Bypass`, schtasks / registry-Run persistence, `format d:` |
|
|
48
|
+
| `exfil/*` | MEDIUM | network sends in files that read env vars / local files, raw-IP URLs, abuse-prone TLDs (`.top` / `.xyz` / …) |
|
|
49
|
+
| `privilege/*` | HIGH / MEDIUM | `--dangerously-skip-permissions` in agent docs, "never ask for confirmation" (EN + VN), `allowed-tools: Bash(*)` in SKILL.md frontmatter |
|
|
50
|
+
|
|
51
|
+
The rule set is deliberately bilingual (EN/VN) and aimed at the
|
|
52
|
+
agentic-skill attack surface, not generic secret scanning.
|
|
53
|
+
|
|
54
|
+
## Install
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install agent-vetter
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Python 3.10+. No other dependencies. Install from git instead:
|
|
61
|
+
`pip install git+https://github.com/sinhduc01/agentscan.git`.
|
|
62
|
+
```bash
|
|
63
|
+
agentscan ./path/to/skill-pack # human-readable table
|
|
64
|
+
agentscan . --json # machine-readable, for CI
|
|
65
|
+
agentscan . --fail-on medium # exit 1 on >= medium (default: high)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Exit code `1` when a finding at or above the threshold is found, so it plugs
|
|
69
|
+
directly into CI, pre-install hooks, or a skill registry gate:
|
|
70
|
+
|
|
71
|
+
```yaml
|
|
72
|
+
- name: Scan agent assets
|
|
73
|
+
run: agentscan skills/ .mcp.json AGENTS.md --fail-on medium
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Example finding:
|
|
77
|
+
|
|
78
|
+
```
|
|
79
|
+
CRITICAL secret/aws-access-key scripts/deploy.py:4
|
|
80
|
+
AWS access key ID
|
|
81
|
+
> AWS_KEY = "AKIAQ7…01
|
|
82
|
+
fix: Rotate the key in IAM; move credentials to environment
|
|
83
|
+
variables or a secret manager.
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## LLM deep-scan (opt-in)
|
|
87
|
+
|
|
88
|
+
The deterministic core is offline and free by design. `--llm` adds an
|
|
89
|
+
advisory second pass that (1) explains *why* each finding is dangerous and
|
|
90
|
+
(2) flags semantic risks patterns can't see — multi-stage injection chains,
|
|
91
|
+
social engineering, instructions that weaken safety controls:
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
export OPENAI_API_KEY=… # any OpenAI-compatible endpoint
|
|
95
|
+
export OPENAI_BASE_URL=http://… # optional — self-hosted vLLM works
|
|
96
|
+
export AGENTSCAN_LLM_MODEL=qwen # optional — default gpt-4o-mini
|
|
97
|
+
|
|
98
|
+
agentscan ./skill --llm --json
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
- LLM results are **advisory by default**: the exit code is still driven
|
|
102
|
+
only by deterministic findings. Add `--llm-fail` to let LLM findings at
|
|
103
|
+
HIGH or above fail the scan.
|
|
104
|
+
- A missing API key or unreachable endpoint prints a warning and never
|
|
105
|
+
breaks the scan.
|
|
106
|
+
- One chat-completion call: the scanner findings plus up to 8 instruction
|
|
107
|
+
documents (12 KB each) are sent to the model; the reply is strict JSON.
|
|
108
|
+
- LLM severity labels are non-deterministic across runs — treat `--llm-fail`
|
|
109
|
+
as a review signal, not a hard gate.
|
|
110
|
+
|
|
111
|
+
## GitHub Action
|
|
112
|
+
|
|
113
|
+
```yaml
|
|
114
|
+
- uses: sinhduc01/agentscan@main # composite action, installs the tool itself
|
|
115
|
+
with:
|
|
116
|
+
path: skills/
|
|
117
|
+
fail-on: high
|
|
118
|
+
# version: 0.1.0 # pin a PyPI release instead of git@main
|
|
119
|
+
# json: true # machine-readable output
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
The repo's own CI runs the test suite and scans a clean skill fixture with
|
|
123
|
+
the action on every push/PR.
|
|
124
|
+
|
|
125
|
+
## Why deterministic rules first?
|
|
126
|
+
|
|
127
|
+
- **Testable**: every rule has a fixture; the suite runs in 0.1 s.
|
|
128
|
+
- **Offline & free**: a pre-install security gate that needs an API key or a
|
|
129
|
+
network call is not a gate.
|
|
130
|
+
- **Reproducible**: same input, same findings, no model drift.
|
|
131
|
+
|
|
132
|
+
The LLM pass is therefore opt-in and advisory — it improves triage, it does
|
|
133
|
+
not change what "safe" means.
|
|
134
|
+
|
|
135
|
+
## Limitations (read before relying on this)
|
|
136
|
+
|
|
137
|
+
- Static analysis: obfuscated payloads (decode → exec in two stages across
|
|
138
|
+
files) and purely semantic malice are out of scope for the deterministic
|
|
139
|
+
core — the LLM pass narrows, but does not close, that gap.
|
|
140
|
+
- Heuristics produce false positives: a legitimate telemetry POST that reads
|
|
141
|
+
an env var will be flagged `exfil/possible-exfiltration`.
|
|
142
|
+
- This is a first-pass filter, not a substitute for reviewing a skill's code.
|
|
143
|
+
|
|
144
|
+
## Development
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
pip install -e .
|
|
148
|
+
pytest
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Layout: `agentscan/rules/` holds one module per rule family
|
|
152
|
+
(`secrets`, `injection`, `dangerous`, `exfil`, `privilege`, `windows`);
|
|
153
|
+
adding a rule is one tuple in a module's `_PATTERNS` list plus a fixture
|
|
154
|
+
line in `tests/fixtures/`. `agentscan/llm.py` holds the advisory LLM pass.
|
|
155
|
+
|
|
156
|
+
## Roadmap
|
|
157
|
+
|
|
158
|
+
- pre-commit hook
|
|
159
|
+
- more targets: Rust/Go MCP servers, `.mcp.json` tool-grant analysis
|
|
160
|
+
- ignore/allowlist file for repository-wide scans
|
|
161
|
+
|
|
162
|
+
## License
|
|
163
|
+
|
|
164
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="assets/demo.gif" alt="agentscan demo" width="880">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
# agentscan
|
|
6
|
+
|
|
7
|
+
**Snyk for AI agents.** A static security scanner for AI agent assets: skill
|
|
8
|
+
packs (Claude Code / Codex / Cursor), MCP server code, and agent instruction
|
|
9
|
+
docs (`AGENTS.md`, `CLAUDE.md`, `.cursorrules`, `.mcp.json`, `.env`) —
|
|
10
|
+
including Windows skill scripts (`.ps1` / `.bat`).
|
|
11
|
+
|
|
12
|
+
Everyone is now installing skills into their agents. A skill is third-party
|
|
13
|
+
code *and* instructions. A malicious one ships prompt injection, hardcoded
|
|
14
|
+
keys, `curl | sh`, and exfiltration to a raw IP — and your agent runs it with
|
|
15
|
+
your permissions.
|
|
16
|
+
|
|
17
|
+
agentscan catches that **before** the skill is loaded: fast, offline,
|
|
18
|
+
deterministic rules — zero dependencies — and every rule is unit-tested
|
|
19
|
+
against a fixture corpus. An optional LLM pass adds semantic depth
|
|
20
|
+
(§ LLM deep-scan).
|
|
21
|
+
|
|
22
|
+
## What it detects
|
|
23
|
+
|
|
24
|
+
| Family | Severity | Examples |
|
|
25
|
+
|---|---|---|
|
|
26
|
+
| `secret/*` | CRITICAL | AWS access keys, OpenAI / Anthropic / GitHub / Google / Slack / ServiceNow tokens, PEM private keys, GCP service-account JSON, JWTs |
|
|
27
|
+
| `injection/*` | CRITICAL / HIGH | "ignore all previous instructions", fetch-and-follow remote commands (English **and Vietnamese**), zero-width / invisible text, hidden HTML comments, white-on-white spans, base64 blobs, pastebin / ngrok URLs |
|
|
28
|
+
| `dangerous/*` | CRITICAL | `curl \| sh`, base64-decode-then-execute, `eval` of network-fetched content, `rm -rf ~` / `--no-preserve-root`, `chmod 777`, crontab persistence |
|
|
29
|
+
| `windows/*` | CRITICAL / HIGH / MEDIUM | `iex (iwr …)`, `Invoke-Expression`, encoded commands (`-enc`), certutil / mshta / bitsadmin downloads, `Set-ExecutionPolicy Bypass`, schtasks / registry-Run persistence, `format d:` |
|
|
30
|
+
| `exfil/*` | MEDIUM | network sends in files that read env vars / local files, raw-IP URLs, abuse-prone TLDs (`.top` / `.xyz` / …) |
|
|
31
|
+
| `privilege/*` | HIGH / MEDIUM | `--dangerously-skip-permissions` in agent docs, "never ask for confirmation" (EN + VN), `allowed-tools: Bash(*)` in SKILL.md frontmatter |
|
|
32
|
+
|
|
33
|
+
The rule set is deliberately bilingual (EN/VN) and aimed at the
|
|
34
|
+
agentic-skill attack surface, not generic secret scanning.
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install agent-vetter
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Python 3.10+. No other dependencies. Install from git instead:
|
|
43
|
+
`pip install git+https://github.com/sinhduc01/agentscan.git`.
|
|
44
|
+
```bash
|
|
45
|
+
agentscan ./path/to/skill-pack # human-readable table
|
|
46
|
+
agentscan . --json # machine-readable, for CI
|
|
47
|
+
agentscan . --fail-on medium # exit 1 on >= medium (default: high)
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Exit code `1` when a finding at or above the threshold is found, so it plugs
|
|
51
|
+
directly into CI, pre-install hooks, or a skill registry gate:
|
|
52
|
+
|
|
53
|
+
```yaml
|
|
54
|
+
- name: Scan agent assets
|
|
55
|
+
run: agentscan skills/ .mcp.json AGENTS.md --fail-on medium
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Example finding:
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
CRITICAL secret/aws-access-key scripts/deploy.py:4
|
|
62
|
+
AWS access key ID
|
|
63
|
+
> AWS_KEY = "AKIAQ7…01
|
|
64
|
+
fix: Rotate the key in IAM; move credentials to environment
|
|
65
|
+
variables or a secret manager.
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## LLM deep-scan (opt-in)
|
|
69
|
+
|
|
70
|
+
The deterministic core is offline and free by design. `--llm` adds an
|
|
71
|
+
advisory second pass that (1) explains *why* each finding is dangerous and
|
|
72
|
+
(2) flags semantic risks patterns can't see — multi-stage injection chains,
|
|
73
|
+
social engineering, instructions that weaken safety controls:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
export OPENAI_API_KEY=… # any OpenAI-compatible endpoint
|
|
77
|
+
export OPENAI_BASE_URL=http://… # optional — self-hosted vLLM works
|
|
78
|
+
export AGENTSCAN_LLM_MODEL=qwen # optional — default gpt-4o-mini
|
|
79
|
+
|
|
80
|
+
agentscan ./skill --llm --json
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
- LLM results are **advisory by default**: the exit code is still driven
|
|
84
|
+
only by deterministic findings. Add `--llm-fail` to let LLM findings at
|
|
85
|
+
HIGH or above fail the scan.
|
|
86
|
+
- A missing API key or unreachable endpoint prints a warning and never
|
|
87
|
+
breaks the scan.
|
|
88
|
+
- One chat-completion call: the scanner findings plus up to 8 instruction
|
|
89
|
+
documents (12 KB each) are sent to the model; the reply is strict JSON.
|
|
90
|
+
- LLM severity labels are non-deterministic across runs — treat `--llm-fail`
|
|
91
|
+
as a review signal, not a hard gate.
|
|
92
|
+
|
|
93
|
+
## GitHub Action
|
|
94
|
+
|
|
95
|
+
```yaml
|
|
96
|
+
- uses: sinhduc01/agentscan@main # composite action, installs the tool itself
|
|
97
|
+
with:
|
|
98
|
+
path: skills/
|
|
99
|
+
fail-on: high
|
|
100
|
+
# version: 0.1.0 # pin a PyPI release instead of git@main
|
|
101
|
+
# json: true # machine-readable output
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
The repo's own CI runs the test suite and scans a clean skill fixture with
|
|
105
|
+
the action on every push/PR.
|
|
106
|
+
|
|
107
|
+
## Why deterministic rules first?
|
|
108
|
+
|
|
109
|
+
- **Testable**: every rule has a fixture; the suite runs in 0.1 s.
|
|
110
|
+
- **Offline & free**: a pre-install security gate that needs an API key or a
|
|
111
|
+
network call is not a gate.
|
|
112
|
+
- **Reproducible**: same input, same findings, no model drift.
|
|
113
|
+
|
|
114
|
+
The LLM pass is therefore opt-in and advisory — it improves triage, it does
|
|
115
|
+
not change what "safe" means.
|
|
116
|
+
|
|
117
|
+
## Limitations (read before relying on this)
|
|
118
|
+
|
|
119
|
+
- Static analysis: obfuscated payloads (decode → exec in two stages across
|
|
120
|
+
files) and purely semantic malice are out of scope for the deterministic
|
|
121
|
+
core — the LLM pass narrows, but does not close, that gap.
|
|
122
|
+
- Heuristics produce false positives: a legitimate telemetry POST that reads
|
|
123
|
+
an env var will be flagged `exfil/possible-exfiltration`.
|
|
124
|
+
- This is a first-pass filter, not a substitute for reviewing a skill's code.
|
|
125
|
+
|
|
126
|
+
## Development
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
pip install -e .
|
|
130
|
+
pytest
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Layout: `agentscan/rules/` holds one module per rule family
|
|
134
|
+
(`secrets`, `injection`, `dangerous`, `exfil`, `privilege`, `windows`);
|
|
135
|
+
adding a rule is one tuple in a module's `_PATTERNS` list plus a fixture
|
|
136
|
+
line in `tests/fixtures/`. `agentscan/llm.py` holds the advisory LLM pass.
|
|
137
|
+
|
|
138
|
+
## Roadmap
|
|
139
|
+
|
|
140
|
+
- pre-commit hook
|
|
141
|
+
- more targets: Rust/Go MCP servers, `.mcp.json` tool-grant analysis
|
|
142
|
+
- ignore/allowlist file for repository-wide scans
|
|
143
|
+
|
|
144
|
+
## License
|
|
145
|
+
|
|
146
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
name: agentscan
|
|
2
|
+
description: >-
|
|
3
|
+
Scan AI agent assets (skill packs, MCP servers, agent configs) for hardcoded
|
|
4
|
+
secrets, prompt injection, dangerous commands, and data exfiltration.
|
|
5
|
+
|
|
6
|
+
inputs:
|
|
7
|
+
path:
|
|
8
|
+
description: "File or directory to scan (default: current directory)"
|
|
9
|
+
required: false
|
|
10
|
+
default: "."
|
|
11
|
+
fail-on:
|
|
12
|
+
description: Fail when a finding at or above this severity is present (critical|high|medium|low|info)
|
|
13
|
+
required: false
|
|
14
|
+
default: "high"
|
|
15
|
+
version:
|
|
16
|
+
description: agentscan version to install (e.g. 0.1.0) or "git" for the latest commit on main
|
|
17
|
+
required: false
|
|
18
|
+
default: "git"
|
|
19
|
+
json:
|
|
20
|
+
description: Emit machine-readable JSON output
|
|
21
|
+
required: false
|
|
22
|
+
default: "false"
|
|
23
|
+
runs:
|
|
24
|
+
using: composite
|
|
25
|
+
steps:
|
|
26
|
+
- name: Install agentscan
|
|
27
|
+
shell: bash
|
|
28
|
+
env:
|
|
29
|
+
AGENTSCAN_VERSION: ${{ inputs.version }}
|
|
30
|
+
run: |
|
|
31
|
+
if [ "$AGENTSCAN_VERSION" = "git" ]; then
|
|
32
|
+
python -m pip install --quiet "git+https://github.com/sinhduc01/agentscan.git@main"
|
|
33
|
+
else
|
|
34
|
+
python -m pip install --quiet "agent-vetter==$AGENTSCAN_VERSION"
|
|
35
|
+
fi
|
|
36
|
+
|
|
37
|
+
- name: Run agentscan
|
|
38
|
+
shell: bash
|
|
39
|
+
env:
|
|
40
|
+
AGENTSCAN_JSON: ${{ inputs.json }}
|
|
41
|
+
run: |
|
|
42
|
+
if [ "$AGENTSCAN_JSON" = "true" ]; then
|
|
43
|
+
agentscan "${{ inputs.path }}" --fail-on "${{ inputs.fail-on }}" --json --no-color
|
|
44
|
+
else
|
|
45
|
+
agentscan "${{ inputs.path }}" --fail-on "${{ inputs.fail-on }}" --no-color
|
|
46
|
+
fi
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from . import __version__, llm
|
|
10
|
+
from .models import Finding, Severity
|
|
11
|
+
from .scanner import scan, scan_with_docs
|
|
12
|
+
|
|
13
|
+
RESET = "\x1b[0m"
|
|
14
|
+
_SEV = {s.name.lower(): s for s in Severity}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _render(findings: list[Finding], scanned: int, root: Path, no_color: bool,
|
|
18
|
+
llm_report: dict | None = None, model: str = "") -> str:
|
|
19
|
+
sev = lambda s: "" if no_color else s.color # noqa: E731
|
|
20
|
+
out = [f"agentscan {__version__} — {scanned} files scanned in {root}"]
|
|
21
|
+
if not findings:
|
|
22
|
+
out.append(" No findings — PASS")
|
|
23
|
+
else:
|
|
24
|
+
for f in findings:
|
|
25
|
+
out.append(
|
|
26
|
+
f" {sev(f.severity)}{f.severity.name:<8}{RESET} "
|
|
27
|
+
f"{f.rule_id:<36} {f.path}:{f.line}"
|
|
28
|
+
)
|
|
29
|
+
out.append(f" {'':<8} {f.title}")
|
|
30
|
+
if f.snippet:
|
|
31
|
+
out.append(f" {'':<8} > {f.snippet[:140]}")
|
|
32
|
+
if f.fix:
|
|
33
|
+
out.append(f" {'':<8} fix: {f.fix}")
|
|
34
|
+
out.append("")
|
|
35
|
+
counts: dict[str, int] = {}
|
|
36
|
+
for f in findings:
|
|
37
|
+
counts[f.severity.name] = counts.get(f.severity.name, 0) + 1
|
|
38
|
+
summary = ", ".join(
|
|
39
|
+
f"{n} {k.lower()}" for k, n in sorted(counts.items(), key=lambda kv: -Severity[kv[0]])
|
|
40
|
+
)
|
|
41
|
+
out.append(f"{len(findings)} findings ({summary}) — FAIL")
|
|
42
|
+
if llm_report is not None:
|
|
43
|
+
out.append("")
|
|
44
|
+
out.append(f" LLM review ({model}):")
|
|
45
|
+
for e in llm_report.get("explanations", []):
|
|
46
|
+
out.append(f" {e.get('rule', '?')} — risk: {e.get('risk', '')}")
|
|
47
|
+
rec = e.get("recommendation", "")
|
|
48
|
+
if rec:
|
|
49
|
+
out.append(f" {'':<4}recommendation: {rec}")
|
|
50
|
+
for x in llm_report.get("extra_findings", []):
|
|
51
|
+
sev_name = str(x.get("severity", "?")).upper()
|
|
52
|
+
out.append(
|
|
53
|
+
f" LLM-{sev_name:<5} {x.get('title', '?')}"
|
|
54
|
+
f" {x.get('path', '?')}:{x.get('line', 0)}"
|
|
55
|
+
)
|
|
56
|
+
if x.get("message"):
|
|
57
|
+
out.append(f" {'':<4}{x['message']}")
|
|
58
|
+
return "\n".join(out)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def main(argv: list[str] | None = None) -> int:
|
|
62
|
+
ap = argparse.ArgumentParser(
|
|
63
|
+
prog="agentscan",
|
|
64
|
+
description="Static security scanner for AI agent skill packs, "
|
|
65
|
+
"MCP servers, and agent configs.",
|
|
66
|
+
)
|
|
67
|
+
ap.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
68
|
+
ap.add_argument("path", nargs="?", default=".",
|
|
69
|
+
help="file or directory to scan (default: current directory)")
|
|
70
|
+
ap.add_argument("--json", action="store_true", help="machine-readable output")
|
|
71
|
+
ap.add_argument("--fail-on", default="high",
|
|
72
|
+
choices=["critical", "high", "medium", "low", "info"],
|
|
73
|
+
help="exit 1 when a finding at or above this severity "
|
|
74
|
+
"is present (default: high)")
|
|
75
|
+
ap.add_argument("--no-color", action="store_true")
|
|
76
|
+
ap.add_argument("--llm", action="store_true",
|
|
77
|
+
help="opt-in LLM deep review: explains findings and flags "
|
|
78
|
+
"semantic risks (needs OPENAI_API_KEY; OPENAI_BASE_URL "
|
|
79
|
+
"and AGENTSCAN_LLM_MODEL are honored)")
|
|
80
|
+
ap.add_argument("--llm-model",
|
|
81
|
+
default=os.environ.get("AGENTSCAN_LLM_MODEL", llm.DEFAULT_MODEL),
|
|
82
|
+
help="model name for the LLM review (default: %(default)s)")
|
|
83
|
+
ap.add_argument("--llm-fail", action="store_true",
|
|
84
|
+
help="let LLM extra findings at HIGH or above affect the exit code")
|
|
85
|
+
args = ap.parse_args(argv)
|
|
86
|
+
threshold = Severity[args.fail_on.upper()]
|
|
87
|
+
|
|
88
|
+
root = Path(args.path)
|
|
89
|
+
if not root.exists():
|
|
90
|
+
ap.error(f"path not found: {root}")
|
|
91
|
+
|
|
92
|
+
llm_report: dict | None = None
|
|
93
|
+
docs: list[tuple[str, str]] = []
|
|
94
|
+
if args.llm:
|
|
95
|
+
scanned, findings, docs = scan_with_docs(root)
|
|
96
|
+
else:
|
|
97
|
+
scanned, findings = scan(root)
|
|
98
|
+
findings.sort(key=Finding.sort_key)
|
|
99
|
+
|
|
100
|
+
failed = any(f.severity >= threshold for f in findings)
|
|
101
|
+
|
|
102
|
+
if args.llm:
|
|
103
|
+
api_key = os.environ.get("OPENAI_API_KEY", "")
|
|
104
|
+
base_url = os.environ.get("OPENAI_BASE_URL", llm.DEFAULT_BASE_URL)
|
|
105
|
+
if not api_key:
|
|
106
|
+
print(" warning: --llm set but OPENAI_API_KEY is not set; "
|
|
107
|
+
"skipping LLM review", file=sys.stderr)
|
|
108
|
+
else:
|
|
109
|
+
try:
|
|
110
|
+
llm_report = llm.llm_review(findings, docs, args.llm_model,
|
|
111
|
+
api_key, base_url)
|
|
112
|
+
except llm.LLMError as e:
|
|
113
|
+
print(f" warning: LLM review unavailable: {e}", file=sys.stderr)
|
|
114
|
+
if llm_report and args.llm_fail:
|
|
115
|
+
for x in llm_report.get("extra_findings", []):
|
|
116
|
+
if _SEV.get(str(x.get("severity", "")).lower(), Severity.INFO) >= Severity.HIGH:
|
|
117
|
+
failed = True
|
|
118
|
+
|
|
119
|
+
if args.json:
|
|
120
|
+
payload = {
|
|
121
|
+
"scanned_files": scanned,
|
|
122
|
+
"result": "fail" if failed else "pass",
|
|
123
|
+
"findings": [
|
|
124
|
+
{
|
|
125
|
+
"rule": f.rule_id,
|
|
126
|
+
"severity": f.severity.name,
|
|
127
|
+
"title": f.title,
|
|
128
|
+
"path": f.path,
|
|
129
|
+
"line": f.line,
|
|
130
|
+
"message": f.message,
|
|
131
|
+
"fix": f.fix,
|
|
132
|
+
}
|
|
133
|
+
for f in findings
|
|
134
|
+
],
|
|
135
|
+
}
|
|
136
|
+
if llm_report is not None:
|
|
137
|
+
payload["llm"] = {"model": args.llm_model, **llm_report}
|
|
138
|
+
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
|
139
|
+
else:
|
|
140
|
+
print(_render(findings, scanned, root, args.no_color,
|
|
141
|
+
llm_report, args.llm_model))
|
|
142
|
+
|
|
143
|
+
return 1 if failed else 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
if __name__ == "__main__":
|
|
147
|
+
sys.exit(main())
|