vulnhound 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vulnhound-0.1.0/.claude-plugin/marketplace.json +15 -0
- vulnhound-0.1.0/.claude-plugin/plugin.json +9 -0
- vulnhound-0.1.0/.github/workflows/ci.yml +18 -0
- vulnhound-0.1.0/.github/workflows/release.yml +41 -0
- vulnhound-0.1.0/.gitignore +9 -0
- vulnhound-0.1.0/CHANGELOG.md +6 -0
- vulnhound-0.1.0/LICENSE +21 -0
- vulnhound-0.1.0/PKG-INFO +97 -0
- vulnhound-0.1.0/README.md +79 -0
- vulnhound-0.1.0/README.tr.md +79 -0
- vulnhound-0.1.0/SECURITY.md +15 -0
- vulnhound-0.1.0/benchmarks/PERF_NOTES.md +196 -0
- vulnhound-0.1.0/benchmarks/cves.json +218 -0
- vulnhound-0.1.0/benchmarks/eval_fixtures.py +23 -0
- vulnhound-0.1.0/benchmarks/fetch_cves.py +131 -0
- vulnhound-0.1.0/benchmarks/index_speed.py +70 -0
- vulnhound-0.1.0/benchmarks/judge_prep.py +69 -0
- vulnhound-0.1.0/benchmarks/precision.py +101 -0
- vulnhound-0.1.0/benchmarks/precision_labels.json +414 -0
- vulnhound-0.1.0/benchmarks/real_projects.py +134 -0
- vulnhound-0.1.0/benchmarks/run_cves.py +152 -0
- vulnhound-0.1.0/benchmarks/slice_tokens.py +44 -0
- vulnhound-0.1.0/pyproject.toml +46 -0
- vulnhound-0.1.0/skills/vulnhound/SKILL.md +37 -0
- vulnhound-0.1.0/src/vulnhound/__init__.py +3 -0
- vulnhound-0.1.0/src/vulnhound/__main__.py +3 -0
- vulnhound-0.1.0/src/vulnhound/cache.py +70 -0
- vulnhound-0.1.0/src/vulnhound/cli.py +250 -0
- vulnhound-0.1.0/src/vulnhound/evaluate.py +77 -0
- vulnhound-0.1.0/src/vulnhound/group.py +75 -0
- vulnhound-0.1.0/src/vulnhound/index/__init__.py +5 -0
- vulnhound-0.1.0/src/vulnhound/index/core.py +151 -0
- vulnhound-0.1.0/src/vulnhound/index/extract.py +182 -0
- vulnhound-0.1.0/src/vulnhound/index/resolve.py +152 -0
- vulnhound-0.1.0/src/vulnhound/models.py +82 -0
- vulnhound-0.1.0/src/vulnhound/poc.py +23 -0
- vulnhound-0.1.0/src/vulnhound/project.py +70 -0
- vulnhound-0.1.0/src/vulnhound/prove.py +129 -0
- vulnhound-0.1.0/src/vulnhound/report.py +181 -0
- vulnhound-0.1.0/src/vulnhound/sandbox.py +592 -0
- vulnhound-0.1.0/src/vulnhound/slicer.py +420 -0
- vulnhound-0.1.0/src/vulnhound/store.py +64 -0
- vulnhound-0.1.0/src/vulnhound/taint/__init__.py +1 -0
- vulnhound-0.1.0/src/vulnhound/taint/engine.py +1883 -0
- vulnhound-0.1.0/src/vulnhound/taint/specs.py +987 -0
- vulnhound-0.1.0/tests/fixtures/README.md +11 -0
- vulnhound-0.1.0/tests/fixtures/safe/closure_global.py +28 -0
- vulnhound-0.1.0/tests/fixtures/safe/constant_urls.py +11 -0
- vulnhound-0.1.0/tests/fixtures/safe/db_lookups.py +64 -0
- vulnhound-0.1.0/tests/fixtures/safe/django_safe.py +10 -0
- vulnhound-0.1.0/tests/fixtures/safe/drf_tornado_safe.py +25 -0
- vulnhound-0.1.0/tests/fixtures/safe/fastapi_safe.py +17 -0
- vulnhound-0.1.0/tests/fixtures/safe/flask_cmd_quoted.py +17 -0
- vulnhound-0.1.0/tests/fixtures/safe/flask_files_safe.py +21 -0
- vulnhound-0.1.0/tests/fixtures/safe/flask_sql_param.py +21 -0
- vulnhound-0.1.0/tests/fixtures/safe/guards_complete.py +102 -0
- vulnhound-0.1.0/tests/fixtures/safe/guards_positive.py +84 -0
- vulnhound-0.1.0/tests/fixtures/safe/loop_carried.py +27 -0
- vulnhound-0.1.0/tests/fixtures/safe/object_fields.py +31 -0
- vulnhound-0.1.0/tests/fixtures/safe/pkg_helpers/__init__.py +0 -0
- vulnhound-0.1.0/tests/fixtures/safe/pkg_helpers/app.py +14 -0
- vulnhound-0.1.0/tests/fixtures/safe/pkg_helpers/util.py +11 -0
- vulnhound-0.1.0/tests/fixtures/safe/script_safe.py +11 -0
- vulnhound-0.1.0/tests/fixtures/safe/sqlalchemy_safe.py +16 -0
- vulnhound-0.1.0/tests/fixtures/safe/template_safe.py +15 -0
- vulnhound-0.1.0/tests/fixtures/safe/url_position.py +89 -0
- vulnhound-0.1.0/tests/fixtures/safe/yaml_safe.py +14 -0
- vulnhound-0.1.0/tests/fixtures/vuln/closure_global.py +23 -0
- vulnhound-0.1.0/tests/fixtures/vuln/db_lookups.py +38 -0
- vulnhound-0.1.0/tests/fixtures/vuln/django_views.py +24 -0
- vulnhound-0.1.0/tests/fixtures/vuln/drf_tornado.py +30 -0
- vulnhound-0.1.0/tests/fixtures/vuln/fastapi_app.py +25 -0
- vulnhound-0.1.0/tests/fixtures/vuln/flask_cmdi.py +20 -0
- vulnhound-0.1.0/tests/fixtures/vuln/flask_deser.py +17 -0
- vulnhound-0.1.0/tests/fixtures/vuln/flask_files.py +28 -0
- vulnhound-0.1.0/tests/fixtures/vuln/flask_sqli.py +23 -0
- vulnhound-0.1.0/tests/fixtures/vuln/flask_ssrf.py +11 -0
- vulnhound-0.1.0/tests/fixtures/vuln/flask_ssti_redirect.py +15 -0
- vulnhound-0.1.0/tests/fixtures/vuln/guards_incomplete.py +47 -0
- vulnhound-0.1.0/tests/fixtures/vuln/guards_positive.py +52 -0
- vulnhound-0.1.0/tests/fixtures/vuln/jinja_env.py +13 -0
- vulnhound-0.1.0/tests/fixtures/vuln/loop_carried.py +26 -0
- vulnhound-0.1.0/tests/fixtures/vuln/object_fields.py +28 -0
- vulnhound-0.1.0/tests/fixtures/vuln/path_helper_pathlib.py +12 -0
- vulnhound-0.1.0/tests/fixtures/vuln/pkg_helpers/__init__.py +0 -0
- vulnhound-0.1.0/tests/fixtures/vuln/pkg_helpers/app.py +14 -0
- vulnhound-0.1.0/tests/fixtures/vuln/pkg_helpers/util.py +9 -0
- vulnhound-0.1.0/tests/fixtures/vuln/pkg_inter/db.py +7 -0
- vulnhound-0.1.0/tests/fixtures/vuln/pkg_inter/routes.py +16 -0
- vulnhound-0.1.0/tests/fixtures/vuln/pkg_inter/shell.py +10 -0
- vulnhound-0.1.0/tests/fixtures/vuln/script_argv.py +12 -0
- vulnhound-0.1.0/tests/fixtures/vuln/sqlalchemy_built.py +11 -0
- vulnhound-0.1.0/tests/fixtures/vuln/url_position.py +57 -0
- vulnhound-0.1.0/tests/test_cli.py +83 -0
- vulnhound-0.1.0/tests/test_engine.py +1082 -0
- vulnhound-0.1.0/tests/test_eval.py +50 -0
- vulnhound-0.1.0/tests/test_group.py +49 -0
- vulnhound-0.1.0/tests/test_index.py +189 -0
- vulnhound-0.1.0/tests/test_plugin.py +36 -0
- vulnhound-0.1.0/tests/test_prove.py +183 -0
- vulnhound-0.1.0/tests/test_report.py +81 -0
- vulnhound-0.1.0/tests/test_sandbox.py +295 -0
- vulnhound-0.1.0/tests/test_slicer.py +187 -0
- vulnhound-0.1.0/tests/test_specs.py +144 -0
- vulnhound-0.1.0/tests/test_store.py +15 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "vulnhound",
|
|
3
|
+
"description": "vulnhound: token-thrifty vulnerability finder for Python projects",
|
|
4
|
+
"owner": {
|
|
5
|
+
"name": "bu-ra34"
|
|
6
|
+
},
|
|
7
|
+
"plugins": [
|
|
8
|
+
{
|
|
9
|
+
"name": "vulnhound",
|
|
10
|
+
"source": "./",
|
|
11
|
+
"description": "Token-thrifty vulnerability finder for Python: finds, proves, fixes without reading the whole repo.",
|
|
12
|
+
"version": "0.1.0"
|
|
13
|
+
}
|
|
14
|
+
]
|
|
15
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "vulnhound",
|
|
3
|
+
"description": "Token-thrifty vulnerability finder for Python: finds real vulnerabilities, proves them, fixes them.",
|
|
4
|
+
"version": "0.1.0",
|
|
5
|
+
"author": { "name": "bu-ra34" },
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"repository": "https://github.com/bu-ra34/vulnhound",
|
|
8
|
+
"keywords": ["security", "vulnerability", "python", "taint-analysis", "audit"]
|
|
9
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
pull_request:
|
|
5
|
+
jobs:
|
|
6
|
+
test:
|
|
7
|
+
runs-on: ubuntu-latest
|
|
8
|
+
strategy:
|
|
9
|
+
matrix:
|
|
10
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v4
|
|
13
|
+
- uses: actions/setup-python@v5
|
|
14
|
+
with:
|
|
15
|
+
python-version: ${{ matrix.python-version }}
|
|
16
|
+
- run: pip install -e '.[dev]'
|
|
17
|
+
- run: ruff check src tests
|
|
18
|
+
- run: pytest
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Publishes to PyPI with Trusted Publishing (OIDC) — no API token stored.
|
|
4
|
+
# One-time setup on PyPI: add a "pending publisher" for project `vulnhound`,
|
|
5
|
+
# owner `bu-ra34`, repo `vulnhound`, workflow `release.yml`, environment `pypi`.
|
|
6
|
+
# Then publish a GitHub Release (or push a v* tag) to trigger this.
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
release:
|
|
10
|
+
types: [published]
|
|
11
|
+
push:
|
|
12
|
+
tags: ["v*"]
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
build:
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.12"
|
|
22
|
+
- run: python -m pip install --upgrade build twine
|
|
23
|
+
- run: python -m build
|
|
24
|
+
- run: python -m twine check dist/*
|
|
25
|
+
- uses: actions/upload-artifact@v4
|
|
26
|
+
with:
|
|
27
|
+
name: dist
|
|
28
|
+
path: dist/
|
|
29
|
+
|
|
30
|
+
publish:
|
|
31
|
+
needs: build
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
environment: pypi
|
|
34
|
+
permissions:
|
|
35
|
+
id-token: write # required for Trusted Publishing
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/download-artifact@v4
|
|
38
|
+
with:
|
|
39
|
+
name: dist
|
|
40
|
+
path: dist/
|
|
41
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 - 2026-11-18
|
|
4
|
+
|
|
5
|
+
- Project skeleton: CLI (`version`, `estimate`), Claude Code plugin and skill, CI.
|
|
6
|
+
- `vulnhound prove <id> --poc FILE`: run a PoC in a sandbox (network off, resource-limited) and report PROVEN / SAFE / UNVERIFIED; `vulnhound.poc` helper (`proven`, `safe`, `canary_path`).
|
vulnhound-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 bu-ra34
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
vulnhound-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: vulnhound
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Token-thrifty vulnerability finder for Python projects: finds, proves, fixes.
|
|
5
|
+
Project-URL: Repository, https://github.com/bu-ra34/vulnhound
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: claude-code,security,static-analysis,taint-analysis,vulnerability
|
|
9
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Security
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Provides-Extra: dev
|
|
15
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
16
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# vulnhound
|
|
20
|
+
|
|
21
|
+
**Finds real vulnerabilities. Proves them. Fixes them. Without burning your tokens.**
|
|
22
|
+
|
|
23
|
+
> Status: pre-alpha. Only `estimate` and `version` work today; the rest is on the roadmap.
|
|
24
|
+
|
|
25
|
+
[Türkçe](README.tr.md)
|
|
26
|
+
|
|
27
|
+
## Why
|
|
28
|
+
|
|
29
|
+
Asking an LLM to security-review a codebase usually means feeding it the whole repo.
|
|
30
|
+
That is expensive, slow, and most of those tokens are code that has nothing to do with
|
|
31
|
+
any vulnerability. vulnhound avoids this: cheap static analysis finds candidate flows
|
|
32
|
+
first, so the LLM only reads a short list and a few small code slices.
|
|
33
|
+
|
|
34
|
+
## How it works
|
|
35
|
+
|
|
36
|
+
- **The core is the eyes.** A stdlib-only Python program builds an AST index and runs
|
|
37
|
+
taint analysis from untrusted sources to dangerous sinks. It never calls an LLM and
|
|
38
|
+
prints short JSON.
|
|
39
|
+
- **Claude is the brain.** Inside Claude Code, the bundled skill tells Claude to run the
|
|
40
|
+
CLI, read only that output plus small slices, judge each finding, write a PoC for the
|
|
41
|
+
likely-real ones, prove it in a sandbox, and patch it.
|
|
42
|
+
|
|
43
|
+
Zero runtime dependencies. Python 3.10+. MIT licensed.
|
|
44
|
+
|
|
45
|
+
## Install
|
|
46
|
+
|
|
47
|
+
As a Claude Code plugin:
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
/plugin marketplace add bu-ra34/vulnhound
|
|
51
|
+
/plugin install vulnhound@vulnhound
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The CLI itself (the skill needs it on your PATH):
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
pip install vulnhound
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## CLI
|
|
61
|
+
|
|
62
|
+
| Command | What it does |
|
|
63
|
+
|---|---|
|
|
64
|
+
| `vulnhound analyze [path] [--json] [--no-cache] [--min-severity S]` | Find candidate vulnerabilities. Exit 1 if any. Findings sharing a sink/helper are grouped (`--no-group` to list all); tests/examples/docs dirs and `test_*.py` are skipped unless `--include-tests`; `--include-local-sources` adds local/CLI inputs as sources. |
|
|
65
|
+
| `vulnhound slice <id>` | Minimal code slice for one finding. |
|
|
66
|
+
| `vulnhound prove <id> --poc FILE [--root DIR] [--backend auto\|docker\|unshare] [--timeout S] [--json] [--verbose]` | Run a PoC in a sandbox (no network, resource-limited). Prints PROVEN (exit 0), SAFE (1) or UNVERIFIED (3). |
|
|
67
|
+
| `vulnhound estimate [path] [--json]` | Rough token cost (bytes/4) of reading the whole repo. |
|
|
68
|
+
| `vulnhound report [--format json\|sarif\|markdown] [-o FILE]` | Render findings. |
|
|
69
|
+
| `vulnhound version` | Print version. |
|
|
70
|
+
|
|
71
|
+
Exit codes: 0 ok / no findings, 1 findings found, 2 usage error (e.g. unknown id), 3 UNVERIFIED (prove).
|
|
72
|
+
|
|
73
|
+
## Proving a finding
|
|
74
|
+
|
|
75
|
+
`vulnhound prove <id> --poc poc.py` runs a small standalone Python PoC in a sandbox (network off, memory/CPU/time limited, project mounted read-only) and reports a verdict. The PoC triggers the vulnerable code with a malicious input and calls `proven("detail")` from `vulnhound.poc` when the exploit works, or `safe("detail")` when it is blocked. `canary_path()` returns a secret file outside the project, for path-traversal PoCs to read. No marker and exit 0 means SAFE; sandbox missing, timeout or a crashing PoC means UNVERIFIED. The PoC never runs on the host.
|
|
76
|
+
|
|
77
|
+
Caveat: the sandbox needs docker or unshare. The PoC runs against your project, so third-party imports must be importable inside it: the `unshare` backend sees the host venv read-only, while the `docker` backend uses a clean python image, so PoCs needing framework imports may report UNVERIFIED there. Pick `--backend unshare` when the PoC imports your installed deps.
|
|
78
|
+
|
|
79
|
+
## Roadmap
|
|
80
|
+
|
|
81
|
+
- [x] M0: skeleton (CLI, plugin, skill, CI)
|
|
82
|
+
- [ ] M1: project index (AST, call graph, cache)
|
|
83
|
+
- [ ] M2: taint engine
|
|
84
|
+
- [ ] M3: framework support and `slice`
|
|
85
|
+
- [ ] M4: sandboxed proof (`prove`)
|
|
86
|
+
- [ ] M5: benchmark
|
|
87
|
+
- [ ] M6: release
|
|
88
|
+
|
|
89
|
+
No benchmark results are claimed yet; they will be published after M5.
|
|
90
|
+
|
|
91
|
+
## Security
|
|
92
|
+
|
|
93
|
+
Scan only code you own. See [SECURITY.md](SECURITY.md).
|
|
94
|
+
|
|
95
|
+
## License
|
|
96
|
+
|
|
97
|
+
MIT
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# vulnhound
|
|
2
|
+
|
|
3
|
+
**Finds real vulnerabilities. Proves them. Fixes them. Without burning your tokens.**
|
|
4
|
+
|
|
5
|
+
> Status: pre-alpha. Only `estimate` and `version` work today; the rest is on the roadmap.
|
|
6
|
+
|
|
7
|
+
[Türkçe](README.tr.md)
|
|
8
|
+
|
|
9
|
+
## Why
|
|
10
|
+
|
|
11
|
+
Asking an LLM to security-review a codebase usually means feeding it the whole repo.
|
|
12
|
+
That is expensive, slow, and most of those tokens are code that has nothing to do with
|
|
13
|
+
any vulnerability. vulnhound avoids this: cheap static analysis finds candidate flows
|
|
14
|
+
first, so the LLM only reads a short list and a few small code slices.
|
|
15
|
+
|
|
16
|
+
## How it works
|
|
17
|
+
|
|
18
|
+
- **The core is the eyes.** A stdlib-only Python program builds an AST index and runs
|
|
19
|
+
taint analysis from untrusted sources to dangerous sinks. It never calls an LLM and
|
|
20
|
+
prints short JSON.
|
|
21
|
+
- **Claude is the brain.** Inside Claude Code, the bundled skill tells Claude to run the
|
|
22
|
+
CLI, read only that output plus small slices, judge each finding, write a PoC for the
|
|
23
|
+
likely-real ones, prove it in a sandbox, and patch it.
|
|
24
|
+
|
|
25
|
+
Zero runtime dependencies. Python 3.10+. MIT licensed.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
As a Claude Code plugin:
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
/plugin marketplace add bu-ra34/vulnhound
|
|
33
|
+
/plugin install vulnhound@vulnhound
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The CLI itself (the skill needs it on your PATH):
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
pip install vulnhound
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## CLI
|
|
43
|
+
|
|
44
|
+
| Command | What it does |
|
|
45
|
+
|---|---|
|
|
46
|
+
| `vulnhound analyze [path] [--json] [--no-cache] [--min-severity S]` | Find candidate vulnerabilities. Exit 1 if any. Findings sharing a sink/helper are grouped (`--no-group` to list all); tests/examples/docs dirs and `test_*.py` are skipped unless `--include-tests`; `--include-local-sources` adds local/CLI inputs as sources. |
|
|
47
|
+
| `vulnhound slice <id>` | Minimal code slice for one finding. |
|
|
48
|
+
| `vulnhound prove <id> --poc FILE [--root DIR] [--backend auto\|docker\|unshare] [--timeout S] [--json] [--verbose]` | Run a PoC in a sandbox (no network, resource-limited). Prints PROVEN (exit 0), SAFE (1) or UNVERIFIED (3). |
|
|
49
|
+
| `vulnhound estimate [path] [--json]` | Rough token cost (bytes/4) of reading the whole repo. |
|
|
50
|
+
| `vulnhound report [--format json\|sarif\|markdown] [-o FILE]` | Render findings. |
|
|
51
|
+
| `vulnhound version` | Print version. |
|
|
52
|
+
|
|
53
|
+
Exit codes: 0 ok / no findings, 1 findings found, 2 usage error (e.g. unknown id), 3 UNVERIFIED (prove).
|
|
54
|
+
|
|
55
|
+
## Proving a finding
|
|
56
|
+
|
|
57
|
+
`vulnhound prove <id> --poc poc.py` runs a small standalone Python PoC in a sandbox (network off, memory/CPU/time limited, project mounted read-only) and reports a verdict. The PoC triggers the vulnerable code with a malicious input and calls `proven("detail")` from `vulnhound.poc` when the exploit works, or `safe("detail")` when it is blocked. `canary_path()` returns a secret file outside the project, for path-traversal PoCs to read. No marker and exit 0 means SAFE; sandbox missing, timeout or a crashing PoC means UNVERIFIED. The PoC never runs on the host.
|
|
58
|
+
|
|
59
|
+
Caveat: the sandbox needs docker or unshare. The PoC runs against your project, so third-party imports must be importable inside it: the `unshare` backend sees the host venv read-only, while the `docker` backend uses a clean python image, so PoCs needing framework imports may report UNVERIFIED there. Pick `--backend unshare` when the PoC imports your installed deps.
|
|
60
|
+
|
|
61
|
+
## Roadmap
|
|
62
|
+
|
|
63
|
+
- [x] M0: skeleton (CLI, plugin, skill, CI)
|
|
64
|
+
- [ ] M1: project index (AST, call graph, cache)
|
|
65
|
+
- [ ] M2: taint engine
|
|
66
|
+
- [ ] M3: framework support and `slice`
|
|
67
|
+
- [ ] M4: sandboxed proof (`prove`)
|
|
68
|
+
- [ ] M5: benchmark
|
|
69
|
+
- [ ] M6: release
|
|
70
|
+
|
|
71
|
+
No benchmark results are claimed yet; they will be published after M5.
|
|
72
|
+
|
|
73
|
+
## Security
|
|
74
|
+
|
|
75
|
+
Scan only code you own. See [SECURITY.md](SECURITY.md).
|
|
76
|
+
|
|
77
|
+
## License
|
|
78
|
+
|
|
79
|
+
MIT
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# vulnhound
|
|
2
|
+
|
|
3
|
+
**Gerçek zafiyetleri bulur. Kanıtlar. Düzeltir. Token'larını yakmadan.**
|
|
4
|
+
|
|
5
|
+
> Durum: pre-alpha. Şu an yalnızca `estimate` ve `version` çalışıyor; gerisi yol haritasında.
|
|
6
|
+
|
|
7
|
+
[English](README.md)
|
|
8
|
+
|
|
9
|
+
## Neden
|
|
10
|
+
|
|
11
|
+
Bir LLM'den kod tabanını güvenlik açısından incelemesini istemek genellikle tüm repoyu
|
|
12
|
+
ona okutmak demektir. Bu pahalı ve yavaştır; harcanan token'ların çoğu zafiyetle ilgisiz
|
|
13
|
+
koddur. vulnhound önce ucuz statik analizle aday akışları bulur; LLM yalnızca kısa bir
|
|
14
|
+
liste ve küçük kod parçaları okur.
|
|
15
|
+
|
|
16
|
+
## Nasıl çalışır
|
|
17
|
+
|
|
18
|
+
- **Çekirdek gözdür.** Yalnızca standart kütüphane kullanan Python programı bir AST
|
|
19
|
+
indeksi oluşturur ve güvenilmeyen kaynaklardan tehlikeli çıkışlara (sink) taint analizi
|
|
20
|
+
yapar. Hiçbir zaman LLM çağırmaz, kısa JSON üretir.
|
|
21
|
+
- **Claude beyindir.** Claude Code içinde, pakete dahil skill Claude'a CLI'ı çalıştırmasını,
|
|
22
|
+
yalnızca bu çıktıyı ve küçük kod parçalarını okumasını, her bulguyu değerlendirmesini,
|
|
23
|
+
muhtemelen gerçek olanlar için PoC yazıp sandbox içinde kanıtlamasını ve yama yapmasını söyler.
|
|
24
|
+
|
|
25
|
+
Çalışma zamanı bağımlılığı yok. Python 3.10+. MIT lisanslı.
|
|
26
|
+
|
|
27
|
+
## Kurulum
|
|
28
|
+
|
|
29
|
+
Claude Code eklentisi olarak:
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
/plugin marketplace add bu-ra34/vulnhound
|
|
33
|
+
/plugin install vulnhound@vulnhound
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
CLI'ın kendisi (skill'in PATH üzerinde olmasına ihtiyacı var):
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
pip install vulnhound
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## CLI
|
|
43
|
+
|
|
44
|
+
| Komut | Ne yapar |
|
|
45
|
+
|---|---|
|
|
46
|
+
| `vulnhound analyze [path] [--json] [--no-cache] [--min-severity S]` | Aday zafiyetleri bulur. Bulgu varsa çıkış kodu 1. Aynı sink/yardımcıyı paylaşan bulgular gruplanır (`--no-group` hepsini listeler); tests/examples/docs dizinleri ve `test_*.py` `--include-tests` verilmedikçe atlanır; `--include-local-sources` yerel/CLI girdilerini kaynak sayar. |
|
|
47
|
+
| `vulnhound slice <id>` | Bir bulgu için minimal kod parçası. |
|
|
48
|
+
| `vulnhound prove <id> --poc FILE [--root DIR] [--backend auto\|docker\|unshare] [--timeout S] [--json] [--verbose]` | PoC'yi sandbox içinde (ağsız, kaynak sınırlı) çalıştırır. PROVEN (çıkış 0), SAFE (1) veya UNVERIFIED (3) yazar. |
|
|
49
|
+
| `vulnhound estimate [path] [--json]` | Tüm repoyu okumanın yaklaşık token maliyeti (bayt/4). |
|
|
50
|
+
| `vulnhound report [--format json\|sarif\|markdown] [-o FILE]` | Bulguları raporlar. |
|
|
51
|
+
| `vulnhound version` | Sürümü yazar. |
|
|
52
|
+
|
|
53
|
+
Çıkış kodları: 0 tamam / bulgu yok, 1 bulgu var, 2 kullanım hatası (örn. bilinmeyen id), 3 UNVERIFIED (prove).
|
|
54
|
+
|
|
55
|
+
## Bir bulguyu kanıtlama
|
|
56
|
+
|
|
57
|
+
`vulnhound prove <id> --poc poc.py`, küçük bağımsız bir Python PoC'sini sandbox içinde (ağ kapalı, bellek/CPU/süre sınırlı, proje salt okunur) çalıştırır ve karar bildirir. PoC, zafiyetli kodu kötü niyetli girdiyle tetikler; sömürü işe yararsa `vulnhound.poc` içinden `proven("ayrıntı")`, engellenirse `safe("ayrıntı")` çağırır. `canary_path()`, path traversal PoC'lerinin okuması için proje dışında gizli bir dosya verir. İşaret yok ve çıkış 0 ise SAFE; sandbox yoksa, zaman aşımı olursa ya da PoC çökerse UNVERIFIED. PoC hiçbir zaman ana makinede çalışmaz.
|
|
58
|
+
|
|
59
|
+
Not: sandbox için docker veya unshare gerekir. PoC projenize karşı çalışır, bu yüzden üçüncü taraf importları sandbox içinde bulunabilir olmalıdır: `unshare` arka ucu ana makinenin venv'ini salt okunur görür, `docker` arka ucu ise temiz bir python imajı kullanır; framework importu gereken PoC'ler orada UNVERIFIED çıkabilir. PoC kurulu bağımlılıkları import ediyorsa `--backend unshare` seçin.
|
|
60
|
+
|
|
61
|
+
## Yol haritası
|
|
62
|
+
|
|
63
|
+
- [x] M0: iskelet (CLI, eklenti, skill, CI)
|
|
64
|
+
- [ ] M1: proje indeksi (AST, çağrı grafı, önbellek)
|
|
65
|
+
- [ ] M2: taint motoru
|
|
66
|
+
- [ ] M3: framework desteği ve `slice`
|
|
67
|
+
- [ ] M4: sandbox içinde kanıt (`prove`)
|
|
68
|
+
- [ ] M5: benchmark
|
|
69
|
+
- [ ] M6: sürüm
|
|
70
|
+
|
|
71
|
+
Henüz ölçülmüş bir benchmark sonucu iddia edilmiyor; M5 sonrası yayımlanacak.
|
|
72
|
+
|
|
73
|
+
## Güvenlik
|
|
74
|
+
|
|
75
|
+
Yalnızca kendi kodunuzu tarayın. Bkz. [SECURITY.md](SECURITY.md).
|
|
76
|
+
|
|
77
|
+
## Lisans
|
|
78
|
+
|
|
79
|
+
MIT
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Security Policy
|
|
2
|
+
|
|
3
|
+
## Reporting a vulnerability in vulnhound
|
|
4
|
+
|
|
5
|
+
Please report privately via GitHub Security Advisories on this repository
|
|
6
|
+
(Security tab, "Report a vulnerability"). Do not open public issues for
|
|
7
|
+
undisclosed vulnerabilities. Expect an acknowledgement within a few days;
|
|
8
|
+
we will coordinate a fix and disclosure timeline with you.
|
|
9
|
+
|
|
10
|
+
## Using vulnhound responsibly
|
|
11
|
+
|
|
12
|
+
- Scan only code you own or are explicitly authorized to test.
|
|
13
|
+
- Proof-of-concept scripts are never run directly on the host. They must be run
|
|
14
|
+
through `vulnhound prove`, which executes them in a sandbox.
|
|
15
|
+
- vulnhound is pre-alpha: findings are leads to verify, not guarantees.
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# Real-project measurements, hot spots, precision sample
|
|
2
|
+
|
|
3
|
+
Reproduce: `python benchmarks/real_projects.py ROOT [--json out.json] [--dump dir]`
|
|
4
|
+
(ROOT has one extracted sdist per sub-directory; code is only parsed, never imported).
|
|
5
|
+
Snapshot of the engine/slicer as of this run (other agents were editing engine.py and
|
|
6
|
+
slicer.py concurrently, so finding counts will drift). Python 3.13, one core, tokens = bytes/4.
|
|
7
|
+
|
|
8
|
+
## Projects (PyPI sdists, `.py` files only)
|
|
9
|
+
|
|
10
|
+
| project | files | lines | funcs | naive tok | cold idx s | warm idx s | analyze s | findings (C/H/M/L) | json B | slice B | Claude reads tok | touched-files tok | x vs repo | x vs touched |
|
|
11
|
+
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
12
|
+
| httpbin 0.10.4 | 8 | 3,316 | 180 | 24,237 | 0.03 | 0.00 | 0.04 | 0 | 169 | 0 | 42 | 0 | 574 | - |
|
|
13
|
+
| fastapi-users 15.0.5 | 89 | 9,247 | 684 | 87,148 | 0.11 | 0.02 | 0.12 | 0 | 181 | 0 | 45 | 0 | 1,926 | - |
|
|
14
|
+
| flask-admin 2.2.1 | 115 | 30,741 | 1,471 | 244,048 | 0.31 | 0.03 | 0.51 | 18 (0/1/0/17) | 13,020 | 17,708 | 7,682 | 30,807 | 31.8 | 4.0 |
|
|
15
|
+
| flask-appbuilder 5.3.0 | 295 | 45,632 | 2,219 | 387,812 | 0.47 | 0.06 | 0.86 | 11 (0/0/0/11) | 9,792 | 12,235 | 5,506 | 9,476 | 70.4 | 1.7 |
|
|
16
|
+
| label-studio 1.23.2 | 669 | 85,594 | 3,816 | 828,225 | 1.28 | 0.18 | 1.53 | 17 (0/11/1/5) | 16,116 | 36,029 | 13,036 | 36,783 | 63.5 | 2.8 |
|
|
17
|
+
| apache-superset 6.1.0 | 1,177 | 200,225 | 6,624 | 1,782,886 | 1.89 | 0.23 | 3.78 | 7 (0/1/2/4) | 6,464 | 12,462 | 4,731 | 44,780 | 376.8 | 9.5 |
|
|
18
|
+
| wagtail 8.1 | 1,332 | 277,900 | 16,348 | 2,547,854 | 3.35 | 0.44 | 7.13 | 20 (0/0/0/20) | 16,212 | 32,749 | 12,240 | 74,307 | 208.2 | 6.1 |
|
|
19
|
+
| mlflow 3.18.0 | 1,315 | 396,534 | 15,611 | 4,049,292 | 3.80 | 0.38 | 9.53 | 104 (6/32/39/27) | 108,614 | 229,747 | 84,590 | 480,751 | 47.9 | 5.7 |
|
|
20
|
+
|
|
21
|
+
"Claude reads" = `analyze --json` + every finding's slice. "touched-files" = whole files that
|
|
22
|
+
contain a finding's source, sink or path step (the realistic naive baseline).
|
|
23
|
+
Peak RSS for mlflow analyze: 528 MB. `slice` for all findings is cheap (mlflow 0.77 s total).
|
|
24
|
+
|
|
25
|
+
Takeaways:
|
|
26
|
+
- Versus reading the whole repo the saving is 30x-1,900x, but that baseline is a straw man.
|
|
27
|
+
Versus the files that actually contain findings it is only 1.7x-9.5x, and it **shrinks when
|
|
28
|
+
there are many findings**: mlflow's 104 findings cost 84k tokens (JSON ~1.04 KB + slice ~2.2 KB
|
|
29
|
+
each), which is 3.4 KB per finding that, per the precision sample below, is mostly noise.
|
|
30
|
+
Precision (and de-duplication of findings sharing a sink) matters more for token cost than slicing.
|
|
31
|
+
- A project with 0 findings costs ~40 tokens, the best case.
|
|
32
|
+
- Cost is dominated by repeated findings: flask-admin has 17 open-redirect findings that are one
|
|
33
|
+
`get_redirect_target()` helper reached from 17 callers; wagtail 12 findings from one helper.
|
|
34
|
+
Group by (rule, source) or (rule, sink) and emit one finding with N call sites.
|
|
35
|
+
- Slicer bug: `build_slice` raised `AttributeError: 'match_case' object has no attribute 'lineno'`
|
|
36
|
+
(slicer.py `_header_span`, `match` statements) for VH-f8ada2 in mlflow. The benchmark records
|
|
37
|
+
it in `slice_errors`.
|
|
38
|
+
|
|
39
|
+
## Scaling
|
|
40
|
+
|
|
41
|
+
cold index ~ 3.8 s/400k lines, analyze ~ 2.5x the cold index time (analyze 9.5 s on mlflow vs
|
|
42
|
+
3.8 s cold index). Linear in code size. 108k functions: ~29 s index / 70 s analyze matches.
|
|
43
|
+
|
|
44
|
+
## Hot spots (cProfile on mlflow, analyze with warm cache: 14.7 s profiled / 9.5 s real)
|
|
45
|
+
|
|
46
|
+
1. **Garbage collector: ~40% of analyze wall time.** Same code, `gc.disable()` vs enabled:
|
|
47
|
+
- parse all 1,315 modules (`ProjectIndex.ast_for`): 4.0-4.2 s -> 1.4-1.6 s
|
|
48
|
+
- full `analyze()`: 8.7 / 9.7 s -> 5.1 / 5.8 s
|
|
49
|
+
The engine keeps ~15.6k function ASTs alive, so every young-generation collection walks a
|
|
50
|
+
huge live heap. Fix (one line, in `cmd_analyze` or `engine.analyze`): `gc.disable()` (or
|
|
51
|
+
`gc.freeze()` after the AST load, or `gc.set_threshold(200_000, 20, 20)`) for the run.
|
|
52
|
+
Cold `build_index` is not affected (trees die immediately): 3.7-4.1 s either way.
|
|
53
|
+
2. **Every module is parsed twice per cold run, and once per warm run.** The index cache stores
|
|
54
|
+
only the extraction JSON, so `Engine.load()` calls `ast_for` for all modules: 1,315 x
|
|
55
|
+
`ast.parse` = 4.0 s of 14.7 profiled (27%, `ast.parse` 2.4 s + C `compile` 1.6 s); `Engine.load`
|
|
56
|
+
alone is 2.1 s even with the GC off. Options: (a) text pre-filter: only parse/analyze modules
|
|
57
|
+
whose source text mentions a configured source or sink name (most modules mention none: of
|
|
58
|
+
15.6k functions only 17,960 analyze_function runs happen anyway, so the saving is parse + walk);
|
|
59
|
+
(b) on a cold run keep the trees from extraction (`extract_module` already parsed them) instead
|
|
60
|
+
of re-parsing: saves ~1.5 s on mlflow at the cost of memory; (c) lazy per-module parse when
|
|
61
|
+
the first function of that module is analyzed.
|
|
62
|
+
3. `_collect_nodes` (engine.py `visit`): 222k recursive calls, 1.65 s cumulative (11%); it walks
|
|
63
|
+
every statement via `ast.iter_child_nodes` just to find def/class nodes. The extractor already
|
|
64
|
+
knows each function's `line`; map `(module, line)` -> node with one pass over `ast.walk` only
|
|
65
|
+
statement-bearing containers, or skip subtrees of simple statements.
|
|
66
|
+
4. `Analyzer.ev`/`call` (taint transfer): 646k `ev` calls (7.4 s cumulative, 0.9 s self) and
|
|
67
|
+
114k `call`; `chain_source` 222k calls = 3.0 s cumulative (20%), of which `specs.match_source`
|
|
68
|
+
144k calls = 0.9 s and `Engine.resolve` 340k calls = 1.8 s (12%). `chain_source` is recomputed for
|
|
69
|
+
the same attribute chains over and over (e.g. `request.args.get`); memoize on the AST node id
|
|
70
|
+
or on `(module, chain_string)` per module. `_chain` is called 785k times for the same reason.
|
|
71
|
+
5. Reanalysis: 17,960 `analyze_function` runs for 15,611 functions (15% re-analysis via the
|
|
72
|
+
field/summary worklist); small, fine.
|
|
73
|
+
6. Index (cold, profile 8.5 s / 3.8 s real): `_Extractor.visit` 1.28M recursive calls, 4.8 s of
|
|
74
|
+
6.5 s extraction cumulative (50%+ of it is `ast.iter_child_nodes` generator overhead), C parse
|
|
75
|
+
1.5 s, cache JSON write 0.86 s (1,315 `write_text`, 1 file per module; ~0.3 s real).
|
|
76
|
+
I tried inlining `iter_child_nodes` in `_Extractor.visit` (same order): only ~7% faster on
|
|
77
|
+
extraction (2.89 s -> 2.68 s best-of-3) so I reverted it; not worth the code. Real gains need
|
|
78
|
+
fewer nodes visited (skip Name/Constant/expr leaves by checking `type(node)` against a
|
|
79
|
+
precomputed skip set) or multiprocessing across files for cold runs.
|
|
80
|
+
7. Warm index (0.4 s on mlflow) is fine: read + sha256 of every file + JSON load.
|
|
81
|
+
Could use mtime+size to skip hashing, but the gain is a few hundred ms.
|
|
82
|
+
|
|
83
|
+
No engine/index/cache code was changed (index change attempted and reverted; tests: 68 pass).
|
|
84
|
+
Highest-value single change: item 1 (disable GC during analyze), ~3.5 s of 9.5 s on mlflow.
|
|
85
|
+
|
|
86
|
+
## Precision sample (15 findings, read from their slices)
|
|
87
|
+
|
|
88
|
+
Sampled across mlflow (7), label-studio (3), flask-admin (2), wagtail, superset.
|
|
89
|
+
Result: **0 true positives, 15 false positives, 0 unclear** (with the caveat that I judged only
|
|
90
|
+
from the slice plus a look at the elided lines; none of these is a plausible vulnerability).
|
|
91
|
+
This is a first estimate, n=15 and not random (spread across rules and severities), but it
|
|
92
|
+
suggests precision on real code is currently far below what the token argument needs.
|
|
93
|
+
|
|
94
|
+
| # | finding | rule/sev | verdict | why |
|
|
95
|
+
|---|---|---|---|---|
|
|
96
|
+
| 1 | mlflow VH-15232b gateway_api -> `exec` in `recreate_function` | code-exec critical | FP | the flow is a chain of unrelated call hops; the server explicitly refuses custom-code scorers (`_serialized_scorer_is_custom_code` raises) and `model_validate` returns a non-executing handle (comment in the slice) |
|
|
97
|
+
| 2 | mlflow VH-9e1627 stdin_server `open(request["output_file"])` | path-traversal high | FP | a local CLI reading stdin; stdin is not attacker input here |
|
|
98
|
+
| 3 | mlflow VH-943ad8 `session_id` -> `read_text` | path-traversal high | FP | `get_session_file` calls `SessionManager.validate_session_id` (hidden by slice elision) |
|
|
99
|
+
| 4 | mlflow VH-01d4c4 upload_artifact_handler `open(file_path)` | path-traversal high | FP | `path = validate_path_is_safe(path)` is in the slice; engine does not treat it as a sanitizer (and the slice pads with `_decode`/`escape_char` bodies) |
|
|
100
|
+
| 5 | mlflow VH-bd4d0b judges/environment_behavior.py | path-traversal medium | FP | test helper reading env vars |
|
|
101
|
+
| 6 | mlflow VH-a45beb scoring_server/app.py | path-traversal medium | FP | env var set by the server operator, not a user |
|
|
102
|
+
| 7 | mlflow VH-96a1ac `os.environ` -> `__import__` | code-exec medium | FP | taint travels through `.format()` into a pip-requirements string; `__import__(module or package)` gets a package name from a static table |
|
|
103
|
+
| 8 | mlflow VH-ab77d4 `os.environ.copy()` -> `Popen` | cmdi low | FP | env dict passed as `env=`, not as the command line |
|
|
104
|
+
| 9 | label-studio VH-c25acd `request.GET['d']` -> `open` | path-traversal high | FP | `safe_join(DOCUMENT_ROOT, path)` plus permission check precede the sink |
|
|
105
|
+
| 10 | label-studio VH-d2a298 `sys.argv` -> `io.open` | path-traversal high | FP | server CLI config-file argument |
|
|
106
|
+
| 11 | label-studio VH-bf3260 `next` -> `redirect` | open-redirect low | FP | `url_has_allowed_host_and_scheme` check reassigns `next_page` on failure (guard-then-reassign) |
|
|
107
|
+
| 12 | flask-admin VH-5b9c8a `request.args['db']` -> `send_file` | path-traversal high | FP | `db` is a Mongo database name; `send_file` gets a GridFS object, not a path |
|
|
108
|
+
| 13 | flask-admin VH-3c60c1 `request.values` -> `redirect` | open-redirect low | FP | the source line is inside `get_redirect_target`, which returns only if `is_safe_url(target)` |
|
|
109
|
+
| 14 | wagtail VH-1f9c9e `request.POST['next']` -> `redirect` | open-redirect low | FP | `url_has_allowed_host_and_scheme(... allowed_hosts={host})` returns "" otherwise |
|
|
110
|
+
| 15 | superset VH-b20faf `request.files` -> `db.session.execute(insert(...))` | sqli high | FP | executes a SQLAlchemy `insert().values()` of integer ids; parametrized |
|
|
111
|
+
|
|
112
|
+
FP causes (counts of the 15): validators/sanitizers not modelled or not honoured when they live
|
|
113
|
+
in a callee or a guard-then-reassign (3, 4, 9, 11, 13, 14, 15 partly, 1): 8; non-attacker
|
|
114
|
+
sources (env, argv, stdin) treated as untrusted (2, 5, 6, 10, 7): 5; wrong semantic (value is
|
|
115
|
+
not a path / not a command, 8, 12, 15): 3 (overlapping).
|
|
116
|
+
|
|
117
|
+
Actionable for the engine (not changed here):
|
|
118
|
+
- Treat as sanitizers any callee whose name matches `validate_*|is_safe*|safe_*|get_safe_*|
|
|
119
|
+
*_is_safe|url_has_allowed_host_and_scheme|safe_join|secure_filename|is_safe_url|is_safe_redirect_url`,
|
|
120
|
+
and honour guard patterns `if not safe(x): x = default` / `if not safe(x): raise|return`.
|
|
121
|
+
- Drop `os.environ`, `sys.argv`, `sys.stdin` as sources by default (or demote to `low` with an
|
|
122
|
+
explicit `--include-local-sources` flag); drop `env=` kwargs as cmdi sink args.
|
|
123
|
+
- Skip or demote findings under `tests/`, `test_*.py`, `examples/`: mlflow has 15+ of those.
|
|
124
|
+
- Do not report source-in-a-safe-helper findings (flask-admin, wagtail, flask-appbuilder: the
|
|
125
|
+
source line is inside `get_redirect_target`/`get_valid_next_url...`, the helper is the sanitizer).
|
|
126
|
+
- Slice elision (`…`) hid the actual guard in #3; keep calls to `validate_*`/`check_*` lines of
|
|
127
|
+
the path function and show `if` guards that dominate the sink.
|
|
128
|
+
|
|
129
|
+
## Round 2 results (validators/guards, grouping, test exclusion)
|
|
130
|
+
|
|
131
|
+
Same 8 projects, CLI defaults. Total findings fell from about 200 to 16
|
|
132
|
+
(superset 3, flask-admin 2, label-studio 2, mlflow 4, wagtail 5, others 0).
|
|
133
|
+
|
|
134
|
+
Manual triage of all 16 slices: 0 true positives, 13 false positives, 3 unclear
|
|
135
|
+
(label-studio storage-URI redirect, mlflow scorer deserialization, mlflow
|
|
136
|
+
skills install `custom_path`; the last is behind a deny-remote-access policy).
|
|
137
|
+
|
|
138
|
+
Remaining false-positive classes:
|
|
139
|
+
1. Open redirect to a same-host URL: `request.path`, `request.url` reduced to
|
|
140
|
+
path+query, or `reverse(...) + "?" + query`. Only attacker control over the
|
|
141
|
+
start of the URL (scheme/host) should count.
|
|
142
|
+
2. Taint through database lookups: a request id used to fetch an object
|
|
143
|
+
(`DAO.find_by_id(id)`, `GridFS.get(...)`), whose stored content then reaches
|
|
144
|
+
a sink. Values loaded by id are not attacker-supplied content.
|
|
145
|
+
3. Positive-branch guards: `if x and is_safe_url(x): return x` is not yet
|
|
146
|
+
treated as a guard on the returned value.
|
|
147
|
+
4. Guards whose result is reassigned (`path = validate_path_is_safe(path)`)
|
|
148
|
+
followed by helper calls still produce a finding in mlflow.
|
|
149
|
+
|
|
150
|
+
## Precision benchmark (benchmarks/precision.py)
|
|
151
|
+
|
|
152
|
+
`python benchmarks/precision.py CORPUS_DIR [LABELS.json] [--json OUT] [--strict] [NAME ...]`
|
|
153
|
+
runs the engine with CLI defaults (grouped, tests excluded) over every project in
|
|
154
|
+
CORPUS_DIR and matches group representatives to `benchmarks/precision_labels.json`
|
|
155
|
+
by key `project|rule|sink_path:line|source_path:line`. It prints per project
|
|
156
|
+
findings / tp / fp / unclear / UNLABELED, then overall precision = tp/(tp+fp),
|
|
157
|
+
and lists unlabeled findings with their keys (triage these and add labels).
|
|
158
|
+
`--strict` exits 1 if any finding is unlabeled. Labels no longer reported are
|
|
159
|
+
counted as stale. Corpus: the 8 earlier projects plus django-cms 5.1.3, flaskbb
|
|
160
|
+
3.0.0, gradio 6.32.0, jupyter-server 2.21.1, nautobot 3.2.9, netbox 4.7.5,
|
|
161
|
+
pretalx 2026.2.1, saleor 2.10.1, streamlit 1.67.0 (PyPI sdists, .py only).
|
|
162
|
+
|
|
163
|
+
Baseline (17 projects, engine as of Round 2): 67 findings; 1 tp, 65 fp, 1 unclear;
|
|
164
|
+
precision 1/66 = 0.015. fp categories: same-host-redirect 32, guarded 26,
|
|
165
|
+
db-lookup 3, wrong-sink-semantics 2, not-attacker-source 2. The 16 original
|
|
166
|
+
findings: 0 tp / 15 fp / 1 unclear (the manager's two other unclear items,
|
|
167
|
+
label-studio storage-URI redirect and mlflow scorer deserialization, are fp:
|
|
168
|
+
bucket-gated presign redirect; guardrail rejects custom-code scorers and the
|
|
169
|
+
server never exec()s). Only tp: one gradio path-traversal finding (details withheld pending responsible disclosure).
|
|
170
|
+
|
|
171
|
+
Note: true-positive labels for issues not yet publicly disclosed are kept out of
|
|
172
|
+
this repository until the maintainers have been notified. `precision.py` reports
|
|
173
|
+
such findings as UNLABELED.
|
|
174
|
+
|
|
175
|
+
## Engine + Claude experiment
|
|
176
|
+
|
|
177
|
+
Question: can a Claude reviewer, given only each finding's slice (not the label),
|
|
178
|
+
keep the real bug and drop the false positives? Three Claude reviewers judged the
|
|
179
|
+
21 grouped findings in isolation, following the skill's judging questions, with no
|
|
180
|
+
access to the ground-truth labels.
|
|
181
|
+
|
|
182
|
+
Result against the hidden labels:
|
|
183
|
+
- Real bug (gradio path traversal) kept as real: 1/1.
|
|
184
|
+
- False positives dropped: 19/19.
|
|
185
|
+
- The 1 label-"unclear" finding (mlflow custom_path) was dropped as a false positive.
|
|
186
|
+
|
|
187
|
+
So the two stages together, on this corpus, surfaced exactly 1 finding to the user,
|
|
188
|
+
the real one, with no false positives. The engine alone has ~5% precision; the
|
|
189
|
+
engine as a pre-filter plus Claude as the judge is what the product is designed
|
|
190
|
+
to be, and on this set it worked.
|
|
191
|
+
|
|
192
|
+
Caveats: 21 findings with a single true positive is a small sample — the "1/1 TP
|
|
193
|
+
kept" number is one data point, not a rate. Reviewers judged mostly from the slice
|
|
194
|
+
and occasionally opened the referenced lines, which mirrors the skill workflow.
|
|
195
|
+
Harness: benchmarks/judge_prep.py dumps slices with hidden labels; the judging and
|
|
196
|
+
scoring were run out-of-tree.
|