deployproof 0.1.7__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deployproof-0.1.7/src/deployproof.egg-info → deployproof-0.2.0}/PKG-INFO +107 -10
- {deployproof-0.1.7 → deployproof-0.2.0}/README.md +106 -9
- {deployproof-0.1.7 → deployproof-0.2.0}/pyproject.toml +1 -1
- deployproof-0.2.0/src/deployproof/__init__.py +2 -0
- deployproof-0.2.0/src/deployproof/cli.py +179 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof/dependencies.py +137 -35
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof/diff.py +5 -1
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof/mutator.py +307 -117
- deployproof-0.2.0/src/deployproof/reporter.py +333 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof/secrets.py +55 -15
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof/symlinks.py +173 -156
- {deployproof-0.1.7 → deployproof-0.2.0/src/deployproof.egg-info}/PKG-INFO +107 -10
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof.egg-info/SOURCES.txt +1 -0
- deployproof-0.2.0/tests/test_baseline_collection_error.py +120 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_cli.py +3 -3
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_control_flow.py +4 -3
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_dependencies_extractor.py +69 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_dependencies_pypi.py +8 -0
- deployproof-0.2.0/tests/test_mutator.py +375 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_secrets.py +26 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_stress_regressions.py +10 -3
- deployproof-0.1.7/src/deployproof/__init__.py +0 -2
- deployproof-0.1.7/src/deployproof/cli.py +0 -397
- deployproof-0.1.7/src/deployproof/reporter.py +0 -623
- deployproof-0.1.7/tests/test_mutator.py +0 -137
- {deployproof-0.1.7 → deployproof-0.2.0}/LICENSE +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/setup.cfg +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof/control_flow.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof/mocks.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof/wsl.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof.egg-info/dependency_links.txt +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof.egg-info/entry_points.txt +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof.egg-info/requires.txt +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/src/deployproof.egg-info/top_level.txt +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_cli_dependency_scan.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_diff.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_edge_cases_validation.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_mocks.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_mutation_suites.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_symlinks.py +0 -0
- {deployproof-0.1.7 → deployproof-0.2.0}/tests/test_wsl.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deployproof
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A deterministic AI-code deployability checker that catches what coverage misses.
|
|
5
5
|
Author-email: SVS Praveen <svspraveens@gmail.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -32,9 +32,11 @@ Dynamic: license-file
|
|
|
32
32
|
|
|
33
33
|
> Deterministic pre-push verification for AI-assisted codebases: AST mutation testing, credential scanning, sandbox-escape detection, mock-usage alerts, swallowed-exception checks, and dependency hallucination defense.
|
|
34
34
|
|
|
35
|
-
[](https://pypi.org/project/deployproof/)
|
|
36
|
+
[](https://pypi.org/project/deployproof/)
|
|
37
|
+
[](https://github.com/SVSPraveen/DeployProof/actions/workflows/ci.yml)
|
|
38
|
+
[](https://github.com/SVSPraveen/DeployProof)
|
|
39
|
+
[](stress_fixtures/)
|
|
38
40
|
[](LICENSE)
|
|
39
41
|
|
|
40
42
|
---
|
|
@@ -65,7 +67,100 @@ Run all verification checks against changes in the current session (git diff):
|
|
|
65
67
|
deployproof check
|
|
66
68
|
```
|
|
67
69
|
|
|
68
|
-
Output includes a section for each check — symlink scan, secrets scan, dependency scan, mock detection, control flow analysis, and mutation score — with a pass/fail line at the bottom.
|
|
70
|
+
Output includes a section for each check — symlink scan, secrets scan, dependency scan, mock detection, control flow analysis, and mutation score — with a pass/fail line at the bottom.
|
|
71
|
+
|
|
72
|
+
### Exit Codes:
|
|
73
|
+
* `0` — **PASSED**: All verification checks passed and mutation score meets threshold.
|
|
74
|
+
* `1` — **FAILED**: Code quality or security gate triggered (mutation score below threshold, untested files, hardcoded secrets, sandbox-escape symlinks, hallucinated packages, or strict flags).
|
|
75
|
+
* `2` — **ERROR**: Test environment failure (test suite failed to collect or execute before mutation testing began due to missing dependencies or broken imports).
|
|
76
|
+
|
|
77
|
+
### Example Walkthrough
|
|
78
|
+
|
|
79
|
+
Given a newly written function `calculator.py`:
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
def calculate_discount(price: float, rate: float) -> float:
|
|
83
|
+
if rate > 0.5:
|
|
84
|
+
return price * 0.5
|
|
85
|
+
return price * (1.0 - rate)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
With an AI-generated test that achieves 100% line coverage by only asserting standard discounts (`rate = 0.2`):
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
def test_calculate_discount_basic():
|
|
92
|
+
assert calculate_discount(100.0, 0.2) == 80.0
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Running `deployproof check` mutates AST operators and detects that boundary conditions and threshold caps are untested:
|
|
96
|
+
|
|
97
|
+
```
|
|
98
|
+
$ deployproof check
|
|
99
|
+
|
|
100
|
+
DeployProof - LOCAL PRE-CHECK (approximate) - not the verified score
|
|
101
|
+
====================================================================
|
|
102
|
+
|
|
103
|
+
Target Scope (1 file evaluated):
|
|
104
|
+
* calculator.py
|
|
105
|
+
|
|
106
|
+
Symlink & Sandbox Escape Scan (CWE-61/CWE-451):
|
|
107
|
+
Clean: No symlinks or sandbox-escape traversal links detected across 1 session file.
|
|
108
|
+
|
|
109
|
+
Secrets & Credentials Pre-Push Scan:
|
|
110
|
+
Clean: No hardcoded secrets or tracked .env files detected across 1 session file.
|
|
111
|
+
|
|
112
|
+
Dependency & Slopsquatting Scan (PyPI Registry & Age Analysis):
|
|
113
|
+
Clean: No new external packages introduced across 1 session file.
|
|
114
|
+
|
|
115
|
+
Mock Usage Introduced (flagged for review):
|
|
116
|
+
Clean: No modified test files in scope.
|
|
117
|
+
|
|
118
|
+
Control Flow & Error Handling (flagged for review):
|
|
119
|
+
Clean: No bare excepts, swallowed exceptions, or unreachable code detected across 1 session file.
|
|
120
|
+
|
|
121
|
+
Local Pre-Check Mutation Verification:
|
|
122
|
+
Score: 57.1% (4/7 mutants killed)
|
|
123
|
+
Status: FAILED (score 57.1% below 80.0%) (threshold: 80.0%)
|
|
124
|
+
Time: 2.27s
|
|
125
|
+
|
|
126
|
+
Skipped Constructs: None (No known unsupported constructs detected)
|
|
127
|
+
|
|
128
|
+
Surviving Mutants (3 unverified changes):
|
|
129
|
+
|
|
130
|
+
[1] calculator.py:2
|
|
131
|
+
Mutation: Replace numeric constant '0.5' with '1.5'
|
|
132
|
+
Original: if rate > 0.5:
|
|
133
|
+
Mutated: if rate > 1.5:
|
|
134
|
+
|
|
135
|
+
[2] calculator.py:3
|
|
136
|
+
Mutation: Replace numeric constant '0.5' with '1.5'
|
|
137
|
+
Original: return price * 0.5
|
|
138
|
+
Mutated: return price * 1.5
|
|
139
|
+
|
|
140
|
+
[3] calculator.py:3
|
|
141
|
+
Mutation: Replace binary operator '*' with '/'
|
|
142
|
+
Original: return price * 0.5
|
|
143
|
+
Mutated: return price / 0.5
|
|
144
|
+
|
|
145
|
+
====================================================================
|
|
146
|
+
Notice: Local pre-check only. Full verified score runs in CI on push (via mutmut).
|
|
147
|
+
Pre-check FAILED: Score 57.1% is below threshold 80.0% (3 surviving mutants).
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Adding tests for threshold cap (`rate = 0.8`) and exact boundary (`rate = 0.5`) kills all mutants:
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
Local Pre-Check Mutation Verification:
|
|
154
|
+
Score: 100.0% (7/7 mutants killed)
|
|
155
|
+
Status: PASSED (threshold: 80.0%)
|
|
156
|
+
Time: 2.31s
|
|
157
|
+
|
|
158
|
+
Surviving Mutants: None (All generated mutants caught by test suite)
|
|
159
|
+
|
|
160
|
+
====================================================================
|
|
161
|
+
Notice: Local pre-check only. Full verified score runs in CI on push (via mutmut).
|
|
162
|
+
Pre-check clean: 100% of tested basic mutations caught.
|
|
163
|
+
```
|
|
69
164
|
|
|
70
165
|
### CLI Options & Flags
|
|
71
166
|
|
|
@@ -80,6 +175,8 @@ Output includes a section for each check — symlink scan, secrets scan, depende
|
|
|
80
175
|
| `deployproof check --base <ref>` | Base git ref (branch/commit/tag) to diff against. |
|
|
81
176
|
| `deployproof check --wsl` | Delegate mutation testing to `mutmut` inside WSL (Windows only). |
|
|
82
177
|
|
|
178
|
+
> **Note:** `deployproof check --wsl` (Windows only) is newer and less battle-tested than the core checks — [file an issue](https://github.com/SVSPraveen/DeployProof/issues) if you hit something.
|
|
179
|
+
|
|
83
180
|
### Machine-Readable Output (`--json`)
|
|
84
181
|
|
|
85
182
|
DeployProof provides a stable structured JSON schema for CI/CD pipelines, IDEs, and automation:
|
|
@@ -92,7 +189,7 @@ deployproof check --json
|
|
|
92
189
|
|
|
93
190
|
```json
|
|
94
191
|
{
|
|
95
|
-
"version": "0.
|
|
192
|
+
"version": "0.2.0",
|
|
96
193
|
"status": "passed",
|
|
97
194
|
"summary": {
|
|
98
195
|
"target_files_count": 1,
|
|
@@ -165,7 +262,7 @@ deployproof check --json
|
|
|
165
262
|
|
|
166
263
|
## What It Checks
|
|
167
264
|
|
|
168
|
-
- **Mutation Score** — Mutates AST operators (`>=`, `==`, `and`, `or`, `*`, numeric constants, comparisons) in modified files and runs your test suite against each mutant. Reports surviving mutants and a percentage score. Does not use line coverage.
|
|
265
|
+
- **Mutation Score** — Mutates AST operators (`>=`, `==`, `and`, `or`, `*`, numeric constants, comparisons) in modified files and runs your test suite against each mutant. Reports surviving mutants and a percentage score. Does not use line coverage. Features atomic file restoration protected by `SIGINT`/`SIGTERM`/`SIGBREAK` signal handlers to ensure interrupted runs never leave mutated code on disk.
|
|
169
266
|
- **Secrets and Credentials** — Scans modified files for hardcoded API keys (OpenAI, Anthropic, AWS, GitHub, Stripe, private keys) and tracked `.env` files using pattern matching and entropy analysis.
|
|
170
267
|
- **Symlink and Sandbox Escape** — Resolves symbolic links and flags any whose target escapes the repository root (CWE-61 / CWE-451). Catches the class of path-traversal trick used in the GhostApproval disclosure (Wiz Research, July 2026).
|
|
171
268
|
- **Dependency and Slopsquatting** — For each new import, dynamic import (`importlib.import_module`, `__import__`), or manifest entry (including recursive `-r` includes) introduced in the diff, queries the PyPI JSON API and checks registration age. Packages that don't exist (HTTP 404) are flagged HIGH RISK; packages registered within the last 30 days are flagged MEDIUM RISK.
|
|
@@ -181,17 +278,17 @@ deployproof check --json
|
|
|
181
278
|
|
|
182
279
|
## See It Catch Real Bugs
|
|
183
280
|
|
|
184
|
-
Clone this repository and run the standalone stress-test suite to see DeployProof evaluate
|
|
281
|
+
Clone this repository and run the standalone stress-test suite to see DeployProof evaluate 11 planted edge cases:
|
|
185
282
|
|
|
186
283
|
```bash
|
|
187
284
|
python stress_fixtures/run_stress_tests.py
|
|
188
285
|
```
|
|
189
286
|
|
|
190
|
-
Fixtures cover weak test suites, zero-test orphan modules, planted OpenAI/AWS credentials,
|
|
287
|
+
Fixtures cover weak test suites, zero-test orphan modules, planted OpenAI/AWS credentials, GhostApproval sandbox-escape traps, swallowed exceptions/dead code, and mock-masked broken implementations.
|
|
191
288
|
|
|
192
289
|
## Status & Roadmap
|
|
193
290
|
|
|
194
|
-
- **Current (v0.
|
|
291
|
+
- **Current (v0.2.0):** Diff-scoped AST mutation testing (with recursive `test/`/`tests/` discovery, AST column-offset snippet reconstruction, and `SIGINT`/`SIGTERM`/`SIGBREAK` signal-safe disk restoration), baseline test-collection failure isolation with distinct exit code `2`, entropy-driven value-based secrets scanner (including unquoted .env values and prefix validation), GhostApproval symlink sandbox-escape detector, PyPI dependency hallucination / slopsquatting scanner (with import-to-distribution translation, recursive `-r` requirements scanning, and dynamic import detection via `importlib` / `__import__`), mock-introduction detector (`--strict-mocks`), control-flow / swallowed-exception scanner (`--strict-error-handling`), 11/11 launch-day stress test suite, 93 unit tests, machine-readable `--json` output, and Tier 2 CI verification via GitHub Actions (mutmut).
|
|
195
292
|
- **Next:** Multi-language mutation support and expanded ecosystem rule packs.
|
|
196
293
|
|
|
197
294
|
## Contributing
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
> Deterministic pre-push verification for AI-assisted codebases: AST mutation testing, credential scanning, sandbox-escape detection, mock-usage alerts, swallowed-exception checks, and dependency hallucination defense.
|
|
4
4
|
|
|
5
|
-
[](https://pypi.org/project/deployproof/)
|
|
6
|
+
[](https://pypi.org/project/deployproof/)
|
|
7
|
+
[](https://github.com/SVSPraveen/DeployProof/actions/workflows/ci.yml)
|
|
8
|
+
[](https://github.com/SVSPraveen/DeployProof)
|
|
9
|
+
[](stress_fixtures/)
|
|
8
10
|
[](LICENSE)
|
|
9
11
|
|
|
10
12
|
---
|
|
@@ -35,7 +37,100 @@ Run all verification checks against changes in the current session (git diff):
|
|
|
35
37
|
deployproof check
|
|
36
38
|
```
|
|
37
39
|
|
|
38
|
-
Output includes a section for each check — symlink scan, secrets scan, dependency scan, mock detection, control flow analysis, and mutation score — with a pass/fail line at the bottom.
|
|
40
|
+
Output includes a section for each check — symlink scan, secrets scan, dependency scan, mock detection, control flow analysis, and mutation score — with a pass/fail line at the bottom.
|
|
41
|
+
|
|
42
|
+
### Exit Codes:
|
|
43
|
+
* `0` — **PASSED**: All verification checks passed and mutation score meets threshold.
|
|
44
|
+
* `1` — **FAILED**: Code quality or security gate triggered (mutation score below threshold, untested files, hardcoded secrets, sandbox-escape symlinks, hallucinated packages, or strict flags).
|
|
45
|
+
* `2` — **ERROR**: Test environment failure (test suite failed to collect or execute before mutation testing began due to missing dependencies or broken imports).
|
|
46
|
+
|
|
47
|
+
### Example Walkthrough
|
|
48
|
+
|
|
49
|
+
Given a newly written function `calculator.py`:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
def calculate_discount(price: float, rate: float) -> float:
|
|
53
|
+
if rate > 0.5:
|
|
54
|
+
return price * 0.5
|
|
55
|
+
return price * (1.0 - rate)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
With an AI-generated test that achieves 100% line coverage by only asserting standard discounts (`rate = 0.2`):
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
def test_calculate_discount_basic():
|
|
62
|
+
assert calculate_discount(100.0, 0.2) == 80.0
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
Running `deployproof check` mutates AST operators and detects that boundary conditions and threshold caps are untested:
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
$ deployproof check
|
|
69
|
+
|
|
70
|
+
DeployProof - LOCAL PRE-CHECK (approximate) - not the verified score
|
|
71
|
+
====================================================================
|
|
72
|
+
|
|
73
|
+
Target Scope (1 file evaluated):
|
|
74
|
+
* calculator.py
|
|
75
|
+
|
|
76
|
+
Symlink & Sandbox Escape Scan (CWE-61/CWE-451):
|
|
77
|
+
Clean: No symlinks or sandbox-escape traversal links detected across 1 session file.
|
|
78
|
+
|
|
79
|
+
Secrets & Credentials Pre-Push Scan:
|
|
80
|
+
Clean: No hardcoded secrets or tracked .env files detected across 1 session file.
|
|
81
|
+
|
|
82
|
+
Dependency & Slopsquatting Scan (PyPI Registry & Age Analysis):
|
|
83
|
+
Clean: No new external packages introduced across 1 session file.
|
|
84
|
+
|
|
85
|
+
Mock Usage Introduced (flagged for review):
|
|
86
|
+
Clean: No modified test files in scope.
|
|
87
|
+
|
|
88
|
+
Control Flow & Error Handling (flagged for review):
|
|
89
|
+
Clean: No bare excepts, swallowed exceptions, or unreachable code detected across 1 session file.
|
|
90
|
+
|
|
91
|
+
Local Pre-Check Mutation Verification:
|
|
92
|
+
Score: 57.1% (4/7 mutants killed)
|
|
93
|
+
Status: FAILED (score 57.1% below 80.0%) (threshold: 80.0%)
|
|
94
|
+
Time: 2.27s
|
|
95
|
+
|
|
96
|
+
Skipped Constructs: None (No known unsupported constructs detected)
|
|
97
|
+
|
|
98
|
+
Surviving Mutants (3 unverified changes):
|
|
99
|
+
|
|
100
|
+
[1] calculator.py:2
|
|
101
|
+
Mutation: Replace numeric constant '0.5' with '1.5'
|
|
102
|
+
Original: if rate > 0.5:
|
|
103
|
+
Mutated: if rate > 1.5:
|
|
104
|
+
|
|
105
|
+
[2] calculator.py:3
|
|
106
|
+
Mutation: Replace numeric constant '0.5' with '1.5'
|
|
107
|
+
Original: return price * 0.5
|
|
108
|
+
Mutated: return price * 1.5
|
|
109
|
+
|
|
110
|
+
[3] calculator.py:3
|
|
111
|
+
Mutation: Replace binary operator '*' with '/'
|
|
112
|
+
Original: return price * 0.5
|
|
113
|
+
Mutated: return price / 0.5
|
|
114
|
+
|
|
115
|
+
====================================================================
|
|
116
|
+
Notice: Local pre-check only. Full verified score runs in CI on push (via mutmut).
|
|
117
|
+
Pre-check FAILED: Score 57.1% is below threshold 80.0% (3 surviving mutants).
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Adding tests for threshold cap (`rate = 0.8`) and exact boundary (`rate = 0.5`) kills all mutants:
|
|
121
|
+
|
|
122
|
+
```
|
|
123
|
+
Local Pre-Check Mutation Verification:
|
|
124
|
+
Score: 100.0% (7/7 mutants killed)
|
|
125
|
+
Status: PASSED (threshold: 80.0%)
|
|
126
|
+
Time: 2.31s
|
|
127
|
+
|
|
128
|
+
Surviving Mutants: None (All generated mutants caught by test suite)
|
|
129
|
+
|
|
130
|
+
====================================================================
|
|
131
|
+
Notice: Local pre-check only. Full verified score runs in CI on push (via mutmut).
|
|
132
|
+
Pre-check clean: 100% of tested basic mutations caught.
|
|
133
|
+
```
|
|
39
134
|
|
|
40
135
|
### CLI Options & Flags
|
|
41
136
|
|
|
@@ -50,6 +145,8 @@ Output includes a section for each check — symlink scan, secrets scan, depende
|
|
|
50
145
|
| `deployproof check --base <ref>` | Base git ref (branch/commit/tag) to diff against. |
|
|
51
146
|
| `deployproof check --wsl` | Delegate mutation testing to `mutmut` inside WSL (Windows only). |
|
|
52
147
|
|
|
148
|
+
> **Note:** `deployproof check --wsl` (Windows only) is newer and less battle-tested than the core checks — [file an issue](https://github.com/SVSPraveen/DeployProof/issues) if you hit something.
|
|
149
|
+
|
|
53
150
|
### Machine-Readable Output (`--json`)
|
|
54
151
|
|
|
55
152
|
DeployProof provides a stable structured JSON schema for CI/CD pipelines, IDEs, and automation:
|
|
@@ -62,7 +159,7 @@ deployproof check --json
|
|
|
62
159
|
|
|
63
160
|
```json
|
|
64
161
|
{
|
|
65
|
-
"version": "0.
|
|
162
|
+
"version": "0.2.0",
|
|
66
163
|
"status": "passed",
|
|
67
164
|
"summary": {
|
|
68
165
|
"target_files_count": 1,
|
|
@@ -135,7 +232,7 @@ deployproof check --json
|
|
|
135
232
|
|
|
136
233
|
## What It Checks
|
|
137
234
|
|
|
138
|
-
- **Mutation Score** — Mutates AST operators (`>=`, `==`, `and`, `or`, `*`, numeric constants, comparisons) in modified files and runs your test suite against each mutant. Reports surviving mutants and a percentage score. Does not use line coverage.
|
|
235
|
+
- **Mutation Score** — Mutates AST operators (`>=`, `==`, `and`, `or`, `*`, numeric constants, comparisons) in modified files and runs your test suite against each mutant. Reports surviving mutants and a percentage score. Does not use line coverage. Features atomic file restoration protected by `SIGINT`/`SIGTERM`/`SIGBREAK` signal handlers to ensure interrupted runs never leave mutated code on disk.
|
|
139
236
|
- **Secrets and Credentials** — Scans modified files for hardcoded API keys (OpenAI, Anthropic, AWS, GitHub, Stripe, private keys) and tracked `.env` files using pattern matching and entropy analysis.
|
|
140
237
|
- **Symlink and Sandbox Escape** — Resolves symbolic links and flags any whose target escapes the repository root (CWE-61 / CWE-451). Catches the class of path-traversal trick used in the GhostApproval disclosure (Wiz Research, July 2026).
|
|
141
238
|
- **Dependency and Slopsquatting** — For each new import, dynamic import (`importlib.import_module`, `__import__`), or manifest entry (including recursive `-r` includes) introduced in the diff, queries the PyPI JSON API and checks registration age. Packages that don't exist (HTTP 404) are flagged HIGH RISK; packages registered within the last 30 days are flagged MEDIUM RISK.
|
|
@@ -151,17 +248,17 @@ deployproof check --json
|
|
|
151
248
|
|
|
152
249
|
## See It Catch Real Bugs
|
|
153
250
|
|
|
154
|
-
Clone this repository and run the standalone stress-test suite to see DeployProof evaluate
|
|
251
|
+
Clone this repository and run the standalone stress-test suite to see DeployProof evaluate 11 planted edge cases:
|
|
155
252
|
|
|
156
253
|
```bash
|
|
157
254
|
python stress_fixtures/run_stress_tests.py
|
|
158
255
|
```
|
|
159
256
|
|
|
160
|
-
Fixtures cover weak test suites, zero-test orphan modules, planted OpenAI/AWS credentials,
|
|
257
|
+
Fixtures cover weak test suites, zero-test orphan modules, planted OpenAI/AWS credentials, GhostApproval sandbox-escape traps, swallowed exceptions/dead code, and mock-masked broken implementations.
|
|
161
258
|
|
|
162
259
|
## Status & Roadmap
|
|
163
260
|
|
|
164
|
-
- **Current (v0.
|
|
261
|
+
- **Current (v0.2.0):** Diff-scoped AST mutation testing (with recursive `test/`/`tests/` discovery, AST column-offset snippet reconstruction, and `SIGINT`/`SIGTERM`/`SIGBREAK` signal-safe disk restoration), baseline test-collection failure isolation with distinct exit code `2`, entropy-driven value-based secrets scanner (including unquoted .env values and prefix validation), GhostApproval symlink sandbox-escape detector, PyPI dependency hallucination / slopsquatting scanner (with import-to-distribution translation, recursive `-r` requirements scanning, and dynamic import detection via `importlib` / `__import__`), mock-introduction detector (`--strict-mocks`), control-flow / swallowed-exception scanner (`--strict-error-handling`), 11/11 launch-day stress test suite, 93 unit tests, machine-readable `--json` output, and Tier 2 CI verification via GitHub Actions (mutmut).
|
|
165
262
|
- **Next:** Multi-language mutation support and expanded ecosystem rule packs.
|
|
166
263
|
|
|
167
264
|
## Contributing
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "deployproof"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "A deterministic AI-code deployability checker that catches what coverage misses."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Command-line interface for DeployProof."""
|
|
2
|
+
import argparse
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import List, Optional
|
|
6
|
+
from deployproof import __version__
|
|
7
|
+
from deployproof.dependencies import extract_all_new_dependencies, scan_dependencies
|
|
8
|
+
from deployproof.control_flow import scan_session_files_for_control_flow
|
|
9
|
+
from deployproof.diff import DiffScopeError, InvalidBaseRefError, NotAGitRepositoryError, get_git_root, is_test_file, resolve_changed_python_files, resolve_changed_session_files
|
|
10
|
+
from deployproof.mocks import scan_session_files_for_mocks
|
|
11
|
+
from deployproof.mutator import run_mutation_tests
|
|
12
|
+
from deployproof.reporter import LARGE_FILE_LOC_THRESHOLD, format_json_report, format_report
|
|
13
|
+
from deployproof.secrets import scan_session_files_for_secrets
|
|
14
|
+
from deployproof.symlinks import scan_session_files_for_symlinks
|
|
15
|
+
from deployproof.wsl import check_wsl_readiness, run_wsl_mutmut
|
|
16
|
+
|
|
17
|
+
def create_parser() -> argparse.ArgumentParser:
|
|
18
|
+
"""Create and configure the command-line argument parser."""
|
|
19
|
+
parser = argparse.ArgumentParser(prog='deployproof', description='DeployProof: A deterministic AI-code deployability checker.')
|
|
20
|
+
parser.add_argument('-v', '--version', action='version', version=f'%(prog)s {__version__}', help="Show program's version number and exit.")
|
|
21
|
+
subparsers = parser.add_subparsers(dest='command', help='Available commands')
|
|
22
|
+
check_parser = subparsers.add_parser('check', help='Run deployability verification checks against session changes.')
|
|
23
|
+
check_parser.add_argument('--base', type=str, default=None, help='Base git ref (branch/commit/tag) to diff against.')
|
|
24
|
+
check_parser.add_argument('--files', nargs='+', type=str, default=None, help='Explicit files to evaluate (bypasses git diff).')
|
|
25
|
+
check_parser.add_argument('--tests', nargs='+', type=str, default=None, help='Specific test file(s) or directories to execute.')
|
|
26
|
+
check_parser.add_argument('--threshold', type=float, default=80.0, help='Minimum mutation score percentage to pass (default: 80.0).')
|
|
27
|
+
check_parser.add_argument('--timeout', type=float, default=10.0, help='Test runner timeout in seconds per mutant (default: 10.0).')
|
|
28
|
+
check_parser.add_argument('--wsl', action='store_true', default=False, help='Delegate mutation testing to mutmut inside WSL (Windows only).')
|
|
29
|
+
check_parser.add_argument('--strict-mocks', action='store_true', default=False, help='Fail the verification gate if new mock or monkeypatch usage is introduced in test files.')
|
|
30
|
+
check_parser.add_argument('--strict-error-handling', action='store_true', default=False, help='Fail the verification gate if bare excepts, swallowed exceptions, or unreachable code are detected.')
|
|
31
|
+
check_parser.add_argument('--json', action='store_true', default=False, help='Output check results as a structured JSON object.')
|
|
32
|
+
subparsers.add_parser('init', help='Initialize DeployProof configuration in the current repository.')
|
|
33
|
+
return parser
|
|
34
|
+
|
|
35
|
+
def handle_check(args: argparse.Namespace) -> int:
|
|
36
|
+
"""Handle the 'check' subcommand."""
|
|
37
|
+
cwd = Path.cwd().resolve()
|
|
38
|
+
repo_root: Optional[Path] = None
|
|
39
|
+
try:
|
|
40
|
+
if args.files:
|
|
41
|
+
session_files = [Path(f).resolve() for f in args.files]
|
|
42
|
+
if session_files:
|
|
43
|
+
try:
|
|
44
|
+
repo_root = get_git_root(session_files[0].parent)
|
|
45
|
+
except DiffScopeError:
|
|
46
|
+
repo_root = session_files[0].parent
|
|
47
|
+
else:
|
|
48
|
+
repo_root = cwd
|
|
49
|
+
else:
|
|
50
|
+
repo_root = get_git_root(cwd)
|
|
51
|
+
session_files = resolve_changed_session_files(cwd=cwd, base=args.base)
|
|
52
|
+
except NotAGitRepositoryError:
|
|
53
|
+
print('Error: Not a git repository. Initialize git or specify files with --files.', file=sys.stderr)
|
|
54
|
+
return 1
|
|
55
|
+
except InvalidBaseRefError as e:
|
|
56
|
+
print(f'Error: {e}', file=sys.stderr)
|
|
57
|
+
return 1
|
|
58
|
+
except DiffScopeError as e:
|
|
59
|
+
print(f'Error: {e}', file=sys.stderr)
|
|
60
|
+
return 1
|
|
61
|
+
if not session_files:
|
|
62
|
+
if getattr(args, 'json', False):
|
|
63
|
+
import json
|
|
64
|
+
print(json.dumps({'version': __version__, 'status': 'passed', 'message': 'No modified files detected in current session.', 'scope': {'target_files': []}, 'summary': {'target_files_count': 0, 'mutation_score': 100.0, 'threshold': args.threshold, 'secrets_found': 0, 'symlink_escapes_found': 0, 'dependency_findings': {'high_risk': 0, 'medium_risk': 0, 'ok': 0, 'unknown': 0, 'unscanned': 0}, 'mock_usages_found': 0, 'control_flow_findings': 0, 'strict_mocks_active': getattr(args, 'strict_mocks', False), 'strict_mocks_triggered': False, 'strict_error_handling_active': getattr(args, 'strict_error_handling', False), 'strict_error_handling_triggered': False}, 'control_flow': {'clean': True, 'strict_gate_triggered': False, 'findings': []}}, indent=2))
|
|
65
|
+
return 0
|
|
66
|
+
else:
|
|
67
|
+
print('DeployProof: No modified files detected in current session.')
|
|
68
|
+
print('Working tree is clean. Use --base <ref> or --files <path...> to evaluate specific files.')
|
|
69
|
+
return 0
|
|
70
|
+
symlink_result = scan_session_files_for_symlinks(session_files, repo_root=repo_root or cwd)
|
|
71
|
+
secrets_result = scan_session_files_for_secrets(session_files)
|
|
72
|
+
extracted_deps = extract_all_new_dependencies(session_files, root=repo_root or cwd, base=args.base)
|
|
73
|
+
dependency_result = scan_dependencies(extracted_deps)
|
|
74
|
+
mock_result = scan_session_files_for_mocks(session_files=session_files, root=repo_root or cwd, base=args.base)
|
|
75
|
+
control_flow_result = scan_session_files_for_control_flow(session_files=session_files, root=repo_root or cwd, base=args.base)
|
|
76
|
+
if args.files:
|
|
77
|
+
non_test_files = [f for f in session_files if f.is_file() and f.suffix == '.py' and (not is_test_file(f))]
|
|
78
|
+
target_files = non_test_files if non_test_files else [f for f in session_files if f.is_file() and f.suffix == '.py']
|
|
79
|
+
else:
|
|
80
|
+
target_files = [f for f in session_files if f.is_file() and f.suffix == '.py' and (not is_test_file(f))]
|
|
81
|
+
if not getattr(args, 'json', False):
|
|
82
|
+
for f in target_files:
|
|
83
|
+
try:
|
|
84
|
+
loc = len(f.read_text(encoding='utf-8', errors='replace').splitlines())
|
|
85
|
+
if loc < LARGE_FILE_LOC_THRESHOLD:
|
|
86
|
+
try:
|
|
87
|
+
rel = f.relative_to(repo_root or cwd)
|
|
88
|
+
except ValueError:
|
|
89
|
+
rel = f
|
|
90
|
+
print(f"Notice: Large file '{rel}' ({loc} LOC) detected - mutation testing may take several minutes.")
|
|
91
|
+
except Exception:
|
|
92
|
+
pass
|
|
93
|
+
if getattr(args, 'wsl', False):
|
|
94
|
+
wsl_ready, wsl_msg = check_wsl_readiness()
|
|
95
|
+
if wsl_ready:
|
|
96
|
+
if not getattr(args, 'json', False):
|
|
97
|
+
print('DeployProof - Delegating to mutmut in WSL...')
|
|
98
|
+
wsl_res = run_wsl_mutmut(repo_root or cwd, target_files)
|
|
99
|
+
if wsl_res.get('success'):
|
|
100
|
+
if not getattr(args, 'json', False):
|
|
101
|
+
print(wsl_res.get('stdout', ''))
|
|
102
|
+
has_security_issue = bool(secrets_result.findings or symlink_result.escape_findings or dependency_result.high_risk_count > 0 or (getattr(args, 'strict_mocks', False) and mock_result.total_findings > 0) or (getattr(args, 'strict_error_handling', False) and control_flow_result.total_findings > 0))
|
|
103
|
+
return 1 if has_security_issue else 0
|
|
104
|
+
elif not getattr(args, 'json', False):
|
|
105
|
+
print(f"WSL execution error: {wsl_res.get('error') or wsl_res.get('stderr')}", file=sys.stderr)
|
|
106
|
+
print('Falling back to Tier 1 local pre-check...')
|
|
107
|
+
elif not getattr(args, 'json', False):
|
|
108
|
+
print(wsl_msg)
|
|
109
|
+
print('-' * 68)
|
|
110
|
+
result = run_mutation_tests(target_files=target_files, repo_root=repo_root or cwd, test_runner_timeout=args.timeout, extra_pytest_args=args.tests)
|
|
111
|
+
if getattr(args, 'json', False):
|
|
112
|
+
report_text = format_json_report(result=result, target_files=target_files, secrets_result=secrets_result, symlink_result=symlink_result, dependency_result=dependency_result, mock_result=mock_result, control_flow_result=control_flow_result, strict_mocks=getattr(args, 'strict_mocks', False), strict_error_handling=getattr(args, 'strict_error_handling', False), repo_root=repo_root or cwd, threshold=args.threshold, version=__version__)
|
|
113
|
+
else:
|
|
114
|
+
report_text = format_report(result=result, target_files=target_files, secrets_result=secrets_result, symlink_result=symlink_result, dependency_result=dependency_result, mock_result=mock_result, control_flow_result=control_flow_result, strict_mocks=getattr(args, 'strict_mocks', False), strict_error_handling=getattr(args, 'strict_error_handling', False), repo_root=repo_root or cwd, threshold=args.threshold)
|
|
115
|
+
print(report_text)
|
|
116
|
+
if result.collection_error:
|
|
117
|
+
return 2
|
|
118
|
+
strict_mocks_triggered = bool(getattr(args, 'strict_mocks', False) and mock_result.total_findings > 0)
|
|
119
|
+
strict_error_triggered = bool(getattr(args, 'strict_error_handling', True) and control_flow_result.total_findings > 0)
|
|
120
|
+
if result.mutation_score is not None and result.mutation_score < args.threshold or len(result.untested_files) > 0 or len(secrets_result.findings) > 0 or (len(symlink_result.escape_findings) > 0) or (dependency_result.high_risk_count > 0) or strict_mocks_triggered or strict_error_triggered:
|
|
121
|
+
return 1
|
|
122
|
+
return 0
|
|
123
|
+
|
|
124
|
+
def handle_init(args: argparse.Namespace) -> int:
|
|
125
|
+
"""Handle the 'init' subcommand to initialize configuration and git hooks."""
|
|
126
|
+
cwd = Path.cwd().resolve()
|
|
127
|
+
try:
|
|
128
|
+
repo_root = get_git_root(cwd)
|
|
129
|
+
except DiffScopeError:
|
|
130
|
+
repo_root = cwd
|
|
131
|
+
print(f'DeployProof: Initializing in {repo_root}...')
|
|
132
|
+
config_path = repo_root / '.deployproof.json'
|
|
133
|
+
if not config_path.exists():
|
|
134
|
+
import json
|
|
135
|
+
default_config = {'version': __version__, 'threshold': 80.0, 'timeout': 10.0, 'secrets_scanning': True, 'symlink_scanning': True, 'dependency_scanning': True}
|
|
136
|
+
config_path.write_text(json.dumps(default_config, indent=2) + '\n', encoding='utf-8')
|
|
137
|
+
print(f' [+] Created configuration file: {config_path.name}')
|
|
138
|
+
else:
|
|
139
|
+
print(f' [.] Configuration file already exists: {config_path.name}')
|
|
140
|
+
hooks_dir = repo_root / '.git' / 'hooks'
|
|
141
|
+
if hooks_dir.is_dir():
|
|
142
|
+
pre_push_hook = hooks_dir / 'pre-push'
|
|
143
|
+
hook_script = '#!/usr/bin/env sh\n# DeployProof deterministic pre-push verification gate\ndeployproof check\n'
|
|
144
|
+
pre_push_hook.write_text(hook_script, encoding='utf-8')
|
|
145
|
+
try:
|
|
146
|
+
import stat
|
|
147
|
+
pre_push_hook.chmod(pre_push_hook.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
|
148
|
+
except Exception:
|
|
149
|
+
pass
|
|
150
|
+
print(' [+] Installed git pre-push hook: .git/hooks/pre-push')
|
|
151
|
+
else:
|
|
152
|
+
print(' [i] Note: .git/hooks directory not found. Initialize git to enable automatic pre-push gating.')
|
|
153
|
+
print("\nDeployProof initialization complete. Run 'deployproof check' to verify your session changes.")
|
|
154
|
+
return 0
|
|
155
|
+
|
|
156
|
+
def main(argv: Optional[List[str]]=None) -> int:
|
|
157
|
+
"""Entry point for the DeployProof CLI."""
|
|
158
|
+
if hasattr(sys.stdout, 'reconfigure'):
|
|
159
|
+
try:
|
|
160
|
+
sys.stdout.reconfigure(errors='replace')
|
|
161
|
+
except Exception:
|
|
162
|
+
pass
|
|
163
|
+
if hasattr(sys.stderr, 'reconfigure'):
|
|
164
|
+
try:
|
|
165
|
+
sys.stderr.reconfigure(errors='replace')
|
|
166
|
+
except Exception:
|
|
167
|
+
pass
|
|
168
|
+
parser = create_parser()
|
|
169
|
+
args = parser.parse_args(argv)
|
|
170
|
+
if not args.command:
|
|
171
|
+
parser.print_help()
|
|
172
|
+
return 0
|
|
173
|
+
if args.command == 'check':
|
|
174
|
+
return handle_check(args)
|
|
175
|
+
elif args.command == 'init':
|
|
176
|
+
return handle_init(args)
|
|
177
|
+
return 0
|
|
178
|
+
if __name__ == '__main__':
|
|
179
|
+
sys.exit(main())
|