guard-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guard_cli-0.1.0/LICENSE +21 -0
- guard_cli-0.1.0/PKG-INFO +241 -0
- guard_cli-0.1.0/README.md +220 -0
- guard_cli-0.1.0/guard/__init__.py +2 -0
- guard_cli-0.1.0/guard/agent_integration.py +122 -0
- guard_cli-0.1.0/guard/api.py +397 -0
- guard_cli-0.1.0/guard/backend.py +475 -0
- guard_cli-0.1.0/guard/cli.py +320 -0
- guard_cli-0.1.0/guard/db.py +271 -0
- guard_cli-0.1.0/guard/decision.py +316 -0
- guard_cli-0.1.0/guard/doctor_cmd.py +475 -0
- guard_cli-0.1.0/guard/events.py +77 -0
- guard_cli-0.1.0/guard/feedback.py +86 -0
- guard_cli-0.1.0/guard/hook.py +281 -0
- guard_cli-0.1.0/guard/ingest/__init__.py +1 -0
- guard_cli-0.1.0/guard/ingest/coverage.py +403 -0
- guard_cli-0.1.0/guard/ingest/static.py +290 -0
- guard_cli-0.1.0/guard/init_cmd.py +437 -0
- guard_cli-0.1.0/guard/integrations/__init__.py +1 -0
- guard_cli-0.1.0/guard/integrations/claude.py +195 -0
- guard_cli-0.1.0/guard/state.py +104 -0
- guard_cli-0.1.0/guard/thin_slice.py +278 -0
- guard_cli-0.1.0/guard_cli/__init__.py +37 -0
- guard_cli-0.1.0/guard_cli/api.py +2 -0
- guard_cli-0.1.0/guard_cli/backend.py +2 -0
- guard_cli-0.1.0/guard_cli/cli.py +3 -0
- guard_cli-0.1.0/guard_cli/db.py +2 -0
- guard_cli-0.1.0/guard_cli/decision.py +2 -0
- guard_cli-0.1.0/guard_cli/doctor_cmd.py +2 -0
- guard_cli-0.1.0/guard_cli/events.py +2 -0
- guard_cli-0.1.0/guard_cli/feedback.py +2 -0
- guard_cli-0.1.0/guard_cli/hook.py +2 -0
- guard_cli-0.1.0/guard_cli/ingest/__init__.py +1 -0
- guard_cli-0.1.0/guard_cli/ingest/coverage.py +1 -0
- guard_cli-0.1.0/guard_cli/ingest/static.py +1 -0
- guard_cli-0.1.0/guard_cli/init_cmd.py +2 -0
- guard_cli-0.1.0/guard_cli/integrations/__init__.py +1 -0
- guard_cli-0.1.0/guard_cli/integrations/claude.py +1 -0
- guard_cli-0.1.0/guard_cli/state.py +2 -0
- guard_cli-0.1.0/guard_cli/thin_slice.py +2 -0
- guard_cli-0.1.0/guard_cli.egg-info/PKG-INFO +241 -0
- guard_cli-0.1.0/guard_cli.egg-info/SOURCES.txt +57 -0
- guard_cli-0.1.0/guard_cli.egg-info/dependency_links.txt +1 -0
- guard_cli-0.1.0/guard_cli.egg-info/entry_points.txt +3 -0
- guard_cli-0.1.0/guard_cli.egg-info/requires.txt +16 -0
- guard_cli-0.1.0/guard_cli.egg-info/top_level.txt +2 -0
- guard_cli-0.1.0/pyproject.toml +39 -0
- guard_cli-0.1.0/setup.cfg +4 -0
- guard_cli-0.1.0/tests/test_api_and_hook.py +347 -0
- guard_cli-0.1.0/tests/test_backend.py +134 -0
- guard_cli-0.1.0/tests/test_coverage.py +124 -0
- guard_cli-0.1.0/tests/test_db.py +146 -0
- guard_cli-0.1.0/tests/test_decision.py +186 -0
- guard_cli-0.1.0/tests/test_doctor_cmd.py +157 -0
- guard_cli-0.1.0/tests/test_feedback.py +70 -0
- guard_cli-0.1.0/tests/test_init_cmd.py +107 -0
- guard_cli-0.1.0/tests/test_integration_init_doctor.py +167 -0
- guard_cli-0.1.0/tests/test_static.py +57 -0
- guard_cli-0.1.0/tests/test_thin_slice.py +81 -0
guard_cli-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Guard Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
guard_cli-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: guard-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Graph-powered safety layer for AI coding agents built on FalkorDB
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: falkordb>=1.0.0
|
|
10
|
+
Requires-Dist: rich>=12.0.0
|
|
11
|
+
Requires-Dist: coverage>=7.0.0
|
|
12
|
+
Provides-Extra: embedded
|
|
13
|
+
Requires-Dist: falkordblite>=0.10.0; python_version >= "3.12" and extra == "embedded"
|
|
14
|
+
Provides-Extra: mcp
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
17
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
|
|
18
|
+
Requires-Dist: coverage>=7.0.0; extra == "dev"
|
|
19
|
+
Requires-Dist: rich>=12.0.0; extra == "dev"
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
# Guard: Graph-Powered Safety Layer for AI Coding Agents
|
|
23
|
+
|
|
24
|
+
[](https://pypi.org/project/guard-cli/)
|
|
25
|
+
[](https://opensource.org/licenses/MIT)
|
|
26
|
+
[](https://www.python.org/downloads/)
|
|
27
|
+
[](https://falkordb.com/)
|
|
28
|
+
|
|
29
|
+
> **A high-velocity, graph-native regression prevention engine built on FalkorDB.**
|
|
30
|
+
> Guard selects the exact minimal subset of tests to run after an AI coding agent modifies code, measures a multi-factor **Trust Score**, and deterministically triggers hard fallbacks to full test suites when graph confidence drops.
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## Quickstart
|
|
35
|
+
|
|
36
|
+
### 1. Install Guard
|
|
37
|
+
```bash
|
|
38
|
+
uv tool install guard-cli
|
|
39
|
+
# Or with embedded FalkorDB support on Python 3.12+:
|
|
40
|
+
# uv tool install "guard-cli[embedded]"
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### 2. Initialize in your Repository
|
|
44
|
+
```bash
|
|
45
|
+
cd /path/to/your/repo
|
|
46
|
+
guard init
|
|
47
|
+
```
|
|
48
|
+
`guard init` auto-detects your Python environment, sets up FalkorDB (server, docker, or embedded), ingests static call graphs & dynamic test coverage, and configures agent integrations (Claude Code Stop hook, AGENTS.md, .gitignore).
|
|
49
|
+
|
|
50
|
+
### 3. Check System Health
|
|
51
|
+
```bash
|
|
52
|
+
guard doctor
|
|
53
|
+
```
|
|
54
|
+
Verifies backend connectivity, graph integrity, git freshness, and integration settings.
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## Verified Benchmark Results (Held-Out Evaluation)
|
|
59
|
+
|
|
60
|
+
Evaluated against **`pallets/flask`** (494 tests) across 24 held-out mutants under a frozen protocol (`evals/split.json`, `PLAN.md`).
|
|
61
|
+
|
|
62
|
+
All metrics are evaluated and reported as **Paired Metrics** (Escape Rate $\leftrightarrow$ Fallback Rate $\leftrightarrow$ Savings Ratio):
|
|
63
|
+
|
|
64
|
+
| Strategy / Engine | Escape Rate (Paired) | Savings Ratio (Paired) | Fallback Rate (Paired) | Killable Evaluated | Escapes |
|
|
65
|
+
| :--- | :---: | :---: | :---: | :---: | :---: |
|
|
66
|
+
| **`run-all`** (Ground Truth) | 0.0% (0/17) | 0.0% | 100.0% (24/24) | 17 | None |
|
|
67
|
+
| **`same-file`** (Heuristic) | 17.6% (3/17) | 77.3% | 0.0% (0/24) | 17 | `m08`, `m16`, `m30` |
|
|
68
|
+
| **`naive-guess`** (Agent Guess) | 17.6% (3/17) | 77.5% | 0.0% (0/24) | 17 | `m08`, `m16`, `m30` |
|
|
69
|
+
| **`guard` (Full System)** | **0.0% (0/17)** | **67.3%** | **25.0% (6/24)** | 17 | **0 (Zero)** |
|
|
70
|
+
| *Ablation 1: `guard-no-confidence`* | 0.0% (0/17) | 77.1% | 20.8% (5/24) | 17 | None |
|
|
71
|
+
| *Ablation 2: `guard-no-widening`* | 0.0% (0/17) | 73.3% | 25.0% (6/24) | 17 | None |
|
|
72
|
+
|
|
73
|
+
> **Key Result**: Guard achieved a **0.0% Escape Rate** while delivering **67.3% Test-Time Savings** compared to `run-all`, completely eliminating the 17.6% escape rate of naive heuristics.
|
|
74
|
+
|
|
75
|
+
### Case Study: Root Cause of `m08` Escape Under `same-file`
|
|
76
|
+
- **Mutant `m08`** inverted an `if not` condition in `src/flask/helpers.py:45`.
|
|
77
|
+
- `same-file` and `naive-guess` selected only `tests/test_helpers.py` (30 tests) $\to$ **0 failures reported (Escape)**.
|
|
78
|
+
- `guard` traversed FalkorDB `COVERED_BY` edges and detected dependent execution paths in `tests/test_basic.py`, `tests/test_blueprints.py`, and `tests/test_templating.py` (26 tests) $\to$ **Caught failure in 0.13s (0.0% escape)**.
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## Problem Statement & Motivation
|
|
83
|
+
|
|
84
|
+
When AI coding agents (Claude Code, Cursor, Codex, OpenCode) modify complex codebases, running the entire test suite on every step slows iteration cycles to a crawl. Conversely, naive heuristics introduce catastrophic silent escapes:
|
|
85
|
+
|
|
86
|
+
- **Same-File Heuristic (`same-file`)**: Only runs tests in the matching test file (e.g., changes in `src/flask/helpers.py` $\to$ run `tests/test_helpers.py`). Foundational helpers, middleware, and lifecycle hooks are heavily exercised across disparate modules (`test_basic.py`, `test_blueprints.py`, `test_templating.py`). On the held-out benchmark, **`same-file` suffered a 17.6% Escape Rate** (3 missed failures).
|
|
87
|
+
- **Agent Guessing (`naive-guess`)**: LLMs predicting test paths based on file names or semantic similarity produce identical blind spots and hallucinate coverage.
|
|
88
|
+
- **Run-All Suite (`run-all`)**: 100% safe, but destroys developer velocity and agent autonomy on non-trivial suites.
|
|
89
|
+
|
|
90
|
+
**Guard's Core Thesis**: Code relationships and test executions form a graph topology. By indexing static AST relationships and per-test dynamic coverage contexts inside **FalkorDB**, Guard computes exact dependency subgraphs, measures empirical test confidence, dynamically widens around ambiguous dynamic dispatch, and enforces hard overrides when evidence is absent.
|
|
91
|
+
|
|
92
|
+
---
|
|
93
|
+
|
|
94
|
+
## Architecture & System Design
|
|
95
|
+
|
|
96
|
+
Guard operates as a deterministic, graph-native verification pipeline connecting static code structure, dynamic coverage telemetry, and autonomous agent loops:
|
|
97
|
+
|
|
98
|
+
1. **AST Diff Analyzer (`guard.thin_slice`)**: Inspects unified git diffs, identifies modified lines, and resolves them to exact function boundaries while flagging test file modifications.
|
|
99
|
+
2. **FalkorDB Graph Engine (`guard.db`)**: Maintains structural call/import relationships and empirical per-test coverage contexts (`COVERED_BY`) in FalkorDB for sub-millisecond Cypher graph traversals.
|
|
100
|
+
3. **Decision Engine (`guard.decision`)**: Pre-emptively checks hard safety overrides (uncovered functions, dynamic dispatch, touched tests), computes the multi-factor Trust Score, and routes execution to the appropriate policy level.
|
|
101
|
+
4. **Self-Healing Feedback Loop (`guard.feedback`)**: Injects high-confidence `miss_repair` edges whenever a missed failure is detected, continuously hardening the graph against future escapes.
|
|
102
|
+
|
|
103
|
+
```
|
|
104
|
+
+-----------------------+
|
|
105
|
+
| AI Agent Git Diff |
|
|
106
|
+
+-----------+-----------+
|
|
107
|
+
|
|
|
108
|
+
v
|
|
109
|
+
+-----------------------+
|
|
110
|
+
| AST Diff Analyzer | -> Identifies modified functions & test files
|
|
111
|
+
+-----------+-----------+
|
|
112
|
+
|
|
|
113
|
+
v
|
|
114
|
+
+-----------------------+
|
|
115
|
+
| FalkorDB Graph Engine| -> Queries COVERED_BY edges, call paths,
|
|
116
|
+
+-----------+-----------+ and dynamic dispatch topology
|
|
117
|
+
|
|
|
118
|
+
v
|
|
119
|
+
+-----------------------+
|
|
120
|
+
| Decision Engine | -> Evaluates hard overrides & Trust Score
|
|
121
|
+
+-----------+-----------+
|
|
122
|
+
|
|
|
123
|
+
+---------+---------+
|
|
124
|
+
| |
|
|
125
|
+
Trust >= 0.8 Trust < 0.8 / Override
|
|
126
|
+
| |
|
|
127
|
+
v v
|
|
128
|
+
+-----------------+ +-----------------+
|
|
129
|
+
| Targeted Subset | | Full Test Suite |
|
|
130
|
+
+-----------------+ +-----------------+
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### FalkorDB Graph Schema
|
|
134
|
+
|
|
135
|
+
#### Nodes
|
|
136
|
+
- `(:File {path})`: Source files and test files in the repository.
|
|
137
|
+
- `(:Function {id, file, name, qualname, start, end, has_dynamic_dispatch, no_evidence})`: Function and method definitions.
|
|
138
|
+
- `(:Test {id, file, name, qualname, start, end, flake_rate})`: Pytest test units.
|
|
139
|
+
- `(:Run {id, timestamp, commit_sha, duration_ms, total_tests, passed, failed})`: Historical test execution runs.
|
|
140
|
+
|
|
141
|
+
#### Edges
|
|
142
|
+
- `(:Function)-[:DEFINED_IN]->(:File)`: Static structural hierarchy.
|
|
143
|
+
- `(:Test)-[:DEFINED_IN]->(:File)`: Static test placement.
|
|
144
|
+
- `(:Function)-[:CALLS {source: 'static', confidence: 0.6, last_verified_sha}]->(:Function)`: Multi-hop static call graph.
|
|
145
|
+
- `(:File)-[:IMPORTS {source: 'static', confidence: 0.6}]->(:File)`: Module import topology.
|
|
146
|
+
- `(:Function)-[:COVERED_BY {source: 'coverage'|'miss_repair', confidence, evidence_count, last_verified_sha}]->(:Test)`: Verified dynamic execution evidence connecting modified functions directly to covering tests.
|
|
147
|
+
- `(:Test)-[:FAILED_IN {source: 'test_run', error_message}]->(:Run)`: Historical test failure records.
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
151
|
+
## Decision Engine & Trust Model
|
|
152
|
+
|
|
153
|
+
Guard evaluates code changes using a strict two-stage process: **Hard Safety Overrides** followed by the **Empirical Trust Score Policy**.
|
|
154
|
+
|
|
155
|
+
### 1. Hard Safety Overrides (Pre-Emptive Gating)
|
|
156
|
+
Before computing continuous scores, Guard enforces three non-bypassable guardrails:
|
|
157
|
+
1. **Test Files Touched**: If any test file is modified or added, Guard forces `run-all` with a safety warning.
|
|
158
|
+
2. **Missing Evidence (`no_evidence`)**: If any modified function lacks dynamic coverage evidence in FalkorDB, Guard automatically falls back to `run-all` (never guess when unmapped).
|
|
159
|
+
3. **Dynamic Dispatch Flagged**: If AST analysis flags dynamic dispatch (`getattr`, `setattr`, `eval`, `**kwargs` dispatch), Guard automatically triggers **Dynamic Widening**, expanding the candidate test set to include all module tests.
|
|
160
|
+
|
|
161
|
+
### 2. Trust Score Formula
|
|
162
|
+
When no hard overrides force `run-all`, Guard computes the composite trust metric:
|
|
163
|
+
|
|
164
|
+
$$\text{Trust} = 0.5 \cdot \text{EvidenceRatio} + 0.3 \cdot \text{Freshness} + 0.2 \cdot (1 - \text{UncertainRatio})$$
|
|
165
|
+
|
|
166
|
+
- $\text{EvidenceRatio}$: Mean of the highest covering-edge confidence per modified function ($0.0$ for functions without covering edges).
|
|
167
|
+
- $\text{Freshness}$: Proportion of evidence edges whose `last_verified_sha` matches the current commit SHA.
|
|
168
|
+
- $\text{UncertainRatio}$: Proportion of modified functions flagged with dynamic dispatch patterns.
|
|
169
|
+
|
|
170
|
+
### 3. Three-Level Execution Policy
|
|
171
|
+
- **$\text{Trust} \ge 0.8$ (`subset`)**: Execute only the directly covering tests mapped via high-confidence `COVERED_BY` edges.
|
|
172
|
+
- **$0.5 \le \text{Trust} < 0.8$ (`medium`)**: Execute the covering subset plus all tests associated with the modified modules.
|
|
173
|
+
- **$\text{Trust} < 0.5$ (`run-all`)**: Enforce full test suite fallback.
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
## Claude Code & Agent Integration
|
|
178
|
+
|
|
179
|
+
Guard integrates seamlessly into autonomous coding workflows:
|
|
180
|
+
|
|
181
|
+
- **Claude Code Stop Hook**: Installed via `guard hook install` into `.claude/settings.local.json`. Whenever Claude attempts to conclude a task, the Stop hook evaluates the diff. If tests fail, it blocks completion (Exit 2) and injects a concise $\le$ 30 line failure report.
|
|
182
|
+
- **Idle-Turn Optimization**: Caches working-tree SHA-256 fingerprints and skips unchanged turns and non-Python diffs.
|
|
183
|
+
- **Loop Guard**: Caps consecutive failures at 3 attempts, handing control back to the user before burning tokens.
|
|
184
|
+
- **Audit Log**: Appends all hook executions, test results, and decisions to `.guard/events.jsonl`.
|
|
185
|
+
|
|
186
|
+
---
|
|
187
|
+
|
|
188
|
+
## CLI Reference
|
|
189
|
+
|
|
190
|
+
### `guard init`
|
|
191
|
+
Initializes Guard in a repository, detects backends, and configures integrations:
|
|
192
|
+
```bash
|
|
193
|
+
guard init [--backend server|docker|embedded] [--skip-coverage]
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### `guard doctor`
|
|
197
|
+
Verifies system health, backend reachability, graph counts, and settings:
|
|
198
|
+
```bash
|
|
199
|
+
guard doctor [--fix]
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
### `guard run`
|
|
203
|
+
Executes agent pre-completion test verification on the current working tree diff:
|
|
204
|
+
```bash
|
|
205
|
+
guard run [--repo /path/to/repo]
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
### `guard check`
|
|
209
|
+
Computes the test selection decision without executing tests (outputs JSON schema v1.0 or text):
|
|
210
|
+
```bash
|
|
211
|
+
guard check [--repo /path/to/repo] [--json]
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### `guard hook`
|
|
215
|
+
Manages Claude Code and agent hook configurations:
|
|
216
|
+
```bash
|
|
217
|
+
guard hook install [--shared]
|
|
218
|
+
guard hook status
|
|
219
|
+
guard hook uninstall
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
### `guard feedback`
|
|
223
|
+
Repairs the graph after an unexpected failure:
|
|
224
|
+
```bash
|
|
225
|
+
guard feedback --function "src/flask/helpers.py::url_for" --test "tests/test_helpers.py::test_url_for"
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
---
|
|
229
|
+
|
|
230
|
+
## Documentation & Visualizations
|
|
231
|
+
|
|
232
|
+
Additional architectural diagrams, interactive visualizations, and test reports are available in the [`docs/`](docs/) directory:
|
|
233
|
+
- [Interactive 3D Flow Architecture](docs/project_flow_3d.html)
|
|
234
|
+
- [Safety Layer Overview](docs/falkordb-safety-layer.html)
|
|
235
|
+
- [Verification Reports](reports/)
|
|
236
|
+
|
|
237
|
+
---
|
|
238
|
+
|
|
239
|
+
## License
|
|
240
|
+
|
|
241
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# Guard: Graph-Powered Safety Layer for AI Coding Agents
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/guard-cli/)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
[](https://www.python.org/downloads/)
|
|
6
|
+
[](https://falkordb.com/)
|
|
7
|
+
|
|
8
|
+
> **A high-velocity, graph-native regression prevention engine built on FalkorDB.**
|
|
9
|
+
> Guard selects the exact minimal subset of tests to run after an AI coding agent modifies code, measures a multi-factor **Trust Score**, and deterministically triggers hard fallbacks to full test suites when graph confidence drops.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Quickstart
|
|
14
|
+
|
|
15
|
+
### 1. Install Guard
|
|
16
|
+
```bash
|
|
17
|
+
uv tool install guard-cli
|
|
18
|
+
# Or with embedded FalkorDB support on Python 3.12+:
|
|
19
|
+
# uv tool install "guard-cli[embedded]"
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
### 2. Initialize in your Repository
|
|
23
|
+
```bash
|
|
24
|
+
cd /path/to/your/repo
|
|
25
|
+
guard init
|
|
26
|
+
```
|
|
27
|
+
`guard init` auto-detects your Python environment, sets up FalkorDB (server, docker, or embedded), ingests static call graphs & dynamic test coverage, and configures agent integrations (Claude Code Stop hook, AGENTS.md, .gitignore).
|
|
28
|
+
|
|
29
|
+
### 3. Check System Health
|
|
30
|
+
```bash
|
|
31
|
+
guard doctor
|
|
32
|
+
```
|
|
33
|
+
Verifies backend connectivity, graph integrity, git freshness, and integration settings.
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## Verified Benchmark Results (Held-Out Evaluation)
|
|
38
|
+
|
|
39
|
+
Evaluated against **`pallets/flask`** (494 tests) across 24 held-out mutants under a frozen protocol (`evals/split.json`, `PLAN.md`).
|
|
40
|
+
|
|
41
|
+
All metrics are evaluated and reported as **Paired Metrics** (Escape Rate $\leftrightarrow$ Fallback Rate $\leftrightarrow$ Savings Ratio):
|
|
42
|
+
|
|
43
|
+
| Strategy / Engine | Escape Rate (Paired) | Savings Ratio (Paired) | Fallback Rate (Paired) | Killable Evaluated | Escapes |
|
|
44
|
+
| :--- | :---: | :---: | :---: | :---: | :---: |
|
|
45
|
+
| **`run-all`** (Ground Truth) | 0.0% (0/17) | 0.0% | 100.0% (24/24) | 17 | None |
|
|
46
|
+
| **`same-file`** (Heuristic) | 17.6% (3/17) | 77.3% | 0.0% (0/24) | 17 | `m08`, `m16`, `m30` |
|
|
47
|
+
| **`naive-guess`** (Agent Guess) | 17.6% (3/17) | 77.5% | 0.0% (0/24) | 17 | `m08`, `m16`, `m30` |
|
|
48
|
+
| **`guard` (Full System)** | **0.0% (0/17)** | **67.3%** | **25.0% (6/24)** | 17 | **0 (Zero)** |
|
|
49
|
+
| *Ablation 1: `guard-no-confidence`* | 0.0% (0/17) | 77.1% | 20.8% (5/24) | 17 | None |
|
|
50
|
+
| *Ablation 2: `guard-no-widening`* | 0.0% (0/17) | 73.3% | 25.0% (6/24) | 17 | None |
|
|
51
|
+
|
|
52
|
+
> **Key Result**: Guard achieved a **0.0% Escape Rate** while delivering **67.3% Test-Time Savings** compared to `run-all`, completely eliminating the 17.6% escape rate of naive heuristics.
|
|
53
|
+
|
|
54
|
+
### Case Study: Root Cause of `m08` Escape Under `same-file`
|
|
55
|
+
- **Mutant `m08`** inverted an `if not` condition in `src/flask/helpers.py:45`.
|
|
56
|
+
- `same-file` and `naive-guess` selected only `tests/test_helpers.py` (30 tests) $\to$ **0 failures reported (Escape)**.
|
|
57
|
+
- `guard` traversed FalkorDB `COVERED_BY` edges and detected dependent execution paths in `tests/test_basic.py`, `tests/test_blueprints.py`, and `tests/test_templating.py` (26 tests) $\to$ **Caught failure in 0.13s (0.0% escape)**.
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## Problem Statement & Motivation
|
|
62
|
+
|
|
63
|
+
When AI coding agents (Claude Code, Cursor, Codex, OpenCode) modify complex codebases, running the entire test suite on every step slows iteration cycles to a crawl. Conversely, naive heuristics introduce catastrophic silent escapes:
|
|
64
|
+
|
|
65
|
+
- **Same-File Heuristic (`same-file`)**: Only runs tests in the matching test file (e.g., changes in `src/flask/helpers.py` $\to$ run `tests/test_helpers.py`). Foundational helpers, middleware, and lifecycle hooks are heavily exercised across disparate modules (`test_basic.py`, `test_blueprints.py`, `test_templating.py`). On the held-out benchmark, **`same-file` suffered a 17.6% Escape Rate** (3 missed failures).
|
|
66
|
+
- **Agent Guessing (`naive-guess`)**: LLMs predicting test paths based on file names or semantic similarity produce identical blind spots and hallucinate coverage.
|
|
67
|
+
- **Run-All Suite (`run-all`)**: 100% safe, but destroys developer velocity and agent autonomy on non-trivial suites.
|
|
68
|
+
|
|
69
|
+
**Guard's Core Thesis**: Code relationships and test executions form a graph topology. By indexing static AST relationships and per-test dynamic coverage contexts inside **FalkorDB**, Guard computes exact dependency subgraphs, measures empirical test confidence, dynamically widens around ambiguous dynamic dispatch, and enforces hard overrides when evidence is absent.
|
|
70
|
+
|
|
71
|
+
---
|
|
72
|
+
|
|
73
|
+
## Architecture & System Design
|
|
74
|
+
|
|
75
|
+
Guard operates as a deterministic, graph-native verification pipeline connecting static code structure, dynamic coverage telemetry, and autonomous agent loops:
|
|
76
|
+
|
|
77
|
+
1. **AST Diff Analyzer (`guard.thin_slice`)**: Inspects unified git diffs, identifies modified lines, and resolves them to exact function boundaries while flagging test file modifications.
|
|
78
|
+
2. **FalkorDB Graph Engine (`guard.db`)**: Maintains structural call/import relationships and empirical per-test coverage contexts (`COVERED_BY`) in FalkorDB for sub-millisecond Cypher graph traversals.
|
|
79
|
+
3. **Decision Engine (`guard.decision`)**: Pre-emptively checks hard safety overrides (uncovered functions, dynamic dispatch, touched tests), computes the multi-factor Trust Score, and routes execution to the appropriate policy level.
|
|
80
|
+
4. **Self-Healing Feedback Loop (`guard.feedback`)**: Injects high-confidence `miss_repair` edges whenever a missed failure is detected, continuously hardening the graph against future escapes.
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
+-----------------------+
|
|
84
|
+
| AI Agent Git Diff |
|
|
85
|
+
+-----------+-----------+
|
|
86
|
+
|
|
|
87
|
+
v
|
|
88
|
+
+-----------------------+
|
|
89
|
+
| AST Diff Analyzer | -> Identifies modified functions & test files
|
|
90
|
+
+-----------+-----------+
|
|
91
|
+
|
|
|
92
|
+
v
|
|
93
|
+
+-----------------------+
|
|
94
|
+
| FalkorDB Graph Engine| -> Queries COVERED_BY edges, call paths,
|
|
95
|
+
+-----------+-----------+ and dynamic dispatch topology
|
|
96
|
+
|
|
|
97
|
+
v
|
|
98
|
+
+-----------------------+
|
|
99
|
+
| Decision Engine | -> Evaluates hard overrides & Trust Score
|
|
100
|
+
+-----------+-----------+
|
|
101
|
+
|
|
|
102
|
+
+---------+---------+
|
|
103
|
+
| |
|
|
104
|
+
Trust >= 0.8 Trust < 0.8 / Override
|
|
105
|
+
| |
|
|
106
|
+
v v
|
|
107
|
+
+-----------------+ +-----------------+
|
|
108
|
+
| Targeted Subset | | Full Test Suite |
|
|
109
|
+
+-----------------+ +-----------------+
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### FalkorDB Graph Schema
|
|
113
|
+
|
|
114
|
+
#### Nodes
|
|
115
|
+
- `(:File {path})`: Source files and test files in the repository.
|
|
116
|
+
- `(:Function {id, file, name, qualname, start, end, has_dynamic_dispatch, no_evidence})`: Function and method definitions.
|
|
117
|
+
- `(:Test {id, file, name, qualname, start, end, flake_rate})`: Pytest test units.
|
|
118
|
+
- `(:Run {id, timestamp, commit_sha, duration_ms, total_tests, passed, failed})`: Historical test execution runs.
|
|
119
|
+
|
|
120
|
+
#### Edges
|
|
121
|
+
- `(:Function)-[:DEFINED_IN]->(:File)`: Static structural hierarchy.
|
|
122
|
+
- `(:Test)-[:DEFINED_IN]->(:File)`: Static test placement.
|
|
123
|
+
- `(:Function)-[:CALLS {source: 'static', confidence: 0.6, last_verified_sha}]->(:Function)`: Multi-hop static call graph.
|
|
124
|
+
- `(:File)-[:IMPORTS {source: 'static', confidence: 0.6}]->(:File)`: Module import topology.
|
|
125
|
+
- `(:Function)-[:COVERED_BY {source: 'coverage'|'miss_repair', confidence, evidence_count, last_verified_sha}]->(:Test)`: Verified dynamic execution evidence connecting modified functions directly to covering tests.
|
|
126
|
+
- `(:Test)-[:FAILED_IN {source: 'test_run', error_message}]->(:Run)`: Historical test failure records.
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## Decision Engine & Trust Model
|
|
131
|
+
|
|
132
|
+
Guard evaluates code changes using a strict two-stage process: **Hard Safety Overrides** followed by the **Empirical Trust Score Policy**.
|
|
133
|
+
|
|
134
|
+
### 1. Hard Safety Overrides (Pre-Emptive Gating)
|
|
135
|
+
Before computing continuous scores, Guard enforces three non-bypassable guardrails:
|
|
136
|
+
1. **Test Files Touched**: If any test file is modified or added, Guard forces `run-all` with a safety warning.
|
|
137
|
+
2. **Missing Evidence (`no_evidence`)**: If any modified function lacks dynamic coverage evidence in FalkorDB, Guard automatically falls back to `run-all` (never guess when unmapped).
|
|
138
|
+
3. **Dynamic Dispatch Flagged**: If AST analysis flags dynamic dispatch (`getattr`, `setattr`, `eval`, `**kwargs` dispatch), Guard automatically triggers **Dynamic Widening**, expanding the candidate test set to include all module tests.
|
|
139
|
+
|
|
140
|
+
### 2. Trust Score Formula
|
|
141
|
+
When no hard overrides force `run-all`, Guard computes the composite trust metric:
|
|
142
|
+
|
|
143
|
+
$$\text{Trust} = 0.5 \cdot \text{EvidenceRatio} + 0.3 \cdot \text{Freshness} + 0.2 \cdot (1 - \text{UncertainRatio})$$
|
|
144
|
+
|
|
145
|
+
- $\text{EvidenceRatio}$: Mean of the highest covering-edge confidence per modified function ($0.0$ for functions without covering edges).
|
|
146
|
+
- $\text{Freshness}$: Proportion of evidence edges whose `last_verified_sha` matches the current commit SHA.
|
|
147
|
+
- $\text{UncertainRatio}$: Proportion of modified functions flagged with dynamic dispatch patterns.
|
|
148
|
+
|
|
149
|
+
### 3. Three-Level Execution Policy
|
|
150
|
+
- **$\text{Trust} \ge 0.8$ (`subset`)**: Execute only the directly covering tests mapped via high-confidence `COVERED_BY` edges.
|
|
151
|
+
- **$0.5 \le \text{Trust} < 0.8$ (`medium`)**: Execute the covering subset plus all tests associated with the modified modules.
|
|
152
|
+
- **$\text{Trust} < 0.5$ (`run-all`)**: Enforce full test suite fallback.
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## Claude Code & Agent Integration
|
|
157
|
+
|
|
158
|
+
Guard integrates seamlessly into autonomous coding workflows:
|
|
159
|
+
|
|
160
|
+
- **Claude Code Stop Hook**: Installed via `guard hook install` into `.claude/settings.local.json`. Whenever Claude attempts to conclude a task, the Stop hook evaluates the diff. If tests fail, it blocks completion (Exit 2) and injects a concise $\le$ 30 line failure report.
|
|
161
|
+
- **Idle-Turn Optimization**: Caches working-tree SHA-256 fingerprints and skips unchanged turns and non-Python diffs.
|
|
162
|
+
- **Loop Guard**: Caps consecutive failures at 3 attempts, handing control back to the user before burning tokens.
|
|
163
|
+
- **Audit Log**: Appends all hook executions, test results, and decisions to `.guard/events.jsonl`.
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
## CLI Reference
|
|
168
|
+
|
|
169
|
+
### `guard init`
|
|
170
|
+
Initializes Guard in a repository, detects backends, and configures integrations:
|
|
171
|
+
```bash
|
|
172
|
+
guard init [--backend server|docker|embedded] [--skip-coverage]
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
### `guard doctor`
|
|
176
|
+
Verifies system health, backend reachability, graph counts, and settings:
|
|
177
|
+
```bash
|
|
178
|
+
guard doctor [--fix]
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
### `guard run`
|
|
182
|
+
Executes agent pre-completion test verification on the current working tree diff:
|
|
183
|
+
```bash
|
|
184
|
+
guard run [--repo /path/to/repo]
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
### `guard check`
|
|
188
|
+
Computes the test selection decision without executing tests (outputs JSON schema v1.0 or text):
|
|
189
|
+
```bash
|
|
190
|
+
guard check [--repo /path/to/repo] [--json]
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### `guard hook`
|
|
194
|
+
Manages Claude Code and agent hook configurations:
|
|
195
|
+
```bash
|
|
196
|
+
guard hook install [--shared]
|
|
197
|
+
guard hook status
|
|
198
|
+
guard hook uninstall
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
### `guard feedback`
|
|
202
|
+
Repairs the graph after an unexpected failure:
|
|
203
|
+
```bash
|
|
204
|
+
guard feedback --function "src/flask/helpers.py::url_for" --test "tests/test_helpers.py::test_url_for"
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
---
|
|
208
|
+
|
|
209
|
+
## Documentation & Visualizations
|
|
210
|
+
|
|
211
|
+
Additional architectural diagrams, interactive visualizations, and test reports are available in the [`docs/`](docs/) directory:
|
|
212
|
+
- [Interactive 3D Flow Architecture](docs/project_flow_3d.html)
|
|
213
|
+
- [Safety Layer Overview](docs/falkordb-safety-layer.html)
|
|
214
|
+
- [Verification Reports](reports/)
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## License
|
|
219
|
+
|
|
220
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Agent integration and test execution runner (Task T7).
|
|
2
|
+
|
|
3
|
+
Provides `guard run`: an agent calls this command before declaring work complete.
|
|
4
|
+
- Calls `guard check` on proposed changes.
|
|
5
|
+
- If trust is low or changes are uncovered (fallback), forces a full test suite run.
|
|
6
|
+
- If trust is high, runs only the selected test subset.
|
|
7
|
+
- Blocks task completion if any test fails.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
import os
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
import subprocess
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from guard.thin_slice import guard_check
|
|
18
|
+
|
|
19
|
+
REPO_ROOT = Path("target_repos/flask").resolve()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def apply_patch(diff_file: Path, repo_path: Path | None = None) -> bool:
|
|
23
|
+
"""Applies patch to target repo."""
|
|
24
|
+
target = repo_path or REPO_ROOT
|
|
25
|
+
proc = subprocess.run(["git", "apply", str(diff_file)], cwd=str(target), capture_output=True, text=True)
|
|
26
|
+
if proc.returncode != 0:
|
|
27
|
+
patch_proc = subprocess.run(["patch", "-p1", "-i", str(diff_file)], cwd=str(target), capture_output=True)
|
|
28
|
+
return patch_proc.returncode == 0
|
|
29
|
+
return True
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def reset_repo(repo_path: Path | None = None):
|
|
33
|
+
"""Restores target repo to clean state."""
|
|
34
|
+
target = repo_path or REPO_ROOT
|
|
35
|
+
subprocess.run(["git", "checkout", "--", "."], cwd=str(target), capture_output=True)
|
|
36
|
+
subprocess.run(["git", "clean", "-fdx"], cwd=str(target), capture_output=True)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def count_collected_tests(test_targets: list[str], repo_path: Path | None = None) -> int:
|
|
40
|
+
"""Counts how many test items are collected for the targets."""
|
|
41
|
+
target = repo_path or REPO_ROOT
|
|
42
|
+
cmd = [sys.executable, "-m", "pytest", "--collect-only", "-q"] + test_targets
|
|
43
|
+
proc = subprocess.run(cmd, cwd=str(target), capture_output=True, text=True)
|
|
44
|
+
lines = proc.stdout.strip().splitlines()
|
|
45
|
+
for line in reversed(lines):
|
|
46
|
+
if "test" in line:
|
|
47
|
+
parts = line.split()
|
|
48
|
+
for p in parts:
|
|
49
|
+
if p.isdigit():
|
|
50
|
+
return int(p)
|
|
51
|
+
return len(test_targets)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def run_agent_guard(
|
|
55
|
+
diff_file: str | Path | None = None,
|
|
56
|
+
repo_path: str | Path | None = None,
|
|
57
|
+
graph_name: str = "guard",
|
|
58
|
+
) -> tuple[dict[str, Any], int]:
|
|
59
|
+
"""Runs Guard check and executes tests according to trust level.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
(report_dict, exit_code)
|
|
63
|
+
"""
|
|
64
|
+
repo = Path(repo_path).resolve() if repo_path else REPO_ROOT
|
|
65
|
+
diff_path = Path(diff_file).resolve() if diff_file else None
|
|
66
|
+
|
|
67
|
+
# 1. Run guard check
|
|
68
|
+
check_res = guard_check(
|
|
69
|
+
diff_file=diff_path,
|
|
70
|
+
repo_path=repo,
|
|
71
|
+
graph_name=graph_name,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
forced_full = check_res.is_fallback or (check_res.status == "run-all")
|
|
75
|
+
tests_to_run = ["tests"] if forced_full else check_res.selected_tests
|
|
76
|
+
|
|
77
|
+
# 2. If a patch file was passed, apply it temporarily for test execution
|
|
78
|
+
applied = False
|
|
79
|
+
if diff_path and diff_path.exists():
|
|
80
|
+
reset_repo()
|
|
81
|
+
applied = apply_patch(diff_path)
|
|
82
|
+
if not applied:
|
|
83
|
+
return {
|
|
84
|
+
"status": "error",
|
|
85
|
+
"forced_full": True,
|
|
86
|
+
"fallback_reason": "Failed to apply patch diff",
|
|
87
|
+
"changed_functions": [],
|
|
88
|
+
"tests_run": [],
|
|
89
|
+
"tests_run_count": 0,
|
|
90
|
+
"duration_s": 0.0,
|
|
91
|
+
"tests_passed": False,
|
|
92
|
+
"exit_code": 1,
|
|
93
|
+
"output_summary": "Error: Failed to apply patch cleanly to target repo.",
|
|
94
|
+
}, 1
|
|
95
|
+
|
|
96
|
+
try:
|
|
97
|
+
# Count collected tests
|
|
98
|
+
test_count = count_collected_tests(tests_to_run)
|
|
99
|
+
|
|
100
|
+
# 3. Execute pytest
|
|
101
|
+
cmd = [sys.executable, "-m", "pytest", "-q", "--tb=short"] + tests_to_run
|
|
102
|
+
start = time.time()
|
|
103
|
+
proc = subprocess.run(cmd, cwd=str(repo), capture_output=True, text=True)
|
|
104
|
+
duration_s = time.time() - start
|
|
105
|
+
|
|
106
|
+
passed = (proc.returncode == 0)
|
|
107
|
+
report = {
|
|
108
|
+
"status": check_res.status,
|
|
109
|
+
"forced_full": forced_full,
|
|
110
|
+
"fallback_reason": check_res.fallback_reason,
|
|
111
|
+
"changed_functions": check_res.changed_functions,
|
|
112
|
+
"tests_run": tests_to_run,
|
|
113
|
+
"tests_run_count": test_count,
|
|
114
|
+
"duration_s": round(duration_s, 3),
|
|
115
|
+
"tests_passed": passed,
|
|
116
|
+
"exit_code": proc.returncode,
|
|
117
|
+
"output_summary": proc.stdout.strip().splitlines()[-1] if proc.stdout.strip() else "",
|
|
118
|
+
}
|
|
119
|
+
return report, proc.returncode
|
|
120
|
+
finally:
|
|
121
|
+
if applied:
|
|
122
|
+
reset_repo()
|