guard-cli 0.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- guard/__init__.py +2 -0
- guard/agent_integration.py +119 -0
- guard/cli.py +194 -0
- guard/db.py +243 -0
- guard/decision.py +316 -0
- guard/feedback.py +86 -0
- guard/ingest/__init__.py +1 -0
- guard/ingest/coverage.py +390 -0
- guard/ingest/static.py +290 -0
- guard/thin_slice.py +266 -0
- guard_cli-0.1.0rc1.dist-info/METADATA +210 -0
- guard_cli-0.1.0rc1.dist-info/RECORD +15 -0
- guard_cli-0.1.0rc1.dist-info/WHEEL +5 -0
- guard_cli-0.1.0rc1.dist-info/entry_points.txt +2 -0
- guard_cli-0.1.0rc1.dist-info/top_level.txt +1 -0
guard/thin_slice.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""Thin slice end-to-end decision engine (Task T5a).
|
|
2
|
+
|
|
3
|
+
Parses a git diff, maps changed line numbers to FalkorDB Function nodes,
|
|
4
|
+
queries covering tests from COVERED_BY edges, and applies crude decision logic:
|
|
5
|
+
- If ANY changed function has no_evidence=true (or 0 covering tests) -> run all.
|
|
6
|
+
- Otherwise -> select the covering tests.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
import re
|
|
10
|
+
import subprocess
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from guard import db
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class ChangedFunction:
|
|
20
|
+
id: str
|
|
21
|
+
file: str
|
|
22
|
+
name: str
|
|
23
|
+
start: int
|
|
24
|
+
end: int
|
|
25
|
+
no_evidence: bool
|
|
26
|
+
has_dynamic_dispatch: bool = False
|
|
27
|
+
covering_tests: list[str] = field(default_factory=list)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class CheckResult:
|
|
32
|
+
status: str # "select" | "run-all"
|
|
33
|
+
is_fallback: bool
|
|
34
|
+
fallback_reason: str | None
|
|
35
|
+
changed_functions: list[str]
|
|
36
|
+
selected_tests: list[str]
|
|
37
|
+
execution_time_ms: float = 0.0
|
|
38
|
+
|
|
39
|
+
def to_dict(self) -> dict[str, Any]:
|
|
40
|
+
return {
|
|
41
|
+
"status": self.status,
|
|
42
|
+
"is_fallback": self.is_fallback,
|
|
43
|
+
"fallback_reason": self.fallback_reason,
|
|
44
|
+
"changed_functions": self.changed_functions,
|
|
45
|
+
"selected_tests": self.selected_tests,
|
|
46
|
+
"selected_tests_count": len(self.selected_tests),
|
|
47
|
+
"execution_time_ms": self.execution_time_ms,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def parse_unified_diff(diff_text: str) -> dict[str, set[int]]:
|
|
52
|
+
"""Parses a unified diff into a map of {relative_file_path: set(modified_lines)}."""
|
|
53
|
+
file_lines: dict[str, set[int]] = {}
|
|
54
|
+
current_file = None
|
|
55
|
+
current_line = 0
|
|
56
|
+
|
|
57
|
+
hunk_regex = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
|
|
58
|
+
|
|
59
|
+
for line in diff_text.splitlines():
|
|
60
|
+
if line.startswith("+++ b/"):
|
|
61
|
+
raw_path = line[6:].strip()
|
|
62
|
+
# Normalize path: strip repo prefixes if present
|
|
63
|
+
current_file = raw_path
|
|
64
|
+
if current_file not in file_lines:
|
|
65
|
+
file_lines[current_file] = set()
|
|
66
|
+
current_line = 0
|
|
67
|
+
continue
|
|
68
|
+
elif line.startswith("+++ "):
|
|
69
|
+
raw_path = line[4:].strip()
|
|
70
|
+
if raw_path != "/dev/null":
|
|
71
|
+
current_file = raw_path.lstrip("b/")
|
|
72
|
+
if current_file not in file_lines:
|
|
73
|
+
file_lines[current_file] = set()
|
|
74
|
+
current_line = 0
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
if not current_file:
|
|
78
|
+
continue
|
|
79
|
+
|
|
80
|
+
hunk_match = hunk_regex.match(line)
|
|
81
|
+
if hunk_match:
|
|
82
|
+
current_line = int(hunk_match.group(1))
|
|
83
|
+
continue
|
|
84
|
+
|
|
85
|
+
if line.startswith("+") and not line.startswith("+++"):
|
|
86
|
+
file_lines[current_file].add(current_line)
|
|
87
|
+
current_line += 1
|
|
88
|
+
elif line.startswith("-") and not line.startswith("---"):
|
|
89
|
+
# Deletion occurs at current_line
|
|
90
|
+
file_lines[current_file].add(current_line)
|
|
91
|
+
elif line.startswith(" "):
|
|
92
|
+
current_line += 1
|
|
93
|
+
|
|
94
|
+
return file_lines
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def resolve_changed_functions(g, file_lines: dict[str, set[int]]) -> list[ChangedFunction]:
|
|
98
|
+
"""Queries FalkorDB to find Function nodes spanning the modified lines."""
|
|
99
|
+
changed_funcs: dict[str, ChangedFunction] = {}
|
|
100
|
+
|
|
101
|
+
for file_path, lines in file_lines.items():
|
|
102
|
+
if not lines:
|
|
103
|
+
continue
|
|
104
|
+
|
|
105
|
+
# In FalkorDB, File nodes use paths like 'src/flask/config.py'
|
|
106
|
+
# Query functions defined in this file
|
|
107
|
+
q = (
|
|
108
|
+
"MATCH (f:Function) "
|
|
109
|
+
"WHERE f.file = $file OR f.file ENDS WITH $file OR $file ENDS WITH f.file "
|
|
110
|
+
"RETURN f.id, f.file, f.name, f.start, f.end, coalesce(f.no_evidence, false), coalesce(f.has_dynamic_dispatch, false)"
|
|
111
|
+
)
|
|
112
|
+
res = g.query(q, {"file": file_path}).result_set
|
|
113
|
+
|
|
114
|
+
for fid, ffile, fname, start, end, no_evidence, dyn in res:
|
|
115
|
+
start_i = int(start)
|
|
116
|
+
end_i = int(end)
|
|
117
|
+
# Check if any modified line falls within [start, end]
|
|
118
|
+
if any(start_i <= lineno <= end_i for lineno in lines):
|
|
119
|
+
if fid not in changed_funcs:
|
|
120
|
+
changed_funcs[fid] = ChangedFunction(
|
|
121
|
+
id=fid,
|
|
122
|
+
file=ffile,
|
|
123
|
+
name=fname,
|
|
124
|
+
start=start_i,
|
|
125
|
+
end=end_i,
|
|
126
|
+
no_evidence=bool(no_evidence),
|
|
127
|
+
has_dynamic_dispatch=bool(dyn),
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
return list(changed_funcs.values())
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def thin_slice_decision(
|
|
134
|
+
g,
|
|
135
|
+
changed_functions: list[ChangedFunction],
|
|
136
|
+
test_files_touched: bool = False,
|
|
137
|
+
no_confidence_filter: bool = False,
|
|
138
|
+
no_dynamic_widening: bool = False,
|
|
139
|
+
) -> CheckResult:
|
|
140
|
+
"""Decision logic:
|
|
141
|
+
- If test files were touched or no functions matched -> run all.
|
|
142
|
+
- If ANY changed function has no_evidence=true (and not no_confidence_filter) -> run all.
|
|
143
|
+
- Otherwise -> collect covering tests via COVERED_BY edges (with confidence filter unless ablated).
|
|
144
|
+
- If dynamic dispatch is present and widening enabled -> include module tests.
|
|
145
|
+
"""
|
|
146
|
+
if test_files_touched:
|
|
147
|
+
return CheckResult(
|
|
148
|
+
status="run-all",
|
|
149
|
+
is_fallback=True,
|
|
150
|
+
fallback_reason="Test files were modified in diff",
|
|
151
|
+
changed_functions=[f.id for f in changed_functions],
|
|
152
|
+
selected_tests=["tests"],
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
if not changed_functions:
|
|
156
|
+
return CheckResult(
|
|
157
|
+
status="run-all",
|
|
158
|
+
is_fallback=True,
|
|
159
|
+
fallback_reason="No parsed functions matched diff line changes",
|
|
160
|
+
changed_functions=[],
|
|
161
|
+
selected_tests=["tests"],
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Hard override: If any function has no evidence -> fallback to run all (unless confidence filter ablated)
|
|
165
|
+
if not no_confidence_filter:
|
|
166
|
+
for fn in changed_functions:
|
|
167
|
+
if fn.no_evidence:
|
|
168
|
+
return CheckResult(
|
|
169
|
+
status="run-all",
|
|
170
|
+
is_fallback=True,
|
|
171
|
+
fallback_reason=f"Uncovered function has no_evidence=true: {fn.id}",
|
|
172
|
+
changed_functions=[f.id for f in changed_functions],
|
|
173
|
+
selected_tests=["tests"],
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# Collect covering tests from graph
|
|
177
|
+
all_tests = set()
|
|
178
|
+
min_conf = 0.0 if no_confidence_filter else 0.8
|
|
179
|
+
for fn in changed_functions:
|
|
180
|
+
covering = db.get_covering_tests(g, fn.id, min_confidence=min_conf) if g is not None else []
|
|
181
|
+
fn_tests = [c["test_id"] for c in covering]
|
|
182
|
+
if not fn_tests and not no_confidence_filter:
|
|
183
|
+
# No covering tests found
|
|
184
|
+
return CheckResult(
|
|
185
|
+
status="run-all",
|
|
186
|
+
is_fallback=True,
|
|
187
|
+
fallback_reason=f"No covering tests found with confidence >= 0.8 for: {fn.id}",
|
|
188
|
+
changed_functions=[f.id for f in changed_functions],
|
|
189
|
+
selected_tests=["tests"],
|
|
190
|
+
)
|
|
191
|
+
fn.covering_tests = fn_tests
|
|
192
|
+
all_tests.update(fn_tests)
|
|
193
|
+
|
|
194
|
+
# Dynamic dispatch widening
|
|
195
|
+
if not no_dynamic_widening and g is not None:
|
|
196
|
+
dynamic_funcs = [fn for fn in changed_functions if fn.has_dynamic_dispatch]
|
|
197
|
+
if dynamic_funcs:
|
|
198
|
+
for fn in dynamic_funcs:
|
|
199
|
+
stem = Path(fn.file).stem
|
|
200
|
+
mod_q = "MATCH (t:Test) WHERE t.file CONTAINS $mod_stem RETURN t.id"
|
|
201
|
+
try:
|
|
202
|
+
mod_res = g.query(mod_q, {"mod_stem": f"test_{stem}"}).result_set
|
|
203
|
+
for r in mod_res:
|
|
204
|
+
all_tests.add(r[0])
|
|
205
|
+
except Exception:
|
|
206
|
+
pass
|
|
207
|
+
|
|
208
|
+
if not all_tests:
|
|
209
|
+
return CheckResult(
|
|
210
|
+
status="run-all",
|
|
211
|
+
is_fallback=True,
|
|
212
|
+
fallback_reason="No covering tests found",
|
|
213
|
+
changed_functions=[f.id for f in changed_functions],
|
|
214
|
+
selected_tests=["tests"],
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
sorted_tests = sorted(all_tests)
|
|
218
|
+
return CheckResult(
|
|
219
|
+
status="select",
|
|
220
|
+
is_fallback=False,
|
|
221
|
+
fallback_reason=None,
|
|
222
|
+
changed_functions=[f.id for f in changed_functions],
|
|
223
|
+
selected_tests=sorted_tests,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def guard_check(
|
|
228
|
+
diff_text: str | None = None,
|
|
229
|
+
diff_file: Path | str | None = None,
|
|
230
|
+
repo_path: Path | str | None = None,
|
|
231
|
+
graph_name: str = "guard",
|
|
232
|
+
no_confidence_filter: bool = False,
|
|
233
|
+
no_dynamic_widening: bool = False,
|
|
234
|
+
) -> CheckResult:
|
|
235
|
+
"""Evaluates a diff and returns the test selection decision."""
|
|
236
|
+
import time
|
|
237
|
+
start_time = time.time()
|
|
238
|
+
|
|
239
|
+
if diff_text is None:
|
|
240
|
+
if diff_file:
|
|
241
|
+
diff_text = Path(diff_file).read_text(encoding="utf-8")
|
|
242
|
+
else:
|
|
243
|
+
# Run git diff HEAD in repo_path
|
|
244
|
+
cwd = str(repo_path) if repo_path else None
|
|
245
|
+
proc = subprocess.run(["git", "diff", "HEAD"], cwd=cwd, capture_output=True, text=True)
|
|
246
|
+
diff_text = proc.stdout
|
|
247
|
+
|
|
248
|
+
file_lines = parse_unified_diff(diff_text)
|
|
249
|
+
|
|
250
|
+
# Check if test files were touched
|
|
251
|
+
test_files_touched = any(
|
|
252
|
+
"test" in p.lower() or p.startswith("tests/")
|
|
253
|
+
for p in file_lines.keys()
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
g = db.get_graph(graph_name)
|
|
257
|
+
changed_funcs = resolve_changed_functions(g, file_lines)
|
|
258
|
+
result = thin_slice_decision(
|
|
259
|
+
g,
|
|
260
|
+
changed_funcs,
|
|
261
|
+
test_files_touched=test_files_touched,
|
|
262
|
+
no_confidence_filter=no_confidence_filter,
|
|
263
|
+
no_dynamic_widening=no_dynamic_widening,
|
|
264
|
+
)
|
|
265
|
+
result.execution_time_ms = round((time.time() - start_time) * 1000.0, 2)
|
|
266
|
+
return result
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: guard-cli
|
|
3
|
+
Version: 0.1.0rc1
|
|
4
|
+
Summary: Graph-powered safety layer for AI coding agents built on FalkorDB
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: falkordb>=1.0.0
|
|
8
|
+
Provides-Extra: dev
|
|
9
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
10
|
+
Requires-Dist: pytest-cov>=5.0.0; extra == "dev"
|
|
11
|
+
Requires-Dist: coverage>=7.0.0; extra == "dev"
|
|
12
|
+
|
|
13
|
+
# Guard: Graph-Powered Safety Layer for AI Coding Agents
|
|
14
|
+
|
|
15
|
+
> **A high-velocity, graph-native regression prevention engine built on FalkorDB.**
|
|
16
|
+
> Guard selects the exact minimal subset of tests to run after an AI coding agent modifies code, measures a multi-factor **Trust Score**, and deterministically triggers hard fallbacks to full test suites when graph confidence drops.
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## 1. Problem Statement & Motivation
|
|
21
|
+
|
|
22
|
+
When AI coding agents (Claude Code, Cursor, Codex, OpenCode) modify complex codebases, running the entire test suite on every step slows iteration cycles to a crawl. Conversely, naive heuristics used by developers or agents to speed things up introduce catastrophic silent escapes:
|
|
23
|
+
|
|
24
|
+
- **Same-File Heuristic (`same-file`)**: Only runs tests in the matching test file (e.g., changes in `src/flask/helpers.py` $\to$ run `tests/test_helpers.py`). In real production architectures, foundational helpers, middleware, and lifecycle hooks are heavily exercised across disparate modules (`test_basic.py`, `test_blueprints.py`, `test_templating.py`). On the held-out benchmark, **`same-file` suffered a 17.6% Escape Rate** (3 missed failures).
|
|
25
|
+
- **Agent Guessing (`naive-guess`)**: LLMs predicting test paths based on file names or semantic similarity produce identical blind spots and hallucinate coverage.
|
|
26
|
+
- **Run-All Suite (`run-all`)**: 100% safe, but destroys developer velocity and agent autonomy on non-trivial suites.
|
|
27
|
+
|
|
28
|
+
**Guard's Core Thesis**: Code relationships and test executions form a graph topology. By indexing static AST relationships and per-test dynamic coverage contexts inside **FalkorDB**, Guard computes exact dependency subgraphs, measures empirical test confidence, dynamically widens around ambiguous dynamic dispatch, and enforces hard overrides when evidence is absent.
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## 2. Verified Benchmark Results (Held-Out Evaluation)
|
|
33
|
+
|
|
34
|
+
Guard was evaluated against **`pallets/flask`** (494 tests) across 24 held-out mutants under a frozen protocol (`evals/split.json`, `PLAN.md`).
|
|
35
|
+
|
|
36
|
+
All metrics are evaluated and reported as **Paired Metrics** (Escape Rate $\leftrightarrow$ Fallback Rate $\leftrightarrow$ Savings Ratio):
|
|
37
|
+
|
|
38
|
+
| Strategy / Engine | Escape Rate (Paired) | Savings Ratio (Paired) | Fallback Rate (Paired) | Killable Evaluated | Escapes |
|
|
39
|
+
| :--- | :---: | :---: | :---: | :---: | :---: |
|
|
40
|
+
| **`run-all`** (Ground Truth) | 0.0% (0/17) | 0.0% | 100.0% (24/24) | 17 | None |
|
|
41
|
+
| **`same-file`** (Heuristic) | 17.6% (3/17) | 77.3% | 0.0% (0/24) | 17 | `m08`, `m16`, `m30` |
|
|
42
|
+
| **`naive-guess`** (Agent Guess) | 17.6% (3/17) | 77.5% | 0.0% (0/24) | 17 | `m08`, `m16`, `m30` |
|
|
43
|
+
| **`guard` (Full System)** | **0.0% (0/17)** | **67.3%** | **25.0% (6/24)** | 17 | **0 (Zero)** |
|
|
44
|
+
| *Ablation 1: `guard-no-confidence`* | 0.0% (0/17) | 77.1% | 20.8% (5/24) | 17 | None |
|
|
45
|
+
| *Ablation 2: `guard-no-widening`* | 0.0% (0/17) | 73.3% | 25.0% (6/24) | 17 | None |
|
|
46
|
+
|
|
47
|
+
> **Key Result**: Guard achieved a **0.0% Escape Rate** while delivering **67.3% Test-Time Savings** compared to `run-all`, completely eliminating the 17.6% escape rate of `same-file`.
|
|
48
|
+
|
|
49
|
+
### Case Study: Root Cause of `m08` Escape Under `same-file`
|
|
50
|
+
- **Mutant `m08`** inverted an `if not` statement in `src/flask/helpers.py:45`.
|
|
51
|
+
- `same-file` and `naive-guess` selected `tests/test_helpers.py` (30 tests) $\to$ **0 failures reported (Escape)**.
|
|
52
|
+
- `guard` traversed FalkorDB `COVERED_BY` edges and detected dependent execution paths in `tests/test_basic.py`, `tests/test_blueprints.py`, and `tests/test_templating.py` (26 tests) $\to$ **Caught failure immediately (0.0% escape)**.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## 3. Architecture & System Design
|
|
57
|
+
|
|
58
|
+
Guard operates as a deterministic, graph-native verification pipeline connecting static code structure, dynamic coverage telemetry, and autonomous agent loops:
|
|
59
|
+
|
|
60
|
+
1. **AST Diff Analyzer (`guard.thin_slice`)**: Inspects unified git diffs, identifies modified lines, and resolves them to exact function boundaries while flagging test file modifications.
|
|
61
|
+
2. **FalkorDB Graph Engine (`guard.db`)**: Maintains structural call/import relationships and empirical per-test coverage contexts (`COVERED_BY`) in FalkorDB for sub-millisecond Cypher graph traversals.
|
|
62
|
+
3. **Decision Engine (`guard.decision`)**: Pre-emptively checks hard safety overrides (uncovered functions, dynamic dispatch, touched tests), computes the multi-factor Trust Score, and routes execution to the appropriate policy level.
|
|
63
|
+
4. **Self-Healing Feedback Loop (`guard.feedback`)**: Injects high-confidence `miss_repair` edges whenever a missed failure is detected, continuously hardening the graph against future escapes.
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
### 3.1. FalkorDB Graph Schema
|
|
67
|
+
|
|
68
|
+
The graph layer leverages FalkorDB for low-latency Cypher queries and multi-hop graph traversals:
|
|
69
|
+
|
|
70
|
+
#### Nodes
|
|
71
|
+
- `(:File {path})`: Source files and test files in the repository.
|
|
72
|
+
- `(:Function {id, file, name, qualname, start, end, has_dynamic_dispatch, no_evidence})`: Function and method definitions.
|
|
73
|
+
- `(:Test {id, file, name, qualname, start, end, flake_rate})`: Pytest test units.
|
|
74
|
+
- `(:Run {id, timestamp, commit_sha, duration_ms, total_tests, passed, failed})`: Historical test execution runs.
|
|
75
|
+
|
|
76
|
+
#### Edges
|
|
77
|
+
- `(:Function)-[:DEFINED_IN]->(:File)`: Static structural hierarchy.
|
|
78
|
+
- `(:Test)-[:DEFINED_IN]->(:File)`: Static test placement.
|
|
79
|
+
- `(:Function)-[:CALLS {source: 'static', confidence: 0.6, last_verified_sha}]->(:Function)`: Multi-hop static call graph.
|
|
80
|
+
- `(:File)-[:IMPORTS {source: 'static', confidence: 0.6}]->(:File)`: Module import topology.
|
|
81
|
+
- `(:Function)-[:COVERED_BY {source: 'coverage'|'miss_repair', confidence, evidence_count, last_verified_sha}]->(:Test)`: Verified dynamic execution evidence connecting modified functions directly to covering tests.
|
|
82
|
+
- `(:Test)-[:FAILED_IN {source: 'test_run', error_message}]->(:Run)`: Historical test failure records.
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## 4. Decision Engine & Trust Model
|
|
87
|
+
|
|
88
|
+
Guard evaluates code changes using a strict two-stage process: **Hard Safety Overrides** followed by the **Empirical Trust Score Policy**.
|
|
89
|
+
|
|
90
|
+
### 4.1. Hard Safety Overrides (Pre-Emptive Gating)
|
|
91
|
+
Before computing continuous scores, Guard enforces three non-bypassable guardrails:
|
|
92
|
+
1. **Test Files Touched**: If any test file is modified or added, Guard forces `run-all` with a safety warning (modifying assertions could invalidate test isolation).
|
|
93
|
+
2. **Missing Evidence (`no_evidence`)**: If any modified function lacks dynamic coverage evidence in FalkorDB, Guard automatically falls back to `run-all` (never guess when unmapped).
|
|
94
|
+
3. **Dynamic Dispatch Flagged**: If AST analysis flags dynamic dispatch (`getattr`, `setattr`, `eval`, `**kwargs` dispatch), Guard automatically triggers **Dynamic Widening**, expanding the candidate test set to include all module tests.
|
|
95
|
+
|
|
96
|
+
### 4.2. Trust Score Formula
|
|
97
|
+
When no hard overrides force `run-all`, Guard computes the composite trust metric:
|
|
98
|
+
|
|
99
|
+
$$\text{Trust} = 0.5 \cdot \text{EvidenceRatio} + 0.3 \cdot \text{Freshness} + 0.2 \cdot (1 - \text{UncertainRatio})$$
|
|
100
|
+
|
|
101
|
+
- $\text{EvidenceRatio}$: Mean of the highest covering-edge confidence per modified function ($0.0$ for functions without covering edges).
|
|
102
|
+
- $\text{Freshness}$: Proportion of evidence edges whose `last_verified_sha` matches the current commit SHA.
|
|
103
|
+
- $\text{UncertainRatio}$: Proportion of modified functions flagged with dynamic dispatch patterns.
|
|
104
|
+
|
|
105
|
+
### 4.3. Three-Level Execution Policy
|
|
106
|
+
- **$\text{Trust} \ge 0.8$ (`subset`)**: Execute only the directly covering tests mapped via high-confidence `COVERED_BY` edges.
|
|
107
|
+
- **$0.5 \le \text{Trust} < 0.8$ (`medium`)**: Execute the covering subset plus all tests associated with the modified modules.
|
|
108
|
+
- **$\text{Trust} < 0.5$ (`run-all`)**: Enforce full test suite fallback.
|
|
109
|
+
|
|
110
|
+
---
|
|
111
|
+
|
|
112
|
+
## 5. Self-Healing Feedback Loop
|
|
113
|
+
|
|
114
|
+
When an unexpected regression or missed test failure occurs:
|
|
115
|
+
1. The failure event is captured via `guard feedback --function <func_id> --test <test_id>`.
|
|
116
|
+
2. Guard writes a synthetic repair edge `(:Function)-[:COVERED_BY {source: 'miss_repair', confidence: 1.0}]->(:Test)`.
|
|
117
|
+
3. The `no_evidence` flag on the function is cleared.
|
|
118
|
+
4. Future decisions for this function or its callers automatically select the repaired test, preventing recurring escapes.
|
|
119
|
+
|
|
120
|
+
---
|
|
121
|
+
|
|
122
|
+
## 6. Installation & Quickstart
|
|
123
|
+
|
|
124
|
+
### Prerequisites
|
|
125
|
+
- Python 3.10+
|
|
126
|
+
- Docker & Docker Compose (for FalkorDB)
|
|
127
|
+
|
|
128
|
+
### 1. Start FalkorDB
|
|
129
|
+
```bash
|
|
130
|
+
docker compose up -d
|
|
131
|
+
# FalkorDB is live on localhost:6379
|
|
132
|
+
# Web UI is accessible on http://localhost:3000
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
### 2. Install Guard
|
|
136
|
+
```bash
|
|
137
|
+
pip install -e .
|
|
138
|
+
# Or install with dev dependencies:
|
|
139
|
+
pip install -e ".[dev]"
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
### 3. Ingest Target Repository
|
|
143
|
+
```bash
|
|
144
|
+
# 1. Ingest static AST topology into FalkorDB
|
|
145
|
+
guard ingest-static /path/to/target/repo --include-tests
|
|
146
|
+
|
|
147
|
+
# 2. Ingest per-test coverage contexts (runs pytest with --cov-context=test)
|
|
148
|
+
guard ingest-coverage /path/to/target/repo --runs 3
|
|
149
|
+
|
|
150
|
+
# 3. View database statistics
|
|
151
|
+
guard stats
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## 7. CLI Reference
|
|
157
|
+
|
|
158
|
+
### `guard run`
|
|
159
|
+
Executes agent pre-completion test verification on the current working tree diff:
|
|
160
|
+
```bash
|
|
161
|
+
guard run --repo /path/to/target/repo
|
|
162
|
+
```
|
|
163
|
+
*Exits with code 0 on success, or code 1 if tests fail or trust is breached, blocking autonomous agents from committing broken code.*
|
|
164
|
+
|
|
165
|
+
### `guard check`
|
|
166
|
+
Computes the test selection decision without executing tests (outputs JSON or human-readable format):
|
|
167
|
+
```bash
|
|
168
|
+
guard check --repo /path/to/target/repo --json
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### `guard callers`
|
|
172
|
+
Traverses multi-hop static call graph in FalkorDB up to $N$ hops:
|
|
173
|
+
```bash
|
|
174
|
+
guard callers "src/flask/app.py::Flask.dispatch_request" --hops 4 --min-confidence 0.6
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### `guard covering-tests`
|
|
178
|
+
Queries all tests covering a specific function ordered by edge confidence:
|
|
179
|
+
```bash
|
|
180
|
+
guard covering-tests "src/flask/helpers.py::url_for"
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
### `guard feedback`
|
|
184
|
+
Repairs the graph after a missed failure:
|
|
185
|
+
```bash
|
|
186
|
+
guard feedback --function "src/flask/helpers.py::url_for" --test "tests/test_helpers.py::test_url_for"
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
---
|
|
190
|
+
|
|
191
|
+
## 8. Architectural Trade-offs & Limits
|
|
192
|
+
|
|
193
|
+
### What Graph-Native Architecture Enables
|
|
194
|
+
- **Deterministic Traceability**: Every skipped test has an explicit, queryable reason (`guard check --json` outputs edge justifications for all known tests).
|
|
195
|
+
- **Cross-Module Awareness**: Identifies deep, non-obvious test linkages across package boundaries that escape keyword search or lexical matching.
|
|
196
|
+
- **Fail-Safe Convergence**: Combines static structure with empirical dynamic execution contexts so that gaps trigger defensive full runs rather than silent failures.
|
|
197
|
+
|
|
198
|
+
### Known Limits and Edge Cases
|
|
199
|
+
1. **Dynamic Python Metaprogramming**: Heavy dynamic dispatch (`getattr(module, var)()`, custom metaclasses, dynamic decorators) cannot be completely resolved via static AST. *Mitigation*: Guard detects dynamic syntax patterns and triggers dynamic widening or full fallbacks.
|
|
200
|
+
2. **Initial Ingestion Cost**: Generating fine-grained per-test coverage contexts requires an initial baseline run (`pytest --cov-context=test`), which can incur a 1.5x–2.0x slowdown during graph ingestion.
|
|
201
|
+
3. **Ecosystem Scope**: Currently tuned for Python repositories using `pytest`.
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## 9. Verification & Quality Invariants
|
|
206
|
+
|
|
207
|
+
Guard development adheres to strict engineering rules (`PLAN.md` R1–R9):
|
|
208
|
+
- **Frozen Benchmarks**: Acceptance claims and split files are immutable and verified through independent verifier runs (`reports/verify_t*.md`).
|
|
209
|
+
- **Zero Flake Tolerance**: Flaky tests are detected during ingest runs and excluded from dynamic evidence edges.
|
|
210
|
+
- **Complete Test Coverage**: Pure decision logic, parsers, and graph writers are covered with unit and integration tests under `tests/`.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
guard/__init__.py,sha256=7k8BGYVwznPf9SRb_g3qURokR4KFGdg6kXgk2a9g23g,87
|
|
2
|
+
guard/agent_integration.py,sha256=K7-JHCEhH73FSVkXSyd4wHMLmpu6oJ7ocrttbUWC2rg,4264
|
|
3
|
+
guard/cli.py,sha256=fXp7kWlXO5a5u9bS-8uCS6dzwE88JT06imiHPcqQgQo,6641
|
|
4
|
+
guard/db.py,sha256=fovdVIkoLNNUd-cn0lWXJ12mtuwJuMaTDDnmidhKZzs,8089
|
|
5
|
+
guard/decision.py,sha256=4ce5HKe7hSJkSvXIBraF-deJw6Eb4KL5f2qIMDPeJtU,10475
|
|
6
|
+
guard/feedback.py,sha256=uNE0KfyRRUmjbhKI79zc_ZxI6WdCRjMd03G0cBm-KJE,2786
|
|
7
|
+
guard/thin_slice.py,sha256=f6rhhUsMRjrOvTWLvrdNWjtwyzrARAS88KlIl12Qu0I,9480
|
|
8
|
+
guard/ingest/__init__.py,sha256=pa21RdOiIC9igJzjRXnErQdbMINGd2kP3KuzZllVuSE,35
|
|
9
|
+
guard/ingest/coverage.py,sha256=GixQBSphd_FTUXb-NEXOrxSVLnpp7SA3lqHPWorQsNg,13581
|
|
10
|
+
guard/ingest/static.py,sha256=abE6u_XhaYG1u9B52T_GQBEVBN3vU_tvp4qRNDAvkmc,10290
|
|
11
|
+
guard_cli-0.1.0rc1.dist-info/METADATA,sha256=WrkFHkwy31U488Vu6oBIkMHY6qHpuL2fhYMrwviymLY,11923
|
|
12
|
+
guard_cli-0.1.0rc1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
13
|
+
guard_cli-0.1.0rc1.dist-info/entry_points.txt,sha256=o2V98n4yg8cN6puiafJO8uHqCjnkvRukHTrbn9bfmlM,41
|
|
14
|
+
guard_cli-0.1.0rc1.dist-info/top_level.txt,sha256=wDYWf5msUK3ABsBg4by00UMHAthwEG1wNePS4M571Lo,6
|
|
15
|
+
guard_cli-0.1.0rc1.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
guard
|