evolveguard-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evolveguard/__init__.py +93 -0
- evolveguard/cli.py +255 -0
- evolveguard/diff/__init__.py +200 -0
- evolveguard/errors.py +17 -0
- evolveguard/fixtures.py +106 -0
- evolveguard/formatters.py +90 -0
- evolveguard/parser/__init__.py +0 -0
- evolveguard/parser/skillmd.py +249 -0
- evolveguard/paths.py +78 -0
- evolveguard/record/__init__.py +30 -0
- evolveguard/replay/__init__.py +35 -0
- evolveguard/report/__init__.py +95 -0
- evolveguard/snapshot.py +113 -0
- evolveguard/types.py +292 -0
- evolveguard_cli-0.1.0.dist-info/METADATA +275 -0
- evolveguard_cli-0.1.0.dist-info/RECORD +19 -0
- evolveguard_cli-0.1.0.dist-info/WHEEL +4 -0
- evolveguard_cli-0.1.0.dist-info/entry_points.txt +2 -0
- evolveguard_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
evolveguard/snapshot.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Loads a skill file from disk and builds per-fixture tool-call sequence
|
|
3
|
+
snapshots against its capability surface. Ported from src/evolveguard/snapshot.ts.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from fnmatch import fnmatch
|
|
10
|
+
from typing import List
|
|
11
|
+
|
|
12
|
+
from .errors import EvolveGuardError
|
|
13
|
+
from .parser.skillmd import derive_capability_surface, parse_skill_file
|
|
14
|
+
from .types import CapabilityEntry, ExpectedToolCall, Fixture, FixtureSnapshot
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class LoadedSkill:
|
|
19
|
+
name: str
|
|
20
|
+
skill_path: str
|
|
21
|
+
skill_dir: str
|
|
22
|
+
capability_surface: List[CapabilityEntry]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def load_skill(skill_path: str) -> LoadedSkill:
|
|
26
|
+
"""Reads a skill file from disk, parses it, and derives its full capability surface."""
|
|
27
|
+
try:
|
|
28
|
+
with open(skill_path, "r", encoding="utf-8") as fh:
|
|
29
|
+
content = fh.read()
|
|
30
|
+
except OSError as err:
|
|
31
|
+
raise EvolveGuardError(
|
|
32
|
+
f'Could not read skill file at "{skill_path}".',
|
|
33
|
+
str(err),
|
|
34
|
+
"Pass a valid path to a SKILL.md or MEMORY.md file.",
|
|
35
|
+
) from err
|
|
36
|
+
|
|
37
|
+
skill_dir = os.path.dirname(os.path.abspath(skill_path))
|
|
38
|
+
parsed = parse_skill_file(content, skill_path)
|
|
39
|
+
capability_surface = derive_capability_surface(parsed, skill_dir)
|
|
40
|
+
|
|
41
|
+
return LoadedSkill(
|
|
42
|
+
name=parsed.name,
|
|
43
|
+
skill_path=os.path.abspath(skill_path),
|
|
44
|
+
skill_dir=skill_dir,
|
|
45
|
+
capability_surface=capability_surface,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _scope_glob_match(text: str, pattern: str) -> bool:
|
|
50
|
+
"""
|
|
51
|
+
A minimal, first-party glob matcher used only to compare two declared
|
|
52
|
+
filesystem scopes (e.g. "./workspace/**" vs "./**") for compatibility --
|
|
53
|
+
it is never run against the filesystem. `fnmatch` treats `*` as matching
|
|
54
|
+
any character including path separators, which mirrors this project's
|
|
55
|
+
only actual use case (checking whether one scope glob subsumes another),
|
|
56
|
+
without pulling in a third-party glob-matching dependency.
|
|
57
|
+
"""
|
|
58
|
+
return fnmatch(text, pattern)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def build_fixture_snapshots(
|
|
62
|
+
surface: List[CapabilityEntry], fixtures: List[Fixture]
|
|
63
|
+
) -> List[FixtureSnapshot]:
|
|
64
|
+
"""
|
|
65
|
+
Builds the per-fixture tool-call sequence snapshot for a given capability
|
|
66
|
+
surface. A fixture with no expected_tool_calls is treated as exercising
|
|
67
|
+
the skill's entire surface (a broad smoke-test fixture); a fixture that
|
|
68
|
+
declares specific expected tool calls only snapshots the surface entries
|
|
69
|
+
matching those tools (and, if scope_matches is given, only entries whose
|
|
70
|
+
declared/inferred scope satisfies that glob).
|
|
71
|
+
"""
|
|
72
|
+
snapshots: List[FixtureSnapshot] = []
|
|
73
|
+
|
|
74
|
+
for fixture in fixtures:
|
|
75
|
+
expected = fixture.expected_tool_calls or []
|
|
76
|
+
|
|
77
|
+
if not expected:
|
|
78
|
+
snapshots.append(
|
|
79
|
+
FixtureSnapshot(
|
|
80
|
+
id=fixture.id,
|
|
81
|
+
prompt=fixture.prompt,
|
|
82
|
+
expected_tool_calls=[],
|
|
83
|
+
tool_call_sequence=surface,
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
continue
|
|
87
|
+
|
|
88
|
+
def matches(entry: CapabilityEntry) -> bool:
|
|
89
|
+
for exp in expected:
|
|
90
|
+
if exp.tool != entry.tool:
|
|
91
|
+
continue
|
|
92
|
+
if not exp.scope_matches:
|
|
93
|
+
return True
|
|
94
|
+
if not entry.scope:
|
|
95
|
+
continue
|
|
96
|
+
if _scope_glob_match(entry.scope, exp.scope_matches) or _scope_glob_match(
|
|
97
|
+
exp.scope_matches, entry.scope
|
|
98
|
+
):
|
|
99
|
+
return True
|
|
100
|
+
return False
|
|
101
|
+
|
|
102
|
+
tool_call_sequence = [entry for entry in surface if matches(entry)]
|
|
103
|
+
|
|
104
|
+
snapshots.append(
|
|
105
|
+
FixtureSnapshot(
|
|
106
|
+
id=fixture.id,
|
|
107
|
+
prompt=fixture.prompt,
|
|
108
|
+
expected_tool_calls=expected,
|
|
109
|
+
tool_call_sequence=tool_call_sequence,
|
|
110
|
+
)
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
return snapshots
|
evolveguard/types.py
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Shared types for EvolveGuard's parse -> record -> replay -> diff pipeline.
|
|
3
|
+
Ported from src/evolveguard/types.ts.
|
|
4
|
+
|
|
5
|
+
Design note (see README "How it works"): EvolveGuard never invokes a live
|
|
6
|
+
LLM agent. A "capability surface" is a deterministic snapshot derived from
|
|
7
|
+
a skill file's own declared frontmatter scope plus static evidence found
|
|
8
|
+
in any bundled hook scripts it references -- the same static-analysis
|
|
9
|
+
mechanism SkillGuard uses for its declared-vs-actual scope check, applied
|
|
10
|
+
here to before/after comparison instead of security auditing.
|
|
11
|
+
|
|
12
|
+
Every dataclass below round-trips through `to_dict()`/`from_dict()` using
|
|
13
|
+
the exact same camelCase JSON key names the TypeScript package writes, so
|
|
14
|
+
a baseline or report file produced by one distribution (npm or PyPI) can
|
|
15
|
+
be read by the other -- schemaVersion is the compatibility contract.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from typing import Any, Dict, List, Optional
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class EvidenceRef:
|
|
25
|
+
"""File:line evidence, present only for inferred entries."""
|
|
26
|
+
|
|
27
|
+
file: str
|
|
28
|
+
line: int
|
|
29
|
+
|
|
30
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
31
|
+
return {"file": self.file, "line": self.line}
|
|
32
|
+
|
|
33
|
+
@staticmethod
|
|
34
|
+
def from_dict(d: Dict[str, Any]) -> "EvidenceRef":
|
|
35
|
+
return EvidenceRef(file=d["file"], line=d["line"])
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class CapabilityEntry:
|
|
40
|
+
"""A single capability a skill declares or is inferred to use."""
|
|
41
|
+
|
|
42
|
+
tool: str
|
|
43
|
+
source: str # "declared" | "inferred"
|
|
44
|
+
scope: Optional[str] = None
|
|
45
|
+
evidence: Optional[List[EvidenceRef]] = None
|
|
46
|
+
|
|
47
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
48
|
+
d: Dict[str, Any] = {"tool": self.tool, "source": self.source}
|
|
49
|
+
if self.scope is not None:
|
|
50
|
+
d["scope"] = self.scope
|
|
51
|
+
if self.evidence is not None:
|
|
52
|
+
d["evidence"] = [e.to_dict() for e in self.evidence]
|
|
53
|
+
return d
|
|
54
|
+
|
|
55
|
+
@staticmethod
|
|
56
|
+
def from_dict(d: Dict[str, Any]) -> "CapabilityEntry":
|
|
57
|
+
evidence = d.get("evidence")
|
|
58
|
+
return CapabilityEntry(
|
|
59
|
+
tool=d["tool"],
|
|
60
|
+
source=d["source"],
|
|
61
|
+
scope=d.get("scope"),
|
|
62
|
+
evidence=[EvidenceRef.from_dict(e) for e in evidence]
|
|
63
|
+
if evidence is not None
|
|
64
|
+
else None,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
CapabilitySurface = List[CapabilityEntry]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class DeclaredScope:
|
|
73
|
+
"""Declared scope parsed directly out of a skill file's YAML frontmatter."""
|
|
74
|
+
|
|
75
|
+
tools: List[str] = field(default_factory=list)
|
|
76
|
+
network: bool = False
|
|
77
|
+
filesystem: str = "none" # "none" | "read-only" | "read-write"
|
|
78
|
+
scope: str = "./**"
|
|
79
|
+
hooks: List[str] = field(default_factory=list)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass
|
|
83
|
+
class ParsedSkillFile:
|
|
84
|
+
name: str
|
|
85
|
+
has_frontmatter: bool
|
|
86
|
+
declared_scope: DeclaredScope
|
|
87
|
+
body: str
|
|
88
|
+
description: Optional[str] = None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass
|
|
92
|
+
class ExpectedToolCall:
|
|
93
|
+
tool: str
|
|
94
|
+
scope_matches: Optional[str] = None
|
|
95
|
+
|
|
96
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
97
|
+
d: Dict[str, Any] = {"tool": self.tool}
|
|
98
|
+
if self.scope_matches is not None:
|
|
99
|
+
d["scopeMatches"] = self.scope_matches
|
|
100
|
+
return d
|
|
101
|
+
|
|
102
|
+
@staticmethod
|
|
103
|
+
def from_dict(d: Dict[str, Any]) -> "ExpectedToolCall":
|
|
104
|
+
return ExpectedToolCall(tool=d["tool"], scope_matches=d.get("scopeMatches"))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass
|
|
108
|
+
class Fixture:
|
|
109
|
+
"""A single labeled fixture from the user-supplied fixtures.json."""
|
|
110
|
+
|
|
111
|
+
id: str
|
|
112
|
+
prompt: str
|
|
113
|
+
expected_tool_calls: Optional[List[ExpectedToolCall]] = None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@dataclass
|
|
117
|
+
class FixtureSnapshot:
|
|
118
|
+
"""The tool-call sequence snapshot recorded (or replayed) for one fixture."""
|
|
119
|
+
|
|
120
|
+
id: str
|
|
121
|
+
prompt: str
|
|
122
|
+
expected_tool_calls: List[ExpectedToolCall]
|
|
123
|
+
tool_call_sequence: List[CapabilityEntry]
|
|
124
|
+
|
|
125
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
126
|
+
return {
|
|
127
|
+
"id": self.id,
|
|
128
|
+
"prompt": self.prompt,
|
|
129
|
+
"expectedToolCalls": [e.to_dict() for e in self.expected_tool_calls],
|
|
130
|
+
"toolCallSequence": [e.to_dict() for e in self.tool_call_sequence],
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def from_dict(d: Dict[str, Any]) -> "FixtureSnapshot":
|
|
135
|
+
return FixtureSnapshot(
|
|
136
|
+
id=d["id"],
|
|
137
|
+
prompt=d["prompt"],
|
|
138
|
+
expected_tool_calls=[
|
|
139
|
+
ExpectedToolCall.from_dict(e) for e in d.get("expectedToolCalls", [])
|
|
140
|
+
],
|
|
141
|
+
tool_call_sequence=[
|
|
142
|
+
CapabilityEntry.from_dict(e) for e in d.get("toolCallSequence", [])
|
|
143
|
+
],
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@dataclass
|
|
148
|
+
class Baseline:
|
|
149
|
+
schema_version: int
|
|
150
|
+
skill_name: str
|
|
151
|
+
skill_path: str
|
|
152
|
+
recorded_at: str
|
|
153
|
+
full_capability_surface: List[CapabilityEntry]
|
|
154
|
+
fixtures: List[FixtureSnapshot]
|
|
155
|
+
|
|
156
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
157
|
+
return {
|
|
158
|
+
"schemaVersion": self.schema_version,
|
|
159
|
+
"skillName": self.skill_name,
|
|
160
|
+
"skillPath": self.skill_path,
|
|
161
|
+
"recordedAt": self.recorded_at,
|
|
162
|
+
"fullCapabilitySurface": [e.to_dict() for e in self.full_capability_surface],
|
|
163
|
+
"fixtures": [f.to_dict() for f in self.fixtures],
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
@staticmethod
|
|
167
|
+
def from_dict(d: Dict[str, Any]) -> "Baseline":
|
|
168
|
+
return Baseline(
|
|
169
|
+
schema_version=d["schemaVersion"],
|
|
170
|
+
skill_name=d["skillName"],
|
|
171
|
+
skill_path=d["skillPath"],
|
|
172
|
+
recorded_at=d["recordedAt"],
|
|
173
|
+
full_capability_surface=[
|
|
174
|
+
CapabilityEntry.from_dict(e) for e in d.get("fullCapabilitySurface", [])
|
|
175
|
+
],
|
|
176
|
+
fixtures=[FixtureSnapshot.from_dict(f) for f in d.get("fixtures", [])],
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@dataclass
|
|
181
|
+
class ReplayResult:
|
|
182
|
+
schema_version: int
|
|
183
|
+
skill_name: str
|
|
184
|
+
skill_path: str
|
|
185
|
+
replayed_at: str
|
|
186
|
+
full_capability_surface: List[CapabilityEntry]
|
|
187
|
+
fixtures: List[FixtureSnapshot]
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
@dataclass
|
|
191
|
+
class CapabilityChange:
|
|
192
|
+
kind: str # "added" | "removed" | "scope-widened" | "scope-changed"
|
|
193
|
+
tool: str
|
|
194
|
+
message: str
|
|
195
|
+
baseline_scope: Optional[str] = None
|
|
196
|
+
new_scope: Optional[str] = None
|
|
197
|
+
|
|
198
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
199
|
+
d: Dict[str, Any] = {"kind": self.kind, "tool": self.tool, "message": self.message}
|
|
200
|
+
if self.baseline_scope is not None:
|
|
201
|
+
d["baselineScope"] = self.baseline_scope
|
|
202
|
+
if self.new_scope is not None:
|
|
203
|
+
d["newScope"] = self.new_scope
|
|
204
|
+
return d
|
|
205
|
+
|
|
206
|
+
@staticmethod
|
|
207
|
+
def from_dict(d: Dict[str, Any]) -> "CapabilityChange":
|
|
208
|
+
return CapabilityChange(
|
|
209
|
+
kind=d["kind"],
|
|
210
|
+
tool=d["tool"],
|
|
211
|
+
message=d["message"],
|
|
212
|
+
baseline_scope=d.get("baselineScope"),
|
|
213
|
+
new_scope=d.get("newScope"),
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@dataclass
|
|
218
|
+
class FixtureDiff:
|
|
219
|
+
id: str
|
|
220
|
+
prompt: str
|
|
221
|
+
verdict: str # "PASS" | "DRIFT"
|
|
222
|
+
changes: List[CapabilityChange]
|
|
223
|
+
|
|
224
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
225
|
+
return {
|
|
226
|
+
"id": self.id,
|
|
227
|
+
"prompt": self.prompt,
|
|
228
|
+
"verdict": self.verdict,
|
|
229
|
+
"changes": [c.to_dict() for c in self.changes],
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
@staticmethod
|
|
233
|
+
def from_dict(d: Dict[str, Any]) -> "FixtureDiff":
|
|
234
|
+
return FixtureDiff(
|
|
235
|
+
id=d["id"],
|
|
236
|
+
prompt=d["prompt"],
|
|
237
|
+
verdict=d["verdict"],
|
|
238
|
+
changes=[CapabilityChange.from_dict(c) for c in d.get("changes", [])],
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
@dataclass
|
|
243
|
+
class Summary:
|
|
244
|
+
pass_count: int
|
|
245
|
+
drift: int
|
|
246
|
+
total: int
|
|
247
|
+
|
|
248
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
249
|
+
return {"pass": self.pass_count, "drift": self.drift, "total": self.total}
|
|
250
|
+
|
|
251
|
+
@staticmethod
|
|
252
|
+
def from_dict(d: Dict[str, Any]) -> "Summary":
|
|
253
|
+
return Summary(pass_count=d["pass"], drift=d["drift"], total=d["total"])
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
@dataclass
|
|
257
|
+
class EvolveGuardReport:
|
|
258
|
+
schema_version: int
|
|
259
|
+
skill_name: str
|
|
260
|
+
skill_path: str
|
|
261
|
+
checked_at: str
|
|
262
|
+
results: List[FixtureDiff]
|
|
263
|
+
surface_changes: List[CapabilityChange]
|
|
264
|
+
summary: Summary
|
|
265
|
+
exit_code: int # 0 | 1 | 2
|
|
266
|
+
|
|
267
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
268
|
+
return {
|
|
269
|
+
"schemaVersion": self.schema_version,
|
|
270
|
+
"skillName": self.skill_name,
|
|
271
|
+
"skillPath": self.skill_path,
|
|
272
|
+
"checkedAt": self.checked_at,
|
|
273
|
+
"results": [r.to_dict() for r in self.results],
|
|
274
|
+
"surfaceChanges": [c.to_dict() for c in self.surface_changes],
|
|
275
|
+
"summary": self.summary.to_dict(),
|
|
276
|
+
"exitCode": self.exit_code,
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
@staticmethod
|
|
280
|
+
def from_dict(d: Dict[str, Any]) -> "EvolveGuardReport":
|
|
281
|
+
return EvolveGuardReport(
|
|
282
|
+
schema_version=d["schemaVersion"],
|
|
283
|
+
skill_name=d["skillName"],
|
|
284
|
+
skill_path=d["skillPath"],
|
|
285
|
+
checked_at=d["checkedAt"],
|
|
286
|
+
results=[FixtureDiff.from_dict(r) for r in d.get("results", [])],
|
|
287
|
+
surface_changes=[
|
|
288
|
+
CapabilityChange.from_dict(c) for c in d.get("surfaceChanges", [])
|
|
289
|
+
],
|
|
290
|
+
summary=Summary.from_dict(d["summary"]),
|
|
291
|
+
exit_code=d["exitCode"],
|
|
292
|
+
)
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: evolveguard-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Regression-testing CI gate for self-edited Claude Agent Skills (SKILL.md, MEMORY.md): golden-transcript record/replay against a skill's own declared and inferred capability surface, zero hosted infrastructure.
|
|
5
|
+
Project-URL: Homepage, https://github.com/RudrenduPaul/evolveguard
|
|
6
|
+
Project-URL: Repository, https://github.com/RudrenduPaul/evolveguard
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/RudrenduPaul/evolveguard/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/RudrenduPaul/evolveguard/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: Documentation, https://github.com/RudrenduPaul/evolveguard/blob/main/docs/getting-started.md
|
|
10
|
+
Project-URL: Author - Rudrendu Paul, https://github.com/RudrenduPaul
|
|
11
|
+
Project-URL: Author - Sourav Nandy, https://github.com/Sourav-nandy-ai
|
|
12
|
+
Author: Rudrendu Paul, Sourav Nandy
|
|
13
|
+
License-Expression: MIT
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Keywords: agent-skills,ai-agents,ci-gate,claude-code,cli-tool,developer-tools,eval-harness,regression-testing
|
|
16
|
+
Classifier: Development Status :: 3 - Alpha
|
|
17
|
+
Classifier: Environment :: Console
|
|
18
|
+
Classifier: Intended Audience :: Developers
|
|
19
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
20
|
+
Classifier: Operating System :: OS Independent
|
|
21
|
+
Classifier: Programming Language :: Python :: 3
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
27
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
28
|
+
Classifier: Topic :: Software Development :: Testing
|
|
29
|
+
Requires-Python: >=3.9
|
|
30
|
+
Requires-Dist: pyyaml<7,>=6.0
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: build<2,>=1.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest<9,>=7.0; extra == 'dev'
|
|
34
|
+
Requires-Dist: twine<7,>=5.0; extra == 'dev'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# evolveguard (Python)
|
|
38
|
+
|
|
39
|
+
Regression-testing CI gate for self-edited Claude Agent Skills -- `SKILL.md`
|
|
40
|
+
manifests and Claude Code auto-memory `MEMORY.md` files -- catching
|
|
41
|
+
behavioral drift before an edit ships.
|
|
42
|
+
|
|
43
|
+
[](https://pypi.org/project/evolveguard/)
|
|
44
|
+
[](https://github.com/RudrenduPaul/evolveguard/blob/main/LICENSE)
|
|
45
|
+
[](https://pypi.org/project/evolveguard/)
|
|
46
|
+
[](https://github.com/RudrenduPaul/evolveguard/actions/workflows/ci.yml)
|
|
47
|
+
|
|
48
|
+
## Why this exists
|
|
49
|
+
|
|
50
|
+
Claude Code's Agent Skills can be authored by a human, or by an agent
|
|
51
|
+
itself: `/skillify` turns a workflow into a `SKILL.md`, and the auto-memory
|
|
52
|
+
system in this same environment writes `MEMORY.md` files that quietly
|
|
53
|
+
change what an agent does in its next session. None of that gets a
|
|
54
|
+
regression check by default. A skill edit that breaks a working workflow
|
|
55
|
+
looks exactly like a skill edit that fixes one, until someone notices the
|
|
56
|
+
agent stopped doing something it used to do. evolveguard records a
|
|
57
|
+
baseline of a skill's own capability surface (what tools it's declared or
|
|
58
|
+
shown to use), then re-derives that surface every time the skill file
|
|
59
|
+
changes and diffs the result against the baseline: same tool-call
|
|
60
|
+
sequence, or a flagged drift with a specific reason.
|
|
61
|
+
|
|
62
|
+
## Install
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install evolveguard
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
or with [uv](https://docs.astral.sh/uv/):
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
uv add evolveguard
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
> **Not yet live on PyPI.** The package is fully built, tested, and
|
|
75
|
+
> publish-ready (wheel + sdist built, inspected, and verified end to end
|
|
76
|
+
> from a fresh venv install), but the first `twine upload` on this account
|
|
77
|
+
> is currently blocked by PyPI's own new-project-creation anti-abuse
|
|
78
|
+
> throttle (`429 Too many new projects created`), confirmed across
|
|
79
|
+
> repeated upload attempts -- not a code or readiness issue. Until that
|
|
80
|
+
> clears, clone the repo and install from source:
|
|
81
|
+
>
|
|
82
|
+
> ```bash
|
|
83
|
+
> git clone https://github.com/RudrenduPaul/evolveguard.git
|
|
84
|
+
> cd evolveguard/python && pip install -e .
|
|
85
|
+
> ```
|
|
86
|
+
|
|
87
|
+
**About the npm package:** `evolveguard` is also registered as an npm
|
|
88
|
+
package name, but publishing it is currently blocked by a separate,
|
|
89
|
+
unrelated account-level 2FA constraint (the npm account's second factor is
|
|
90
|
+
security-key/passkey only, with no authenticator-app or OTP fallback
|
|
91
|
+
configured). Clone the repo and run `npm run build && npm link` if you
|
|
92
|
+
need the TypeScript CLI locally in the meantime.
|
|
93
|
+
|
|
94
|
+
## Quickstart
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
# 1. Record a baseline against a skill and its labeled fixtures
|
|
98
|
+
evolveguard record ./skills/my-skill/SKILL.md --fixtures ./fixtures/my-skill.json
|
|
99
|
+
# writes ./skills/my-skill/.evolveguard-baseline.json
|
|
100
|
+
|
|
101
|
+
# 2. Edit the skill (by hand, or let an agent edit it)
|
|
102
|
+
|
|
103
|
+
# 3. Check for drift
|
|
104
|
+
evolveguard check ./skills/my-skill/SKILL.md
|
|
105
|
+
# writes ./evolveguard-report.json, exits 1 if drift was found
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
A fixtures file is a JSON array of labeled prompts and the tool-call
|
|
109
|
+
shapes each one is expected to touch:
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
[
|
|
113
|
+
{
|
|
114
|
+
"id": "scan-a-monorepo",
|
|
115
|
+
"prompt": "scan a monorepo",
|
|
116
|
+
"expectedToolCalls": [{ "tool": "fs.read" }, { "tool": "fs.write" }]
|
|
117
|
+
}
|
|
118
|
+
]
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`expectedToolCalls` is optional -- omit it and the fixture is treated as
|
|
122
|
+
exercising the skill's entire capability surface. `scopeMatches` (a glob)
|
|
123
|
+
narrows a tool to a specific filesystem scope, e.g.
|
|
124
|
+
`{ "tool": "fs.write", "scopeMatches": "./workspace/**" }`.
|
|
125
|
+
|
|
126
|
+
Or call the library directly (the agent-native path):
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from evolveguard import record_baseline, replay_skill, diff_all, write_baseline, read_baseline
|
|
130
|
+
|
|
131
|
+
baseline = record_baseline("./SKILL.md", "./fixtures.json")
|
|
132
|
+
write_baseline("./.evolveguard-baseline.json", baseline)
|
|
133
|
+
|
|
134
|
+
# ... skill gets edited ...
|
|
135
|
+
|
|
136
|
+
saved = read_baseline("./.evolveguard-baseline.json")
|
|
137
|
+
replay = replay_skill("./SKILL.md", saved)
|
|
138
|
+
report = diff_all(saved, replay)
|
|
139
|
+
print(report.summary, report.exit_code)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## How it works
|
|
143
|
+
|
|
144
|
+
evolveguard does not run a live LLM agent, and it does not replay a real
|
|
145
|
+
conversation transcript. It is a static, deterministic tool by design:
|
|
146
|
+
|
|
147
|
+
1. **`record`** parses a skill file's YAML frontmatter (declared `tools`,
|
|
148
|
+
`network`, `filesystem`, `scope`, and any bundled `hooks`), scans the
|
|
149
|
+
skill's body text and any hook scripts for static evidence of network
|
|
150
|
+
calls or filesystem writes, and combines both into a **capability
|
|
151
|
+
surface** -- the set of tools the skill is declared or shown to use.
|
|
152
|
+
Each fixture's `expectedToolCalls` filters that surface down to the
|
|
153
|
+
tools the fixture author says it cares about; the result is the
|
|
154
|
+
recorded baseline.
|
|
155
|
+
2. **`check`** re-reads the (possibly edited) skill file, re-derives its
|
|
156
|
+
capability surface with the exact same logic, and re-filters it per
|
|
157
|
+
fixture.
|
|
158
|
+
3. **`diff`** compares baseline vs. current per fixture (PASS if the tool
|
|
159
|
+
set and scopes match, DRIFT with a specific reason otherwise), and
|
|
160
|
+
separately diffs the whole capability surface so a new capability that
|
|
161
|
+
no fixture's `expectedToolCalls` happened to cover still gets caught.
|
|
162
|
+
|
|
163
|
+
evolveguard detects changes in what a skill is _declared or shown_ to be
|
|
164
|
+
capable of. It can't tell you whether a live agent run would actually
|
|
165
|
+
behave differently on a given prompt -- that's a real, intentional scope
|
|
166
|
+
limit. The tradeoff: it just needs a `SKILL.md` file and a fixtures file,
|
|
167
|
+
with nothing hosted and no SDK to integrate against, which is also why it
|
|
168
|
+
runs fully offline in a pre-commit hook or CI job. This Python package is
|
|
169
|
+
a genuine, independent port of the pipeline -- not a wrapper around the
|
|
170
|
+
Node binary. See the
|
|
171
|
+
[project README](https://github.com/RudrenduPaul/evolveguard#readme) for
|
|
172
|
+
the fuller design writeup and the comparison against Braintrust and
|
|
173
|
+
agent-eval.
|
|
174
|
+
|
|
175
|
+
## CLI command reference
|
|
176
|
+
|
|
177
|
+
```
|
|
178
|
+
usage: evolveguard [-h] [-V] {record,check,report,mcp} ...
|
|
179
|
+
|
|
180
|
+
Regression-testing CLI for self-edited Claude Agent Skills (SKILL.md,
|
|
181
|
+
MEMORY.md) -- golden-transcript record/replay against a skill's own
|
|
182
|
+
declared and inferred capability surface, zero hosted infrastructure.
|
|
183
|
+
|
|
184
|
+
positional arguments:
|
|
185
|
+
{record,check,report,mcp}
|
|
186
|
+
record Record a golden-transcript baseline for a skill
|
|
187
|
+
against a set of labeled fixtures
|
|
188
|
+
check Replay the fixtures from a baseline against the
|
|
189
|
+
current (possibly edited) skill and report drift
|
|
190
|
+
report Print a previously generated evolveguard-report.json
|
|
191
|
+
mcp [coming soon] Expose record/check/report as MCP
|
|
192
|
+
tools for a coding agent to call mid-session
|
|
193
|
+
|
|
194
|
+
options:
|
|
195
|
+
-h, --help show this help message and exit
|
|
196
|
+
-V, --version show program's version number and exit
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
`evolveguard record <skillPath> --fixtures <path> [--baseline <path>] [--json]`,
|
|
200
|
+
`evolveguard check <skillPath> [--baseline <path>] [--report <path>] [--allow-drift] [--json]`,
|
|
201
|
+
and `evolveguard report [reportPath] [--json]` mirror the npm CLI's flags
|
|
202
|
+
and defaults exactly -- see the
|
|
203
|
+
[project README's CLI reference](https://github.com/RudrenduPaul/evolveguard#cli-command-reference)
|
|
204
|
+
for the full `--help` output of each subcommand.
|
|
205
|
+
|
|
206
|
+
**Exit codes:** `0` all fixtures PASS and no surface-level drift, `1` at
|
|
207
|
+
least one DRIFT was found (pass `--allow-drift` to still exit 0 while
|
|
208
|
+
still reporting it), `2` a usage error or a file that failed to parse.
|
|
209
|
+
|
|
210
|
+
## Agent-native usage
|
|
211
|
+
|
|
212
|
+
Every subcommand supports `--json` for structured output an agent can
|
|
213
|
+
parse directly:
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
evolveguard check ./SKILL.md --json
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
`evolveguard mcp` is documented but not implemented yet in either
|
|
220
|
+
distribution -- call `record`/`check`/`report --json` directly as a
|
|
221
|
+
subprocess (or the library functions in-process) from your coding agent
|
|
222
|
+
until it ships.
|
|
223
|
+
|
|
224
|
+
## False-positive rate
|
|
225
|
+
|
|
226
|
+
No accuracy claim ships without the command that produced it. From a
|
|
227
|
+
clone of the repo:
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
cd python && pytest tests/test_benchmark.py -v
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
against `fixtures/labeled-non-breaking-edits/` (shared with the
|
|
234
|
+
TypeScript test suite, not duplicated) -- a small, hand-labeled corpus of
|
|
235
|
+
5 real before/after `SKILL.md` pairs: 2 labeled non-breaking (a wording
|
|
236
|
+
tweak, a typo fix) and 3 labeled breaking (a filesystem-scope widen, a new
|
|
237
|
+
write capability, and a hook script gaining a network call). As of this
|
|
238
|
+
release: **0% false positives** (0 of 2 non-breaking cases flagged as
|
|
239
|
+
drift), matching the npm package's own documented result on the same
|
|
240
|
+
corpus. The corpus is small and will grow as more real skill edits are
|
|
241
|
+
reported.
|
|
242
|
+
|
|
243
|
+
## Contributing
|
|
244
|
+
|
|
245
|
+
See [CONTRIBUTING.md](https://github.com/RudrenduPaul/evolveguard/blob/main/CONTRIBUTING.md).
|
|
246
|
+
There is no enforced minimum coverage threshold; the bar is that the full
|
|
247
|
+
pytest suite (`pytest` from `python/`) passes and new behavior ships with
|
|
248
|
+
tests.
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
cd python
|
|
252
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
253
|
+
pip install -e ".[dev]"
|
|
254
|
+
pytest
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
## Security
|
|
258
|
+
|
|
259
|
+
evolveguard reads local files you point it at and never executes any of
|
|
260
|
+
them -- no `eval`, no subprocess, no dynamic import of scan-target
|
|
261
|
+
content, no network calls. Hook script paths declared in a skill's
|
|
262
|
+
frontmatter are resolved and validated against that skill's own directory
|
|
263
|
+
before being read, including a symlink-escape check, so a malicious or
|
|
264
|
+
broken skill file cannot make evolveguard read outside its own folder.
|
|
265
|
+
Baselines and reports are read and written as plain JSON, never pickled or
|
|
266
|
+
otherwise deserialized as executable data. See
|
|
267
|
+
[SECURITY.md](https://github.com/RudrenduPaul/evolveguard/blob/main/SECURITY.md)
|
|
268
|
+
for the disclosure process. **Honest note**: this project does not
|
|
269
|
+
currently publish SLSA provenance, Sigstore signatures, or an SBOM, and
|
|
270
|
+
has no OpenSSF Scorecard badge set up -- none of that infrastructure
|
|
271
|
+
exists yet for either distribution, so it isn't claimed here.
|
|
272
|
+
|
|
273
|
+
## License
|
|
274
|
+
|
|
275
|
+
MIT, see [LICENSE](https://github.com/RudrenduPaul/evolveguard/blob/main/LICENSE).
|