evolveguard 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evolveguard-0.1.0/.gitignore +11 -0
- evolveguard-0.1.0/LICENSE +21 -0
- evolveguard-0.1.0/PKG-INFO +275 -0
- evolveguard-0.1.0/README.md +239 -0
- evolveguard-0.1.0/examples/01-basic-record-replay/demo.py +68 -0
- evolveguard-0.1.0/examples/02-ci-gate/gate.py +85 -0
- evolveguard-0.1.0/examples/03-agent-native-json/agent_report.py +63 -0
- evolveguard-0.1.0/examples/README.md +30 -0
- evolveguard-0.1.0/pyproject.toml +68 -0
- evolveguard-0.1.0/src/evolveguard/__init__.py +93 -0
- evolveguard-0.1.0/src/evolveguard/cli.py +255 -0
- evolveguard-0.1.0/src/evolveguard/diff/__init__.py +200 -0
- evolveguard-0.1.0/src/evolveguard/errors.py +17 -0
- evolveguard-0.1.0/src/evolveguard/fixtures.py +106 -0
- evolveguard-0.1.0/src/evolveguard/formatters.py +90 -0
- evolveguard-0.1.0/src/evolveguard/parser/__init__.py +0 -0
- evolveguard-0.1.0/src/evolveguard/parser/skillmd.py +249 -0
- evolveguard-0.1.0/src/evolveguard/paths.py +78 -0
- evolveguard-0.1.0/src/evolveguard/record/__init__.py +30 -0
- evolveguard-0.1.0/src/evolveguard/replay/__init__.py +35 -0
- evolveguard-0.1.0/src/evolveguard/report/__init__.py +95 -0
- evolveguard-0.1.0/src/evolveguard/snapshot.py +113 -0
- evolveguard-0.1.0/src/evolveguard/types.py +292 -0
- evolveguard-0.1.0/tests/__init__.py +0 -0
- evolveguard-0.1.0/tests/test_benchmark.py +82 -0
- evolveguard-0.1.0/tests/test_cli.py +176 -0
- evolveguard-0.1.0/tests/test_diff.py +167 -0
- evolveguard-0.1.0/tests/test_fixtures.py +63 -0
- evolveguard-0.1.0/tests/test_paths.py +54 -0
- evolveguard-0.1.0/tests/test_record.py +34 -0
- evolveguard-0.1.0/tests/test_replay.py +40 -0
- evolveguard-0.1.0/tests/test_report.py +91 -0
- evolveguard-0.1.0/tests/test_skillmd.py +135 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rudrendu Paul and Sourav Nandy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: evolveguard
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Regression-testing CI gate for self-edited Claude Agent Skills (SKILL.md, MEMORY.md): golden-transcript record/replay against a skill's own declared and inferred capability surface, zero hosted infrastructure.
|
|
5
|
+
Project-URL: Homepage, https://github.com/RudrenduPaul/evolveguard
|
|
6
|
+
Project-URL: Repository, https://github.com/RudrenduPaul/evolveguard
|
|
7
|
+
Project-URL: Bug Tracker, https://github.com/RudrenduPaul/evolveguard/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/RudrenduPaul/evolveguard/blob/main/CHANGELOG.md
|
|
9
|
+
Project-URL: Documentation, https://github.com/RudrenduPaul/evolveguard/blob/main/docs/getting-started.md
|
|
10
|
+
Project-URL: Author - Rudrendu Paul, https://github.com/RudrenduPaul
|
|
11
|
+
Project-URL: Author - Sourav Nandy, https://github.com/Sourav-nandy-ai
|
|
12
|
+
Author: Rudrendu Paul, Sourav Nandy
|
|
13
|
+
License-Expression: MIT
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Keywords: agent-skills,ai-agents,ci-gate,claude-code,cli-tool,developer-tools,eval-harness,regression-testing
|
|
16
|
+
Classifier: Development Status :: 3 - Alpha
|
|
17
|
+
Classifier: Environment :: Console
|
|
18
|
+
Classifier: Intended Audience :: Developers
|
|
19
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
20
|
+
Classifier: Operating System :: OS Independent
|
|
21
|
+
Classifier: Programming Language :: Python :: 3
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
27
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
28
|
+
Classifier: Topic :: Software Development :: Testing
|
|
29
|
+
Requires-Python: >=3.9
|
|
30
|
+
Requires-Dist: pyyaml<7,>=6.0
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: build<2,>=1.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: pytest<9,>=7.0; extra == 'dev'
|
|
34
|
+
Requires-Dist: twine<7,>=5.0; extra == 'dev'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# evolveguard (Python)
|
|
38
|
+
|
|
39
|
+
Regression-testing CI gate for self-edited Claude Agent Skills -- `SKILL.md`
|
|
40
|
+
manifests and Claude Code auto-memory `MEMORY.md` files -- catching
|
|
41
|
+
behavioral drift before an edit ships.
|
|
42
|
+
|
|
43
|
+
[](https://pypi.org/project/evolveguard/)
|
|
44
|
+
[](https://github.com/RudrenduPaul/evolveguard/blob/main/LICENSE)
|
|
45
|
+
[](https://pypi.org/project/evolveguard/)
|
|
46
|
+
[](https://github.com/RudrenduPaul/evolveguard/actions/workflows/ci.yml)
|
|
47
|
+
|
|
48
|
+
## Why this exists
|
|
49
|
+
|
|
50
|
+
Claude Code's Agent Skills can be authored by a human, or by an agent
|
|
51
|
+
itself: `/skillify` turns a workflow into a `SKILL.md`, and the auto-memory
|
|
52
|
+
system in this same environment writes `MEMORY.md` files that quietly
|
|
53
|
+
change what an agent does in its next session. None of that gets a
|
|
54
|
+
regression check by default. A skill edit that breaks a working workflow
|
|
55
|
+
looks exactly like a skill edit that fixes one, until someone notices the
|
|
56
|
+
agent stopped doing something it used to do. evolveguard records a
|
|
57
|
+
baseline of a skill's own capability surface (what tools it's declared or
|
|
58
|
+
shown to use), then re-derives that surface every time the skill file
|
|
59
|
+
changes and diffs the result against the baseline: same tool-call
|
|
60
|
+
sequence, or a flagged drift with a specific reason.
|
|
61
|
+
|
|
62
|
+
## Install
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install evolveguard
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
or with [uv](https://docs.astral.sh/uv/):
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
uv add evolveguard
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
> **Not yet live on PyPI.** The package is fully built, tested, and
|
|
75
|
+
> publish-ready (wheel + sdist built, inspected, and verified end to end
|
|
76
|
+
> from a fresh venv install), but the first `twine upload` on this account
|
|
77
|
+
> is currently blocked by PyPI's own new-project-creation anti-abuse
|
|
78
|
+
> throttle (`429 Too many new projects created`), confirmed across
|
|
79
|
+
> repeated upload attempts -- not a code or readiness issue. Until that
|
|
80
|
+
> clears, clone the repo and install from source:
|
|
81
|
+
>
|
|
82
|
+
> ```bash
|
|
83
|
+
> git clone https://github.com/RudrenduPaul/evolveguard.git
|
|
84
|
+
> cd evolveguard/python && pip install -e .
|
|
85
|
+
> ```
|
|
86
|
+
|
|
87
|
+
**About the npm package:** `evolveguard` is also registered as an npm
|
|
88
|
+
package name, but publishing it is currently blocked by a separate,
|
|
89
|
+
unrelated account-level 2FA constraint (the npm account's second factor is
|
|
90
|
+
security-key/passkey only, with no authenticator-app or OTP fallback
|
|
91
|
+
configured). Clone the repo and run `npm run build && npm link` if you
|
|
92
|
+
need the TypeScript CLI locally in the meantime.
|
|
93
|
+
|
|
94
|
+
## Quickstart
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
# 1. Record a baseline against a skill and its labeled fixtures
|
|
98
|
+
evolveguard record ./skills/my-skill/SKILL.md --fixtures ./fixtures/my-skill.json
|
|
99
|
+
# writes ./skills/my-skill/.evolveguard-baseline.json
|
|
100
|
+
|
|
101
|
+
# 2. Edit the skill (by hand, or let an agent edit it)
|
|
102
|
+
|
|
103
|
+
# 3. Check for drift
|
|
104
|
+
evolveguard check ./skills/my-skill/SKILL.md
|
|
105
|
+
# writes ./evolveguard-report.json, exits 1 if drift was found
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
A fixtures file is a JSON array of labeled prompts and the tool-call
|
|
109
|
+
shapes each one is expected to touch:
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
[
|
|
113
|
+
{
|
|
114
|
+
"id": "scan-a-monorepo",
|
|
115
|
+
"prompt": "scan a monorepo",
|
|
116
|
+
"expectedToolCalls": [{ "tool": "fs.read" }, { "tool": "fs.write" }]
|
|
117
|
+
}
|
|
118
|
+
]
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`expectedToolCalls` is optional -- omit it and the fixture is treated as
|
|
122
|
+
exercising the skill's entire capability surface. `scopeMatches` (a glob)
|
|
123
|
+
narrows a tool to a specific filesystem scope, e.g.
|
|
124
|
+
`{ "tool": "fs.write", "scopeMatches": "./workspace/**" }`.
|
|
125
|
+
|
|
126
|
+
Or call the library directly (the agent-native path):
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from evolveguard import record_baseline, replay_skill, diff_all, write_baseline, read_baseline
|
|
130
|
+
|
|
131
|
+
baseline = record_baseline("./SKILL.md", "./fixtures.json")
|
|
132
|
+
write_baseline("./.evolveguard-baseline.json", baseline)
|
|
133
|
+
|
|
134
|
+
# ... skill gets edited ...
|
|
135
|
+
|
|
136
|
+
saved = read_baseline("./.evolveguard-baseline.json")
|
|
137
|
+
replay = replay_skill("./SKILL.md", saved)
|
|
138
|
+
report = diff_all(saved, replay)
|
|
139
|
+
print(report.summary, report.exit_code)
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## How it works
|
|
143
|
+
|
|
144
|
+
evolveguard does not run a live LLM agent, and it does not replay a real
|
|
145
|
+
conversation transcript. It is a static, deterministic tool by design:
|
|
146
|
+
|
|
147
|
+
1. **`record`** parses a skill file's YAML frontmatter (declared `tools`,
|
|
148
|
+
`network`, `filesystem`, `scope`, and any bundled `hooks`), scans the
|
|
149
|
+
skill's body text and any hook scripts for static evidence of network
|
|
150
|
+
calls or filesystem writes, and combines both into a **capability
|
|
151
|
+
surface** -- the set of tools the skill is declared or shown to use.
|
|
152
|
+
Each fixture's `expectedToolCalls` filters that surface down to the
|
|
153
|
+
tools the fixture author says it cares about; the result is the
|
|
154
|
+
recorded baseline.
|
|
155
|
+
2. **`check`** re-reads the (possibly edited) skill file, re-derives its
|
|
156
|
+
capability surface with the exact same logic, and re-filters it per
|
|
157
|
+
fixture.
|
|
158
|
+
3. **`diff`** compares baseline vs. current per fixture (PASS if the tool
|
|
159
|
+
set and scopes match, DRIFT with a specific reason otherwise), and
|
|
160
|
+
separately diffs the whole capability surface so a new capability that
|
|
161
|
+
no fixture's `expectedToolCalls` happened to cover still gets caught.
|
|
162
|
+
|
|
163
|
+
evolveguard detects changes in what a skill is _declared or shown_ to be
|
|
164
|
+
capable of. It can't tell you whether a live agent run would actually
|
|
165
|
+
behave differently on a given prompt -- that's a real, intentional scope
|
|
166
|
+
limit. The tradeoff: it just needs a `SKILL.md` file and a fixtures file,
|
|
167
|
+
with nothing hosted and no SDK to integrate against, which is also why it
|
|
168
|
+
runs fully offline in a pre-commit hook or CI job. This Python package is
|
|
169
|
+
a genuine, independent port of the pipeline -- not a wrapper around the
|
|
170
|
+
Node binary. See the
|
|
171
|
+
[project README](https://github.com/RudrenduPaul/evolveguard#readme) for
|
|
172
|
+
the fuller design writeup and the comparison against Braintrust and
|
|
173
|
+
agent-eval.
|
|
174
|
+
|
|
175
|
+
## CLI command reference
|
|
176
|
+
|
|
177
|
+
```
|
|
178
|
+
usage: evolveguard [-h] [-V] {record,check,report,mcp} ...
|
|
179
|
+
|
|
180
|
+
Regression-testing CLI for self-edited Claude Agent Skills (SKILL.md,
|
|
181
|
+
MEMORY.md) -- golden-transcript record/replay against a skill's own
|
|
182
|
+
declared and inferred capability surface, zero hosted infrastructure.
|
|
183
|
+
|
|
184
|
+
positional arguments:
|
|
185
|
+
{record,check,report,mcp}
|
|
186
|
+
record Record a golden-transcript baseline for a skill
|
|
187
|
+
against a set of labeled fixtures
|
|
188
|
+
check Replay the fixtures from a baseline against the
|
|
189
|
+
current (possibly edited) skill and report drift
|
|
190
|
+
report Print a previously generated evolveguard-report.json
|
|
191
|
+
mcp [coming soon] Expose record/check/report as MCP
|
|
192
|
+
tools for a coding agent to call mid-session
|
|
193
|
+
|
|
194
|
+
options:
|
|
195
|
+
-h, --help show this help message and exit
|
|
196
|
+
-V, --version show program's version number and exit
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
`evolveguard record <skillPath> --fixtures <path> [--baseline <path>] [--json]`,
|
|
200
|
+
`evolveguard check <skillPath> [--baseline <path>] [--report <path>] [--allow-drift] [--json]`,
|
|
201
|
+
and `evolveguard report [reportPath] [--json]` mirror the npm CLI's flags
|
|
202
|
+
and defaults exactly -- see the
|
|
203
|
+
[project README's CLI reference](https://github.com/RudrenduPaul/evolveguard#cli-command-reference)
|
|
204
|
+
for the full `--help` output of each subcommand.
|
|
205
|
+
|
|
206
|
+
**Exit codes:** `0` all fixtures PASS and no surface-level drift, `1` at
|
|
207
|
+
least one DRIFT was found (pass `--allow-drift` to still exit 0 while
|
|
208
|
+
still reporting it), `2` a usage error or a file that failed to parse.
|
|
209
|
+
|
|
210
|
+
## Agent-native usage
|
|
211
|
+
|
|
212
|
+
Every subcommand supports `--json` for structured output an agent can
|
|
213
|
+
parse directly:
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
evolveguard check ./SKILL.md --json
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
`evolveguard mcp` is documented but not implemented yet in either
|
|
220
|
+
distribution -- call `record`/`check`/`report --json` directly as a
|
|
221
|
+
subprocess (or the library functions in-process) from your coding agent
|
|
222
|
+
until it ships.
|
|
223
|
+
|
|
224
|
+
## False-positive rate
|
|
225
|
+
|
|
226
|
+
No accuracy claim ships without the command that produced it. From a
|
|
227
|
+
clone of the repo:
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
cd python && pytest tests/test_benchmark.py -v
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
against `fixtures/labeled-non-breaking-edits/` (shared with the
|
|
234
|
+
TypeScript test suite, not duplicated) -- a small, hand-labeled corpus of
|
|
235
|
+
5 real before/after `SKILL.md` pairs: 2 labeled non-breaking (a wording
|
|
236
|
+
tweak, a typo fix) and 3 labeled breaking (a filesystem-scope widen, a new
|
|
237
|
+
write capability, and a hook script gaining a network call). As of this
|
|
238
|
+
release: **0% false positives** (0 of 2 non-breaking cases flagged as
|
|
239
|
+
drift), matching the npm package's own documented result on the same
|
|
240
|
+
corpus. The corpus is small and will grow as more real skill edits are
|
|
241
|
+
reported.
|
|
242
|
+
|
|
243
|
+
## Contributing
|
|
244
|
+
|
|
245
|
+
See [CONTRIBUTING.md](https://github.com/RudrenduPaul/evolveguard/blob/main/CONTRIBUTING.md).
|
|
246
|
+
There is no enforced minimum coverage threshold; the bar is that the full
|
|
247
|
+
pytest suite (`pytest` from `python/`) passes and new behavior ships with
|
|
248
|
+
tests.
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
cd python
|
|
252
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
253
|
+
pip install -e ".[dev]"
|
|
254
|
+
pytest
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
## Security
|
|
258
|
+
|
|
259
|
+
evolveguard reads local files you point it at and never executes any of
|
|
260
|
+
them -- no `eval`, no subprocess, no dynamic import of scan-target
|
|
261
|
+
content, no network calls. Hook script paths declared in a skill's
|
|
262
|
+
frontmatter are resolved and validated against that skill's own directory
|
|
263
|
+
before being read, including a symlink-escape check, so a malicious or
|
|
264
|
+
broken skill file cannot make evolveguard read outside its own folder.
|
|
265
|
+
Baselines and reports are read and written as plain JSON, never pickled or
|
|
266
|
+
otherwise deserialized as executable data. See
|
|
267
|
+
[SECURITY.md](https://github.com/RudrenduPaul/evolveguard/blob/main/SECURITY.md)
|
|
268
|
+
for the disclosure process. **Honest note**: this project does not
|
|
269
|
+
currently publish SLSA provenance, Sigstore signatures, or an SBOM, and
|
|
270
|
+
has no OpenSSF Scorecard badge set up -- none of that infrastructure
|
|
271
|
+
exists yet for either distribution, so it isn't claimed here.
|
|
272
|
+
|
|
273
|
+
## License
|
|
274
|
+
|
|
275
|
+
MIT, see [LICENSE](https://github.com/RudrenduPaul/evolveguard/blob/main/LICENSE).
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
# evolveguard (Python)
|
|
2
|
+
|
|
3
|
+
Regression-testing CI gate for self-edited Claude Agent Skills -- `SKILL.md`
|
|
4
|
+
manifests and Claude Code auto-memory `MEMORY.md` files -- catching
|
|
5
|
+
behavioral drift before an edit ships.
|
|
6
|
+
|
|
7
|
+
[](https://pypi.org/project/evolveguard/)
|
|
8
|
+
[](https://github.com/RudrenduPaul/evolveguard/blob/main/LICENSE)
|
|
9
|
+
[](https://pypi.org/project/evolveguard/)
|
|
10
|
+
[](https://github.com/RudrenduPaul/evolveguard/actions/workflows/ci.yml)
|
|
11
|
+
|
|
12
|
+
## Why this exists
|
|
13
|
+
|
|
14
|
+
Claude Code's Agent Skills can be authored by a human, or by an agent
|
|
15
|
+
itself: `/skillify` turns a workflow into a `SKILL.md`, and the auto-memory
|
|
16
|
+
system in this same environment writes `MEMORY.md` files that quietly
|
|
17
|
+
change what an agent does in its next session. None of that gets a
|
|
18
|
+
regression check by default. A skill edit that breaks a working workflow
|
|
19
|
+
looks exactly like a skill edit that fixes one, until someone notices the
|
|
20
|
+
agent stopped doing something it used to do. evolveguard records a
|
|
21
|
+
baseline of a skill's own capability surface (what tools it's declared or
|
|
22
|
+
shown to use), then re-derives that surface every time the skill file
|
|
23
|
+
changes and diffs the result against the baseline: same tool-call
|
|
24
|
+
sequence, or a flagged drift with a specific reason.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install evolveguard
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
or with [uv](https://docs.astral.sh/uv/):
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
uv add evolveguard
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
> **Not yet live on PyPI.** The package is fully built, tested, and
|
|
39
|
+
> publish-ready (wheel + sdist built, inspected, and verified end to end
|
|
40
|
+
> from a fresh venv install), but the first `twine upload` on this account
|
|
41
|
+
> is currently blocked by PyPI's own new-project-creation anti-abuse
|
|
42
|
+
> throttle (`429 Too many new projects created`), confirmed across
|
|
43
|
+
> repeated upload attempts -- not a code or readiness issue. Until that
|
|
44
|
+
> clears, clone the repo and install from source:
|
|
45
|
+
>
|
|
46
|
+
> ```bash
|
|
47
|
+
> git clone https://github.com/RudrenduPaul/evolveguard.git
|
|
48
|
+
> cd evolveguard/python && pip install -e .
|
|
49
|
+
> ```
|
|
50
|
+
|
|
51
|
+
**About the npm package:** `evolveguard` is also registered as an npm
|
|
52
|
+
package name, but publishing it is currently blocked by a separate,
|
|
53
|
+
unrelated account-level 2FA constraint (the npm account's second factor is
|
|
54
|
+
security-key/passkey only, with no authenticator-app or OTP fallback
|
|
55
|
+
configured). Clone the repo and run `npm run build && npm link` if you
|
|
56
|
+
need the TypeScript CLI locally in the meantime.
|
|
57
|
+
|
|
58
|
+
## Quickstart
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
# 1. Record a baseline against a skill and its labeled fixtures
|
|
62
|
+
evolveguard record ./skills/my-skill/SKILL.md --fixtures ./fixtures/my-skill.json
|
|
63
|
+
# writes ./skills/my-skill/.evolveguard-baseline.json
|
|
64
|
+
|
|
65
|
+
# 2. Edit the skill (by hand, or let an agent edit it)
|
|
66
|
+
|
|
67
|
+
# 3. Check for drift
|
|
68
|
+
evolveguard check ./skills/my-skill/SKILL.md
|
|
69
|
+
# writes ./evolveguard-report.json, exits 1 if drift was found
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
A fixtures file is a JSON array of labeled prompts and the tool-call
|
|
73
|
+
shapes each one is expected to touch:
|
|
74
|
+
|
|
75
|
+
```json
|
|
76
|
+
[
|
|
77
|
+
{
|
|
78
|
+
"id": "scan-a-monorepo",
|
|
79
|
+
"prompt": "scan a monorepo",
|
|
80
|
+
"expectedToolCalls": [{ "tool": "fs.read" }, { "tool": "fs.write" }]
|
|
81
|
+
}
|
|
82
|
+
]
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`expectedToolCalls` is optional -- omit it and the fixture is treated as
|
|
86
|
+
exercising the skill's entire capability surface. `scopeMatches` (a glob)
|
|
87
|
+
narrows a tool to a specific filesystem scope, e.g.
|
|
88
|
+
`{ "tool": "fs.write", "scopeMatches": "./workspace/**" }`.
|
|
89
|
+
|
|
90
|
+
Or call the library directly (the agent-native path):
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from evolveguard import record_baseline, replay_skill, diff_all, write_baseline, read_baseline
|
|
94
|
+
|
|
95
|
+
baseline = record_baseline("./SKILL.md", "./fixtures.json")
|
|
96
|
+
write_baseline("./.evolveguard-baseline.json", baseline)
|
|
97
|
+
|
|
98
|
+
# ... skill gets edited ...
|
|
99
|
+
|
|
100
|
+
saved = read_baseline("./.evolveguard-baseline.json")
|
|
101
|
+
replay = replay_skill("./SKILL.md", saved)
|
|
102
|
+
report = diff_all(saved, replay)
|
|
103
|
+
print(report.summary, report.exit_code)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## How it works
|
|
107
|
+
|
|
108
|
+
evolveguard does not run a live LLM agent, and it does not replay a real
|
|
109
|
+
conversation transcript. It is a static, deterministic tool by design:
|
|
110
|
+
|
|
111
|
+
1. **`record`** parses a skill file's YAML frontmatter (declared `tools`,
|
|
112
|
+
`network`, `filesystem`, `scope`, and any bundled `hooks`), scans the
|
|
113
|
+
skill's body text and any hook scripts for static evidence of network
|
|
114
|
+
calls or filesystem writes, and combines both into a **capability
|
|
115
|
+
surface** -- the set of tools the skill is declared or shown to use.
|
|
116
|
+
Each fixture's `expectedToolCalls` filters that surface down to the
|
|
117
|
+
tools the fixture author says it cares about; the result is the
|
|
118
|
+
recorded baseline.
|
|
119
|
+
2. **`check`** re-reads the (possibly edited) skill file, re-derives its
|
|
120
|
+
capability surface with the exact same logic, and re-filters it per
|
|
121
|
+
fixture.
|
|
122
|
+
3. **`diff`** compares baseline vs. current per fixture (PASS if the tool
|
|
123
|
+
set and scopes match, DRIFT with a specific reason otherwise), and
|
|
124
|
+
separately diffs the whole capability surface so a new capability that
|
|
125
|
+
no fixture's `expectedToolCalls` happened to cover still gets caught.
|
|
126
|
+
|
|
127
|
+
evolveguard detects changes in what a skill is _declared or shown_ to be
|
|
128
|
+
capable of. It can't tell you whether a live agent run would actually
|
|
129
|
+
behave differently on a given prompt -- that's a real, intentional scope
|
|
130
|
+
limit. The tradeoff: it just needs a `SKILL.md` file and a fixtures file,
|
|
131
|
+
with nothing hosted and no SDK to integrate against, which is also why it
|
|
132
|
+
runs fully offline in a pre-commit hook or CI job. This Python package is
|
|
133
|
+
a genuine, independent port of the pipeline -- not a wrapper around the
|
|
134
|
+
Node binary. See the
|
|
135
|
+
[project README](https://github.com/RudrenduPaul/evolveguard#readme) for
|
|
136
|
+
the fuller design writeup and the comparison against Braintrust and
|
|
137
|
+
agent-eval.
|
|
138
|
+
|
|
139
|
+
## CLI command reference
|
|
140
|
+
|
|
141
|
+
```
|
|
142
|
+
usage: evolveguard [-h] [-V] {record,check,report,mcp} ...
|
|
143
|
+
|
|
144
|
+
Regression-testing CLI for self-edited Claude Agent Skills (SKILL.md,
|
|
145
|
+
MEMORY.md) -- golden-transcript record/replay against a skill's own
|
|
146
|
+
declared and inferred capability surface, zero hosted infrastructure.
|
|
147
|
+
|
|
148
|
+
positional arguments:
|
|
149
|
+
{record,check,report,mcp}
|
|
150
|
+
record Record a golden-transcript baseline for a skill
|
|
151
|
+
against a set of labeled fixtures
|
|
152
|
+
check Replay the fixtures from a baseline against the
|
|
153
|
+
current (possibly edited) skill and report drift
|
|
154
|
+
report Print a previously generated evolveguard-report.json
|
|
155
|
+
mcp [coming soon] Expose record/check/report as MCP
|
|
156
|
+
tools for a coding agent to call mid-session
|
|
157
|
+
|
|
158
|
+
options:
|
|
159
|
+
-h, --help show this help message and exit
|
|
160
|
+
-V, --version show program's version number and exit
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
`evolveguard record <skillPath> --fixtures <path> [--baseline <path>] [--json]`,
|
|
164
|
+
`evolveguard check <skillPath> [--baseline <path>] [--report <path>] [--allow-drift] [--json]`,
|
|
165
|
+
and `evolveguard report [reportPath] [--json]` mirror the npm CLI's flags
|
|
166
|
+
and defaults exactly -- see the
|
|
167
|
+
[project README's CLI reference](https://github.com/RudrenduPaul/evolveguard#cli-command-reference)
|
|
168
|
+
for the full `--help` output of each subcommand.
|
|
169
|
+
|
|
170
|
+
**Exit codes:** `0` all fixtures PASS and no surface-level drift, `1` at
|
|
171
|
+
least one DRIFT was found (pass `--allow-drift` to still exit 0 while
|
|
172
|
+
still reporting it), `2` a usage error or a file that failed to parse.
|
|
173
|
+
|
|
174
|
+
## Agent-native usage
|
|
175
|
+
|
|
176
|
+
Every subcommand supports `--json` for structured output an agent can
|
|
177
|
+
parse directly:
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
evolveguard check ./SKILL.md --json
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
`evolveguard mcp` is documented but not implemented yet in either
|
|
184
|
+
distribution -- call `record`/`check`/`report --json` directly as a
|
|
185
|
+
subprocess (or the library functions in-process) from your coding agent
|
|
186
|
+
until it ships.
|
|
187
|
+
|
|
188
|
+
## False-positive rate
|
|
189
|
+
|
|
190
|
+
No accuracy claim ships without the command that produced it. From a
|
|
191
|
+
clone of the repo:
|
|
192
|
+
|
|
193
|
+
```bash
|
|
194
|
+
cd python && pytest tests/test_benchmark.py -v
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
against `fixtures/labeled-non-breaking-edits/` (shared with the
|
|
198
|
+
TypeScript test suite, not duplicated) -- a small, hand-labeled corpus of
|
|
199
|
+
5 real before/after `SKILL.md` pairs: 2 labeled non-breaking (a wording
|
|
200
|
+
tweak, a typo fix) and 3 labeled breaking (a filesystem-scope widen, a new
|
|
201
|
+
write capability, and a hook script gaining a network call). As of this
|
|
202
|
+
release: **0% false positives** (0 of 2 non-breaking cases flagged as
|
|
203
|
+
drift), matching the npm package's own documented result on the same
|
|
204
|
+
corpus. The corpus is small and will grow as more real skill edits are
|
|
205
|
+
reported.
|
|
206
|
+
|
|
207
|
+
## Contributing
|
|
208
|
+
|
|
209
|
+
See [CONTRIBUTING.md](https://github.com/RudrenduPaul/evolveguard/blob/main/CONTRIBUTING.md).
|
|
210
|
+
There is no enforced minimum coverage threshold; the bar is that the full
|
|
211
|
+
pytest suite (`pytest` from `python/`) passes and new behavior ships with
|
|
212
|
+
tests.
|
|
213
|
+
|
|
214
|
+
```bash
|
|
215
|
+
cd python
|
|
216
|
+
python3 -m venv .venv && source .venv/bin/activate
|
|
217
|
+
pip install -e ".[dev]"
|
|
218
|
+
pytest
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
## Security
|
|
222
|
+
|
|
223
|
+
evolveguard reads local files you point it at and never executes any of
|
|
224
|
+
them -- no `eval`, no subprocess, no dynamic import of scan-target
|
|
225
|
+
content, no network calls. Hook script paths declared in a skill's
|
|
226
|
+
frontmatter are resolved and validated against that skill's own directory
|
|
227
|
+
before being read, including a symlink-escape check, so a malicious or
|
|
228
|
+
broken skill file cannot make evolveguard read outside its own folder.
|
|
229
|
+
Baselines and reports are read and written as plain JSON, never pickled or
|
|
230
|
+
otherwise deserialized as executable data. See
|
|
231
|
+
[SECURITY.md](https://github.com/RudrenduPaul/evolveguard/blob/main/SECURITY.md)
|
|
232
|
+
for the disclosure process. **Honest note**: this project does not
|
|
233
|
+
currently publish SLSA provenance, Sigstore signatures, or an SBOM, and
|
|
234
|
+
has no OpenSSF Scorecard badge set up -- none of that infrastructure
|
|
235
|
+
exists yet for either distribution, so it isn't claimed here.
|
|
236
|
+
|
|
237
|
+
## License
|
|
238
|
+
|
|
239
|
+
MIT, see [LICENSE](https://github.com/RudrenduPaul/evolveguard/blob/main/LICENSE).
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
01 -- basic record/replay.
|
|
4
|
+
|
|
5
|
+
Demonstrates the core library call sequence: record_baseline() against a
|
|
6
|
+
skill and its labeled fixtures, then replay_skill() + diff_all() after an
|
|
7
|
+
edit, printing the human-readable verdict. Uses the repo's own bundled
|
|
8
|
+
fixtures/labeled-non-breaking-edits/case-03-add-write-capability/ corpus
|
|
9
|
+
case (a filesystem: read-only -> read-write edit -- the same "scan a
|
|
10
|
+
monorepo" example used in the project README's own quickstart), copied
|
|
11
|
+
into a scratch directory so running this script never modifies the repo's
|
|
12
|
+
fixtures.
|
|
13
|
+
|
|
14
|
+
Run:
|
|
15
|
+
python3 examples/01-basic-record-replay/demo.py
|
|
16
|
+
"""
|
|
17
|
+
import shutil
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
from evolveguard import diff_all, record_baseline, replay_skill, write_baseline
|
|
23
|
+
|
|
24
|
+
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
25
|
+
CASE_DIR = (
|
|
26
|
+
REPO_ROOT / "fixtures" / "labeled-non-breaking-edits" / "case-03-add-write-capability"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def main() -> int:
|
|
31
|
+
if not CASE_DIR.exists():
|
|
32
|
+
print(
|
|
33
|
+
f"Fixture case not found at {CASE_DIR} -- run this from a full repo clone.",
|
|
34
|
+
file=sys.stderr,
|
|
35
|
+
)
|
|
36
|
+
return 2
|
|
37
|
+
|
|
38
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
39
|
+
tmp_path = Path(tmp)
|
|
40
|
+
skill_path = tmp_path / "SKILL.md"
|
|
41
|
+
fixtures_path = tmp_path / "fixtures.json"
|
|
42
|
+
shutil.copy(CASE_DIR / "before" / "SKILL.md", skill_path)
|
|
43
|
+
shutil.copy(CASE_DIR / "fixtures.json", fixtures_path)
|
|
44
|
+
|
|
45
|
+
baseline = record_baseline(str(skill_path), str(fixtures_path))
|
|
46
|
+
write_baseline(str(tmp_path / ".evolveguard-baseline.json"), baseline)
|
|
47
|
+
print(
|
|
48
|
+
f"Recorded baseline for '{baseline.skill_name}': "
|
|
49
|
+
f"{len(baseline.fixtures)} fixture(s)."
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
# ... the skill gets edited: filesystem widens from read-only to read-write ...
|
|
53
|
+
shutil.copy(CASE_DIR / "after" / "SKILL.md", skill_path)
|
|
54
|
+
|
|
55
|
+
replay = replay_skill(str(skill_path), baseline)
|
|
56
|
+
report = diff_all(baseline, replay)
|
|
57
|
+
|
|
58
|
+
print(f"\nCheck result: {report.summary.pass_count} PASS, {report.summary.drift} DRIFT")
|
|
59
|
+
for result in report.results:
|
|
60
|
+
print(f" [{result.verdict}] {result.prompt}")
|
|
61
|
+
for change in result.changes:
|
|
62
|
+
print(f" -> {change.message}")
|
|
63
|
+
|
|
64
|
+
return report.exit_code
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
sys.exit(main())
|