wikiskill 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wikiskill-0.1.1/LICENSE +21 -0
- wikiskill-0.1.1/PKG-INFO +207 -0
- wikiskill-0.1.1/README.md +163 -0
- wikiskill-0.1.1/pyproject.toml +39 -0
- wikiskill-0.1.1/setup.cfg +4 -0
- wikiskill-0.1.1/tests/test_backends.py +292 -0
- wikiskill-0.1.1/tests/test_compare.py +89 -0
- wikiskill-0.1.1/tests/test_core.py +115 -0
- wikiskill-0.1.1/tests/test_harness.py +286 -0
- wikiskill-0.1.1/tests/test_transfer.py +63 -0
- wikiskill-0.1.1/wikiskill/__init__.py +8 -0
- wikiskill-0.1.1/wikiskill/agents.py +46 -0
- wikiskill-0.1.1/wikiskill/bench.py +461 -0
- wikiskill-0.1.1/wikiskill/cli.py +273 -0
- wikiskill-0.1.1/wikiskill/compare.py +124 -0
- wikiskill-0.1.1/wikiskill/gating.py +183 -0
- wikiskill-0.1.1/wikiskill/harness.py +179 -0
- wikiskill-0.1.1/wikiskill/prompts.py +142 -0
- wikiskill-0.1.1/wikiskill/scoring.py +62 -0
- wikiskill-0.1.1/wikiskill/tasks.py +109 -0
- wikiskill-0.1.1/wikiskill/traces.py +72 -0
- wikiskill-0.1.1/wikiskill/transfer.py +79 -0
- wikiskill-0.1.1/wikiskill/wiki.py +61 -0
- wikiskill-0.1.1/wikiskill.egg-info/PKG-INFO +207 -0
- wikiskill-0.1.1/wikiskill.egg-info/SOURCES.txt +26 -0
- wikiskill-0.1.1/wikiskill.egg-info/dependency_links.txt +1 -0
- wikiskill-0.1.1/wikiskill.egg-info/entry_points.txt +2 -0
- wikiskill-0.1.1/wikiskill.egg-info/top_level.txt +1 -0
wikiskill-0.1.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ashutosh Singh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
wikiskill-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wikiskill
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: WikiSkill (arXiv:2608.27454) for Hermes Agent โ self-evolving agent skills via a persistent knowledge wiki
|
|
5
|
+
License: MIT License
|
|
6
|
+
|
|
7
|
+
Copyright (c) 2026 Ashutosh Singh
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|
|
26
|
+
|
|
27
|
+
Project-URL: Homepage, https://github.com/ashutoshsinghpr7/wikiskill
|
|
28
|
+
Project-URL: Repository, https://github.com/ashutoshsinghpr7/wikiskill
|
|
29
|
+
Project-URL: arXiv, https://arxiv.org/abs/2608.27454
|
|
30
|
+
Keywords: agents,skills,skill-evolution,wiki,llm,hermes,self-improving,agent-experience,persistent-knowledge,arxiv-2608.27454
|
|
31
|
+
Classifier: Development Status :: 3 - Alpha
|
|
32
|
+
Classifier: Intended Audience :: Developers
|
|
33
|
+
Classifier: Intended Audience :: Science/Research
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Programming Language :: Python :: 3
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
39
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
40
|
+
Requires-Python: >=3.10
|
|
41
|
+
Description-Content-Type: text/markdown
|
|
42
|
+
License-File: LICENSE
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# ๐ง WikiSkill
|
|
46
|
+
|
|
47
|
+
**Compile agent experience into a persistent wiki โ and let skills evolve themselves.**
|
|
48
|
+
|
|
49
|
+
> ๐ **Docs site**: [ashutoshsinghpr7.github.io/wikiskill](https://ashutoshsinghpr7.github.io/wikiskill) ยท **arXiv**: [2608.27454](https://arxiv.org/abs/2608.27454)
|
|
50
|
+
|
|
51
|
+
A faithful, production-minded implementation of **[WikiSkill: Compiling Agent Experience into Persistent Knowledge for Skill Evolution](https://arxiv.org/abs/2608.27454)** (arXiv:2608.27454, Google Research). The loop is **agent-agnostic** โ **Hermes Agent** is the reference backend (built natively), **Claude Code** ships in the box, and Codex/OpenCode are on the roadmap ([issue #13](https://github.com/ashutoshsinghpr7/wikiskill/issues/13)). Your agent becomes both the *student* and the *teacher*.
|
|
52
|
+
|
|
53
|
+
[](https://www.python.org)
|
|
54
|
+
[](LICENSE)
|
|
55
|
+
[](https://github.com/ashutoshsinghpr7/wikiskill/actions)
|
|
56
|
+
[](https://arxiv.org/abs/2608.27454)
|
|
57
|
+
|
|
58
|
+
---
|
|
59
|
+
|
|
60
|
+
## What this is
|
|
61
|
+
|
|
62
|
+
Agents fail. They also *learn* โ but the lessons usually die with the session. WikiSkill fixes that by keeping a **persistent knowledge wiki** alongside the skill set, and running a closed evolution loop:
|
|
63
|
+
|
|
64
|
+
1. The agent runs **training tasks** with its current skills โ raw execution traces
|
|
65
|
+
2. A **Wiki Maintainer** agent distills the traces into pattern pages (root causes, fixes)
|
|
66
|
+
3. A **Skill Proposer** agent reads the wiki + traces and proposes one skill change (create or patch)
|
|
67
|
+
4. **Gating**: the change is validated on held-out tasks โ **strictly better** than the best score so far โ kept; otherwise rolled back. **The wiki is never rolled back.**
|
|
68
|
+
|
|
69
|
+
Over iterations, knowledge compounds in the wiki while only *proven* improvements touch the skills.
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
73
|
+
โ EVOLUTION LOOP (Algorithm 1) โ
|
|
74
|
+
โ โ
|
|
75
|
+
tasks โโโโโโบ โ Inference Agent โโโบ raw/traces/ (immutable) โ
|
|
76
|
+
โ โ โ
|
|
77
|
+
โ โผ โ
|
|
78
|
+
โ Wiki Maintainer โโโบ wiki/patterns/, index, log โ
|
|
79
|
+
โ โ โ
|
|
80
|
+
โ โผ โ
|
|
81
|
+
โ Skill Proposer โโโบ proposal (create/patch skill) โ
|
|
82
|
+
โ โ โ
|
|
83
|
+
โ โผ โ
|
|
84
|
+
โ GATE: val score > R_best? โโyesโโโบ keep, R_best=R โ
|
|
85
|
+
โ โ no โ
|
|
86
|
+
โ โผ โ
|
|
87
|
+
โ rollback skills; wiki retained forever โ
|
|
88
|
+
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Why Hermes?
|
|
92
|
+
|
|
93
|
+
This is not a toy simulator. Every component is a **real Hermes agent turn**:
|
|
94
|
+
|
|
95
|
+
| WikiSkill (paper) | This repo |
|
|
96
|
+
|---|---|
|
|
97
|
+
| Inference Agent | `hermes chat --oneshot` in an **isolated `HERMES_HOME` profile** |
|
|
98
|
+
| Raw Layer | Full session JSONL transcripts, exported via `hermes sessions export` |
|
|
99
|
+
| Wiki Layer | `wiki/` โ git-tracked, maintained by a real agent, never rolled back |
|
|
100
|
+
| Skill Layer | Real `SKILL.md` packages (frontmatter + instructions), git-managed |
|
|
101
|
+
| Wiki Maintainer | Agent turn with the paper's **Appendix E.2 prompt** (extracted verbatim) |
|
|
102
|
+
| Skill Proposer | Agent turn with the paper's **Appendix E.3 ReAct prompt** |
|
|
103
|
+
| Gating | Strict `R_val > R_best`; `git reset --hard` on reject |
|
|
104
|
+
|
|
105
|
+
**Why the isolated profile matters:** gating is only meaningful if the agent sees *exactly* the candidate skill set. Each evolution workspace gets its own `HERMES_HOME` (bundled skills opted out, empty memory, skills symlinked per stage) โ your real profile is never touched.
|
|
106
|
+
|
|
107
|
+
## Quickstart (60 seconds)
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pip install -e . # installs the `wikiskill` CLI
|
|
111
|
+
wikiskill init demo # workspace + 22-task auto-graded bench (13 train / 9 val)
|
|
112
|
+
wikiskill status
|
|
113
|
+
wikiskill evolve --iters 3 # full Algorithm 1 loop with your default model
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
That's it. Each evolution workspace lives at `workspaces/<domain>/`:
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
workspaces/demo/
|
|
120
|
+
โโโ raw/traces/iter-01/{train,val}/<task>.jsonl # immutable execution traces
|
|
121
|
+
โโโ wiki/ # persistent knowledge (never rolled back)
|
|
122
|
+
โ โโโ index.md ยท log.md ยท skill-impact.md ยท patterns/*.md
|
|
123
|
+
โโโ skills/active/ # git-managed evolving skill set (Sโ = โ
)
|
|
124
|
+
โโโ skills/framework/ # maintainer + proposer agent skills
|
|
125
|
+
โโโ bench/tasks/<id>/ # task sandboxes (inputs + grader)
|
|
126
|
+
โโโ runs/ # per-run stdout, proposals, state
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## CLI
|
|
130
|
+
|
|
131
|
+
| Command | What it does |
|
|
132
|
+
|---|---|
|
|
133
|
+
| `wikiskill init <domain> [--backend claude]` | Create workspace + demo bench (pins the agent backend) |
|
|
134
|
+
| `wikiskill bench --reset` | Regenerate tasks (deterministic, seed=42) |
|
|
135
|
+
| `wikiskill status` | Workspace state: scores, skills, wiki, history |
|
|
136
|
+
| `wikiskill evolve --iters N [--model M] [--provider P] [--max-turns N] [--no-early-stop]` | The full loop (`--model`/`--provider` patch the isolated profile's default model, e.g. `google/gemini-2.5-flash-lite` + `openrouter`) |
|
|
137
|
+
| `wikiskill run-task <id>` | Single inference rollout (debug) |
|
|
138
|
+
| `wikiskill compare <wsA> <wsB> [--iters N]` | Paired statistical comparison: per-task win/loss/tie + two-sided exact-binomial p-value (answers "did the skill actually help?" โ see [docs/COMPARING.md](docs/COMPARING.md)) |
|
|
139
|
+
|
|
140
|
+
## Bring your own tasks
|
|
141
|
+
|
|
142
|
+
Tasks are plain JSON (`tasks.json`); anything auto-gradable works:
|
|
143
|
+
|
|
144
|
+
```json
|
|
145
|
+
{
|
|
146
|
+
"id": "spec-format1-1", "split": "train",
|
|
147
|
+
"title": "Format products according to spec",
|
|
148
|
+
"prompt": "Read spec.md and products.json...",
|
|
149
|
+
"sandbox": {"spec.md": "...", "products.json": "..."},
|
|
150
|
+
"grader": {"type": "exact", "file": "output.txt", "expected": "alpha|35|active\n..."}
|
|
151
|
+
}
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Graders: `exact`, `contains`, `json_field`, `code_stdout` (runs the produced script). Missing deliverables score **0**, never crash.
|
|
155
|
+
|
|
156
|
+
## Live results so far
|
|
157
|
+
|
|
158
|
+
Honest numbers from real agent runs on the bundled bench:
|
|
159
|
+
|
|
160
|
+
| Setup | Baseline (Sโ) | What happened |
|
|
161
|
+
|---|---|---|
|
|
162
|
+
| deepseek-v4-flash, 15 turns | **1.0** | Algorithm 1 **early-stop** โ nothing to evolve |
|
|
163
|
+
| deepseek-v4-flash, 8 turns | **1.0** | same |
|
|
164
|
+
| deepseek-v4-flash, forced | 1.0 | proposer created `spec_literal_transform` โ R_val=1.0, not > R_best โ **rejected** |
|
|
165
|
+
| deepseek-v4-flash, forced | 1.0 | maintainer distilled **4 pattern pages** (incl. `execute_code` blocked in sandbox, ripgrep binary misses); proposer created `exact-match-sandbox-task` โ R_val=0.8889 (skill *hurt*) โ **rejected** |
|
|
166
|
+
| gemma-3-4b (free, OpenRouter) | โ | **invalid run, thrown out** โ dead agent sessions were phantom-graded against stale sandboxes. The maintainer's pattern page caught the framework's own bug; fixed + regression-tested (see [docs/RUNS.md](docs/RUNS.md) Run 4) |
|
|
167
|
+
| **gemini-2.5-flash-lite (free, OpenRouter), 8 turns** | **0.6667** (real) | small model fails at Sโ โ maintainer distilled 5 patterns โ proposer created `find-secret` โ **R_val=0.4444, the skill hurt (2 regressions) โ rejected**. Full loop live on a genuinely weak model, ~$0.09/iteration |
|
|
168
|
+
| gemini-2.5-flash-lite, 3 iterations (issue #5) | **0.4444** (real) | compounding run: train as low as 0.2308, 6/48 launch failures (detected + honest 0.0s), maintainer distilled 1 pattern, proposer **declined (`no_action`)** โ nothing to gate, r_best preserved. Honest negative: accumulation needs a stronger model (see [docs/RUNS.md](docs/RUNS.md) Run 6) |
|
|
169
|
+
|
|
170
|
+
The gating mechanism has caught both a neutral and a *harmful* proposal live. Full logs in [docs/RUNS.md](docs/RUNS.md).
|
|
171
|
+
|
|
172
|
+
## Design decisions worth knowing
|
|
173
|
+
|
|
174
|
+
- **`--in` doesn't pin the agent's CWD** in single-query runs โ every inference prompt embeds an absolute `WORKING DIRECTORY` and forbids exploring outside it.
|
|
175
|
+
- **Sessions live in `state.db`**, not loose files โ transcripts are materialized via `hermes sessions export --format jsonl`.
|
|
176
|
+
- **Rejected proposals are never lost** โ their full content is embedded in `wiki/skill-impact.md` so future proposers don't repeat them (per Appendix E.3).
|
|
177
|
+
- **The demo bench has traps**: subtle-spec tasks and multi-bug debug scripts whose bugs *don't* compensate (verified at generation time).
|
|
178
|
+
|
|
179
|
+
## How this compares to other community implementations
|
|
180
|
+
|
|
181
|
+
We audited the three repos that appeared alongside the paper (see [docs/RUNS.md](docs/RUNS.md)). This is the only one that: runs on a real agent stack (Hermes), gates skills through a fully isolated profile, ships verbatim Appendix E prompts, and has a live-verified end-to-end loop (maintainer โ proposer โ gate โ rollback).
|
|
182
|
+
|
|
183
|
+
## Agent backends
|
|
184
|
+
|
|
185
|
+
The loop runs on any supported agent CLI โ the raw/wiki/skill layers are
|
|
186
|
+
backend-agnostic ([issue #13](https://github.com/ashutoshsinghpr7/wikiskill/issues/13)).
|
|
187
|
+
|
|
188
|
+
| Backend | Pin a workspace | Notes |
|
|
189
|
+
|---|---|---|
|
|
190
|
+
| `hermes` (default) | `wikiskill init demo --backend hermes` | reference implementation; isolated `HERMES_HOME` per workspace |
|
|
191
|
+
| `claude` | `wikiskill init demo --backend claude` | Claude Code 2.x (`claude -p`), isolated `CLAUDE_CONFIG_DIR`, transcripts normalized from the stream-json output; `claude auth login` required once |
|
|
192
|
+
|
|
193
|
+
Each workspace pins its backend in `workspaces/<domain>/workspace.json`; switch
|
|
194
|
+
anytime with `wikiskill evolve <domain> --backend claude`. Skills evolved on
|
|
195
|
+
one backend transfer to another via `wikiskill transfer` (same SKILL.md format).
|
|
196
|
+
|
|
197
|
+
## Roadmap
|
|
198
|
+
|
|
199
|
+
- [ ] Multi-iteration compounding run (the paper's key ablation: wiki accumulation matters)
|
|
200
|
+
- [ ] Skill *transfer* across models (paper: skills evolved by one model help another)
|
|
201
|
+
- [ ] Real-task domains (your recurring workflows as graded task sets)
|
|
202
|
+
- [ ] `compare` command for statistical run comparison
|
|
203
|
+
- [ ] Cron-driven overnight evolution (`hermes cron`)
|
|
204
|
+
|
|
205
|
+
## License
|
|
206
|
+
|
|
207
|
+
MIT โ see [LICENSE](LICENSE). Based on arXiv:2608.27454 (Google Research); all prompts in `skills/` are adapted from the paper's Appendix E. Inspired by [Karpathy's LLM Wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f).
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# ๐ง WikiSkill
|
|
2
|
+
|
|
3
|
+
**Compile agent experience into a persistent wiki โ and let skills evolve themselves.**
|
|
4
|
+
|
|
5
|
+
> ๐ **Docs site**: [ashutoshsinghpr7.github.io/wikiskill](https://ashutoshsinghpr7.github.io/wikiskill) ยท **arXiv**: [2608.27454](https://arxiv.org/abs/2608.27454)
|
|
6
|
+
|
|
7
|
+
A faithful, production-minded implementation of **[WikiSkill: Compiling Agent Experience into Persistent Knowledge for Skill Evolution](https://arxiv.org/abs/2608.27454)** (arXiv:2608.27454, Google Research). The loop is **agent-agnostic** โ **Hermes Agent** is the reference backend (built natively), **Claude Code** ships in the box, and Codex/OpenCode are on the roadmap ([issue #13](https://github.com/ashutoshsinghpr7/wikiskill/issues/13)). Your agent becomes both the *student* and the *teacher*.
|
|
8
|
+
|
|
9
|
+
[](https://www.python.org)
|
|
10
|
+
[](LICENSE)
|
|
11
|
+
[](https://github.com/ashutoshsinghpr7/wikiskill/actions)
|
|
12
|
+
[](https://arxiv.org/abs/2608.27454)
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## What this is
|
|
17
|
+
|
|
18
|
+
Agents fail. They also *learn* โ but the lessons usually die with the session. WikiSkill fixes that by keeping a **persistent knowledge wiki** alongside the skill set, and running a closed evolution loop:
|
|
19
|
+
|
|
20
|
+
1. The agent runs **training tasks** with its current skills โ raw execution traces
|
|
21
|
+
2. A **Wiki Maintainer** agent distills the traces into pattern pages (root causes, fixes)
|
|
22
|
+
3. A **Skill Proposer** agent reads the wiki + traces and proposes one skill change (create or patch)
|
|
23
|
+
4. **Gating**: the change is validated on held-out tasks โ **strictly better** than the best score so far โ kept; otherwise rolled back. **The wiki is never rolled back.**
|
|
24
|
+
|
|
25
|
+
Over iterations, knowledge compounds in the wiki while only *proven* improvements touch the skills.
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
29
|
+
โ EVOLUTION LOOP (Algorithm 1) โ
|
|
30
|
+
โ โ
|
|
31
|
+
tasks โโโโโโบ โ Inference Agent โโโบ raw/traces/ (immutable) โ
|
|
32
|
+
โ โ โ
|
|
33
|
+
โ โผ โ
|
|
34
|
+
โ Wiki Maintainer โโโบ wiki/patterns/, index, log โ
|
|
35
|
+
โ โ โ
|
|
36
|
+
โ โผ โ
|
|
37
|
+
โ Skill Proposer โโโบ proposal (create/patch skill) โ
|
|
38
|
+
โ โ โ
|
|
39
|
+
โ โผ โ
|
|
40
|
+
โ GATE: val score > R_best? โโyesโโโบ keep, R_best=R โ
|
|
41
|
+
โ โ no โ
|
|
42
|
+
โ โผ โ
|
|
43
|
+
โ rollback skills; wiki retained forever โ
|
|
44
|
+
โโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโโ
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Why Hermes?
|
|
48
|
+
|
|
49
|
+
This is not a toy simulator. Every component is a **real Hermes agent turn**:
|
|
50
|
+
|
|
51
|
+
| WikiSkill (paper) | This repo |
|
|
52
|
+
|---|---|
|
|
53
|
+
| Inference Agent | `hermes chat --oneshot` in an **isolated `HERMES_HOME` profile** |
|
|
54
|
+
| Raw Layer | Full session JSONL transcripts, exported via `hermes sessions export` |
|
|
55
|
+
| Wiki Layer | `wiki/` โ git-tracked, maintained by a real agent, never rolled back |
|
|
56
|
+
| Skill Layer | Real `SKILL.md` packages (frontmatter + instructions), git-managed |
|
|
57
|
+
| Wiki Maintainer | Agent turn with the paper's **Appendix E.2 prompt** (extracted verbatim) |
|
|
58
|
+
| Skill Proposer | Agent turn with the paper's **Appendix E.3 ReAct prompt** |
|
|
59
|
+
| Gating | Strict `R_val > R_best`; `git reset --hard` on reject |
|
|
60
|
+
|
|
61
|
+
**Why the isolated profile matters:** gating is only meaningful if the agent sees *exactly* the candidate skill set. Each evolution workspace gets its own `HERMES_HOME` (bundled skills opted out, empty memory, skills symlinked per stage) โ your real profile is never touched.
|
|
62
|
+
|
|
63
|
+
## Quickstart (60 seconds)
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
pip install -e . # installs the `wikiskill` CLI
|
|
67
|
+
wikiskill init demo # workspace + 22-task auto-graded bench (13 train / 9 val)
|
|
68
|
+
wikiskill status
|
|
69
|
+
wikiskill evolve --iters 3 # full Algorithm 1 loop with your default model
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
That's it. Each evolution workspace lives at `workspaces/<domain>/`:
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
workspaces/demo/
|
|
76
|
+
โโโ raw/traces/iter-01/{train,val}/<task>.jsonl # immutable execution traces
|
|
77
|
+
โโโ wiki/ # persistent knowledge (never rolled back)
|
|
78
|
+
โ โโโ index.md ยท log.md ยท skill-impact.md ยท patterns/*.md
|
|
79
|
+
โโโ skills/active/ # git-managed evolving skill set (Sโ = โ
)
|
|
80
|
+
โโโ skills/framework/ # maintainer + proposer agent skills
|
|
81
|
+
โโโ bench/tasks/<id>/ # task sandboxes (inputs + grader)
|
|
82
|
+
โโโ runs/ # per-run stdout, proposals, state
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## CLI
|
|
86
|
+
|
|
87
|
+
| Command | What it does |
|
|
88
|
+
|---|---|
|
|
89
|
+
| `wikiskill init <domain> [--backend claude]` | Create workspace + demo bench (pins the agent backend) |
|
|
90
|
+
| `wikiskill bench --reset` | Regenerate tasks (deterministic, seed=42) |
|
|
91
|
+
| `wikiskill status` | Workspace state: scores, skills, wiki, history |
|
|
92
|
+
| `wikiskill evolve --iters N [--model M] [--provider P] [--max-turns N] [--no-early-stop]` | The full loop (`--model`/`--provider` patch the isolated profile's default model, e.g. `google/gemini-2.5-flash-lite` + `openrouter`) |
|
|
93
|
+
| `wikiskill run-task <id>` | Single inference rollout (debug) |
|
|
94
|
+
| `wikiskill compare <wsA> <wsB> [--iters N]` | Paired statistical comparison: per-task win/loss/tie + two-sided exact-binomial p-value (answers "did the skill actually help?" โ see [docs/COMPARING.md](docs/COMPARING.md)) |
|
|
95
|
+
|
|
96
|
+
## Bring your own tasks
|
|
97
|
+
|
|
98
|
+
Tasks are plain JSON (`tasks.json`); anything auto-gradable works:
|
|
99
|
+
|
|
100
|
+
```json
|
|
101
|
+
{
|
|
102
|
+
"id": "spec-format1-1", "split": "train",
|
|
103
|
+
"title": "Format products according to spec",
|
|
104
|
+
"prompt": "Read spec.md and products.json...",
|
|
105
|
+
"sandbox": {"spec.md": "...", "products.json": "..."},
|
|
106
|
+
"grader": {"type": "exact", "file": "output.txt", "expected": "alpha|35|active\n..."}
|
|
107
|
+
}
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Graders: `exact`, `contains`, `json_field`, `code_stdout` (runs the produced script). Missing deliverables score **0**, never crash.
|
|
111
|
+
|
|
112
|
+
## Live results so far
|
|
113
|
+
|
|
114
|
+
Honest numbers from real agent runs on the bundled bench:
|
|
115
|
+
|
|
116
|
+
| Setup | Baseline (Sโ) | What happened |
|
|
117
|
+
|---|---|---|
|
|
118
|
+
| deepseek-v4-flash, 15 turns | **1.0** | Algorithm 1 **early-stop** โ nothing to evolve |
|
|
119
|
+
| deepseek-v4-flash, 8 turns | **1.0** | same |
|
|
120
|
+
| deepseek-v4-flash, forced | 1.0 | proposer created `spec_literal_transform` โ R_val=1.0, not > R_best โ **rejected** |
|
|
121
|
+
| deepseek-v4-flash, forced | 1.0 | maintainer distilled **4 pattern pages** (incl. `execute_code` blocked in sandbox, ripgrep binary misses); proposer created `exact-match-sandbox-task` โ R_val=0.8889 (skill *hurt*) โ **rejected** |
|
|
122
|
+
| gemma-3-4b (free, OpenRouter) | โ | **invalid run, thrown out** โ dead agent sessions were phantom-graded against stale sandboxes. The maintainer's pattern page caught the framework's own bug; fixed + regression-tested (see [docs/RUNS.md](docs/RUNS.md) Run 4) |
|
|
123
|
+
| **gemini-2.5-flash-lite (free, OpenRouter), 8 turns** | **0.6667** (real) | small model fails at Sโ โ maintainer distilled 5 patterns โ proposer created `find-secret` โ **R_val=0.4444, the skill hurt (2 regressions) โ rejected**. Full loop live on a genuinely weak model, ~$0.09/iteration |
|
|
124
|
+
| gemini-2.5-flash-lite, 3 iterations (issue #5) | **0.4444** (real) | compounding run: train as low as 0.2308, 6/48 launch failures (detected + honest 0.0s), maintainer distilled 1 pattern, proposer **declined (`no_action`)** โ nothing to gate, r_best preserved. Honest negative: accumulation needs a stronger model (see [docs/RUNS.md](docs/RUNS.md) Run 6) |
|
|
125
|
+
|
|
126
|
+
The gating mechanism has caught both a neutral and a *harmful* proposal live. Full logs in [docs/RUNS.md](docs/RUNS.md).
|
|
127
|
+
|
|
128
|
+
## Design decisions worth knowing
|
|
129
|
+
|
|
130
|
+
- **`--in` doesn't pin the agent's CWD** in single-query runs โ every inference prompt embeds an absolute `WORKING DIRECTORY` and forbids exploring outside it.
|
|
131
|
+
- **Sessions live in `state.db`**, not loose files โ transcripts are materialized via `hermes sessions export --format jsonl`.
|
|
132
|
+
- **Rejected proposals are never lost** โ their full content is embedded in `wiki/skill-impact.md` so future proposers don't repeat them (per Appendix E.3).
|
|
133
|
+
- **The demo bench has traps**: subtle-spec tasks and multi-bug debug scripts whose bugs *don't* compensate (verified at generation time).
|
|
134
|
+
|
|
135
|
+
## How this compares to other community implementations
|
|
136
|
+
|
|
137
|
+
We audited the three repos that appeared alongside the paper (see [docs/RUNS.md](docs/RUNS.md)). This is the only one that: runs on a real agent stack (Hermes), gates skills through a fully isolated profile, ships verbatim Appendix E prompts, and has a live-verified end-to-end loop (maintainer โ proposer โ gate โ rollback).
|
|
138
|
+
|
|
139
|
+
## Agent backends
|
|
140
|
+
|
|
141
|
+
The loop runs on any supported agent CLI โ the raw/wiki/skill layers are
|
|
142
|
+
backend-agnostic ([issue #13](https://github.com/ashutoshsinghpr7/wikiskill/issues/13)).
|
|
143
|
+
|
|
144
|
+
| Backend | Pin a workspace | Notes |
|
|
145
|
+
|---|---|---|
|
|
146
|
+
| `hermes` (default) | `wikiskill init demo --backend hermes` | reference implementation; isolated `HERMES_HOME` per workspace |
|
|
147
|
+
| `claude` | `wikiskill init demo --backend claude` | Claude Code 2.x (`claude -p`), isolated `CLAUDE_CONFIG_DIR`, transcripts normalized from the stream-json output; `claude auth login` required once |
|
|
148
|
+
|
|
149
|
+
Each workspace pins its backend in `workspaces/<domain>/workspace.json`; switch
|
|
150
|
+
anytime with `wikiskill evolve <domain> --backend claude`. Skills evolved on
|
|
151
|
+
one backend transfer to another via `wikiskill transfer` (same SKILL.md format).
|
|
152
|
+
|
|
153
|
+
## Roadmap
|
|
154
|
+
|
|
155
|
+
- [ ] Multi-iteration compounding run (the paper's key ablation: wiki accumulation matters)
|
|
156
|
+
- [ ] Skill *transfer* across models (paper: skills evolved by one model help another)
|
|
157
|
+
- [ ] Real-task domains (your recurring workflows as graded task sets)
|
|
158
|
+
- [ ] `compare` command for statistical run comparison
|
|
159
|
+
- [ ] Cron-driven overnight evolution (`hermes cron`)
|
|
160
|
+
|
|
161
|
+
## License
|
|
162
|
+
|
|
163
|
+
MIT โ see [LICENSE](LICENSE). Based on arXiv:2608.27454 (Google Research); all prompts in `skills/` are adapted from the paper's Appendix E. Inspired by [Karpathy's LLM Wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f).
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "wikiskill"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "WikiSkill (arXiv:2608.27454) for Hermes Agent โ self-evolving agent skills via a persistent knowledge wiki"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { file = "LICENSE" }
|
|
12
|
+
keywords = [
|
|
13
|
+
"agents", "skills", "skill-evolution", "wiki", "llm", "hermes",
|
|
14
|
+
"self-improving", "agent-experience", "persistent-knowledge",
|
|
15
|
+
"arxiv-2608.27454",
|
|
16
|
+
]
|
|
17
|
+
classifiers = [
|
|
18
|
+
"Development Status :: 3 - Alpha",
|
|
19
|
+
"Intended Audience :: Developers",
|
|
20
|
+
"Intended Audience :: Science/Research",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
27
|
+
]
|
|
28
|
+
dependencies = []
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
wikiskill = "wikiskill.cli:main"
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Homepage = "https://github.com/ashutoshsinghpr7/wikiskill"
|
|
35
|
+
Repository = "https://github.com/ashutoshsinghpr7/wikiskill"
|
|
36
|
+
arXiv = "https://arxiv.org/abs/2608.27454"
|
|
37
|
+
|
|
38
|
+
[tool.setuptools]
|
|
39
|
+
packages = ["wikiskill"]
|