wikiskill 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ashutosh Singh
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,207 @@
1
+ Metadata-Version: 2.4
2
+ Name: wikiskill
3
+ Version: 0.1.1
4
+ Summary: WikiSkill (arXiv:2608.27454) for Hermes Agent โ€” self-evolving agent skills via a persistent knowledge wiki
5
+ License: MIT License
6
+
7
+ Copyright (c) 2026 Ashutosh Singh
8
+
9
+ Permission is hereby granted, free of charge, to any person obtaining a copy
10
+ of this software and associated documentation files (the "Software"), to deal
11
+ in the Software without restriction, including without limitation the rights
12
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13
+ copies of the Software, and to permit persons to whom the Software is
14
+ furnished to do so, subject to the following conditions:
15
+
16
+ The above copyright notice and this permission notice shall be included in all
17
+ copies or substantial portions of the Software.
18
+
19
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25
+ SOFTWARE.
26
+
27
+ Project-URL: Homepage, https://github.com/ashutoshsinghpr7/wikiskill
28
+ Project-URL: Repository, https://github.com/ashutoshsinghpr7/wikiskill
29
+ Project-URL: arXiv, https://arxiv.org/abs/2608.27454
30
+ Keywords: agents,skills,skill-evolution,wiki,llm,hermes,self-improving,agent-experience,persistent-knowledge,arxiv-2608.27454
31
+ Classifier: Development Status :: 3 - Alpha
32
+ Classifier: Intended Audience :: Developers
33
+ Classifier: Intended Audience :: Science/Research
34
+ Classifier: License :: OSI Approved :: MIT License
35
+ Classifier: Programming Language :: Python :: 3
36
+ Classifier: Programming Language :: Python :: 3.10
37
+ Classifier: Programming Language :: Python :: 3.11
38
+ Classifier: Programming Language :: Python :: 3.12
39
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
40
+ Requires-Python: >=3.10
41
+ Description-Content-Type: text/markdown
42
+ License-File: LICENSE
43
+ Dynamic: license-file
44
+
45
+ # ๐Ÿง  WikiSkill
46
+
47
+ **Compile agent experience into a persistent wiki โ€” and let skills evolve themselves.**
48
+
49
+ > ๐Ÿ“š **Docs site**: [ashutoshsinghpr7.github.io/wikiskill](https://ashutoshsinghpr7.github.io/wikiskill) ยท **arXiv**: [2608.27454](https://arxiv.org/abs/2608.27454)
50
+
51
+ A faithful, production-minded implementation of **[WikiSkill: Compiling Agent Experience into Persistent Knowledge for Skill Evolution](https://arxiv.org/abs/2608.27454)** (arXiv:2608.27454, Google Research). The loop is **agent-agnostic** โ€” **Hermes Agent** is the reference backend (built natively), **Claude Code** ships in the box, and Codex/OpenCode are on the roadmap ([issue #13](https://github.com/ashutoshsinghpr7/wikiskill/issues/13)). Your agent becomes both the *student* and the *teacher*.
52
+
53
+ [![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org)
54
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
55
+ [![CI](https://img.shields.io/github/actions/workflow/status/ashutoshsinghpr7/wikiskill/ci.yml?label=CI)](https://github.com/ashutoshsinghpr7/wikiskill/actions)
56
+ [![arXiv](https://img.shields.io/badge/arXiv-2608.27454-red.svg)](https://arxiv.org/abs/2608.27454)
57
+
58
+ ---
59
+
60
+ ## What this is
61
+
62
+ Agents fail. They also *learn* โ€” but the lessons usually die with the session. WikiSkill fixes that by keeping a **persistent knowledge wiki** alongside the skill set, and running a closed evolution loop:
63
+
64
+ 1. The agent runs **training tasks** with its current skills โ†’ raw execution traces
65
+ 2. A **Wiki Maintainer** agent distills the traces into pattern pages (root causes, fixes)
66
+ 3. A **Skill Proposer** agent reads the wiki + traces and proposes one skill change (create or patch)
67
+ 4. **Gating**: the change is validated on held-out tasks โ€” **strictly better** than the best score so far โ†’ kept; otherwise rolled back. **The wiki is never rolled back.**
68
+
69
+ Over iterations, knowledge compounds in the wiki while only *proven* improvements touch the skills.
70
+
71
+ ```
72
+ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”
73
+ โ”‚ EVOLUTION LOOP (Algorithm 1) โ”‚
74
+ โ”‚ โ”‚
75
+ tasks โ”€โ”€โ”€โ”€โ”€โ–บ โ”‚ Inference Agent โ”€โ”€โ–บ raw/traces/ (immutable) โ”‚
76
+ โ”‚ โ”‚ โ”‚
77
+ โ”‚ โ–ผ โ”‚
78
+ โ”‚ Wiki Maintainer โ”€โ”€โ–บ wiki/patterns/, index, log โ”‚
79
+ โ”‚ โ”‚ โ”‚
80
+ โ”‚ โ–ผ โ”‚
81
+ โ”‚ Skill Proposer โ”€โ”€โ–บ proposal (create/patch skill) โ”‚
82
+ โ”‚ โ”‚ โ”‚
83
+ โ”‚ โ–ผ โ”‚
84
+ โ”‚ GATE: val score > R_best? โ”€โ”€yesโ”€โ”€โ–บ keep, R_best=R โ”‚
85
+ โ”‚ โ”‚ no โ”‚
86
+ โ”‚ โ–ผ โ”‚
87
+ โ”‚ rollback skills; wiki retained forever โ”‚
88
+ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜
89
+ ```
90
+
91
+ ## Why Hermes?
92
+
93
+ This is not a toy simulator. Every component is a **real Hermes agent turn**:
94
+
95
+ | WikiSkill (paper) | This repo |
96
+ |---|---|
97
+ | Inference Agent | `hermes chat --oneshot` in an **isolated `HERMES_HOME` profile** |
98
+ | Raw Layer | Full session JSONL transcripts, exported via `hermes sessions export` |
99
+ | Wiki Layer | `wiki/` โ€” git-tracked, maintained by a real agent, never rolled back |
100
+ | Skill Layer | Real `SKILL.md` packages (frontmatter + instructions), git-managed |
101
+ | Wiki Maintainer | Agent turn with the paper's **Appendix E.2 prompt** (extracted verbatim) |
102
+ | Skill Proposer | Agent turn with the paper's **Appendix E.3 ReAct prompt** |
103
+ | Gating | Strict `R_val > R_best`; `git reset --hard` on reject |
104
+
105
+ **Why the isolated profile matters:** gating is only meaningful if the agent sees *exactly* the candidate skill set. Each evolution workspace gets its own `HERMES_HOME` (bundled skills opted out, empty memory, skills symlinked per stage) โ€” your real profile is never touched.
106
+
107
+ ## Quickstart (60 seconds)
108
+
109
+ ```bash
110
+ pip install -e . # installs the `wikiskill` CLI
111
+ wikiskill init demo # workspace + 22-task auto-graded bench (13 train / 9 val)
112
+ wikiskill status
113
+ wikiskill evolve --iters 3 # full Algorithm 1 loop with your default model
114
+ ```
115
+
116
+ That's it. Each evolution workspace lives at `workspaces/<domain>/`:
117
+
118
+ ```
119
+ workspaces/demo/
120
+ โ”œโ”€โ”€ raw/traces/iter-01/{train,val}/<task>.jsonl # immutable execution traces
121
+ โ”œโ”€โ”€ wiki/ # persistent knowledge (never rolled back)
122
+ โ”‚ โ”œโ”€โ”€ index.md ยท log.md ยท skill-impact.md ยท patterns/*.md
123
+ โ”œโ”€โ”€ skills/active/ # git-managed evolving skill set (Sโ‚€ = โˆ…)
124
+ โ”œโ”€โ”€ skills/framework/ # maintainer + proposer agent skills
125
+ โ”œโ”€โ”€ bench/tasks/<id>/ # task sandboxes (inputs + grader)
126
+ โ””โ”€โ”€ runs/ # per-run stdout, proposals, state
127
+ ```
128
+
129
+ ## CLI
130
+
131
+ | Command | What it does |
132
+ |---|---|
133
+ | `wikiskill init <domain> [--backend claude]` | Create workspace + demo bench (pins the agent backend) |
134
+ | `wikiskill bench --reset` | Regenerate tasks (deterministic, seed=42) |
135
+ | `wikiskill status` | Workspace state: scores, skills, wiki, history |
136
+ | `wikiskill evolve --iters N [--model M] [--provider P] [--max-turns N] [--no-early-stop]` | The full loop (`--model`/`--provider` patch the isolated profile's default model, e.g. `google/gemini-2.5-flash-lite` + `openrouter`) |
137
+ | `wikiskill run-task <id>` | Single inference rollout (debug) |
138
+ | `wikiskill compare <wsA> <wsB> [--iters N]` | Paired statistical comparison: per-task win/loss/tie + two-sided exact-binomial p-value (answers "did the skill actually help?" โ€” see [docs/COMPARING.md](docs/COMPARING.md)) |
139
+
140
+ ## Bring your own tasks
141
+
142
+ Tasks are plain JSON (`tasks.json`); anything auto-gradable works:
143
+
144
+ ```json
145
+ {
146
+ "id": "spec-format1-1", "split": "train",
147
+ "title": "Format products according to spec",
148
+ "prompt": "Read spec.md and products.json...",
149
+ "sandbox": {"spec.md": "...", "products.json": "..."},
150
+ "grader": {"type": "exact", "file": "output.txt", "expected": "alpha|35|active\n..."}
151
+ }
152
+ ```
153
+
154
+ Graders: `exact`, `contains`, `json_field`, `code_stdout` (runs the produced script). Missing deliverables score **0**, never crash.
155
+
156
+ ## Live results so far
157
+
158
+ Honest numbers from real agent runs on the bundled bench:
159
+
160
+ | Setup | Baseline (Sโ‚€) | What happened |
161
+ |---|---|---|
162
+ | deepseek-v4-flash, 15 turns | **1.0** | Algorithm 1 **early-stop** โ€” nothing to evolve |
163
+ | deepseek-v4-flash, 8 turns | **1.0** | same |
164
+ | deepseek-v4-flash, forced | 1.0 | proposer created `spec_literal_transform` โ†’ R_val=1.0, not > R_best โ†’ **rejected** |
165
+ | deepseek-v4-flash, forced | 1.0 | maintainer distilled **4 pattern pages** (incl. `execute_code` blocked in sandbox, ripgrep binary misses); proposer created `exact-match-sandbox-task` โ†’ R_val=0.8889 (skill *hurt*) โ†’ **rejected** |
166
+ | gemma-3-4b (free, OpenRouter) | โ€” | **invalid run, thrown out** โ€” dead agent sessions were phantom-graded against stale sandboxes. The maintainer's pattern page caught the framework's own bug; fixed + regression-tested (see [docs/RUNS.md](docs/RUNS.md) Run 4) |
167
+ | **gemini-2.5-flash-lite (free, OpenRouter), 8 turns** | **0.6667** (real) | small model fails at Sโ‚€ โ†’ maintainer distilled 5 patterns โ†’ proposer created `find-secret` โ†’ **R_val=0.4444, the skill hurt (2 regressions) โ†’ rejected**. Full loop live on a genuinely weak model, ~$0.09/iteration |
168
+ | gemini-2.5-flash-lite, 3 iterations (issue #5) | **0.4444** (real) | compounding run: train as low as 0.2308, 6/48 launch failures (detected + honest 0.0s), maintainer distilled 1 pattern, proposer **declined (`no_action`)** โ€” nothing to gate, r_best preserved. Honest negative: accumulation needs a stronger model (see [docs/RUNS.md](docs/RUNS.md) Run 6) |
169
+
170
+ The gating mechanism has caught both a neutral and a *harmful* proposal live. Full logs in [docs/RUNS.md](docs/RUNS.md).
171
+
172
+ ## Design decisions worth knowing
173
+
174
+ - **`--in` doesn't pin the agent's CWD** in single-query runs โ†’ every inference prompt embeds an absolute `WORKING DIRECTORY` and forbids exploring outside it.
175
+ - **Sessions live in `state.db`**, not loose files โ†’ transcripts are materialized via `hermes sessions export --format jsonl`.
176
+ - **Rejected proposals are never lost** โ€” their full content is embedded in `wiki/skill-impact.md` so future proposers don't repeat them (per Appendix E.3).
177
+ - **The demo bench has traps**: subtle-spec tasks and multi-bug debug scripts whose bugs *don't* compensate (verified at generation time).
178
+
179
+ ## How this compares to other community implementations
180
+
181
+ We audited the three repos that appeared alongside the paper (see [docs/RUNS.md](docs/RUNS.md)). This is the only one that: runs on a real agent stack (Hermes), gates skills through a fully isolated profile, ships verbatim Appendix E prompts, and has a live-verified end-to-end loop (maintainer โ†’ proposer โ†’ gate โ†’ rollback).
182
+
183
+ ## Agent backends
184
+
185
+ The loop runs on any supported agent CLI โ€” the raw/wiki/skill layers are
186
+ backend-agnostic ([issue #13](https://github.com/ashutoshsinghpr7/wikiskill/issues/13)).
187
+
188
+ | Backend | Pin a workspace | Notes |
189
+ |---|---|---|
190
+ | `hermes` (default) | `wikiskill init demo --backend hermes` | reference implementation; isolated `HERMES_HOME` per workspace |
191
+ | `claude` | `wikiskill init demo --backend claude` | Claude Code 2.x (`claude -p`), isolated `CLAUDE_CONFIG_DIR`, transcripts normalized from the stream-json output; `claude auth login` required once |
192
+
193
+ Each workspace pins its backend in `workspaces/<domain>/workspace.json`; switch
194
+ anytime with `wikiskill evolve <domain> --backend claude`. Skills evolved on
195
+ one backend transfer to another via `wikiskill transfer` (same SKILL.md format).
196
+
197
+ ## Roadmap
198
+
199
+ - [ ] Multi-iteration compounding run (the paper's key ablation: wiki accumulation matters)
200
+ - [ ] Skill *transfer* across models (paper: skills evolved by one model help another)
201
+ - [ ] Real-task domains (your recurring workflows as graded task sets)
202
+ - [ ] `compare` command for statistical run comparison
203
+ - [ ] Cron-driven overnight evolution (`hermes cron`)
204
+
205
+ ## License
206
+
207
+ MIT โ€” see [LICENSE](LICENSE). Based on arXiv:2608.27454 (Google Research); all prompts in `skills/` are adapted from the paper's Appendix E. Inspired by [Karpathy's LLM Wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f).
@@ -0,0 +1,163 @@
1
+ # ๐Ÿง  WikiSkill
2
+
3
+ **Compile agent experience into a persistent wiki โ€” and let skills evolve themselves.**
4
+
5
+ > ๐Ÿ“š **Docs site**: [ashutoshsinghpr7.github.io/wikiskill](https://ashutoshsinghpr7.github.io/wikiskill) ยท **arXiv**: [2608.27454](https://arxiv.org/abs/2608.27454)
6
+
7
+ A faithful, production-minded implementation of **[WikiSkill: Compiling Agent Experience into Persistent Knowledge for Skill Evolution](https://arxiv.org/abs/2608.27454)** (arXiv:2608.27454, Google Research). The loop is **agent-agnostic** โ€” **Hermes Agent** is the reference backend (built natively), **Claude Code** ships in the box, and Codex/OpenCode are on the roadmap ([issue #13](https://github.com/ashutoshsinghpr7/wikiskill/issues/13)). Your agent becomes both the *student* and the *teacher*.
8
+
9
+ [![Python](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org)
10
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
11
+ [![CI](https://img.shields.io/github/actions/workflow/status/ashutoshsinghpr7/wikiskill/ci.yml?label=CI)](https://github.com/ashutoshsinghpr7/wikiskill/actions)
12
+ [![arXiv](https://img.shields.io/badge/arXiv-2608.27454-red.svg)](https://arxiv.org/abs/2608.27454)
13
+
14
+ ---
15
+
16
+ ## What this is
17
+
18
+ Agents fail. They also *learn* โ€” but the lessons usually die with the session. WikiSkill fixes that by keeping a **persistent knowledge wiki** alongside the skill set, and running a closed evolution loop:
19
+
20
+ 1. The agent runs **training tasks** with its current skills โ†’ raw execution traces
21
+ 2. A **Wiki Maintainer** agent distills the traces into pattern pages (root causes, fixes)
22
+ 3. A **Skill Proposer** agent reads the wiki + traces and proposes one skill change (create or patch)
23
+ 4. **Gating**: the change is validated on held-out tasks โ€” **strictly better** than the best score so far โ†’ kept; otherwise rolled back. **The wiki is never rolled back.**
24
+
25
+ Over iterations, knowledge compounds in the wiki while only *proven* improvements touch the skills.
26
+
27
+ ```
28
+ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”
29
+ โ”‚ EVOLUTION LOOP (Algorithm 1) โ”‚
30
+ โ”‚ โ”‚
31
+ tasks โ”€โ”€โ”€โ”€โ”€โ–บ โ”‚ Inference Agent โ”€โ”€โ–บ raw/traces/ (immutable) โ”‚
32
+ โ”‚ โ”‚ โ”‚
33
+ โ”‚ โ–ผ โ”‚
34
+ โ”‚ Wiki Maintainer โ”€โ”€โ–บ wiki/patterns/, index, log โ”‚
35
+ โ”‚ โ”‚ โ”‚
36
+ โ”‚ โ–ผ โ”‚
37
+ โ”‚ Skill Proposer โ”€โ”€โ–บ proposal (create/patch skill) โ”‚
38
+ โ”‚ โ”‚ โ”‚
39
+ โ”‚ โ–ผ โ”‚
40
+ โ”‚ GATE: val score > R_best? โ”€โ”€yesโ”€โ”€โ–บ keep, R_best=R โ”‚
41
+ โ”‚ โ”‚ no โ”‚
42
+ โ”‚ โ–ผ โ”‚
43
+ โ”‚ rollback skills; wiki retained forever โ”‚
44
+ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜
45
+ ```
46
+
47
+ ## Why Hermes?
48
+
49
+ This is not a toy simulator. Every component is a **real Hermes agent turn**:
50
+
51
+ | WikiSkill (paper) | This repo |
52
+ |---|---|
53
+ | Inference Agent | `hermes chat --oneshot` in an **isolated `HERMES_HOME` profile** |
54
+ | Raw Layer | Full session JSONL transcripts, exported via `hermes sessions export` |
55
+ | Wiki Layer | `wiki/` โ€” git-tracked, maintained by a real agent, never rolled back |
56
+ | Skill Layer | Real `SKILL.md` packages (frontmatter + instructions), git-managed |
57
+ | Wiki Maintainer | Agent turn with the paper's **Appendix E.2 prompt** (extracted verbatim) |
58
+ | Skill Proposer | Agent turn with the paper's **Appendix E.3 ReAct prompt** |
59
+ | Gating | Strict `R_val > R_best`; `git reset --hard` on reject |
60
+
61
+ **Why the isolated profile matters:** gating is only meaningful if the agent sees *exactly* the candidate skill set. Each evolution workspace gets its own `HERMES_HOME` (bundled skills opted out, empty memory, skills symlinked per stage) โ€” your real profile is never touched.
62
+
63
+ ## Quickstart (60 seconds)
64
+
65
+ ```bash
66
+ pip install -e . # installs the `wikiskill` CLI
67
+ wikiskill init demo # workspace + 22-task auto-graded bench (13 train / 9 val)
68
+ wikiskill status
69
+ wikiskill evolve --iters 3 # full Algorithm 1 loop with your default model
70
+ ```
71
+
72
+ That's it. Each evolution workspace lives at `workspaces/<domain>/`:
73
+
74
+ ```
75
+ workspaces/demo/
76
+ โ”œโ”€โ”€ raw/traces/iter-01/{train,val}/<task>.jsonl # immutable execution traces
77
+ โ”œโ”€โ”€ wiki/ # persistent knowledge (never rolled back)
78
+ โ”‚ โ”œโ”€โ”€ index.md ยท log.md ยท skill-impact.md ยท patterns/*.md
79
+ โ”œโ”€โ”€ skills/active/ # git-managed evolving skill set (Sโ‚€ = โˆ…)
80
+ โ”œโ”€โ”€ skills/framework/ # maintainer + proposer agent skills
81
+ โ”œโ”€โ”€ bench/tasks/<id>/ # task sandboxes (inputs + grader)
82
+ โ””โ”€โ”€ runs/ # per-run stdout, proposals, state
83
+ ```
84
+
85
+ ## CLI
86
+
87
+ | Command | What it does |
88
+ |---|---|
89
+ | `wikiskill init <domain> [--backend claude]` | Create workspace + demo bench (pins the agent backend) |
90
+ | `wikiskill bench --reset` | Regenerate tasks (deterministic, seed=42) |
91
+ | `wikiskill status` | Workspace state: scores, skills, wiki, history |
92
+ | `wikiskill evolve --iters N [--model M] [--provider P] [--max-turns N] [--no-early-stop]` | The full loop (`--model`/`--provider` patch the isolated profile's default model, e.g. `google/gemini-2.5-flash-lite` + `openrouter`) |
93
+ | `wikiskill run-task <id>` | Single inference rollout (debug) |
94
+ | `wikiskill compare <wsA> <wsB> [--iters N]` | Paired statistical comparison: per-task win/loss/tie + two-sided exact-binomial p-value (answers "did the skill actually help?" โ€” see [docs/COMPARING.md](docs/COMPARING.md)) |
95
+
96
+ ## Bring your own tasks
97
+
98
+ Tasks are plain JSON (`tasks.json`); anything auto-gradable works:
99
+
100
+ ```json
101
+ {
102
+ "id": "spec-format1-1", "split": "train",
103
+ "title": "Format products according to spec",
104
+ "prompt": "Read spec.md and products.json...",
105
+ "sandbox": {"spec.md": "...", "products.json": "..."},
106
+ "grader": {"type": "exact", "file": "output.txt", "expected": "alpha|35|active\n..."}
107
+ }
108
+ ```
109
+
110
+ Graders: `exact`, `contains`, `json_field`, `code_stdout` (runs the produced script). Missing deliverables score **0**, never crash.
111
+
112
+ ## Live results so far
113
+
114
+ Honest numbers from real agent runs on the bundled bench:
115
+
116
+ | Setup | Baseline (Sโ‚€) | What happened |
117
+ |---|---|---|
118
+ | deepseek-v4-flash, 15 turns | **1.0** | Algorithm 1 **early-stop** โ€” nothing to evolve |
119
+ | deepseek-v4-flash, 8 turns | **1.0** | same |
120
+ | deepseek-v4-flash, forced | 1.0 | proposer created `spec_literal_transform` โ†’ R_val=1.0, not > R_best โ†’ **rejected** |
121
+ | deepseek-v4-flash, forced | 1.0 | maintainer distilled **4 pattern pages** (incl. `execute_code` blocked in sandbox, ripgrep binary misses); proposer created `exact-match-sandbox-task` โ†’ R_val=0.8889 (skill *hurt*) โ†’ **rejected** |
122
+ | gemma-3-4b (free, OpenRouter) | โ€” | **invalid run, thrown out** โ€” dead agent sessions were phantom-graded against stale sandboxes. The maintainer's pattern page caught the framework's own bug; fixed + regression-tested (see [docs/RUNS.md](docs/RUNS.md) Run 4) |
123
+ | **gemini-2.5-flash-lite (free, OpenRouter), 8 turns** | **0.6667** (real) | small model fails at Sโ‚€ โ†’ maintainer distilled 5 patterns โ†’ proposer created `find-secret` โ†’ **R_val=0.4444, the skill hurt (2 regressions) โ†’ rejected**. Full loop live on a genuinely weak model, ~$0.09/iteration |
124
+ | gemini-2.5-flash-lite, 3 iterations (issue #5) | **0.4444** (real) | compounding run: train as low as 0.2308, 6/48 launch failures (detected + honest 0.0s), maintainer distilled 1 pattern, proposer **declined (`no_action`)** โ€” nothing to gate, r_best preserved. Honest negative: accumulation needs a stronger model (see [docs/RUNS.md](docs/RUNS.md) Run 6) |
125
+
126
+ The gating mechanism has caught both a neutral and a *harmful* proposal live. Full logs in [docs/RUNS.md](docs/RUNS.md).
127
+
128
+ ## Design decisions worth knowing
129
+
130
+ - **`--in` doesn't pin the agent's CWD** in single-query runs โ†’ every inference prompt embeds an absolute `WORKING DIRECTORY` and forbids exploring outside it.
131
+ - **Sessions live in `state.db`**, not loose files โ†’ transcripts are materialized via `hermes sessions export --format jsonl`.
132
+ - **Rejected proposals are never lost** โ€” their full content is embedded in `wiki/skill-impact.md` so future proposers don't repeat them (per Appendix E.3).
133
+ - **The demo bench has traps**: subtle-spec tasks and multi-bug debug scripts whose bugs *don't* compensate (verified at generation time).
134
+
135
+ ## How this compares to other community implementations
136
+
137
+ We audited the three repos that appeared alongside the paper (see [docs/RUNS.md](docs/RUNS.md)). This is the only one that: runs on a real agent stack (Hermes), gates skills through a fully isolated profile, ships verbatim Appendix E prompts, and has a live-verified end-to-end loop (maintainer โ†’ proposer โ†’ gate โ†’ rollback).
138
+
139
+ ## Agent backends
140
+
141
+ The loop runs on any supported agent CLI โ€” the raw/wiki/skill layers are
142
+ backend-agnostic ([issue #13](https://github.com/ashutoshsinghpr7/wikiskill/issues/13)).
143
+
144
+ | Backend | Pin a workspace | Notes |
145
+ |---|---|---|
146
+ | `hermes` (default) | `wikiskill init demo --backend hermes` | reference implementation; isolated `HERMES_HOME` per workspace |
147
+ | `claude` | `wikiskill init demo --backend claude` | Claude Code 2.x (`claude -p`), isolated `CLAUDE_CONFIG_DIR`, transcripts normalized from the stream-json output; `claude auth login` required once |
148
+
149
+ Each workspace pins its backend in `workspaces/<domain>/workspace.json`; switch
150
+ anytime with `wikiskill evolve <domain> --backend claude`. Skills evolved on
151
+ one backend transfer to another via `wikiskill transfer` (same SKILL.md format).
152
+
153
+ ## Roadmap
154
+
155
+ - [ ] Multi-iteration compounding run (the paper's key ablation: wiki accumulation matters)
156
+ - [ ] Skill *transfer* across models (paper: skills evolved by one model help another)
157
+ - [ ] Real-task domains (your recurring workflows as graded task sets)
158
+ - [ ] `compare` command for statistical run comparison
159
+ - [ ] Cron-driven overnight evolution (`hermes cron`)
160
+
161
+ ## License
162
+
163
+ MIT โ€” see [LICENSE](LICENSE). Based on arXiv:2608.27454 (Google Research); all prompts in `skills/` are adapted from the paper's Appendix E. Inspired by [Karpathy's LLM Wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f).
@@ -0,0 +1,39 @@
1
+ [build-system]
2
+ requires = ["setuptools>=64"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "wikiskill"
7
+ version = "0.1.1"
8
+ description = "WikiSkill (arXiv:2608.27454) for Hermes Agent โ€” self-evolving agent skills via a persistent knowledge wiki"
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { file = "LICENSE" }
12
+ keywords = [
13
+ "agents", "skills", "skill-evolution", "wiki", "llm", "hermes",
14
+ "self-improving", "agent-experience", "persistent-knowledge",
15
+ "arxiv-2608.27454",
16
+ ]
17
+ classifiers = [
18
+ "Development Status :: 3 - Alpha",
19
+ "Intended Audience :: Developers",
20
+ "Intended Audience :: Science/Research",
21
+ "License :: OSI Approved :: MIT License",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3.10",
24
+ "Programming Language :: Python :: 3.11",
25
+ "Programming Language :: Python :: 3.12",
26
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
27
+ ]
28
+ dependencies = []
29
+
30
+ [project.scripts]
31
+ wikiskill = "wikiskill.cli:main"
32
+
33
+ [project.urls]
34
+ Homepage = "https://github.com/ashutoshsinghpr7/wikiskill"
35
+ Repository = "https://github.com/ashutoshsinghpr7/wikiskill"
36
+ arXiv = "https://arxiv.org/abs/2608.27454"
37
+
38
+ [tool.setuptools]
39
+ packages = ["wikiskill"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+