skilljab 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skilljab-0.1.0/LICENSE +21 -0
- skilljab-0.1.0/PKG-INFO +235 -0
- skilljab-0.1.0/README.md +211 -0
- skilljab-0.1.0/pyproject.toml +40 -0
- skilljab-0.1.0/setup.cfg +4 -0
- skilljab-0.1.0/skilljab/__init__.py +7 -0
- skilljab-0.1.0/skilljab/check.py +53 -0
- skilljab-0.1.0/skilljab/cli.py +152 -0
- skilljab-0.1.0/skilljab/graveyard.py +47 -0
- skilljab-0.1.0/skilljab/inject.py +56 -0
- skilljab-0.1.0/skilljab/lineup.py +25 -0
- skilljab-0.1.0/skilljab/pack/diversity/personas.yaml +29 -0
- skilljab-0.1.0/skilljab/pack/graveyard/seed.json +34 -0
- skilljab-0.1.0/skilljab/pack/symptoms.yaml +40 -0
- skilljab-0.1.0/skilljab/plandiff.py +40 -0
- skilljab-0.1.0/skilljab/project.py +153 -0
- skilljab-0.1.0/skilljab/render_skill.py +75 -0
- skilljab-0.1.0/skilljab/report.py +83 -0
- skilljab-0.1.0/skilljab/report_assets/report.html +231 -0
- skilljab-0.1.0/skilljab/runner.py +131 -0
- skilljab-0.1.0/skilljab/simulate.py +85 -0
- skilljab-0.1.0/skilljab/stones/__init__.py +32 -0
- skilljab-0.1.0/skilljab/stones/_common.py +14 -0
- skilljab-0.1.0/skilljab/stones/batch_shift.py +18 -0
- skilljab-0.1.0/skilljab/stones/catalog.yaml +79 -0
- skilljab-0.1.0/skilljab/stones/correlated_block.py +14 -0
- skilljab-0.1.0/skilljab/stones/duplicates.py +8 -0
- skilljab-0.1.0/skilljab/stones/heavy_tails.py +12 -0
- skilljab-0.1.0/skilljab/stones/measurement_error.py +9 -0
- skilljab-0.1.0/skilljab/stones/mnar_missing.py +11 -0
- skilljab-0.1.0/skilljab/stones/outliers.py +12 -0
- skilljab-0.1.0/skilljab/stones/rare_category.py +11 -0
- skilljab-0.1.0/skilljab/stones/target_leakage.py +9 -0
- skilljab-0.1.0/skilljab/stones/type_corruption.py +14 -0
- skilljab-0.1.0/skilljab/stones/unit_mix.py +8 -0
- skilljab-0.1.0/skilljab/timing.py +84 -0
- skilljab-0.1.0/skilljab/tree.py +44 -0
- skilljab-0.1.0/skilljab/util.py +39 -0
- skilljab-0.1.0/skilljab.egg-info/PKG-INFO +235 -0
- skilljab-0.1.0/skilljab.egg-info/SOURCES.txt +46 -0
- skilljab-0.1.0/skilljab.egg-info/dependency_links.txt +1 -0
- skilljab-0.1.0/skilljab.egg-info/entry_points.txt +2 -0
- skilljab-0.1.0/skilljab.egg-info/requires.txt +6 -0
- skilljab-0.1.0/skilljab.egg-info/top_level.txt +1 -0
- skilljab-0.1.0/tests/test_engine_loop.py +95 -0
- skilljab-0.1.0/tests/test_plugin.py +55 -0
- skilljab-0.1.0/tests/test_reasoning_helpers.py +129 -0
- skilljab-0.1.0/tests/test_simulate_and_stones.py +71 -0
skilljab-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Alsamman M. Alsamman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
skilljab-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: skilljab
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Immunize an analysis by sabotaging it before it sees real data - and keep the antibodies as a skill.
|
|
5
|
+
Author-email: "Alsamman M. Alsamman" <aalsamman100@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/AlsammanAlsamman/skilljab
|
|
8
|
+
Project-URL: Repository, https://github.com/AlsammanAlsamman/skilljab
|
|
9
|
+
Keywords: data-analysis,simulation,chaos-engineering,statistics,pipelines,claude-code,skills
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: numpy>=1.24
|
|
19
|
+
Requires-Dist: pandas>=2.0
|
|
20
|
+
Requires-Dist: pyyaml>=6.0
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
23
|
+
Dynamic: license-file
|
|
24
|
+
|
|
25
|
+
<p align="center">
|
|
26
|
+
<img src="https://raw.githubusercontent.com/AlsammanAlsamman/skilljab/main/assets/icon/skilljab-mascot-v1.jpeg" alt="SkillJab mascot — a winking syringe with a boxing glove, jabbing a bar chart" width="260">
|
|
27
|
+
</p>
|
|
28
|
+
|
|
29
|
+
<h1 align="center">SkillJab</h1>
|
|
30
|
+
|
|
31
|
+
<p align="center"><b>Immunize an analysis by sabotaging it before it sees real data — and keep the antibodies as a skill.</b></p>
|
|
32
|
+
|
|
33
|
+
<p align="center">
|
|
34
|
+
<code>pipx install skilljab</code> · Claude Code plugin · Python 3.10+ · MIT
|
|
35
|
+
</p>
|
|
36
|
+
|
|
37
|
+
---
|
|
38
|
+
|
|
39
|
+
Every analyst has the same scar: a pipeline that ran for six hours and was wrong the whole time, because of something that was public knowledge — the dense-LD region, the leaked column, the duplicated join key, the spreadsheet that wrote `1,234`. AI assistants make this *worse*: they write clean, competent, silently wrong code, and the code looks so good you trust it.
|
|
40
|
+
|
|
41
|
+
SkillJab is a **jab** — a small controlled hit, like a vaccine shot or a sparring punch — delivered to a *miniature* of your analysis before the real run:
|
|
42
|
+
|
|
43
|
+
1. **Plant a truth.** Generate a tiny dataset where you know the answer.
|
|
44
|
+
2. **Prove recovery.** If the pipeline can't find a truth you planted, it's wrong before any real data.
|
|
45
|
+
3. **Throw stones.** Drop generic perturbations into the miniature — the Ghost (missing-not-at-random), the Clique (correlated block), the Time Traveler (leakage), the Twin (duplicates), the Drifter (batch shift)…
|
|
46
|
+
4. **Find the silent failures.** The result moved *and no check fired*. That's the only thing you see.
|
|
47
|
+
5. **Wake up the AI.** Each silent failure forces an explanation — which is when the model remembers the thing it knew but wasn't applying ("oh — this is the HLA region").
|
|
48
|
+
6. **Keep the antibody.** Explanation → checker + mitigation + heads-up → rendered into a per-pipeline `SKILL.md` that Claude loads every time it touches that analysis.
|
|
49
|
+
|
|
50
|
+
The skill is the accumulated antibodies. The report is a crash test.
|
|
51
|
+
|
|
52
|
+
## What it looks like
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
$ skilljab round run --skill .claude/skills/gwas
|
|
56
|
+
{ "class": "silent", "worst_rel_err": 0.9, "checks_fired": [] }
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
> **Round 1: The Time Traveler (target_leakage) hit stage `clean` — `beta_x1` moved 90% from the planted truth and nothing warned you.**
|
|
60
|
+
|
|
61
|
+
…then after `/skilljab:improve`:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
$ skilljab round run --skill .claude/skills/gwas
|
|
65
|
+
{ "class": "caught", "checks_fired": [{"script": "checks/no_leaky_columns.py", "message": "predictors nearly identical to outcome: ['x_score']"}] }
|
|
66
|
+
$ skilljab status --skill .claude/skills/gwas
|
|
67
|
+
{ "jabbed": true, "reason": "clean", "skill_version": 1, "n_antibodies": 1 }
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The report (`history/report.html`, self-contained, no external requests) shows star ratings per stage, a twin-result slider (drag the dose, watch your own estimates move), breaking-point curves, a stone × stage fragility matrix, predicted lap times at your real N, and a vaccination card of rounds × stones turning from red to green.
|
|
71
|
+
|
|
72
|
+
## Two halves
|
|
73
|
+
|
|
74
|
+
| | **Engine** — `skilljab` CLI (Python) | **Plugin** — Claude Code |
|
|
75
|
+
|---|---|---|
|
|
76
|
+
| does | simulate, inject, run, grade, sweep, diff plans, build the decision tree, score the lineup, predict time, render `SKILL.md` and the report | interview, plan ×N, elicit, judge, sabotage, analyze blind, explain, write antibodies |
|
|
77
|
+
| never | calls an LLM, decides what a failure *means* | touches data, computes a verdict, grades itself |
|
|
78
|
+
|
|
79
|
+
The engine grades; the AI reasons. Any fact that could come from either comes from the engine — that's what keeps the skill honest.
|
|
80
|
+
|
|
81
|
+
### Blind roles
|
|
82
|
+
|
|
83
|
+
Five subagents, each with only the files it's allowed to see:
|
|
84
|
+
|
|
85
|
+
- **planner ×N** — different persona + constraint each (statistician, domain expert, cluster admin, Reviewer 2, an outsider from another field…); never sees the other plans.
|
|
86
|
+
- **judge** — runs the *lineup*: "one of these plans failed — which, and why?" Never sees stone results, so its hit rate can be measured; sometimes a known culprit is planted to calibrate it.
|
|
87
|
+
- **saboteur** — picks and doses stones; writes `private/stones.json`.
|
|
88
|
+
- **analyst** — runs the pipeline blind with the current skill loaded; forbidden from opening `private/`.
|
|
89
|
+
- **explainer** — the only role that sees what was planted; does the wake-up, writes the antibody.
|
|
90
|
+
|
|
91
|
+
### The funnel
|
|
92
|
+
|
|
93
|
+
Search wide and cheap, prove narrow and expensive:
|
|
94
|
+
|
|
95
|
+
0. **Divergence map** — diff N plans; where they disagree is where uncertainty lives.
|
|
96
|
+
1. **Déjà vu** — `skilljab graveyard search` shows *setups* of similar past failures; predict the outcome, then reveal.
|
|
97
|
+
2. **Blurry-friend probe** — describe a symptom badly ("slow at step 3, warned about a parameter"); the model enumerates the neighbourhood; the user *recognizes* what they couldn't recall.
|
|
98
|
+
3. **Stones** — build, dose, run; only silent failures survive.
|
|
99
|
+
|
|
100
|
+
Elicitation proposes; simulation disposes. Nothing reaches the skill without a stone or a user's "that one."
|
|
101
|
+
|
|
102
|
+
## Install
|
|
103
|
+
|
|
104
|
+
SkillJab is a command-line tool, so install it with **pipx** (isolated, on your PATH, no fight with your system Python — on Debian/Ubuntu plain `pip install` is blocked by PEP 668):
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
# from PyPI (once published)
|
|
108
|
+
pipx install skilljab
|
|
109
|
+
|
|
110
|
+
# from GitHub, before/without PyPI
|
|
111
|
+
pipx install git+https://github.com/AlsammanAlsamman/skilljab.git
|
|
112
|
+
|
|
113
|
+
# from a local checkout (add -e to develop against it)
|
|
114
|
+
pipx install .
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
No pipx? `sudo apt install pipx && pipx ensurepath` (Debian/Ubuntu), `brew install pipx` (macOS), or use a venv: `python3 -m venv ~/.skilljab && ~/.skilljab/bin/pip install skilljab`.
|
|
118
|
+
|
|
119
|
+
Then the Claude Code plugin — point Claude at the plugin directory:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
claude --plugin-dir /path/to/skilljab/plugin # this session only
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Requires Python ≥ 3.10. `/usr/bin/time` (GNU time) is used for per-stage peak memory when present; otherwise a coarser fallback.
|
|
126
|
+
|
|
127
|
+
## Quick start (Claude Code)
|
|
128
|
+
|
|
129
|
+
```
|
|
130
|
+
/skilljab:build examples/toy_regression/pipeline.yaml
|
|
131
|
+
/skilljab:test
|
|
132
|
+
/skilljab:improve
|
|
133
|
+
/skilljab:report
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
or the whole vaccine in one go: `/skilljab:jab`. When you half-remember something odd from a past run: `/skilljab:recall`.
|
|
137
|
+
|
|
138
|
+
## Quick start (CLI only)
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
cd examples/toy_regression
|
|
142
|
+
skilljab init --name toy --pipeline pipeline.yaml --spec spec.yaml --skill-dir .claude/skills/toy
|
|
143
|
+
skilljab baseline --skill .claude/skills/toy --target-n 200000 # proves clean recovery, fits timing
|
|
144
|
+
skilljab round new --skill .claude/skills/toy
|
|
145
|
+
skilljab round stones --skill .claude/skills/toy target_leakage outliers --level 0.6
|
|
146
|
+
skilljab round run --skill .claude/skills/toy # -> "silent"
|
|
147
|
+
skilljab sweep --skill .claude/skills/toy --stone outliers --levels 5
|
|
148
|
+
skilljab report --skill .claude/skills/toy # history/report.html
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
## Your pipeline's contract
|
|
152
|
+
|
|
153
|
+
A `pipeline.yaml` of stages as shell commands — any language, any tool:
|
|
154
|
+
|
|
155
|
+
```yaml
|
|
156
|
+
name: gwas
|
|
157
|
+
stages:
|
|
158
|
+
- id: qc cmd: "plink2 --bfile {in} --geno 0.02 --make-bed --out {out}" out: qc.csv
|
|
159
|
+
- id: prune cmd: "Rscript prune.R {in} {out}" out: pruned.csv
|
|
160
|
+
- id: fit cmd: "python fit.py {in} {out}" out: result.json
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
The last stage writes `result.json = {"estimates": {"beta_x1": 0.79, ...}}` with the same estimand names as the planted truth in `sim/spec.yaml`. Checkers are scripts called at stage boundaries — `python3 check.py <stage_output>` — that exit `1` (or print `{"fired": true, "message": "..."}`) when they fire.
|
|
164
|
+
|
|
165
|
+
Three generators ship (`tabular_regression`, `tabular_classification`, `two_group_lift`); adding one is a function that returns `(DataFrame, {estimand: truth})`.
|
|
166
|
+
|
|
167
|
+
## The stones
|
|
168
|
+
|
|
169
|
+
| character | stone | what it does |
|
|
170
|
+
|---|---|---|
|
|
171
|
+
| The Clique | `correlated_block` | near-identical copies of a feature |
|
|
172
|
+
| The Ghost | `mnar_missing` | blanks the top values (missing-not-at-random) |
|
|
173
|
+
| The Twin | `duplicates` | re-appends rows |
|
|
174
|
+
| The Spike | `outliers` | a few impossible values |
|
|
175
|
+
| The Drifter | `batch_shift` | a hidden batch confounded with the outcome |
|
|
176
|
+
| The Blur | `measurement_error` | noise in a predictor (attenuation) |
|
|
177
|
+
| The Time Traveler | `target_leakage` | a feature computed from the outcome |
|
|
178
|
+
| The Metric Martian | `unit_mix` | two units in one column |
|
|
179
|
+
| The Long Tail | `heavy_tails` | Student-t noise |
|
|
180
|
+
| The Unicorn | `rare_category` | a level with almost no support |
|
|
181
|
+
| The Typo | `type_corruption` | `'1,234'`, `' 12 '`, `'NA'` in a numeric column |
|
|
182
|
+
|
|
183
|
+
They're deliberately domain-neutral. A generic stone is enough to make the model's own knowledge do the specialization: "a correlated block broke PCA" becomes "exclude chr6:25–35Mb before PCA" in the explainer's hands.
|
|
184
|
+
|
|
185
|
+
## Layout
|
|
186
|
+
|
|
187
|
+
```
|
|
188
|
+
skilljab/ engine (pip package): simulate · stones/ · inject · runner · check · project ·
|
|
189
|
+
plandiff · tree · lineup · timing · render_skill · graveyard · report · pack/
|
|
190
|
+
plugin/ Claude Code plugin: commands/ · agents/ · skills/skilljab-core · hooks/
|
|
191
|
+
examples/ toy_regression — a naive OLS pipeline that SkillJab breaks in four ways
|
|
192
|
+
tests/ 47 tests: every stone, the full loop, helpers, CLI, plugin, hook
|
|
193
|
+
docs/DESIGN.md the design and the thinking behind it
|
|
194
|
+
docs/sessions/ transcripts of the design discussions
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
Per pipeline, in *your* repo:
|
|
198
|
+
|
|
199
|
+
```
|
|
200
|
+
.claude/skills/<name>/
|
|
201
|
+
├── SKILL.md rendered — never hand-edit
|
|
202
|
+
├── antibodies.json source of truth, with provenance and evidence class
|
|
203
|
+
├── tree.json decisions with evidence on the edges
|
|
204
|
+
├── checks/ the antibodies' checkers
|
|
205
|
+
├── sim/spec.yaml how to build this pipeline's miniature
|
|
206
|
+
└── history/ baseline · sizes · round-NNN · sweeps · timing · lineup log · graveyard · report
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
## Honest limits
|
|
210
|
+
|
|
211
|
+
- **Miniatures can lie.** Some failures only appear at scale (memory cliffs, rare variants, convergence). The timing page extrapolates from three sizes and says so; treat ranges as optimistic.
|
|
212
|
+
- **Simulation only helps where you can plant a truth.** Quantitative analyses, yes. Design and strategy, no — there the simulation would just be the model's own assumptions fed back to itself.
|
|
213
|
+
- **Stones must bite.** A stone dosed too gently on a tiny miniature proves nothing; use `skilljab sweep` to find the knee.
|
|
214
|
+
- **The graveyard is local** in this version. A shared, anonymized one is the obvious next step and the one thing that would make déjà vu real rather than manufactured.
|
|
215
|
+
|
|
216
|
+
## Development
|
|
217
|
+
|
|
218
|
+
```bash
|
|
219
|
+
python3 -m venv .venv && . .venv/bin/activate
|
|
220
|
+
pip install -e ".[dev]"
|
|
221
|
+
python -m pytest -q # 47 tests, ~35 s
|
|
222
|
+
claude plugin validate plugin --strict
|
|
223
|
+
python scripts/export_session.py # save the current Claude Code session into docs/sessions/
|
|
224
|
+
|
|
225
|
+
pipx run build && pipx run twine check dist/* # build + verify the wheel and sdist
|
|
226
|
+
pipx run twine upload dist/* # publish to PyPI (needs an API token)
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
## Author
|
|
230
|
+
|
|
231
|
+
**Alsamman M. Alsamman** — aalsamman100@gmail.com · [github.com/AlsammanAlsamman](https://github.com/AlsammanAlsamman)
|
|
232
|
+
|
|
233
|
+
## Why the name
|
|
234
|
+
|
|
235
|
+
A *jab* is a vaccine shot and a boxing punch. Both are a small, deliberate hit that makes you stronger before the real fight. Read the design notes in [`docs/DESIGN.md`](docs/DESIGN.md) for the whole argument — the pre-mortem, the recoverability test, why generic stones beat a hand-written pitfall library, the lineup, and the tricks for making an AI say "oh yeah, I should have told you."
|
skilljab-0.1.0/README.md
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/AlsammanAlsamman/skilljab/main/assets/icon/skilljab-mascot-v1.jpeg" alt="SkillJab mascot — a winking syringe with a boxing glove, jabbing a bar chart" width="260">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">SkillJab</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center"><b>Immunize an analysis by sabotaging it before it sees real data — and keep the antibodies as a skill.</b></p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<code>pipx install skilljab</code> · Claude Code plugin · Python 3.10+ · MIT
|
|
11
|
+
</p>
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
Every analyst has the same scar: a pipeline that ran for six hours and was wrong the whole time, because of something that was public knowledge — the dense-LD region, the leaked column, the duplicated join key, the spreadsheet that wrote `1,234`. AI assistants make this *worse*: they write clean, competent, silently wrong code, and the code looks so good you trust it.
|
|
16
|
+
|
|
17
|
+
SkillJab is a **jab** — a small controlled hit, like a vaccine shot or a sparring punch — delivered to a *miniature* of your analysis before the real run:
|
|
18
|
+
|
|
19
|
+
1. **Plant a truth.** Generate a tiny dataset where you know the answer.
|
|
20
|
+
2. **Prove recovery.** If the pipeline can't find a truth you planted, it's wrong before any real data.
|
|
21
|
+
3. **Throw stones.** Drop generic perturbations into the miniature — the Ghost (missing-not-at-random), the Clique (correlated block), the Time Traveler (leakage), the Twin (duplicates), the Drifter (batch shift)…
|
|
22
|
+
4. **Find the silent failures.** The result moved *and no check fired*. That's the only thing you see.
|
|
23
|
+
5. **Wake up the AI.** Each silent failure forces an explanation — which is when the model remembers the thing it knew but wasn't applying ("oh — this is the HLA region").
|
|
24
|
+
6. **Keep the antibody.** Explanation → checker + mitigation + heads-up → rendered into a per-pipeline `SKILL.md` that Claude loads every time it touches that analysis.
|
|
25
|
+
|
|
26
|
+
The skill is the accumulated antibodies. The report is a crash test.
|
|
27
|
+
|
|
28
|
+
## What it looks like
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
$ skilljab round run --skill .claude/skills/gwas
|
|
32
|
+
{ "class": "silent", "worst_rel_err": 0.9, "checks_fired": [] }
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
> **Round 1: The Time Traveler (target_leakage) hit stage `clean` — `beta_x1` moved 90% from the planted truth and nothing warned you.**
|
|
36
|
+
|
|
37
|
+
…then after `/skilljab:improve`:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
$ skilljab round run --skill .claude/skills/gwas
|
|
41
|
+
{ "class": "caught", "checks_fired": [{"script": "checks/no_leaky_columns.py", "message": "predictors nearly identical to outcome: ['x_score']"}] }
|
|
42
|
+
$ skilljab status --skill .claude/skills/gwas
|
|
43
|
+
{ "jabbed": true, "reason": "clean", "skill_version": 1, "n_antibodies": 1 }
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The report (`history/report.html`, self-contained, no external requests) shows star ratings per stage, a twin-result slider (drag the dose, watch your own estimates move), breaking-point curves, a stone × stage fragility matrix, predicted lap times at your real N, and a vaccination card of rounds × stones turning from red to green.
|
|
47
|
+
|
|
48
|
+
## Two halves
|
|
49
|
+
|
|
50
|
+
| | **Engine** — `skilljab` CLI (Python) | **Plugin** — Claude Code |
|
|
51
|
+
|---|---|---|
|
|
52
|
+
| does | simulate, inject, run, grade, sweep, diff plans, build the decision tree, score the lineup, predict time, render `SKILL.md` and the report | interview, plan ×N, elicit, judge, sabotage, analyze blind, explain, write antibodies |
|
|
53
|
+
| never | calls an LLM, decides what a failure *means* | touches data, computes a verdict, grades itself |
|
|
54
|
+
|
|
55
|
+
The engine grades; the AI reasons. Any fact that could come from either comes from the engine — that's what keeps the skill honest.
|
|
56
|
+
|
|
57
|
+
### Blind roles
|
|
58
|
+
|
|
59
|
+
Five subagents, each with only the files it's allowed to see:
|
|
60
|
+
|
|
61
|
+
- **planner ×N** — different persona + constraint each (statistician, domain expert, cluster admin, Reviewer 2, an outsider from another field…); never sees the other plans.
|
|
62
|
+
- **judge** — runs the *lineup*: "one of these plans failed — which, and why?" Never sees stone results, so its hit rate can be measured; sometimes a known culprit is planted to calibrate it.
|
|
63
|
+
- **saboteur** — picks and doses stones; writes `private/stones.json`.
|
|
64
|
+
- **analyst** — runs the pipeline blind with the current skill loaded; forbidden from opening `private/`.
|
|
65
|
+
- **explainer** — the only role that sees what was planted; does the wake-up, writes the antibody.
|
|
66
|
+
|
|
67
|
+
### The funnel
|
|
68
|
+
|
|
69
|
+
Search wide and cheap, prove narrow and expensive:
|
|
70
|
+
|
|
71
|
+
0. **Divergence map** — diff N plans; where they disagree is where uncertainty lives.
|
|
72
|
+
1. **Déjà vu** — `skilljab graveyard search` shows *setups* of similar past failures; predict the outcome, then reveal.
|
|
73
|
+
2. **Blurry-friend probe** — describe a symptom badly ("slow at step 3, warned about a parameter"); the model enumerates the neighbourhood; the user *recognizes* what they couldn't recall.
|
|
74
|
+
3. **Stones** — build, dose, run; only silent failures survive.
|
|
75
|
+
|
|
76
|
+
Elicitation proposes; simulation disposes. Nothing reaches the skill without a stone or a user's "that one."
|
|
77
|
+
|
|
78
|
+
## Install
|
|
79
|
+
|
|
80
|
+
SkillJab is a command-line tool, so install it with **pipx** (isolated, on your PATH, no fight with your system Python — on Debian/Ubuntu plain `pip install` is blocked by PEP 668):
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
# from PyPI (once published)
|
|
84
|
+
pipx install skilljab
|
|
85
|
+
|
|
86
|
+
# from GitHub, before/without PyPI
|
|
87
|
+
pipx install git+https://github.com/AlsammanAlsamman/skilljab.git
|
|
88
|
+
|
|
89
|
+
# from a local checkout (add -e to develop against it)
|
|
90
|
+
pipx install .
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
No pipx? `sudo apt install pipx && pipx ensurepath` (Debian/Ubuntu), `brew install pipx` (macOS), or use a venv: `python3 -m venv ~/.skilljab && ~/.skilljab/bin/pip install skilljab`.
|
|
94
|
+
|
|
95
|
+
Then the Claude Code plugin — point Claude at the plugin directory:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
claude --plugin-dir /path/to/skilljab/plugin # this session only
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Requires Python ≥ 3.10. `/usr/bin/time` (GNU time) is used for per-stage peak memory when present; otherwise a coarser fallback.
|
|
102
|
+
|
|
103
|
+
## Quick start (Claude Code)
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
/skilljab:build examples/toy_regression/pipeline.yaml
|
|
107
|
+
/skilljab:test
|
|
108
|
+
/skilljab:improve
|
|
109
|
+
/skilljab:report
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
or the whole vaccine in one go: `/skilljab:jab`. When you half-remember something odd from a past run: `/skilljab:recall`.
|
|
113
|
+
|
|
114
|
+
## Quick start (CLI only)
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
cd examples/toy_regression
|
|
118
|
+
skilljab init --name toy --pipeline pipeline.yaml --spec spec.yaml --skill-dir .claude/skills/toy
|
|
119
|
+
skilljab baseline --skill .claude/skills/toy --target-n 200000 # proves clean recovery, fits timing
|
|
120
|
+
skilljab round new --skill .claude/skills/toy
|
|
121
|
+
skilljab round stones --skill .claude/skills/toy target_leakage outliers --level 0.6
|
|
122
|
+
skilljab round run --skill .claude/skills/toy # -> "silent"
|
|
123
|
+
skilljab sweep --skill .claude/skills/toy --stone outliers --levels 5
|
|
124
|
+
skilljab report --skill .claude/skills/toy # history/report.html
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## Your pipeline's contract
|
|
128
|
+
|
|
129
|
+
A `pipeline.yaml` of stages as shell commands — any language, any tool:
|
|
130
|
+
|
|
131
|
+
```yaml
|
|
132
|
+
name: gwas
|
|
133
|
+
stages:
|
|
134
|
+
- id: qc cmd: "plink2 --bfile {in} --geno 0.02 --make-bed --out {out}" out: qc.csv
|
|
135
|
+
- id: prune cmd: "Rscript prune.R {in} {out}" out: pruned.csv
|
|
136
|
+
- id: fit cmd: "python fit.py {in} {out}" out: result.json
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
The last stage writes `result.json = {"estimates": {"beta_x1": 0.79, ...}}` with the same estimand names as the planted truth in `sim/spec.yaml`. Checkers are scripts called at stage boundaries — `python3 check.py <stage_output>` — that exit `1` (or print `{"fired": true, "message": "..."}`) when they fire.
|
|
140
|
+
|
|
141
|
+
Three generators ship (`tabular_regression`, `tabular_classification`, `two_group_lift`); adding one is a function that returns `(DataFrame, {estimand: truth})`.
|
|
142
|
+
|
|
143
|
+
## The stones
|
|
144
|
+
|
|
145
|
+
| character | stone | what it does |
|
|
146
|
+
|---|---|---|
|
|
147
|
+
| The Clique | `correlated_block` | near-identical copies of a feature |
|
|
148
|
+
| The Ghost | `mnar_missing` | blanks the top values (missing-not-at-random) |
|
|
149
|
+
| The Twin | `duplicates` | re-appends rows |
|
|
150
|
+
| The Spike | `outliers` | a few impossible values |
|
|
151
|
+
| The Drifter | `batch_shift` | a hidden batch confounded with the outcome |
|
|
152
|
+
| The Blur | `measurement_error` | noise in a predictor (attenuation) |
|
|
153
|
+
| The Time Traveler | `target_leakage` | a feature computed from the outcome |
|
|
154
|
+
| The Metric Martian | `unit_mix` | two units in one column |
|
|
155
|
+
| The Long Tail | `heavy_tails` | Student-t noise |
|
|
156
|
+
| The Unicorn | `rare_category` | a level with almost no support |
|
|
157
|
+
| The Typo | `type_corruption` | `'1,234'`, `' 12 '`, `'NA'` in a numeric column |
|
|
158
|
+
|
|
159
|
+
They're deliberately domain-neutral. A generic stone is enough to make the model's own knowledge do the specialization: "a correlated block broke PCA" becomes "exclude chr6:25–35Mb before PCA" in the explainer's hands.
|
|
160
|
+
|
|
161
|
+
## Layout
|
|
162
|
+
|
|
163
|
+
```
|
|
164
|
+
skilljab/ engine (pip package): simulate · stones/ · inject · runner · check · project ·
|
|
165
|
+
plandiff · tree · lineup · timing · render_skill · graveyard · report · pack/
|
|
166
|
+
plugin/ Claude Code plugin: commands/ · agents/ · skills/skilljab-core · hooks/
|
|
167
|
+
examples/ toy_regression — a naive OLS pipeline that SkillJab breaks in four ways
|
|
168
|
+
tests/ 47 tests: every stone, the full loop, helpers, CLI, plugin, hook
|
|
169
|
+
docs/DESIGN.md the design and the thinking behind it
|
|
170
|
+
docs/sessions/ transcripts of the design discussions
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
Per pipeline, in *your* repo:
|
|
174
|
+
|
|
175
|
+
```
|
|
176
|
+
.claude/skills/<name>/
|
|
177
|
+
├── SKILL.md rendered — never hand-edit
|
|
178
|
+
├── antibodies.json source of truth, with provenance and evidence class
|
|
179
|
+
├── tree.json decisions with evidence on the edges
|
|
180
|
+
├── checks/ the antibodies' checkers
|
|
181
|
+
├── sim/spec.yaml how to build this pipeline's miniature
|
|
182
|
+
└── history/ baseline · sizes · round-NNN · sweeps · timing · lineup log · graveyard · report
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
## Honest limits
|
|
186
|
+
|
|
187
|
+
- **Miniatures can lie.** Some failures only appear at scale (memory cliffs, rare variants, convergence). The timing page extrapolates from three sizes and says so; treat ranges as optimistic.
|
|
188
|
+
- **Simulation only helps where you can plant a truth.** Quantitative analyses, yes. Design and strategy, no — there the simulation would just be the model's own assumptions fed back to itself.
|
|
189
|
+
- **Stones must bite.** A stone dosed too gently on a tiny miniature proves nothing; use `skilljab sweep` to find the knee.
|
|
190
|
+
- **The graveyard is local** in this version. A shared, anonymized one is the obvious next step and the one thing that would make déjà vu real rather than manufactured.
|
|
191
|
+
|
|
192
|
+
## Development
|
|
193
|
+
|
|
194
|
+
```bash
|
|
195
|
+
python3 -m venv .venv && . .venv/bin/activate
|
|
196
|
+
pip install -e ".[dev]"
|
|
197
|
+
python -m pytest -q # 47 tests, ~35 s
|
|
198
|
+
claude plugin validate plugin --strict
|
|
199
|
+
python scripts/export_session.py # save the current Claude Code session into docs/sessions/
|
|
200
|
+
|
|
201
|
+
pipx run build && pipx run twine check dist/* # build + verify the wheel and sdist
|
|
202
|
+
pipx run twine upload dist/* # publish to PyPI (needs an API token)
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
## Author
|
|
206
|
+
|
|
207
|
+
**Alsamman M. Alsamman** — aalsamman100@gmail.com · [github.com/AlsammanAlsamman](https://github.com/AlsammanAlsamman)
|
|
208
|
+
|
|
209
|
+
## Why the name
|
|
210
|
+
|
|
211
|
+
A *jab* is a vaccine shot and a boxing punch. Both are a small, deliberate hit that makes you stronger before the real fight. Read the design notes in [`docs/DESIGN.md`](docs/DESIGN.md) for the whole argument — the pre-mortem, the recoverability test, why generic stones beat a hand-written pitfall library, the lineup, and the tricks for making an AI say "oh yeah, I should have told you."
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "skilljab"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Immunize an analysis by sabotaging it before it sees real data - and keep the antibodies as a skill."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "Alsamman M. Alsamman", email = "aalsamman100@gmail.com"}]
|
|
13
|
+
keywords = ["data-analysis", "simulation", "chaos-engineering", "statistics", "pipelines", "claude-code", "skills"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Scientific/Engineering",
|
|
20
|
+
]
|
|
21
|
+
dependencies = ["numpy>=1.24", "pandas>=2.0", "pyyaml>=6.0"]
|
|
22
|
+
|
|
23
|
+
[project.urls]
|
|
24
|
+
Homepage = "https://github.com/AlsammanAlsamman/skilljab"
|
|
25
|
+
Repository = "https://github.com/AlsammanAlsamman/skilljab"
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
dev = ["pytest>=7"]
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
skilljab = "skilljab.cli:main"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
include = ["skilljab*"]
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.package-data]
|
|
37
|
+
skilljab = ["stones/catalog.yaml", "pack/*.yaml", "pack/diversity/*.yaml", "pack/graveyard/*.json", "report_assets/*"]
|
|
38
|
+
|
|
39
|
+
[tool.pytest.ini_options]
|
|
40
|
+
testpaths = ["tests"]
|
skilljab-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""SkillJab engine - deterministic tooling that Claude Code drives.
|
|
2
|
+
|
|
3
|
+
The engine simulates, injects stones, runs pipelines, grades results, diffs plans,
|
|
4
|
+
merges decision trees, scores the lineup, predicts time, renders SKILL.md and the
|
|
5
|
+
report. It never calls an LLM and never decides what a failure *means*.
|
|
6
|
+
"""
|
|
7
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Grade a run against the planted truth and the checkers -> verdict.json
|
|
2
|
+
|
|
3
|
+
Classes:
|
|
4
|
+
crashed pipeline did not produce a result
|
|
5
|
+
silent result degraded and NO checker fired <- the only thing the user must see
|
|
6
|
+
caught result degraded and a checker fired
|
|
7
|
+
false_alarm result fine but a checker fired
|
|
8
|
+
harmless result fine, nothing fired
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
from .util import read_json, write_json
|
|
12
|
+
|
|
13
|
+
def grade(truth, run, baseline_run=None):
|
|
14
|
+
tol = truth.get("tolerance", {}); rel_tol = float(tol.get("relative", 0.25)); abs_tol = float(tol.get("absolute", 0.1))
|
|
15
|
+
est = (run.get("result") or {}).get("estimates", {}) if run.get("result") else {}
|
|
16
|
+
per = {}
|
|
17
|
+
degraded = False
|
|
18
|
+
for name, tv in truth.get("estimands", {}).items():
|
|
19
|
+
if run.get("crashed") or name not in est or est[name] is None:
|
|
20
|
+
per[name] = {"truth": tv, "estimate": None, "abs_err": None, "rel_err": None, "degraded": True}
|
|
21
|
+
degraded = True; continue
|
|
22
|
+
ev = float(est[name]); ae = abs(ev - float(tv)); re = ae / (abs(float(tv)) if abs(float(tv)) > 1e-12 else 1.0)
|
|
23
|
+
d = (ae > abs_tol) and (re > rel_tol)
|
|
24
|
+
per[name] = {"truth": tv, "estimate": ev, "abs_err": round(ae, 6), "rel_err": round(re, 6), "degraded": d}
|
|
25
|
+
degraded = degraded or d
|
|
26
|
+
fired = [{"stage": s["id"], **c} for s in run.get("stages", []) for c in s.get("checks", []) if c.get("fired")]
|
|
27
|
+
if run.get("crashed"):
|
|
28
|
+
cls = "crashed"
|
|
29
|
+
elif degraded and not fired:
|
|
30
|
+
cls = "silent"
|
|
31
|
+
elif degraded and fired:
|
|
32
|
+
cls = "caught"
|
|
33
|
+
elif fired:
|
|
34
|
+
cls = "false_alarm"
|
|
35
|
+
else:
|
|
36
|
+
cls = "harmless"
|
|
37
|
+
# per-stage time impact vs baseline
|
|
38
|
+
time_impact = {}
|
|
39
|
+
if baseline_run:
|
|
40
|
+
base = {s["id"]: s["seconds"] for s in baseline_run.get("stages", [])}
|
|
41
|
+
for s in run.get("stages", []):
|
|
42
|
+
b = base.get(s["id"])
|
|
43
|
+
if b and b > 0:
|
|
44
|
+
time_impact[s["id"]] = round(s["seconds"] / b, 3)
|
|
45
|
+
worst = max([p["rel_err"] for p in per.values() if p["rel_err"] is not None] or [None if run.get("crashed") else 0.0], key=lambda x: (x is not None, x))
|
|
46
|
+
return {"class": cls, "degraded": degraded, "crashed": bool(run.get("crashed")), "worst_rel_err": worst,
|
|
47
|
+
"estimands": per, "checks_fired": fired, "time_impact": time_impact,
|
|
48
|
+
"stones": run.get("stones_applied", []), "n_rows": run.get("n_rows"), "label": run.get("label")}
|
|
49
|
+
|
|
50
|
+
def check_cli(truth_path, run_path, out_path, baseline_path=None):
|
|
51
|
+
v = grade(read_json(truth_path), read_json(run_path), read_json(baseline_path) if baseline_path else None)
|
|
52
|
+
write_json(out_path, v)
|
|
53
|
+
return v
|