skilljab 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. skilljab-0.1.0/LICENSE +21 -0
  2. skilljab-0.1.0/PKG-INFO +235 -0
  3. skilljab-0.1.0/README.md +211 -0
  4. skilljab-0.1.0/pyproject.toml +40 -0
  5. skilljab-0.1.0/setup.cfg +4 -0
  6. skilljab-0.1.0/skilljab/__init__.py +7 -0
  7. skilljab-0.1.0/skilljab/check.py +53 -0
  8. skilljab-0.1.0/skilljab/cli.py +152 -0
  9. skilljab-0.1.0/skilljab/graveyard.py +47 -0
  10. skilljab-0.1.0/skilljab/inject.py +56 -0
  11. skilljab-0.1.0/skilljab/lineup.py +25 -0
  12. skilljab-0.1.0/skilljab/pack/diversity/personas.yaml +29 -0
  13. skilljab-0.1.0/skilljab/pack/graveyard/seed.json +34 -0
  14. skilljab-0.1.0/skilljab/pack/symptoms.yaml +40 -0
  15. skilljab-0.1.0/skilljab/plandiff.py +40 -0
  16. skilljab-0.1.0/skilljab/project.py +153 -0
  17. skilljab-0.1.0/skilljab/render_skill.py +75 -0
  18. skilljab-0.1.0/skilljab/report.py +83 -0
  19. skilljab-0.1.0/skilljab/report_assets/report.html +231 -0
  20. skilljab-0.1.0/skilljab/runner.py +131 -0
  21. skilljab-0.1.0/skilljab/simulate.py +85 -0
  22. skilljab-0.1.0/skilljab/stones/__init__.py +32 -0
  23. skilljab-0.1.0/skilljab/stones/_common.py +14 -0
  24. skilljab-0.1.0/skilljab/stones/batch_shift.py +18 -0
  25. skilljab-0.1.0/skilljab/stones/catalog.yaml +79 -0
  26. skilljab-0.1.0/skilljab/stones/correlated_block.py +14 -0
  27. skilljab-0.1.0/skilljab/stones/duplicates.py +8 -0
  28. skilljab-0.1.0/skilljab/stones/heavy_tails.py +12 -0
  29. skilljab-0.1.0/skilljab/stones/measurement_error.py +9 -0
  30. skilljab-0.1.0/skilljab/stones/mnar_missing.py +11 -0
  31. skilljab-0.1.0/skilljab/stones/outliers.py +12 -0
  32. skilljab-0.1.0/skilljab/stones/rare_category.py +11 -0
  33. skilljab-0.1.0/skilljab/stones/target_leakage.py +9 -0
  34. skilljab-0.1.0/skilljab/stones/type_corruption.py +14 -0
  35. skilljab-0.1.0/skilljab/stones/unit_mix.py +8 -0
  36. skilljab-0.1.0/skilljab/timing.py +84 -0
  37. skilljab-0.1.0/skilljab/tree.py +44 -0
  38. skilljab-0.1.0/skilljab/util.py +39 -0
  39. skilljab-0.1.0/skilljab.egg-info/PKG-INFO +235 -0
  40. skilljab-0.1.0/skilljab.egg-info/SOURCES.txt +46 -0
  41. skilljab-0.1.0/skilljab.egg-info/dependency_links.txt +1 -0
  42. skilljab-0.1.0/skilljab.egg-info/entry_points.txt +2 -0
  43. skilljab-0.1.0/skilljab.egg-info/requires.txt +6 -0
  44. skilljab-0.1.0/skilljab.egg-info/top_level.txt +1 -0
  45. skilljab-0.1.0/tests/test_engine_loop.py +95 -0
  46. skilljab-0.1.0/tests/test_plugin.py +55 -0
  47. skilljab-0.1.0/tests/test_reasoning_helpers.py +129 -0
  48. skilljab-0.1.0/tests/test_simulate_and_stones.py +71 -0
skilljab-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Alsamman M. Alsamman
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,235 @@
1
+ Metadata-Version: 2.4
2
+ Name: skilljab
3
+ Version: 0.1.0
4
+ Summary: Immunize an analysis by sabotaging it before it sees real data - and keep the antibodies as a skill.
5
+ Author-email: "Alsamman M. Alsamman" <aalsamman100@gmail.com>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/AlsammanAlsamman/skilljab
8
+ Project-URL: Repository, https://github.com/AlsammanAlsamman/skilljab
9
+ Keywords: data-analysis,simulation,chaos-engineering,statistics,pipelines,claude-code,skills
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Scientific/Engineering
15
+ Requires-Python: >=3.10
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+ Requires-Dist: numpy>=1.24
19
+ Requires-Dist: pandas>=2.0
20
+ Requires-Dist: pyyaml>=6.0
21
+ Provides-Extra: dev
22
+ Requires-Dist: pytest>=7; extra == "dev"
23
+ Dynamic: license-file
24
+
25
+ <p align="center">
26
+ <img src="https://raw.githubusercontent.com/AlsammanAlsamman/skilljab/main/assets/icon/skilljab-mascot-v1.jpeg" alt="SkillJab mascot — a winking syringe with a boxing glove, jabbing a bar chart" width="260">
27
+ </p>
28
+
29
+ <h1 align="center">SkillJab</h1>
30
+
31
+ <p align="center"><b>Immunize an analysis by sabotaging it before it sees real data — and keep the antibodies as a skill.</b></p>
32
+
33
+ <p align="center">
34
+ <code>pipx install skilljab</code> · Claude Code plugin · Python 3.10+ · MIT
35
+ </p>
36
+
37
+ ---
38
+
39
+ Every analyst has the same scar: a pipeline that ran for six hours and was wrong the whole time, because of something that was public knowledge — the dense-LD region, the leaked column, the duplicated join key, the spreadsheet that wrote `1,234`. AI assistants make this *worse*: they write clean, competent, silently wrong code, and the code looks so good you trust it.
40
+
41
+ SkillJab is a **jab** — a small controlled hit, like a vaccine shot or a sparring punch — delivered to a *miniature* of your analysis before the real run:
42
+
43
+ 1. **Plant a truth.** Generate a tiny dataset where you know the answer.
44
+ 2. **Prove recovery.** If the pipeline can't find a truth you planted, it's wrong before any real data.
45
+ 3. **Throw stones.** Drop generic perturbations into the miniature — the Ghost (missing-not-at-random), the Clique (correlated block), the Time Traveler (leakage), the Twin (duplicates), the Drifter (batch shift)…
46
+ 4. **Find the silent failures.** The result moved *and no check fired*. That's the only thing you see.
47
+ 5. **Wake up the AI.** Each silent failure forces an explanation — which is when the model remembers the thing it knew but wasn't applying ("oh — this is the HLA region").
48
+ 6. **Keep the antibody.** Explanation → checker + mitigation + heads-up → rendered into a per-pipeline `SKILL.md` that Claude loads every time it touches that analysis.
49
+
50
+ The skill is the accumulated antibodies. The report is a crash test.
51
+
52
+ ## What it looks like
53
+
54
+ ```
55
+ $ skilljab round run --skill .claude/skills/gwas
56
+ { "class": "silent", "worst_rel_err": 0.9, "checks_fired": [] }
57
+ ```
58
+
59
+ > **Round 1: The Time Traveler (target_leakage) hit stage `clean` — `beta_x1` moved 90% from the planted truth and nothing warned you.**
60
+
61
+ …then after `/skilljab:improve`:
62
+
63
+ ```
64
+ $ skilljab round run --skill .claude/skills/gwas
65
+ { "class": "caught", "checks_fired": [{"script": "checks/no_leaky_columns.py", "message": "predictors nearly identical to outcome: ['x_score']"}] }
66
+ $ skilljab status --skill .claude/skills/gwas
67
+ { "jabbed": true, "reason": "clean", "skill_version": 1, "n_antibodies": 1 }
68
+ ```
69
+
70
+ The report (`history/report.html`, self-contained, no external requests) shows star ratings per stage, a twin-result slider (drag the dose, watch your own estimates move), breaking-point curves, a stone × stage fragility matrix, predicted lap times at your real N, and a vaccination card of rounds × stones turning from red to green.
71
+
72
+ ## Two halves
73
+
74
+ | | **Engine** — `skilljab` CLI (Python) | **Plugin** — Claude Code |
75
+ |---|---|---|
76
+ | does | simulate, inject, run, grade, sweep, diff plans, build the decision tree, score the lineup, predict time, render `SKILL.md` and the report | interview, plan ×N, elicit, judge, sabotage, analyze blind, explain, write antibodies |
77
+ | never | calls an LLM, decides what a failure *means* | touches data, computes a verdict, grades itself |
78
+
79
+ The engine grades; the AI reasons. Any fact that could come from either comes from the engine — that's what keeps the skill honest.
80
+
81
+ ### Blind roles
82
+
83
+ Five subagents, each with only the files it's allowed to see:
84
+
85
+ - **planner ×N** — different persona + constraint each (statistician, domain expert, cluster admin, Reviewer 2, an outsider from another field…); never sees the other plans.
86
+ - **judge** — runs the *lineup*: "one of these plans failed — which, and why?" Never sees stone results, so its hit rate can be measured; sometimes a known culprit is planted to calibrate it.
87
+ - **saboteur** — picks and doses stones; writes `private/stones.json`.
88
+ - **analyst** — runs the pipeline blind with the current skill loaded; forbidden from opening `private/`.
89
+ - **explainer** — the only role that sees what was planted; does the wake-up, writes the antibody.
90
+
91
+ ### The funnel
92
+
93
+ Search wide and cheap, prove narrow and expensive:
94
+
95
+ 0. **Divergence map** — diff N plans; where they disagree is where uncertainty lives.
96
+ 1. **Déjà vu** — `skilljab graveyard search` shows *setups* of similar past failures; predict the outcome, then reveal.
97
+ 2. **Blurry-friend probe** — describe a symptom badly ("slow at step 3, warned about a parameter"); the model enumerates the neighbourhood; the user *recognizes* what they couldn't recall.
98
+ 3. **Stones** — build, dose, run; only silent failures survive.
99
+
100
+ Elicitation proposes; simulation disposes. Nothing reaches the skill without a stone or a user's "that one."
101
+
102
+ ## Install
103
+
104
+ SkillJab is a command-line tool, so install it with **pipx** (isolated, on your PATH, no fight with your system Python — on Debian/Ubuntu plain `pip install` is blocked by PEP 668):
105
+
106
+ ```bash
107
+ # from PyPI (once published)
108
+ pipx install skilljab
109
+
110
+ # from GitHub, before/without PyPI
111
+ pipx install git+https://github.com/AlsammanAlsamman/skilljab.git
112
+
113
+ # from a local checkout (add -e to develop against it)
114
+ pipx install .
115
+ ```
116
+
117
+ No pipx? `sudo apt install pipx && pipx ensurepath` (Debian/Ubuntu), `brew install pipx` (macOS), or use a venv: `python3 -m venv ~/.skilljab && ~/.skilljab/bin/pip install skilljab`.
118
+
119
+ Then the Claude Code plugin — point Claude at the plugin directory:
120
+
121
+ ```bash
122
+ claude --plugin-dir /path/to/skilljab/plugin # this session only
123
+ ```
124
+
125
+ Requires Python ≥ 3.10. `/usr/bin/time` (GNU time) is used for per-stage peak memory when present; otherwise a coarser fallback.
126
+
127
+ ## Quick start (Claude Code)
128
+
129
+ ```
130
+ /skilljab:build examples/toy_regression/pipeline.yaml
131
+ /skilljab:test
132
+ /skilljab:improve
133
+ /skilljab:report
134
+ ```
135
+
136
+ or the whole vaccine in one go: `/skilljab:jab`. When you half-remember something odd from a past run: `/skilljab:recall`.
137
+
138
+ ## Quick start (CLI only)
139
+
140
+ ```bash
141
+ cd examples/toy_regression
142
+ skilljab init --name toy --pipeline pipeline.yaml --spec spec.yaml --skill-dir .claude/skills/toy
143
+ skilljab baseline --skill .claude/skills/toy --target-n 200000 # proves clean recovery, fits timing
144
+ skilljab round new --skill .claude/skills/toy
145
+ skilljab round stones --skill .claude/skills/toy target_leakage outliers --level 0.6
146
+ skilljab round run --skill .claude/skills/toy # -> "silent"
147
+ skilljab sweep --skill .claude/skills/toy --stone outliers --levels 5
148
+ skilljab report --skill .claude/skills/toy # history/report.html
149
+ ```
150
+
151
+ ## Your pipeline's contract
152
+
153
+ A `pipeline.yaml` of stages as shell commands — any language, any tool:
154
+
155
+ ```yaml
156
+ name: gwas
157
+ stages:
158
+ - id: qc cmd: "plink2 --bfile {in} --geno 0.02 --make-bed --out {out}" out: qc.csv
159
+ - id: prune cmd: "Rscript prune.R {in} {out}" out: pruned.csv
160
+ - id: fit cmd: "python fit.py {in} {out}" out: result.json
161
+ ```
162
+
163
+ The last stage writes `result.json = {"estimates": {"beta_x1": 0.79, ...}}` with the same estimand names as the planted truth in `sim/spec.yaml`. Checkers are scripts called at stage boundaries — `python3 check.py <stage_output>` — that exit `1` (or print `{"fired": true, "message": "..."}`) when they fire.
164
+
165
+ Three generators ship (`tabular_regression`, `tabular_classification`, `two_group_lift`); adding one is a function that returns `(DataFrame, {estimand: truth})`.
166
+
167
+ ## The stones
168
+
169
+ | character | stone | what it does |
170
+ |---|---|---|
171
+ | The Clique | `correlated_block` | near-identical copies of a feature |
172
+ | The Ghost | `mnar_missing` | blanks the top values (missing-not-at-random) |
173
+ | The Twin | `duplicates` | re-appends rows |
174
+ | The Spike | `outliers` | a few impossible values |
175
+ | The Drifter | `batch_shift` | a hidden batch confounded with the outcome |
176
+ | The Blur | `measurement_error` | noise in a predictor (attenuation) |
177
+ | The Time Traveler | `target_leakage` | a feature computed from the outcome |
178
+ | The Metric Martian | `unit_mix` | two units in one column |
179
+ | The Long Tail | `heavy_tails` | Student-t noise |
180
+ | The Unicorn | `rare_category` | a level with almost no support |
181
+ | The Typo | `type_corruption` | `'1,234'`, `' 12 '`, `'NA'` in a numeric column |
182
+
183
+ They're deliberately domain-neutral. A generic stone is enough to make the model's own knowledge do the specialization: "a correlated block broke PCA" becomes "exclude chr6:25–35Mb before PCA" in the explainer's hands.
184
+
185
+ ## Layout
186
+
187
+ ```
188
+ skilljab/ engine (pip package): simulate · stones/ · inject · runner · check · project ·
189
+ plandiff · tree · lineup · timing · render_skill · graveyard · report · pack/
190
+ plugin/ Claude Code plugin: commands/ · agents/ · skills/skilljab-core · hooks/
191
+ examples/ toy_regression — a naive OLS pipeline that SkillJab breaks in four ways
192
+ tests/ 47 tests: every stone, the full loop, helpers, CLI, plugin, hook
193
+ docs/DESIGN.md the design and the thinking behind it
194
+ docs/sessions/ transcripts of the design discussions
195
+ ```
196
+
197
+ Per pipeline, in *your* repo:
198
+
199
+ ```
200
+ .claude/skills/<name>/
201
+ ├── SKILL.md rendered — never hand-edit
202
+ ├── antibodies.json source of truth, with provenance and evidence class
203
+ ├── tree.json decisions with evidence on the edges
204
+ ├── checks/ the antibodies' checkers
205
+ ├── sim/spec.yaml how to build this pipeline's miniature
206
+ └── history/ baseline · sizes · round-NNN · sweeps · timing · lineup log · graveyard · report
207
+ ```
208
+
209
+ ## Honest limits
210
+
211
+ - **Miniatures can lie.** Some failures only appear at scale (memory cliffs, rare variants, convergence). The timing page extrapolates from three sizes and says so; treat ranges as optimistic.
212
+ - **Simulation only helps where you can plant a truth.** Quantitative analyses, yes. Design and strategy, no — there the simulation would just be the model's own assumptions fed back to itself.
213
+ - **Stones must bite.** A stone dosed too gently on a tiny miniature proves nothing; use `skilljab sweep` to find the knee.
214
+ - **The graveyard is local** in this version. A shared, anonymized one is the obvious next step and the one thing that would make déjà vu real rather than manufactured.
215
+
216
+ ## Development
217
+
218
+ ```bash
219
+ python3 -m venv .venv && . .venv/bin/activate
220
+ pip install -e ".[dev]"
221
+ python -m pytest -q # 47 tests, ~35 s
222
+ claude plugin validate plugin --strict
223
+ python scripts/export_session.py # save the current Claude Code session into docs/sessions/
224
+
225
+ pipx run build && pipx run twine check dist/* # build + verify the wheel and sdist
226
+ pipx run twine upload dist/* # publish to PyPI (needs an API token)
227
+ ```
228
+
229
+ ## Author
230
+
231
+ **Alsamman M. Alsamman** — aalsamman100@gmail.com · [github.com/AlsammanAlsamman](https://github.com/AlsammanAlsamman)
232
+
233
+ ## Why the name
234
+
235
+ A *jab* is a vaccine shot and a boxing punch. Both are a small, deliberate hit that makes you stronger before the real fight. Read the design notes in [`docs/DESIGN.md`](docs/DESIGN.md) for the whole argument — the pre-mortem, the recoverability test, why generic stones beat a hand-written pitfall library, the lineup, and the tricks for making an AI say "oh yeah, I should have told you."
@@ -0,0 +1,211 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/AlsammanAlsamman/skilljab/main/assets/icon/skilljab-mascot-v1.jpeg" alt="SkillJab mascot — a winking syringe with a boxing glove, jabbing a bar chart" width="260">
3
+ </p>
4
+
5
+ <h1 align="center">SkillJab</h1>
6
+
7
+ <p align="center"><b>Immunize an analysis by sabotaging it before it sees real data — and keep the antibodies as a skill.</b></p>
8
+
9
+ <p align="center">
10
+ <code>pipx install skilljab</code> · Claude Code plugin · Python 3.10+ · MIT
11
+ </p>
12
+
13
+ ---
14
+
15
+ Every analyst has the same scar: a pipeline that ran for six hours and was wrong the whole time, because of something that was public knowledge — the dense-LD region, the leaked column, the duplicated join key, the spreadsheet that wrote `1,234`. AI assistants make this *worse*: they write clean, competent, silently wrong code, and the code looks so good you trust it.
16
+
17
+ SkillJab is a **jab** — a small controlled hit, like a vaccine shot or a sparring punch — delivered to a *miniature* of your analysis before the real run:
18
+
19
+ 1. **Plant a truth.** Generate a tiny dataset where you know the answer.
20
+ 2. **Prove recovery.** If the pipeline can't find a truth you planted, it's wrong before any real data.
21
+ 3. **Throw stones.** Drop generic perturbations into the miniature — the Ghost (missing-not-at-random), the Clique (correlated block), the Time Traveler (leakage), the Twin (duplicates), the Drifter (batch shift)…
22
+ 4. **Find the silent failures.** The result moved *and no check fired*. That's the only thing you see.
23
+ 5. **Wake up the AI.** Each silent failure forces an explanation — which is when the model remembers the thing it knew but wasn't applying ("oh — this is the HLA region").
24
+ 6. **Keep the antibody.** Explanation → checker + mitigation + heads-up → rendered into a per-pipeline `SKILL.md` that Claude loads every time it touches that analysis.
25
+
26
+ The skill is the accumulated antibodies. The report is a crash test.
27
+
28
+ ## What it looks like
29
+
30
+ ```
31
+ $ skilljab round run --skill .claude/skills/gwas
32
+ { "class": "silent", "worst_rel_err": 0.9, "checks_fired": [] }
33
+ ```
34
+
35
+ > **Round 1: The Time Traveler (target_leakage) hit stage `clean` — `beta_x1` moved 90% from the planted truth and nothing warned you.**
36
+
37
+ …then after `/skilljab:improve`:
38
+
39
+ ```
40
+ $ skilljab round run --skill .claude/skills/gwas
41
+ { "class": "caught", "checks_fired": [{"script": "checks/no_leaky_columns.py", "message": "predictors nearly identical to outcome: ['x_score']"}] }
42
+ $ skilljab status --skill .claude/skills/gwas
43
+ { "jabbed": true, "reason": "clean", "skill_version": 1, "n_antibodies": 1 }
44
+ ```
45
+
46
+ The report (`history/report.html`, self-contained, no external requests) shows star ratings per stage, a twin-result slider (drag the dose, watch your own estimates move), breaking-point curves, a stone × stage fragility matrix, predicted lap times at your real N, and a vaccination card of rounds × stones turning from red to green.
47
+
48
+ ## Two halves
49
+
50
+ | | **Engine** — `skilljab` CLI (Python) | **Plugin** — Claude Code |
51
+ |---|---|---|
52
+ | does | simulate, inject, run, grade, sweep, diff plans, build the decision tree, score the lineup, predict time, render `SKILL.md` and the report | interview, plan ×N, elicit, judge, sabotage, analyze blind, explain, write antibodies |
53
+ | never | calls an LLM, decides what a failure *means* | touches data, computes a verdict, grades itself |
54
+
55
+ The engine grades; the AI reasons. Any fact that could come from either comes from the engine — that's what keeps the skill honest.
56
+
57
+ ### Blind roles
58
+
59
+ Five subagents, each with only the files it's allowed to see:
60
+
61
+ - **planner ×N** — different persona + constraint each (statistician, domain expert, cluster admin, Reviewer 2, an outsider from another field…); never sees the other plans.
62
+ - **judge** — runs the *lineup*: "one of these plans failed — which, and why?" Never sees stone results, so its hit rate can be measured; sometimes a known culprit is planted to calibrate it.
63
+ - **saboteur** — picks and doses stones; writes `private/stones.json`.
64
+ - **analyst** — runs the pipeline blind with the current skill loaded; forbidden from opening `private/`.
65
+ - **explainer** — the only role that sees what was planted; does the wake-up, writes the antibody.
66
+
67
+ ### The funnel
68
+
69
+ Search wide and cheap, prove narrow and expensive:
70
+
71
+ 0. **Divergence map** — diff N plans; where they disagree is where uncertainty lives.
72
+ 1. **Déjà vu** — `skilljab graveyard search` shows *setups* of similar past failures; predict the outcome, then reveal.
73
+ 2. **Blurry-friend probe** — describe a symptom badly ("slow at step 3, warned about a parameter"); the model enumerates the neighbourhood; the user *recognizes* what they couldn't recall.
74
+ 3. **Stones** — build, dose, run; only silent failures survive.
75
+
76
+ Elicitation proposes; simulation disposes. Nothing reaches the skill without a stone or a user's "that one."
77
+
78
+ ## Install
79
+
80
+ SkillJab is a command-line tool, so install it with **pipx** (isolated, on your PATH, no fight with your system Python — on Debian/Ubuntu plain `pip install` is blocked by PEP 668):
81
+
82
+ ```bash
83
+ # from PyPI (once published)
84
+ pipx install skilljab
85
+
86
+ # from GitHub, before/without PyPI
87
+ pipx install git+https://github.com/AlsammanAlsamman/skilljab.git
88
+
89
+ # from a local checkout (add -e to develop against it)
90
+ pipx install .
91
+ ```
92
+
93
+ No pipx? `sudo apt install pipx && pipx ensurepath` (Debian/Ubuntu), `brew install pipx` (macOS), or use a venv: `python3 -m venv ~/.skilljab && ~/.skilljab/bin/pip install skilljab`.
94
+
95
+ Then the Claude Code plugin — point Claude at the plugin directory:
96
+
97
+ ```bash
98
+ claude --plugin-dir /path/to/skilljab/plugin # this session only
99
+ ```
100
+
101
+ Requires Python ≥ 3.10. `/usr/bin/time` (GNU time) is used for per-stage peak memory when present; otherwise a coarser fallback.
102
+
103
+ ## Quick start (Claude Code)
104
+
105
+ ```
106
+ /skilljab:build examples/toy_regression/pipeline.yaml
107
+ /skilljab:test
108
+ /skilljab:improve
109
+ /skilljab:report
110
+ ```
111
+
112
+ or the whole vaccine in one go: `/skilljab:jab`. When you half-remember something odd from a past run: `/skilljab:recall`.
113
+
114
+ ## Quick start (CLI only)
115
+
116
+ ```bash
117
+ cd examples/toy_regression
118
+ skilljab init --name toy --pipeline pipeline.yaml --spec spec.yaml --skill-dir .claude/skills/toy
119
+ skilljab baseline --skill .claude/skills/toy --target-n 200000 # proves clean recovery, fits timing
120
+ skilljab round new --skill .claude/skills/toy
121
+ skilljab round stones --skill .claude/skills/toy target_leakage outliers --level 0.6
122
+ skilljab round run --skill .claude/skills/toy # -> "silent"
123
+ skilljab sweep --skill .claude/skills/toy --stone outliers --levels 5
124
+ skilljab report --skill .claude/skills/toy # history/report.html
125
+ ```
126
+
127
+ ## Your pipeline's contract
128
+
129
+ A `pipeline.yaml` of stages as shell commands — any language, any tool:
130
+
131
+ ```yaml
132
+ name: gwas
133
+ stages:
134
+ - id: qc cmd: "plink2 --bfile {in} --geno 0.02 --make-bed --out {out}" out: qc.csv
135
+ - id: prune cmd: "Rscript prune.R {in} {out}" out: pruned.csv
136
+ - id: fit cmd: "python fit.py {in} {out}" out: result.json
137
+ ```
138
+
139
+ The last stage writes `result.json = {"estimates": {"beta_x1": 0.79, ...}}` with the same estimand names as the planted truth in `sim/spec.yaml`. Checkers are scripts called at stage boundaries — `python3 check.py <stage_output>` — that exit `1` (or print `{"fired": true, "message": "..."}`) when they fire.
140
+
141
+ Three generators ship (`tabular_regression`, `tabular_classification`, `two_group_lift`); adding one is a function that returns `(DataFrame, {estimand: truth})`.
142
+
143
+ ## The stones
144
+
145
+ | character | stone | what it does |
146
+ |---|---|---|
147
+ | The Clique | `correlated_block` | near-identical copies of a feature |
148
+ | The Ghost | `mnar_missing` | blanks the top values (missing-not-at-random) |
149
+ | The Twin | `duplicates` | re-appends rows |
150
+ | The Spike | `outliers` | a few impossible values |
151
+ | The Drifter | `batch_shift` | a hidden batch confounded with the outcome |
152
+ | The Blur | `measurement_error` | noise in a predictor (attenuation) |
153
+ | The Time Traveler | `target_leakage` | a feature computed from the outcome |
154
+ | The Metric Martian | `unit_mix` | two units in one column |
155
+ | The Long Tail | `heavy_tails` | Student-t noise |
156
+ | The Unicorn | `rare_category` | a level with almost no support |
157
+ | The Typo | `type_corruption` | `'1,234'`, `' 12 '`, `'NA'` in a numeric column |
158
+
159
+ They're deliberately domain-neutral. A generic stone is enough to make the model's own knowledge do the specialization: "a correlated block broke PCA" becomes "exclude chr6:25–35Mb before PCA" in the explainer's hands.
160
+
161
+ ## Layout
162
+
163
+ ```
164
+ skilljab/ engine (pip package): simulate · stones/ · inject · runner · check · project ·
165
+ plandiff · tree · lineup · timing · render_skill · graveyard · report · pack/
166
+ plugin/ Claude Code plugin: commands/ · agents/ · skills/skilljab-core · hooks/
167
+ examples/ toy_regression — a naive OLS pipeline that SkillJab breaks in four ways
168
+ tests/ 47 tests: every stone, the full loop, helpers, CLI, plugin, hook
169
+ docs/DESIGN.md the design and the thinking behind it
170
+ docs/sessions/ transcripts of the design discussions
171
+ ```
172
+
173
+ Per pipeline, in *your* repo:
174
+
175
+ ```
176
+ .claude/skills/<name>/
177
+ ├── SKILL.md rendered — never hand-edit
178
+ ├── antibodies.json source of truth, with provenance and evidence class
179
+ ├── tree.json decisions with evidence on the edges
180
+ ├── checks/ the antibodies' checkers
181
+ ├── sim/spec.yaml how to build this pipeline's miniature
182
+ └── history/ baseline · sizes · round-NNN · sweeps · timing · lineup log · graveyard · report
183
+ ```
184
+
185
+ ## Honest limits
186
+
187
+ - **Miniatures can lie.** Some failures only appear at scale (memory cliffs, rare variants, convergence). The timing page extrapolates from three sizes and says so; treat ranges as optimistic.
188
+ - **Simulation only helps where you can plant a truth.** Quantitative analyses, yes. Design and strategy, no — there the simulation would just be the model's own assumptions fed back to itself.
189
+ - **Stones must bite.** A stone dosed too gently on a tiny miniature proves nothing; use `skilljab sweep` to find the knee.
190
+ - **The graveyard is local** in this version. A shared, anonymized one is the obvious next step and the one thing that would make déjà vu real rather than manufactured.
191
+
192
+ ## Development
193
+
194
+ ```bash
195
+ python3 -m venv .venv && . .venv/bin/activate
196
+ pip install -e ".[dev]"
197
+ python -m pytest -q # 47 tests, ~35 s
198
+ claude plugin validate plugin --strict
199
+ python scripts/export_session.py # save the current Claude Code session into docs/sessions/
200
+
201
+ pipx run build && pipx run twine check dist/* # build + verify the wheel and sdist
202
+ pipx run twine upload dist/* # publish to PyPI (needs an API token)
203
+ ```
204
+
205
+ ## Author
206
+
207
+ **Alsamman M. Alsamman** — aalsamman100@gmail.com · [github.com/AlsammanAlsamman](https://github.com/AlsammanAlsamman)
208
+
209
+ ## Why the name
210
+
211
+ A *jab* is a vaccine shot and a boxing punch. Both are a small, deliberate hit that makes you stronger before the real fight. Read the design notes in [`docs/DESIGN.md`](docs/DESIGN.md) for the whole argument — the pre-mortem, the recoverability test, why generic stones beat a hand-written pitfall library, the lineup, and the tricks for making an AI say "oh yeah, I should have told you."
@@ -0,0 +1,40 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "skilljab"
7
+ version = "0.1.0"
8
+ description = "Immunize an analysis by sabotaging it before it sees real data - and keep the antibodies as a skill."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = {text = "MIT"}
12
+ authors = [{name = "Alsamman M. Alsamman", email = "aalsamman100@gmail.com"}]
13
+ keywords = ["data-analysis", "simulation", "chaos-engineering", "statistics", "pipelines", "claude-code", "skills"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Science/Research",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python :: 3",
19
+ "Topic :: Scientific/Engineering",
20
+ ]
21
+ dependencies = ["numpy>=1.24", "pandas>=2.0", "pyyaml>=6.0"]
22
+
23
+ [project.urls]
24
+ Homepage = "https://github.com/AlsammanAlsamman/skilljab"
25
+ Repository = "https://github.com/AlsammanAlsamman/skilljab"
26
+
27
+ [project.optional-dependencies]
28
+ dev = ["pytest>=7"]
29
+
30
+ [project.scripts]
31
+ skilljab = "skilljab.cli:main"
32
+
33
+ [tool.setuptools.packages.find]
34
+ include = ["skilljab*"]
35
+
36
+ [tool.setuptools.package-data]
37
+ skilljab = ["stones/catalog.yaml", "pack/*.yaml", "pack/diversity/*.yaml", "pack/graveyard/*.json", "report_assets/*"]
38
+
39
+ [tool.pytest.ini_options]
40
+ testpaths = ["tests"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,7 @@
1
+ """SkillJab engine - deterministic tooling that Claude Code drives.
2
+
3
+ The engine simulates, injects stones, runs pipelines, grades results, diffs plans,
4
+ merges decision trees, scores the lineup, predicts time, renders SKILL.md and the
5
+ report. It never calls an LLM and never decides what a failure *means*.
6
+ """
7
+ __version__ = "0.1.0"
@@ -0,0 +1,53 @@
1
+ """Grade a run against the planted truth and the checkers -> verdict.json
2
+
3
+ Classes:
4
+ crashed pipeline did not produce a result
5
+ silent result degraded and NO checker fired <- the only thing the user must see
6
+ caught result degraded and a checker fired
7
+ false_alarm result fine but a checker fired
8
+ harmless result fine, nothing fired
9
+ """
10
+ from __future__ import annotations
11
+ from .util import read_json, write_json
12
+
13
+ def grade(truth, run, baseline_run=None):
14
+ tol = truth.get("tolerance", {}); rel_tol = float(tol.get("relative", 0.25)); abs_tol = float(tol.get("absolute", 0.1))
15
+ est = (run.get("result") or {}).get("estimates", {}) if run.get("result") else {}
16
+ per = {}
17
+ degraded = False
18
+ for name, tv in truth.get("estimands", {}).items():
19
+ if run.get("crashed") or name not in est or est[name] is None:
20
+ per[name] = {"truth": tv, "estimate": None, "abs_err": None, "rel_err": None, "degraded": True}
21
+ degraded = True; continue
22
+ ev = float(est[name]); ae = abs(ev - float(tv)); re = ae / (abs(float(tv)) if abs(float(tv)) > 1e-12 else 1.0)
23
+ d = (ae > abs_tol) and (re > rel_tol)
24
+ per[name] = {"truth": tv, "estimate": ev, "abs_err": round(ae, 6), "rel_err": round(re, 6), "degraded": d}
25
+ degraded = degraded or d
26
+ fired = [{"stage": s["id"], **c} for s in run.get("stages", []) for c in s.get("checks", []) if c.get("fired")]
27
+ if run.get("crashed"):
28
+ cls = "crashed"
29
+ elif degraded and not fired:
30
+ cls = "silent"
31
+ elif degraded and fired:
32
+ cls = "caught"
33
+ elif fired:
34
+ cls = "false_alarm"
35
+ else:
36
+ cls = "harmless"
37
+ # per-stage time impact vs baseline
38
+ time_impact = {}
39
+ if baseline_run:
40
+ base = {s["id"]: s["seconds"] for s in baseline_run.get("stages", [])}
41
+ for s in run.get("stages", []):
42
+ b = base.get(s["id"])
43
+ if b and b > 0:
44
+ time_impact[s["id"]] = round(s["seconds"] / b, 3)
45
+ worst = max([p["rel_err"] for p in per.values() if p["rel_err"] is not None] or [None if run.get("crashed") else 0.0], key=lambda x: (x is not None, x))
46
+ return {"class": cls, "degraded": degraded, "crashed": bool(run.get("crashed")), "worst_rel_err": worst,
47
+ "estimands": per, "checks_fired": fired, "time_impact": time_impact,
48
+ "stones": run.get("stones_applied", []), "n_rows": run.get("n_rows"), "label": run.get("label")}
49
+
50
+ def check_cli(truth_path, run_path, out_path, baseline_path=None):
51
+ v = grade(read_json(truth_path), read_json(run_path), read_json(baseline_path) if baseline_path else None)
52
+ write_json(out_path, v)
53
+ return v