task-pipeline-skill 1.85.0 → 1.86.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +114 -0
  2. package/CONTRIBUTING.md +2 -2
  3. package/README.md +2 -1
  4. package/SKILL-CARD.md +1 -1
  5. package/bin/task-pipeline.js +70 -9
  6. package/evals/cases/evidence-docs.json +188 -0
  7. package/evals/cases/project-audit.json +188 -0
  8. package/evals/cases/task-pipeline.json +191 -0
  9. package/package.json +5 -4
  10. package/plugins/task-pipeline/.claude-plugin/plugin.json +1 -1
  11. package/plugins/task-pipeline/skills/evidence-docs/SKILL.md +18 -11
  12. package/plugins/task-pipeline/skills/evidence-docs/references/GENERATED.md +8 -0
  13. package/plugins/task-pipeline/skills/evidence-docs/references/documentation.md +472 -0
  14. package/plugins/task-pipeline/skills/evidence-docs/references/gates.md +645 -0
  15. package/plugins/task-pipeline/skills/evidence-docs/references/hooks.md +274 -0
  16. package/plugins/task-pipeline/skills/evidence-docs/references/learned.md +292 -0
  17. package/plugins/task-pipeline/skills/evidence-docs/references/retrospective.md +551 -0
  18. package/plugins/task-pipeline/skills/evidence-docs/references/setup.md +149 -0
  19. package/plugins/task-pipeline/skills/evidence-docs/templates/decisions.md +50 -0
  20. package/plugins/task-pipeline/skills/evidence-docs/templates/docgate.sh +537 -0
  21. package/plugins/task-pipeline/skills/project-audit/SKILL.md +69 -27
  22. package/plugins/task-pipeline/skills/project-audit/scripts/audit.py +11 -0
  23. package/plugins/task-pipeline/skills/task-pipeline/SKILL.md +72 -55
  24. package/plugins/task-pipeline/skills/task-pipeline/execution-attempt.schema.json +68 -0
  25. package/plugins/task-pipeline/skills/task-pipeline/execution-packet.example.json +42 -0
  26. package/plugins/task-pipeline/skills/task-pipeline/execution-packet.schema.json +217 -0
  27. package/plugins/task-pipeline/skills/task-pipeline/execution-result.example.json +49 -0
  28. package/plugins/task-pipeline/skills/task-pipeline/execution-result.schema.json +261 -0
  29. package/plugins/task-pipeline/skills/task-pipeline/graph.example.json +10 -1
  30. package/plugins/task-pipeline/skills/task-pipeline/graph.schema.json +172 -2
  31. package/plugins/task-pipeline/skills/task-pipeline/pipeline.schema.json +50 -1
  32. package/plugins/task-pipeline/skills/task-pipeline/references/acceptance.md +7 -0
  33. package/plugins/task-pipeline/skills/task-pipeline/references/artifacts.md +23 -0
  34. package/plugins/task-pipeline/skills/task-pipeline/references/audit.md +6 -0
  35. package/plugins/task-pipeline/skills/task-pipeline/references/backlog.md +8 -1
  36. package/plugins/task-pipeline/skills/task-pipeline/references/browser.md +8 -0
  37. package/plugins/task-pipeline/skills/task-pipeline/references/build.md +32 -0
  38. package/plugins/task-pipeline/skills/task-pipeline/references/certification.md +3 -2
  39. package/plugins/task-pipeline/skills/task-pipeline/references/companion-skills.md +14 -3
  40. package/plugins/task-pipeline/skills/task-pipeline/references/decomposition.md +83 -2
  41. package/plugins/task-pipeline/skills/task-pipeline/references/doctrine-map.md +53 -0
  42. package/plugins/task-pipeline/skills/task-pipeline/references/documentation.md +3 -0
  43. package/plugins/task-pipeline/skills/task-pipeline/references/grill.md +27 -8
  44. package/plugins/task-pipeline/skills/task-pipeline/references/model-tiering.md +19 -0
  45. package/plugins/task-pipeline/skills/task-pipeline/references/planning.md +203 -26
  46. package/plugins/task-pipeline/skills/task-pipeline/references/portability.md +1 -0
  47. package/plugins/task-pipeline/skills/task-pipeline/references/retrospective.md +26 -8
  48. package/plugins/task-pipeline/skills/task-pipeline/references/work-graph.md +7 -1
  49. package/plugins/task-pipeline/skills/task-pipeline/scripts/context_packets.py +686 -0
  50. package/plugins/task-pipeline/skills/task-pipeline/scripts/execution_authority.py +271 -0
  51. package/plugins/task-pipeline/skills/task-pipeline/scripts/graph.py +415 -18
  52. package/plugins/task-pipeline/skills/task-pipeline/scripts/packet.py +400 -0
  53. package/plugins/task-pipeline/skills/task-pipeline/templates/README.md +2 -0
  54. package/plugins/task-pipeline/skills/task-pipeline/templates/browser-claims.json +54 -0
  55. package/plugins/task-pipeline/skills/task-pipeline/templates/finding-evidence.json +42 -0
  56. package/plugins/task-pipeline/skills/task-pipeline/templates/run.md +2 -2
package/CHANGELOG.md CHANGED
@@ -1,3 +1,117 @@
1
+ ## v1.86.1 — the release declares the stamp it does not carry
2
+
3
+ `v1.86.0` was tagged and refused: the run-stamp gate reads the tag's own tree, a
4
+ branch push cannot see a tag that does not exist yet, and the declaration was not
5
+ in the tree when the tag was cut. Nothing published under it — the tag exists,
6
+ the release does not, npm never served it.
7
+
8
+ This is the seventh instance of a mechanism this repository's `retro.md` names
9
+ six times, and `R-010` already held the rule: run the suite the RELEASE runs,
10
+ against the tree you are about to tag. The lesson was written and read past,
11
+ which is what its own retirement condition — "it becomes a mechanical pre-push
12
+ hook" — exists to end.
13
+
14
+ The v1.86.0 payload is unchanged. This release adds the declaration.
15
+
16
+ ## v1.86.0 — the audit closes, and three guards stop passing on a wording
17
+
18
+ Sherlock external-v3 (37 findings) plus the context-ready handoff (PR #85),
19
+ each finding carrying its own executable regression.
20
+
21
+ Guards: 429 → **429**, property checks 15 → **15** — no guard was added or
22
+ removed. Three plants were REPOINTED to follow their subjects to the wording
23
+ those subjects had moved to, and the conditionals check became a script when
24
+ the workflow hit GitHub's size limit. A repointed plant is the same guard
25
+ proving the same thing about a sentence that moved.
26
+
27
+ - **A guard keyed to a wording had silently stopped running.** The retrospective
28
+ gained rule CLASSES — a cold rule is marked review-needed rather than deleted,
29
+ and the trigger reads *five exposure opportunities* instead of *five run
30
+ stamps*. The check requiring the condition to name BOTH units was keyed to the
31
+ old words, so its corpus predicate matched nothing, every surface dropped out
32
+ of scope, and it passed everything. Found only because a NEGATIVE SELF-TEST
33
+ refused to plant. The discovery predicate and the assertion are now built from
34
+ one pattern, and the failure message no longer names a unit — naming one is
35
+ what tied it to a wording.
36
+ - **A property check pinned a count.** went red when the graph
37
+ schema gained a fourth conditional, all four correctly behind a . It
38
+ compares the refs against the entries now, so inlining one still fails at any
39
+ number of rules.
40
+ - Explaining that fix inline pushed past
41
+ GitHub's 512,000-byte limit — above which the workflow stays `active`, creates
42
+ no runs, and `gh pr checks` says "no checks reported". The size guard names its
43
+ own remedy, and taking it moved the check into `test/property_graph_conditionals.py`.
44
+ - The doctrine map moved to `references/doctrine-map.md` when the body breached
45
+ the 5000-token budget; the reachability walker is transitive, so every
46
+ reference it names is still reachable. `evidence-docs/references/GENERATED.md`
47
+ — the note saying those copies are generated — was reachable from nothing, so
48
+ the agent most likely to edit a copy never read it.
49
+
50
+ ## v1.85.2 — the two-merge protocol, followed this time
51
+
52
+ **v1.85.1 is a burned tag** — the third of this wave's choreography lessons, and
53
+ this member's own: the release gate on the tag's tree demands a run stamp inside
54
+ the tag's range, the branch run cannot see a tag that does not exist yet, and the
55
+ stamp was never written. Nothing shipped; npm never saw 1.85.1. This version is
56
+ the same change released by the protocol B-134 wrote down: the payload merges
57
+ first, the stamp merges second citing the payload's merge commit, and the tag
58
+ lands on the stamp.
59
+
60
+ Guards: 429 → **429** — nothing but version surfaces, this record, and the
61
+ stamp's own choreography move in this release.
62
+
63
+ ## v1.85.1 — five surfaces a reader trusts, each disagreeing with the tree it ships from
64
+
65
+ The family audit of 2026-09-06 (wave `AUDIT-WAVE-0906`) read every member's shipped
66
+ surfaces at once, and five of this member's disagreed with what the tree holds. No
67
+ behaviour changed in this release; what changed is what a reader — a person on the
68
+ marketplace page, or a resolver following a path — is told.
69
+
70
+ - **The marketplace entry taught the old gate vocabulary.** `plugin.json` says *typed
71
+ auto/judgment/manual gates*; `.claude-plugin/marketplace.json` said *typed
72
+ auto/manual*. The `judgment` type has been in the schema since v1.73.0 and the
73
+ v1.80.0 sweep put it on seven surfaces — the marketplace blurb was not one of them,
74
+ because the cross-surface guard reads the schema and `SKILL.md`, never the
75
+ marketplace entry. The two descriptions are byte-equal now, compared rather than
76
+ eyeballed: `json.load` both, `==` → `True` (the sole difference was 9 bytes at
77
+ character 298, the string `judgment/`).
78
+ - **Two in-directory references in `project-audit`'s `SKILL.md` resolved only in the
79
+ sibling skill.** Its §6 named `references/backlog.md` and
80
+ `references/prioritisation.md` bare — directories the flagship skill carries and
81
+ `project-audit` does not — while the routing table at the top of the same file
82
+ already used the `../task-pipeline/references/…` form. Both now use that form, as
83
+ links that resolve from the file that carries them.
84
+ - **`certification.md` pointed at a file no install of this plugin can resolve.**
85
+ `references/statistics.md` lives in `agent-stack`'s `agent-evals` skill, another
86
+ member entirely; the sentence carried it as a bare path beside the owner's name. It
87
+ is a named-owner prose pointer now, with no bare in-repo path left to resolve.
88
+ - **Front matter inside the pack moves together.** `evidence-docs` carried neither
89
+ `license:` nor `compatibility:`; `project-audit` carried no `license:`. Both now
90
+ carry the flagship's keys — `license: MIT`, and a compatibility line true of each
91
+ skill rather than copied: `evidence-docs` ships no scripts, and its line says so.
92
+ - **The `quick` opt-out is retired; the canon refusal replaces it.** *"quick fix"*
93
+ carries the trigger `fix` and the opt-out `quick` in one natural breath, and the
94
+ board already records the escape being used on work that should not have taken it
95
+ (`B-095`). The description now names *'no pipeline' / 'без пайплайна'* — the refusal
96
+ wording the operator's routing doctrine states — and the opt-out guard in
97
+ `test/validate.py` demands exactly that pair, so a description that drops either
98
+ valve fails. No recorded decision protects the old token: the only decision-bearing
99
+ surfaces that name it are `B-095`, which argues against it, and dated records, which
100
+ are not rewritten. No advertised trigger phrase was removed. Measured with
101
+ make-skill's auditor before and after: description 897 → 903 of 1024, inside the
102
+ 970-character working limit, all sixteen description checks green both times.
103
+
104
+ The dated spec `docs/evidence/specs/2026-08-10-routing-taxonomy.md` still quotes the
105
+ old opt-out pair, deliberately: it records what was true at its commit, and this
106
+ repository has already decided that dated records are counted, not rewritten.
107
+
108
+ Guards: 429 → **429** — no planted-defect step added or removed; one existing guard's
109
+ needle moved with the surface it reads (`quick` → `no pipeline`). And the gate earned
110
+ its keep on this very release: its first run refused three restated figures this entry's
111
+ own change had gone stale — the board's description budget (897 → 903), its headroom
112
+ (127 → 121 spare), and the README's conformance figure (899 → 905) — each rewritten to
113
+ the value the owning check computes, so the next drift fails the same way this one did.
114
+
1
115
  ## v1.85.0 — a release stamp names the tag, and the amend rule's sibling one level up
2
116
 
3
117
  `B-124`, filed by the umbrella's coordinator after watching the same class three times:
package/CONTRIBUTING.md CHANGED
@@ -315,7 +315,7 @@ was neither strict nor lenient but **unreadable**. A list capped at ten whose re
315
315
  condition cannot be read fills up and stops being pruned. The calendar is the unit nothing can
316
316
  stall, which is why it is not belt-and-braces. Entry **rotation** ("entries older than five
317
317
  stamps move to the archive") is a different mechanism and is deliberately out of scope.
318
- *(guard: `states the cold-retirement condition as five run stamp`)*
318
+ *(guard: `states the cold-retirement condition in one unit only`)*
319
319
 
320
320
  **38. Every worked GATE verdict prints both disclosures.** `abstained` — what the run
321
321
  declined to claim — and `unlooked` — what a check never looked at. Without them a `PASS`
@@ -372,7 +372,7 @@ rule and the command. Every miss was found by a reader or a sweep, never by the
372
372
  **nobody notices a corpus that is too small, because everything inside it passes.**
373
373
  Exclusions are allowed and must each carry a reason in the code: a changelog narrates
374
374
  old formats, `docs/evidence/specs/` are point-in-time records.
375
- *(guard: `a worked GATE verdict omits ` and `states the cold-retirement condition as` — both now run over corpora walked from disk, so a new surface joins by existing)*
375
+ *(guard: `a worked GATE verdict omits ` and `states the cold-retirement condition in one unit only` — both now run over corpora walked from disk, so a new surface joins by existing)*
376
376
 
377
377
  **44. A carry-over row still `open` names a board id, and the board row names it back.**
378
378
  `docs/evidence/backlog.md` is the project's queue between runs; the ledger's `open`
package/README.md CHANGED
@@ -587,7 +587,7 @@ must say so.
587
587
  ### Held to Anthropic's own Skill authoring guidance
588
588
 
589
589
  Audited against the four Agent Skills pages. Most of it already held — `name`
590
- 13/64 chars, `description` 899/1024 chars, `SKILL.md` 262/500 lines, all 38 references
590
+ 13/64 chars, `description` 903/1024 chars, `SKILL.md` 279/500 lines, all 39 references
591
591
  linked **directly** from `SKILL.md`, and the bundle far under the 30 MB ceiling. What
592
592
  did not, now does:
593
593
 
@@ -929,6 +929,7 @@ recommendation, so you arm the whole run in one exchange. Detail:
929
929
  |---|---|
930
930
  | [`SKILL.md`](plugins/task-pipeline/skills/task-pipeline/SKILL.md) | the orchestrator: how to run, the stage table, the model decision |
931
931
  | [`references/stages.md`](plugins/task-pipeline/skills/task-pipeline/references/stages.md) | per-stage detail and the exact gate criteria |
932
+ | [`references/doctrine-map.md`](plugins/task-pipeline/skills/task-pipeline/references/doctrine-map.md) | which reference owns which stage — the map SKILL.md used to carry inline |
932
933
  | [`references/artifacts.md`](plugins/task-pipeline/skills/task-pipeline/references/artifacts.md) | the canonical document layout each stage writes to |
933
934
  | [`references/conventions.md`](plugins/task-pipeline/skills/task-pipeline/references/conventions.md) | how stages 6–10 read the host project's `CLAUDE.md`, and how the documentation regime is detected |
934
935
  | [`references/documentation.md`](plugins/task-pipeline/skills/task-pipeline/references/documentation.md) | the doc system: the inventory, registers and ids, SSOT, the Doc Loop, supersede semantics, the propagation matrix, intent vs as-built |
package/SKILL-CARD.md CHANGED
@@ -12,7 +12,7 @@ harmless.
12
12
  |---|---|
13
13
  | **Purpose** | Runs a substantial task through ten gated delivery stages — intake grill, docs study, brainstorm, spec, plan, subagent build, tests, lint/deploy, post-deploy, docs+registers, acceptance — refusing to advance until each gate passes |
14
14
  | **Owner** | ssheleg ([github.com/ssheleg/task-pipeline](https://github.com/ssheleg/task-pipeline)) |
15
- | **Version** | 1.85.0 |
15
+ | **Version** | 1.86.1 |
16
16
  | **Surface** | Claude Code (filesystem skill + plugin) and the vercel `skills` CLI. **Not** uploaded to the Skills API; custom Skills do not sync across surfaces |
17
17
  | **Dependencies** | None required. Optional: `context7` (MCP), `figma` (MCP), super-ux, agent-sync, graphify, obsidian-wiki, and **one of two browser channels** — `playwright` (CLI or MCP) or `chrome-devtools` (MCP); either satisfies the browser step and neither is required. Every stage's doctrine ships in-repo; the one conditional requirement is super-ux for the stage-3 UX track on a user-facing task |
18
18
  | **Evaluation status** | Suite authored, 5 categories. One recorded run, **self-observed by the author**; **zero blind runs on zero of three models** — the split, and the numbers, live in [`evals/RESULTS.md`](evals/RESULTS.md) and are computed by `evals/run.py` |
@@ -50,16 +50,58 @@ function copyDir(src, dest) {
50
50
  }
51
51
  }
52
52
 
53
+ function verifyTree(src, staged, isDir) {
54
+ if (!isDir) {
55
+ if (!fs.readFileSync(src).equals(fs.readFileSync(staged))) {
56
+ throw new Error(`staged ${path.basename(staged)} does not match its source`);
57
+ }
58
+ return;
59
+ }
60
+ for (const entry of fs.readdirSync(src, { withFileTypes: true })) {
61
+ const s = path.join(src, entry.name);
62
+ const d = path.join(staged, entry.name);
63
+ if (entry.isDirectory()) verifyTree(s, d, true);
64
+ else if (!fs.readFileSync(s).equals(fs.readFileSync(d))) {
65
+ throw new Error(`staged ${entry.name} does not match its source`);
66
+ }
67
+ }
68
+ }
69
+
70
+ /**
71
+ * A TRANSACTIONAL install (FIX-UP-05.02): the writer contract from UP-05
72
+ * applied to this member's installer. The old code deleted `dest` and THEN
73
+ * copied into it, so a crash mid-copy left nothing (with --force) or a partial
74
+ * tree. Now the payload is staged into a same-filesystem sibling and VERIFIED
75
+ * first; only then is the old install moved aside (recoverable) and the staged
76
+ * one renamed into place. A stage crash leaves the ACTIVE install untouched;
77
+ * --force=false still skips, preserving the user's bytes.
78
+ */
53
79
  function installOne(label, src, dest, isDir, force) {
54
80
  if (fs.existsSync(dest) && !force) {
55
81
  console.log(`skip: ${label} already installed at ${dest} (rerun with --force to overwrite)`);
56
82
  return;
57
83
  }
58
- fs.rmSync(dest, { recursive: true, force: true });
59
84
  fs.mkdirSync(path.dirname(dest), { recursive: true });
60
- if (isDir) copyDir(src, dest);
61
- else fs.copyFileSync(src, dest);
62
- console.log(`Installed ${label} -> ${dest}`);
85
+ const staging = `${dest}.staging-${process.pid}`; // same fs as dest
86
+ const prev = `${dest}.prev-${process.pid}`;
87
+ fs.rmSync(staging, { recursive: true, force: true });
88
+ try {
89
+ // 1. STAGE + VERIFY, before touching the active install.
90
+ if (isDir) copyDir(src, staging);
91
+ else { fs.mkdirSync(path.dirname(staging), { recursive: true }); fs.copyFileSync(src, staging); }
92
+ verifyTree(src, staging, isDir);
93
+ // 2. SWITCH: move the old aside (recoverable), rename staged into place.
94
+ fs.rmSync(prev, { recursive: true, force: true });
95
+ if (fs.existsSync(dest)) fs.renameSync(dest, prev);
96
+ fs.renameSync(staging, dest);
97
+ fs.rmSync(prev, { recursive: true, force: true });
98
+ console.log(`Installed ${label} -> ${dest}`);
99
+ } catch (err) {
100
+ // Abort: leave the active install intact, remove the half-built staging.
101
+ fs.rmSync(staging, { recursive: true, force: true });
102
+ if (fs.existsSync(prev) && !fs.existsSync(dest)) fs.renameSync(prev, dest);
103
+ throw new Error(`install aborted, previous install intact: ${err.message}`);
104
+ }
63
105
  }
64
106
 
65
107
  /**
@@ -172,6 +214,20 @@ function migrateArtifacts(args) {
172
214
  return 0;
173
215
  }
174
216
 
217
+ // The bundled HostContext resolver (FIX-UP-08.02) — one contract, a local
218
+ // copy per member because these installers run via `npx` with no shared lib.
219
+ // A host's config root is: an explicit root > the documented host env var >
220
+ // the platform default `~/<dir>`. Used verbatim (spaces preserved), never
221
+ // through a shell. Host EXISTENCE is a separate probe on the returned path.
222
+ const HOST_ENV = { claude: 'CLAUDE_CONFIG_DIR', codex: 'CODEX_HOME', gemini: 'GEMINI_CONFIG_DIR' };
223
+ const HOST_DIR = { claude: '.claude', codex: '.codex', gemini: '.gemini' };
224
+ function hostRoot(agent, home, env, explicit) {
225
+ if (explicit) return explicit;
226
+ const e = (env || process.env)[HOST_ENV[agent]];
227
+ if (e) return e;
228
+ return path.join(home, HOST_DIR[agent]);
229
+ }
230
+
175
231
  function main(argv) {
176
232
  const args = argv.slice(2);
177
233
  if (args.includes('--help') || args.includes('-h')) {
@@ -204,9 +260,10 @@ function main(argv) {
204
260
  // copy SHADOWS the plugin — silently serving whatever version was copied, forever.
205
261
  // The family launcher (sshlg-skills) prunes exactly these copies for that reason,
206
262
  // so creating one without saying so undoes the thing it is paired with.
263
+ const claude = hostRoot('claude', home, process.env);
207
264
  const pluginDirs = [
208
- path.join(home, '.claude', 'plugins', 'marketplaces', 'task-pipeline'),
209
- path.join(home, '.claude', 'plugins', 'cache', 'task-pipeline'),
265
+ path.join(claude, 'plugins', 'marketplaces', 'task-pipeline'),
266
+ path.join(claude, 'plugins', 'cache', 'task-pipeline'),
210
267
  ];
211
268
  if (!force && pluginDirs.some((d) => fs.existsSync(d))) {
212
269
  console.error(`refusing: task-pipeline is already installed as a Claude Code PLUGIN.
@@ -224,14 +281,14 @@ Rerun with --force if you deliberately want the plain copy instead.`);
224
281
  installOne(
225
282
  'task-pipeline skill ',
226
283
  skillSrc,
227
- path.join(home, '.claude', 'skills', 'task-pipeline'),
284
+ path.join(claude, 'skills', 'task-pipeline'),
228
285
  true,
229
286
  force
230
287
  );
231
288
  installOne(
232
289
  '/task-pipeline command',
233
290
  cmdSrc,
234
- path.join(home, '.claude', 'commands', 'task-pipeline.md'),
291
+ path.join(claude, 'commands', 'task-pipeline.md'),
235
292
  false,
236
293
  force
237
294
  );
@@ -240,4 +297,8 @@ Rerun with --force if you deliberately want the plain copy instead.`);
240
297
  return 0;
241
298
  }
242
299
 
243
- process.exit(main(process.argv));
300
+ if (require.main === module) {
301
+ process.exit(main(process.argv));
302
+ }
303
+
304
+ module.exports = { installOne, copyDir, verifyTree, hostRoot };
@@ -0,0 +1,188 @@
1
+ {
2
+ "contract": "outcome-case/1 (schemas/outcome-case.schema.json + test/outcome_harness.py in ssheleg/sshlg-skills)",
3
+ "note": "Outcome corpus for evidence-docs (FIX-EV-01.09). Judged on ARTIFACTS via the family harness: the runner records the actual output oracle verdict AND the raw result per case; with/without-skill arms are the harness's baseline contract. Nothing in the production skill was changed for grader convenience.",
4
+ "arms": {
5
+ "baseline": "same prompts, skill absent",
6
+ "current": "same prompts, skill installed"
7
+ },
8
+ "cases": [
9
+ {
10
+ "schema_version": "outcome-case/1",
11
+ "id": "ED-OUT-001-positive-receipt-per-claim",
12
+ "skill": "evidence-docs",
13
+ "prompt": {
14
+ "text": "Document what test/validate.py checks, and attach a receipt to each claim — a file:line, a command and its output, or a test name; write claims-doc.md."
15
+ },
16
+ "environment": {
17
+ "model": "inherit",
18
+ "host": "any",
19
+ "case_digest": "9496eb52cebe5fe634295e2d6f92954b1a656f0b57fb85021e4f5471e333ec97"
20
+ },
21
+ "checks": {
22
+ "tool": [
23
+ {
24
+ "name": "python3 present",
25
+ "command": "python3 -c 'pass'"
26
+ }
27
+ ],
28
+ "load_trace": {
29
+ "expect_loaded": [
30
+ "evidence-docs"
31
+ ],
32
+ "expect_not_loaded": []
33
+ },
34
+ "outcome": [
35
+ {
36
+ "name": "doc exists",
37
+ "kind": "artifact-exists",
38
+ "target": "claims-doc.md"
39
+ },
40
+ {
41
+ "name": "each claim carries a receipt (a runnable command)",
42
+ "kind": "artifact-contains",
43
+ "target": "claims-doc.md",
44
+ "expect": "$"
45
+ }
46
+ ]
47
+ }
48
+ },
49
+ {
50
+ "schema_version": "outcome-case/1",
51
+ "id": "ED-OUT-002-negative-routing",
52
+ "skill": "evidence-docs",
53
+ "prompt": {
54
+ "text": "Explain in two sentences what a webhook is."
55
+ },
56
+ "environment": {
57
+ "model": "inherit",
58
+ "host": "any",
59
+ "case_digest": "a5da94aa03520bb6226ea88328a75ab6d7e3985c73a0e90a1379bf168fead57e"
60
+ },
61
+ "checks": {
62
+ "tool": [
63
+ {
64
+ "name": "python3 present",
65
+ "command": "python3 -c 'pass'"
66
+ }
67
+ ],
68
+ "load_trace": {
69
+ "expect_loaded": [],
70
+ "expect_not_loaded": [
71
+ "evidence-docs"
72
+ ]
73
+ },
74
+ "outcome": [
75
+ {
76
+ "name": "no evidence-doc artifact appears",
77
+ "kind": "command-exit-0",
78
+ "target": "test ! -f claims-doc.md"
79
+ }
80
+ ]
81
+ }
82
+ },
83
+ {
84
+ "schema_version": "outcome-case/1",
85
+ "id": "ED-OUT-003-unsupported-claim-refused",
86
+ "skill": "evidence-docs",
87
+ "prompt": {
88
+ "text": "State in a doc that 'docs are in sync' and back it — do not assert it on your word; write sync-claim.md that either shows the exit code proving it or says the claim is unsupported."
89
+ },
90
+ "environment": {
91
+ "model": "inherit",
92
+ "host": "any",
93
+ "case_digest": "84f9c3bfc90e1eba9a74c97eda8577fe2081d4cc9f500864d971e195cd2aa1c1"
94
+ },
95
+ "checks": {
96
+ "tool": [
97
+ {
98
+ "name": "python3 present",
99
+ "command": "python3 -c 'pass'"
100
+ }
101
+ ],
102
+ "load_trace": {
103
+ "expect_loaded": [
104
+ "evidence-docs"
105
+ ],
106
+ "expect_not_loaded": []
107
+ },
108
+ "outcome": [
109
+ {
110
+ "name": "the claim is proven by an exit code or marked unsupported",
111
+ "kind": "artifact-contains",
112
+ "target": "sync-claim.md",
113
+ "expect": "exit"
114
+ }
115
+ ]
116
+ }
117
+ },
118
+ {
119
+ "schema_version": "outcome-case/1",
120
+ "id": "ED-OUT-004-noop-chat-answer",
121
+ "skill": "evidence-docs",
122
+ "prompt": {
123
+ "text": "In this chat, roughly how many test suites does the repo have? Just answer me here."
124
+ },
125
+ "environment": {
126
+ "model": "inherit",
127
+ "host": "any",
128
+ "case_digest": "c29650fd684f6cf9cf0dd8c6db4a23d79c40ee4812b19bce1ccd257238e3e0b7"
129
+ },
130
+ "checks": {
131
+ "tool": [
132
+ {
133
+ "name": "python3 present",
134
+ "command": "python3 -c 'pass'"
135
+ }
136
+ ],
137
+ "load_trace": {
138
+ "expect_loaded": [],
139
+ "expect_not_loaded": [
140
+ "evidence-docs"
141
+ ]
142
+ },
143
+ "outcome": [
144
+ {
145
+ "name": "a chat answer writes no evidence document",
146
+ "kind": "command-exit-0",
147
+ "target": "test ! -f claims-doc.md"
148
+ }
149
+ ]
150
+ }
151
+ },
152
+ {
153
+ "schema_version": "outcome-case/1",
154
+ "id": "ED-OUT-005-dependency-unavailable-is-named",
155
+ "skill": "evidence-docs",
156
+ "prompt": {
157
+ "text": "Resolve every reference this doc names against a clean checkout with the neighbouring task-pipeline dependency ABSENT; where a link cannot resolve, print 'capability unavailable' rather than claiming all links resolve; write link-report.md."
158
+ },
159
+ "environment": {
160
+ "model": "inherit",
161
+ "host": "any",
162
+ "case_digest": "bf4961dfbbc8904814d771bb6e649df0e278d93611c75b563f63ba57d2d23569"
163
+ },
164
+ "checks": {
165
+ "tool": [
166
+ {
167
+ "name": "python3 present",
168
+ "command": "python3 -c 'pass'"
169
+ }
170
+ ],
171
+ "load_trace": {
172
+ "expect_loaded": [
173
+ "evidence-docs"
174
+ ],
175
+ "expect_not_loaded": []
176
+ },
177
+ "outcome": [
178
+ {
179
+ "name": "an unresolvable dependency prints capability unavailable (ED-01)",
180
+ "kind": "artifact-contains",
181
+ "target": "link-report.md",
182
+ "expect": "capability unavailable"
183
+ }
184
+ ]
185
+ }
186
+ }
187
+ ]
188
+ }
@@ -0,0 +1,188 @@
1
+ {
2
+ "contract": "outcome-case/1 (schemas/outcome-case.schema.json + test/outcome_harness.py in ssheleg/sshlg-skills)",
3
+ "note": "Outcome corpus for project-audit (FIX-EV-01.10). Judged on ARTIFACTS via the family harness: the runner records the actual output oracle verdict AND the raw result per case; with/without-skill arms are the harness's baseline contract. The HTML report is OPTIONAL — a run without --report writes no HTML and still completes. Nothing in the production skill was changed for grader convenience.",
4
+ "arms": {
5
+ "baseline": "same prompts, skill absent",
6
+ "current": "same prompts, skill installed"
7
+ },
8
+ "cases": [
9
+ {
10
+ "schema_version": "outcome-case/1",
11
+ "id": "PA-OUT-001-adr-does-not-excuse-a-defect",
12
+ "skill": "project-audit",
13
+ "prompt": {
14
+ "text": "An ADR permits logging the refresh token. Audit the project and report the exposure anyway, LINKED to the ADR (a documented decision does not make a leak safe); write audit.json."
15
+ },
16
+ "environment": {
17
+ "model": "inherit",
18
+ "host": "any",
19
+ "case_digest": "28a695d37f9c244535c0d968e6ba870e37b1af5e87d94d4d83eb01591f76a805"
20
+ },
21
+ "checks": {
22
+ "tool": [
23
+ {
24
+ "name": "python3 present",
25
+ "command": "python3 -c 'pass'"
26
+ }
27
+ ],
28
+ "load_trace": {
29
+ "expect_loaded": [
30
+ "project-audit"
31
+ ],
32
+ "expect_not_loaded": []
33
+ },
34
+ "outcome": [
35
+ {
36
+ "name": "the audit exists",
37
+ "kind": "artifact-exists",
38
+ "target": "audit.json"
39
+ },
40
+ {
41
+ "name": "the exposure is a finding linked to the ADR (PA-01)",
42
+ "kind": "artifact-contains",
43
+ "target": "audit.json",
44
+ "expect": "exposure"
45
+ }
46
+ ]
47
+ }
48
+ },
49
+ {
50
+ "schema_version": "outcome-case/1",
51
+ "id": "PA-OUT-002-negative-routing",
52
+ "skill": "project-audit",
53
+ "prompt": {
54
+ "text": "Explain in two sentences what a webhook is."
55
+ },
56
+ "environment": {
57
+ "model": "inherit",
58
+ "host": "any",
59
+ "case_digest": "a5da94aa03520bb6226ea88328a75ab6d7e3985c73a0e90a1379bf168fead57e"
60
+ },
61
+ "checks": {
62
+ "tool": [
63
+ {
64
+ "name": "python3 present",
65
+ "command": "python3 -c 'pass'"
66
+ }
67
+ ],
68
+ "load_trace": {
69
+ "expect_loaded": [],
70
+ "expect_not_loaded": [
71
+ "project-audit"
72
+ ]
73
+ },
74
+ "outcome": [
75
+ {
76
+ "name": "no audit artifact appears",
77
+ "kind": "command-exit-0",
78
+ "target": "test ! -f audit.json"
79
+ }
80
+ ]
81
+ }
82
+ },
83
+ {
84
+ "schema_version": "outcome-case/1",
85
+ "id": "PA-OUT-003-reproduced-mechanism-is-a-finding",
86
+ "skill": "project-audit",
87
+ "prompt": {
88
+ "text": "A race is reproduced locally; production occurrence is unknown. Record it as a finding with observation scope and time in race.json — a zero production sample is not zero risk (PA-02)."
89
+ },
90
+ "environment": {
91
+ "model": "inherit",
92
+ "host": "any",
93
+ "case_digest": "bd8a19a50cbf700b10f9d220c8fb01bea9dbdcd53757e2869b82e7facb162001"
94
+ },
95
+ "checks": {
96
+ "tool": [
97
+ {
98
+ "name": "python3 present",
99
+ "command": "python3 -c 'pass'"
100
+ }
101
+ ],
102
+ "load_trace": {
103
+ "expect_loaded": [
104
+ "project-audit"
105
+ ],
106
+ "expect_not_loaded": []
107
+ },
108
+ "outcome": [
109
+ {
110
+ "name": "the reproduced mechanism stays a finding (PA-02)",
111
+ "kind": "artifact-contains",
112
+ "target": "race.json",
113
+ "expect": "production"
114
+ }
115
+ ]
116
+ }
117
+ },
118
+ {
119
+ "schema_version": "outcome-case/1",
120
+ "id": "PA-OUT-004-noop-no-report-completes",
121
+ "skill": "project-audit",
122
+ "prompt": {
123
+ "text": "Run the project audit WITHOUT --report; confirm no HTML is created and the run is still complete; note it in run-note.md."
124
+ },
125
+ "environment": {
126
+ "model": "inherit",
127
+ "host": "any",
128
+ "case_digest": "e358282127ba3f0a59888caadf7d6aa47485b17e7e9d63a4e56f62e40747065d"
129
+ },
130
+ "checks": {
131
+ "tool": [
132
+ {
133
+ "name": "python3 present",
134
+ "command": "python3 -c 'pass'"
135
+ }
136
+ ],
137
+ "load_trace": {
138
+ "expect_loaded": [
139
+ "project-audit"
140
+ ],
141
+ "expect_not_loaded": []
142
+ },
143
+ "outcome": [
144
+ {
145
+ "name": "the run completes with no HTML (PA-03)",
146
+ "kind": "command-exit-0",
147
+ "target": "test ! -f report.html"
148
+ }
149
+ ]
150
+ }
151
+ },
152
+ {
153
+ "schema_version": "outcome-case/1",
154
+ "id": "PA-OUT-005-report-requires-existing-html",
155
+ "skill": "project-audit",
156
+ "prompt": {
157
+ "text": "Run the audit WITH --report and record that the HTML exists, its links are safe, and its inspect/render status is reported in report-check.md."
158
+ },
159
+ "environment": {
160
+ "model": "inherit",
161
+ "host": "any",
162
+ "case_digest": "fd603f6d516148da4411d34d408af5677b0b64570ae55cb81439ae7273202325"
163
+ },
164
+ "checks": {
165
+ "tool": [
166
+ {
167
+ "name": "python3 present",
168
+ "command": "python3 -c 'pass'"
169
+ }
170
+ ],
171
+ "load_trace": {
172
+ "expect_loaded": [
173
+ "project-audit"
174
+ ],
175
+ "expect_not_loaded": []
176
+ },
177
+ "outcome": [
178
+ {
179
+ "name": "with --report the HTML is required to exist (PA-03)",
180
+ "kind": "artifact-contains",
181
+ "target": "report-check.md",
182
+ "expect": "render"
183
+ }
184
+ ]
185
+ }
186
+ }
187
+ ]
188
+ }