task-pipeline-skill 1.85.2 → 1.86.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -0
- package/CONTRIBUTING.md +2 -2
- package/README.md +2 -1
- package/SKILL-CARD.md +1 -1
- package/bin/task-pipeline.js +70 -9
- package/evals/cases/evidence-docs.json +188 -0
- package/evals/cases/project-audit.json +188 -0
- package/evals/cases/task-pipeline.json +191 -0
- package/package.json +5 -4
- package/plugins/task-pipeline/.claude-plugin/plugin.json +1 -1
- package/plugins/task-pipeline/skills/evidence-docs/SKILL.md +16 -11
- package/plugins/task-pipeline/skills/evidence-docs/references/GENERATED.md +8 -0
- package/plugins/task-pipeline/skills/evidence-docs/references/documentation.md +472 -0
- package/plugins/task-pipeline/skills/evidence-docs/references/gates.md +645 -0
- package/plugins/task-pipeline/skills/evidence-docs/references/hooks.md +279 -0
- package/plugins/task-pipeline/skills/evidence-docs/references/learned.md +292 -0
- package/plugins/task-pipeline/skills/evidence-docs/references/retrospective.md +551 -0
- package/plugins/task-pipeline/skills/evidence-docs/references/setup.md +149 -0
- package/plugins/task-pipeline/skills/evidence-docs/templates/decisions.md +50 -0
- package/plugins/task-pipeline/skills/evidence-docs/templates/docgate.sh +537 -0
- package/plugins/task-pipeline/skills/project-audit/SKILL.md +66 -25
- package/plugins/task-pipeline/skills/project-audit/scripts/audit.py +11 -0
- package/plugins/task-pipeline/skills/task-pipeline/SKILL.md +71 -54
- package/plugins/task-pipeline/skills/task-pipeline/execution-attempt.schema.json +68 -0
- package/plugins/task-pipeline/skills/task-pipeline/execution-packet.example.json +42 -0
- package/plugins/task-pipeline/skills/task-pipeline/execution-packet.schema.json +217 -0
- package/plugins/task-pipeline/skills/task-pipeline/execution-result.example.json +49 -0
- package/plugins/task-pipeline/skills/task-pipeline/execution-result.schema.json +261 -0
- package/plugins/task-pipeline/skills/task-pipeline/graph.example.json +10 -1
- package/plugins/task-pipeline/skills/task-pipeline/graph.schema.json +172 -2
- package/plugins/task-pipeline/skills/task-pipeline/pipeline.schema.json +50 -1
- package/plugins/task-pipeline/skills/task-pipeline/references/acceptance.md +7 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/artifacts.md +23 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/audit.md +6 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/backlog.md +8 -1
- package/plugins/task-pipeline/skills/task-pipeline/references/browser.md +8 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/build.md +32 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/companion-skills.md +14 -3
- package/plugins/task-pipeline/skills/task-pipeline/references/decomposition.md +83 -2
- package/plugins/task-pipeline/skills/task-pipeline/references/doctrine-map.md +53 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/documentation.md +3 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/grill.md +27 -8
- package/plugins/task-pipeline/skills/task-pipeline/references/hooks.md +10 -5
- package/plugins/task-pipeline/skills/task-pipeline/references/model-tiering.md +19 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/planning.md +203 -26
- package/plugins/task-pipeline/skills/task-pipeline/references/portability.md +1 -0
- package/plugins/task-pipeline/skills/task-pipeline/references/retrospective.md +26 -8
- package/plugins/task-pipeline/skills/task-pipeline/references/work-graph.md +7 -1
- package/plugins/task-pipeline/skills/task-pipeline/scripts/context_packets.py +686 -0
- package/plugins/task-pipeline/skills/task-pipeline/scripts/execution_authority.py +271 -0
- package/plugins/task-pipeline/skills/task-pipeline/scripts/graph.py +415 -18
- package/plugins/task-pipeline/skills/task-pipeline/scripts/packet.py +400 -0
- package/plugins/task-pipeline/skills/task-pipeline/templates/README.md +2 -0
- package/plugins/task-pipeline/skills/task-pipeline/templates/browser-claims.json +54 -0
- package/plugins/task-pipeline/skills/task-pipeline/templates/finding-evidence.json +42 -0
- package/plugins/task-pipeline/skills/task-pipeline/templates/hooks.example.json +2 -2
- package/plugins/task-pipeline/skills/task-pipeline/templates/run.md +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,90 @@
|
|
|
1
|
+
## v1.86.3 — the release declares the stamp it does not carry, eighth time
|
|
2
|
+
|
|
3
|
+
`v1.86.2` was tagged and refused by its own release workflow: `npm run test:all` against
|
|
4
|
+
the tag's tree found the version named nowhere in `retro.md`'s `## Releases that carry no
|
|
5
|
+
stamp`. Nothing published under it — the tag exists, the release does not, npm never
|
|
6
|
+
served it. The tree had passed `npm test` locally and CI's `validate` job; the release
|
|
7
|
+
runs `test:all`, and `R-010` says to run exactly that against the tree you are about to
|
|
8
|
+
tag. Read that evening, skipped that evening.
|
|
9
|
+
|
|
10
|
+
The v1.86.2 payload is unchanged. This release adds the declaration for both versions.
|
|
11
|
+
|
|
12
|
+
## v1.86.2 — the filter this skill exported filtered nothing
|
|
13
|
+
|
|
14
|
+
Claude Code 2.1.270 started printing `hooks.json: unknown key "if" … ignored` at session
|
|
15
|
+
start for a sibling plugin. The same shape has been in this skill's one shipped template
|
|
16
|
+
since 2026-08-03, and every project that copied it into `.claude/settings.json` got a
|
|
17
|
+
documentation gate that ran on **every** Bash call — a red gate refused `ls`, not commits.
|
|
18
|
+
|
|
19
|
+
Guards: 429 → **429**, property checks 15 → **15** — the regression for this release is
|
|
20
|
+
`test/audit_regressions/fix-hk-02.py`, run by `npm test`, and NOT a workflow step:
|
|
21
|
+
`validate.yml` sits at 511,941 of GitHub's 512,000 bytes (#91), and a 59-byte headroom holds
|
|
22
|
+
no step. It plants the old shape into a copy and requires the validator to refuse it by name.
|
|
23
|
+
|
|
24
|
+
- **`"if": "Bash(git commit *)"` sat beside `matcher`, and a matcher group is only
|
|
25
|
+
`matcher` + `hooks`.** Read out of the 2.1.270 binary's schema: `if` exists on a command
|
|
26
|
+
handler only. Moved there in `templates/hooks.example.json`; the group's `_note` moved to
|
|
27
|
+
a top-level `_filter_note` for the same reason, since the block is copied verbatim into a
|
|
28
|
+
settings file.
|
|
29
|
+
- **`references/hooks.md` said "beside `matcher`" and now says "inside the handler,
|
|
30
|
+
beside `type` and `command`"** — the worked example too; the evidence-docs mirror was
|
|
31
|
+
regenerated with `fix-ed-01.01.py --sync`. The dated 2026-08-03 documentation-track plan
|
|
32
|
+
keeps its wording.
|
|
33
|
+
- **A new validator check refuses any key the hook schema does not know** at either level
|
|
34
|
+
and requires exactly one handler-level `Bash(git commit *)`; `fix-hk-02.py` plants the
|
|
35
|
+
pre-v1.86.2 shape back and watches the check fire. `claude plugin validate
|
|
36
|
+
--strict` passes the defective file, so a repo gate is the only place this fails before a
|
|
37
|
+
session start.
|
|
38
|
+
|
|
39
|
+
## v1.86.1 — the release declares the stamp it does not carry
|
|
40
|
+
|
|
41
|
+
`v1.86.0` was tagged and refused: the run-stamp gate reads the tag's own tree, a
|
|
42
|
+
branch push cannot see a tag that does not exist yet, and the declaration was not
|
|
43
|
+
in the tree when the tag was cut. Nothing published under it — the tag exists,
|
|
44
|
+
the release does not, npm never served it.
|
|
45
|
+
|
|
46
|
+
This is the seventh instance of a mechanism this repository's `retro.md` names
|
|
47
|
+
six times, and `R-010` already held the rule: run the suite the RELEASE runs,
|
|
48
|
+
against the tree you are about to tag. The lesson was written and read past,
|
|
49
|
+
which is what its own retirement condition — "it becomes a mechanical pre-push
|
|
50
|
+
hook" — exists to end.
|
|
51
|
+
|
|
52
|
+
The v1.86.0 payload is unchanged. This release adds the declaration.
|
|
53
|
+
|
|
54
|
+
## v1.86.0 — the audit closes, and three guards stop passing on a wording
|
|
55
|
+
|
|
56
|
+
Sherlock external-v3 (37 findings) plus the context-ready handoff (PR #85),
|
|
57
|
+
each finding carrying its own executable regression.
|
|
58
|
+
|
|
59
|
+
Guards: 429 → **429**, property checks 15 → **15** — no guard was added or
|
|
60
|
+
removed. Three plants were REPOINTED to follow their subjects to the wording
|
|
61
|
+
those subjects had moved to, and the conditionals check became a script when
|
|
62
|
+
the workflow hit GitHub's size limit. A repointed plant is the same guard
|
|
63
|
+
proving the same thing about a sentence that moved.
|
|
64
|
+
|
|
65
|
+
- **A guard keyed to a wording had silently stopped running.** The retrospective
|
|
66
|
+
gained rule CLASSES — a cold rule is marked review-needed rather than deleted,
|
|
67
|
+
and the trigger reads *five exposure opportunities* instead of *five run
|
|
68
|
+
stamps*. The check requiring the condition to name BOTH units was keyed to the
|
|
69
|
+
old words, so its corpus predicate matched nothing, every surface dropped out
|
|
70
|
+
of scope, and it passed everything. Found only because a NEGATIVE SELF-TEST
|
|
71
|
+
refused to plant. The discovery predicate and the assertion are now built from
|
|
72
|
+
one pattern, and the failure message no longer names a unit — naming one is
|
|
73
|
+
what tied it to a wording.
|
|
74
|
+
- **A property check pinned a count.** went red when the graph
|
|
75
|
+
schema gained a fourth conditional, all four correctly behind a . It
|
|
76
|
+
compares the refs against the entries now, so inlining one still fails at any
|
|
77
|
+
number of rules.
|
|
78
|
+
- Explaining that fix inline pushed past
|
|
79
|
+
GitHub's 512,000-byte limit — above which the workflow stays `active`, creates
|
|
80
|
+
no runs, and `gh pr checks` says "no checks reported". The size guard names its
|
|
81
|
+
own remedy, and taking it moved the check into `test/property_graph_conditionals.py`.
|
|
82
|
+
- The doctrine map moved to `references/doctrine-map.md` when the body breached
|
|
83
|
+
the 5000-token budget; the reachability walker is transitive, so every
|
|
84
|
+
reference it names is still reachable. `evidence-docs/references/GENERATED.md`
|
|
85
|
+
— the note saying those copies are generated — was reachable from nothing, so
|
|
86
|
+
the agent most likely to edit a copy never read it.
|
|
87
|
+
|
|
1
88
|
## v1.85.2 — the two-merge protocol, followed this time
|
|
2
89
|
|
|
3
90
|
**v1.85.1 is a burned tag** — the third of this wave's choreography lessons, and
|
package/CONTRIBUTING.md
CHANGED
|
@@ -315,7 +315,7 @@ was neither strict nor lenient but **unreadable**. A list capped at ten whose re
|
|
|
315
315
|
condition cannot be read fills up and stops being pruned. The calendar is the unit nothing can
|
|
316
316
|
stall, which is why it is not belt-and-braces. Entry **rotation** ("entries older than five
|
|
317
317
|
stamps move to the archive") is a different mechanism and is deliberately out of scope.
|
|
318
|
-
*(guard: `states the cold-retirement condition
|
|
318
|
+
*(guard: `states the cold-retirement condition in one unit only`)*
|
|
319
319
|
|
|
320
320
|
**38. Every worked GATE verdict prints both disclosures.** `abstained` — what the run
|
|
321
321
|
declined to claim — and `unlooked` — what a check never looked at. Without them a `PASS`
|
|
@@ -372,7 +372,7 @@ rule and the command. Every miss was found by a reader or a sweep, never by the
|
|
|
372
372
|
**nobody notices a corpus that is too small, because everything inside it passes.**
|
|
373
373
|
Exclusions are allowed and must each carry a reason in the code: a changelog narrates
|
|
374
374
|
old formats, `docs/evidence/specs/` are point-in-time records.
|
|
375
|
-
*(guard: `a worked GATE verdict omits ` and `states the cold-retirement condition
|
|
375
|
+
*(guard: `a worked GATE verdict omits ` and `states the cold-retirement condition in one unit only` — both now run over corpora walked from disk, so a new surface joins by existing)*
|
|
376
376
|
|
|
377
377
|
**44. A carry-over row still `open` names a board id, and the board row names it back.**
|
|
378
378
|
`docs/evidence/backlog.md` is the project's queue between runs; the ledger's `open`
|
package/README.md
CHANGED
|
@@ -587,7 +587,7 @@ must say so.
|
|
|
587
587
|
### Held to Anthropic's own Skill authoring guidance
|
|
588
588
|
|
|
589
589
|
Audited against the four Agent Skills pages. Most of it already held — `name`
|
|
590
|
-
13/64 chars, `description` 903/1024 chars, `SKILL.md`
|
|
590
|
+
13/64 chars, `description` 903/1024 chars, `SKILL.md` 279/500 lines, all 39 references
|
|
591
591
|
linked **directly** from `SKILL.md`, and the bundle far under the 30 MB ceiling. What
|
|
592
592
|
did not, now does:
|
|
593
593
|
|
|
@@ -929,6 +929,7 @@ recommendation, so you arm the whole run in one exchange. Detail:
|
|
|
929
929
|
|---|---|
|
|
930
930
|
| [`SKILL.md`](plugins/task-pipeline/skills/task-pipeline/SKILL.md) | the orchestrator: how to run, the stage table, the model decision |
|
|
931
931
|
| [`references/stages.md`](plugins/task-pipeline/skills/task-pipeline/references/stages.md) | per-stage detail and the exact gate criteria |
|
|
932
|
+
| [`references/doctrine-map.md`](plugins/task-pipeline/skills/task-pipeline/references/doctrine-map.md) | which reference owns which stage — the map SKILL.md used to carry inline |
|
|
932
933
|
| [`references/artifacts.md`](plugins/task-pipeline/skills/task-pipeline/references/artifacts.md) | the canonical document layout each stage writes to |
|
|
933
934
|
| [`references/conventions.md`](plugins/task-pipeline/skills/task-pipeline/references/conventions.md) | how stages 6–10 read the host project's `CLAUDE.md`, and how the documentation regime is detected |
|
|
934
935
|
| [`references/documentation.md`](plugins/task-pipeline/skills/task-pipeline/references/documentation.md) | the doc system: the inventory, registers and ids, SSOT, the Doc Loop, supersede semantics, the propagation matrix, intent vs as-built |
|
package/SKILL-CARD.md
CHANGED
|
@@ -12,7 +12,7 @@ harmless.
|
|
|
12
12
|
|---|---|
|
|
13
13
|
| **Purpose** | Runs a substantial task through ten gated delivery stages — intake grill, docs study, brainstorm, spec, plan, subagent build, tests, lint/deploy, post-deploy, docs+registers, acceptance — refusing to advance until each gate passes |
|
|
14
14
|
| **Owner** | ssheleg ([github.com/ssheleg/task-pipeline](https://github.com/ssheleg/task-pipeline)) |
|
|
15
|
-
| **Version** | 1.
|
|
15
|
+
| **Version** | 1.86.3 |
|
|
16
16
|
| **Surface** | Claude Code (filesystem skill + plugin) and the vercel `skills` CLI. **Not** uploaded to the Skills API; custom Skills do not sync across surfaces |
|
|
17
17
|
| **Dependencies** | None required. Optional: `context7` (MCP), `figma` (MCP), super-ux, agent-sync, graphify, obsidian-wiki, and **one of two browser channels** — `playwright` (CLI or MCP) or `chrome-devtools` (MCP); either satisfies the browser step and neither is required. Every stage's doctrine ships in-repo; the one conditional requirement is super-ux for the stage-3 UX track on a user-facing task |
|
|
18
18
|
| **Evaluation status** | Suite authored, 5 categories. One recorded run, **self-observed by the author**; **zero blind runs on zero of three models** — the split, and the numbers, live in [`evals/RESULTS.md`](evals/RESULTS.md) and are computed by `evals/run.py` |
|
package/bin/task-pipeline.js
CHANGED
|
@@ -50,16 +50,58 @@ function copyDir(src, dest) {
|
|
|
50
50
|
}
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
function verifyTree(src, staged, isDir) {
|
|
54
|
+
if (!isDir) {
|
|
55
|
+
if (!fs.readFileSync(src).equals(fs.readFileSync(staged))) {
|
|
56
|
+
throw new Error(`staged ${path.basename(staged)} does not match its source`);
|
|
57
|
+
}
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
for (const entry of fs.readdirSync(src, { withFileTypes: true })) {
|
|
61
|
+
const s = path.join(src, entry.name);
|
|
62
|
+
const d = path.join(staged, entry.name);
|
|
63
|
+
if (entry.isDirectory()) verifyTree(s, d, true);
|
|
64
|
+
else if (!fs.readFileSync(s).equals(fs.readFileSync(d))) {
|
|
65
|
+
throw new Error(`staged ${entry.name} does not match its source`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* A TRANSACTIONAL install (FIX-UP-05.02): the writer contract from UP-05
|
|
72
|
+
* applied to this member's installer. The old code deleted `dest` and THEN
|
|
73
|
+
* copied into it, so a crash mid-copy left nothing (with --force) or a partial
|
|
74
|
+
* tree. Now the payload is staged into a same-filesystem sibling and VERIFIED
|
|
75
|
+
* first; only then is the old install moved aside (recoverable) and the staged
|
|
76
|
+
* one renamed into place. A stage crash leaves the ACTIVE install untouched;
|
|
77
|
+
* --force=false still skips, preserving the user's bytes.
|
|
78
|
+
*/
|
|
53
79
|
function installOne(label, src, dest, isDir, force) {
|
|
54
80
|
if (fs.existsSync(dest) && !force) {
|
|
55
81
|
console.log(`skip: ${label} already installed at ${dest} (rerun with --force to overwrite)`);
|
|
56
82
|
return;
|
|
57
83
|
}
|
|
58
|
-
fs.rmSync(dest, { recursive: true, force: true });
|
|
59
84
|
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
85
|
+
const staging = `${dest}.staging-${process.pid}`; // same fs as dest
|
|
86
|
+
const prev = `${dest}.prev-${process.pid}`;
|
|
87
|
+
fs.rmSync(staging, { recursive: true, force: true });
|
|
88
|
+
try {
|
|
89
|
+
// 1. STAGE + VERIFY, before touching the active install.
|
|
90
|
+
if (isDir) copyDir(src, staging);
|
|
91
|
+
else { fs.mkdirSync(path.dirname(staging), { recursive: true }); fs.copyFileSync(src, staging); }
|
|
92
|
+
verifyTree(src, staging, isDir);
|
|
93
|
+
// 2. SWITCH: move the old aside (recoverable), rename staged into place.
|
|
94
|
+
fs.rmSync(prev, { recursive: true, force: true });
|
|
95
|
+
if (fs.existsSync(dest)) fs.renameSync(dest, prev);
|
|
96
|
+
fs.renameSync(staging, dest);
|
|
97
|
+
fs.rmSync(prev, { recursive: true, force: true });
|
|
98
|
+
console.log(`Installed ${label} -> ${dest}`);
|
|
99
|
+
} catch (err) {
|
|
100
|
+
// Abort: leave the active install intact, remove the half-built staging.
|
|
101
|
+
fs.rmSync(staging, { recursive: true, force: true });
|
|
102
|
+
if (fs.existsSync(prev) && !fs.existsSync(dest)) fs.renameSync(prev, dest);
|
|
103
|
+
throw new Error(`install aborted, previous install intact: ${err.message}`);
|
|
104
|
+
}
|
|
63
105
|
}
|
|
64
106
|
|
|
65
107
|
/**
|
|
@@ -172,6 +214,20 @@ function migrateArtifacts(args) {
|
|
|
172
214
|
return 0;
|
|
173
215
|
}
|
|
174
216
|
|
|
217
|
+
// The bundled HostContext resolver (FIX-UP-08.02) — one contract, a local
|
|
218
|
+
// copy per member because these installers run via `npx` with no shared lib.
|
|
219
|
+
// A host's config root is: an explicit root > the documented host env var >
|
|
220
|
+
// the platform default `~/<dir>`. Used verbatim (spaces preserved), never
|
|
221
|
+
// through a shell. Host EXISTENCE is a separate probe on the returned path.
|
|
222
|
+
const HOST_ENV = { claude: 'CLAUDE_CONFIG_DIR', codex: 'CODEX_HOME', gemini: 'GEMINI_CONFIG_DIR' };
|
|
223
|
+
const HOST_DIR = { claude: '.claude', codex: '.codex', gemini: '.gemini' };
|
|
224
|
+
function hostRoot(agent, home, env, explicit) {
|
|
225
|
+
if (explicit) return explicit;
|
|
226
|
+
const e = (env || process.env)[HOST_ENV[agent]];
|
|
227
|
+
if (e) return e;
|
|
228
|
+
return path.join(home, HOST_DIR[agent]);
|
|
229
|
+
}
|
|
230
|
+
|
|
175
231
|
function main(argv) {
|
|
176
232
|
const args = argv.slice(2);
|
|
177
233
|
if (args.includes('--help') || args.includes('-h')) {
|
|
@@ -204,9 +260,10 @@ function main(argv) {
|
|
|
204
260
|
// copy SHADOWS the plugin — silently serving whatever version was copied, forever.
|
|
205
261
|
// The family launcher (sshlg-skills) prunes exactly these copies for that reason,
|
|
206
262
|
// so creating one without saying so undoes the thing it is paired with.
|
|
263
|
+
const claude = hostRoot('claude', home, process.env);
|
|
207
264
|
const pluginDirs = [
|
|
208
|
-
path.join(
|
|
209
|
-
path.join(
|
|
265
|
+
path.join(claude, 'plugins', 'marketplaces', 'task-pipeline'),
|
|
266
|
+
path.join(claude, 'plugins', 'cache', 'task-pipeline'),
|
|
210
267
|
];
|
|
211
268
|
if (!force && pluginDirs.some((d) => fs.existsSync(d))) {
|
|
212
269
|
console.error(`refusing: task-pipeline is already installed as a Claude Code PLUGIN.
|
|
@@ -224,14 +281,14 @@ Rerun with --force if you deliberately want the plain copy instead.`);
|
|
|
224
281
|
installOne(
|
|
225
282
|
'task-pipeline skill ',
|
|
226
283
|
skillSrc,
|
|
227
|
-
path.join(
|
|
284
|
+
path.join(claude, 'skills', 'task-pipeline'),
|
|
228
285
|
true,
|
|
229
286
|
force
|
|
230
287
|
);
|
|
231
288
|
installOne(
|
|
232
289
|
'/task-pipeline command',
|
|
233
290
|
cmdSrc,
|
|
234
|
-
path.join(
|
|
291
|
+
path.join(claude, 'commands', 'task-pipeline.md'),
|
|
235
292
|
false,
|
|
236
293
|
force
|
|
237
294
|
);
|
|
@@ -240,4 +297,8 @@ Rerun with --force if you deliberately want the plain copy instead.`);
|
|
|
240
297
|
return 0;
|
|
241
298
|
}
|
|
242
299
|
|
|
243
|
-
|
|
300
|
+
if (require.main === module) {
|
|
301
|
+
process.exit(main(process.argv));
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
module.exports = { installOne, copyDir, verifyTree, hostRoot };
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
{
|
|
2
|
+
"contract": "outcome-case/1 (schemas/outcome-case.schema.json + test/outcome_harness.py in ssheleg/sshlg-skills)",
|
|
3
|
+
"note": "Outcome corpus for evidence-docs (FIX-EV-01.09). Judged on ARTIFACTS via the family harness: the runner records the actual output oracle verdict AND the raw result per case; with/without-skill arms are the harness's baseline contract. Nothing in the production skill was changed for grader convenience.",
|
|
4
|
+
"arms": {
|
|
5
|
+
"baseline": "same prompts, skill absent",
|
|
6
|
+
"current": "same prompts, skill installed"
|
|
7
|
+
},
|
|
8
|
+
"cases": [
|
|
9
|
+
{
|
|
10
|
+
"schema_version": "outcome-case/1",
|
|
11
|
+
"id": "ED-OUT-001-positive-receipt-per-claim",
|
|
12
|
+
"skill": "evidence-docs",
|
|
13
|
+
"prompt": {
|
|
14
|
+
"text": "Document what test/validate.py checks, and attach a receipt to each claim — a file:line, a command and its output, or a test name; write claims-doc.md."
|
|
15
|
+
},
|
|
16
|
+
"environment": {
|
|
17
|
+
"model": "inherit",
|
|
18
|
+
"host": "any",
|
|
19
|
+
"case_digest": "9496eb52cebe5fe634295e2d6f92954b1a656f0b57fb85021e4f5471e333ec97"
|
|
20
|
+
},
|
|
21
|
+
"checks": {
|
|
22
|
+
"tool": [
|
|
23
|
+
{
|
|
24
|
+
"name": "python3 present",
|
|
25
|
+
"command": "python3 -c 'pass'"
|
|
26
|
+
}
|
|
27
|
+
],
|
|
28
|
+
"load_trace": {
|
|
29
|
+
"expect_loaded": [
|
|
30
|
+
"evidence-docs"
|
|
31
|
+
],
|
|
32
|
+
"expect_not_loaded": []
|
|
33
|
+
},
|
|
34
|
+
"outcome": [
|
|
35
|
+
{
|
|
36
|
+
"name": "doc exists",
|
|
37
|
+
"kind": "artifact-exists",
|
|
38
|
+
"target": "claims-doc.md"
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"name": "each claim carries a receipt (a runnable command)",
|
|
42
|
+
"kind": "artifact-contains",
|
|
43
|
+
"target": "claims-doc.md",
|
|
44
|
+
"expect": "$"
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"schema_version": "outcome-case/1",
|
|
51
|
+
"id": "ED-OUT-002-negative-routing",
|
|
52
|
+
"skill": "evidence-docs",
|
|
53
|
+
"prompt": {
|
|
54
|
+
"text": "Explain in two sentences what a webhook is."
|
|
55
|
+
},
|
|
56
|
+
"environment": {
|
|
57
|
+
"model": "inherit",
|
|
58
|
+
"host": "any",
|
|
59
|
+
"case_digest": "a5da94aa03520bb6226ea88328a75ab6d7e3985c73a0e90a1379bf168fead57e"
|
|
60
|
+
},
|
|
61
|
+
"checks": {
|
|
62
|
+
"tool": [
|
|
63
|
+
{
|
|
64
|
+
"name": "python3 present",
|
|
65
|
+
"command": "python3 -c 'pass'"
|
|
66
|
+
}
|
|
67
|
+
],
|
|
68
|
+
"load_trace": {
|
|
69
|
+
"expect_loaded": [],
|
|
70
|
+
"expect_not_loaded": [
|
|
71
|
+
"evidence-docs"
|
|
72
|
+
]
|
|
73
|
+
},
|
|
74
|
+
"outcome": [
|
|
75
|
+
{
|
|
76
|
+
"name": "no evidence-doc artifact appears",
|
|
77
|
+
"kind": "command-exit-0",
|
|
78
|
+
"target": "test ! -f claims-doc.md"
|
|
79
|
+
}
|
|
80
|
+
]
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"schema_version": "outcome-case/1",
|
|
85
|
+
"id": "ED-OUT-003-unsupported-claim-refused",
|
|
86
|
+
"skill": "evidence-docs",
|
|
87
|
+
"prompt": {
|
|
88
|
+
"text": "State in a doc that 'docs are in sync' and back it — do not assert it on your word; write sync-claim.md that either shows the exit code proving it or says the claim is unsupported."
|
|
89
|
+
},
|
|
90
|
+
"environment": {
|
|
91
|
+
"model": "inherit",
|
|
92
|
+
"host": "any",
|
|
93
|
+
"case_digest": "84f9c3bfc90e1eba9a74c97eda8577fe2081d4cc9f500864d971e195cd2aa1c1"
|
|
94
|
+
},
|
|
95
|
+
"checks": {
|
|
96
|
+
"tool": [
|
|
97
|
+
{
|
|
98
|
+
"name": "python3 present",
|
|
99
|
+
"command": "python3 -c 'pass'"
|
|
100
|
+
}
|
|
101
|
+
],
|
|
102
|
+
"load_trace": {
|
|
103
|
+
"expect_loaded": [
|
|
104
|
+
"evidence-docs"
|
|
105
|
+
],
|
|
106
|
+
"expect_not_loaded": []
|
|
107
|
+
},
|
|
108
|
+
"outcome": [
|
|
109
|
+
{
|
|
110
|
+
"name": "the claim is proven by an exit code or marked unsupported",
|
|
111
|
+
"kind": "artifact-contains",
|
|
112
|
+
"target": "sync-claim.md",
|
|
113
|
+
"expect": "exit"
|
|
114
|
+
}
|
|
115
|
+
]
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"schema_version": "outcome-case/1",
|
|
120
|
+
"id": "ED-OUT-004-noop-chat-answer",
|
|
121
|
+
"skill": "evidence-docs",
|
|
122
|
+
"prompt": {
|
|
123
|
+
"text": "In this chat, roughly how many test suites does the repo have? Just answer me here."
|
|
124
|
+
},
|
|
125
|
+
"environment": {
|
|
126
|
+
"model": "inherit",
|
|
127
|
+
"host": "any",
|
|
128
|
+
"case_digest": "c29650fd684f6cf9cf0dd8c6db4a23d79c40ee4812b19bce1ccd257238e3e0b7"
|
|
129
|
+
},
|
|
130
|
+
"checks": {
|
|
131
|
+
"tool": [
|
|
132
|
+
{
|
|
133
|
+
"name": "python3 present",
|
|
134
|
+
"command": "python3 -c 'pass'"
|
|
135
|
+
}
|
|
136
|
+
],
|
|
137
|
+
"load_trace": {
|
|
138
|
+
"expect_loaded": [],
|
|
139
|
+
"expect_not_loaded": [
|
|
140
|
+
"evidence-docs"
|
|
141
|
+
]
|
|
142
|
+
},
|
|
143
|
+
"outcome": [
|
|
144
|
+
{
|
|
145
|
+
"name": "a chat answer writes no evidence document",
|
|
146
|
+
"kind": "command-exit-0",
|
|
147
|
+
"target": "test ! -f claims-doc.md"
|
|
148
|
+
}
|
|
149
|
+
]
|
|
150
|
+
}
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"schema_version": "outcome-case/1",
|
|
154
|
+
"id": "ED-OUT-005-dependency-unavailable-is-named",
|
|
155
|
+
"skill": "evidence-docs",
|
|
156
|
+
"prompt": {
|
|
157
|
+
"text": "Resolve every reference this doc names against a clean checkout with the neighbouring task-pipeline dependency ABSENT; where a link cannot resolve, print 'capability unavailable' rather than claiming all links resolve; write link-report.md."
|
|
158
|
+
},
|
|
159
|
+
"environment": {
|
|
160
|
+
"model": "inherit",
|
|
161
|
+
"host": "any",
|
|
162
|
+
"case_digest": "bf4961dfbbc8904814d771bb6e649df0e278d93611c75b563f63ba57d2d23569"
|
|
163
|
+
},
|
|
164
|
+
"checks": {
|
|
165
|
+
"tool": [
|
|
166
|
+
{
|
|
167
|
+
"name": "python3 present",
|
|
168
|
+
"command": "python3 -c 'pass'"
|
|
169
|
+
}
|
|
170
|
+
],
|
|
171
|
+
"load_trace": {
|
|
172
|
+
"expect_loaded": [
|
|
173
|
+
"evidence-docs"
|
|
174
|
+
],
|
|
175
|
+
"expect_not_loaded": []
|
|
176
|
+
},
|
|
177
|
+
"outcome": [
|
|
178
|
+
{
|
|
179
|
+
"name": "an unresolvable dependency prints capability unavailable (ED-01)",
|
|
180
|
+
"kind": "artifact-contains",
|
|
181
|
+
"target": "link-report.md",
|
|
182
|
+
"expect": "capability unavailable"
|
|
183
|
+
}
|
|
184
|
+
]
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
]
|
|
188
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
{
|
|
2
|
+
"contract": "outcome-case/1 (schemas/outcome-case.schema.json + test/outcome_harness.py in ssheleg/sshlg-skills)",
|
|
3
|
+
"note": "Outcome corpus for project-audit (FIX-EV-01.10). Judged on ARTIFACTS via the family harness: the runner records the actual output oracle verdict AND the raw result per case; with/without-skill arms are the harness's baseline contract. The HTML report is OPTIONAL — a run without --report writes no HTML and still completes. Nothing in the production skill was changed for grader convenience.",
|
|
4
|
+
"arms": {
|
|
5
|
+
"baseline": "same prompts, skill absent",
|
|
6
|
+
"current": "same prompts, skill installed"
|
|
7
|
+
},
|
|
8
|
+
"cases": [
|
|
9
|
+
{
|
|
10
|
+
"schema_version": "outcome-case/1",
|
|
11
|
+
"id": "PA-OUT-001-adr-does-not-excuse-a-defect",
|
|
12
|
+
"skill": "project-audit",
|
|
13
|
+
"prompt": {
|
|
14
|
+
"text": "An ADR permits logging the refresh token. Audit the project and report the exposure anyway, LINKED to the ADR (a documented decision does not make a leak safe); write audit.json."
|
|
15
|
+
},
|
|
16
|
+
"environment": {
|
|
17
|
+
"model": "inherit",
|
|
18
|
+
"host": "any",
|
|
19
|
+
"case_digest": "28a695d37f9c244535c0d968e6ba870e37b1af5e87d94d4d83eb01591f76a805"
|
|
20
|
+
},
|
|
21
|
+
"checks": {
|
|
22
|
+
"tool": [
|
|
23
|
+
{
|
|
24
|
+
"name": "python3 present",
|
|
25
|
+
"command": "python3 -c 'pass'"
|
|
26
|
+
}
|
|
27
|
+
],
|
|
28
|
+
"load_trace": {
|
|
29
|
+
"expect_loaded": [
|
|
30
|
+
"project-audit"
|
|
31
|
+
],
|
|
32
|
+
"expect_not_loaded": []
|
|
33
|
+
},
|
|
34
|
+
"outcome": [
|
|
35
|
+
{
|
|
36
|
+
"name": "the audit exists",
|
|
37
|
+
"kind": "artifact-exists",
|
|
38
|
+
"target": "audit.json"
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"name": "the exposure is a finding linked to the ADR (PA-01)",
|
|
42
|
+
"kind": "artifact-contains",
|
|
43
|
+
"target": "audit.json",
|
|
44
|
+
"expect": "exposure"
|
|
45
|
+
}
|
|
46
|
+
]
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
"schema_version": "outcome-case/1",
|
|
51
|
+
"id": "PA-OUT-002-negative-routing",
|
|
52
|
+
"skill": "project-audit",
|
|
53
|
+
"prompt": {
|
|
54
|
+
"text": "Explain in two sentences what a webhook is."
|
|
55
|
+
},
|
|
56
|
+
"environment": {
|
|
57
|
+
"model": "inherit",
|
|
58
|
+
"host": "any",
|
|
59
|
+
"case_digest": "a5da94aa03520bb6226ea88328a75ab6d7e3985c73a0e90a1379bf168fead57e"
|
|
60
|
+
},
|
|
61
|
+
"checks": {
|
|
62
|
+
"tool": [
|
|
63
|
+
{
|
|
64
|
+
"name": "python3 present",
|
|
65
|
+
"command": "python3 -c 'pass'"
|
|
66
|
+
}
|
|
67
|
+
],
|
|
68
|
+
"load_trace": {
|
|
69
|
+
"expect_loaded": [],
|
|
70
|
+
"expect_not_loaded": [
|
|
71
|
+
"project-audit"
|
|
72
|
+
]
|
|
73
|
+
},
|
|
74
|
+
"outcome": [
|
|
75
|
+
{
|
|
76
|
+
"name": "no audit artifact appears",
|
|
77
|
+
"kind": "command-exit-0",
|
|
78
|
+
"target": "test ! -f audit.json"
|
|
79
|
+
}
|
|
80
|
+
]
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"schema_version": "outcome-case/1",
|
|
85
|
+
"id": "PA-OUT-003-reproduced-mechanism-is-a-finding",
|
|
86
|
+
"skill": "project-audit",
|
|
87
|
+
"prompt": {
|
|
88
|
+
"text": "A race is reproduced locally; production occurrence is unknown. Record it as a finding with observation scope and time in race.json — a zero production sample is not zero risk (PA-02)."
|
|
89
|
+
},
|
|
90
|
+
"environment": {
|
|
91
|
+
"model": "inherit",
|
|
92
|
+
"host": "any",
|
|
93
|
+
"case_digest": "bd8a19a50cbf700b10f9d220c8fb01bea9dbdcd53757e2869b82e7facb162001"
|
|
94
|
+
},
|
|
95
|
+
"checks": {
|
|
96
|
+
"tool": [
|
|
97
|
+
{
|
|
98
|
+
"name": "python3 present",
|
|
99
|
+
"command": "python3 -c 'pass'"
|
|
100
|
+
}
|
|
101
|
+
],
|
|
102
|
+
"load_trace": {
|
|
103
|
+
"expect_loaded": [
|
|
104
|
+
"project-audit"
|
|
105
|
+
],
|
|
106
|
+
"expect_not_loaded": []
|
|
107
|
+
},
|
|
108
|
+
"outcome": [
|
|
109
|
+
{
|
|
110
|
+
"name": "the reproduced mechanism stays a finding (PA-02)",
|
|
111
|
+
"kind": "artifact-contains",
|
|
112
|
+
"target": "race.json",
|
|
113
|
+
"expect": "production"
|
|
114
|
+
}
|
|
115
|
+
]
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"schema_version": "outcome-case/1",
|
|
120
|
+
"id": "PA-OUT-004-noop-no-report-completes",
|
|
121
|
+
"skill": "project-audit",
|
|
122
|
+
"prompt": {
|
|
123
|
+
"text": "Run the project audit WITHOUT --report; confirm no HTML is created and the run is still complete; note it in run-note.md."
|
|
124
|
+
},
|
|
125
|
+
"environment": {
|
|
126
|
+
"model": "inherit",
|
|
127
|
+
"host": "any",
|
|
128
|
+
"case_digest": "e358282127ba3f0a59888caadf7d6aa47485b17e7e9d63a4e56f62e40747065d"
|
|
129
|
+
},
|
|
130
|
+
"checks": {
|
|
131
|
+
"tool": [
|
|
132
|
+
{
|
|
133
|
+
"name": "python3 present",
|
|
134
|
+
"command": "python3 -c 'pass'"
|
|
135
|
+
}
|
|
136
|
+
],
|
|
137
|
+
"load_trace": {
|
|
138
|
+
"expect_loaded": [
|
|
139
|
+
"project-audit"
|
|
140
|
+
],
|
|
141
|
+
"expect_not_loaded": []
|
|
142
|
+
},
|
|
143
|
+
"outcome": [
|
|
144
|
+
{
|
|
145
|
+
"name": "the run completes with no HTML (PA-03)",
|
|
146
|
+
"kind": "command-exit-0",
|
|
147
|
+
"target": "test ! -f report.html"
|
|
148
|
+
}
|
|
149
|
+
]
|
|
150
|
+
}
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
"schema_version": "outcome-case/1",
|
|
154
|
+
"id": "PA-OUT-005-report-requires-existing-html",
|
|
155
|
+
"skill": "project-audit",
|
|
156
|
+
"prompt": {
|
|
157
|
+
"text": "Run the audit WITH --report and record that the HTML exists, its links are safe, and its inspect/render status is reported in report-check.md."
|
|
158
|
+
},
|
|
159
|
+
"environment": {
|
|
160
|
+
"model": "inherit",
|
|
161
|
+
"host": "any",
|
|
162
|
+
"case_digest": "fd603f6d516148da4411d34d408af5677b0b64570ae55cb81439ae7273202325"
|
|
163
|
+
},
|
|
164
|
+
"checks": {
|
|
165
|
+
"tool": [
|
|
166
|
+
{
|
|
167
|
+
"name": "python3 present",
|
|
168
|
+
"command": "python3 -c 'pass'"
|
|
169
|
+
}
|
|
170
|
+
],
|
|
171
|
+
"load_trace": {
|
|
172
|
+
"expect_loaded": [
|
|
173
|
+
"project-audit"
|
|
174
|
+
],
|
|
175
|
+
"expect_not_loaded": []
|
|
176
|
+
},
|
|
177
|
+
"outcome": [
|
|
178
|
+
{
|
|
179
|
+
"name": "with --report the HTML is required to exist (PA-03)",
|
|
180
|
+
"kind": "artifact-contains",
|
|
181
|
+
"target": "report-check.md",
|
|
182
|
+
"expect": "render"
|
|
183
|
+
}
|
|
184
|
+
]
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
]
|
|
188
|
+
}
|