shapeup-sdlc 3.0.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/kernel/probe/stats.mjs +3 -80
- package/kernel/report/export.mjs +9 -13
- package/kernel/report/facts.mjs +16 -154
- package/package.json +1 -1
- package/skills/tech-lead/references/gates.md +9 -7
- package/skills/tech-lead/references/protocol.md +4 -4
- package/skills/tech-lead/workflows/shapeup-run.js +14 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "shapeup-sdlc-plugin",
|
|
3
3
|
"displayName": "ShapeUp SDLC Plugin",
|
|
4
|
-
"version": "3.0.
|
|
4
|
+
"version": "3.0.1",
|
|
5
5
|
"description": "Shape Up SDLC harness for Claude Code: shaping, intake, orient, scope-mapping, building (T0-verified, sandboxed, scope-contracted), evaluation and QA skills orchestrated by a tech-lead.",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Liberty Nguyen",
|
package/kernel/probe/stats.mjs
CHANGED
|
@@ -23,7 +23,6 @@ import { readFileSync, readdirSync, existsSync } from "node:fs";
|
|
|
23
23
|
import { resolve, join } from "node:path";
|
|
24
24
|
import { validate } from "../verify/envelope.mjs";
|
|
25
25
|
import { runArgs } from "../lib/argv.mjs";
|
|
26
|
-
import { collectRun } from "../report/export.mjs";
|
|
27
26
|
import { localDir, decisions as decisionsPath, metricsDir as metricsDirPath, SHARED } from "../lib/paths.mjs";
|
|
28
27
|
|
|
29
28
|
/**
|
|
@@ -284,79 +283,6 @@ export function readDecisions(cwd) {
|
|
|
284
283
|
} catch { return []; }
|
|
285
284
|
}
|
|
286
285
|
|
|
287
|
-
/**
|
|
288
|
-
* Project run economics for every run in the checkout — measurement-table row 4.
|
|
289
|
-
*
|
|
290
|
-
* WHY IT LIVES HERE and reads the run trace rather than the metrics shards: the same reason
|
|
291
|
-
* `--ratchet` reads `trials.jsonl` and `--hooks` reads `decisions.jsonl`. A harvest row is written
|
|
292
|
-
* once at SHIP S.6 and carries counts, never durations or cost — so a run that never shipped, which
|
|
293
|
-
* is exactly the run whose cost you want to see, has no harvest row at all. The journal has one row
|
|
294
|
-
* per agent call from the first dispatch onwards.
|
|
295
|
-
*
|
|
296
|
-
* FACTS ONLY, unchanged: sums, counts and durations over rows that already exist. Nothing here is
|
|
297
|
-
* divided by an expectation or compared to a target, because that would be a grade.
|
|
298
|
-
*
|
|
299
|
-
* @param {string} cwd - Project root.
|
|
300
|
-
* @param {(string|null)} [slug=null] - Restrict to one feature slug.
|
|
301
|
-
* @returns {object} `{runs, per_run[]}` — one economics block per run, newest last, each tagged
|
|
302
|
-
* with its run_id and slug.
|
|
303
|
-
*/
|
|
304
|
-
export function economicsReport(cwd, slug = null) {
|
|
305
|
-
const root = localDir(cwd);
|
|
306
|
-
if (!existsSync(root)) return { runs: 0, per_run: [] };
|
|
307
|
-
let slugs;
|
|
308
|
-
try { slugs = slug ? [slug] : readdirSync(root); } catch { return { runs: 0, per_run: [] }; }
|
|
309
|
-
const per_run = [];
|
|
310
|
-
for (const s of slugs.sort()) {
|
|
311
|
-
let collected = null;
|
|
312
|
-
try { collected = collectRun(cwd, s); } catch { collected = null; }
|
|
313
|
-
if (!collected) continue; // no receipt ⇒ not a run, which is not an error
|
|
314
|
-
per_run.push({ run_id: collected.run_id, slug: s, ...collected.economics });
|
|
315
|
-
}
|
|
316
|
-
return { runs: per_run.length, per_run };
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
/**
|
|
320
|
-
* Format a dollar figure, keeping "no cost row was recorded" visibly different from "$0.0000".
|
|
321
|
-
* @param {(number|null|undefined)} v - A cost in USD, or null when nothing recorded one.
|
|
322
|
-
* @returns {string} e.g. `$1.2000`, or `—` when the value is absent.
|
|
323
|
-
*/
|
|
324
|
-
function money(v) {
|
|
325
|
-
return v === null || v === undefined ? "—" : `$${v.toFixed(4)}`;
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
/**
|
|
329
|
-
* Render the economics report as text.
|
|
330
|
-
* @param {object} r - Output of {@link economicsReport}.
|
|
331
|
-
* @returns {string} The multi-line report.
|
|
332
|
-
*/
|
|
333
|
-
function renderEconomics(r) {
|
|
334
|
-
const lines = [`economics: ${r.runs} run(s) with a receipt`];
|
|
335
|
-
if (!r.runs) {
|
|
336
|
-
lines.push("", "(no run trace in this checkout — this reads the run's own records, so it is empty");
|
|
337
|
-
lines.push(" until a run opens, and stays readable after one ends until the trace is cleaned.)");
|
|
338
|
-
return lines.join("\n");
|
|
339
|
-
}
|
|
340
|
-
for (const e of r.per_run) {
|
|
341
|
-
lines.push("", ` ${e.run_id ?? e.slug}`,
|
|
342
|
-
` agent calls ${e.agent_calls} (${e.retried_calls} retried, ${e.failed_calls} failed, ${e.killed_calls} killed)`,
|
|
343
|
-
` cost ${money(e.cost_usd)} attributed ${money(e.cost_attributed_usd)} · unattributed ${money(e.cost_unattributed_usd)}`,
|
|
344
|
-
` wall clock ${e.wall_ms_total === null ? "—" : `${Math.round(e.wall_ms_total / 1000)}s`}`,
|
|
345
|
-
` to first write ${e.calls_to_first_write ?? "—"} call(s)` +
|
|
346
|
-
`${e.seconds_to_first_write === null ? "" : ` · ${e.seconds_to_first_write}s`}`,
|
|
347
|
-
` dispatches ${e.dispatches} (${e.dispatches_answered} answered, ${e.dispatches_costed} costed)`);
|
|
348
|
-
for (const m of e.by_model) {
|
|
349
|
-
lines.push(` ${String(m.model).padEnd(22)}${String(m.calls).padStart(3)} call(s) ${money(m.cost_usd)}`);
|
|
350
|
-
}
|
|
351
|
-
// The gap is named, never left as a quiet shortfall in the total.
|
|
352
|
-
if (e.dispatches_costed < e.dispatches) {
|
|
353
|
-
lines.push(` ⓘ ${e.dispatches - e.dispatches_costed} dispatch(es) carry no cost row — the journal exists only on the`,
|
|
354
|
-
" workflow lane, so a prose-lane or --tiny dispatch has no agent call to join to.");
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
return lines.join("\n");
|
|
358
|
-
}
|
|
359
|
-
|
|
360
286
|
/**
|
|
361
287
|
* Render a StatsReport as a human-readable fixed-width table.
|
|
362
288
|
* @param {object} report - A validated StatsReport (see {@link aggregate}).
|
|
@@ -395,7 +321,7 @@ function renderTable(report) {
|
|
|
395
321
|
/** The typed argv contract (see `./lib/argv.mjs`). */
|
|
396
322
|
export const ARGV_SPEC = {
|
|
397
323
|
usage: "harness.mjs probe stats [--cwd <dir>] [--metrics-dir <dir>] [--slug <slug>] [--format json|table] " +
|
|
398
|
-
"[--ratchet] [--hooks]
|
|
324
|
+
"[--ratchet] [--hooks]",
|
|
399
325
|
_: { arity: 0, max: 0, name: "(no positional operands)" },
|
|
400
326
|
cwd: { type: "path" },
|
|
401
327
|
"metrics-dir": { type: "path" },
|
|
@@ -403,7 +329,6 @@ export const ARGV_SPEC = {
|
|
|
403
329
|
format: { type: "enum", values: ["json", "table"], default: "json" },
|
|
404
330
|
ratchet: { type: "flag" },
|
|
405
331
|
hooks: { type: "flag" },
|
|
406
|
-
economics: { type: "flag" },
|
|
407
332
|
};
|
|
408
333
|
|
|
409
334
|
/**
|
|
@@ -461,7 +386,7 @@ function renderHooks(r) {
|
|
|
461
386
|
}
|
|
462
387
|
|
|
463
388
|
/**
|
|
464
|
-
* Aggregate the metric shards, or report the ratchet
|
|
389
|
+
* Aggregate the metric shards, or report the ratchet and hook ledgers.
|
|
465
390
|
*
|
|
466
391
|
* @param {string[]} rawArgv - The subcommand's own arguments (harness.mjs strips the verb words).
|
|
467
392
|
* @returns {(Promise<void>|void)} Settles when the subcommand has written its output; most paths
|
|
@@ -475,16 +400,14 @@ export async function cli(rawArgv) {
|
|
|
475
400
|
|
|
476
401
|
// The two exit measurements are separate modes: each reads a different ledger, and neither is a
|
|
477
402
|
// StatsReport (which is schema-locked to the harvest shards).
|
|
478
|
-
if (args.ratchet || args.hooks
|
|
403
|
+
if (args.ratchet || args.hooks) {
|
|
479
404
|
const out = {};
|
|
480
405
|
if (args.ratchet) out.ratchet = ratchetReport(readAllTrials(cwd, args.slug ?? null));
|
|
481
406
|
if (args.hooks) out.hooks = hooksReport(readDecisions(cwd));
|
|
482
|
-
if (args.economics) out.economics = economicsReport(cwd, args.slug ?? null);
|
|
483
407
|
if (format === "table") {
|
|
484
408
|
const parts = [];
|
|
485
409
|
if (out.ratchet) parts.push(renderRatchet(out.ratchet));
|
|
486
410
|
if (out.hooks) parts.push(renderHooks(out.hooks));
|
|
487
|
-
if (out.economics) parts.push(renderEconomics(out.economics));
|
|
488
411
|
console.log(parts.join("\n\n"));
|
|
489
412
|
} else {
|
|
490
413
|
console.log(JSON.stringify(out, null, 2));
|
package/kernel/report/export.mjs
CHANGED
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
//
|
|
4
4
|
// WHY THIS EXISTS (from the storage design, not from a preference for tooling).
|
|
5
5
|
//
|
|
6
|
-
// Every record this exports already exists. Orders, results,
|
|
7
|
-
// verdicts
|
|
8
|
-
//
|
|
6
|
+
// Every record this exports already exists. Orders, results, trial rows, T0 verdicts, criterion
|
|
7
|
+
// verdicts and hook decisions are all written during a normal run, all as JSON, all
|
|
8
|
+
// schema-registered. They live in `.shapeup/<slug>/` — the LOCAL tier, which ADR-0001
|
|
9
9
|
// defines as gitignored, machine-local and REGENERABLE. That definition is correct for run state
|
|
10
10
|
// and fatal for measurement: the trial-row contract says it out loud — a measurement left there
|
|
11
11
|
// "answers the question exactly once and then deletes itself".
|
|
@@ -46,10 +46,10 @@ import { join, resolve } from "node:path";
|
|
|
46
46
|
import { runArgs } from "../lib/argv.mjs";
|
|
47
47
|
import { splitFrontmatter } from "../lib/contract.mjs";
|
|
48
48
|
import { runIdFromReceipt, readReceipt } from "../lib/paths.mjs";
|
|
49
|
-
import { TABLES, runRow,
|
|
49
|
+
import { TABLES, runRow, dispatchFacts } from "./facts.mjs";
|
|
50
50
|
import {
|
|
51
51
|
localDir, activeScope, receipt as receiptPath, harnessRun, ordersDir, resultsDir,
|
|
52
|
-
|
|
52
|
+
trials as trialsPath, verdictsDir, evaluationDir, decisions as decisionsPath,
|
|
53
53
|
exportsDir, exportRunDir,
|
|
54
54
|
} from "../lib/paths.mjs";
|
|
55
55
|
|
|
@@ -174,8 +174,8 @@ function criterionRows(dir, runId, t) {
|
|
|
174
174
|
*
|
|
175
175
|
* @param {string} cwd - Project root.
|
|
176
176
|
* @param {string} slug - The feature slug whose run to export.
|
|
177
|
-
* @returns {(object|null)} `{run_id, slug, tables:{…},
|
|
178
|
-
*
|
|
177
|
+
* @returns {(object|null)} `{run_id, slug, tables:{…}, defects}` — or null when the slug has no
|
|
178
|
+
* readable receipt, which is the definition of "not a run".
|
|
179
179
|
*/
|
|
180
180
|
export function collectRun(cwd, slug) {
|
|
181
181
|
const t = tally();
|
|
@@ -188,10 +188,8 @@ export function collectRun(cwd, slug) {
|
|
|
188
188
|
|
|
189
189
|
const orders = readJsonDir(ordersDir(cwd, slug), t);
|
|
190
190
|
const results = readJsonDir(resultsDir(cwd, slug), t);
|
|
191
|
-
const journal = readJsonl(join(workflowRunDir(cwd, slug), "journal.jsonl"), t);
|
|
192
191
|
|
|
193
|
-
const { dispatch, ac_result, discovery, file_touched } = dispatchFacts({ orders, results,
|
|
194
|
-
const agent_call = journal.map((j) => agentCallRow(j, runId));
|
|
192
|
+
const { dispatch, ac_result, discovery, file_touched } = dispatchFacts({ orders, results, runId });
|
|
195
193
|
const run = runRow({ receipt: rec, ledger, runId });
|
|
196
194
|
|
|
197
195
|
// Hook decisions are checkout-wide, so they are FILTERED to this run rather than read from a
|
|
@@ -205,13 +203,12 @@ export function collectRun(cwd, slug) {
|
|
|
205
203
|
slug,
|
|
206
204
|
tables: {
|
|
207
205
|
run: run ? [run] : [],
|
|
208
|
-
dispatch, ac_result, discovery, file_touched,
|
|
206
|
+
dispatch, ac_result, discovery, file_touched,
|
|
209
207
|
trial: readJsonl(trialsPath(cwd, slug), t).map((r) => ({ ...r, run_id: r.run_id ?? runId ?? null })),
|
|
210
208
|
t0_verdict: readJsonDir(verdictsDir(cwd, slug), t).map((a) => t0Row(a, runId)),
|
|
211
209
|
criterion_verdict: criterionRows(evaluationDir(cwd, slug), runId, t),
|
|
212
210
|
hook_decision,
|
|
213
211
|
},
|
|
214
|
-
economics: economics({ agent_call, dispatch, run }),
|
|
215
212
|
defects: { records_skipped: t.skipped },
|
|
216
213
|
};
|
|
217
214
|
}
|
|
@@ -246,7 +243,6 @@ export function writeRun(collected, outDir, format = "jsonl") {
|
|
|
246
243
|
// The defect count is a first-class manifest field, not a log line. A short table with no
|
|
247
244
|
// record of why is indistinguishable from a short run.
|
|
248
245
|
records_skipped: collected.defects.records_skipped,
|
|
249
|
-
economics: collected.economics,
|
|
250
246
|
};
|
|
251
247
|
writeFileSync(join(outDir, "manifest.json"), JSON.stringify(manifest, null, 2) + "\n", "utf8");
|
|
252
248
|
return manifest;
|
package/kernel/report/facts.mjs
CHANGED
|
@@ -2,45 +2,31 @@
|
|
|
2
2
|
//
|
|
3
3
|
// WHY THIS FILE EXISTS.
|
|
4
4
|
//
|
|
5
|
-
// The pipeline already writes JSON at every boundary — an order in, a result out, a
|
|
6
|
-
// per
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
5
|
+
// The pipeline already writes JSON at every boundary — an order in, a result out, a decision row
|
|
6
|
+
// per hook evaluation, a trial row per T0 run. What it never had was a way to READ them together.
|
|
7
|
+
// Each record answers a question about itself; none of them answers "what did this run do",
|
|
8
|
+
// because that question needs a join and nothing on disk was joinable (see `mintRunId` in
|
|
9
|
+
// `lib/paths.mjs` for why).
|
|
10
10
|
//
|
|
11
11
|
// This module is the projection half. It takes parsed records and returns flat rows — a star
|
|
12
12
|
// schema whose grain is the DISPATCH, which is the finest unit the harness actually plans in:
|
|
13
|
-
// one compiled order, one worker, one result. Everything else
|
|
14
|
-
// (`
|
|
15
|
-
// `discovery`, `file_touched`).
|
|
13
|
+
// one compiled order, one worker, one result. Everything else hangs off it as a child table
|
|
14
|
+
// (`ac_result`, `discovery`, `file_touched`).
|
|
16
15
|
//
|
|
17
|
-
//
|
|
16
|
+
// FACTS ONLY — the rule ``harness probe stats`` states in its own header. Every field below is a
|
|
17
|
+
// count, a duration, a copied enum or an id. No field here is a score, a rate of quality, or a
|
|
18
|
+
// judgement, because a computed grade in the read plane is a second judge behind spec-evaluator
|
|
19
|
+
// and the architecture forbids one. `n_ac_fail` is a fact; "AC health" is not.
|
|
18
20
|
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
// spec-evaluator and the architecture forbids one. `n_ac_fail` is a fact; "AC health" is not.
|
|
23
|
-
//
|
|
24
|
-
// 2. NEVER FABRICATE A JOIN. The agent-call join (below) is real but partial, so every dispatch
|
|
25
|
-
// row carries `agent_join` naming HOW it was joined — or `null` when it wasn't. An analyst
|
|
26
|
-
// summing `cost_usd` must be able to see what share of dispatches had no cost row at all,
|
|
27
|
-
// because the alternative is a total that silently under-reports and looks authoritative.
|
|
28
|
-
// This is the same defect shape as `allow` with no receipt: an absent value and a zero value
|
|
29
|
-
// must not share a signature.
|
|
30
|
-
//
|
|
31
|
-
// THE AGENT-CALL JOIN, stated exactly. `journal.jsonl` is the only record carrying `cost_usd` and
|
|
32
|
-
// wall-clock, and its rows name no order — the launcher is generic and never parsed the prompt it
|
|
33
|
-
// was given. But the workflow's dispatch prompt asks the worker for a `result_path`, and that path
|
|
34
|
-
// is `<run root>/results/<stem>.json`, whose stem is the order id's suffix. So a journal row whose
|
|
35
|
-
// returned object carries `result_path` joins to exactly one order, deterministically, with no
|
|
36
|
-
// heuristics. Rows without it — the mechanical `set-active-order` couriers, a failed dispatch that
|
|
37
|
-
// returned nothing — join to no order and are counted as unattributed rather than dropped.
|
|
21
|
+
// A dispatch with no matching result is not dropped: `answered: false` says so, because an absent
|
|
22
|
+
// value and a zero value must not share a signature. This module carries no cost or wall-clock
|
|
23
|
+
// instrumentation — there is no run-scoped record of either to project.
|
|
38
24
|
|
|
39
25
|
/** Every fact table this module can produce, in dependency order. Exported so the writer, the
|
|
40
26
|
* manifest and the tests enumerate one list instead of three. */
|
|
41
27
|
export const TABLES = [
|
|
42
28
|
"run", "dispatch", "ac_result", "discovery", "file_touched",
|
|
43
|
-
"
|
|
29
|
+
"trial", "t0_verdict", "criterion_verdict", "hook_decision",
|
|
44
30
|
];
|
|
45
31
|
|
|
46
32
|
/** Coerce anything to a finite number, or null. Keeps `0` and rejects `NaN`/`""`/undefined. */
|
|
@@ -84,18 +70,6 @@ export function parseOrderStem(orderId) {
|
|
|
84
70
|
return { scope_id: null, round: null, attempt: null };
|
|
85
71
|
}
|
|
86
72
|
|
|
87
|
-
/**
|
|
88
|
-
* The order stem a journal row refers to, via the `result_path` its schema'd reply carries.
|
|
89
|
-
* @param {object} row - One `journal.jsonl` row.
|
|
90
|
-
* @returns {(string|null)} The stem, or null when the row named no result path.
|
|
91
|
-
*/
|
|
92
|
-
export function journalOrderStem(row) {
|
|
93
|
-
const p = row?.result?.result_path;
|
|
94
|
-
if (typeof p !== "string" || !p) return null;
|
|
95
|
-
const base = p.split(/[/\\]/).pop() || "";
|
|
96
|
-
return base.replace(/\.json$/i, "") || null;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
73
|
/**
|
|
100
74
|
* Project the run dimension — one row, the thing every fact table's `run_id` points at.
|
|
101
75
|
* @param {object} o - Sources (destructured):
|
|
@@ -132,35 +106,6 @@ export function runRow({ receipt, ledger = null, runId = null }) {
|
|
|
132
106
|
};
|
|
133
107
|
}
|
|
134
108
|
|
|
135
|
-
/**
|
|
136
|
-
* Project one agent-call row from a journal row.
|
|
137
|
-
* @param {object} row - One `journal.jsonl` row.
|
|
138
|
-
* @param {(string|null)} runId - The run key to stamp when the row itself carries none.
|
|
139
|
-
* @returns {object} A flat `agent_call` fact row.
|
|
140
|
-
*/
|
|
141
|
-
export function agentCallRow(row, runId = null) {
|
|
142
|
-
const sessions = Array.isArray(row?.sessions) ? row.sessions : [];
|
|
143
|
-
return {
|
|
144
|
-
run_id: row?.run_id ?? runId ?? null,
|
|
145
|
-
seq: num(row?.seq),
|
|
146
|
-
phase: row?.phase ?? null,
|
|
147
|
-
label: row?.label ?? null,
|
|
148
|
-
model: row?.model ?? null,
|
|
149
|
-
permission_mode: row?.permission_mode ?? null,
|
|
150
|
-
started_at: row?.started_at ?? null,
|
|
151
|
-
wall_ms: num(row?.wall_ms),
|
|
152
|
-
// The launcher retries a schema-invalid reply once, so `attempts` > 1 is a fact about the
|
|
153
|
-
// WORKER's compliance and is kept distinct from `ok`, which is about the final outcome.
|
|
154
|
-
attempts: num(row?.attempts),
|
|
155
|
-
ok: typeof row?.ok === "boolean" ? row.ok : null,
|
|
156
|
-
cost_usd: sumOrNull(sessions.map((s) => s?.cost_usd)),
|
|
157
|
-
sessions: sessions.length,
|
|
158
|
-
errored: sessions.some((s) => s?.is_error === true),
|
|
159
|
-
killed: sessions.some((s) => s?.killed === true),
|
|
160
|
-
order_stem: journalOrderStem(row),
|
|
161
|
-
};
|
|
162
|
-
}
|
|
163
|
-
|
|
164
109
|
/**
|
|
165
110
|
* Project the dispatch fact table and its three child tables.
|
|
166
111
|
*
|
|
@@ -170,24 +115,14 @@ export function agentCallRow(row, runId = null) {
|
|
|
170
115
|
* @param {object} o - Sources (destructured):
|
|
171
116
|
* @param {Array<object>} o.orders - Parsed WorkOrders.
|
|
172
117
|
* @param {Array<object>} [o.results] - Parsed WorkResults; joined on `order_id`.
|
|
173
|
-
* @param {Array<object>} [o.journal] - Parsed journal rows; joined on the result-path stem.
|
|
174
118
|
* @param {(string|null)} [o.runId] - Run key for orders that carry none (pre-v1.8 traces).
|
|
175
119
|
* @returns {{dispatch:Array<object>, ac_result:Array<object>, discovery:Array<object>,
|
|
176
120
|
* file_touched:Array<object>}} The fact table and its children, each row already carrying
|
|
177
121
|
* `run_id` + `order_id` so every table stands alone in the warehouse.
|
|
178
122
|
*/
|
|
179
|
-
export function dispatchFacts({ orders, results = [],
|
|
123
|
+
export function dispatchFacts({ orders, results = [], runId = null }) {
|
|
180
124
|
const byOrderId = new Map();
|
|
181
125
|
for (const r of results) if (r?.order_id) byOrderId.set(r.order_id, r);
|
|
182
|
-
const byStem = new Map();
|
|
183
|
-
for (const j of journal) {
|
|
184
|
-
const stem = journalOrderStem(j);
|
|
185
|
-
// Last write wins: a dispatch retried after a failure legitimately produces two journal rows
|
|
186
|
-
// for one order, and the one that finished it is the one whose cost the dispatch carries. The
|
|
187
|
-
// discarded row is NOT lost — it remains its own `agent_call` row, so the retry is still
|
|
188
|
-
// visible and the two totals differ by exactly the retries.
|
|
189
|
-
if (stem) byStem.set(stem, j);
|
|
190
|
-
}
|
|
191
126
|
|
|
192
127
|
const dispatch = [], ac_result = [], discovery = [], file_touched = [];
|
|
193
128
|
|
|
@@ -198,8 +133,6 @@ export function dispatchFacts({ orders, results = [], journal = [], runId = null
|
|
|
198
133
|
const stem = orderStem(id);
|
|
199
134
|
const { scope_id, round, attempt } = parseOrderStem(id);
|
|
200
135
|
const result = byOrderId.get(id) || null;
|
|
201
|
-
const call = stem ? byStem.get(stem) || null : null;
|
|
202
|
-
const agent = call ? agentCallRow(call, rid) : null;
|
|
203
136
|
const taskResults = Array.isArray(result?.task_results) ? result.task_results : [];
|
|
204
137
|
const discoveries = Array.isArray(result?.discoveries) ? result.discoveries : [];
|
|
205
138
|
const filesTouched = Array.isArray(result?.files_touched) ? result.files_touched : [];
|
|
@@ -270,78 +203,7 @@ export function dispatchFacts({ orders, results = [], journal = [], runId = null
|
|
|
270
203
|
verdict_overall: result?.verdict?.overall ?? null,
|
|
271
204
|
assumptions: Array.isArray(result?.assumptions) ? result.assumptions.length : 0,
|
|
272
205
|
deviations: Array.isArray(result?.deviations) ? result.deviations.length : 0,
|
|
273
|
-
// The agent-call leg. Present only where the join held; `agent_join` says which.
|
|
274
|
-
agent_seq: agent?.seq ?? null,
|
|
275
|
-
model: agent?.model ?? null,
|
|
276
|
-
wall_ms: agent?.wall_ms ?? null,
|
|
277
|
-
cost_usd: agent?.cost_usd ?? null,
|
|
278
|
-
agent_attempts: agent?.attempts ?? null,
|
|
279
|
-
agent_ok: agent?.ok ?? null,
|
|
280
|
-
agent_join: agent ? "result_path" : null,
|
|
281
206
|
});
|
|
282
207
|
}
|
|
283
208
|
return { dispatch, ac_result, discovery, file_touched };
|
|
284
209
|
}
|
|
285
|
-
|
|
286
|
-
/**
|
|
287
|
-
* The run-economics projection — measurement-table row 4, computed from records the harness
|
|
288
|
-
* already writes.
|
|
289
|
-
*
|
|
290
|
-
* Every field is a count, a sum or a duration over recorded rows. Nothing here is normalised
|
|
291
|
-
* against a baseline, because no baseline dataset exists: this reports what a run cost, never
|
|
292
|
-
* whether that was good, which would be a grade.
|
|
293
|
-
*
|
|
294
|
-
* @param {object} o - Sources (destructured):
|
|
295
|
-
* @param {Array<object>} o.agent_call - Rows from {@link agentCallRow}.
|
|
296
|
-
* @param {Array<object>} [o.dispatch] - Rows from {@link dispatchFacts}; supplies the first write.
|
|
297
|
-
* @param {(object|null)} [o.run] - The run row; supplies `started_at` for the latency measures.
|
|
298
|
-
* @returns {object} `{agent_calls, cost_usd, cost_attributed_usd, cost_unattributed_usd,
|
|
299
|
-
* wall_ms_total, calls_to_first_write, seconds_to_first_write, by_model[], retried_calls,
|
|
300
|
-
* failed_calls, dispatches, dispatches_answered, dispatches_costed}` — with nulls, never zeros,
|
|
301
|
-
* wherever the underlying record was absent.
|
|
302
|
-
*/
|
|
303
|
-
export function economics({ agent_call, dispatch = [], run = null }) {
|
|
304
|
-
const calls = Array.isArray(agent_call) ? agent_call : [];
|
|
305
|
-
const wroteBy = new Set(dispatch.filter((d) => (d.files_touched || 0) > 0).map((d) => d.stem));
|
|
306
|
-
|
|
307
|
-
// "Turns to first write" — the harness's own definition of the metric the design doc names but
|
|
308
|
-
// has never had an instrument for. Measured in AGENT CALLS, because a call is the unit that
|
|
309
|
-
// costs money; the seconds figure is reported beside it so a run that is slow and a run that is
|
|
310
|
-
// chatty stay distinguishable.
|
|
311
|
-
const ordered = [...calls].sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0));
|
|
312
|
-
const firstWriteIdx = ordered.findIndex((c) => c.order_stem && wroteBy.has(c.order_stem));
|
|
313
|
-
const firstWrite = firstWriteIdx === -1 ? null : ordered[firstWriteIdx];
|
|
314
|
-
const startMs = run?.started_at ? Date.parse(run.started_at) : NaN;
|
|
315
|
-
const firstMs = firstWrite?.started_at ? Date.parse(firstWrite.started_at) : NaN;
|
|
316
|
-
|
|
317
|
-
const byModel = new Map();
|
|
318
|
-
for (const c of calls) {
|
|
319
|
-
const k = c.model || "(unnamed)";
|
|
320
|
-
const m = byModel.get(k) || { model: k, calls: 0, cost_usd: null, wall_ms: null };
|
|
321
|
-
m.calls++;
|
|
322
|
-
if (num(c.cost_usd) !== null) m.cost_usd = (m.cost_usd ?? 0) + c.cost_usd;
|
|
323
|
-
if (num(c.wall_ms) !== null) m.wall_ms = (m.wall_ms ?? 0) + c.wall_ms;
|
|
324
|
-
byModel.set(k, m);
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
const costed = new Set(dispatch.filter((d) => num(d.cost_usd) !== null).map((d) => d.order_id));
|
|
328
|
-
return {
|
|
329
|
-
agent_calls: calls.length,
|
|
330
|
-
cost_usd: sumOrNull(calls.map((c) => c.cost_usd)),
|
|
331
|
-
// The unattributed share is reported, never hidden: it is what the dispatch table's cost
|
|
332
|
-
// column is MISSING, and an analyst who cannot see it will read a partial total as a full one.
|
|
333
|
-
cost_attributed_usd: sumOrNull(calls.filter((c) => c.order_stem).map((c) => c.cost_usd)),
|
|
334
|
-
cost_unattributed_usd: sumOrNull(calls.filter((c) => !c.order_stem).map((c) => c.cost_usd)),
|
|
335
|
-
wall_ms_total: sumOrNull(calls.map((c) => c.wall_ms)),
|
|
336
|
-
calls_to_first_write: firstWriteIdx === -1 ? null : firstWriteIdx + 1,
|
|
337
|
-
seconds_to_first_write: Number.isFinite(startMs) && Number.isFinite(firstMs)
|
|
338
|
-
? Math.round((firstMs - startMs) / 1000) : null,
|
|
339
|
-
retried_calls: calls.filter((c) => (c.attempts ?? 0) > 1).length,
|
|
340
|
-
failed_calls: calls.filter((c) => c.ok === false).length,
|
|
341
|
-
killed_calls: calls.filter((c) => c.killed).length,
|
|
342
|
-
dispatches: dispatch.length,
|
|
343
|
-
dispatches_answered: dispatch.filter((d) => d.answered).length,
|
|
344
|
-
dispatches_costed: costed.size,
|
|
345
|
-
by_model: [...byModel.values()].sort((a, b) => a.model.localeCompare(b.model)),
|
|
346
|
-
};
|
|
347
|
-
}
|
package/package.json
CHANGED
|
@@ -436,17 +436,19 @@ S.6 Harvest one signal row → append to `.shapeup/metrics/<machine-id>.jsonl`
|
|
|
436
436
|
S.7 Export the run's records → one keyed dataset, before the trace is superseded.
|
|
437
437
|
node "${CLAUDE_PLUGIN_ROOT}/kernel/harness.mjs" report export --slug <slug>
|
|
438
438
|
Same argument as S.6, applied to the records the harvest row does NOT carry: orders,
|
|
439
|
-
results,
|
|
440
|
-
|
|
439
|
+
results, T0 verdicts, trial rows, criterion verdicts and this run's hook decisions all
|
|
440
|
+
live in the LOCAL tier, which is regenerable and gets wiped.
|
|
441
441
|
The export freezes them as fact tables under `.shapeup/exports/<run_id>/` (JSONL, one
|
|
442
442
|
object per line), keyed by run id so a second run of the same feature is a second
|
|
443
443
|
dataset rather than an overwrite. `--out <dir>` sends it somewhere durable instead.
|
|
444
444
|
It is READ-ONLY: it writes nothing into the trace, so it may be re-run at any time.
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
445
|
+
NOT COST/WALL-CLOCK. This harness carries no run-economics record — the design once
|
|
446
|
+
called for one derived from a per-agent-call journal, but nothing in the pipeline ever
|
|
447
|
+
wrote that journal, so the derivation and its reporting command were removed rather than
|
|
448
|
+
left presenting nulls as measurements. Even a real cost record would not belong in the
|
|
449
|
+
harvest row's schema either way — that row's contract rejects clock fields on purpose
|
|
450
|
+
(see protocol.md "Rejected fields"), because a signal feed that carried a duration would
|
|
451
|
+
become a velocity feed on the next person who read it.
|
|
450
452
|
```
|
|
451
453
|
|
|
452
454
|
---
|
|
@@ -816,10 +816,10 @@ fixtures run in isolation and do not consume it.
|
|
|
816
816
|
drives both toward 0 — measured from the build trace, no manual grading.
|
|
817
817
|
- **Rejected fields:** `time_spent` / velocity (no clock; Shape Up forbids counting hours
|
|
818
818
|
— `round_count` is the legitimate effort proxy) and `run_quality_score` (second judge).
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
819
|
+
Even a real cost or wall-clock record would not belong here: a signal feed that carried a
|
|
820
|
+
duration would become a velocity feed on the next person who read it. This harness does not
|
|
821
|
+
currently carry one — the design called for one derived from a per-agent-call journal, but
|
|
822
|
+
nothing in the pipeline ever writes it, so there is no such record to reject or admit.
|
|
823
823
|
|
|
824
824
|
### Row template
|
|
825
825
|
```json
|
|
@@ -992,6 +992,20 @@ if (!rs.has_spec_tree) {
|
|
|
992
992
|
// ---- WIRE + GATE L1a.5 ------------------------------------------------------------------------
|
|
993
993
|
phase("Wire");
|
|
994
994
|
if (!rs.has_wiring_map) {
|
|
995
|
+
// FAIL FAST, at the orchestrator: `rs` already carries has_project_profile from the resume
|
|
996
|
+
// snapshot taken before ORIENT — the SAME fact solution-architect would have to discover for
|
|
997
|
+
// itself. Dispatching without it spends a full worker turn only for the worker to hit its own
|
|
998
|
+
// documented rule ("profile absent ⇒ ESCALATE, do not invent an entry point" — solution-
|
|
999
|
+
// architect/SKILL.md) and escalate; an escalation writes no wiring-map.md, so `has_wiring_map`
|
|
1000
|
+
// stays false and every relaunch re-dispatches and re-escalates identically. The orchestrator
|
|
1001
|
+
// holds the state a gate needs; it should not hand the check to the LLM it is about to pay for.
|
|
1002
|
+
if (!rs.has_project_profile) {
|
|
1003
|
+
return withWarnings(aborted("WIRE",
|
|
1004
|
+
`missing SHARED project-profile.md at ${rs.project_profile_path} — GATE L0 writes it ` +
|
|
1005
|
+
`({schema_version:1, archetype, entry_point}; references/gates.md GATE L0 §PROFILE) before ` +
|
|
1006
|
+
`this workflow launches. WIRE cannot resolve an entry_call_site without an entry_point to ` +
|
|
1007
|
+
`resolve against. Write the profile, then relaunch.`));
|
|
1008
|
+
}
|
|
995
1009
|
log(`WIRE — dispatching (slug ${slug})`);
|
|
996
1010
|
const w = await worker({
|
|
997
1011
|
skill: "solution-architect", operation: "wire", schema: PHASE_OK, phase: "Wire", label: "wire",
|