pi-workflow-engine 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/pi-workflow-engine/index.ts +255 -97
- package/.pi/extensions/pi-workflow-engine/src/advisory-challenge.ts +75 -0
- package/.pi/extensions/pi-workflow-engine/src/advisory-evidence.ts +90 -0
- package/.pi/extensions/pi-workflow-engine/src/advisory-schema.ts +13 -5
- package/.pi/extensions/pi-workflow-engine/src/agent-attempt.ts +3 -1
- package/.pi/extensions/pi-workflow-engine/src/agent-limits.ts +35 -0
- package/.pi/extensions/pi-workflow-engine/src/agent-replay.ts +27 -4
- package/.pi/extensions/pi-workflow-engine/src/agent-retry.ts +116 -0
- package/.pi/extensions/pi-workflow-engine/src/agent-runner-types.ts +49 -45
- package/.pi/extensions/pi-workflow-engine/src/agent-runner.ts +95 -26
- package/.pi/extensions/pi-workflow-engine/src/agent-session-providers.ts +50 -0
- package/.pi/extensions/pi-workflow-engine/src/agent-session.ts +158 -203
- package/.pi/extensions/pi-workflow-engine/src/agent-skills.ts +48 -42
- package/.pi/extensions/pi-workflow-engine/src/agent-workspace.ts +26 -4
- package/.pi/extensions/pi-workflow-engine/src/background-workflow-tool.ts +75 -0
- package/.pi/extensions/pi-workflow-engine/src/background-workflows.ts +492 -0
- package/.pi/extensions/pi-workflow-engine/src/budget.ts +3 -3
- package/.pi/extensions/pi-workflow-engine/src/cancellation.ts +38 -0
- package/.pi/extensions/pi-workflow-engine/src/command-completions.ts +36 -0
- package/.pi/extensions/pi-workflow-engine/src/concurrency.ts +82 -10
- package/.pi/extensions/pi-workflow-engine/src/diff-capture.ts +16 -130
- package/.pi/extensions/pi-workflow-engine/src/discovery.ts +9 -5
- package/.pi/extensions/pi-workflow-engine/src/dynamax.ts +109 -60
- package/.pi/extensions/pi-workflow-engine/src/engine.ts +166 -18
- package/.pi/extensions/pi-workflow-engine/src/filesystem-error.ts +4 -0
- package/.pi/extensions/pi-workflow-engine/src/inline-workflow.ts +2 -2
- package/.pi/extensions/pi-workflow-engine/src/journal.ts +13 -6
- package/.pi/extensions/pi-workflow-engine/src/model-profile-command.ts +167 -0
- package/.pi/extensions/pi-workflow-engine/src/model-profiles.ts +310 -0
- package/.pi/extensions/pi-workflow-engine/src/options.ts +88 -1
- package/.pi/extensions/pi-workflow-engine/src/pi-compat.ts +32 -0
- package/.pi/extensions/pi-workflow-engine/src/progress-types.ts +50 -0
- package/.pi/extensions/pi-workflow-engine/src/progress.ts +72 -117
- package/.pi/extensions/pi-workflow-engine/src/provider-usage-limit.ts +189 -0
- package/.pi/extensions/pi-workflow-engine/src/research-contract.ts +89 -0
- package/.pi/extensions/pi-workflow-engine/src/research-evidence.ts +289 -0
- package/.pi/extensions/pi-workflow-engine/src/resume-context.ts +127 -41
- package/.pi/extensions/pi-workflow-engine/src/review/code-review-orchestration.ts +24 -0
- package/.pi/extensions/pi-workflow-engine/src/review/patch-validation.ts +120 -0
- package/.pi/extensions/pi-workflow-engine/src/review/review-fix-workflow.ts +43 -7
- package/.pi/extensions/pi-workflow-engine/src/review/review-format.ts +2 -1
- package/.pi/extensions/pi-workflow-engine/src/review/review-issues.ts +1 -1
- package/.pi/extensions/pi-workflow-engine/src/review/review-report.ts +1 -1
- package/.pi/extensions/pi-workflow-engine/src/review/review-results-flow.ts +2 -1
- package/.pi/extensions/pi-workflow-engine/src/review/review-results-viewer.ts +11 -30
- package/.pi/extensions/pi-workflow-engine/src/review/review-session-coordinator.ts +5 -5
- package/.pi/extensions/pi-workflow-engine/src/review/review-snapshot.ts +3 -4
- package/.pi/extensions/pi-workflow-engine/src/review-diff-target.ts +130 -0
- package/.pi/extensions/pi-workflow-engine/src/session-identity.ts +6 -0
- package/.pi/extensions/pi-workflow-engine/src/structured-output.ts +32 -0
- package/.pi/extensions/pi-workflow-engine/src/tool-capabilities.ts +44 -0
- package/.pi/extensions/pi-workflow-engine/src/tool-source-identity.ts +1 -4
- package/.pi/extensions/pi-workflow-engine/src/tree-fingerprint.ts +1 -4
- package/.pi/extensions/pi-workflow-engine/src/types.ts +52 -15
- package/.pi/extensions/pi-workflow-engine/src/ui/dynamax-editor-decoration.ts +220 -0
- package/.pi/extensions/pi-workflow-engine/src/ui/workflow-format.ts +26 -6
- package/.pi/extensions/pi-workflow-engine/src/ui/workflow-inspector.ts +88 -46
- package/.pi/extensions/pi-workflow-engine/src/ui/workflow-result-renderer.ts +7 -3
- package/.pi/extensions/pi-workflow-engine/src/ui/workflow-viewer-layout.ts +51 -0
- package/.pi/extensions/pi-workflow-engine/src/ui/workflow-widget.ts +12 -67
- package/.pi/extensions/pi-workflow-engine/src/usage.ts +124 -16
- package/.pi/extensions/pi-workflow-engine/src/workflow-advisory-utils.ts +151 -178
- package/.pi/extensions/pi-workflow-engine/src/workflow-execution.ts +1 -1
- package/.pi/extensions/pi-workflow-engine/src/workflow-run-background.ts +72 -0
- package/.pi/extensions/pi-workflow-engine/src/workflow-run-controller.ts +354 -0
- package/.pi/extensions/pi-workflow-engine/src/workflow-run-history.ts +162 -0
- package/.pi/extensions/pi-workflow-engine/src/workflow-run-record.ts +632 -0
- package/.pi/extensions/pi-workflow-engine/src/workflow-run-store.ts +172 -0
- package/.pi/extensions/pi-workflow-engine/src/workflow-usage-limit-scheduler.ts +80 -0
- package/.pi/extensions/pi-workflow-engine/src/workflows.ts +2 -0
- package/.pi/extensions/pi-workflow-engine/src/worktree.ts +47 -3
- package/.pi/extensions/pi-workflow-engine/workflows/code-review.ts +65 -162
- package/.pi/extensions/pi-workflow-engine/workflows/diagnose.ts +48 -149
- package/.pi/extensions/pi-workflow-engine/workflows/perf-review.ts +28 -77
- package/.pi/extensions/pi-workflow-engine/workflows/refactor-scout.ts +28 -77
- package/.pi/extensions/pi-workflow-engine/workflows/research.ts +170 -0
- package/README.md +72 -141
- package/USAGE.md +270 -17
- package/package.json +1 -1
|
@@ -1,29 +1,24 @@
|
|
|
1
|
+
import { challengeFindings, parseChallengeArgs } from "../src/advisory-challenge.ts";
|
|
1
2
|
import { Type } from "typebox";
|
|
2
3
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
type AdvisoryVerdict,
|
|
9
|
-
} from "../src/advisory-schema.ts";
|
|
10
|
-
import {
|
|
11
|
-
advisoryDedupKey as dedupKey,
|
|
12
|
-
backfillAdvisoryFindings,
|
|
4
|
+
type AdvisoryVerified,
|
|
5
|
+
type AdvisoryLens,
|
|
6
|
+
synthesizeAdvisoryReport,
|
|
7
|
+
finishAdvisoryReport,
|
|
8
|
+
emptyAdvisoryReport,
|
|
13
9
|
formatEvidence,
|
|
14
10
|
formatLocation,
|
|
15
|
-
|
|
16
|
-
|
|
11
|
+
publishVerifiedKeptProgress,
|
|
12
|
+
runLensVerificationPipeline,
|
|
17
13
|
DEFAULT_ADVISORY_TOOL_HINTS,
|
|
18
14
|
DEFAULT_ADVISORY_TOOLS,
|
|
19
15
|
} from "../src/workflow-advisory-utils.ts";
|
|
20
|
-
import { compactResults } from "../src/concurrency.ts";
|
|
21
16
|
import type { WorkflowApi, WorkflowMeta, WorkflowRunStats } from "../src/types.ts";
|
|
22
17
|
|
|
23
18
|
export const meta: WorkflowMeta = {
|
|
24
19
|
name: "diagnose",
|
|
25
20
|
description: "Advisory-only bug diagnosis: scope symptoms → competing hypotheses → independent verify → synthesize likely root causes.",
|
|
26
|
-
phases: [{ title: "Scope" }, { title: "Hypothesize" }, { title: "Verify" }, { title: "Synthesize" }],
|
|
21
|
+
phases: [{ title: "Scope" }, { title: "Hypothesize" }, { title: "Verify" }, { title: "Challenge" }, { title: "Synthesize" }],
|
|
27
22
|
};
|
|
28
23
|
|
|
29
24
|
const ScopeSchema = Type.Object({
|
|
@@ -34,25 +29,7 @@ const ScopeSchema = Type.Object({
|
|
|
34
29
|
constraints: Type.Optional(Type.String({ description: "Safety constraints, missing evidence, or commands intentionally not run." })),
|
|
35
30
|
});
|
|
36
31
|
|
|
37
|
-
|
|
38
|
-
label: string;
|
|
39
|
-
category: string;
|
|
40
|
-
text: string;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
type Candidate = AdvisoryCandidate;
|
|
44
|
-
|
|
45
|
-
interface Hypothesis extends Candidate {
|
|
46
|
-
lens: HypothesisLens;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
interface Verified extends Hypothesis {
|
|
50
|
-
verdict: AdvisoryVerdict["verdict"];
|
|
51
|
-
evidence: string[];
|
|
52
|
-
confidence?: AdvisoryVerdict["confidence"];
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
const HYPOTHESIS_LENSES: HypothesisLens[] = [
|
|
32
|
+
const HYPOTHESIS_LENSES: AdvisoryLens[] = [
|
|
56
33
|
{ label: "recent-change", category: "regression", text: "A recent code change broke a previously working path or changed an implicit contract." },
|
|
57
34
|
{ label: "control-flow", category: "root-cause", text: "Incorrect branching, ordering, async flow, data flow, or state transition causes the symptom." },
|
|
58
35
|
{ label: "configuration", category: "configuration", text: "Configuration, environment, package scripts, or runtime assumptions differ from what the code expects." },
|
|
@@ -60,13 +37,12 @@ const HYPOTHESIS_LENSES: HypothesisLens[] = [
|
|
|
60
37
|
{ label: "test-fixture", category: "test-fixture", text: "The failure is caused by test setup, fixtures, mocks, generated files, or stale local state rather than product code." },
|
|
61
38
|
];
|
|
62
39
|
|
|
63
|
-
const TOOLS = DEFAULT_ADVISORY_TOOLS;
|
|
64
|
-
const TOOL_HINTS = DEFAULT_ADVISORY_TOOL_HINTS;
|
|
65
40
|
const PER_LENS = 4;
|
|
66
41
|
|
|
67
42
|
export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
68
|
-
const { agent,
|
|
69
|
-
const
|
|
43
|
+
const { agent, phase, log, progress, args } = api;
|
|
44
|
+
const challengeConfig = parseChallengeArgs(args);
|
|
45
|
+
const symptom = challengeConfig.args.trim();
|
|
70
46
|
let fileCount = 0;
|
|
71
47
|
let rawCandidateCount = 0;
|
|
72
48
|
let droppedCandidateCount = 0;
|
|
@@ -89,15 +65,15 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
89
65
|
"Inspect relevant files, package/test configuration, and safe diagnostic commands. " +
|
|
90
66
|
"Safe commands are read-only commands such as status, grep, listing files, typecheck/test commands, or commands explicitly requested by the user. " +
|
|
91
67
|
"Do not run mutation, install, commit, network, or destructive commands. Return scoped files, observations, and constraints. Structured output only.",
|
|
92
|
-
{ phase: "Scope", label: "scope", tools:
|
|
68
|
+
{ phase: "Scope", label: "scope", tools: DEFAULT_ADVISORY_TOOLS, toolHints: DEFAULT_ADVISORY_TOOL_HINTS, profile: "medium", schema: ScopeSchema },
|
|
93
69
|
);
|
|
94
70
|
|
|
95
71
|
if (!scope) {
|
|
96
|
-
return
|
|
72
|
+
return finishAdvisoryReport(emptyAdvisoryReport(
|
|
97
73
|
"Diagnosis could not establish a scope.",
|
|
98
74
|
["Provide the failing command, error message, or regression description and rerun diagnose."],
|
|
99
75
|
makeStats(0, 0),
|
|
100
|
-
);
|
|
76
|
+
), []);
|
|
101
77
|
}
|
|
102
78
|
|
|
103
79
|
fileCount = scope.files.length;
|
|
@@ -112,92 +88,48 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
112
88
|
`## Observations\n${scope.observations.map((observation) => `- ${observation}`).join("\n") || "(none)"}\n\n` +
|
|
113
89
|
`## Constraints\n${scope.constraints ?? "(none noted)"}\n`;
|
|
114
90
|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
title: candidate.summary,
|
|
134
|
-
subtitle: formatLocation(candidate),
|
|
135
|
-
status: "pending",
|
|
136
|
-
details: candidate.impact,
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
|
-
return candidates;
|
|
140
|
-
}),
|
|
141
|
-
);
|
|
142
|
-
|
|
143
|
-
const hypotheses = dedupe(compactResults(perLens).flat(), (dropped) => {
|
|
144
|
-
droppedCandidateCount += dropped;
|
|
145
|
-
progress({ type: "counter_delta", key: "dropped", label: "dropped", delta: dropped });
|
|
91
|
+
const pipelineResult = await runLensVerificationPipeline({
|
|
92
|
+
api,
|
|
93
|
+
lenses: HYPOTHESIS_LENSES,
|
|
94
|
+
perLens: PER_LENS,
|
|
95
|
+
finderPhase: "Hypothesize",
|
|
96
|
+
finderPrompt: (lens) =>
|
|
97
|
+
`## Diagnose hypothesis generator — ${lens.label}\n\n${scopeBlock}\n` +
|
|
98
|
+
"This workflow is advisory-only: diagnose and recommend validation/fix plans, but do not edit files.\n" +
|
|
99
|
+
`Consider ONLY this hypothesis lens:\n${lens.text}\n\n` +
|
|
100
|
+
`Surface up to ${PER_LENS} root-cause hypotheses. Use category exactly "${lens.category}". ` +
|
|
101
|
+
"Each hypothesis must include a one-line summary, locations, impact explaining how it produces the symptom, and an optional recommendation for the next validation step. Structured output only.",
|
|
102
|
+
verifierPrompt: (candidate) =>
|
|
103
|
+
`## Diagnose verifier\n\n${scopeBlock}\n## Hypothesis\n` +
|
|
104
|
+
`Location: ${formatLocation(candidate)}\nCategory: ${candidate.category}\nSummary: ${candidate.summary}\nImpact: ${candidate.impact}\n` +
|
|
105
|
+
`Recommended validation: ${candidate.recommendation ?? "(none supplied)"}\n\n` +
|
|
106
|
+
"Read relevant files and, when useful, run only safe read-only diagnostic commands from the scoped command list or commands explicitly requested by the user. " +
|
|
107
|
+
"Do not run mutation, install, commit, network, or destructive commands. Return CONFIRMED, PLAUSIBLE, NOT_SUBSTANTIATED, or REFUTED with evidence. " +
|
|
108
|
+
"Use NOT_SUBSTANTIATED when evidence is missing; REFUTED requires disproof. Structured output only.",
|
|
146
109
|
});
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
const location = primaryLocation(hypothesis);
|
|
153
|
-
const judged = await agent(
|
|
154
|
-
`## Diagnose verifier\n\n${scopeBlock}\n## Hypothesis\n` +
|
|
155
|
-
`Location: ${formatLocation(hypothesis)}\nCategory: ${hypothesis.category}\nSummary: ${hypothesis.summary}\nImpact: ${hypothesis.impact}\n` +
|
|
156
|
-
`Recommended validation: ${hypothesis.recommendation ?? "(none supplied)"}\n\n` +
|
|
157
|
-
"Read relevant files and, when useful, run only safe read-only diagnostic commands from the scoped command list or commands explicitly requested by the user. " +
|
|
158
|
-
"Do not run mutation, install, commit, network, or destructive commands. Return CONFIRMED, PLAUSIBLE, or REFUTED with evidence. " +
|
|
159
|
-
"Default toward REFUTED if evidence does not connect the hypothesis to the symptom. Structured output only.",
|
|
160
|
-
{
|
|
161
|
-
phase: "Verify",
|
|
162
|
-
label: `verify:${location.file.split("/").pop() ?? location.file}`,
|
|
163
|
-
tools: TOOLS,
|
|
164
|
-
toolHints: TOOL_HINTS,
|
|
165
|
-
thinkingLevel: "low",
|
|
166
|
-
schema: AdvisoryVerdictSchema,
|
|
167
|
-
},
|
|
168
|
-
);
|
|
169
|
-
if (!judged) return null;
|
|
170
|
-
recordVerdictProgress(progress, hypothesis, judged, () => {
|
|
171
|
-
refutedCandidateCount += 1;
|
|
172
|
-
});
|
|
173
|
-
return { ...hypothesis, verdict: judged.verdict, evidence: judged.evidence, confidence: judged.confidence };
|
|
174
|
-
}),
|
|
175
|
-
),
|
|
176
|
-
);
|
|
177
|
-
|
|
110
|
+
rawCandidateCount += pipelineResult.rawCandidates;
|
|
111
|
+
droppedCandidateCount += pipelineResult.dropped;
|
|
112
|
+
refutedCandidateCount += pipelineResult.refuted;
|
|
113
|
+
const { coverage } = pipelineResult;
|
|
114
|
+
const verified = await challengeFindings(api, pipelineResult.verified, scopeBlock, challengeConfig.options, coverage);
|
|
178
115
|
const surviving = verified.filter((finding) => finding.verdict !== "REFUTED");
|
|
179
116
|
const refuted = verified.filter((finding) => finding.verdict === "REFUTED");
|
|
180
117
|
const stats = makeStats(verified.length, surviving.length);
|
|
181
|
-
|
|
182
|
-
progress({ type: "counter", key: "kept", label: "kept", value: surviving.length });
|
|
183
|
-
progress({ type: "summary", key: "verified", value: verified.length });
|
|
184
|
-
progress({ type: "summary", key: "kept", value: surviving.length });
|
|
185
|
-
log(`${verified.length} verified → ${surviving.length} kept`);
|
|
118
|
+
publishVerifiedKeptProgress({ progress, log }, verified.length, surviving.length);
|
|
186
119
|
|
|
187
120
|
if (surviving.length === 0) {
|
|
188
|
-
return
|
|
121
|
+
return finishAdvisoryReport(emptyAdvisoryReport(
|
|
189
122
|
"No root-cause hypothesis survived verification.",
|
|
190
123
|
["Capture the exact failing command and error output.", "Rerun diagnose with a narrower symptom or more evidence."],
|
|
191
124
|
stats,
|
|
192
|
-
);
|
|
125
|
+
), coverage, verified);
|
|
193
126
|
}
|
|
194
127
|
|
|
195
|
-
phase("Synthesize");
|
|
196
128
|
const ranked = [...surviving].sort((a, b) => rank(a) - rank(b));
|
|
197
129
|
const block = ranked
|
|
198
130
|
.map(
|
|
199
131
|
(finding, index) =>
|
|
200
|
-
`### [${index}] ${formatLocation(finding)} (${finding.verdict}, ${finding.category})\n` +
|
|
132
|
+
`### [${index}] IDs: ${finding.sourceCandidateIds.join(", ")} ${formatLocation(finding)} (${finding.verdict}, ${finding.category})\n` +
|
|
201
133
|
`${finding.summary}\nImpact: ${finding.impact}\nEvidence: ${formatEvidence(finding.evidence)}\nValidation/fix plan: ${finding.recommendation ?? "(none supplied)"}`,
|
|
202
134
|
)
|
|
203
135
|
.join("\n\n");
|
|
@@ -206,50 +138,17 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
206
138
|
.map((finding) => `- ${finding.summary} — REFUTED because ${formatEvidence(finding.evidence)}`)
|
|
207
139
|
.join("\n");
|
|
208
140
|
|
|
209
|
-
const
|
|
141
|
+
const resolved = await synthesizeAdvisoryReport(api,
|
|
210
142
|
`## Synthesis: final diagnosis report\n\n${ranked.length} hypotheses survived independent verification.\n\n${block}\n\n` +
|
|
211
143
|
`## Refuted hypotheses for context\n${refutedBlock || "(none recorded)"}\n\n` +
|
|
212
|
-
"
|
|
213
|
-
"Use categories such as root-cause, regression, configuration, dependency, or test-fixture. " +
|
|
144
|
+
"Select confirmed, plausible or explicitly unresolved root causes by ID. " +
|
|
214
145
|
"Recommendation must be a validation/fix plan, not a patch. nextSteps must be the minimum commands or code inspections needed to confirm the top diagnosis. Structured output only.",
|
|
215
|
-
|
|
216
|
-
phase: "Synthesize",
|
|
217
|
-
label: "synthesize",
|
|
218
|
-
tools: [],
|
|
219
|
-
thinkingLevel: "medium",
|
|
220
|
-
resume: "read-only",
|
|
221
|
-
schema: AdvisoryReportSchema,
|
|
222
|
-
},
|
|
146
|
+
ranked, coverage,
|
|
223
147
|
);
|
|
224
|
-
|
|
225
|
-
if (!report) return emptyReport("Synthesis produced no output.", ["Inspect verifier evidence manually or rerun diagnose with a narrower symptom."], stats);
|
|
226
|
-
|
|
227
|
-
const findings = backfillAdvisoryFindings(report.findings, ranked, {
|
|
228
|
-
impact: "Impact not restated by synthesis.",
|
|
229
|
-
recommendation: "Validate this diagnosis with the smallest safe reproduction command.",
|
|
230
|
-
});
|
|
231
|
-
return { ...report, findings, stats };
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
function emptyReport(summary: string, nextSteps: string[], stats: WorkflowRunStats): AdvisoryReport & { stats: WorkflowRunStats } {
|
|
235
|
-
return { summary, findings: [], nextSteps, stats };
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
function dedupe(candidates: Hypothesis[], onDropped: (dropped: number) => void): Hypothesis[] {
|
|
239
|
-
const seen = new Set<string>();
|
|
240
|
-
const novel: Hypothesis[] = [];
|
|
241
|
-
for (const candidate of candidates) {
|
|
242
|
-
const key = dedupKey(candidate);
|
|
243
|
-
if (seen.has(key)) continue;
|
|
244
|
-
seen.add(key);
|
|
245
|
-
novel.push(candidate);
|
|
246
|
-
}
|
|
247
|
-
const dropped = candidates.length - novel.length;
|
|
248
|
-
if (dropped > 0) onDropped(dropped);
|
|
249
|
-
return novel;
|
|
148
|
+
return finishAdvisoryReport({ ...resolved, stats: { ...stats, kept: resolved.findings.length } }, coverage, verified);
|
|
250
149
|
}
|
|
251
150
|
|
|
252
|
-
function rank(finding:
|
|
151
|
+
function rank(finding: AdvisoryVerified): number {
|
|
253
152
|
if (finding.verdict === "CONFIRMED") return 0;
|
|
254
153
|
return 1;
|
|
255
154
|
}
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
+
import { challengeFindings, parseChallengeArgs } from "../src/advisory-challenge.ts";
|
|
1
2
|
import { Type } from "typebox";
|
|
2
3
|
import {
|
|
3
|
-
|
|
4
|
-
type
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
import {
|
|
9
|
-
backfillAdvisoryFindings,
|
|
4
|
+
type AdvisoryVerified,
|
|
5
|
+
type AdvisoryLens,
|
|
6
|
+
synthesizeAdvisoryReport,
|
|
7
|
+
finishAdvisoryReport,
|
|
8
|
+
emptyAdvisoryReport,
|
|
10
9
|
formatEvidence,
|
|
11
10
|
formatLocation,
|
|
11
|
+
publishVerifiedKeptProgress,
|
|
12
12
|
runLensVerificationPipeline,
|
|
13
13
|
DEFAULT_ADVISORY_TOOL_HINTS,
|
|
14
14
|
DEFAULT_ADVISORY_TOOLS,
|
|
@@ -18,7 +18,7 @@ import type { WorkflowApi, WorkflowMeta, WorkflowRunStats } from "../src/types.t
|
|
|
18
18
|
export const meta: WorkflowMeta = {
|
|
19
19
|
name: "perf-review",
|
|
20
20
|
description: "Advisory-only performance review: scope slow path → per-lens bottleneck hypotheses → verify evidence → synthesize measurements and safe optimizations.",
|
|
21
|
-
phases: [{ title: "Scope" }, { title: "Find" }, { title: "Verify" }, { title: "Synthesize" }],
|
|
21
|
+
phases: [{ title: "Scope" }, { title: "Find" }, { title: "Verify" }, { title: "Challenge" }, { title: "Synthesize" }],
|
|
22
22
|
};
|
|
23
23
|
|
|
24
24
|
const ScopeSchema = Type.Object({
|
|
@@ -29,22 +29,7 @@ const ScopeSchema = Type.Object({
|
|
|
29
29
|
knownMeasurements: Type.Optional(Type.String({ description: "Existing measurements, timings, or explicit lack of measurements." })),
|
|
30
30
|
});
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
label: string;
|
|
34
|
-
category: string;
|
|
35
|
-
text: string;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
type Candidate = AdvisoryCandidate;
|
|
39
|
-
|
|
40
|
-
interface Verified extends Candidate {
|
|
41
|
-
verdict: AdvisoryVerdict["verdict"];
|
|
42
|
-
evidence: string[];
|
|
43
|
-
confidence?: AdvisoryVerdict["confidence"];
|
|
44
|
-
lens: PerfLens;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
const PERF_LENSES: PerfLens[] = [
|
|
32
|
+
const PERF_LENSES: AdvisoryLens[] = [
|
|
48
33
|
{ label: "algorithmic", category: "algorithmic", text: "Complexity, repeated scans, avoidable nested loops, or data-structure choices that grow poorly with input size." },
|
|
49
34
|
{ label: "io", category: "io", text: "Filesystem, subprocess, network, or other I/O costs on hot paths or startup paths." },
|
|
50
35
|
{ label: "concurrency", category: "concurrency", text: "Unnecessary serialization, missing batching, excessive fan-out, contention, or concurrency limits." },
|
|
@@ -53,13 +38,12 @@ const PERF_LENSES: PerfLens[] = [
|
|
|
53
38
|
{ label: "measurement", category: "measurement", text: "Missing, misleading, noisy, or insufficient benchmark/measurement design." },
|
|
54
39
|
];
|
|
55
40
|
|
|
56
|
-
const TOOLS = DEFAULT_ADVISORY_TOOLS;
|
|
57
|
-
const TOOL_HINTS = DEFAULT_ADVISORY_TOOL_HINTS;
|
|
58
41
|
const PER_LENS = 4;
|
|
59
42
|
|
|
60
43
|
export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
61
|
-
const { agent,
|
|
62
|
-
const
|
|
44
|
+
const { agent, phase, log, progress, args } = api;
|
|
45
|
+
const challengeConfig = parseChallengeArgs(args);
|
|
46
|
+
const target = challengeConfig.args.trim() || "repository performance";
|
|
63
47
|
let fileCount = 0;
|
|
64
48
|
let rawCandidateCount = 0;
|
|
65
49
|
let droppedCandidateCount = 0;
|
|
@@ -80,15 +64,15 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
80
64
|
"Inspect repository structure, scripts, likely hot-path files, and any existing benchmark or measurement commands. " +
|
|
81
65
|
"Prefer identifying what to measure before claiming bottlenecks. Return files, commands, summary, and known measurements or the lack of them. " +
|
|
82
66
|
`This workflow will fan out across ${PERF_LENSES.length} lenses with up to ${PER_LENS} candidates per lens. Structured output only.`,
|
|
83
|
-
{ phase: "Scope", label: "scope", tools:
|
|
67
|
+
{ phase: "Scope", label: "scope", tools: DEFAULT_ADVISORY_TOOLS, toolHints: DEFAULT_ADVISORY_TOOL_HINTS, profile: "medium", schema: ScopeSchema },
|
|
84
68
|
);
|
|
85
69
|
|
|
86
70
|
if (!scope || scope.files.length === 0) {
|
|
87
|
-
return
|
|
71
|
+
return finishAdvisoryReport(emptyAdvisoryReport(
|
|
88
72
|
"No performance-relevant files were identified.",
|
|
89
73
|
["Provide a slow command, workload, file path, or user-visible latency concern to review."],
|
|
90
74
|
makeStats(0, 0),
|
|
91
|
-
);
|
|
75
|
+
), []);
|
|
92
76
|
}
|
|
93
77
|
|
|
94
78
|
fileCount = scope.files.length;
|
|
@@ -103,14 +87,10 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
103
87
|
`## Summary\n${scope.summary}\n\n## Known measurements\n${scope.knownMeasurements ?? "(none known)"}\n` +
|
|
104
88
|
(args.trim() ? `\n## User instructions (verbatim)\n${args.trim()}\n` : "");
|
|
105
89
|
|
|
106
|
-
phase("Find");
|
|
107
90
|
const pipelineResult = await runLensVerificationPipeline({
|
|
108
|
-
api
|
|
91
|
+
api,
|
|
109
92
|
lenses: PERF_LENSES,
|
|
110
|
-
schedulingMode: "finder-barrier",
|
|
111
93
|
perLens: PER_LENS,
|
|
112
|
-
tools: TOOLS,
|
|
113
|
-
toolHints: TOOL_HINTS,
|
|
114
94
|
finderPrompt: (lens) =>
|
|
115
95
|
`## Perf-review finder — ${lens.label}\n\n${scopeBlock}\n` +
|
|
116
96
|
"This workflow is advisory-only: identify bottleneck hypotheses, measurement gaps, and safe optimization directions, but do not edit files.\n" +
|
|
@@ -123,75 +103,46 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
123
103
|
`Location: ${formatLocation(candidate)}\nCategory: ${candidate.category}\nSummary: ${candidate.summary}\nImpact: ${candidate.impact}\n` +
|
|
124
104
|
`Recommendation: ${candidate.recommendation ?? "(none supplied)"}\n\n` +
|
|
125
105
|
"Read relevant files and package/scripts. Run only safe read-only measurement or inspection commands when useful. " +
|
|
126
|
-
"Return CONFIRMED, PLAUSIBLE, or REFUTED with evidence from code, scripts, config, or measurement output. " +
|
|
106
|
+
"Return CONFIRMED, PLAUSIBLE, NOT_SUBSTANTIATED, or REFUTED with evidence from code, scripts, config, or measurement output. " +
|
|
127
107
|
"Default toward PLAUSIBLE or REFUTED when no measurement exists; do not overstate a bottleneck. Structured output only.",
|
|
128
|
-
makeVerified: (candidate, lens, judged): Verified => ({
|
|
129
|
-
...candidate,
|
|
130
|
-
verdict: judged.verdict,
|
|
131
|
-
evidence: judged.evidence,
|
|
132
|
-
confidence: judged.confidence,
|
|
133
|
-
lens,
|
|
134
|
-
}),
|
|
135
108
|
});
|
|
136
109
|
rawCandidateCount += pipelineResult.rawCandidates;
|
|
137
110
|
droppedCandidateCount += pipelineResult.dropped;
|
|
138
111
|
refutedCandidateCount += pipelineResult.refuted;
|
|
139
|
-
const
|
|
112
|
+
const { coverage } = pipelineResult;
|
|
113
|
+
const verified = await challengeFindings(api, pipelineResult.verified, scopeBlock, challengeConfig.options, coverage);
|
|
140
114
|
const surviving = verified.filter((finding) => finding.verdict !== "REFUTED");
|
|
141
115
|
const stats = makeStats(verified.length, surviving.length);
|
|
142
|
-
|
|
143
|
-
progress({ type: "counter", key: "kept", label: "kept", value: surviving.length });
|
|
144
|
-
progress({ type: "summary", key: "verified", value: verified.length });
|
|
145
|
-
progress({ type: "summary", key: "kept", value: surviving.length });
|
|
146
|
-
log(`${verified.length} verified → ${surviving.length} kept`);
|
|
116
|
+
publishVerifiedKeptProgress({ progress, log }, verified.length, surviving.length);
|
|
147
117
|
|
|
148
118
|
if (surviving.length === 0) {
|
|
149
|
-
return
|
|
119
|
+
return finishAdvisoryReport(emptyAdvisoryReport(
|
|
150
120
|
"No performance finding survived verification.",
|
|
151
121
|
["Add or run a focused measurement for the target workload before optimizing.", "Rerun perf-review with benchmark output or a narrower slow path."],
|
|
152
122
|
stats,
|
|
153
|
-
);
|
|
123
|
+
), coverage, verified);
|
|
154
124
|
}
|
|
155
125
|
|
|
156
|
-
phase("Synthesize");
|
|
157
126
|
const ranked = [...surviving].sort((a, b) => rank(a) - rank(b));
|
|
158
127
|
const block = ranked
|
|
159
128
|
.map(
|
|
160
129
|
(finding, index) =>
|
|
161
|
-
`### [${index}] ${formatLocation(finding)} (${finding.verdict}, ${finding.category})\n` +
|
|
130
|
+
`### [${index}] IDs: ${finding.sourceCandidateIds.join(", ")} ${formatLocation(finding)} (${finding.verdict}, ${finding.category})\n` +
|
|
162
131
|
`${finding.summary}\nImpact: ${finding.impact}\nEvidence: ${formatEvidence(finding.evidence)}\nRecommendation: ${finding.recommendation ?? "(none supplied)"}`,
|
|
163
132
|
)
|
|
164
133
|
.join("\n\n");
|
|
165
134
|
|
|
166
|
-
const
|
|
135
|
+
const resolved = await synthesizeAdvisoryReport(api,
|
|
167
136
|
`## Synthesis: final perf-review report\n\n${ranked.length} candidates survived independent verification.\n\n${block}\n\n` +
|
|
168
|
-
"
|
|
137
|
+
"Select findings by ID. " +
|
|
169
138
|
"Severity is expected performance impact for the target workload. Prefer measurement recommendations before optimization recommendations when evidence is weak. " +
|
|
170
139
|
"Recommendations must be safe advisory next actions, not patches. Include risky optimizations to avoid in recommendations or nextSteps when relevant. Structured output only.",
|
|
171
|
-
|
|
172
|
-
phase: "Synthesize",
|
|
173
|
-
label: "synthesize",
|
|
174
|
-
tools: [],
|
|
175
|
-
thinkingLevel: "medium",
|
|
176
|
-
resume: "read-only",
|
|
177
|
-
schema: AdvisoryReportSchema,
|
|
178
|
-
},
|
|
140
|
+
ranked, coverage,
|
|
179
141
|
);
|
|
180
|
-
|
|
181
|
-
if (!report) return emptyReport("Synthesis produced no output.", ["Inspect verifier evidence manually or rerun perf-review with a narrower target."], stats);
|
|
182
|
-
|
|
183
|
-
const findings = backfillAdvisoryFindings(report.findings, ranked, {
|
|
184
|
-
impact: "Performance impact not restated by synthesis.",
|
|
185
|
-
recommendation: "Measure the target workload before changing code.",
|
|
186
|
-
});
|
|
187
|
-
return { ...report, findings, stats };
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
function emptyReport(summary: string, nextSteps: string[], stats: WorkflowRunStats): AdvisoryReport & { stats: WorkflowRunStats } {
|
|
191
|
-
return { summary, findings: [], nextSteps, stats };
|
|
142
|
+
return finishAdvisoryReport({ ...resolved, stats: { ...stats, kept: resolved.findings.length } }, coverage, verified);
|
|
192
143
|
}
|
|
193
144
|
|
|
194
|
-
function rank(finding:
|
|
145
|
+
function rank(finding: AdvisoryVerified): number {
|
|
195
146
|
const verdictRank = finding.verdict === "CONFIRMED" ? 0 : 1;
|
|
196
147
|
const measurementPenalty = finding.category === "measurement" ? 1 : 0;
|
|
197
148
|
return verdictRank + measurementPenalty;
|