pi-workflow-engine 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/pi-workflow-engine/index.ts +1 -8
- package/.pi/extensions/pi-workflow-engine/src/advisory-challenge.ts +5 -2
- package/.pi/extensions/pi-workflow-engine/src/advisory-schema.ts +8 -0
- package/.pi/extensions/pi-workflow-engine/src/agent-limits.ts +4 -0
- package/.pi/extensions/pi-workflow-engine/src/dynamax.ts +5 -7
- package/.pi/extensions/pi-workflow-engine/src/engine.ts +1 -0
- package/.pi/extensions/pi-workflow-engine/src/types.ts +10 -0
- package/.pi/extensions/pi-workflow-engine/src/workflow-advisory-utils.ts +65 -17
- package/.pi/extensions/pi-workflow-engine/workflows/code-review.ts +58 -22
- package/.pi/extensions/pi-workflow-engine/workflows/diagnose.ts +6 -4
- package/.pi/extensions/pi-workflow-engine/workflows/perf-review.ts +6 -4
- package/.pi/extensions/pi-workflow-engine/workflows/refactor-scout.ts +6 -4
- package/USAGE.md +10 -8
- package/package.json +1 -1
- package/skills/workflow-authoring/SKILL.md +4 -2
|
@@ -646,13 +646,6 @@ function registerWorkflowTool(
|
|
|
646
646
|
description: "Maximum delay accepted from a provider reset hint before automatic resume",
|
|
647
647
|
}),
|
|
648
648
|
),
|
|
649
|
-
budget: Type.Optional(
|
|
650
|
-
Type.Integer({
|
|
651
|
-
minimum: WORKFLOW_BUDGET_MIN,
|
|
652
|
-
maximum: WORKFLOW_BUDGET_MAX,
|
|
653
|
-
description: "Optional output-token ceiling for the run; agent() throws once it is exceeded",
|
|
654
|
-
}),
|
|
655
|
-
),
|
|
656
649
|
perf: Type.Optional(Type.Boolean({ description: "Include workflow performance timing aggregates in the result details" })),
|
|
657
650
|
resumeFromRunId: Type.Optional(Type.String({ minLength: 1, description: "Workflow run id to resume from by replaying matching completed agent results" })),
|
|
658
651
|
resumeEditedWorkflow: Type.Optional(
|
|
@@ -681,7 +674,7 @@ function registerWorkflowTool(
|
|
|
681
674
|
if (unavailable) return unavailable;
|
|
682
675
|
}
|
|
683
676
|
const turnBudget = dynamax.turnBudget(ctx);
|
|
684
|
-
const budget = resolveTurnBudget(
|
|
677
|
+
const budget = resolveTurnBudget(turnBudget, { min: WORKFLOW_BUDGET_MIN, max: WORKFLOW_BUDGET_MAX });
|
|
685
678
|
if (budget.kind === "spent") {
|
|
686
679
|
return workflowToolError(
|
|
687
680
|
`The user's +${budget.total} output-token budget for this turn is spent by its earlier workflow runs. Do not start another workflow this turn unless the user raises the budget.`,
|
|
@@ -41,7 +41,7 @@ export function needsChallenge(finding: AdvisoryVerified): boolean {
|
|
|
41
41
|
|
|
42
42
|
/** A bounded recipe using existing agent/parallel calls, not a new runtime primitive. */
|
|
43
43
|
export async function challengeFindings<T extends AdvisoryVerified>(
|
|
44
|
-
api: Pick<WorkflowApi, "agent" | "parallel">,
|
|
44
|
+
api: Pick<WorkflowApi, "agent" | "agents" | "log" | "parallel">,
|
|
45
45
|
findings: T[],
|
|
46
46
|
context: string,
|
|
47
47
|
options: AdvisoryChallengeOptions,
|
|
@@ -49,7 +49,10 @@ export async function challengeFindings<T extends AdvisoryVerified>(
|
|
|
49
49
|
): Promise<T[]> {
|
|
50
50
|
const limit = Number.isFinite(options.maxChallenges) ? Math.max(0, Math.min(10, Math.trunc(options.maxChallenges))) : 0;
|
|
51
51
|
if (limit === 0) return findings;
|
|
52
|
-
const
|
|
52
|
+
const eligible = findings.filter((finding) => finding.verdict !== "REFUTED" && (options.shouldChallenge ?? needsChallenge)(finding)).slice(0, limit);
|
|
53
|
+
// A challenge takes two agents; keep one admission for the synthesis that follows.
|
|
54
|
+
const selected = eligible.slice(0, Math.max(0, Math.floor((api.agents.remaining() - 1) / 2)));
|
|
55
|
+
if (selected.length < eligible.length) api.log(`Challenging ${selected.length} of ${eligible.length} findings to fit the agent limit`);
|
|
53
56
|
const replacements = new Map<string, T>();
|
|
54
57
|
await collectAdvisoryStage(api, "Challenge", selected.map((finding) => ({ id: finding.candidateId, run: async () => {
|
|
55
58
|
// Preserve the candidate as unresolved if either independent stage fails.
|
|
@@ -38,6 +38,14 @@ export const AdvisoryVerdictSchema = Type.Object({
|
|
|
38
38
|
evidence: Type.Array(Type.String({ description: "Quoted or cited evidence supporting the verdict." })),
|
|
39
39
|
});
|
|
40
40
|
|
|
41
|
+
export const AdvisoryBatchVerdictSchema = Type.Object({
|
|
42
|
+
verdicts: Type.Array(Type.Object({
|
|
43
|
+
candidateId: Type.String({ description: "The candidateId of the record this verdict judges." }),
|
|
44
|
+
verdict: AdvisoryVerifierVerdictSchema,
|
|
45
|
+
evidence: Type.Array(Type.String({ description: "Quoted or cited evidence supporting the verdict." })),
|
|
46
|
+
}), { description: "One verdict per candidate record, each judged on its own." }),
|
|
47
|
+
});
|
|
48
|
+
|
|
41
49
|
export const AdvisoryFindingSchema = Type.Object({
|
|
42
50
|
verdict: Type.Optional(AdvisoryVerifierVerdictSchema),
|
|
43
51
|
sourceCandidateIds: Type.Optional(Type.Array(Type.String())),
|
|
@@ -147,20 +147,18 @@ export type DynamaxTurnBudgetResolution =
|
|
|
147
147
|
| { readonly kind: "spent"; readonly total: number };
|
|
148
148
|
|
|
149
149
|
/**
|
|
150
|
-
* A run's budget.
|
|
151
|
-
* its turn
|
|
152
|
-
*
|
|
150
|
+
* A tool run's budget. Only the user sets one: without a turn pool this adds none, and the pool caps
|
|
151
|
+
* every run in its turn at the remainder, clamped to the run budget range. A pool with less than `min`
|
|
152
|
+
* left is spent.
|
|
153
153
|
*/
|
|
154
154
|
export function resolveTurnBudget(
|
|
155
|
-
explicit: number | undefined,
|
|
156
155
|
pool: DynamaxTurnBudget | undefined,
|
|
157
156
|
range: { readonly min: number; readonly max: number },
|
|
158
157
|
): DynamaxTurnBudgetResolution {
|
|
159
|
-
if (pool === undefined) return { kind: "budget", budget:
|
|
158
|
+
if (pool === undefined) return { kind: "budget", budget: undefined };
|
|
160
159
|
const remaining = pool.remaining();
|
|
161
160
|
if (remaining < range.min) return { kind: "spent", total: pool.total };
|
|
162
|
-
|
|
163
|
-
return { kind: "budget", budget: explicit === undefined ? cap : Math.min(explicit, cap) };
|
|
161
|
+
return { kind: "budget", budget: Math.min(range.max, Math.floor(remaining)) };
|
|
164
162
|
}
|
|
165
163
|
|
|
166
164
|
export function markDynamaxOneShot(state: DynamaxState): void {
|
|
@@ -280,10 +280,20 @@ export interface WorkflowApi {
|
|
|
280
280
|
* `agent()` throws `WorkflowBudgetExceededError` once the ceiling is reached.
|
|
281
281
|
*/
|
|
282
282
|
budget: WorkflowBudget;
|
|
283
|
+
/**
|
|
284
|
+
* The run's live-agent limit. `remaining()` is live: admissions left before `agent()` throws
|
|
285
|
+
* `WorkflowAgentLimitError` (journal replays take none). Reserve what later stages need before a fan-out.
|
|
286
|
+
*/
|
|
287
|
+
agents: WorkflowAgentCapacity;
|
|
283
288
|
/** Abort signal for the run, propagated from the host. */
|
|
284
289
|
signal: AbortSignal | undefined;
|
|
285
290
|
}
|
|
286
291
|
|
|
292
|
+
export interface WorkflowAgentCapacity {
|
|
293
|
+
readonly max: number;
|
|
294
|
+
remaining(): number;
|
|
295
|
+
}
|
|
296
|
+
|
|
287
297
|
export type WorkflowRun = (api: WorkflowApi) => Promise<unknown>;
|
|
288
298
|
|
|
289
299
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AdvisoryCandidatesSchema, AdvisoryVerdictSchema, formatAdvisoryLocation, type AdvisoryCandidate, type AdvisoryFinding, type IdentifiedAdvisoryCandidate, type AdvisoryLocation, type AdvisoryReport, type AdvisoryVerdict } from "./advisory-schema.ts";
|
|
1
|
+
import { AdvisoryBatchVerdictSchema, AdvisoryCandidatesSchema, AdvisoryVerdictSchema, formatAdvisoryLocation, type AdvisoryCandidate, type AdvisoryFinding, type IdentifiedAdvisoryCandidate, type AdvisoryLocation, type AdvisoryReport, type AdvisoryVerdict } from "./advisory-schema.ts";
|
|
2
2
|
import { AdvisorySynthesisSchema, SYNTHESIS_ID_INSTRUCTIONS, withAdvisoryCoverage, collectAdvisoryStage, dedupeCandidates, identifyCandidates, mergeCandidateInto, repoPathCandidates, uniqueLocations, type AdvisoryStageCoverage, type AdvisorySynthesis } from "./advisory-evidence.ts";
|
|
3
3
|
import { isFatalWorkflowError } from "./cancellation.ts";
|
|
4
4
|
import { WorkflowClassifierUnavailableError } from "./classify.ts";
|
|
@@ -54,15 +54,29 @@ export function publishVerifiedKeptProgress(
|
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
export interface LensVerificationPipelineOptions {
|
|
57
|
-
api: Pick<WorkflowApi, "agent" | "classify" | "parallel" | "phase" | "progress" | "log">;
|
|
57
|
+
api: Pick<WorkflowApi, "agent" | "agents" | "classify" | "parallel" | "phase" | "progress" | "log">;
|
|
58
58
|
lenses: readonly AdvisoryLens[];
|
|
59
59
|
perLens: number;
|
|
60
60
|
finderPhase?: "Find" | "Hypothesize";
|
|
61
|
+
finderProfile?: AgentOptions["profile"];
|
|
61
62
|
finderPrompt(lens: AdvisoryLens): string;
|
|
62
|
-
|
|
63
|
+
/** One verifier's prompt. It gets several candidates only when the agent limit leaves fewer verifiers than candidates. */
|
|
64
|
+
verifierPrompt(candidates: readonly IdentifiedAdvisoryCandidate[]): string;
|
|
63
65
|
boundCandidate?(candidate: IdentifiedAdvisoryCandidate, lens: AdvisoryLens): IdentifiedAdvisoryCandidate | undefined;
|
|
64
66
|
}
|
|
65
67
|
|
|
68
|
+
/** The candidate section of a verifier prompt: one record, or a batch judged in one pass. */
|
|
69
|
+
export function verifierCandidates(
|
|
70
|
+
candidates: readonly IdentifiedAdvisoryCandidate[],
|
|
71
|
+
heading: { one: string; many: string },
|
|
72
|
+
fields: (candidate: IdentifiedAdvisoryCandidate) => string,
|
|
73
|
+
): string {
|
|
74
|
+
if (candidates.length === 1) return `## ${heading.one}\n${fields(candidates[0]!)}`;
|
|
75
|
+
return `## ${heading.many} (${candidates.length}: the run's agent limit leaves one verifier for several)\n` +
|
|
76
|
+
candidates.map((candidate) => `### candidateId ${candidate.candidateId}\n${fields(candidate)}`).join("\n\n") +
|
|
77
|
+
"\n\nJudge each on its own evidence and return one verdict per candidateId.";
|
|
78
|
+
}
|
|
79
|
+
|
|
66
80
|
export interface AdvisoryBackfillDefaults {
|
|
67
81
|
impact: string;
|
|
68
82
|
recommendation?: string;
|
|
@@ -72,7 +86,7 @@ export interface AdvisoryBackfillDefaults {
|
|
|
72
86
|
export async function runLensVerificationPipeline(
|
|
73
87
|
options: LensVerificationPipelineOptions,
|
|
74
88
|
): Promise<LensVerificationPipelineResult> {
|
|
75
|
-
const { api, lenses, perLens, finderPhase = "Find", finderPrompt, verifierPrompt, boundCandidate } = options;
|
|
89
|
+
const { api, lenses, perLens, finderPhase = "Find", finderProfile = "small", finderPrompt, verifierPrompt, boundCandidate } = options;
|
|
76
90
|
const coverage: AdvisoryStageCoverage[] = [];
|
|
77
91
|
let rawCandidates = 0;
|
|
78
92
|
let overCap = 0;
|
|
@@ -81,7 +95,7 @@ export async function runLensVerificationPipeline(
|
|
|
81
95
|
const result = await api.agent(finderPrompt(lens), {
|
|
82
96
|
phase: finderPhase, label: `${finderPhase.toLowerCase()}:${lens.label}`,
|
|
83
97
|
tools: DEFAULT_ADVISORY_TOOLS, toolHints: DEFAULT_ADVISORY_TOOL_HINTS,
|
|
84
|
-
profile:
|
|
98
|
+
profile: finderProfile, schema: AdvisoryCandidatesSchema,
|
|
85
99
|
});
|
|
86
100
|
rawCandidates += result.candidates.length;
|
|
87
101
|
api.progress({ type: "counter_delta", key: "candidates", label: "candidates", delta: result.candidates.length });
|
|
@@ -111,21 +125,55 @@ export async function runLensVerificationPipeline(
|
|
|
111
125
|
}
|
|
112
126
|
const dropped = overCap + duplicateOrOutOfScope;
|
|
113
127
|
api.phase("Verify");
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
}
|
|
125
|
-
})
|
|
128
|
+
// Keep one admission for the synthesis that follows. A host-set agent limit can leave fewer verifiers than
|
|
129
|
+
// candidates; then each verifier judges a batch, so no candidate goes unverified and synthesis still runs.
|
|
130
|
+
const batches = verificationBatches(candidates, Math.max(1, api.agents.remaining() - 1));
|
|
131
|
+
if (batches.length < candidates.length) api.log(`Verifying ${candidates.length} candidates in ${batches.length} batch(es) to fit the agent limit`);
|
|
132
|
+
// Each candidate keeps its own coverage branch, so a failed batch still lists every candidate it held.
|
|
133
|
+
const verified = await collectAdvisoryStage(api, "Verify", batches.flatMap((batch) => {
|
|
134
|
+
let judged: Promise<AdvisoryVerified[]> | undefined;
|
|
135
|
+
return batch.map((candidate, index) => ({
|
|
136
|
+
id: candidate.candidateId, candidate,
|
|
137
|
+
run: async () => (await (judged ??= verifyBatch(api, batch, verifierPrompt)))[index]!,
|
|
138
|
+
}));
|
|
139
|
+
}), coverage);
|
|
126
140
|
return { verified, rawCandidates, dropped, coverage };
|
|
127
141
|
}
|
|
128
142
|
|
|
143
|
+
/** Split candidates into at most `count` batches, keeping candidates in one file together where it fits. */
|
|
144
|
+
function verificationBatches(candidates: readonly IdentifiedAdvisoryCandidate[], count: number): IdentifiedAdvisoryCandidate[][] {
|
|
145
|
+
if (candidates.length <= count) return candidates.map((candidate) => [candidate]);
|
|
146
|
+
const byFile = [...candidates].sort((a, b) => primaryLocation(a).file.localeCompare(primaryLocation(b).file));
|
|
147
|
+
const size = Math.ceil(byFile.length / count);
|
|
148
|
+
return Array.from({ length: Math.ceil(byFile.length / size) }, (_, index) => byFile.slice(index * size, (index + 1) * size));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function verifyBatch(
|
|
152
|
+
api: Pick<WorkflowApi, "agent" | "progress">,
|
|
153
|
+
batch: readonly IdentifiedAdvisoryCandidate[],
|
|
154
|
+
verifierPrompt: LensVerificationPipelineOptions["verifierPrompt"],
|
|
155
|
+
): Promise<AdvisoryVerified[]> {
|
|
156
|
+
const location = primaryLocation(batch[0]!);
|
|
157
|
+
const options = {
|
|
158
|
+
phase: "Verify", label: `verify:${location.file.split("/").pop() ?? location.file}${batch.length > 1 ? ` +${batch.length - 1}` : ""}`,
|
|
159
|
+
tools: DEFAULT_ADVISORY_TOOLS, toolHints: DEFAULT_ADVISORY_TOOL_HINTS,
|
|
160
|
+
} as const;
|
|
161
|
+
if (batch.length === 1) {
|
|
162
|
+
const candidate = batch[0]!;
|
|
163
|
+
const judged = await api.agent(`${verifierPrompt(batch)}\nCandidate record: ${JSON.stringify(candidate)}`, { ...options, profile: "small", schema: AdvisoryVerdictSchema });
|
|
164
|
+
recordVerdictProgress(api.progress, candidate, judged);
|
|
165
|
+
return [{ ...candidate, verdict: judged.verdict, evidence: judged.evidence }];
|
|
166
|
+
}
|
|
167
|
+
// One agent does several verifiers' work, so it gets the medium route.
|
|
168
|
+
const judged = await api.agent(`${verifierPrompt(batch)}\nCandidate records: ${JSON.stringify(batch)}`, { ...options, profile: "medium", schema: AdvisoryBatchVerdictSchema });
|
|
169
|
+
const verdicts = new Map(judged.verdicts.map((verdict) => [verdict.candidateId, verdict]));
|
|
170
|
+
return batch.map((candidate) => {
|
|
171
|
+
const verdict = verdicts.get(candidate.candidateId) ?? { verdict: "NOT_SUBSTANTIATED" as const, evidence: ["The batch verifier returned no verdict for this candidate."] };
|
|
172
|
+
recordVerdictProgress(api.progress, candidate, verdict);
|
|
173
|
+
return { ...candidate, verdict: verdict.verdict, evidence: verdict.evidence };
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
|
|
129
177
|
/** TypeSafe Jev routes in preference order. Jev is the classifier the duplicate merge was measured with. */
|
|
130
178
|
export const DUPLICATE_CANDIDATE_CLASSIFIERS = [
|
|
131
179
|
"typesafe/jev-latest",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { challengeFindings, parseChallengeArgs } from "../src/advisory-challenge.ts";
|
|
2
|
-
import { Type } from "typebox";
|
|
2
|
+
import { Type, type Static } from "typebox";
|
|
3
3
|
import {
|
|
4
4
|
type AdvisoryVerified,
|
|
5
5
|
type AdvisoryLens,
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
formatEvidence,
|
|
10
10
|
formatLocation,
|
|
11
11
|
runLensVerificationPipeline,
|
|
12
|
+
verifierCandidates,
|
|
12
13
|
EMPTY_LENS_REVIEW_STATS,
|
|
13
14
|
verdictConfidence,
|
|
14
15
|
DEFAULT_ADVISORY_TOOL_HINTS,
|
|
@@ -35,6 +36,16 @@ const ScopeSchema = Type.Object({
|
|
|
35
36
|
conventions: Type.Optional(Type.String({ description: "Relevant AGENTS.md / project conventions" })),
|
|
36
37
|
});
|
|
37
38
|
|
|
39
|
+
// With review instructions, scope splits them by stage: finders get only where to look, because a reporting bar in a
|
|
40
|
+
// finder prompt makes finders filter their own candidates before any verifier sees them; the bar goes to synthesis.
|
|
41
|
+
const InstructedScopeSchema = Type.Object({
|
|
42
|
+
...ScopeSchema.properties,
|
|
43
|
+
conventions: Type.Optional(Type.String({ description: "Repository conventions from AGENTS.md and project docs only. Do not restate the user's review instructions, filtering rules or already-reported findings here." })),
|
|
44
|
+
focus: Type.String({ description: "Where the user wants reviewers to look: areas, files, risks and lenses named in the instructions. Exclude any filtering, evidence, reporting or deduplication rules; write (none) if the instructions name no focus." }),
|
|
45
|
+
reportingBar: Type.String({ description: "The user's filtering, evidence, severity and reporting rules restated so a reader without file access can apply them, listing every already-reported issue to exclude (read any files the instructions reference). Write (none) if the instructions set no bar." }),
|
|
46
|
+
});
|
|
47
|
+
type Scope = Static<typeof ScopeSchema> & Partial<Pick<Static<typeof InstructedScopeSchema>, "focus" | "reportingBar">>;
|
|
48
|
+
|
|
38
49
|
// The review lenses — this is the part you customise to your codebase's real failure modes.
|
|
39
50
|
const ANGLES: AdvisoryLens[] = [
|
|
40
51
|
{ label: "logic-bugs", category: "bug", text: "Off-by-one errors, wrong conditionals, incorrect return values, broken control flow." },
|
|
@@ -54,7 +65,7 @@ export function buildCodeReviewScopeBlock(input: {
|
|
|
54
65
|
readonly summary: string;
|
|
55
66
|
readonly conventions?: string;
|
|
56
67
|
readonly diffText: string;
|
|
57
|
-
readonly
|
|
68
|
+
readonly instructions?: { readonly heading: string; readonly text: string };
|
|
58
69
|
}): string {
|
|
59
70
|
const diffBlock = input.diffText
|
|
60
71
|
? `\n## Diff (review is bounded to these changed lines)\n\`\`\`diff\n${
|
|
@@ -67,7 +78,7 @@ export function buildCodeReviewScopeBlock(input: {
|
|
|
67
78
|
`## Diff command\n${input.diffCommand}\n\n## Changed files\n${input.files.map((file) => `- ${file}`).join("\n")}\n\n` +
|
|
68
79
|
`## Summary\n${input.summary}\n\n## Conventions\n${input.conventions ?? "(none noted)"}\n` +
|
|
69
80
|
diffBlock +
|
|
70
|
-
(input.
|
|
81
|
+
(input.instructions?.text ? `\n## ${input.instructions.heading}\n${input.instructions.text}\n` : "")
|
|
71
82
|
);
|
|
72
83
|
}
|
|
73
84
|
|
|
@@ -82,7 +93,7 @@ export default async function run(api: WorkflowApi, dependencies: CodeReviewDepe
|
|
|
82
93
|
|
|
83
94
|
// ─── Phase 0: Scope ───
|
|
84
95
|
phase("Scope");
|
|
85
|
-
const
|
|
96
|
+
const scopePrompt =
|
|
86
97
|
"Establish the scope of a code review.\n" +
|
|
87
98
|
(target
|
|
88
99
|
? `Target / instructions (verbatim): "${target}". If it names a PR number, branch, ref range, or files, build the matching diff command (use 'gh pr diff <number>' for a PR). Otherwise use the default selection below.\n`
|
|
@@ -98,9 +109,14 @@ export default async function run(api: WorkflowApi, dependencies: CodeReviewDepe
|
|
|
98
109
|
"4. Run the chosen command to confirm the diff is non-empty.\n\n" +
|
|
99
110
|
"Then: list the changed files, summarize the change in one paragraph (mention the PR if one was found), " +
|
|
100
111
|
"and read any relevant AGENTS.md or project docs noting conventions a reviewer should know.\n" +
|
|
101
|
-
"Return diffCommand exactly as a reviewer should run it. Structured output only."
|
|
102
|
-
|
|
103
|
-
|
|
112
|
+
"Return diffCommand exactly as a reviewer should run it. Structured output only.";
|
|
113
|
+
const scopeOptions = { phase: "Scope", label: "scope", tools: DEFAULT_ADVISORY_TOOLS, toolHints: DEFAULT_ADVISORY_TOOL_HINTS, profile: "medium" } as const;
|
|
114
|
+
const scope: Scope = target
|
|
115
|
+
? await agent(
|
|
116
|
+
`${scopePrompt}\nKeep the user's instructions out of conventions. Return focus (where to look) and reportingBar (the self-contained filtering and reporting bar, with every already-reported issue spelled out) as separate fields.`,
|
|
117
|
+
{ ...scopeOptions, schema: InstructedScopeSchema },
|
|
118
|
+
)
|
|
119
|
+
: await agent(scopePrompt, { ...scopeOptions, schema: ScopeSchema });
|
|
104
120
|
|
|
105
121
|
progress({ type: "summary", key: "files", value: scope.files.join(", ") || "(none)" });
|
|
106
122
|
const diffTarget = parseAllowedDiffCommand(scope.diffCommand);
|
|
@@ -148,19 +164,27 @@ export default async function run(api: WorkflowApi, dependencies: CodeReviewDepe
|
|
|
148
164
|
: {}),
|
|
149
165
|
};
|
|
150
166
|
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
167
|
+
const blockFor = (instructions?: { heading: string; text: string }) =>
|
|
168
|
+
`Reviewed snapshot identity: ${JSON.stringify(reviewContext.snapshot ?? "unavailable")}\n` + buildCodeReviewScopeBlock({
|
|
169
|
+
diffCommand,
|
|
170
|
+
files: scope.files,
|
|
171
|
+
summary: scope.summary,
|
|
172
|
+
conventions: scope.conventions,
|
|
173
|
+
diffText,
|
|
174
|
+
...(instructions ? { instructions } : {}),
|
|
175
|
+
});
|
|
176
|
+
const scopeBlock = blockFor({ heading: "User instructions (verbatim)", text: target });
|
|
177
|
+
const focus = stated(scope.focus);
|
|
178
|
+
const reportingBar = stated(scope.reportingBar);
|
|
179
|
+
const finderBlock = blockFor(focus ? { heading: "Review focus (from the user's instructions)", text: focus } : undefined);
|
|
180
|
+
if (focus) log(`review focus: ${focus}`);
|
|
159
181
|
|
|
160
182
|
const pipelineResult = await runLensVerificationPipeline({
|
|
161
183
|
api,
|
|
162
184
|
lenses: ANGLES,
|
|
163
185
|
perLens: PER_ANGLE,
|
|
186
|
+
// Finders on `small` (thinking low by default) returned nothing on real PRs; they are the stage that must dig.
|
|
187
|
+
finderProfile: "medium",
|
|
164
188
|
boundCandidate: (candidate, lens) => {
|
|
165
189
|
const reviewAnchor = candidate.reviewAnchor
|
|
166
190
|
? diffAnchor(changed, candidate.reviewAnchor)
|
|
@@ -168,18 +192,18 @@ export default async function run(api: WorkflowApi, dependencies: CodeReviewDepe
|
|
|
168
192
|
return reviewAnchor ? { ...candidate, category: lens.category, reviewAnchor } : undefined;
|
|
169
193
|
},
|
|
170
194
|
finderPrompt: (lens) =>
|
|
171
|
-
`## Code-review finder — ${lens.label}\n\n${
|
|
195
|
+
`## Code-review finder — ${lens.label}\n\n${finderBlock}\n` +
|
|
172
196
|
`Review the change through ONLY this lens:\n${lens.text}\n` +
|
|
173
197
|
"Only flag issues on lines that are part of the diff above (run the diff command if it is not shown). " +
|
|
174
198
|
"Set reviewAnchor to the changed line causing the issue; locations and discoveryEvidence may include unchanged callers and other files. " +
|
|
175
199
|
`Surface up to ${PER_ANGLE} candidates. Use category exactly "${lens.category}". Each candidate must include a one-line summary, ` +
|
|
176
200
|
"locations with the changed file and a line that appears in the diff, and impact describing the concrete failure or maintenance scenario. " +
|
|
177
201
|
"Pass through anything with a nameable impact — a separate verifier judges them next. Structured output only.",
|
|
178
|
-
verifierPrompt: (
|
|
179
|
-
`## Code-review verifier\n\n${scopeBlock}\n
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
"Run the diff command, read the relevant file(s), and return exactly one verdict (CONFIRMED / PLAUSIBLE / NOT_SUBSTANTIATED / REFUTED) " +
|
|
202
|
+
verifierPrompt: (candidates) =>
|
|
203
|
+
`## Code-review verifier\n\n${scopeBlock}\n` +
|
|
204
|
+
verifierCandidates(candidates, { one: "Candidate", many: "Candidates" }, (candidate) =>
|
|
205
|
+
`Location: ${formatLocation(candidate)}\nCategory: ${candidate.category}\nSummary: ${candidate.summary}\nImpact: ${candidate.impact}`) + "\n\n" +
|
|
206
|
+
"Run the diff command, read the relevant file(s), and return exactly one verdict per candidate (CONFIRMED / PLAUSIBLE / NOT_SUBSTANTIATED / REFUTED) " +
|
|
183
207
|
"with evidence quoting the line(s). Use NOT_SUBSTANTIATED when evidence is insufficient; use REFUTED only for concrete disproof. Structured output only.",
|
|
184
208
|
});
|
|
185
209
|
const verified = await challengeFindings(api, pipelineResult.verified, scopeBlock, challengeConfig.options, pipelineResult.coverage);
|
|
@@ -194,11 +218,23 @@ export default async function run(api: WorkflowApi, dependencies: CodeReviewDepe
|
|
|
194
218
|
synthesisPrompt: (block, ranked) =>
|
|
195
219
|
`## Synthesis: final code-review report\n\n${ranked.length} findings survived independent verification.\n\n${block}\n\n` +
|
|
196
220
|
"Merge findings with the same root cause, rank most-severe first (correctness bugs above cleanups), and produce the final advisory report. " +
|
|
197
|
-
"Return summary, ID selections with severity (low/medium/high) and advisory recommendation, and nextSteps. Evidence and confidence are reconstructed from verified records. Structured output only."
|
|
221
|
+
"Return summary, ID selections with severity (low/medium/high) and advisory recommendation, and nextSteps. Evidence and confidence are reconstructed from verified records. Structured output only." +
|
|
222
|
+
// Only a stated bar filters: a focus alone must not drop verified findings outside it.
|
|
223
|
+
(reportingBar
|
|
224
|
+
? `\n\n## Reporting bar (the user's rules with referenced context resolved)\n${reportingBar}` +
|
|
225
|
+
`\n\n## User instructions (verbatim)\n${target}\n\n` +
|
|
226
|
+
"Apply the reporting bar when choosing which IDs to report: leave out findings it excludes, and say in the summary what you left out and why."
|
|
227
|
+
: ""),
|
|
198
228
|
});
|
|
199
229
|
return { ...report, reviewContext };
|
|
200
230
|
}
|
|
201
231
|
|
|
232
|
+
/** A scope field's text, or undefined when the scope agent left it blank or wrote (none). */
|
|
233
|
+
function stated(value: string | undefined): string | undefined {
|
|
234
|
+
const text = value?.trim();
|
|
235
|
+
return text && text !== "(none)" ? text : undefined;
|
|
236
|
+
}
|
|
237
|
+
|
|
202
238
|
function rank(finding: AdvisoryVerified): number {
|
|
203
239
|
return (finding.category === "cleanup" ? 2 : 0) + (finding.verdict !== "CONFIRMED" ? 1 : 0);
|
|
204
240
|
}
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
formatLocation,
|
|
9
9
|
formatRankedFinding,
|
|
10
10
|
runLensVerificationPipeline,
|
|
11
|
+
verifierCandidates,
|
|
11
12
|
DEFAULT_ADVISORY_TOOL_HINTS,
|
|
12
13
|
DEFAULT_ADVISORY_TOOLS,
|
|
13
14
|
} from "../src/workflow-advisory-utils.ts";
|
|
@@ -77,10 +78,11 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
77
78
|
`Consider ONLY this hypothesis lens:\n${lens.text}\n\n` +
|
|
78
79
|
`Surface up to ${PER_LENS} root-cause hypotheses. Use category exactly "${lens.category}". ` +
|
|
79
80
|
"Each hypothesis must include a one-line summary, locations, impact explaining how it produces the symptom, and an optional recommendation for the next validation step. Structured output only.",
|
|
80
|
-
verifierPrompt: (
|
|
81
|
-
`## Diagnose verifier\n\n${scopeBlock}\n
|
|
82
|
-
|
|
83
|
-
|
|
81
|
+
verifierPrompt: (candidates) =>
|
|
82
|
+
`## Diagnose verifier\n\n${scopeBlock}\n` +
|
|
83
|
+
verifierCandidates(candidates, { one: "Hypothesis", many: "Hypotheses" }, (candidate) =>
|
|
84
|
+
`Location: ${formatLocation(candidate)}\nCategory: ${candidate.category}\nSummary: ${candidate.summary}\nImpact: ${candidate.impact}\n` +
|
|
85
|
+
`Recommended validation: ${candidate.recommendation ?? "(none supplied)"}`) + "\n\n" +
|
|
84
86
|
"Read relevant files and, when useful, run only safe read-only diagnostic commands from the scoped command list or commands explicitly requested by the user. " +
|
|
85
87
|
"Do not run mutation, install, commit, network, or destructive commands. Return CONFIRMED, PLAUSIBLE, NOT_SUBSTANTIATED, or REFUTED with evidence. " +
|
|
86
88
|
"Use NOT_SUBSTANTIATED when evidence is missing; REFUTED requires disproof. Structured output only.",
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
EMPTY_LENS_REVIEW_STATS,
|
|
10
10
|
formatLocation,
|
|
11
11
|
runLensVerificationPipeline,
|
|
12
|
+
verifierCandidates,
|
|
12
13
|
DEFAULT_ADVISORY_TOOL_HINTS,
|
|
13
14
|
DEFAULT_ADVISORY_TOOLS,
|
|
14
15
|
} from "../src/workflow-advisory-utils.ts";
|
|
@@ -85,10 +86,11 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
85
86
|
`Surface up to ${PER_LENS} candidates. Use category exactly "${lens.category}". ` +
|
|
86
87
|
"Each candidate must include a one-line summary, locations, impact stating the suspected performance consequence and workload where it matters, " +
|
|
87
88
|
"and an optional recommendation. Prefer measurement recommendations when evidence is weak. Structured output only.",
|
|
88
|
-
verifierPrompt: (
|
|
89
|
-
`## Perf-review verifier\n\n${scopeBlock}\n
|
|
90
|
-
|
|
91
|
-
|
|
89
|
+
verifierPrompt: (candidates) =>
|
|
90
|
+
`## Perf-review verifier\n\n${scopeBlock}\n` +
|
|
91
|
+
verifierCandidates(candidates, { one: "Candidate", many: "Candidates" }, (candidate) =>
|
|
92
|
+
`Location: ${formatLocation(candidate)}\nCategory: ${candidate.category}\nSummary: ${candidate.summary}\nImpact: ${candidate.impact}\n` +
|
|
93
|
+
`Recommendation: ${candidate.recommendation ?? "(none supplied)"}`) + "\n\n" +
|
|
92
94
|
"Read relevant files and package/scripts. Run only safe read-only measurement or inspection commands when useful. " +
|
|
93
95
|
"Return CONFIRMED, PLAUSIBLE, NOT_SUBSTANTIATED, or REFUTED with evidence from code, scripts, config, or measurement output. " +
|
|
94
96
|
"Default toward PLAUSIBLE or REFUTED when no measurement exists; do not overstate a bottleneck. Structured output only.",
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
formatLocation,
|
|
11
11
|
formatRankedFinding,
|
|
12
12
|
runLensVerificationPipeline,
|
|
13
|
+
verifierCandidates,
|
|
13
14
|
DEFAULT_ADVISORY_TOOL_HINTS,
|
|
14
15
|
DEFAULT_ADVISORY_TOOLS,
|
|
15
16
|
} from "../src/workflow-advisory-utils.ts";
|
|
@@ -83,10 +84,11 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
|
|
|
83
84
|
`Surface up to ${PER_LENS} candidates. Use category exactly "${lens.category}". ` +
|
|
84
85
|
"Each candidate must include a one-line summary, locations, impact on maintainability or future correctness, and an optional safe first recommendation. " +
|
|
85
86
|
"Only include opportunities where a small, reviewable first step is plausible. Structured output only.",
|
|
86
|
-
verifierPrompt: (
|
|
87
|
-
`## Refactor-scout verifier\n\n${scopeBlock}\n
|
|
88
|
-
|
|
89
|
-
|
|
87
|
+
verifierPrompt: (candidates) =>
|
|
88
|
+
`## Refactor-scout verifier\n\n${scopeBlock}\n` +
|
|
89
|
+
verifierCandidates(candidates, { one: "Candidate", many: "Candidates" }, (candidate) =>
|
|
90
|
+
`Location: ${formatLocation(candidate)}\nCategory: ${candidate.category}\nSummary: ${candidate.summary}\nImpact: ${candidate.impact}\n` +
|
|
91
|
+
`Recommendation: ${candidate.recommendation ?? "(none supplied)"}`) + "\n\n" +
|
|
90
92
|
"Read the relevant files and return CONFIRMED, PLAUSIBLE, NOT_SUBSTANTIATED, or REFUTED. " +
|
|
91
93
|
"Default toward REFUTED if the opportunity is generic, too broad, not evidenced by code, or lacks a safe first step. " +
|
|
92
94
|
"Evidence must quote or cite code. Structured output only.",
|
package/USAGE.md
CHANGED
|
@@ -606,7 +606,7 @@ Only tune these when a workflow is too slow, too expensive, or too noisy:
|
|
|
606
606
|
| `--max-agents=N` / `PI_WORKFLOW_MAX_AGENTS=N` | Cap live model calls across the complete run, including sub-workflows. Default `64`; values are clamped to `1`–`10000`. Replay hits do not consume the cap. The `workflow` tool field is `maxAgents`. This is independent of the `/workflow:size` prompt guideline. |
|
|
607
607
|
| `--agent-timeout-ms=N` / `PI_WORKFLOW_AGENT_TIMEOUT_MS=N` | Abort one live agent after this duration. Default `1800000` (30 minutes); values are clamped to `1000`–`86400000`. The session is aborted and its isolated worktree is cleaned up. The `workflow` tool field is `agentTimeoutMs`. |
|
|
608
608
|
| `--agent-retries=N` / `PI_WORKFLOW_AGENT_RETRIES=N` | Restart the whole agent after a transient provider or transport failure that pi's in-session turn retry could not recover. Default `0`; values are clamped to `0`–`10`. Backoff starts at one second, doubles, and caps at 30 seconds. The `workflow` tool field is `agentRetries`. |
|
|
609
|
-
| `--budget=N` / `PI_WORKFLOW_BUDGET=N` | Set an output-token ceiling across recorded subagent attempts, including failed provider attempts. `agent()` throws `WorkflowBudgetExceededError` before starting another model request once the ceiling is reached; agents already running may overshoot because the engine does not reserve per-agent estimates. |
|
|
609
|
+
| `--budget=N` / `PI_WORKFLOW_BUDGET=N` | Set an output-token ceiling across recorded subagent attempts, including failed provider attempts. `agent()` throws `WorkflowBudgetExceededError` before starting another model request once the ceiling is reached; agents already running may overshoot because the engine does not reserve per-agent estimates. The `workflow` tool has no budget field; see [Turn budget directive](#turn-budget-directive). |
|
|
610
610
|
| `--perf` / `PI_WORKFLOW_PERF=1` | Include internal timing aggregates. Usage/cost totals are reported separately from perf. |
|
|
611
611
|
| `PI_WORKFLOW_LANE_ITEM_LIMIT=N` | Cap retained progress lane items. |
|
|
612
612
|
|
|
@@ -636,16 +636,18 @@ do not count, and values under 1,000 tokens are ignored so a `+1` in ordinary
|
|
|
636
636
|
prose never caps a run. The first such token in a typed prompt wins; prompts
|
|
637
637
|
that extensions send are not read.
|
|
638
638
|
|
|
639
|
-
While the pool is active,
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
can only lower that ceiling, is refused the same way once the pool is spent, and
|
|
645
|
-
its spend is subtracted too. The pool clears when the run that
|
|
639
|
+
While the pool is active, every `workflow` tool call runs with the pool's
|
|
640
|
+
remainder as its budget (clamped to the run budget range), and each
|
|
641
|
+
synchronous run's output tokens are subtracted once it finishes, failed or not.
|
|
642
|
+
When less than one token remains, the tool refuses the call and tells the agent
|
|
643
|
+
the turn's budget is spent. The pool clears when the run that
|
|
646
644
|
received the prompt settles, so it still covers the run after pi retries a
|
|
647
645
|
provider error or compacts an overflowing context within that run.
|
|
648
646
|
|
|
647
|
+
The `workflow` tool has no `budget` field, so the host agent cannot choose a
|
|
648
|
+
ceiling: one it guesses can run out before a run's final stage. Without a pool,
|
|
649
|
+
a tool run takes `PI_WORKFLOW_BUDGET` if it is set and is otherwise unbudgeted.
|
|
650
|
+
|
|
649
651
|
Background runs receive the current remainder as their budget but do not draw
|
|
650
652
|
the pool down, because their usage lands after the turn. Parallel `workflow`
|
|
651
653
|
calls in one assistant message each see the same remainder before any of them
|
package/package.json
CHANGED
|
@@ -46,6 +46,7 @@ export default async function run(api) {
|
|
|
46
46
|
| `workflow(nameOrRef, args?)` | Runs another workflow as a sub-step and returns its result: a registered workflow by name, or an inline-format script file via `{ scriptPath }`, relative to the workflow cwd. `args` is a string or any JSON value, which the child reads as `input` and `args`. One level of nesting only. Rejects on an unknown name, an unreadable `scriptPath`, or a script that does not compile. |
|
|
47
47
|
| `phase(title)` / `log(message)` / `progress(event)` | Drive the live progress tree, logs, counters, and lanes. |
|
|
48
48
|
| `budget` | `{ total, spent(), remaining() }` in output tokens; `total` is `null` when the run has no budget. |
|
|
49
|
+
| `agents` | `{ max, remaining() }`: the run's live-agent limit and the admissions left before `agent()` throws. It is a live reading, not a reservation: agents running alongside, and retries, take admissions too, and journal replays take none, so a resumed run reads more. |
|
|
49
50
|
| `input` | The arguments verbatim: the `/workflow` command's string, or the JSON value the tool call or `workflow()` passed; `""` when none was given. |
|
|
50
51
|
| `args`, `cwd`, `signal` | The arguments as text (a string as is, any other JSON value as JSON text, `""` when none), the host working directory, and the run's abort signal. |
|
|
51
52
|
|
|
@@ -104,6 +105,7 @@ Host-written scripts in practice reached for `Promise.all` and never for `pipeli
|
|
|
104
105
|
## Quality patterns
|
|
105
106
|
|
|
106
107
|
- **Adversarial verify.** Give each finding to a separate verifier told to refute it, and keep only what survives. Thin evidence means not substantiated, not confirmed.
|
|
108
|
+
- **Filter after discovery, not during it.** When the user sets a reporting bar (evidence or experiment requirements, severity or MVP floors, net-LOC rules, dedupe against posted comments), give finders only where to look and ask for every candidate with a nameable impact; apply the bar in verify and synthesis. Finders told that returning nothing is preferred filter out their own candidates before any verifier sees them, and the run reports nothing.
|
|
107
109
|
- **Perspective-diverse verify.** For findings that matter, use two or three verifiers with different methods (read the callers, reproduce it, check the tests) and require agreement.
|
|
108
110
|
- **Judge panel.** Generate several independent answers or designs, then have a judge (or several) pick or merge them against stated criteria.
|
|
109
111
|
- **Loop until dry.** Repeat discovery rounds, deduping each round against everything seen so far, not only the last round. Stop when a round adds nothing new, or at a round cap or budget guard.
|
|
@@ -116,7 +118,7 @@ Host-written scripts in practice reached for `Promise.all` and never for `pipeli
|
|
|
116
118
|
|
|
117
119
|
- **Scout inline first, then orchestrate.** Use your own tools to find the work list (the files, failing tests, or findings) before writing the script, then fan out over that concrete list. Embed the list in the script or pass it as a JSON `args` value and read it from `api.input`.
|
|
118
120
|
- **One single-phase workflow per turn.** For multi-phase work (understand, design, implement, review), run one workflow per phase and read each result before choosing the next, rather than one script that guesses every phase in advance.
|
|
119
|
-
- **Scale to the request and the size guideline.** A focused question needs a few agents, a broad sweep more. The `<workflows>` section's size guideline line (set by the user with `/workflow:size`) bounds each workflow unless the user asks for more. The run's `maxAgents` limit (default 64) is a separate hard backstop, not a target.
|
|
121
|
+
- **Scale to the request and the size guideline.** A focused question needs a few agents, a broad sweep more. The `<workflows>` section's size guideline line (set by the user with `/workflow:size`) bounds each workflow unless the user asks for more. The run's `maxAgents` limit (default 64) is a separate hard backstop, not a target. Under a tight limit, size a fan-out from `api.agents.remaining()` and keep the agents later stages need, such as a final synthesize, rather than letting the fan-out spend them; the built-in review workflows batch verification this way.
|
|
120
122
|
- **Use `classify()` as a cheap gate** between stages: dedupe, triage, or route items without a full agent. It rejects when no classifier model is configured, so call it inside `api.parallel` or a `try`/`catch` and fall back to an agent or to keeping the item. Results are not journaled, so a resumed run classifies again.
|
|
121
123
|
|
|
122
124
|
```js
|
|
@@ -129,7 +131,7 @@ const keep = answers.real?.type === "bool" ? answers.real.probability > 0.5 : tr
|
|
|
129
131
|
|
|
130
132
|
## Budget
|
|
131
133
|
|
|
132
|
-
`api.budget.total` is the output-token ceiling (`null` when unset); `spent()` and `remaining()` are live. When the user's prompt carries a `+500k`-style directive, every run that turn is capped at the remaining pool
|
|
134
|
+
`api.budget.total` is the output-token ceiling (`null` when unset); `spent()` and `remaining()` are live. Only the user sets a budget: the `workflow` tool takes none. When the user's prompt carries a `+500k`-style directive, every run that turn is capped at the remaining pool, and synchronous runs draw the pool down; once it is spent the tool refuses further runs that turn. Once the ceiling is reached `agent()` rejects with `WorkflowBudgetExceededError` (a `null` slot inside `parallel`); agents already running can overshoot. Scale fleets from `total`, and guard every loop with it so an unbudgeted run still terminates:
|
|
133
135
|
|
|
134
136
|
```js
|
|
135
137
|
const width = api.budget.total ? Math.max(2, Math.min(8, Math.floor(api.budget.total / 40_000))) : 4;
|