pi-workflow-engine 0.12.0 → 0.13.0

package/.pi/extensions/pi-workflow-engine/workflows/perf-review.ts

@@ -1,11 +1,10 @@
+import { challengeFindings, parseChallengeArgs } from "../src/advisory-challenge.ts";
 import { Type } from "typebox";
 import {
-  AdvisoryReportSchema,
-  type AdvisoryCandidate,
-  type AdvisoryVerdict,
-} from "../src/advisory-schema.ts";
-import {
-  backfillAdvisoryFindings,
+  type AdvisoryVerified,
+  type AdvisoryLens,
+  synthesizeAdvisoryReport,
+  finishAdvisoryReport,
   emptyAdvisoryReport,
   formatEvidence,
   formatLocation,
@@ -19,7 +18,7 @@ import type { WorkflowApi, WorkflowMeta, WorkflowRunStats } from "../src/types.t
 export const meta: WorkflowMeta = {
   name: "perf-review",
   description: "Advisory-only performance review: scope slow path → per-lens bottleneck hypotheses → verify evidence → synthesize measurements and safe optimizations.",
-  phases: [{ title: "Scope" }, { title: "Find" }, { title: "Verify" }, { title: "Synthesize" }],
+  phases: [{ title: "Scope" }, { title: "Find" }, { title: "Verify" }, { title: "Challenge" }, { title: "Synthesize" }],
 };
 
 const ScopeSchema = Type.Object({
@@ -30,22 +29,7 @@ const ScopeSchema = Type.Object({
   knownMeasurements: Type.Optional(Type.String({ description: "Existing measurements, timings, or explicit lack of measurements." })),
 });
 
-interface PerfLens {
-  label: string;
-  category: string;
-  text: string;
-}
-
-type Candidate = AdvisoryCandidate;
-
-interface Verified extends Candidate {
-  verdict: AdvisoryVerdict["verdict"];
-  evidence: string[];
-  confidence?: AdvisoryVerdict["confidence"];
-  lens: PerfLens;
-}
-
-const PERF_LENSES: PerfLens[] = [
+const PERF_LENSES: AdvisoryLens[] = [
   { label: "algorithmic", category: "algorithmic", text: "Complexity, repeated scans, avoidable nested loops, or data-structure choices that grow poorly with input size." },
   { label: "io", category: "io", text: "Filesystem, subprocess, network, or other I/O costs on hot paths or startup paths." },
   { label: "concurrency", category: "concurrency", text: "Unnecessary serialization, missing batching, excessive fan-out, contention, or concurrency limits." },
@@ -54,13 +38,12 @@ const PERF_LENSES: PerfLens[] = [
   { label: "measurement", category: "measurement", text: "Missing, misleading, noisy, or insufficient benchmark/measurement design." },
 ];
 
-const TOOLS = DEFAULT_ADVISORY_TOOLS;
-const TOOL_HINTS = DEFAULT_ADVISORY_TOOL_HINTS;
 const PER_LENS = 4;
 
 export default async function run(api: WorkflowApi): Promise<unknown> {
-  const { agent, parallel, pipeline, phase, log, progress, args } = api;
-  const target = args.trim() || "repository performance";
+  const { agent, phase, log, progress, args } = api;
+  const challengeConfig = parseChallengeArgs(args);
+  const target = challengeConfig.args.trim() || "repository performance";
   let fileCount = 0;
   let rawCandidateCount = 0;
   let droppedCandidateCount = 0;
@@ -81,15 +64,15 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
       "Inspect repository structure, scripts, likely hot-path files, and any existing benchmark or measurement commands. " +
       "Prefer identifying what to measure before claiming bottlenecks. Return files, commands, summary, and known measurements or the lack of them. " +
       `This workflow will fan out across ${PERF_LENSES.length} lenses with up to ${PER_LENS} candidates per lens. Structured output only.`,
-    { phase: "Scope", label: "scope", tools: TOOLS, toolHints: TOOL_HINTS, profile: "medium", schema: ScopeSchema },
+    { phase: "Scope", label: "scope", tools: DEFAULT_ADVISORY_TOOLS, toolHints: DEFAULT_ADVISORY_TOOL_HINTS, profile: "medium", schema: ScopeSchema },
   );
 
   if (!scope || scope.files.length === 0) {
-    return emptyAdvisoryReport(
+    return finishAdvisoryReport(emptyAdvisoryReport(
       "No performance-relevant files were identified.",
       ["Provide a slow command, workload, file path, or user-visible latency concern to review."],
       makeStats(0, 0),
-    );
+    ), []);
   }
 
   fileCount = scope.files.length;
@@ -104,14 +87,10 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
     `## Summary\n${scope.summary}\n\n## Known measurements\n${scope.knownMeasurements ?? "(none known)"}\n` +
     (args.trim() ? `\n## User instructions (verbatim)\n${args.trim()}\n` : "");
 
-  phase("Find");
   const pipelineResult = await runLensVerificationPipeline({
-    api: { agent, parallel, pipeline, progress, log },
+    api,
     lenses: PERF_LENSES,
-    schedulingMode: "finder-barrier",
     perLens: PER_LENS,
-    tools: TOOLS,
-    toolHints: TOOL_HINTS,
     finderPrompt: (lens) =>
       `## Perf-review finder — ${lens.label}\n\n${scopeBlock}\n` +
       "This workflow is advisory-only: identify bottleneck hypotheses, measurement gaps, and safe optimization directions, but do not edit files.\n" +
@@ -124,67 +103,46 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
       `Location: ${formatLocation(candidate)}\nCategory: ${candidate.category}\nSummary: ${candidate.summary}\nImpact: ${candidate.impact}\n` +
       `Recommendation: ${candidate.recommendation ?? "(none supplied)"}\n\n` +
       "Read relevant files and package/scripts. Run only safe read-only measurement or inspection commands when useful. " +
-      "Return CONFIRMED, PLAUSIBLE, or REFUTED with evidence from code, scripts, config, or measurement output. " +
+      "Return CONFIRMED, PLAUSIBLE, NOT_SUBSTANTIATED, or REFUTED with evidence from code, scripts, config, or measurement output. " +
       "Default toward PLAUSIBLE or REFUTED when no measurement exists; do not overstate a bottleneck. Structured output only.",
-    makeVerified: (candidate, lens, judged): Verified => ({
-      ...candidate,
-      verdict: judged.verdict,
-      evidence: judged.evidence,
-      confidence: judged.confidence,
-      lens,
-    }),
   });
   rawCandidateCount += pipelineResult.rawCandidates;
   droppedCandidateCount += pipelineResult.dropped;
   refutedCandidateCount += pipelineResult.refuted;
-  const verified = pipelineResult.verified;
+  const { coverage } = pipelineResult;
+  const verified = await challengeFindings(api, pipelineResult.verified, scopeBlock, challengeConfig.options, coverage);
   const surviving = verified.filter((finding) => finding.verdict !== "REFUTED");
   const stats = makeStats(verified.length, surviving.length);
   publishVerifiedKeptProgress({ progress, log }, verified.length, surviving.length);
 
   if (surviving.length === 0) {
-    return emptyAdvisoryReport(
+    return finishAdvisoryReport(emptyAdvisoryReport(
       "No performance finding survived verification.",
       ["Add or run a focused measurement for the target workload before optimizing.", "Rerun perf-review with benchmark output or a narrower slow path."],
       stats,
-    );
+    ), coverage, verified);
   }
 
-  phase("Synthesize");
   const ranked = [...surviving].sort((a, b) => rank(a) - rank(b));
   const block = ranked
     .map(
       (finding, index) =>
-        `### [${index}] ${formatLocation(finding)} (${finding.verdict}, ${finding.category})\n` +
+        `### [${index}] IDs: ${finding.sourceCandidateIds.join(", ")} ${formatLocation(finding)} (${finding.verdict}, ${finding.category})\n` +
         `${finding.summary}\nImpact: ${finding.impact}\nEvidence: ${formatEvidence(finding.evidence)}\nRecommendation: ${finding.recommendation ?? "(none supplied)"}`,
     )
     .join("\n\n");
 
-  const report = await agent(
+  const resolved = await synthesizeAdvisoryReport(api,
     `## Synthesis: final perf-review report\n\n${ranked.length} candidates survived independent verification.\n\n${block}\n\n` +
-      "Produce the shared advisory report shape. Categories should be algorithmic, io, concurrency, startup, allocation, or measurement when applicable. " +
+      "Select findings by ID. " +
       "Severity is expected performance impact for the target workload. Prefer measurement recommendations before optimization recommendations when evidence is weak. " +
       "Recommendations must be safe advisory next actions, not patches. Include risky optimizations to avoid in recommendations or nextSteps when relevant. Structured output only.",
-    {
-      phase: "Synthesize",
-      label: "synthesize",
-      tools: [],
-      profile: "medium",
-      resume: "read-only",
-      schema: AdvisoryReportSchema,
-    },
+    ranked, coverage,
   );
-
-  if (!report) return emptyAdvisoryReport("Synthesis produced no output.", ["Inspect verifier evidence manually or rerun perf-review with a narrower target."], stats);
-
-  const findings = backfillAdvisoryFindings(report.findings, ranked, {
-    impact: "Performance impact not restated by synthesis.",
-    recommendation: "Measure the target workload before changing code.",
-  });
-  return { ...report, findings, stats };
+  return finishAdvisoryReport({ ...resolved, stats: { ...stats, kept: resolved.findings.length } }, coverage, verified);
 }
 
-function rank(finding: Verified): number {
+function rank(finding: AdvisoryVerified): number {
   const verdictRank = finding.verdict === "CONFIRMED" ? 0 : 1;
   const measurementPenalty = finding.category === "measurement" ? 1 : 0;
   return verdictRank + measurementPenalty;

package/.pi/extensions/pi-workflow-engine/workflows/refactor-scout.ts

@@ -1,11 +1,10 @@
+import { challengeFindings, parseChallengeArgs } from "../src/advisory-challenge.ts";
 import { Type } from "typebox";
 import {
-  AdvisoryReportSchema,
-  type AdvisoryCandidate,
-  type AdvisoryVerdict,
-} from "../src/advisory-schema.ts";
-import {
-  backfillAdvisoryFindings,
+  type AdvisoryVerified,
+  type AdvisoryLens,
+  synthesizeAdvisoryReport,
+  finishAdvisoryReport,
   emptyAdvisoryReport,
   formatEvidence,
   formatLocation,
@@ -19,7 +18,7 @@ import type { WorkflowApi, WorkflowMeta, WorkflowRunStats } from "../src/types.t
 export const meta: WorkflowMeta = {
   name: "refactor-scout",
   description: "Advisory-only refactor scout: scope → per-lens find → independent verify → synthesize safe refactor opportunities.",
-  phases: [{ title: "Scope" }, { title: "Find" }, { title: "Verify" }, { title: "Synthesize" }],
+  phases: [{ title: "Scope" }, { title: "Find" }, { title: "Verify" }, { title: "Challenge" }, { title: "Synthesize" }],
 };
 
 const ScopeSchema = Type.Object({
@@ -29,22 +28,7 @@ const ScopeSchema = Type.Object({
   conventions: Type.Optional(Type.String({ description: "Relevant project conventions from AGENTS.md / docs." })),
 });
 
-interface RefactorLens {
-  label: string;
-  category: string;
-  text: string;
-}
-
-type Candidate = AdvisoryCandidate;
-
-interface Verified extends Candidate {
-  verdict: AdvisoryVerdict["verdict"];
-  evidence: string[];
-  confidence?: AdvisoryVerdict["confidence"];
-  lens: RefactorLens;
-}
-
-const REFACTOR_LENSES: RefactorLens[] = [
+const REFACTOR_LENSES: AdvisoryLens[] = [
   { label: "duplication", category: "duplication", text: "Repeated logic, copy-pasted structures, or near-duplicate flows that could share one clearer implementation." },
   { label: "complexity", category: "complexity", text: "Oversized functions, tangled control flow, or abstractions that make local reasoning harder than necessary." },
   { label: "type-safety", category: "type-safety", text: "Weak typing, avoidable casts, unchecked shapes, or places stronger types would prevent mistakes." },
@@ -53,13 +37,12 @@ const REFACTOR_LENSES: RefactorLens[] = [
   { label: "conventions", category: "conventions", text: "Departures from project conventions, naming, dependency rules, or local idioms." },
 ];
 
-const TOOLS = DEFAULT_ADVISORY_TOOLS;
-const TOOL_HINTS = DEFAULT_ADVISORY_TOOL_HINTS;
 const PER_LENS = 5;
 
 export default async function run(api: WorkflowApi): Promise<unknown> {
-  const { agent, parallel, pipeline, phase, log, progress, args } = api;
-  const target = args.trim() || ".";
+  const { agent, phase, log, progress, args } = api;
+  const challengeConfig = parseChallengeArgs(args);
+  const target = challengeConfig.args.trim() || ".";
   let fileCount = 0;
   let rawCandidateCount = 0;
   let droppedCandidateCount = 0;
@@ -80,15 +63,15 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
       "Inspect repository structure, the target path or module, and relevant AGENTS.md / project docs conventions. " +
       "Return the concrete files that should be considered, a short summary, and any conventions that affect refactor advice. " +
       `This workflow will fan out across ${REFACTOR_LENSES.length} lenses with up to ${PER_LENS} candidates per lens. Structured output only.`,
-    { phase: "Scope", label: "scope", tools: TOOLS, toolHints: TOOL_HINTS, profile: "medium", schema: ScopeSchema },
+    { phase: "Scope", label: "scope", tools: DEFAULT_ADVISORY_TOOLS, toolHints: DEFAULT_ADVISORY_TOOL_HINTS, profile: "medium", schema: ScopeSchema },
   );
 
   if (!scope || scope.files.length === 0) {
-    return emptyAdvisoryReport(
+    return finishAdvisoryReport(emptyAdvisoryReport(
       "No files were identified for refactor scouting.",
       ["Provide a target path, module, or subsystem to scout for refactor opportunities."],
       makeStats(0, 0),
-    );
+    ), []);
   }
 
   fileCount = scope.files.length;
@@ -101,14 +84,10 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
     `## Summary\n${scope.summary}\n\n## Conventions\n${scope.conventions ?? "(none noted)"}\n` +
     (args.trim() ? `\n## User instructions (verbatim)\n${args.trim()}\n` : "");
 
-  phase("Find");
   const pipelineResult = await runLensVerificationPipeline({
-    api: { agent, parallel, pipeline, progress, log },
+    api,
     lenses: REFACTOR_LENSES,
-    schedulingMode: "finder-barrier",
     perLens: PER_LENS,
-    tools: TOOLS,
-    toolHints: TOOL_HINTS,
     finderPrompt: (lens) =>
       `## Refactor-scout finder — ${lens.label}\n\n${scopeBlock}\n` +
       "This workflow is advisory-only: do not edit files and do not propose broad rewrites.\n" +
@@ -120,65 +99,44 @@ export default async function run(api: WorkflowApi): Promise<unknown> {
       `## Refactor-scout verifier\n\n${scopeBlock}\n## Candidate\n` +
       `Location: ${formatLocation(candidate)}\nCategory: ${candidate.category}\nSummary: ${candidate.summary}\nImpact: ${candidate.impact}\n` +
       `Recommendation: ${candidate.recommendation ?? "(none supplied)"}\n\n` +
-      "Read the relevant files and return CONFIRMED, PLAUSIBLE, or REFUTED. " +
+      "Read the relevant files and return CONFIRMED, PLAUSIBLE, NOT_SUBSTANTIATED, or REFUTED. " +
       "Default toward REFUTED if the opportunity is generic, too broad, not evidenced by code, or lacks a safe first step. " +
       "Evidence must quote or cite code. Structured output only.",
-    makeVerified: (candidate, lens, judged): Verified => ({
-      ...candidate,
-      verdict: judged.verdict,
-      evidence: judged.evidence,
-      confidence: judged.confidence,
-      lens,
-    }),
   });
   rawCandidateCount += pipelineResult.rawCandidates;
   droppedCandidateCount += pipelineResult.dropped;
   refutedCandidateCount += pipelineResult.refuted;
-  const verified = pipelineResult.verified;
+  const { coverage } = pipelineResult;
+  const verified = await challengeFindings(api, pipelineResult.verified, scopeBlock, challengeConfig.options, coverage);
   const surviving = verified.filter((finding) => finding.verdict !== "REFUTED");
   const stats = makeStats(verified.length, surviving.length);
   publishVerifiedKeptProgress({ progress, log }, verified.length, surviving.length);
 
   if (surviving.length === 0) {
-    return emptyAdvisoryReport("No refactor opportunities survived verification.", ["Leave the scoped code unchanged unless a human reviewer has additional context."], stats);
+    return finishAdvisoryReport(emptyAdvisoryReport("No refactor opportunities survived verification.", ["Leave the scoped code unchanged unless a human reviewer has additional context."], stats), coverage, verified);
   }
 
-  phase("Synthesize");
   const ranked = [...surviving].sort((a, b) => rank(a) - rank(b));
   const block = ranked
     .map(
       (finding, index) =>
-        `### [${index}] ${formatLocation(finding)} (${finding.verdict}, ${finding.category})\n` +
+        `### [${index}] IDs: ${finding.sourceCandidateIds.join(", ")} ${formatLocation(finding)} (${finding.verdict}, ${finding.category})\n` +
         `${finding.summary}\nImpact: ${finding.impact}\nEvidence: ${formatEvidence(finding.evidence)}\nSafe first step: ${finding.recommendation ?? "(none supplied)"}`,
     )
     .join("\n\n");
 
-  const report = await agent(
+  const resolved = await synthesizeAdvisoryReport(api,
     `## Synthesis: final refactor-scout report\n\n${ranked.length} opportunities survived independent verification.\n\n${block}\n\n` +
       "Merge findings with the same root cause and rank highest leverage / lowest risk first. " +
-      "Return the shared advisory report shape. Categories should come from the verified candidates. " +
-      "Severity is maintenance or future-correctness impact. Confidence is high for CONFIRMED, medium for PLAUSIBLE unless verifier confidence says otherwise. " +
+      "Select findings by ID. " +
+      "Severity is maintenance or future-correctness impact. " +
       "Recommendations must be safe first refactor steps, not rewrites. Include concrete nextSteps for the host developer. Structured output only.",
-    {
-      phase: "Synthesize",
-      label: "synthesize",
-      tools: [],
-      profile: "medium",
-      resume: "read-only",
-      schema: AdvisoryReportSchema,
-    },
+    ranked, coverage,
   );
-
-  if (!report) return emptyAdvisoryReport("Synthesis produced no output.", ["Inspect verifier evidence manually or rerun the workflow with a narrower target."], stats);
-
-  const findings = backfillAdvisoryFindings(report.findings, ranked, {
-    impact: "Impact not restated by synthesis.",
-    recommendation: "Choose a small, behavior-preserving refactor first step.",
-  });
-  return { ...report, findings, stats };
+  return finishAdvisoryReport({ ...resolved, stats: { ...stats, kept: resolved.findings.length } }, coverage, verified);
 }
 
-function rank(finding: Verified): number {
+function rank(finding: AdvisoryVerified): number {
   const verdictRank = finding.verdict === "CONFIRMED" ? 0 : 1;
   const categoryRank = finding.category === "dead-code" || finding.category === "conventions" ? 2 : 0;
   return verdictRank + categoryRank;

package/USAGE.md

@@ -543,3 +543,62 @@ failures, timeouts, and host aborts are never provider-retried.
 - **Slow run**: lower fan-out, set `thinkingLevel`, or reduce `--concurrency`.
 - **Budget exhausted**: narrow the target, raise `--budget`, reduce fan-out/concurrency, or guard custom loops with `api.budget.remaining()`.
 - **Duplicate command/tool warnings while developing**: avoid loading both the global package and the working copy.
+
+### Advisory evidence and coverage
+
+`code-review`, `diagnose`, `refactor-scout`, and `perf-review` retain candidate IDs
+from discovery through verification and synthesis. Findings contain
+`sourceCandidateIds`; merged findings retain all contributing IDs, locations, and
+evidence. Code review uses `reviewAnchor` for the changed line that caused the
+finding. Other `locations` can identify unchanged callers or related files.
+
+These results include `status` (`complete` or `incomplete`), `coverage` with
+expected/completed/failed branch counts and failure reasons, `gaps`, and the
+`verification` records. Failed verification is not a refutation. Verifiers use
+`NOT_SUBSTANTIATED` when evidence is insufficient and `REFUTED` for concrete
+disproof. A failed finder or verifier prevents a clean conclusion. If synthesis
+fails, the result retains verified records and reports incomplete coverage.
+Synthesis selects or merges IDs; the workflow reconstructs evidence-bearing
+fields from those records.
+
+### Selective adversarial challenges
+
+Add `--challenge` to an advisory workflow to challenge up to three uncertain or
+high-risk findings. `--challenge=N` requires a non-negative integer and sets the
+limit, capped at ten; zero disables it. Invalid values stop the workflow before
+agents start. Ordinary confirmed, low-risk findings bypass this stage.
+
+```text
+/workflow code-review --challenge=2
+/workflow diagnose --challenge failing retry test
+```
+
+The challenger tries to disprove the finding or identify an alternative cause.
+A separate adjudicator receives both sides. Unresolved disagreement remains
+`NOT_SUBSTANTIATED`; finding no counterexample does not upgrade confidence.
+Custom workflows can use the `challengeFindings` recipe with `maxChallenges`
+and a `shouldChallenge` predicate, including selection based on domain-specific
+severity. It uses the existing `agent` and settled `parallel` calls.
+
+### Independently validated repair candidates
+
+The results viewer's fix action produces candidate patches in disposable
+worktrees. Each candidate retains the implementer's report separately from its
+`validation` record: reviewed baseline fingerprint, baseline commit, patch
+SHA-256, evaluator decision, executed commands, output, and process results.
+
+A fresh evaluator receives the exact reviewed baseline plus captured patch.
+The engine reconstructs another workspace and executes the evaluator's focused
+checks. Where applicable, a test-only patch and expected failure text let it
+check that a regression fails on the original baseline and passes after repair.
+Outcomes are `verified`, `rejected`, `blocked`, or `no-patch`. Missing required
+validation blocks verification. An empty patch does not prove that a finding
+needs no repair. Rejected and blocked candidates retain their patch and failure
+evidence for inspection. Cleanup failures also retain the validation evidence and
+record a `cleanupError`; a passed candidate becomes `blocked` if cleanup fails,
+while a rejected candidate remains `rejected`.
+
+Isolated agent results now include `baselineOid`. For independent evaluation,
+`agent()` accepts `candidatePatch: { baselineOid, patch }` with
+`isolation: "worktree"`; the engine checks the fresh baseline before applying
+it. Worktree isolation separates filesystem state; it is not a security sandbox.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-workflow-engine",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "type": "module",
   "description": "Zero-dependency Dynamax workflows for pi: live parallel subagents, typed handoffs, synthesized results.",
   "keywords": [