pi-background-tasks 0.9.0 → 1.0.3

package/src/core/fusion/orchestrator.ts

@@ -1,4 +1,5 @@
 import { randomBytes as nodeRandomBytes } from 'node:crypto';
+import { canonicalJson } from '../attested-pi-run.js';
 import { parseJsonText } from '../common.js';
 import { FUSION_BUDGET_POLICY, FusionBudget, assertChildOutputWithinContract } from './budget.js';
 import {
@@ -9,7 +10,10 @@ import {
 import {
   boundedEvaluationErrors,
   formatEvaluationErrors,
+  parseFusionValidationCandidateReport,
+  renderValidatedFusionValidationReport,
   validateFusionEvaluation,
+  validateFusionFindingAccounting,
 } from './evaluation.js';
 import { FusionChildRunError, runPiChild, type RunPiChildOptions } from './pi-child.js';
 import {
@@ -22,12 +26,14 @@ import {
   type AnonymousFusionCandidate,
 } from './prompts.js';
 import {
-  FUSION_BRAINSTORM_WORKFLOW,
-  resolveWorkflowCapability,
+  assertWorkflowCapability,
+  fusionWorkflowProfile,
   type FusionWorkflowProfile,
 } from './workflows.js';
+import { buildFusionSourcePolicy, sourcePolicyCanonicalBytes } from './source-policy.js';
 import {
-  FUSION_DEFAULT_CAPABILITY,
+  FUSION_INPUT_SCHEMA_VERSION,
+  FUSION_NO_TOOLS_CAPABILITY,
   FUSION_RESULT_SCHEMA_VERSION,
   FusionError,
   addFusionUsage,
@@ -46,6 +52,7 @@ import {
   type FusionSource,
   type FusionStage,
   type FusionUsage,
+  type FusionValidationFindingRecord,
   type ResolvedFusionModel,
   type ResolvedFusionModels,
 } from './types.js';
@@ -62,11 +69,11 @@ export interface FusionWorkflowInput {
   sessionId?: string | undefined;
   canonicalInput: FusionCanonicalInputV3;
   canonicalInputSerialized: string;
-  contextLedger: FusionContextOmissionLedgerV2;
+  contextLedger?: FusionContextOmissionLedgerV2 | undefined;
   config: FusionModelConfigV1;
   models: ResolvedFusionModels;
   candidateCapability?: FusionCapability | undefined;
-  /** Stage framing and capability policy. Defaults to the brainstorm workflow. */
+  /** Mandatory v5 workflow profile. */
   profile?: FusionWorkflowProfile | undefined;
   signal?: AbortSignal | undefined;
   onProgress?: FusionProgressSink | undefined;
@@ -100,6 +107,33 @@ function errorText(error: unknown): string {
   return error instanceof Error ? error.message : String(error);
 }
 
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function hasOnlyKeys(value: Record<string, unknown>, allowed: readonly string[]): boolean {
+  const allowedSet = new Set(allowed);
+  return Object.keys(value).every((key) => allowedSet.has(key));
+}
+
+function isStrictCleanCanonicalInput(value: unknown): boolean {
+  if (!isRecord(value)) return false;
+  if (!hasOnlyKeys(value, ['schema_version', 'workflow', 'cwd', 'request', 'context'])) return false;
+  const request = value['request'];
+  if (!isRecord(request)) return false;
+  if (!hasOnlyKeys(request, ['source', 'authority', 'text', 'sha256'])) return false;
+  const context = value['context'];
+  if (!isRecord(context)) return false;
+  if (!hasOnlyKeys(context, ['kind', 'policy_id', 'declared_sources'])) return false;
+  if (context['kind'] !== 'clean_task') return false;
+  const declaredSources = context['declared_sources'];
+  if (!Array.isArray(declaredSources)) return false;
+  for (const source of declaredSources) {
+    if (!isRecord(source) || !hasOnlyKeys(source, ['url', 'canonical_url', 'purpose', 'sha256'])) return false;
+  }
+  return true;
+}
+
 function asFusionError(error: unknown, artifactDir: string, messageOverride?: string): FusionError {
   if (error instanceof FusionError) {
     const details: FusionErrorDetails = {
@@ -191,6 +225,7 @@ function childOptions(
   signal: AbortSignal,
   slot?: CandidateSlot,
   toolCallLogPath?: string,
+  sourcePolicy?: { path: string; sha256: string },
 ): RunPiChildOptions {
   const out: RunPiChildOptions = {
     stage,
@@ -204,10 +239,14 @@ function childOptions(
   };
   if (slot !== undefined) out.slot = slot;
   if (toolCallLogPath !== undefined) out.toolCallLogPath = toolCallLogPath;
+  if (sourcePolicy !== undefined) out.sourcePolicy = sourcePolicy;
   return out;
 }
 
-function parseEvaluationAttempt(text: string): {
+function parseEvaluationAttempt(
+  text: string,
+  expectedValidationFindings: readonly FusionValidationFindingRecord[] | undefined,
+): {
   evaluation: FusionEvaluationV1 | undefined;
   errors: readonly string[];
 } {
@@ -221,8 +260,21 @@ function parseEvaluationAttempt(text: string): {
     };
   }
   const result = validateFusionEvaluation(parsed);
-  if (result.ok) return { evaluation: result.value, errors: [] };
-  return { evaluation: undefined, errors: result.errors };
+  if (!result.ok) return { evaluation: undefined, errors: result.errors };
+  if (
+    expectedValidationFindings === undefined &&
+    result.value.validation_accounting !== undefined
+  ) {
+    return {
+      evaluation: undefined,
+      errors: ['evaluation.validation_accounting is permitted only for fusion_validate'],
+    };
+  }
+  if (expectedValidationFindings !== undefined) {
+    const accountingErrors = validateEvaluationAccountsForSourceFindings(result.value, expectedValidationFindings);
+    if (accountingErrors.length > 0) return { evaluation: undefined, errors: accountingErrors };
+  }
+  return { evaluation: result.value, errors: [] };
 }
 
 function randomIndex(limit: number, randomBytes: FusionRandomBytes): number {
@@ -318,6 +370,63 @@ function anonymousCandidates(
   };
 }
 
+interface ValidationSourceData {
+  findings: readonly FusionValidationFindingRecord[];
+  verified: readonly string[];
+  limitations: readonly string[];
+}
+
+function validationSourceData(candidates: readonly [AnonymousFusionCandidate, AnonymousFusionCandidate, AnonymousFusionCandidate]): ValidationSourceData {
+  const findings: FusionValidationFindingRecord[] = [];
+  const verified: string[] = [];
+  const limitations: string[] = [];
+  for (const candidate of candidates) {
+    const report = parseFusionValidationCandidateReport(candidate.response, candidate.candidate_id);
+    findings.push(...report.findings);
+    verified.push(...report.verified);
+    limitations.push(...report.limitations);
+  }
+  return { findings, verified, limitations };
+}
+
+function validateEvaluationAccountsForSourceFindings(
+  evaluation: FusionEvaluationV1,
+  sourceFindings: readonly FusionValidationFindingRecord[],
+): readonly string[] {
+  const errors: string[] = [];
+  const accounting = evaluation.validation_accounting;
+  if (accounting === undefined) {
+    return ['validation evaluator output must include validation_accounting'];
+  }
+  const expected = sourceFindings.map((finding) => canonicalJson(finding)).sort();
+  const actual = accounting.findings.map((finding) => canonicalJson(finding)).sort();
+  if (expected.length !== actual.length || expected.some((value, index) => value !== actual[index])) {
+    errors.push('validation evaluator validation_accounting.findings must exactly equal host-assigned source findings');
+  }
+  errors.push(...validateFusionFindingAccounting(accounting));
+  return errors;
+}
+
+
+function resolveRunProfile(input: FusionWorkflowInput): FusionWorkflowProfile {
+  if (input.profile !== undefined) return fusionWorkflowProfile(input.profile.id);
+  const workflow = input.canonicalInput.workflow;
+  const contextKind = input.canonicalInput.context?.kind;
+  if (workflow !== undefined && workflow !== 'reason') {
+    throw new FusionError(`fusion workflow profile is required for ${workflow} runs`, {
+      code: 'orchestration_failed',
+      childCreated: false,
+    });
+  }
+  if (contextKind === 'clean_task') {
+    throw new FusionError('fusion workflow profile is required for clean-task runs', {
+      code: 'orchestration_failed',
+      childCreated: false,
+    });
+  }
+  return fusionWorkflowProfile('reason');
+}
+
 export class FusionOrchestrator {
   private readonly childRunner: FusionChildRunner;
   private readonly randomBytes: FusionRandomBytes;
@@ -334,11 +443,28 @@ export class FusionOrchestrator {
   }
 
   async run(input: FusionWorkflowInput): Promise<FusionRunResult> {
-    const profile = input.profile ?? FUSION_BRAINSTORM_WORKFLOW;
-    // Workflow policy, not caller input: a fixed-capability workflow rejects each
-    // other capability here, before a single child exists, rather than silently
-    // substituting its own and running a review that never read the code.
-    const candidateCapability = resolveWorkflowCapability(profile, input.candidateCapability);
+    if (input.canonicalInput.schema_version !== FUSION_INPUT_SCHEMA_VERSION) {
+      throw new FusionError('fusion orchestrator accepts only v5 canonical input', {
+        code: 'orchestration_failed',
+        childCreated: false,
+      });
+    }
+    const profile = resolveRunProfile(input);
+    const inputWorkflow = input.canonicalInput.workflow ?? profile.id;
+    const inputContextKind = input.canonicalInput.context?.kind ?? 'session_projection';
+    if (inputWorkflow !== profile.id || inputContextKind !== profile.contextKind) {
+      throw new FusionError(
+        `fusion workflow profile ${profile.id} is incompatible with canonical input workflow=${String(inputWorkflow)} context=${String(inputContextKind)}`,
+        { code: 'orchestration_failed', childCreated: false },
+      );
+    }
+    if (profile.contextKind === 'clean_task' && !isStrictCleanCanonicalInput(input.canonicalInput)) {
+      throw new FusionError('clean-task fusion input must not carry parent context fields and must match the strict clean canonical shape', {
+        code: 'orchestration_failed',
+        childCreated: false,
+      });
+    }
+    const candidateCapability = assertWorkflowCapability(profile, input.candidateCapability);
     const storeOptions: CreateFusionArtifactStoreOptions = {
       cwd: input.cwd,
       profile,
@@ -347,24 +473,63 @@ export class FusionOrchestrator {
       models: input.models,
       capabilities: {
         candidate: candidateCapability,
-        evaluation: FUSION_DEFAULT_CAPABILITY,
-        merge: FUSION_DEFAULT_CAPABILITY,
+        evaluation: FUSION_NO_TOOLS_CAPABILITY,
+        merge: FUSION_NO_TOOLS_CAPABILITY,
       },
     };
     if (input.sessionId !== undefined) storeOptions.sessionId = input.sessionId;
     if (this.now !== undefined) storeOptions.now = this.now;
+    let serializedParsed: unknown;
+    try {
+      serializedParsed = parseJsonText(input.canonicalInputSerialized);
+    } catch (error) {
+      throw new FusionError(`fusion canonical input artifact is not valid JSON: ${errorText(error)}`, {
+        code: 'orchestration_failed',
+        childCreated: false,
+      });
+    }
+    if (canonicalJson(serializedParsed) !== canonicalJson(input.canonicalInput)) {
+      throw new FusionError('fusion canonical input serialized bytes do not match canonical input object', {
+        code: 'orchestration_failed',
+        childCreated: false,
+      });
+    }
     const store = await this.createArtifactStore(storeOptions);
     input.onProgress?.({ type: 'state', state: 'initializing' });
     const usage = createEmptyFusionUsage();
     const calibrationWarnings: FusionCalibrationViolation[] = [];
     try {
       await store.writeCanonicalInput(input.canonicalInputSerialized);
-      await store.writeContextLedger(input.contextLedger);
+      if (inputContextKind === 'session_projection') {
+        if (input.contextLedger === undefined) {
+          throw new FusionError('session-projection fusion input requires an omission ledger artifact', {
+            code: 'orchestration_failed',
+            childCreated: false,
+          });
+        }
+        await store.writeContextLedger(input.contextLedger);
+      } else if (input.contextLedger !== undefined) {
+        throw new FusionError('clean-task fusion input must not carry a parent omission ledger', {
+          code: 'orchestration_failed',
+          childCreated: false,
+        });
+      }
+      if (profile.id === 'research') {
+        const cleanContext = input.canonicalInput.context;
+        if (cleanContext?.kind !== 'clean_task') {
+          throw new FusionError('research workflow requires a clean-task canonical input', {
+            code: 'orchestration_failed',
+            childCreated: false,
+          });
+        }
+        const policy = buildFusionSourcePolicy(input.cwd, cleanContext.declared_sources);
+        await store.writeSourcePolicy(sourcePolicyCanonicalBytes(policy));
+      }
       // Deterministic size accounting for the whole workflow, performed before
       // a single child process exists. A rejection here launches zero children.
       const budget = new FusionBudget(
         input.models,
-        input.canonicalInput.conversation_projection.policy.id,
+        input.canonicalInput.context?.policy_id ?? 'fusion-session-projection-v1',
         candidateCapability,
         profile,
       );
@@ -393,8 +558,13 @@ export class FusionOrchestrator {
       input.onProgress?.({ type: 'state', state: 'candidates_complete' });
 
       const shuffled = anonymousCandidates(candidateResults, shuffledSlots(this.randomBytes));
+      const validationData = profile.id === 'validate' ? validationSourceData(shuffled.candidates) : undefined;
       await store.setAnonymousMap(shuffled.map);
-      const blindInput = buildBlindEvaluationInput(input.canonicalInput, shuffled.candidates);
+      const blindInput = buildBlindEvaluationInput(
+        input.canonicalInput,
+        shuffled.candidates,
+        validationData?.findings,
+      );
       await store.writeBlindCandidates(buildEvaluationPrompt(blindInput));
 
       await store.transition('evaluating');
@@ -407,6 +577,7 @@ export class FusionOrchestrator {
         budget,
         calibrationWarnings,
         profile,
+        validationData?.findings,
       );
       await store.writeEvaluationJson(evaluation);
       await store.transition('evaluation_complete');
@@ -428,7 +599,7 @@ export class FusionOrchestrator {
         mergePrompt,
         input.signal ?? new AbortController().signal,
         // Stage policy, not caller input: evaluator and merger are always reasoning-only.
-        FUSION_DEFAULT_CAPABILITY,
+        FUSION_NO_TOOLS_CAPABILITY,
         undefined,
         'md',
       );
@@ -445,12 +616,24 @@ export class FusionOrchestrator {
         merged,
       );
       assertChildOutputWithinContract('merge', merged.text);
-      await store.writeMerged(merged.text);
+      let finalMergedText = merged.text;
+      if (profile.id === 'validate') {
+        const accounting = evaluation.validation_accounting;
+        if (accounting === undefined) {
+          throw new FusionError('fusion_validate evaluation completed without validation accounting', {
+            code: 'evaluation_invalid',
+            stage: 'merge',
+          });
+        }
+        finalMergedText = renderValidatedFusionValidationReport(accounting, validationData);
+      }
+      if (finalMergedText !== merged.text) assertChildOutputWithinContract('merge', finalMergedText);
+      await store.writeMerged(finalMergedText);
       await store.setUsage(usage);
       await store.transition('completed');
       input.onProgress?.({ type: 'completed', runId: store.runId, artifactDir: store.artifactDir });
       return {
-        mergedText: merged.text,
+        mergedText: finalMergedText,
         details: {
           schema_version: FUSION_RESULT_SCHEMA_VERSION,
           run_id: store.runId,
@@ -458,6 +641,8 @@ export class FusionOrchestrator {
           source: input.source,
           status: 'completed',
           artifact_dir: store.artifactDir,
+          context: { kind: inputContextKind, policy_id: input.canonicalInput.context?.policy_id ?? 'fusion-session-projection-v1' },
+          tool_policy: { candidate_tools: profile.candidateTools, evaluation_tools: [], merge_tools: [] },
           models: store.snapshot().models,
           evaluator_attempts: store
             .snapshot()
@@ -596,6 +781,7 @@ export class FusionOrchestrator {
     budget: FusionBudget,
     calibrationWarnings: FusionCalibrationViolation[],
     profile: FusionWorkflowProfile,
+    expectedValidationFindings: readonly FusionValidationFindingRecord[] | undefined,
   ): Promise<FusionEvaluationV1> {
     const firstPrompt = buildEvaluationPrompt(blindInput);
     budget.assertStagePrompt('evaluation', profile.evaluatorSystemPrompt, firstPrompt);
@@ -609,6 +795,7 @@ export class FusionOrchestrator {
       1,
       false,
       profile,
+      expectedValidationFindings,
     );
     if (first.evaluation !== undefined) return first.evaluation;
     const errors = boundedEvaluationErrors(first.errors);
@@ -634,6 +821,7 @@ export class FusionOrchestrator {
       2,
       true,
       profile,
+      expectedValidationFindings,
     );
     if (second.evaluation !== undefined) return second.evaluation;
     throw new FusionError(
@@ -656,6 +844,7 @@ export class FusionOrchestrator {
     attempt: 1 | 2,
     repair: boolean,
     profile: FusionWorkflowProfile,
+    expectedValidationFindings: readonly FusionValidationFindingRecord[] | undefined,
   ): Promise<EvaluationAttemptResult> {
     input.onProgress?.({ type: 'evaluation_started', attempt, repair });
     const systemPrompt = repair
@@ -671,7 +860,7 @@ export class FusionOrchestrator {
       prompt,
       input.signal ?? new AbortController().signal,
       // Stage policy, not caller input: evaluator and merger are always reasoning-only.
-      FUSION_DEFAULT_CAPABILITY,
+      FUSION_NO_TOOLS_CAPABILITY,
       undefined,
       'txt',
       attempt,
@@ -691,7 +880,7 @@ export class FusionOrchestrator {
     await store.setUsage(usage);
     // Bound the evaluator output before it can be embedded in a repair prompt.
     assertChildOutputWithinContract('evaluation', result.text);
-    const parsed = parseEvaluationAttempt(result.text);
+    const parsed = parseEvaluationAttempt(result.text, expectedValidationFindings);
     return { result, evaluation: parsed.evaluation, errors: parsed.errors };
   }
 
@@ -753,6 +942,10 @@ export class FusionOrchestrator {
         capability !== 'reason'
           ? store.childToolCallLogPath(stage, slot, logicalAttempt)
           : undefined;
+      const sourcePolicy =
+        capability === 'research'
+          ? store.sourcePolicyLaunchReference()
+          : undefined;
       try {
         return await this.childRunner(
           childOptions(
@@ -766,6 +959,7 @@ export class FusionOrchestrator {
             signal,
             slot,
             toolCallLogPath,
+            sourcePolicy,
           ),
         );
       } catch (error) {