pi-background-tasks 2.0.0 → 2.1.2

package/src/core/fusion/prompts.ts

@@ -1,4 +1,5 @@
 import { canonicalJson } from '../attested-pi-run.js';
+import { FUSION_CANDIDATE_OUTPUT_CONTRACT_INSTRUCTION } from './output-contract.js';
 import {
   FUSION_EVALUATION_SCHEMA_VERSION,
   FUSION_VALIDATE_CAPABILITY,
@@ -33,6 +34,8 @@ ${FUSION_CANONICAL_INPUT_GUIDE}
 
 Produce the strongest direct answer you can for the request using that context.
 
+${FUSION_CANDIDATE_OUTPUT_CONTRACT_INSTRUCTION}
+
 Do not invent process metadata. Do not mention provider names, model names, slots, or hidden workflow details. Do not specialize the answer; each child receives the same instruction. Output only the answer text.`;
 
 export const FUSION_INSPECT_CANONICAL_INPUT_GUIDE = `The JSON input contains only workflow, cwd, request, and clean-task context. context.kind is "clean_task".
@@ -47,6 +50,8 @@ ${FUSION_INSPECT_CANONICAL_INPUT_GUIDE}
 
 Produce the strongest direct answer you can for the request using that context.
 
+${FUSION_CANDIDATE_OUTPUT_CONTRACT_INSTRUCTION}
+
 Do not invent process metadata. Do not mention provider names, model names, slots, or hidden workflow details. Do not specialize the answer; each child receives the same instruction. Output only the answer text.`;
 
 export const FUSION_RESEARCH_CANONICAL_INPUT_GUIDE = `The JSON input contains only workflow, cwd, request, and clean-task context. Research uses context.kind "clean_task" with declared_sources: the only initial public URLs fusion_web_fetch may initiate. Redirects are followed only by the fetcher after public-address checks; fusion_web_fetch is targeted URL fetch, not search.
@@ -61,6 +66,8 @@ ${FUSION_RESEARCH_CANONICAL_INPUT_GUIDE}
 
 Produce the strongest direct answer you can for the request using that context.
 
+${FUSION_CANDIDATE_OUTPUT_CONTRACT_INSTRUCTION}
+
 Do not invent process metadata. Do not mention provider names, model names, slots, or hidden workflow details. Do not specialize the answer; each child receives the same instruction. Output only the answer text.`;
 
 export function fusionCandidateSystemPrompt(capability: FusionCapability): string {
@@ -198,6 +205,8 @@ Stay in scope. Validate what the request names. Do not propose unrelated refacto
 
 Close with what you verified and what you could not cover.
 
+${FUSION_CANDIDATE_OUTPUT_CONTRACT_INSTRUCTION}
+
 Do not invent process metadata. Do not mention provider names, model names, slots, or hidden workflow details. Do not specialize the report; each child receives the same instruction. Output only the required JSON.`;
 
 /**
@@ -301,7 +310,8 @@ export function buildBlindEvaluationInput(
     canonical_input: canonicalInput,
     candidates,
   };
-  if (validationSourceFindings !== undefined) input.validation_source_findings = validationSourceFindings;
+  if (validationSourceFindings !== undefined)
+    input.validation_source_findings = validationSourceFindings;
   return input;
 }
 

package/src/core/fusion/result-package.ts

@@ -61,7 +61,16 @@ function usage(value: unknown, artifactDir: string): FusionUsage {
   if (!isRecord(value)) fail('details.usage must be an object', artifactDir);
   assertOnlyKeys(
     value,
-    ['input', 'output', 'cacheRead', 'cacheWrite', 'totalTokens', 'cost'],
+    [
+      'input',
+      'output',
+      'cacheRead',
+      'cacheWrite',
+      'cacheWrite1h',
+      'reasoning',
+      'totalTokens',
+      'cost',
+    ],
     'details.usage',
     artifactDir,
   );
@@ -78,11 +87,29 @@ function usage(value: unknown, artifactDir: string): FusionUsage {
       fail(`${label} is invalid`, artifactDir);
     return entry;
   };
+  const output = finiteNonnegative(value['output'], 'details.usage.output');
+  const cacheWrite = finiteNonnegative(value['cacheWrite'], 'details.usage.cacheWrite');
+  const cacheWrite1h =
+    value['cacheWrite1h'] === undefined
+      ? undefined
+      : finiteNonnegative(value['cacheWrite1h'], 'details.usage.cacheWrite1h');
+  const reasoning =
+    value['reasoning'] === undefined
+      ? undefined
+      : finiteNonnegative(value['reasoning'], 'details.usage.reasoning');
+  if (cacheWrite1h !== undefined && cacheWrite1h > cacheWrite) {
+    fail('details.usage.cacheWrite1h must not exceed cacheWrite', artifactDir);
+  }
+  if (reasoning !== undefined && reasoning > output) {
+    fail('details.usage.reasoning must not exceed output', artifactDir);
+  }
   return {
     input: finiteNonnegative(value['input'], 'details.usage.input'),
-    output: finiteNonnegative(value['output'], 'details.usage.output'),
+    output,
     cacheRead: finiteNonnegative(value['cacheRead'], 'details.usage.cacheRead'),
-    cacheWrite: finiteNonnegative(value['cacheWrite'], 'details.usage.cacheWrite'),
+    cacheWrite,
+    ...(cacheWrite1h === undefined ? {} : { cacheWrite1h }),
+    ...(reasoning === undefined ? {} : { reasoning }),
     totalTokens: finiteNonnegative(value['totalTokens'], 'details.usage.totalTokens'),
     cost: {
       input: finiteNonnegative(cost['input'], 'details.usage.cost.input'),

package/src/core/fusion/types.ts

@@ -522,6 +522,8 @@ export function cloneFusionUsage(usage: FusionUsage): FusionUsage {
     output: usage.output,
     cacheRead: usage.cacheRead,
     cacheWrite: usage.cacheWrite,
+    ...(usage.cacheWrite1h === undefined ? {} : { cacheWrite1h: usage.cacheWrite1h }),
+    ...(usage.reasoning === undefined ? {} : { reasoning: usage.reasoning }),
     totalTokens: usage.totalTokens,
     cost: {
       input: usage.cost.input,
@@ -538,6 +540,12 @@ export function addFusionUsage(target: FusionUsage, delta: FusionUsage): void {
   target.output += delta.output;
   target.cacheRead += delta.cacheRead;
   target.cacheWrite += delta.cacheWrite;
+  if (delta.cacheWrite1h !== undefined) {
+    target.cacheWrite1h = (target.cacheWrite1h ?? 0) + delta.cacheWrite1h;
+  }
+  if (delta.reasoning !== undefined) {
+    target.reasoning = (target.reasoning ?? 0) + delta.reasoning;
+  }
   target.totalTokens += delta.totalTokens;
   target.cost.input += delta.cost.input;
   target.cost.output += delta.cost.output;
@@ -546,6 +554,26 @@ export function addFusionUsage(target: FusionUsage, delta: FusionUsage): void {
   target.cost.total += delta.cost.total;
 }
 
+export type FusionRunStageProgressStatus = 'not_started' | 'incomplete' | 'completed';
+
+export interface FusionRunStageProgress {
+  status: FusionRunStageProgressStatus;
+  attempts_recorded: number;
+  children_created: number;
+  children_completed: number;
+  children_failed: number;
+  children_cancelled: number;
+  not_started_slots?: number | undefined;
+}
+
+export interface FusionRunProgress {
+  manifest_state: FusionState;
+  candidates: FusionRunStageProgress;
+  evaluation: FusionRunStageProgress;
+  merge: FusionRunStageProgress;
+  usage_so_far: FusionUsage;
+}
+
 export interface FusionResultBudgetDetails {
   policy_id: string;
   calibration_version: string;
@@ -707,6 +735,7 @@ export interface FusionErrorDetails {
   transient?: boolean;
   childCreated?: boolean;
   budget?: FusionBudgetErrorDetail;
+  runProgress?: FusionRunProgress;
 }
 
 export class FusionError extends Error {
@@ -718,6 +747,7 @@ export class FusionError extends Error {
   readonly transient: boolean;
   readonly childCreated: boolean;
   readonly budget: FusionBudgetErrorDetail | undefined;
+  readonly runProgress: FusionRunProgress | undefined;
 
   constructor(message: string, details: FusionErrorDetails) {
     super(message);
@@ -730,6 +760,7 @@ export class FusionError extends Error {
     this.transient = details.transient ?? false;
     this.childCreated = details.childCreated ?? true;
     this.budget = details.budget;
+    this.runProgress = details.runProgress;
   }
 }
 
@@ -772,6 +803,18 @@ export interface FusionToolCallTrace {
   summary: FusionToolCallLogSummary;
 }
 
+export interface FusionCandidateOutputRecovery {
+  kind: 'same_session_compression';
+  limit_bytes: number;
+  original_record_index: number;
+  replacement_record_index: number | null;
+  original_json_rendered_bytes: number;
+  replacement_json_rendered_bytes: number | null;
+  original_text_sha256: string;
+  original_text: string;
+  status: 'completed' | 'failed';
+}
+
 export interface FusionChildRunResult {
   stage: FusionStage;
   slot?: 1 | 2 | 3;
@@ -786,6 +829,7 @@ export interface FusionChildRunResult {
   firstRequestUsage?: FusionUsage;
   /** Number of provider requests represented by aggregate usage. */
   providerRequestCount?: number;
+  outputRecovery?: FusionCandidateOutputRecovery;
   events: Buffer;
   stderr: Buffer;
   exitCode: number;
@@ -793,11 +837,24 @@ export interface FusionChildRunResult {
   toolCallTrace?: FusionToolCallTrace;
 }
 
+export interface FusionAttemptOutputRecoveryRecord {
+  kind: 'same_session_compression';
+  status: 'completed' | 'failed';
+  limit_bytes: number;
+  original_response_path: string;
+  original_record_index: number;
+  replacement_record_index: number | null;
+  original_json_rendered_bytes: number;
+  replacement_json_rendered_bytes: number | null;
+  original_text_sha256: string;
+}
+
 export interface FusionAttemptArtifactRecord {
   stage: FusionStage;
   slot?: 1 | 2 | 3;
   attempt: number;
   status: 'completed' | 'failed' | 'cancelled';
+  child_created: boolean;
   prompt_path: string;
   events_path?: string;
   stderr_path?: string;
@@ -805,6 +862,7 @@ export interface FusionAttemptArtifactRecord {
   partial_response_path?: string;
   tool_calls_path?: string;
   tool_calls?: FusionToolCallLogSummary;
+  output_recovery?: FusionAttemptOutputRecoveryRecord;
   provider?: string;
   model?: string;
   qualifiedId?: string;

package/src/delegate-child-extension.ts

@@ -36,18 +36,17 @@ import {
  * - enforcing turn and tool-call limits;
  * - committing exactly one result package atomically.
  *
- * Measured Pi 0.83 behaviour this design accounts for (see
- * `tests/scripted-provider/pi-hook-contract.test.ts`):
+ * Measured behavior across the supported Pi lines (see
+ * `tests/scripted-provider/pi-hook-contract.test.ts` and `scripts/test-compat.ts`):
  *
- * - Throwing from a `context` handler does NOT stop the provider call. Pi
- *   catches the exception and dispatches anyway. A throw is therefore never used
- *   as a barrier here.
- * - `ctx.abort()` does not skip the provider call site, but the call receives an
- *   already-aborted signal and the run terminates. That is the barrier used.
- * - Because neither mechanism is a hard admission gate on its own, the guard
- *   ALSO replaces the offending content in the returned message set. Even a
- *   provider that ignored the aborted signal could not transmit the content,
- *   because the content is no longer there.
+ * - Throwing from a `context` handler does NOT stop dispatch. Pi catches the
+ *   exception and continues, so a throw is never used as a barrier here.
+ * - `ctx.abort()` blocks transport in both supported modes: Pi 0.81.1-0.83.0
+ *   invoke the provider with an already-aborted signal, while Pi 0.84.0 skips
+ *   the provider entry point during signal-aware auth resolution.
+ * - The guard ALSO replaces the offending content in the returned message set.
+ *   Even a non-conforming provider could not transmit the content because the
+ *   content is no longer there.
  */
 
 const SPILL_DIRNAME = 'spill';
@@ -471,8 +470,8 @@ export default function delegateChildExtension(pi: ExtensionAPI): void {
       if (verdict.withinBudget) return undefined;
       const message = `delegate child context reached ${String(verdict.measuredTokens)} input tokens against a ${String(verdict.allowedTokens)}-token allowance on route ${seed.route.qualified_id}, over by ${String(verdict.overageTokens)}; estimator family ${verdict.rateSource.family}, source ${verdict.rateSource.source}, backed=${String(verdict.backed)}, dominant_byte_class=${verdict.dominantByteClass}, rate ${String(verdict.rateSource.effective_rate_bytes_per_token_x100)}/100 B/tok + ${String(verdict.rateSource.affine_f_tokens)} tokens`;
       latch('provider_context_budget_exhausted', message);
-      // Barrier one: terminate the run. Measured on Pi 0.83, this hands the
-      // provider call an already-aborted signal and stops further turns.
+      // Barrier one: terminate the run. Depending on the supported Pi line,
+      // this skips provider dispatch or hands it an already-aborted signal.
       ctx.abort();
       // Barrier two: remove the content itself, so the request could not carry it
       // even if a provider ignored the aborted signal. Retaining only the first

package/src/fusion-child-extension.ts

@@ -8,7 +8,7 @@ import {
   readFileSync,
   writeSync,
 } from 'node:fs';
-import { dirname } from 'node:path';
+import { dirname, isAbsolute } from 'node:path';
 import { parseJsonText } from './core/common.js';
 import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
 import { Type, type Static } from 'typebox';
@@ -32,11 +32,13 @@ import {
   parseFusionSourcePolicy,
 } from './core/fusion/source-policy.js';
 import {
+  applyFusionClaudePromptCachingScopeHeader,
   nonAnthropicFusionCacheObservation,
   normalizeFusionClaudeCachePayload,
   type FusionClaudeCacheObservation,
 } from './core/fusion/claude-cache.js';
 import {
+  FUSION_CANDIDATE_OUTPUT_RECOVERY_PATH_ENV,
   FUSION_CHILD_MAX_PROVIDER_REQUESTS,
   FUSION_CHILD_MAX_TOOL_CALLS,
   FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES,
@@ -59,12 +61,19 @@ import {
   type FusionRuntimeGuardCode,
   type FusionRuntimeGuardRecord,
 } from './core/fusion/child-protocol.js';
+import {
+  FUSION_CANDIDATE_MAX_OUTPUT_BYTES,
+  FUSION_CANDIDATE_OUTPUT_COMPRESSION_PROMPT,
+  fusionJsonRenderedTextBytes,
+} from './core/fusion/output-contract.js';
 
 export {
   FUSION_CLAUDE_CACHE_BREAKPOINT_LIMIT,
   FUSION_CLAUDE_CACHE_DEFAULT_RETENTION,
   FUSION_CLAUDE_CACHE_OBSERVATION_SCHEMA_VERSION,
   FUSION_CLAUDE_CACHE_RETENTION_ENV,
+  FUSION_CLAUDE_PROMPT_CACHING_SCOPE_BETA,
+  applyFusionClaudePromptCachingScopeHeader,
   nonAnthropicFusionCacheObservation,
   normalizeFusionClaudeCachePayload,
   resolveFusionClaudeCachePolicy,
@@ -75,6 +84,7 @@ export {
 } from './core/fusion/claude-cache.js';
 
 export {
+  FUSION_CANDIDATE_OUTPUT_RECOVERY_PATH_ENV,
   FUSION_CHILD_MAX_PROVIDER_REQUESTS,
   FUSION_CHILD_MAX_TOOL_CALLS,
   FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES,
@@ -103,6 +113,12 @@ export {
   type FusionRuntimeGuardRecord,
 } from './core/fusion/child-protocol.js';
 
+export {
+  FUSION_CANDIDATE_MAX_OUTPUT_BYTES,
+  FUSION_CANDIDATE_OUTPUT_COMPRESSION_PROMPT,
+  fusionJsonRenderedTextBytes,
+} from './core/fusion/output-contract.js';
+
 const FUSION_CHILD_O_NOFOLLOW = typeof constants.O_NOFOLLOW === 'number' ? constants.O_NOFOLLOW : 0;
 
 const FusionWebFetchParams = Type.Object(
@@ -253,6 +269,24 @@ function createToolCallLog(path: string): void {
   fsyncParentDirectorySync(path);
 }
 
+function createCandidateOutputRecoveryArtifact(path: string, text: string): void {
+  if (!isAbsolute(path)) {
+    throw new Error(`${FUSION_CANDIDATE_OUTPUT_RECOVERY_PATH_ENV} must be an absolute path`);
+  }
+  const bytes = Buffer.from(text, 'utf8');
+  withRegularFileDescriptorSync(
+    path,
+    constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | FUSION_CHILD_O_NOFOLLOW,
+    0o600,
+    'fusion oversized candidate response',
+    (fd) => {
+      writeAllSync(fd, bytes, 'fusion oversized candidate response');
+      fsyncSync(fd);
+    },
+  );
+  fsyncParentDirectorySync(path);
+}
+
 function appendToolCallLogLine(path: string, record: FusionToolCallLogRecord): void {
   // The log is an audit trail, not a payload copy: raw tool arguments/results may
   // contain secrets, so only byte counts and SHA-256 digests are persisted.
@@ -670,6 +704,15 @@ function fusionWebFetchResultText(result: Awaited<ReturnType<typeof fusionWebFet
  */
 export default function fusionChildExtension(pi: ExtensionAPI): void {
   const toolCallLogPath = process.env[FUSION_TOOL_CALL_LOG_PATH_ENV];
+  const candidateOutputRecoveryPath = process.env[FUSION_CANDIDATE_OUTPUT_RECOVERY_PATH_ENV];
+  if (
+    candidateOutputRecoveryPath !== undefined &&
+    (candidateOutputRecoveryPath.trim().length === 0 || !isAbsolute(candidateOutputRecoveryPath))
+  ) {
+    throw new Error(
+      `${FUSION_CANDIDATE_OUTPUT_RECOVERY_PATH_ENV} must be an absolute non-blank path`,
+    );
+  }
   const researchEnabled = process.env[FUSION_RESEARCH_ENABLED_ENV];
   if (researchEnabled !== undefined && researchEnabled !== '1') {
     throw new Error(`${FUSION_RESEARCH_ENABLED_ENV} must be unset or exactly 1`);
@@ -684,10 +727,17 @@ export default function fusionChildExtension(pi: ExtensionAPI): void {
   let providerRequestCount = 0;
   let toolCallCount = 0;
   let runtimeGuardFailed = false;
+  let outputRecoveryFailed = false;
+  let outputRecoveryPhase: 'eligible' | 'queued' | 'finished' = 'eligible';
   let settlementPublished = false;
   const childResultRecords: FusionChildResultMetadata[] = [];
   let pendingCacheObservation: FusionClaudeCacheObservation | undefined;
 
+  pi.on('before_provider_headers', (event, ctx) => {
+    if (ctx.model?.provider !== 'anthropic') return;
+    applyFusionClaudePromptCachingScopeHeader(event.headers);
+  });
+
   pi.on('before_provider_request', async (event, ctx) => {
     providerRequestCount += 1;
     if (runtimeGuardFailed) {
@@ -820,6 +870,15 @@ export default function fusionChildExtension(pi: ExtensionAPI): void {
     pi.on('tool_call', async (event, ctx) => {
       try {
         requireOpen('tool_call');
+        if (outputRecoveryPhase === 'queued') {
+          outputRecoveryFailed = true;
+          latchAuditProcessFailure();
+          ctx.abort();
+          return {
+            block: true,
+            reason: 'fusion candidate output compression is a no-tool continuation',
+          };
+        }
         if (runtimeGuardFailed) {
           ctx.abort();
           return { block: true, reason: 'fusion child runtime guard already refused the run' };
@@ -978,13 +1037,68 @@ export default function fusionChildExtension(pi: ExtensionAPI): void {
     if (event.message.role !== 'assistant') return;
     const cacheObservation = pendingCacheObservation;
     if (cacheObservation === undefined) {
+      // Authentication and other pre-transport failures can produce an assistant
+      // error without ever reaching before_provider_request. Preserve Pi's original
+      // provider diagnostic and let settlement report no_records; emitting a second
+      // cache-policy error would mask the actionable failure. A successful result
+      // without an observation remains a hard invariant violation.
+      if (event.message.stopReason === 'error') return;
       latchAuditProcessFailure();
       throw new Error('fusion child assistant result has no matching cache-policy observation');
     }
     pendingCacheObservation = undefined;
-    const record = buildFusionChildResultMetadata(event.message, cacheObservation);
+    const text = event.message.content
+      .flatMap((part) => (part.type === 'text' ? [part.text] : []))
+      .join('');
+    const renderedBytes = fusionJsonRenderedTextBytes(text);
+    const isReplacement = outputRecoveryPhase === 'queued';
+    const shouldRecover =
+      candidateOutputRecoveryPath !== undefined &&
+      outputRecoveryPhase === 'eligible' &&
+      event.message.stopReason === 'stop' &&
+      renderedBytes > FUSION_CANDIDATE_MAX_OUTPUT_BYTES;
+    const recoveryRole = isReplacement
+      ? 'replacement'
+      : shouldRecover
+        ? 'oversized_original'
+        : 'none';
+    const record = buildFusionChildResultMetadata(event.message, cacheObservation, {
+      candidateLimitBytes:
+        candidateOutputRecoveryPath === undefined ? null : FUSION_CANDIDATE_MAX_OUTPUT_BYTES,
+      recoveryRole,
+    });
+    if (shouldRecover) {
+      try {
+        createCandidateOutputRecoveryArtifact(candidateOutputRecoveryPath, text);
+      } catch (error) {
+        outputRecoveryFailed = true;
+        latchAuditProcessFailure();
+        throw error;
+      }
+    }
     await writeMetadata(record);
     childResultRecords.push(record);
+    if (shouldRecover) {
+      outputRecoveryPhase = 'queued';
+      try {
+        pi.setActiveTools([]);
+        pi.sendUserMessage(FUSION_CANDIDATE_OUTPUT_COMPRESSION_PROMPT, {
+          deliverAs: 'followUp',
+        });
+      } catch (error) {
+        outputRecoveryFailed = true;
+        latchAuditProcessFailure();
+        throw error;
+      }
+      return;
+    }
+    if (isReplacement) {
+      outputRecoveryPhase = 'finished';
+      if (renderedBytes > FUSION_CANDIDATE_MAX_OUTPUT_BYTES) {
+        outputRecoveryFailed = true;
+        latchAuditProcessFailure();
+      }
+    }
   });
   pi.on('agent_settled', async (_event, ctx) => {
     if (settlementPublished) {
@@ -1001,6 +1115,7 @@ export default function fusionChildExtension(pi: ExtensionAPI): void {
       childResultRecords,
       runtimeGuardFailed,
       cacheObservationFailed,
+      outputRecoveryFailed,
     );
     if (settlement.status !== 'complete') latchAuditProcessFailure();
     await writeSettlement(settlement);