pi-background-tasks 0.9.0 → 1.0.3

package/src/fusion-child-extension.ts

@@ -1,6 +1,5 @@
 import { createHash } from 'node:crypto';
-import { closeSync, fsyncSync, openSync, writeSync } from 'node:fs';
-import type { Usage } from '@earendil-works/pi-ai';
+import { closeSync, constants, fstatSync, fsyncSync, openSync, readFileSync, writeSync } from 'node:fs';
 import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
 import { Type, type Static } from 'typebox';
 import {
@@ -13,22 +12,40 @@ import {
   FusionWebFetchError,
   FUSION_WEB_FETCH_TIMEOUT_MS,
 } from './core/fusion/web-fetch.js';
+import {
+  canonicalizeFusionPublicUrl,
+  parseFusionSourcePolicy,
+} from './core/fusion/source-policy.js';
+import {
+  FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES,
+  FUSION_CHILD_RESULT_PREFIX,
+  FUSION_RESEARCH_ENABLED_ENV,
+  FUSION_SOURCE_POLICY_PATH_ENV,
+  FUSION_SOURCE_POLICY_SHA256_ENV,
+  FUSION_TOOL_CALL_LOG_PATH_ENV,
+  FUSION_TOOL_CALL_SEAL_SCHEMA_VERSION,
+  FUSION_TOOL_CALL_SEAL_SUFFIX,
+  buildFusionChildResultMetadata,
+  type FusionChildResultMetadata,
+} from './core/fusion/child-protocol.js';
 
-export const FUSION_CHILD_RESULT_SCHEMA_VERSION =
-  'pi-background-tasks.fusion-child-result.v2' as const;
-export const FUSION_CHILD_RESULT_PREFIX = '\u001ePI_FUSION_CHILD_RESULT ';
-export const FUSION_TOOL_CALL_LOG_PATH_ENV = 'PI_FUSION_TOOL_CALL_LOG_PATH';
-export const FUSION_RESEARCH_ENABLED_ENV = 'PI_FUSION_RESEARCH_ENABLED';
+export {
+  FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES,
+  FUSION_CHILD_RESULT_PREFIX,
+  FUSION_CHILD_RESULT_SCHEMA_VERSION,
+  FUSION_RESEARCH_ENABLED_ENV,
+  FUSION_SOURCE_POLICY_PATH_ENV,
+  FUSION_SOURCE_POLICY_SHA256_ENV,
+  FUSION_TOOL_CALL_LOG_PATH_ENV,
+  FUSION_TOOL_CALL_SEAL_SCHEMA_VERSION,
+  FUSION_TOOL_CALL_SEAL_SUFFIX,
+  buildFusionChildResultMetadata,
+  type FusionChildResultMetadata,
+  type FusionChildResultUsageMetadata,
+  type FusionChildTextBlockMetadata,
+} from './core/fusion/child-protocol.js';
 
-/**
- * Aggregate ceiling on tool-result bytes a single candidate child may accumulate.
- *
- * v1 deliberately has no tool-CALL cap, so this byte budget is the only bound on how much
- * a read-only candidate can pull into its context. 8 MiB is generous for targeted
- * grep/read investigation while still preventing an unbounded read loop from degrading
- * into an opaque provider-side context failure.
- */
-export const FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES = 8 * 1024 * 1024;
+const FUSION_CHILD_O_NOFOLLOW = typeof constants.O_NOFOLLOW === 'number' ? constants.O_NOFOLLOW : 0;
 
 const FusionWebFetchParams = Type.Object(
   {
@@ -58,30 +75,14 @@ interface FusionWebFetchDetails {
 }
 
 interface FusionWebFetchAuditMetadata {
-  url: string;
+  url?: string | undefined;
+  rejected_url_sha256?: string | undefined;
   final_url?: string | undefined;
   http_status?: number | undefined;
   response_bytes?: number | undefined;
   content_sha256?: string | undefined;
 }
 
-export interface FusionChildTextBlockMetadata {
-  utf8_bytes: number;
-  sha256: string;
-}
-
-export type FusionChildResultUsageMetadata = Usage;
-
-export interface FusionChildResultMetadata {
-  schema_version: typeof FUSION_CHILD_RESULT_SCHEMA_VERSION;
-  provider: string;
-  model: string;
-  stop_reason: string;
-  text_blocks: FusionChildTextBlockMetadata[];
-  text_sha256: string;
-  usage: FusionChildResultUsageMetadata;
-}
-
 function sha256(value: string | Buffer): string {
   return createHash('sha256').update(value).digest('hex');
 }
@@ -119,42 +120,32 @@ function appendToolCallLogLine(path: string, record: FusionToolCallLogRecord): v
   }
 }
 
-export function buildFusionChildResultMetadata(message: {
-  provider: string;
-  model: string;
-  stopReason: string;
-  content: ReadonlyArray<{ type: string; text?: string }>;
-  usage: Usage;
-}): FusionChildResultMetadata {
-  const textBlocks = message.content.flatMap((part) =>
-    part.type === 'text' && typeof part.text === 'string' ? [part.text] : [],
-  );
-  const usage: FusionChildResultUsageMetadata = {
-    input: message.usage.input,
-    output: message.usage.output,
-    cacheRead: message.usage.cacheRead,
-    cacheWrite: message.usage.cacheWrite,
-    totalTokens: message.usage.totalTokens,
-    cost: {
-      input: message.usage.cost.input,
-      output: message.usage.cost.output,
-      cacheRead: message.usage.cost.cacheRead,
-      cacheWrite: message.usage.cost.cacheWrite,
-      total: message.usage.cost.total,
-    },
-  };
-  return {
-    schema_version: FUSION_CHILD_RESULT_SCHEMA_VERSION,
-    provider: message.provider,
-    model: message.model,
-    stop_reason: message.stopReason,
-    text_blocks: textBlocks.map((text) => ({
-      utf8_bytes: Buffer.byteLength(text, 'utf8'),
-      sha256: sha256(text),
-    })),
-    text_sha256: sha256(textBlocks.join('')),
-    usage,
-  };
+function writeToolCallLogSeal(
+  path: string,
+  recordCount: number,
+  totalResultBytes: number,
+  complete: boolean,
+): void {
+  const logBytes = readFileSync(path);
+  const seal = {
+    schema_version: FUSION_TOOL_CALL_SEAL_SCHEMA_VERSION,
+    status: complete ? 'complete' : 'failed',
+    record_count: recordCount,
+    total_result_bytes: totalResultBytes,
+    log_sha256: sha256(logBytes),
+  } as const;
+  const bytes = Buffer.from(`${JSON.stringify(seal)}\n`, 'utf8');
+  let fd: number | undefined;
+  try {
+    fd = openSync(`${path}${FUSION_TOOL_CALL_SEAL_SUFFIX}`, 'wx', 0o600);
+    const written = writeSync(fd, bytes);
+    if (written !== bytes.length) {
+      throw new Error(`fusion tool-call seal short write: ${String(written)} of ${String(bytes.length)} bytes`);
+    }
+    fsyncSync(fd);
+  } finally {
+    if (fd !== undefined) closeSync(fd);
+  }
 }
 
 async function writeMetadata(record: FusionChildResultMetadata): Promise<void> {
@@ -199,7 +190,9 @@ function stringField(value: object, key: string): string | undefined {
 }
 
 function fetchAuditMetadataFromObject(value: object, fallbackUrl: string): FusionWebFetchAuditMetadata {
-  const metadata: FusionWebFetchAuditMetadata = { url: stringField(value, 'url') ?? fallbackUrl };
+  const metadata: FusionWebFetchAuditMetadata = {
+    url: stringField(value, 'url') ?? canonicalizeFusionPublicUrl(fallbackUrl),
+  };
   const finalUrl = stringField(value, 'final_url');
   if (finalUrl !== undefined) metadata.final_url = finalUrl;
   const status = numberField(value, 'status');
@@ -211,15 +204,48 @@ function fetchAuditMetadataFromObject(value: object, fallbackUrl: string): Fusio
   return metadata;
 }
 
-function fetchAuditMetadataFromError(error: unknown, fallbackUrl: string): FusionWebFetchAuditMetadata {
-  if (!(error instanceof FusionWebFetchError) || typeof error !== 'object' || error === null) {
-    return { url: fallbackUrl };
+function fetchAuditMetadataFromError(error: unknown, attemptedUrl: string): FusionWebFetchAuditMetadata {
+  const metadata: FusionWebFetchAuditMetadata = { rejected_url_sha256: sha256(attemptedUrl) };
+  if (error instanceof FusionWebFetchError && typeof error === 'object' && error !== null) {
+    const status = numberField(error, 'status');
+    if (status !== undefined) metadata.http_status = status;
+  }
+  return metadata;
+}
+
+
+function readRegularFileNoSymlinkSync(path: string, label: string): Buffer {
+  let fd: number | undefined;
+  try {
+    fd = openSync(path, constants.O_RDONLY | FUSION_CHILD_O_NOFOLLOW);
+  } catch (error) {
+    if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ELOOP') {
+      throw new Error(`${label} at ${path} is a symlink; refusing to follow it`);
+    }
+    throw error;
+  }
+  try {
+    const stats = fstatSync(fd);
+    if (!stats.isFile()) throw new Error(`${label} at ${path} is not a regular file`);
+    return readFileSync(fd);
+  } finally {
+    if (fd !== undefined) closeSync(fd);
   }
-  const result = Reflect.get(error, 'result');
-  if (typeof result === 'object' && result !== null) {
-    return fetchAuditMetadataFromObject(result, fallbackUrl);
+}
+
+function loadDeclaredResearchUrls(): ReadonlySet<string> {
+  const policyPath = process.env[FUSION_SOURCE_POLICY_PATH_ENV];
+  const expectedHash = process.env[FUSION_SOURCE_POLICY_SHA256_ENV];
+  if (policyPath === undefined || expectedHash === undefined) {
+    throw new Error(`${FUSION_WEB_FETCH_TOOL_NAME} research mode requires source policy path and sha256`);
   }
-  return fetchAuditMetadataFromObject(error, fallbackUrl);
+  if (!/^[0-9a-f]{64}$/.test(expectedHash)) throw new Error('fusion source policy hash is malformed');
+  const bytes = readRegularFileNoSymlinkSync(policyPath, 'fusion source policy');
+  if (sha256(bytes) !== expectedHash) throw new Error('fusion source policy hash mismatch');
+  const text = bytes.toString('utf8');
+  if (!Buffer.from(text, 'utf8').equals(bytes)) throw new Error('fusion source policy is not UTF-8');
+  const parsed = parseFusionSourcePolicy(JSON.parse(text));
+  return new Set(parsed.sources.map((source) => source.canonical_url));
 }
 
 function fusionWebFetchResultText(result: Awaited<ReturnType<typeof fusionWebFetch>>): string {
@@ -255,6 +281,7 @@ export default function fusionChildExtension(pi: ExtensionAPI): void {
   if (researchEnabled === '1' && toolCallLogPath === undefined) {
     throw new Error(`${FUSION_WEB_FETCH_TOOL_NAME} research mode requires ${FUSION_TOOL_CALL_LOG_PATH_ENV}`);
   }
+  const declaredResearchUrls = researchEnabled === '1' ? loadDeclaredResearchUrls() : undefined;
   const fetchAuditMetadata = new Map<string, FusionWebFetchAuditMetadata>();
   if (toolCallLogPath !== undefined) {
     // Create the log immediately, before tools can run. Without this, an absent file
@@ -264,54 +291,59 @@ export default function fusionChildExtension(pi: ExtensionAPI): void {
     closeSync(openSync(toolCallLogPath, 'a', 0o600));
     let ordinal = 0;
     let totalToolResultBytes = 0;
+    let auditFailed = false;
     const starts = new Map<string, number>();
     pi.on('tool_call', (event) => {
       starts.set(event.toolCallId, Date.now());
     });
     pi.on('tool_result', (event) => {
-      const start = starts.get(event.toolCallId);
-      if (start === undefined) {
-        throw new Error(`fusion tool-call log missing start for ${event.toolCallId}`);
-      }
-      starts.delete(event.toolCallId);
-      const argumentsBytes = utf8JsonBytes(event.input, 'arguments');
-      const resultBytes = utf8JsonBytes(
-        {
-          content: event.content,
-          details: event.details,
-          isError: event.isError,
-          usage: event.usage,
-        },
-        'result',
-      );
-      const fetchMetadata = fetchAuditMetadata.get(event.toolCallId);
-      fetchAuditMetadata.delete(event.toolCallId);
-      const record: FusionToolCallLogRecord = {
-        schema_version: FUSION_TOOL_CALL_LOG_SCHEMA_VERSION,
-        ordinal,
-        tool_name: event.toolName,
-        arguments_sha256: sha256(argumentsBytes),
-        arguments_bytes: argumentsBytes.length,
-        result_bytes: resultBytes.length,
-        result_sha256: sha256(resultBytes),
-        status: event.isError === true ? 'error' : 'ok',
-        duration_ms: Math.max(0, Date.now() - start),
-        ...(fetchMetadata === undefined ? {} : fetchMetadata),
-      };
-      ordinal += 1;
-      appendToolCallLogLine(toolCallLogPath, record);
-      // Aggregate output ceiling. There is no tool-CALL cap in v1 by design, so bytes are
-      // the only bound on how much a read-only candidate can pull into its context. The
-      // record is durable before this check, so the offending call stays auditable; the
-      // failure is loud rather than a truncation, because a silently shortened tool result
-      // would corrupt the candidate's reasoning with no signal at all.
-      totalToolResultBytes += resultBytes.length;
-      if (totalToolResultBytes > FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES) {
-        throw new Error(
-          `fusion candidate exceeded the aggregate tool-output budget: ${String(totalToolResultBytes)} bytes across ${String(ordinal)} calls exceeds ${String(FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES)}`,
+      try {
+        const start = starts.get(event.toolCallId);
+        if (start === undefined) {
+          throw new Error(`fusion tool-call log missing start for ${event.toolCallId}`);
+        }
+        starts.delete(event.toolCallId);
+        const argumentsBytes = utf8JsonBytes(event.input, 'arguments');
+        const resultBytes = utf8JsonBytes(
+          {
+            content: event.content,
+            details: event.details,
+            isError: event.isError,
+            usage: event.usage,
+          },
+          'result',
         );
+        const fetchMetadata = fetchAuditMetadata.get(event.toolCallId);
+        fetchAuditMetadata.delete(event.toolCallId);
+        const record: FusionToolCallLogRecord = {
+          schema_version: FUSION_TOOL_CALL_LOG_SCHEMA_VERSION,
+          ordinal,
+          tool_name: event.toolName,
+          arguments_sha256: sha256(argumentsBytes),
+          arguments_bytes: argumentsBytes.length,
+          result_bytes: resultBytes.length,
+          result_sha256: sha256(resultBytes),
+          status: event.isError === true ? 'error' : 'ok',
+          duration_ms: Math.max(0, Date.now() - start),
+          ...(fetchMetadata === undefined ? {} : fetchMetadata),
+        };
+        ordinal += 1;
+        appendToolCallLogLine(toolCallLogPath, record);
+        totalToolResultBytes += resultBytes.length;
+        if (totalToolResultBytes > FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES) {
+          throw new Error(
+            `fusion candidate exceeded the aggregate tool-output budget: ${String(totalToolResultBytes)} bytes across ${String(ordinal)} calls exceeds ${String(FUSION_CHILD_MAX_TOTAL_TOOL_RESULT_BYTES)}`,
+          );
+        }
+      } catch (error) {
+        auditFailed = true;
+        throw error;
       }
     });
+    pi.on('agent_end', () => {
+      const complete = !auditFailed && starts.size === 0;
+      writeToolCallLogSeal(toolCallLogPath, ordinal, totalToolResultBytes, complete);
+    });
   }
 
   if (researchEnabled === '1') {
@@ -332,10 +364,17 @@ export default function fusionChildExtension(pi: ExtensionAPI): void {
       },
       async execute(toolCallId, params) {
         try {
+          const canonicalUrl = canonicalizeFusionPublicUrl(params.url);
+          if (params.url !== canonicalUrl) {
+            throw new Error(`${FUSION_WEB_FETCH_TOOL_NAME} URL must exactly match its declared canonical URL`);
+          }
+          if (declaredResearchUrls === undefined || !declaredResearchUrls.has(canonicalUrl)) {
+            throw new Error(`${FUSION_WEB_FETCH_TOOL_NAME} URL was not declared in the research source policy`);
+          }
           const result = await fusionWebFetch(
             params.extract === undefined
-              ? { url: params.url }
-              : { url: params.url, extract: params.extract },
+              ? { url: canonicalUrl }
+              : { url: canonicalUrl, extract: params.extract },
           );
           fetchAuditMetadata.set(toolCallId, fetchAuditMetadataFromObject(result, params.url));
           return {