pi-background-tasks 0.9.0 → 1.0.3

package/src/core/fusion/source-policy.ts

@@ -0,0 +1,257 @@
+import { createHash } from 'node:crypto';
+import { constants } from 'node:fs';
+import { open } from 'node:fs/promises';
+import { isIP } from 'node:net';
+import { canonicalJson } from '../attested-pi-run.js';
+import { isJsonObject, parseJsonText } from '../common.js';
+import {
+  FUSION_SOURCE_POLICY_SCHEMA_VERSION,
+  FusionError,
+  type FusionDeclaredSourceV1,
+  type FusionSourcePolicyV1,
+} from './types.js';
+
+const SHA256_HEX = /^[0-9a-f]{64}$/;
+const O_NOFOLLOW = typeof constants.O_NOFOLLOW === 'number' ? constants.O_NOFOLLOW : 0;
+
+function sha256Text(value: string): string {
+  return createHash('sha256').update(value, 'utf8').digest('hex');
+}
+
+function stripIpv6Brackets(hostname: string): string {
+  return hostname.startsWith('[') && hostname.endsWith(']') ? hostname.slice(1, -1) : hostname;
+}
+
+function isBlockedHostname(hostname: string): boolean {
+  const lower = hostname.toLowerCase();
+  return (
+    lower === 'localhost' ||
+    lower.endsWith('.localhost') ||
+    lower === 'metadata' ||
+    lower === 'metadata.local' ||
+    lower === 'metadata.google.internal' ||
+    lower === 'metadata.goog' ||
+    lower === 'instance-data' ||
+    lower === 'instance-data.ec2.internal'
+  );
+}
+
+function isBlockedIpv4(host: string): boolean {
+  const parts = host.split('.').map((part) => Number.parseInt(part, 10));
+  if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255))
+    return true;
+  const [a = 0, b = 0, c = 0] = parts;
+  if (a === 0 || a === 10 || a === 127) return true;
+  if (a === 100 && b >= 64 && b <= 127) return true;
+  if (a === 169 && b === 254) return true;
+  if (a === 172 && b >= 16 && b <= 31) return true;
+  if (a === 192 && b === 168) return true;
+  if (a === 192 && b === 0 && c === 0) return true;
+  if (a === 192 && b === 0 && c === 2) return true;
+  if (a === 198 && (b === 18 || b === 19)) return true;
+  if (a === 198 && b === 51 && c === 100) return true;
+  if (a === 203 && b === 0 && c === 113) return true;
+  if (a >= 224) return true;
+  return host === '255.255.255.255' || host === '168.63.129.16';
+}
+
+function isBlockedIpv6(host: string): boolean {
+  const lower = host.toLowerCase();
+  return (
+    lower === '::' ||
+    lower === '::1' ||
+    lower.startsWith('::ffff:') ||
+    lower.startsWith('64:ff9b::') ||
+    lower.startsWith('64:ff9b:1:') ||
+    lower.startsWith('100:') ||
+    lower.startsWith('2001:2:') ||
+    lower.startsWith('2001:db8:') ||
+    lower.startsWith('2002:') ||
+    lower.startsWith('fc') ||
+    lower.startsWith('fd') ||
+    lower.startsWith('fe8') ||
+    lower.startsWith('fe9') ||
+    lower.startsWith('fea') ||
+    lower.startsWith('feb') ||
+    lower.startsWith('ff')
+  );
+}
+
+export function canonicalizeFusionPublicUrl(value: string): string {
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch (error) {
+    throw new FusionError(
+      `fusion research declared source URL is malformed: ${error instanceof Error ? error.message : String(error)}`,
+      { code: 'orchestration_failed', childCreated: false },
+    );
+  }
+  if (url.username !== '' || url.password !== '') {
+    throw new FusionError('fusion research source URL must not contain credentials', {
+      code: 'orchestration_failed',
+      childCreated: false,
+    });
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+    throw new FusionError('fusion research source URL must use http or https', {
+      code: 'orchestration_failed',
+      childCreated: false,
+    });
+  }
+  url.username = '';
+  url.password = '';
+  url.hash = '';
+  const normalizedHost = stripIpv6Brackets(url.hostname.toLowerCase().replace(/\.+$/u, ''));
+  url.hostname = normalizedHost;
+  if (isBlockedHostname(normalizedHost)) {
+    throw new FusionError('fusion research source URL must be public, not localhost/metadata', {
+      code: 'orchestration_failed',
+      childCreated: false,
+    });
+  }
+  const ipKind = isIP(normalizedHost);
+  if ((ipKind === 4 && isBlockedIpv4(normalizedHost)) || (ipKind === 6 && isBlockedIpv6(normalizedHost))) {
+    throw new FusionError('fusion research source URL must be public, not private/reserved', {
+      code: 'orchestration_failed',
+      childCreated: false,
+    });
+  }
+  if ((url.protocol === 'http:' && url.port === '80') || (url.protocol === 'https:' && url.port === '443')) {
+    url.port = '';
+  }
+  return url.toString();
+}
+
+export interface DeclaredFusionSourceInput {
+  url: string;
+  purpose: string;
+}
+
+export function normalizeFusionDeclaredSources(
+  sources: readonly DeclaredFusionSourceInput[] = [],
+): readonly FusionDeclaredSourceV1[] {
+  const seen = new Map<string, number>();
+  return sources.map((source, index) => {
+    if (typeof source.url !== 'string' || source.url.trim().length === 0) {
+      throw new FusionError(`fusion research source ${String(index)} requires non-blank URL`, {
+        code: 'orchestration_failed',
+        childCreated: false,
+      });
+    }
+    if (typeof source.purpose !== 'string' || source.purpose.trim().length === 0) {
+      throw new FusionError(`fusion research source ${String(index)} requires non-blank purpose`, {
+        code: 'orchestration_failed',
+        childCreated: false,
+      });
+    }
+    const canonicalUrl = canonicalizeFusionPublicUrl(source.url.trim());
+    const previous = seen.get(canonicalUrl);
+    if (previous !== undefined) {
+      throw new FusionError(
+        `fusion research source ${String(index)} duplicates canonical URL from source ${String(previous)}: ${canonicalUrl}`,
+        { code: 'orchestration_failed', childCreated: false },
+      );
+    }
+    seen.set(canonicalUrl, index);
+    const purpose = source.purpose.trim();
+    return {
+      url: canonicalUrl,
+      canonical_url: canonicalUrl,
+      purpose,
+      sha256: sha256Text(`${canonicalUrl}\u0000${purpose}`),
+    };
+  });
+}
+
+export function buildFusionSourcePolicy(
+  cwd: string,
+  sources: readonly FusionDeclaredSourceV1[],
+): FusionSourcePolicyV1 {
+  const body = {
+    schema_version: FUSION_SOURCE_POLICY_SCHEMA_VERSION,
+    workflow: 'research' as const,
+    cwd,
+    sources,
+  } as const;
+  return { ...body, root_sha256: sha256Text(canonicalJson(body)) };
+}
+
+export function sourcePolicyCanonicalBytes(policy: FusionSourcePolicyV1): string {
+  return canonicalJson(policy);
+}
+
+function requireString(record: Record<PropertyKey, unknown>, key: string, label: string): string {
+  const value = record[key];
+  if (typeof value !== 'string' || value.length === 0) throw new Error(`${label}.${key} must be non-blank string`);
+  return value;
+}
+
+export function parseFusionSourcePolicy(value: unknown): FusionSourcePolicyV1 {
+  if (!isJsonObject(value) || Array.isArray(value)) throw new Error('fusion source policy must be object');
+  const keys = Object.keys(value).sort();
+  const expected = ['cwd', 'root_sha256', 'schema_version', 'sources', 'workflow'];
+  if (keys.join('\0') !== expected.join('\0')) throw new Error('fusion source policy keys mismatch');
+  if (value['schema_version'] !== FUSION_SOURCE_POLICY_SCHEMA_VERSION) throw new Error('fusion source policy schema_version mismatch');
+  if (value['workflow'] !== 'research') throw new Error('fusion source policy workflow must be research');
+  const cwd = requireString(value, 'cwd', 'fusion source policy');
+  const rootSha256 = requireString(value, 'root_sha256', 'fusion source policy');
+  if (!SHA256_HEX.test(rootSha256)) throw new Error('fusion source policy.root_sha256 must be sha256');
+  if (!Array.isArray(value['sources'])) throw new Error('fusion source policy.sources must be array');
+  const sources = value['sources'].map((item, index): FusionDeclaredSourceV1 => {
+    const label = `fusion source policy.sources[${String(index)}]`;
+    if (!isJsonObject(item) || Array.isArray(item)) throw new Error(`${label} must be object`);
+    const itemKeys = Object.keys(item).sort();
+    const itemExpected = ['canonical_url', 'purpose', 'sha256', 'url'];
+    if (itemKeys.join('\0') !== itemExpected.join('\0')) throw new Error(`${label} keys mismatch`);
+    const url = requireString(item, 'url', label);
+    const purpose = requireString(item, 'purpose', label);
+    const canonical_url = requireString(item, 'canonical_url', label);
+    const sha256 = requireString(item, 'sha256', label);
+    if (!SHA256_HEX.test(sha256)) throw new Error(`${label}.sha256 must be sha256`);
+    if (canonicalizeFusionPublicUrl(url) !== canonical_url) throw new Error(`${label}.canonical_url mismatch`);
+    if (url !== canonical_url) throw new Error(`${label}.url must equal canonical_url`);
+    if (purpose.trim() !== purpose) throw new Error(`${label}.purpose must be trimmed`);
+    if (sha256Text(`${canonical_url}\u0000${purpose}`) !== sha256) throw new Error(`${label}.sha256 mismatch`);
+    return { url, canonical_url, purpose, sha256 };
+  });
+  const seen = new Set<string>();
+  for (const [index, source] of sources.entries()) {
+    if (seen.has(source.canonical_url)) {
+      throw new Error(`fusion source policy.sources[${String(index)}].canonical_url duplicate`);
+    }
+    seen.add(source.canonical_url);
+  }
+  const body = { schema_version: FUSION_SOURCE_POLICY_SCHEMA_VERSION, workflow: 'research' as const, cwd, sources } as const;
+  if (sha256Text(canonicalJson(body)) !== rootSha256) throw new Error('fusion source policy root_sha256 mismatch');
+  return { ...body, root_sha256: rootSha256 };
+}
+
+async function readRegularFileNoSymlink(path: string, label: string): Promise<Buffer> {
+  let handle: Awaited<ReturnType<typeof open>>;
+  try {
+    handle = await open(path, constants.O_RDONLY | O_NOFOLLOW);
+  } catch (error) {
+    if (isJsonObject(error) && error['code'] === 'ELOOP') {
+      throw new Error(`${label} at ${path} is a symlink; refusing to follow it`);
+    }
+    throw error;
+  }
+  try {
+    const stats = await handle.stat();
+    if (!stats.isFile()) throw new Error(`${label} at ${path} is not a regular file`);
+    return await handle.readFile();
+  } finally {
+    await handle.close();
+  }
+}
+
+export async function readFusionSourcePolicyFile(path: string, expectedSha256: string): Promise<FusionSourcePolicyV1> {
+  if (!SHA256_HEX.test(expectedSha256)) throw new Error('fusion source policy expected hash is malformed');
+  const bytes = await readRegularFileNoSymlink(path, 'fusion source policy');
+  const actual = createHash('sha256').update(bytes).digest('hex');
+  if (actual !== expectedSha256) throw new Error('fusion source policy artifact hash mismatch');
+  const text = bytes.toString('utf8');
+  if (!Buffer.from(text, 'utf8').equals(bytes)) throw new Error('fusion source policy is not UTF-8');
+  return parseFusionSourcePolicy(parseJsonText(text));
+}

package/src/core/fusion/types.ts

@@ -11,11 +11,16 @@ import type {
 export type FusionThinkingLevel = 'off' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh' | 'max';
 
 export const FUSION_MODEL_CONFIG_SCHEMA_VERSION = 'pi-background-tasks.fusion-models.v1';
-export const FUSION_INPUT_SCHEMA_VERSION = 'pi-background-tasks.fusion-input.v4';
+export const FUSION_LEGACY_INPUT_SCHEMA_VERSION = 'pi-background-tasks.fusion-input.v4';
+export const FUSION_INPUT_SCHEMA_VERSION = 'pi-background-tasks.fusion-input.v5';
 export const FUSION_EVALUATION_SCHEMA_VERSION = 'pi-background-tasks.fusion-evaluation.v1';
-export const FUSION_RESULT_SCHEMA_VERSION = 'pi-background-tasks.fusion-result.v4';
-export const FUSION_MANIFEST_SCHEMA_VERSION = 'pi-background-tasks.fusion-manifest.v3';
+export const FUSION_VALIDATE_CANDIDATE_SCHEMA_VERSION = 'pi-background-tasks.fusion-validation-candidate.v1';
+export const FUSION_LEGACY_RESULT_SCHEMA_VERSION = 'pi-background-tasks.fusion-result.v4';
+export const FUSION_RESULT_SCHEMA_VERSION = 'pi-background-tasks.fusion-result.v5';
+export const FUSION_LEGACY_MANIFEST_SCHEMA_VERSION = 'pi-background-tasks.fusion-manifest.v3';
+export const FUSION_MANIFEST_SCHEMA_VERSION = 'pi-background-tasks.fusion-manifest.v4';
 export const FUSION_CONTEXT_LEDGER_SCHEMA_VERSION = 'pi-background-tasks.fusion-context-ledger.v2';
+export const FUSION_SOURCE_POLICY_SCHEMA_VERSION = 'pi-background-tasks.fusion-source-policy.v1';
 export const FUSION_BUDGET_PLAN_SCHEMA_VERSION = 'pi-background-tasks.fusion-budget-plan.v3';
 export const FUSION_CALIBRATION_VIOLATION_SCHEMA_VERSION =
   'pi-background-tasks.fusion-calibration-violation.v1';
@@ -46,26 +51,51 @@ export type FusionStage = (typeof FUSION_STAGE_VALUES)[number];
 
 export const FUSION_CAPABILITY_VALUES = Object.freeze(['reason', 'inspect', 'research'] as const);
 export type FusionCapability = (typeof FUSION_CAPABILITY_VALUES)[number];
-export const FUSION_DEFAULT_CAPABILITY: FusionCapability = 'reason';
+
+/** No-tools capability for reason candidates, evaluator, repair, and merger. */
+export const FUSION_NO_TOOLS_CAPABILITY: FusionCapability = 'reason';
+/** Legacy default retained for old type imports only. New workflows never default. */
+export const FUSION_BRAINSTORM_DEFAULT_CAPABILITY: FusionCapability = 'inspect';
+/** @deprecated New v5 workflows have no caller capability default. */
+export const FUSION_DEFAULT_CAPABILITY: FusionCapability = FUSION_BRAINSTORM_DEFAULT_CAPABILITY;
 
 export const FUSION_WEB_FETCH_TOOL_NAME = 'fusion_web_fetch' as const;
 export const FUSION_INSPECT_TOOLS = Object.freeze(['read', 'grep', 'find', 'ls'] as const);
+export const FUSION_RESEARCH_TOOLS = Object.freeze([
+  'read',
+  'grep',
+  'find',
+  'ls',
+  FUSION_WEB_FETCH_TOOL_NAME,
+] as const);
 
 /**
  * Workflow identities sharing one orchestrator, one context projection, one
  * evaluation schema, and one artifact store. A workflow selects stage framing and
  * capability policy only; it never changes the canonical input schema.
  */
-export const FUSION_WORKFLOW_IDS = Object.freeze(['brainstorm', 'validate'] as const);
+export const FUSION_WORKFLOW_IDS = Object.freeze([
+  'reason',
+  'investigate',
+  'research',
+  'validate',
+] as const);
 export type FusionWorkflowId = (typeof FUSION_WORKFLOW_IDS)[number];
+export const FUSION_PUBLIC_WORKFLOW_NAMES = Object.freeze([
+  'fusion_reason',
+  'fusion_investigate',
+  'fusion_research',
+  'fusion_validate',
+] as const);
+export type FusionPublicWorkflowName = (typeof FUSION_PUBLIC_WORKFLOW_NAMES)[number];
+export type FusionContextKind = 'session_projection' | 'clean_task';
 
 /**
  * The single capability the validate workflow ever runs candidates with.
  *
- * Deliberately separate from `FUSION_DEFAULT_CAPABILITY`, which must stay the
- * least-privileged `reason` profile for brainstorm. A validator that cannot read
- * the code it is judging produces opinion rather than validation, so this is a
- * fixed workflow policy and not a default a caller may override.
+ * Deliberately separate from the caller-selectable brainstorm default. Although
+ * both workflows currently give candidates read-only inspection, validation pins
+ * that capability as fixed policy rather than exposing a caller override.
  */
 export const FUSION_VALIDATE_CAPABILITY: FusionCapability = 'inspect';
 
@@ -74,6 +104,9 @@ export const FUSION_FORBIDDEN_TOOLS = Object.freeze([
   'edit',
   'write',
   'fusion_brainstorm',
+  'fusion_reason',
+  'fusion_investigate',
+  'fusion_research',
   'fusion_validate',
   'bg_delegate',
   'bg_result',
@@ -302,15 +335,73 @@ export interface FusionConversationProjectionV4 {
 
 export type FusionConversationProjectionV3 = FusionConversationProjectionV4;
 
-export interface FusionCanonicalInputV4 {
+export interface FusionDeclaredSourceV1 {
+  url: string;
+  canonical_url: string;
+  purpose: string;
+  sha256: string;
+}
+
+export interface FusionCleanTaskContextV1 {
+  kind: 'clean_task';
+  policy_id: 'fusion-clean-task-v1';
+  declared_sources: readonly FusionDeclaredSourceV1[];
+}
+
+export interface FusionSessionProjectionContextV1 {
+  kind: 'session_projection';
+  policy_id: 'fusion-session-projection-v1';
+  system_prompt: string;
+  conversation_projection: FusionConversationProjectionV4;
+}
+
+export interface FusionCanonicalInputV5Base {
   schema_version: typeof FUSION_INPUT_SCHEMA_VERSION;
+  workflow?: FusionWorkflowId | undefined;
+  cwd: string;
+  request: FusionCanonicalRequestV3;
+}
+
+export interface FusionSessionProjectionCanonicalInputV5 extends FusionCanonicalInputV5Base {
+  workflow?: FusionWorkflowId | undefined;
+  /** @deprecated v4 readability alias. Present only for session-projection inputs at runtime. */
+  system_prompt: string;
+  /** @deprecated v4 readability alias. Present only for session-projection inputs at runtime. */
+  conversation_projection: FusionConversationProjectionV4;
+  context?: FusionSessionProjectionContextV1 | undefined;
+}
+
+export interface FusionCleanTaskCanonicalInputV5 extends FusionCanonicalInputV5Base {
+  workflow: Exclude<FusionWorkflowId, 'reason'>;
+  context: FusionCleanTaskContextV1;
+}
+
+export type FusionCanonicalInputV5 =
+  | FusionSessionProjectionCanonicalInputV5
+  | FusionCleanTaskCanonicalInputV5;
+
+/** Legacy v4 shape retained for frozen golden fixtures/readability only. */
+export interface FusionCanonicalInputV4 {
+  schema_version: typeof FUSION_LEGACY_INPUT_SCHEMA_VERSION;
   cwd: string;
   system_prompt: string;
   request: FusionCanonicalRequestV3;
   conversation_projection: FusionConversationProjectionV4;
 }
 
-export type FusionCanonicalInputV3 = FusionCanonicalInputV4;
+export type FusionCanonicalInputV3 = FusionCanonicalInputV5;
+
+export interface FusionSourcePolicyV1 {
+  schema_version: typeof FUSION_SOURCE_POLICY_SCHEMA_VERSION;
+  workflow: 'research';
+  cwd: string;
+  sources: readonly FusionDeclaredSourceV1[];
+  root_sha256: string;
+}
+
+export interface FusionSourcePolicyArtifactRef extends FusionArtifactRef {
+  root_sha256: string;
+}
 
 export interface CandidateAssessment {
   candidate_id: FusionCandidateId;
@@ -343,12 +434,56 @@ export interface FusionSynthesisPlan {
   must_avoid: readonly string[];
 }
 
+export type FusionValidationSeverity = 'critical' | 'high' | 'minor';
+
+export interface FusionValidationFindingRecord {
+  id: string;
+  candidate_id: FusionCandidateId;
+  severity: FusionValidationSeverity;
+  location: string;
+  evidence: string;
+  impact: string;
+  summary: string;
+}
+
+export interface FusionValidationFindingDecision {
+  source_id: string;
+  disposition: 'include' | 'exclude';
+  rationale: string;
+  group_id?: string | undefined;
+}
+
+export interface FusionValidationFindingGroup {
+  group_id: string;
+  source_ids: readonly string[];
+  severity: FusionValidationSeverity;
+  location: string;
+  evidence: string;
+  impact: string;
+  summary: string;
+  rationale: string;
+}
+
+export interface FusionValidationFindingAccounting {
+  findings: readonly FusionValidationFindingRecord[];
+  decisions: readonly FusionValidationFindingDecision[];
+  groups: readonly FusionValidationFindingGroup[];
+}
+
+export interface FusionValidationCandidateReportV1 {
+  schema_version: typeof FUSION_VALIDATE_CANDIDATE_SCHEMA_VERSION;
+  findings: readonly Omit<FusionValidationFindingRecord, 'id' | 'candidate_id'>[];
+  verified: readonly string[];
+  limitations: readonly string[];
+}
+
 export interface FusionEvaluationV1 {
   schema_version: typeof FUSION_EVALUATION_SCHEMA_VERSION;
   candidate_assessments: readonly [CandidateAssessment, CandidateAssessment, CandidateAssessment];
   agreements: readonly string[];
   conflicts: readonly FusionConflict[];
   synthesis_plan: FusionSynthesisPlan;
+  validation_accounting?: FusionValidationFindingAccounting | undefined;
 }
 
 /** Exact Pi usage contract used at the child, artifact, and host tool-result boundaries. */
@@ -420,6 +555,8 @@ export interface FusionResultDetails {
   workflow: FusionWorkflowId;
   source: FusionSource;
   status: 'completed';
+  context: { kind: FusionContextKind; policy_id: string };
+  tool_policy: { candidate_tools: readonly string[]; evaluation_tools: readonly []; merge_tools: readonly [] };
   artifact_dir: string;
   models: {
     candidates: readonly [string, string, string];
@@ -596,6 +733,8 @@ export interface FusionToolCallLogRecord {
   status: FusionToolCallLogStatus;
   duration_ms: number;
   url?: string | undefined;
+  /** SHA-256 of a rejected attempted fetch URL; raw rejected URLs are never persisted. */
+  rejected_url_sha256?: string | undefined;
   final_url?: string | undefined;
   http_status?: number | undefined;
   response_bytes?: number | undefined;
@@ -676,6 +815,8 @@ export interface FusionArtifactManifest {
     evaluation: FusionCapability;
     merge: FusionCapability;
   };
+  context: { kind: FusionContextKind; policy_id: string; ledger_artifact?: string; source_policy_artifact?: string };
+  tool_policy: { candidate_tools: readonly string[]; evaluation_tools: readonly []; merge_tools: readonly [] };
   usage: FusionUsage;
   attempts: readonly FusionAttemptArtifactRecord[];
   artifacts: Readonly<Record<string, FusionArtifactRef>>;
@@ -756,6 +897,10 @@ export interface FusionBudgetEmptyRequestVerdict {
 
 export interface FusionBudgetPlanV1 {
   schema_version: typeof FUSION_BUDGET_PLAN_SCHEMA_VERSION;
+  workflow: FusionWorkflowId;
+  context: { kind: FusionContextKind; policy_id: string };
+  fixed_candidate_policy: { capability: FusionCapability; tools: readonly string[] };
+  tool_policy: { candidate_tools: readonly string[]; evaluation_tools: readonly []; merge_tools: readonly [] };
   policy: FusionBudgetPolicyDescriptor;
   routes: readonly FusionRouteCapacity[];
   stages: readonly FusionStageBudgetPlanEntry[];