switchboard-cli 0.1.0-alpha.3 → 0.1.0-alpha.5

package/node_modules/@switchboard/core/src/execution-trace.ts

@@ -1,455 +0,0 @@
-import { randomBytes } from "crypto";
-import { mkdir, writeFile } from "fs/promises";
-import { join } from "path";
-import YAML from "yaml";
-import type { BudgetCategory, ProofArtifact } from "./types";
-import { proofArtifactSchema } from "./types";
-import { captureProofArtifact } from "./proof-capture";
-
-// ---------------------------------------------------------------------------
-// Execution Trace — structured step-by-step record of execution
-// ---------------------------------------------------------------------------
-//
-// The execution trace turns an unstructured transcript blob into a governed
-// artifact that downstream evaluation and optimization can consume.
-//
-// It captures:
-//   - What steps/phases the execution went through
-//   - What tools/commands were invoked and their outcomes
-//   - Where failure occurred (if it did)
-//   - A lightweight summary suitable for the audited receipt
-//
-// Design constraints:
-//   - Extraction is best-effort from raw output — not all executors emit
-//     structured step data, so the trace may be partial
-//   - The trace is a *derived* proof artifact — it is computed from the
-//     captured transcript, not independently observed
-//   - The trace does not replace the transcript — it adds structure on top
-// ---------------------------------------------------------------------------
-
-// ---------------------------------------------------------------------------
-// Types
-// ---------------------------------------------------------------------------
-
-export interface TraceStep {
-  /** Sequential index (0-based). */
-  index: number;
-  /** Short label for this step (e.g. "tool_use", "thinking", "command", "output"). */
-  kind: string;
-  /** What was done — tool name, command, or phase label. */
-  action: string;
-  /** Outcome summary — success/failure/output snippet. */
-  outcome: string;
-  /** Duration hint in ms (0 if unknown). */
-  duration_hint_ms: number;
-  /** Budget category — task, verification, or correction. */
-  budget_category: BudgetCategory;
-}
-
-export interface FailureContext {
-  /** Whether execution failed. */
-  failed: boolean;
-  /** The phase where failure occurred (e.g. "execution", "parsing", "dispatch"). */
-  failure_phase: string;
-  /** Short error classification (e.g. "timeout", "api_error", "parse_error", "crash"). */
-  error_category: string;
-  /** The error message or description. */
-  error_message: string;
-  /** Index of the last successful trace step before failure (-1 if none). */
-  last_successful_step: number;
-  /** Partial work completed before failure. */
-  partial_work_summary: string;
-}
-
-export interface ExecutionTrace {
-  trace_id: string;
-  dispatch_id: string;
-  created_at: string;
-  /** Total steps extracted from the transcript. */
-  step_count: number;
-  /** Number of tool/command invocations found. */
-  tool_use_count: number;
-  /** Execution duration in ms. */
-  duration_ms: number;
-  /** The structured steps. */
-  steps: TraceStep[];
-  /** Failure context (present even on success — failed=false). */
-  failure: FailureContext;
-  /** Summary for embedding on receipt. */
-  summary: string;
-}
-
-export interface TraceSummary {
-  trace_step_count: number;
-  trace_tool_use_count: number;
-  trace_failure_phase: string;
-  trace_error_category: string;
-  trace_duration_ms: number;
-  trace_available: boolean;
-}
-
-// ---------------------------------------------------------------------------
-// Transcript parsing — extract structured steps from raw output
-// ---------------------------------------------------------------------------
-
-// Known tool-use patterns across Claude/Codex output formats
-const TOOL_USE_PATTERNS: Array<{ pattern: RegExp; kind: string }> = [
-  { pattern: /^(?:Tool|tool_use):\s*(.+)/m, kind: "tool_use" },
-  { pattern: /^Running:\s*(.+)/m, kind: "command" },
-  { pattern: /^\$\s+(.+)/m, kind: "command" },
-  { pattern: /^(?:Read|Write|Edit|Bash|Grep|Glob)\s*[:(]/m, kind: "tool_use" },
-  { pattern: /^> (.+)/m, kind: "command" },
-];
-
-// Phase/section markers
-const PHASE_PATTERNS: Array<{ pattern: RegExp; kind: string }> = [
-  { pattern: /^#{1,3}\s+(.+)/m, kind: "phase" },
-  { pattern: /^---+$/m, kind: "separator" },
-  { pattern: /^(?:Step|Phase)\s+\d+/im, kind: "phase" },
-  { pattern: /^(?:Error|FAIL|Failed|Exception)/im, kind: "error" },
-  { pattern: /^(?:✓|✗|PASS|FAIL)\s+(.+)/m, kind: "verification" },
-];
-
-// ---------------------------------------------------------------------------
-// Budget category classification — heuristic from step kind + action text
-// ---------------------------------------------------------------------------
-//
-// Steps are classified into task / verification / correction based on signals
-// in the action text and step kind. The classification is best-effort and
-// conservative: when in doubt, a step is classified as "task".
-//
-// Verification signals: test commands, assertions, validation keywords
-// Correction signals: fix/patch keywords following a verification step,
-//                     retry patterns, error-recovery keywords
-// ---------------------------------------------------------------------------
-
-const VERIFICATION_PATTERNS: RegExp[] = [
-  /\btest\b/i,
-  /\bspec\b/i,
-  /\bassert/i,
-  /\bexpect\b/i,
-  /\bverif/i,
-  /\bvalidat/i,
-  /\bcheck/i,
-  /lint\b/i,
-  /\btypecheck/i,
-  /\btsc\b/i,
-  /\best\b/i,             // jest, vitest
-  /\bvitest\b/i,
-  /\bpytest\b/i,
-  /\bcargo test/i,
-  /\bnpm run test/i,
-  /\bnpx.*test/i,
-  /\bgit diff/i,          // diffing to verify
-  /\bcat.*\.spec\b/i,
-  /\bcat.*\.test\b/i,
-];
-
-const CORRECTION_PATTERNS: RegExp[] = [
-  /\bfix\b/i,
-  /\bpatch\b/i,
-  /\bretry/i,
-  /\bhotfix/i,
-  /\brevert/i,
-  /\bundo\b/i,
-  /\brollback/i,
-  /\bcorrect/i,
-  /\brepair/i,
-  /\bresolv/i,            // resolve, resolving
-  /\bworkaround/i,
-];
-
-/**
- * Classify a trace step into a budget category based on its kind and action.
- *
- * Uses a simple state-aware heuristic:
- * - If the step's action matches verification patterns → verification
- * - If a prior step was verification and this step matches correction patterns → correction
- * - Otherwise → task
- */
-export function classifyStepBudget(
-  step: { kind: string; action: string; outcome: string },
-  priorCategory: BudgetCategory | null,
-): BudgetCategory {
-  const text = `${step.action} ${step.outcome}`;
-
-  // Verification patterns — independent of prior context
-  if (step.kind === "verification") return "verification";
-  if (VERIFICATION_PATTERNS.some(p => p.test(text))) return "verification";
-
-  // Correction patterns — only meaningful after verification or error
-  if (priorCategory === "verification" || step.kind === "error") {
-    if (CORRECTION_PATTERNS.some(p => p.test(text))) return "correction";
-  }
-
-  // Error steps that follow task steps are corrections (fixing an issue)
-  if (step.kind === "error" && priorCategory === "task") return "correction";
-
-  return "task";
-}
-
-/**
- * Extract structured trace steps from a raw execution transcript.
- *
- * This is best-effort — the transcript is unstructured text from
- * different executor surfaces. We extract what we can.
- */
-export function extractTraceSteps(rawOutput: string): TraceStep[] {
-  if (!rawOutput || rawOutput.length === 0) return [];
-
-  const steps: TraceStep[] = [];
-  const lines = rawOutput.split("\n");
-  let stepIndex = 0;
-
-  for (let i = 0; i < lines.length; i++) {
-    const line = lines[i]!.trim();
-    if (line.length === 0) continue;
-
-    // Check tool-use patterns first (higher signal)
-    let matched = false;
-    for (const { pattern, kind } of TOOL_USE_PATTERNS) {
-      const m = line.match(pattern);
-      if (m) {
-        // Look ahead for outcome on next non-empty line
-        let outcome = "";
-        for (let j = i + 1; j < Math.min(i + 5, lines.length); j++) {
-          const next = lines[j]!.trim();
-          if (next.length > 0) {
-            outcome = next.slice(0, 120);
-            break;
-          }
-        }
-        const actionText = (m[1] ?? line).slice(0, 200);
-        const priorCategory = steps.length > 0 ? steps[steps.length - 1]!.budget_category : null;
-        const budget_category = classifyStepBudget({ kind, action: actionText, outcome }, priorCategory);
-        steps.push({
-          index: stepIndex++,
-          kind,
-          action: actionText,
-          outcome,
-          duration_hint_ms: 0,
-          budget_category,
-        });
-        matched = true;
-        break;
-      }
-    }
-    if (matched) continue;
-
-    // Check phase/section patterns
-    for (const { pattern, kind } of PHASE_PATTERNS) {
-      const m = line.match(pattern);
-      if (m) {
-        const actionText = (m[1] ?? line).slice(0, 200);
-        const priorCategory = steps.length > 0 ? steps[steps.length - 1]!.budget_category : null;
-        const budget_category = classifyStepBudget({ kind, action: actionText, outcome: "" }, priorCategory);
-        steps.push({
-          index: stepIndex++,
-          kind,
-          action: actionText,
-          outcome: "",
-          duration_hint_ms: 0,
-          budget_category,
-        });
-        break;
-      }
-    }
-  }
-
-  return steps;
-}
-
-// ---------------------------------------------------------------------------
-// Failure context extraction
-// ---------------------------------------------------------------------------
-
-const ERROR_CATEGORIES: Array<{ pattern: RegExp; category: string }> = [
-  { pattern: /timeout|timed?\s*out/i, category: "timeout" },
-  { pattern: /rate.?limit/i, category: "rate_limit" },
-  { pattern: /api.?error|status\s+[45]\d\d/i, category: "api_error" },
-  { pattern: /parse|JSON\.parse|SyntaxError/i, category: "parse_error" },
-  { pattern: /ENOENT|not found|no such file/i, category: "file_not_found" },
-  { pattern: /permission|EACCES|forbidden/i, category: "permission_error" },
-  { pattern: /crash|segfault|SIGKILL|SIGTERM/i, category: "crash" },
-  { pattern: /out of memory|OOM|heap/i, category: "resource_exhaustion" },
-];
-
-/**
- * Classify an error message into a short category.
- */
-export function classifyError(errorMessage: string): string {
-  if (!errorMessage) return "unknown";
-  for (const { pattern, category } of ERROR_CATEGORIES) {
-    if (pattern.test(errorMessage)) return category;
-  }
-  return "execution_error";
-}
-
-/**
- * Build structured failure context from execution results.
- */
-export function buildFailureContext(input: {
-  failed: boolean;
-  error: string | null;
-  phase?: string;
-  steps?: TraceStep[];
-  rawOutput?: string;
-}): FailureContext {
-  if (!input.failed) {
-    return {
-      failed: false,
-      failure_phase: "",
-      error_category: "",
-      error_message: "",
-      last_successful_step: input.steps?.length
-        ? input.steps.length - 1
-        : -1,
-      partial_work_summary: "",
-    };
-  }
-
-  const errorMsg = input.error ?? "Unknown error";
-  const errorCategory = classifyError(errorMsg);
-  const failurePhase = input.phase ?? "execution";
-
-  // Find last successful step
-  const steps = input.steps ?? [];
-  const lastSuccess = steps.length > 0 ? steps.length - 1 : -1;
-
-  // Summarize partial work from successful steps
-  const completedActions = steps
-    .filter(s => s.kind === "tool_use" || s.kind === "command")
-    .map(s => s.action)
-    .slice(0, 5);
-  const partialSummary = completedActions.length > 0
-    ? `${completedActions.length} tool/command action(s) completed before failure: ${completedActions.join("; ").slice(0, 200)}`
-    : "No tool/command actions completed before failure.";
-
-  return {
-    failed: true,
-    failure_phase: failurePhase,
-    error_category: errorCategory,
-    error_message: errorMsg.slice(0, 500),
-    last_successful_step: lastSuccess,
-    partial_work_summary: partialSummary,
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Trace builder — assembles the full execution trace
-// ---------------------------------------------------------------------------
-
-export function buildExecutionTrace(input: {
-  dispatchId: string;
-  rawOutput: string;
-  durationMs: number;
-  failed: boolean;
-  error?: string | null;
-  failurePhase?: string;
-}): ExecutionTrace {
-  const steps = extractTraceSteps(input.rawOutput);
-  const toolUseCount = steps.filter(
-    s => s.kind === "tool_use" || s.kind === "command",
-  ).length;
-
-  const failure = buildFailureContext({
-    failed: input.failed,
-    error: input.error ?? null,
-    ...(input.failurePhase != null && { phase: input.failurePhase }),
-    steps,
-    rawOutput: input.rawOutput,
-  });
-
-  // Build a one-line summary
-  const summaryParts: string[] = [];
-  summaryParts.push(`${steps.length} step(s)`);
-  summaryParts.push(`${toolUseCount} tool/command use(s)`);
-  summaryParts.push(`${Math.round(input.durationMs / 1000)}s`);
-  if (failure.failed) {
-    summaryParts.push(`failed at ${failure.failure_phase}: ${failure.error_category}`);
-  }
-
-  return {
-    trace_id: `trace-${randomBytes(8).toString("hex")}`,
-    dispatch_id: input.dispatchId,
-    created_at: new Date().toISOString(),
-    step_count: steps.length,
-    tool_use_count: toolUseCount,
-    duration_ms: input.durationMs,
-    steps,
-    failure,
-    summary: summaryParts.join(", "),
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Trace summary — lightweight fields for embedding on receipts
-// ---------------------------------------------------------------------------
-
-export function deriveTraceSummary(trace: ExecutionTrace | null): TraceSummary {
-  if (!trace) {
-    return {
-      trace_step_count: 0,
-      trace_tool_use_count: 0,
-      trace_failure_phase: "",
-      trace_error_category: "",
-      trace_duration_ms: 0,
-      trace_available: false,
-    };
-  }
-
-  return {
-    trace_step_count: trace.step_count,
-    trace_tool_use_count: trace.tool_use_count,
-    trace_failure_phase: trace.failure.failure_phase,
-    trace_error_category: trace.failure.error_category,
-    trace_duration_ms: trace.duration_ms,
-    trace_available: true,
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Persistence — capture trace as a proof artifact
-// ---------------------------------------------------------------------------
-
-/**
- * Persist an execution trace as a YAML proof artifact.
- */
-export async function captureExecutionTrace(input: {
-  proofDir: string;
-  trace: ExecutionTrace;
-}): Promise<ProofArtifact> {
-  return captureProofArtifact({
-    proofDir: input.proofDir,
-    kind: "execution-trace",
-    provenance: "derived",
-    filename: "execution-trace.yaml",
-    content: YAML.stringify(input.trace),
-    description: input.trace.summary,
-  });
-}
-
-/**
- * Persist failure context as a separate proof artifact.
- * Only written when the execution actually failed.
- */
-export async function captureFailureContext(input: {
-  proofDir: string;
-  failure: FailureContext;
-  dispatchId: string;
-}): Promise<ProofArtifact | null> {
-  if (!input.failure.failed) return null;
-
-  return captureProofArtifact({
-    proofDir: input.proofDir,
-    kind: "failure-context",
-    provenance: "derived",
-    filename: "failure-context.yaml",
-    content: YAML.stringify({
-      dispatch_id: input.dispatchId,
-      ...input.failure,
-      captured_at: new Date().toISOString(),
-    }),
-    description: `Failure in ${input.failure.failure_phase}: ${input.failure.error_category} — ${input.failure.error_message.slice(0, 100)}`,
-  });
-}