switchboard-cli 0.1.0-alpha.1 → 0.1.0-alpha.3

package/node_modules/@switchboard/core/src/execution-trace.ts

@@ -0,0 +1,455 @@
+import { randomBytes } from "crypto";
+import { mkdir, writeFile } from "fs/promises";
+import { join } from "path";
+import YAML from "yaml";
+import type { BudgetCategory, ProofArtifact } from "./types";
+import { proofArtifactSchema } from "./types";
+import { captureProofArtifact } from "./proof-capture";
+
+// ---------------------------------------------------------------------------
+// Execution Trace — structured step-by-step record of execution
+// ---------------------------------------------------------------------------
+//
+// The execution trace turns an unstructured transcript blob into a governed
+// artifact that downstream evaluation and optimization can consume.
+//
+// It captures:
+//   - What steps/phases the execution went through
+//   - What tools/commands were invoked and their outcomes
+//   - Where failure occurred (if it did)
+//   - A lightweight summary suitable for the audited receipt
+//
+// Design constraints:
+//   - Extraction is best-effort from raw output — not all executors emit
+//     structured step data, so the trace may be partial
+//   - The trace is a *derived* proof artifact — it is computed from the
+//     captured transcript, not independently observed
+//   - The trace does not replace the transcript — it adds structure on top
+// ---------------------------------------------------------------------------
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface TraceStep {
+  /** Sequential index (0-based). */
+  index: number;
+  /** Short label for this step (e.g. "tool_use", "thinking", "command", "output"). */
+  kind: string;
+  /** What was done — tool name, command, or phase label. */
+  action: string;
+  /** Outcome summary — success/failure/output snippet. */
+  outcome: string;
+  /** Duration hint in ms (0 if unknown). */
+  duration_hint_ms: number;
+  /** Budget category — task, verification, or correction. */
+  budget_category: BudgetCategory;
+}
+
+export interface FailureContext {
+  /** Whether execution failed. */
+  failed: boolean;
+  /** The phase where failure occurred (e.g. "execution", "parsing", "dispatch"). */
+  failure_phase: string;
+  /** Short error classification (e.g. "timeout", "api_error", "parse_error", "crash"). */
+  error_category: string;
+  /** The error message or description. */
+  error_message: string;
+  /** Index of the last successful trace step before failure (-1 if none). */
+  last_successful_step: number;
+  /** Partial work completed before failure. */
+  partial_work_summary: string;
+}
+
+export interface ExecutionTrace {
+  trace_id: string;
+  dispatch_id: string;
+  created_at: string;
+  /** Total steps extracted from the transcript. */
+  step_count: number;
+  /** Number of tool/command invocations found. */
+  tool_use_count: number;
+  /** Execution duration in ms. */
+  duration_ms: number;
+  /** The structured steps. */
+  steps: TraceStep[];
+  /** Failure context (present even on success — failed=false). */
+  failure: FailureContext;
+  /** Summary for embedding on receipt. */
+  summary: string;
+}
+
+export interface TraceSummary {
+  trace_step_count: number;
+  trace_tool_use_count: number;
+  trace_failure_phase: string;
+  trace_error_category: string;
+  trace_duration_ms: number;
+  trace_available: boolean;
+}
+
+// ---------------------------------------------------------------------------
+// Transcript parsing — extract structured steps from raw output
+// ---------------------------------------------------------------------------
+
+// Known tool-use patterns across Claude/Codex output formats
+const TOOL_USE_PATTERNS: Array<{ pattern: RegExp; kind: string }> = [
+  { pattern: /^(?:Tool|tool_use):\s*(.+)/m, kind: "tool_use" },
+  { pattern: /^Running:\s*(.+)/m, kind: "command" },
+  { pattern: /^\$\s+(.+)/m, kind: "command" },
+  { pattern: /^(?:Read|Write|Edit|Bash|Grep|Glob)\s*[:(]/m, kind: "tool_use" },
+  { pattern: /^> (.+)/m, kind: "command" },
+];
+
+// Phase/section markers
+const PHASE_PATTERNS: Array<{ pattern: RegExp; kind: string }> = [
+  { pattern: /^#{1,3}\s+(.+)/m, kind: "phase" },
+  { pattern: /^---+$/m, kind: "separator" },
+  { pattern: /^(?:Step|Phase)\s+\d+/im, kind: "phase" },
+  { pattern: /^(?:Error|FAIL|Failed|Exception)/im, kind: "error" },
+  { pattern: /^(?:✓|✗|PASS|FAIL)\s+(.+)/m, kind: "verification" },
+];
+
+// ---------------------------------------------------------------------------
+// Budget category classification — heuristic from step kind + action text
+// ---------------------------------------------------------------------------
+//
+// Steps are classified into task / verification / correction based on signals
+// in the action text and step kind. The classification is best-effort and
+// conservative: when in doubt, a step is classified as "task".
+//
+// Verification signals: test commands, assertions, validation keywords
+// Correction signals: fix/patch keywords following a verification step,
+//                     retry patterns, error-recovery keywords
+// ---------------------------------------------------------------------------
+
+const VERIFICATION_PATTERNS: RegExp[] = [
+  /\btest\b/i,
+  /\bspec\b/i,
+  /\bassert/i,
+  /\bexpect\b/i,
+  /\bverif/i,
+  /\bvalidat/i,
+  /\bcheck/i,
+  /lint\b/i,
+  /\btypecheck/i,
+  /\btsc\b/i,
+  /\best\b/i,             // jest, vitest
+  /\bvitest\b/i,
+  /\bpytest\b/i,
+  /\bcargo test/i,
+  /\bnpm run test/i,
+  /\bnpx.*test/i,
+  /\bgit diff/i,          // diffing to verify
+  /\bcat.*\.spec\b/i,
+  /\bcat.*\.test\b/i,
+];
+
+const CORRECTION_PATTERNS: RegExp[] = [
+  /\bfix\b/i,
+  /\bpatch\b/i,
+  /\bretry/i,
+  /\bhotfix/i,
+  /\brevert/i,
+  /\bundo\b/i,
+  /\brollback/i,
+  /\bcorrect/i,
+  /\brepair/i,
+  /\bresolv/i,            // resolve, resolving
+  /\bworkaround/i,
+];
+
+/**
+ * Classify a trace step into a budget category based on its kind and action.
+ *
+ * Uses a simple state-aware heuristic:
+ * - If the step's action matches verification patterns → verification
+ * - If a prior step was verification and this step matches correction patterns → correction
+ * - Otherwise → task
+ */
+export function classifyStepBudget(
+  step: { kind: string; action: string; outcome: string },
+  priorCategory: BudgetCategory | null,
+): BudgetCategory {
+  const text = `${step.action} ${step.outcome}`;
+
+  // Verification patterns — independent of prior context
+  if (step.kind === "verification") return "verification";
+  if (VERIFICATION_PATTERNS.some(p => p.test(text))) return "verification";
+
+  // Correction patterns — only meaningful after verification or error
+  if (priorCategory === "verification" || step.kind === "error") {
+    if (CORRECTION_PATTERNS.some(p => p.test(text))) return "correction";
+  }
+
+  // Error steps that follow task steps are corrections (fixing an issue)
+  if (step.kind === "error" && priorCategory === "task") return "correction";
+
+  return "task";
+}
+
+/**
+ * Extract structured trace steps from a raw execution transcript.
+ *
+ * This is best-effort — the transcript is unstructured text from
+ * different executor surfaces. We extract what we can.
+ */
+export function extractTraceSteps(rawOutput: string): TraceStep[] {
+  if (!rawOutput || rawOutput.length === 0) return [];
+
+  const steps: TraceStep[] = [];
+  const lines = rawOutput.split("\n");
+  let stepIndex = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]!.trim();
+    if (line.length === 0) continue;
+
+    // Check tool-use patterns first (higher signal)
+    let matched = false;
+    for (const { pattern, kind } of TOOL_USE_PATTERNS) {
+      const m = line.match(pattern);
+      if (m) {
+        // Look ahead for outcome on next non-empty line
+        let outcome = "";
+        for (let j = i + 1; j < Math.min(i + 5, lines.length); j++) {
+          const next = lines[j]!.trim();
+          if (next.length > 0) {
+            outcome = next.slice(0, 120);
+            break;
+          }
+        }
+        const actionText = (m[1] ?? line).slice(0, 200);
+        const priorCategory = steps.length > 0 ? steps[steps.length - 1]!.budget_category : null;
+        const budget_category = classifyStepBudget({ kind, action: actionText, outcome }, priorCategory);
+        steps.push({
+          index: stepIndex++,
+          kind,
+          action: actionText,
+          outcome,
+          duration_hint_ms: 0,
+          budget_category,
+        });
+        matched = true;
+        break;
+      }
+    }
+    if (matched) continue;
+
+    // Check phase/section patterns
+    for (const { pattern, kind } of PHASE_PATTERNS) {
+      const m = line.match(pattern);
+      if (m) {
+        const actionText = (m[1] ?? line).slice(0, 200);
+        const priorCategory = steps.length > 0 ? steps[steps.length - 1]!.budget_category : null;
+        const budget_category = classifyStepBudget({ kind, action: actionText, outcome: "" }, priorCategory);
+        steps.push({
+          index: stepIndex++,
+          kind,
+          action: actionText,
+          outcome: "",
+          duration_hint_ms: 0,
+          budget_category,
+        });
+        break;
+      }
+    }
+  }
+
+  return steps;
+}
+
+// ---------------------------------------------------------------------------
+// Failure context extraction
+// ---------------------------------------------------------------------------
+
+const ERROR_CATEGORIES: Array<{ pattern: RegExp; category: string }> = [
+  { pattern: /timeout|timed?\s*out/i, category: "timeout" },
+  { pattern: /rate.?limit/i, category: "rate_limit" },
+  { pattern: /api.?error|status\s+[45]\d\d/i, category: "api_error" },
+  { pattern: /parse|JSON\.parse|SyntaxError/i, category: "parse_error" },
+  { pattern: /ENOENT|not found|no such file/i, category: "file_not_found" },
+  { pattern: /permission|EACCES|forbidden/i, category: "permission_error" },
+  { pattern: /crash|segfault|SIGKILL|SIGTERM/i, category: "crash" },
+  { pattern: /out of memory|OOM|heap/i, category: "resource_exhaustion" },
+];
+
+/**
+ * Classify an error message into a short category.
+ */
+export function classifyError(errorMessage: string): string {
+  if (!errorMessage) return "unknown";
+  for (const { pattern, category } of ERROR_CATEGORIES) {
+    if (pattern.test(errorMessage)) return category;
+  }
+  return "execution_error";
+}
+
+/**
+ * Build structured failure context from execution results.
+ */
+export function buildFailureContext(input: {
+  failed: boolean;
+  error: string | null;
+  phase?: string;
+  steps?: TraceStep[];
+  rawOutput?: string;
+}): FailureContext {
+  if (!input.failed) {
+    return {
+      failed: false,
+      failure_phase: "",
+      error_category: "",
+      error_message: "",
+      last_successful_step: input.steps?.length
+        ? input.steps.length - 1
+        : -1,
+      partial_work_summary: "",
+    };
+  }
+
+  const errorMsg = input.error ?? "Unknown error";
+  const errorCategory = classifyError(errorMsg);
+  const failurePhase = input.phase ?? "execution";
+
+  // Find last successful step
+  const steps = input.steps ?? [];
+  const lastSuccess = steps.length > 0 ? steps.length - 1 : -1;
+
+  // Summarize partial work from successful steps
+  const completedActions = steps
+    .filter(s => s.kind === "tool_use" || s.kind === "command")
+    .map(s => s.action)
+    .slice(0, 5);
+  const partialSummary = completedActions.length > 0
+    ? `${completedActions.length} tool/command action(s) completed before failure: ${completedActions.join("; ").slice(0, 200)}`
+    : "No tool/command actions completed before failure.";
+
+  return {
+    failed: true,
+    failure_phase: failurePhase,
+    error_category: errorCategory,
+    error_message: errorMsg.slice(0, 500),
+    last_successful_step: lastSuccess,
+    partial_work_summary: partialSummary,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Trace builder — assembles the full execution trace
+// ---------------------------------------------------------------------------
+
+export function buildExecutionTrace(input: {
+  dispatchId: string;
+  rawOutput: string;
+  durationMs: number;
+  failed: boolean;
+  error?: string | null;
+  failurePhase?: string;
+}): ExecutionTrace {
+  const steps = extractTraceSteps(input.rawOutput);
+  const toolUseCount = steps.filter(
+    s => s.kind === "tool_use" || s.kind === "command",
+  ).length;
+
+  const failure = buildFailureContext({
+    failed: input.failed,
+    error: input.error ?? null,
+    ...(input.failurePhase != null && { phase: input.failurePhase }),
+    steps,
+    rawOutput: input.rawOutput,
+  });
+
+  // Build a one-line summary
+  const summaryParts: string[] = [];
+  summaryParts.push(`${steps.length} step(s)`);
+  summaryParts.push(`${toolUseCount} tool/command use(s)`);
+  summaryParts.push(`${Math.round(input.durationMs / 1000)}s`);
+  if (failure.failed) {
+    summaryParts.push(`failed at ${failure.failure_phase}: ${failure.error_category}`);
+  }
+
+  return {
+    trace_id: `trace-${randomBytes(8).toString("hex")}`,
+    dispatch_id: input.dispatchId,
+    created_at: new Date().toISOString(),
+    step_count: steps.length,
+    tool_use_count: toolUseCount,
+    duration_ms: input.durationMs,
+    steps,
+    failure,
+    summary: summaryParts.join(", "),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Trace summary — lightweight fields for embedding on receipts
+// ---------------------------------------------------------------------------
+
+export function deriveTraceSummary(trace: ExecutionTrace | null): TraceSummary {
+  if (!trace) {
+    return {
+      trace_step_count: 0,
+      trace_tool_use_count: 0,
+      trace_failure_phase: "",
+      trace_error_category: "",
+      trace_duration_ms: 0,
+      trace_available: false,
+    };
+  }
+
+  return {
+    trace_step_count: trace.step_count,
+    trace_tool_use_count: trace.tool_use_count,
+    trace_failure_phase: trace.failure.failure_phase,
+    trace_error_category: trace.failure.error_category,
+    trace_duration_ms: trace.duration_ms,
+    trace_available: true,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Persistence — capture trace as a proof artifact
+// ---------------------------------------------------------------------------
+
+/**
+ * Persist an execution trace as a YAML proof artifact.
+ */
+export async function captureExecutionTrace(input: {
+  proofDir: string;
+  trace: ExecutionTrace;
+}): Promise<ProofArtifact> {
+  return captureProofArtifact({
+    proofDir: input.proofDir,
+    kind: "execution-trace",
+    provenance: "derived",
+    filename: "execution-trace.yaml",
+    content: YAML.stringify(input.trace),
+    description: input.trace.summary,
+  });
+}
+
+/**
+ * Persist failure context as a separate proof artifact.
+ * Only written when the execution actually failed.
+ */
+export async function captureFailureContext(input: {
+  proofDir: string;
+  failure: FailureContext;
+  dispatchId: string;
+}): Promise<ProofArtifact | null> {
+  if (!input.failure.failed) return null;
+
+  return captureProofArtifact({
+    proofDir: input.proofDir,
+    kind: "failure-context",
+    provenance: "derived",
+    filename: "failure-context.yaml",
+    content: YAML.stringify({
+      dispatch_id: input.dispatchId,
+      ...input.failure,
+      captured_at: new Date().toISOString(),
+    }),
+    description: `Failure in ${input.failure.failure_phase}: ${input.failure.error_category} — ${input.failure.error_message.slice(0, 100)}`,
+  });
+}