stagent 0.9.2 → 0.9.5

package/src/lib/workflows/error-analysis.ts

@@ -0,0 +1,249 @@
+import { db } from "@/lib/db";
+import { workflows, agentLogs } from "@/lib/db/schema";
+import { eq, sql } from "drizzle-orm";
+import type { WorkflowDefinition, WorkflowState, StepState } from "./types";
+
+export interface TimelineEvent {
+  timestamp: string;
+  event: string;
+  severity: "success" | "warning" | "error";
+  details: string;
+  stepId?: string;
+}
+
+export interface FixSuggestion {
+  tier: "quick" | "better" | "best";
+  title: string;
+  description: string;
+  action?: string;
+}
+
+export interface DebugAnalysis {
+  rootCause: {
+    type: "budget_exceeded" | "timeout" | "transient" | "unknown";
+    summary: string;
+  };
+  timeline: TimelineEvent[];
+  suggestions: FixSuggestion[];
+  stepErrors: Array<{
+    stepId: string;
+    stepName: string;
+    error: string;
+  }>;
+}
+
+function classifyRootCause(errors: string[]): DebugAnalysis["rootCause"] {
+  const combined = errors.join(" ");
+
+  if (/budget|Budget|maximum budget/i.test(combined)) {
+    return {
+      type: "budget_exceeded",
+      summary: "The workflow exceeded its allocated budget before completing all steps.",
+    };
+  }
+
+  if (/timeout|max turns|Turn limit/i.test(combined)) {
+    return {
+      type: "timeout",
+      summary: "The workflow ran out of turns or hit a timeout before completing.",
+    };
+  }
+
+  if (/connection|rate limit|ECONNREFUSED/i.test(combined)) {
+    return {
+      type: "transient",
+      summary: "A transient network or rate-limit error interrupted the workflow.",
+    };
+  }
+
+  return {
+    type: "unknown",
+    summary: errors.length > 0
+      ? `Workflow failed: ${errors[0].slice(0, 200)}`
+      : "The workflow failed for an unknown reason.",
+  };
+}
+
+function buildSuggestions(type: DebugAnalysis["rootCause"]["type"]): FixSuggestion[] {
+  switch (type) {
+    case "budget_exceeded":
+      return [
+        {
+          tier: "quick",
+          title: "Raise budget to $10",
+          description: "Increase the per-step or workflow budget cap to allow the agent more room to finish.",
+          action: "raise_budget",
+        },
+        {
+          tier: "better",
+          title: "Reduce document context per step",
+          description: "Attach fewer or smaller documents to each step so the agent consumes less budget on context.",
+          action: "reduce_docs",
+        },
+        {
+          tier: "best",
+          title: "Split into smaller workflows",
+          description: "Break the workflow into focused sub-workflows that each stay within budget.",
+          action: "restructure",
+        },
+      ];
+    case "timeout":
+      return [
+        {
+          tier: "quick",
+          title: "Increase max turns to 100",
+          description: "Give the agent more turns to complete complex reasoning chains.",
+        },
+        {
+          tier: "better",
+          title: "Simplify step prompts",
+          description: "Reduce prompt complexity so the agent can finish in fewer turns.",
+        },
+        {
+          tier: "best",
+          title: "Break complex steps into sub-steps",
+          description: "Decompose multi-part steps so each one is tractable within the turn limit.",
+        },
+      ];
+    case "transient":
+      return [
+        {
+          tier: "quick",
+          title: "Retry the workflow",
+          description: "The error was likely temporary. Re-running may succeed without changes.",
+        },
+        {
+          tier: "better",
+          title: "Switch to a different runtime",
+          description: "Use an alternative agent runtime that may have better availability.",
+        },
+        {
+          tier: "best",
+          title: "Add retry logic to step definitions",
+          description: "Configure automatic retries on transient failures for resilient execution.",
+        },
+      ];
+    default:
+      return [
+        {
+          tier: "quick",
+          title: "Check agent logs for details",
+          description: "Review the full agent log output for the failed step to understand the error.",
+        },
+        {
+          tier: "better",
+          title: "Simplify workflow",
+          description: "Reduce the number of steps or prompt complexity to isolate the failure.",
+        },
+        {
+          tier: "best",
+          title: "Contact support",
+          description: "If the issue persists, reach out with the workflow ID and debug timeline.",
+        },
+      ];
+  }
+}
+
+function mapEventSeverity(event: string): TimelineEvent["severity"] {
+  if (event.includes("failed") || event.includes("error")) return "error";
+  if (event.includes("completed") || event.includes("success")) return "success";
+  return "warning";
+}
+
+export async function analyzeWorkflowFailure(workflowId: string): Promise<DebugAnalysis> {
+  // 1. Fetch workflow and parse state
+  const [workflow] = await db
+    .select()
+    .from(workflows)
+    .where(eq(workflows.id, workflowId));
+
+  if (!workflow) {
+    throw new Error(`Workflow ${workflowId} not found`);
+  }
+
+  const definition: WorkflowDefinition & { _state?: WorkflowState } = JSON.parse(
+    workflow.definition
+  );
+  const state = definition._state;
+
+  // 2. Query agent_logs for this workflow
+  const logs = await db
+    .select()
+    .from(agentLogs)
+    .where(sql`${agentLogs.payload} LIKE ${"%" + workflowId + "%"}`)
+    .orderBy(agentLogs.timestamp);
+
+  // 3. Build timeline from agent_logs
+  const timeline: TimelineEvent[] = logs.map((log) => {
+    let details = "";
+    try {
+      const payload = JSON.parse(log.payload ?? "{}");
+      details = payload.error || payload.result?.slice(0, 200) || payload.stepName || log.event;
+    } catch {
+      details = log.event;
+    }
+
+    let stepId: string | undefined;
+    try {
+      const payload = JSON.parse(log.payload ?? "{}");
+      stepId = payload.stepId;
+    } catch {
+      // ignore
+    }
+
+    return {
+      timestamp: log.timestamp instanceof Date
+        ? log.timestamp.toISOString()
+        : new Date(log.timestamp).toISOString(),
+      event: log.event,
+      severity: mapEventSeverity(log.event),
+      details,
+      stepId,
+    };
+  });
+
+  // 4. Extract step errors from _state
+  const stepErrors: DebugAnalysis["stepErrors"] = [];
+  const errorMessages: string[] = [];
+
+  if (state?.stepStates) {
+    for (const ss of state.stepStates) {
+      if (ss.status === "failed" && ss.error) {
+        const stepDef = definition.steps.find((s) => s.id === ss.stepId);
+        stepErrors.push({
+          stepId: ss.stepId,
+          stepName: stepDef?.name ?? ss.stepId,
+          error: ss.error,
+        });
+        errorMessages.push(ss.error);
+      }
+    }
+  }
+
+  // Also gather errors from logs if no step errors found
+  if (errorMessages.length === 0) {
+    for (const log of logs) {
+      try {
+        const payload = JSON.parse(log.payload ?? "{}");
+        if (payload.error) {
+          errorMessages.push(payload.error);
+        }
+      } catch {
+        // ignore
+      }
+    }
+  }
+
+  // 5. Classify root cause
+  const rootCause = classifyRootCause(errorMessages);
+
+  // 6. Generate suggestions
+  const suggestions = buildSuggestions(rootCause.type);
+
+  return {
+    rootCause,
+    timeline,
+    suggestions,
+    stepErrors,
+  };
+}

package/src/lib/workflows/execution-stats.ts

@@ -0,0 +1,252 @@
+import { db } from "@/lib/db";
+import {
+  workflows,
+  usageLedger,
+  agentLogs,
+  workflowExecutionStats,
+  workflowDocumentInputs,
+} from "@/lib/db/schema";
+import { eq, and, sql } from "drizzle-orm";
+import type { WorkflowDefinition } from "./types";
+
+// ── Stats update ────────────────────────────────────────────────────
+
+/**
+ * Update aggregated execution stats after a workflow run completes or fails.
+ * Keyed by (pattern, stepCount) bucket — running averages across multiple runs.
+ */
+export async function updateExecutionStats(workflowId: string): Promise<void> {
+  const [workflow] = await db
+    .select()
+    .from(workflows)
+    .where(eq(workflows.id, workflowId));
+
+  if (!workflow) return;
+
+  const definition: WorkflowDefinition = JSON.parse(workflow.definition);
+  const pattern = definition.pattern;
+  const stepCount = definition.steps.length;
+  const succeeded = workflow.status === "completed";
+
+  // Query usage ledger for this workflow's run
+  const usageEntries = await db
+    .select()
+    .from(usageLedger)
+    .where(eq(usageLedger.workflowId, workflowId));
+
+  const totalCostMicros = usageEntries.reduce(
+    (sum, entry) => sum + (entry.costMicros ?? 0),
+    0
+  );
+  const totalDurationMs = usageEntries.reduce((sum, entry) => {
+    if (entry.startedAt && entry.finishedAt) {
+      return sum + (entry.finishedAt.getTime() - entry.startedAt.getTime());
+    }
+    return sum;
+  }, 0);
+
+  const avgCostPerStep = stepCount > 0 ? Math.round(totalCostMicros / stepCount) : 0;
+  const avgDurationPerStep = stepCount > 0 ? Math.round(totalDurationMs / stepCount) : 0;
+
+  // Count documents per step
+  const docBindings = await db
+    .select()
+    .from(workflowDocumentInputs)
+    .where(eq(workflowDocumentInputs.workflowId, workflowId));
+  const avgDocsPerStep = stepCount > 0 ? docBindings.length / stepCount : 0;
+
+  // Collect runtime distribution
+  const runtimeCounts: Record<string, number> = {};
+  for (const entry of usageEntries) {
+    runtimeCounts[entry.runtimeId] = (runtimeCounts[entry.runtimeId] ?? 0) + 1;
+  }
+
+  // Classify failure type from agent logs
+  let failureType: string | null = null;
+  if (!succeeded) {
+    const failLogs = await db
+      .select()
+      .from(agentLogs)
+      .where(
+        and(
+          eq(agentLogs.event, "workflow_failed"),
+          sql`${agentLogs.payload} LIKE ${"%" + workflowId + "%"}`
+        )
+      );
+
+    for (const log of failLogs) {
+      const payload = log.payload ?? "";
+      if (payload.includes("budget") || payload.includes("Budget")) {
+        failureType = "budget_exceeded";
+      } else if (payload.includes("timeout") || payload.includes("max turns")) {
+        failureType = "timeout";
+      } else if (payload.includes("connection") || payload.includes("rate limit")) {
+        failureType = "transient";
+      } else {
+        failureType = "other";
+      }
+    }
+  }
+
+  // Upsert into stats table keyed by (pattern, stepCount)
+  const bucketId = `${pattern}:${stepCount}`;
+  const now = new Date().toISOString();
+
+  const [existing] = await db
+    .select()
+    .from(workflowExecutionStats)
+    .where(eq(workflowExecutionStats.id, bucketId));
+
+  if (existing) {
+    const n = existing.sampleCount;
+    const newN = n + 1;
+
+    // Running average: newAvg = (oldAvg * n + newValue) / (n + 1)
+    const newAvgCost = Math.round(
+      ((existing.avgCostPerStepMicros ?? 0) * n + avgCostPerStep) / newN
+    );
+    const newAvgDuration = Math.round(
+      ((existing.avgDurationPerStepMs ?? 0) * n + avgDurationPerStep) / newN
+    );
+    const newAvgDocs = ((existing.avgDocsPerStep ?? 0) * n + avgDocsPerStep) / newN;
+    const newSuccessRate =
+      ((existing.successRate ?? 0) * n + (succeeded ? 1 : 0)) / newN;
+
+    // Merge common failures
+    const existingFailures: Record<string, number> = existing.commonFailures
+      ? JSON.parse(existing.commonFailures)
+      : {};
+    if (failureType) {
+      existingFailures[failureType] = (existingFailures[failureType] ?? 0) + 1;
+    }
+
+    // Merge runtime breakdown (convert counts to rates)
+    const existingRuntimes: Record<string, number> = existing.runtimeBreakdown
+      ? JSON.parse(existing.runtimeBreakdown)
+      : {};
+    for (const [rtId, count] of Object.entries(runtimeCounts)) {
+      existingRuntimes[rtId] = (existingRuntimes[rtId] ?? 0) + count;
+    }
+
+    await db
+      .update(workflowExecutionStats)
+      .set({
+        avgCostPerStepMicros: newAvgCost,
+        avgDurationPerStepMs: newAvgDuration,
+        avgDocsPerStep: Math.round(newAvgDocs * 100) / 100,
+        successRate: Math.round(newSuccessRate * 1000) / 1000,
+        commonFailures: JSON.stringify(existingFailures),
+        runtimeBreakdown: JSON.stringify(existingRuntimes),
+        sampleCount: newN,
+        lastUpdated: now,
+      })
+      .where(eq(workflowExecutionStats.id, bucketId));
+  } else {
+    await db.insert(workflowExecutionStats).values({
+      id: bucketId,
+      pattern,
+      stepCount,
+      avgDocsPerStep: Math.round(avgDocsPerStep * 100) / 100,
+      avgCostPerStepMicros: avgCostPerStep,
+      avgDurationPerStepMs: avgDurationPerStep,
+      successRate: succeeded ? 1.0 : 0.0,
+      commonFailures: failureType ? JSON.stringify({ [failureType]: 1 }) : "{}",
+      runtimeBreakdown: JSON.stringify(runtimeCounts),
+      sampleCount: 1,
+      lastUpdated: now,
+      createdAt: now,
+    });
+  }
+}
+
+// ── Optimization hints ──────────────────────────────────────────────
+
+export interface OptimizationHints {
+  budgetRecommendation: number | null;
+  docBindingStrategy: "global" | "per-step";
+  runtimeRecommendation: string | null;
+  patternComparison: { pattern: string; successRate: number } | null;
+  similarWorkflowStats: {
+    avgCostPerStepMicros: number;
+    avgDurationPerStepMs: number;
+    successRate: number;
+    sampleCount: number;
+  } | null;
+}
+
+/**
+ * Get optimization hints based on historical execution data.
+ * Returns sensible defaults when no history exists (cold start).
+ */
+export async function getWorkflowOptimizationHints(
+  pattern: string,
+  stepCount: number,
+  _docCount: number
+): Promise<OptimizationHints> {
+  const bucketId = `${pattern}:${stepCount}`;
+  const [stats] = await db
+    .select()
+    .from(workflowExecutionStats)
+    .where(eq(workflowExecutionStats.id, bucketId));
+
+  if (!stats || stats.sampleCount === 0) {
+    return {
+      budgetRecommendation: null,
+      docBindingStrategy: "global",
+      runtimeRecommendation: null,
+      patternComparison: null,
+      similarWorkflowStats: null,
+    };
+  }
+
+  // Budget recommendation: avg + 50% buffer
+  const avgCostUsd = (stats.avgCostPerStepMicros ?? 0) / 1_000_000;
+  const budgetRecommendation =
+    Math.round(avgCostUsd * 1.5 * stepCount * 100) / 100;
+
+  // Doc binding strategy: per-step if avg docs > 3
+  const docBindingStrategy =
+    (stats.avgDocsPerStep ?? 0) > 3 ? "per-step" : "global";
+
+  // Runtime recommendation: pick highest success rate from breakdown
+  let runtimeRecommendation: string | null = null;
+  if (stats.runtimeBreakdown) {
+    const breakdown: Record<string, number> = JSON.parse(stats.runtimeBreakdown);
+    const totalRuns = Object.values(breakdown).reduce((a, b) => a + b, 0);
+    if (totalRuns > 0) {
+      runtimeRecommendation = Object.entries(breakdown).sort(
+        ([, a], [, b]) => b - a
+      )[0]?.[0] ?? null;
+    }
+  }
+
+  // Pattern comparison: check if alternative pattern has better success rate
+  let patternComparison: { pattern: string; successRate: number } | null = null;
+  const allStats = await db.select().from(workflowExecutionStats);
+  for (const altStats of allStats) {
+    if (
+      altStats.pattern !== pattern &&
+      altStats.sampleCount >= 3 &&
+      (altStats.successRate ?? 0) > (stats.successRate ?? 0) + 0.2
+    ) {
+      patternComparison = {
+        pattern: altStats.pattern,
+        successRate: altStats.successRate ?? 0,
+      };
+      break;
+    }
+  }
+
+  return {
+    budgetRecommendation: budgetRecommendation > 0 ? budgetRecommendation : null,
+    docBindingStrategy,
+    runtimeRecommendation,
+    patternComparison,
+    similarWorkflowStats: {
+      avgCostPerStepMicros: stats.avgCostPerStepMicros ?? 0,
+      avgDurationPerStepMs: stats.avgDurationPerStepMs ?? 0,
+      successRate: stats.successRate ?? 0,
+      sampleCount: stats.sampleCount,
+    },
+  };
+}

package/src/lib/workflows/optimizer.ts

@@ -0,0 +1,193 @@
+import type { WorkflowDefinition } from "./types";
+import { getWorkflowOptimizationHints } from "./execution-stats";
+import { estimateWorkflowCost } from "./cost-estimator";
+
+export type SuggestionType =
+  | "document_binding"
+  | "budget_estimate"
+  | "runtime_recommendation"
+  | "pattern_insight";
+
+export interface OptimizationSuggestion {
+  type: SuggestionType;
+  title: string;
+  description: string;
+  data: Record<string, unknown>;
+  action?: {
+    label: string;
+    type: string;
+    payload: Record<string, unknown>;
+  };
+}
+
+/**
+ * Generate optimization suggestions for a workflow definition.
+ * Analyzes document bindings, budget, runtime, and pattern choices
+ * against historical execution data.
+ *
+ * Graceful on errors — each sub-analysis is independent and failures
+ * are caught so the remaining suggestions still return.
+ */
+export async function generateOptimizationSuggestions(
+  definition: Partial<WorkflowDefinition>,
+  workflowId?: string
+): Promise<OptimizationSuggestion[]> {
+  const suggestions: OptimizationSuggestion[] = [];
+
+  // ── Document Binding Analysis ──────────────────────────────────────
+  try {
+    const steps = definition.steps ?? [];
+    if (steps.length > 0) {
+      // Collect all document IDs referenced at the workflow level (global docs)
+      const globalDocIds: string[] = [];
+      for (const step of steps) {
+        if (step.documentIds) {
+          for (const docId of step.documentIds) {
+            if (!globalDocIds.includes(docId)) {
+              globalDocIds.push(docId);
+            }
+          }
+        }
+      }
+
+      const stepCount = steps.length;
+      const docCount = globalDocIds.length;
+
+      // If many docs across many steps, per-step binding reduces redundant injections
+      if (stepCount > 2 && docCount > 3) {
+        const totalInjections = docCount * stepCount;
+        // Estimate that per-step binding typically needs ~40% of total injections
+        const perStepInjections = Math.max(
+          stepCount,
+          Math.round(totalInjections * 0.4)
+        );
+
+        suggestions.push({
+          type: "document_binding",
+          title: "Reduce document injections with per-step binding",
+          description: `${docCount} docs × ${stepCount} steps = ${totalInjections} injections → only ${perStepInjections} needed with per-step binding`,
+          data: {
+            globalDocCount: docCount,
+            stepCount,
+            totalInjections,
+            perStepInjections,
+          },
+          action: {
+            label: "Enable per-step docs",
+            type: "set_per_step_docs",
+            payload: { strategy: "per-step" },
+          },
+        });
+      }
+    }
+  } catch {
+    // Silently skip document binding analysis on error
+  }
+
+  // ── Budget Estimate ────────────────────────────────────────────────
+  try {
+    if (workflowId) {
+      const costEstimate = await estimateWorkflowCost(workflowId);
+
+      if (costEstimate.steps.length > 0) {
+        const perStepBreakdown = costEstimate.steps.map((s) => ({
+          name: s.name,
+          estimatedCostUsd: s.estimatedCostUsd,
+          budgetCapUsd: s.budgetCapUsd,
+        }));
+
+        suggestions.push({
+          type: "budget_estimate",
+          title: `Estimated cost: $${costEstimate.totalEstimatedCostUsd.toFixed(4)}`,
+          description: costEstimate.overBudget
+            ? `Over budget — total cap is $${costEstimate.totalBudgetCapUsd.toFixed(2)}. ${costEstimate.warnings[0] ?? ""}`
+            : `Within budget cap of $${costEstimate.totalBudgetCapUsd.toFixed(2)}`,
+          data: {
+            totalEstimatedCostUsd: costEstimate.totalEstimatedCostUsd,
+            totalBudgetCapUsd: costEstimate.totalBudgetCapUsd,
+            overBudget: costEstimate.overBudget,
+            perStepBreakdown,
+            warnings: costEstimate.warnings,
+          },
+          action: costEstimate.overBudget
+            ? {
+                label: "Adjust budget",
+                type: "set_budget",
+                payload: {
+                  suggestedBudgetUsd:
+                    Math.round(costEstimate.totalEstimatedCostUsd * 1.3 * 100) / 100,
+                },
+              }
+            : undefined,
+        });
+      }
+    }
+  } catch {
+    // Silently skip budget analysis on error
+  }
+
+  // ── Runtime + Pattern Recommendation ───────────────────────────────
+  try {
+    const pattern = definition.pattern ?? "sequence";
+    const stepCount = definition.steps?.length ?? 0;
+
+    // Count docs across all steps
+    let docCount = 0;
+    for (const step of definition.steps ?? []) {
+      docCount += step.documentIds?.length ?? 0;
+    }
+
+    const hints = await getWorkflowOptimizationHints(pattern, stepCount, docCount);
+
+    // Runtime recommendation
+    if (hints.runtimeRecommendation) {
+      const successPct = hints.similarWorkflowStats
+        ? `${Math.round(hints.similarWorkflowStats.successRate * 100)}%`
+        : "N/A";
+
+      suggestions.push({
+        type: "runtime_recommendation",
+        title: `Recommended runtime: ${hints.runtimeRecommendation}`,
+        description: `Based on ${hints.similarWorkflowStats?.sampleCount ?? 0} similar runs with ${successPct} success rate`,
+        data: {
+          runtimeId: hints.runtimeRecommendation,
+          similarWorkflowStats: hints.similarWorkflowStats,
+        },
+        action: {
+          label: "Use this runtime",
+          type: "set_runtime",
+          payload: { runtimeId: hints.runtimeRecommendation },
+        },
+      });
+    }
+
+    // Pattern comparison — suggest alternative if >20% better success rate
+    if (hints.patternComparison) {
+      const altSuccessPct = Math.round(hints.patternComparison.successRate * 100);
+      const currentSuccessPct = hints.similarWorkflowStats
+        ? Math.round(hints.similarWorkflowStats.successRate * 100)
+        : 0;
+
+      suggestions.push({
+        type: "pattern_insight",
+        title: `Consider "${hints.patternComparison.pattern}" pattern`,
+        description: `${altSuccessPct}% success rate vs ${currentSuccessPct}% for "${pattern}" — a ${altSuccessPct - currentSuccessPct}% improvement`,
+        data: {
+          currentPattern: pattern,
+          suggestedPattern: hints.patternComparison.pattern,
+          currentSuccessRate: currentSuccessPct,
+          suggestedSuccessRate: altSuccessPct,
+        },
+        action: {
+          label: "Switch pattern",
+          type: "change_pattern",
+          payload: { pattern: hints.patternComparison.pattern },
+        },
+      });
+    }
+  } catch {
+    // Silently skip runtime/pattern analysis on error
+  }
+
+  return suggestions;
+}