stagent 0.9.5 → 0.10.0

package/src/lib/workflows/loop-executor.ts

@@ -1,9 +1,22 @@
 import { db } from "@/lib/db";
 import { workflows, agentLogs } from "@/lib/db/schema";
 import { eq } from "drizzle-orm";
-import { executeChildTask, updateWorkflowState } from "./engine";
-import type { WorkflowDefinition, LoopState, IterationState, LoopStopReason } from "./types";
+import { executeChildTask } from "./engine";
+import type {
+  WorkflowDefinition,
+  LoopState,
+  IterationState,
+  LoopStopReason,
+  WorkflowEnrichmentTargetContract,
+} from "./types";
 import { createInitialLoopState } from "./types";
+import {
+  resolvePostAction,
+  shouldSkipPostActionValue,
+  extractPostActionValue,
+} from "./post-action";
+import { updateRow } from "@/lib/data/tables";
+import { normalizeEnrichmentOutput } from "@/lib/tables/enrichment-planner";
 
 /**
  * Execute the loop pattern — autonomous iteration with stop conditions.
@@ -27,8 +40,18 @@ export async function executeLoop(
     assignedAgent,
     agentProfile,
     completionSignals,
+    items,
+    itemVariable,
   } = definition.loopConfig;
-  const loopPrompt = definition.steps[0].prompt;
+
+  // Row-driven loop: iterate exactly once per item (capped at maxIterations).
+  // Items array presence flips the loop into a finite fan-out pattern.
+  const isRowDriven = Array.isArray(items);
+  const rowItems = isRowDriven ? (items as unknown[]) : [];
+  const boundVarName = itemVariable && itemVariable.length > 0 ? itemVariable : "item";
+  const effectiveMax = isRowDriven
+    ? Math.min(rowItems.length, maxIterations)
+    : maxIterations;
 
   // Restore existing state (resume) or create fresh
   const loopState = await restoreOrCreateLoopState(workflowId);
@@ -49,7 +72,7 @@ export async function executeLoop(
   }
 
   try {
-    while (loopState.currentIteration < maxIterations) {
+    while (loopState.currentIteration < effectiveMax) {
       // Check pause: re-fetch workflow status from DB
       const [workflow] = await db
         .select()
@@ -81,14 +104,6 @@ export async function executeLoop(
 
       const iterationNum = loopState.currentIteration + 1;
 
-      // Build iteration prompt
-      const prompt = buildIterationPrompt(
-        loopPrompt,
-        previousOutput,
-        iterationNum,
-        maxIterations
-      );
-
       // Create iteration state
       const iterationState: IterationState = {
         iteration: iterationNum,
@@ -112,14 +127,29 @@ export async function executeLoop(
         timestamp: new Date(),
       });
 
-      // Execute child task
-      const result = await executeChildTask(
-        workflowId,
-        `Loop Iteration ${iterationNum}`,
-        prompt,
-        assignedAgent ?? definition.steps[0].assignedAgent,
-        agentProfile ?? definition.steps[0].agentProfile
-      );
+      const result = isRowDriven
+        ? await executeRowDrivenIteration({
+            workflowId,
+            definition,
+            row: rowItems[loopState.currentIteration],
+            itemVariable: boundVarName,
+            iteration: iterationNum,
+            totalRows: effectiveMax,
+            loopAssignedAgent: assignedAgent,
+            loopAgentProfile: agentProfile,
+          })
+        : await executeChildTask(
+            workflowId,
+            `Loop Iteration ${iterationNum}`,
+            buildIterationPrompt(
+              definition.steps[0].prompt,
+              previousOutput,
+              iterationNum,
+              maxIterations
+            ),
+            assignedAgent ?? definition.steps[0].assignedAgent,
+            agentProfile ?? definition.steps[0].agentProfile
+          );
 
       // Update iteration state
       const iterStartTime = new Date(iterationState.startedAt!).getTime();
@@ -130,19 +160,29 @@ export async function executeLoop(
       if (result.status === "completed") {
         iterationState.status = "completed";
         iterationState.result = result.result;
-        previousOutput = result.result ?? "";
+        if (!isRowDriven) {
+          previousOutput = result.result ?? "";
+        }
       } else {
         iterationState.status = "failed";
         iterationState.error = result.error;
-        await finalizeLoop(workflowId, loopState, "error");
-        return;
+        loopState.currentIteration = iterationNum;
+        await updateLoopState(workflowId, loopState, "active");
+        if (!isRowDriven) {
+          await finalizeLoop(workflowId, loopState, "error");
+          return;
+        }
+        continue;
       }
 
       loopState.currentIteration = iterationNum;
       await updateLoopState(workflowId, loopState, "active");
 
-      // Check completion signal
+      // Check completion signal — only for autonomous loops. Row-driven
+      // loops always run through every item; per-row completion text like
+      // "NOT_FOUND" must not abort the fan-out.
       if (
+        !isRowDriven &&
         result.result &&
         detectCompletionSignal(result.result, completionSignals)
       ) {
@@ -164,6 +204,62 @@ export async function executeLoop(
   }
 }
 
+/**
+ * Build the prompt for a single row-driven iteration.
+ *
+ * Row-driven loops fan out one iteration per item in `loopConfig.items` and
+ * stop when items are exhausted (no LOOP_COMPLETE signal needed). The row
+ * payload is serialized as JSON under the bound variable name so the agent
+ * can read every field without us pre-committing to a templating syntax.
+ */
+export function buildRowIterationPrompt(
+  template: string,
+  row: unknown,
+  itemVariable: string,
+  iteration: number,
+  totalRows: number,
+  previousStepOutput: string,
+  stepOutputs: Record<string, string>
+): string {
+  const resolvedTemplate = resolveRowTemplate(template, {
+    [itemVariable]: row,
+    previous: previousStepOutput,
+    stepOutputs,
+  });
+  const parts: string[] = [];
+  parts.push(`Row ${iteration} of ${totalRows}.`);
+  parts.push(
+    `\nCurrent ${itemVariable}:\n\`\`\`json\n${JSON.stringify(row, null, 2)}\n\`\`\``
+  );
+  if (previousStepOutput) {
+    parts.push(`\nPrevious step output:\n${previousStepOutput}`);
+  }
+  parts.push(`\n---\n\n${resolvedTemplate}`);
+  return parts.join("");
+}
+
+function resolveRowTemplate(
+  template: string,
+  context: Record<string, unknown>
+): string {
+  return template.replace(/\{\{\s*([^}]+)\s*\}\}/g, (_match, path: string) => {
+    const value = readContextPath(context, path.trim());
+    if (value === undefined || value === null) return "";
+    return typeof value === "string" ? value : JSON.stringify(value);
+  });
+}
+
+function readContextPath(value: unknown, path: string): unknown {
+  const parts = path.split(".");
+  let current = value;
+  for (const part of parts) {
+    if (current === null || current === undefined) return undefined;
+    if (typeof current !== "object") return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+  return current;
+}
+
 /**
  * Build the prompt for a single iteration, including previous output context.
  */
@@ -253,6 +349,235 @@ async function restoreOrCreateLoopState(
   return createInitialLoopState();
 }
 
+/**
+ * Apply a `postAction` for a single row-driven iteration. Resolves any
+ * `{{row.field}}` placeholders, runs the skip rules, and writes the value
+ * via `updateRow`. Every outcome is logged to `agent_logs` so enrichment
+ * runs are auditable end-to-end. Errors are caught and logged — never
+ * thrown — so a single bad row can't abort the fan-out.
+ */
+async function applyRowPostAction(params: {
+  workflowId: string;
+  taskId: string;
+  postAction: NonNullable<import("./types").WorkflowStep["postAction"]>;
+  row: unknown;
+  itemVariable: string;
+  taskResult: string;
+  targetContract?: WorkflowEnrichmentTargetContract;
+}): Promise<void> {
+  const {
+    workflowId,
+    taskId,
+    postAction,
+    row,
+    itemVariable,
+    taskResult,
+    targetContract,
+  } = params;
+
+  try {
+    if (postAction.type !== "update_row") {
+      // Future-proofing: unknown variants log + return rather than throwing.
+      await db.insert(agentLogs).values({
+        id: crypto.randomUUID(),
+        taskId,
+        agentType: "loop-executor",
+        event: "post_action_unknown_type",
+        payload: JSON.stringify({ workflowId, postAction }),
+        timestamp: new Date(),
+      });
+      return;
+    }
+
+    const resolved = resolvePostAction(postAction, row, itemVariable);
+    const rawValue = extractPostActionValue(taskResult);
+
+    if (!targetContract && shouldSkipPostActionValue(rawValue)) {
+      await db.insert(agentLogs).values({
+        id: crypto.randomUUID(),
+        taskId,
+        agentType: "loop-executor",
+        event: "post_action_skipped",
+        payload: JSON.stringify({
+          workflowId,
+          rowId: resolved.rowId,
+          column: resolved.column,
+          reason: rawValue.trim() === "" ? "empty" : "not_found",
+        }),
+        timestamp: new Date(),
+      });
+      return;
+    }
+
+    const normalized = targetContract
+      ? normalizeEnrichmentOutput(rawValue, targetContract)
+      : { kind: "valid" as const, value: rawValue };
+
+    if (normalized.kind === "skip") {
+      await db.insert(agentLogs).values({
+        id: crypto.randomUUID(),
+        taskId,
+        agentType: "loop-executor",
+        event: "post_action_skipped",
+        payload: JSON.stringify({
+          workflowId,
+          rowId: resolved.rowId,
+          column: resolved.column,
+          reason: normalized.reason,
+        }),
+        timestamp: new Date(),
+      });
+      return;
+    }
+
+    if (normalized.kind === "invalid") {
+      await db.insert(agentLogs).values({
+        id: crypto.randomUUID(),
+        taskId,
+        agentType: "loop-executor",
+        event: "post_action_contract_invalid",
+        payload: JSON.stringify({
+          workflowId,
+          rowId: resolved.rowId,
+          column: resolved.column,
+          error: normalized.reason,
+          rawValue,
+        }),
+        timestamp: new Date(),
+      });
+      return;
+    }
+
+    if (!resolved.rowId) {
+      await db.insert(agentLogs).values({
+        id: crypto.randomUUID(),
+        taskId,
+        agentType: "loop-executor",
+        event: "post_action_failed",
+        payload: JSON.stringify({
+          workflowId,
+          error: "rowId resolved to empty string — check postAction template",
+          postAction,
+        }),
+        timestamp: new Date(),
+      });
+      return;
+    }
+
+    const updated = await updateRow(resolved.rowId, {
+      data: { [resolved.column]: normalized.value },
+    });
+
+    await db.insert(agentLogs).values({
+      id: crypto.randomUUID(),
+      taskId,
+      agentType: "loop-executor",
+      event: updated ? "post_action_applied" : "post_action_failed",
+      payload: JSON.stringify({
+        workflowId,
+        rowId: resolved.rowId,
+        column: resolved.column,
+        tableId: resolved.tableId,
+        ...(updated ? {} : { error: "row not found" }),
+      }),
+      timestamp: new Date(),
+    });
+  } catch (err) {
+    // Never let postAction failures abort the loop iteration.
+    await db.insert(agentLogs).values({
+      id: crypto.randomUUID(),
+      taskId,
+      agentType: "loop-executor",
+      event: "post_action_failed",
+      payload: JSON.stringify({
+        workflowId,
+        error: err instanceof Error ? err.message : String(err),
+      }),
+      timestamp: new Date(),
+    }).catch(() => {});
+  }
+}
+
+async function executeRowDrivenIteration(params: {
+  workflowId: string;
+  definition: WorkflowDefinition;
+  row: unknown;
+  itemVariable: string;
+  iteration: number;
+  totalRows: number;
+  loopAssignedAgent?: string;
+  loopAgentProfile?: string;
+}): Promise<{ taskId: string; status: string; result?: string; error?: string }> {
+  const {
+    workflowId,
+    definition,
+    row,
+    itemVariable,
+    iteration,
+    totalRows,
+    loopAssignedAgent,
+    loopAgentProfile,
+  } = params;
+
+  let previousStepOutput = "";
+  let lastTaskId = "";
+  const stepOutputs: Record<string, string> = {};
+
+  for (const step of definition.steps) {
+    const prompt = buildRowIterationPrompt(
+      step.prompt,
+      row,
+      itemVariable,
+      iteration,
+      totalRows,
+      previousStepOutput,
+      stepOutputs
+    );
+
+    const result = await executeChildTask(
+      workflowId,
+      `${step.name} · Row ${iteration}`,
+      prompt,
+      loopAssignedAgent ?? step.assignedAgent,
+      loopAgentProfile ?? step.agentProfile,
+      undefined,
+      step.id,
+      step.budgetUsd,
+      step.runtimeId
+    );
+    lastTaskId = result.taskId;
+
+    if (result.status !== "completed") {
+      return {
+        taskId: lastTaskId,
+        status: "failed",
+        error: `${step.name}: ${result.error ?? "Task did not complete successfully"}`,
+      };
+    }
+
+    previousStepOutput = result.result ?? "";
+    stepOutputs[step.id] = previousStepOutput;
+
+    if (step.postAction) {
+      await applyRowPostAction({
+        workflowId,
+        taskId: result.taskId,
+        postAction: step.postAction,
+        row,
+        itemVariable,
+        taskResult: previousStepOutput,
+        targetContract: definition.metadata?.enrichment?.targetContract,
+      });
+    }
+  }
+
+  return {
+    taskId: lastTaskId,
+    status: "completed",
+    result: previousStepOutput,
+  };
+}
+
 /**
  * Finalize a loop with a stop reason and mark the workflow as completed.
  */

package/src/lib/workflows/post-action.ts

@@ -0,0 +1,91 @@
+/**
+ * Post-step action helpers — pure logic for the declarative side-effect
+ * framework that bulk row enrichment uses to write agent results back into
+ * user table cells.
+ *
+ * Dispatch (the actual `updateRow` call) lives in the loop-executor where
+ * it has DB access; this module stays pure so the resolution + skip rules
+ * can be unit-tested without mocking DB.
+ *
+ * See features/bulk-row-enrichment.md.
+ */
+
+import type { StepPostAction } from "./types";
+
+/**
+ * Substitute `{{itemVariable.field}}` placeholders in a postAction definition
+ * against the current loop iteration's row. Supports nested paths via dotted
+ * field names (e.g. `{{row.meta.id}}`). Only `rowId` is templated today —
+ * `tableId` and `column` are static, and templating them would invite SQL
+ * surprises.
+ */
+export function resolvePostAction(
+  action: StepPostAction,
+  row: unknown,
+  itemVariable: string
+): StepPostAction {
+  return {
+    ...action,
+    rowId: substituteRowPath(action.rowId, row, itemVariable),
+  };
+}
+
+/**
+ * Replace `{{itemVariable.path.to.field}}` with the value at that path on row.
+ * Multiple placeholders in the same string are all replaced. Missing paths
+ * resolve to an empty string (caller should validate the result).
+ */
+function substituteRowPath(
+  template: string,
+  row: unknown,
+  itemVariable: string
+): string {
+  // Match {{<itemVariable>.<dotted.path>}} — escape itemVariable for safety
+  const escaped = itemVariable.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  const pattern = new RegExp(`\\{\\{\\s*${escaped}\\.([\\w.]+)\\s*\\}\\}`, "g");
+
+  return template.replace(pattern, (_match, path: string) => {
+    const value = readPath(row, path);
+    if (value === undefined || value === null) return "";
+    return String(value);
+  });
+}
+
+function readPath(obj: unknown, path: string): unknown {
+  const parts = path.split(".");
+  let current: unknown = obj;
+  for (const part of parts) {
+    if (current === null || current === undefined) return undefined;
+    if (typeof current !== "object") return undefined;
+    current = (current as Record<string, unknown>)[part];
+  }
+  return current;
+}
+
+/**
+ * Decide whether the agent's result should be written back to the cell, or
+ * skipped silently. We skip empty/whitespace-only results and the literal
+ * `NOT_FOUND` sentinel (case-insensitive) so an enrichment workflow can
+ * gracefully say "no value for this row" without overwriting a real value
+ * with garbage.
+ *
+ * Substring matches are intentionally NOT skipped — only the trimmed value
+ * being exactly `NOT_FOUND` triggers the skip. This avoids dropping a long
+ * answer that happens to mention the sentinel.
+ */
+export function shouldSkipPostActionValue(value: string): boolean {
+  const trimmed = value.trim();
+  if (trimmed === "") return true;
+  if (trimmed.toUpperCase() === "NOT_FOUND") return true;
+  return false;
+}
+
+/**
+ * Pull the writable value out of a task result. Trims whitespace and tolerates
+ * undefined/null without throwing. The caller should run the result through
+ * `shouldSkipPostActionValue` before writing.
+ */
+export function extractPostActionValue(result: string | undefined | null): string {
+  if (result === undefined || result === null) return "";
+  return result.trim();
+}