sea-dev 1.0.0

package/libs/core/src/pipelines/execute.ts

@@ -0,0 +1,2342 @@
+import {
+  createPipelineRun,
+  createPipelineRunStep,
+  getPipelineWithSteps,
+  getPipelineStepsWithLevels,
+  getParentStepRuns,
+  getParentStepRunsWithDependencies,
+  getRuntimeSubmission,
+  getRuntimeSubmissionWithData,
+  storePipelineOutputData,
+  updatePipelineRunStatus,
+  updatePipelineRunStepStatus,
+  logRequestCall,
+  getSubmissionGroups,
+  getGroupForStepRun,
+  resolveStepBlockersOnCompletion,
+  getPipelineStepById,
+  getStepDocuments,
+  getPipelineRunWithSteps,
+  getStepDependenciesByParentStepId,
+  getStepIntegrationConfig,
+} from "@sea/dal/pipeline";
+import { PIPELINE_RUN_STATUS, PIPELINE_RUN_STEP_STATUS } from "@sea/db";
+import {
+  createSubmissionWithConversation,
+  getDenormSubmission,
+  syncSubmissionToForm,
+  getSubmissionWithData,
+  type DenormSubmission,
+} from "@sea/dal/submission";
+import type { FormGroupHydrated, Member, PipelineRunStep } from "@sea/db/types";
+import { db, schema } from "@sea/db";
+import { and, asc, eq, inArray, or } from "drizzle-orm";
+import type {
+  ExecutePipelineInput,
+  ExecutePipelineResult,
+  StepExecutionResult,
+} from "@sea/schemas/core/pipeline";
+import {
+  addAdditionalPropertiesToJsonSchema,
+  buildExtractionSchemaFromGroup,
+} from "@sea/core/data-extraction/structuring/custom";
+import {
+  jsonSchema,
+  Output,
+  stepCountIs,
+  ToolLoopAgent,
+  zodSchema,
+  type JSONSchema7,
+} from "ai";
+import * as z from "zod/v4";
+import type { createLanguageModel } from "@sea/ai/models";
+import type { DalResult } from "@sea/dal/result";
+import { HatchetClientMgr } from "@sea/clients/hatchet";
+import type { HatchetConfig } from "@sea/config";
+import { createPipelineArtifact } from "../chat/tools/pipeline-artifact";
+import pLimit from "p-limit";
+
+// ============================================================================
+// Serialization helpers
+// ============================================================================
+
+function serializeSubmission(submission: DenormSubmission): string {
+  const htmlParts: string[] = [];
+
+  // Add submission header
+  htmlParts.push(`<h2>${submission.formName}</h2>`);
+  if (submission.name) {
+    htmlParts.push(`<p><strong>Submission:</strong> ${submission.name}</p>`);
+  }
+
+  // Serialize single groups (definition lists)
+  for (const group of submission.groups) {
+    htmlParts.push(`<h3>${group.name}</h3>`);
+    htmlParts.push("<dl>");
+
+    for (const item of group.items) {
+      htmlParts.push(`<dt>${item.name}</dt>`);
+      htmlParts.push(`<dd id="${item.id}">${String(item.value ?? "")}</dd>`);
+    }
+
+    // Add derived fields if any
+    for (const derived of group.derived) {
+      htmlParts.push(`<dt>${derived.name} (derived)</dt>`);
+      htmlParts.push(
+        `<dd id="${derived.id}">${String(derived.value ?? "")}</dd>`,
+      );
+    }
+
+    htmlParts.push("</dl>");
+  }
+
+  // Serialize tables (multi groups)
+  for (const table of submission.tables) {
+    htmlParts.push(`<h3>${table.name}</h3>`);
+    htmlParts.push("<table>");
+
+    // Table header
+    htmlParts.push("<thead><tr>");
+    for (const col of table.columns) {
+      htmlParts.push(`<th>${col}</th>`);
+    }
+    htmlParts.push("</tr></thead>");
+
+    // Table body
+    htmlParts.push("<tbody>");
+    for (const row of table.rows) {
+      htmlParts.push("<tr>");
+
+      // Regular items
+      for (const item of row.items) {
+        htmlParts.push(`<td id="${item.id}">${String(item.value ?? "")}</td>`);
+      }
+
+      // Derived items
+      for (const derived of row.derived) {
+        htmlParts.push(
+          `<td id="${derived.id}">${String(derived.value ?? "")}</td>`,
+        );
+      }
+
+      htmlParts.push("</tr>");
+    }
+    htmlParts.push("</tbody>");
+    htmlParts.push("</table>");
+  }
+
+  return htmlParts.join("\n");
+}
+
+function serializePipelineResources(resources: DenormSubmission[]): string {
+  const htmlParts: string[] = [];
+
+  for (const resource of resources) {
+    htmlParts.push(serializeSubmission(resource));
+    htmlParts.push("<hr>");
+  }
+
+  return htmlParts.join("\n");
+}
+
+// ============================================================================
+// DAG Execution Engine (Phase 1)
+// ============================================================================
+
+/**
+ * Execute pipeline using DAG traversal with level-by-level execution.
+ * This replaces the old sequential execution approach.
+ */
+export async function executePipelineDAG({
+  member,
+  runId,
+  config,
+}: {
+  member: Member;
+  runId: string;
+  config: {
+    model: ReturnType<typeof createLanguageModel>;
+    hatchet?: HatchetConfig;
+  };
+}): Promise<ExecutePipelineResult> {
+  try {
+    // Get pipeline run via DAL
+    const runResult = await getPipelineRunWithSteps({ member, runId });
+    if (!runResult.ok) {
+      return { ok: false, code: 404, error: "Pipeline run not found" };
+    }
+    const run = runResult.data;
+
+    // Update run status to running
+    await updatePipelineRunStatus({
+      member,
+      runId,
+      status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+    });
+
+    // Get execution levels for topological ordering
+    const levelsResult = await getPipelineStepsWithLevels({
+      member,
+      pipelineId: run.pipelineId,
+    });
+
+    if (!levelsResult.ok) {
+      await updatePipelineRunStatus({
+        member,
+        runId,
+        status: PIPELINE_RUN_STATUS.FAILED,
+        errorSummary: "Failed to compute execution levels",
+      });
+      return {
+        ok: false,
+        code: 500,
+        error: "Failed to compute execution levels",
+      };
+    }
+
+    const stepsWithLevels = levelsResult.data;
+
+    // Group steps by execution level
+    const levelMap = new Map<number, typeof stepsWithLevels>();
+    for (const step of stepsWithLevels) {
+      const level = step.executionLevel;
+      if (!levelMap.has(level)) {
+        levelMap.set(level, []);
+      }
+      levelMap.get(level)!.push(step);
+    }
+
+    const levels = Array.from(levelMap.keys()).sort((a, b) => a - b);
+
+    // Execute level by level
+    const limit = pLimit(5); // Concurrent execution limit per level
+    console.log(
+      "[executePipelineDAG] Executing levels:",
+      JSON.stringify(levels, null, 2),
+    );
+
+    for (const level of levels) {
+      const stepsInLevel = levelMap.get(level) || [];
+      console.log(
+        `[executePipelineDAG] Executing level ${level} with ${stepsInLevel.length} steps`,
+      );
+
+      // Execute all steps in this level concurrently (with limit)
+      const levelPromises = stepsInLevel.map((step) =>
+        limit(() =>
+          executeStepByRole({
+            member,
+            runId,
+            stepId: step.id,
+            config,
+          }),
+        ),
+      );
+
+      try {
+        // Wait for all steps in this level to complete (or be blocked/failed)
+        await Promise.all(levelPromises);
+
+        // After level completes, check if any steps are still blocked
+        // If so, we cannot proceed to the next level
+        const levelStepIds = stepsInLevel.map((s) => s.id);
+        const runSteps = await db.query.pipelineRunStep.findMany({
+          where: and(
+            eq(schema.pipelineRunStep.runId, runId),
+            // Filter to steps in this level
+            or(
+              ...levelStepIds.map((id) =>
+                eq(schema.pipelineRunStep.stepId, id),
+              ),
+            ),
+          ),
+        });
+
+        const blockedSteps = runSteps.filter(
+          (rs) => rs.status === PIPELINE_RUN_STEP_STATUS.BLOCKED,
+        );
+        const failedSteps = runSteps.filter(
+          (rs) => rs.status === PIPELINE_RUN_STEP_STATUS.FAILED,
+        );
+
+        if (failedSteps.length > 0) {
+          console.error(
+            `[executePipelineDAG] Level ${level} has ${failedSteps.length} failed step(s)`,
+          );
+          await updatePipelineRunStatus({
+            member,
+            runId,
+            status: PIPELINE_RUN_STATUS.FAILED,
+            errorSummary: `${failedSteps.length} step(s) failed in level ${level}`,
+          });
+          return {
+            ok: false,
+            code: 500,
+            error: `Level ${level} had failed steps`,
+          };
+        }
+
+        if (blockedSteps.length > 0) {
+          console.log(
+            `[executePipelineDAG] Level ${level} has ${blockedSteps.length} blocked step(s), marking run as blocked`,
+          );
+          await updatePipelineRunStatus({
+            member,
+            runId,
+            status: PIPELINE_RUN_STATUS.BLOCKED,
+          });
+          return {
+            ok: false,
+            code: 400,
+            error: `${blockedSteps.length} step(s) are blocked - resolve blockers to continue`,
+          };
+        }
+
+        // After level completes, check for spawned instances and execute them
+        console.log(
+          `[executePipelineDAG] Checking for spawned instances after level ${level}`,
+        );
+
+        // First check all run steps to see what's there
+        const allRunSteps = await db.query.pipelineRunStep.findMany({
+          where: eq(schema.pipelineRunStep.runId, runId),
+        });
+
+        console.log(
+          `[executePipelineDAG] All run steps (${allRunSteps.length}):`,
+          allRunSteps.map((rs) => ({
+            id: rs.id,
+            stepId: rs.stepId,
+            spawnType: rs.spawnType,
+            status: rs.status,
+            stepRunIndex: rs.stepRunIndex,
+          })),
+        );
+
+        const spawnedInstances = await db.query.pipelineRunStep.findMany({
+          where: and(
+            eq(schema.pipelineRunStep.runId, runId),
+            eq(schema.pipelineRunStep.spawnType, "instance"),
+            eq(schema.pipelineRunStep.status, "pending"),
+          ),
+        });
+
+        console.log(
+          `[executePipelineDAG] Spawned instances query result (${spawnedInstances.length}):`,
+          spawnedInstances.map((si) => ({
+            id: si.id,
+            stepId: si.stepId,
+            spawnType: si.spawnType,
+            status: si.status,
+            stepRunIndex: si.stepRunIndex,
+          })),
+        );
+
+        if (spawnedInstances.length > 0) {
+          console.log(
+            `[executePipelineDAG] Found ${spawnedInstances.length} spawned instance(s), executing in parallel (max 10)`,
+          );
+
+          // Execute spawned instances with max 10 concurrency
+          const instanceLimit = pLimit(10);
+          const instancePromises = spawnedInstances.map((instance) =>
+            instanceLimit(() =>
+              executeInstanceStep({
+                member,
+                runId,
+                runStepId: instance.id,
+                stepId: instance.stepId!,
+                config,
+              }),
+            ),
+          );
+
+          await Promise.all(instancePromises);
+
+          // Check if any instances failed or are blocked
+          const completedInstances = await db.query.pipelineRunStep.findMany({
+            where: and(
+              eq(schema.pipelineRunStep.runId, runId),
+              inArray(
+                schema.pipelineRunStep.id,
+                spawnedInstances.map((i) => i.id),
+              ),
+            ),
+          });
+
+          const failedInstances = completedInstances.filter(
+            (i) => i.status === PIPELINE_RUN_STEP_STATUS.FAILED,
+          );
+          const blockedInstances = completedInstances.filter(
+            (i) => i.status === PIPELINE_RUN_STEP_STATUS.BLOCKED,
+          );
+
+          if (failedInstances.length > 0) {
+            console.error(
+              `[executePipelineDAG] ${failedInstances.length} spawned instance(s) failed`,
+            );
+            await updatePipelineRunStatus({
+              member,
+              runId,
+              status: PIPELINE_RUN_STATUS.FAILED,
+              errorSummary: `${failedInstances.length} spawned instance(s) failed in level ${level}`,
+            });
+            return {
+              ok: false,
+              code: 500,
+              error: `Spawned instances failed in level ${level}`,
+            };
+          }
+
+          if (blockedInstances.length > 0) {
+            console.log(
+              `[executePipelineDAG] ${blockedInstances.length} spawned instance(s) blocked`,
+            );
+            await updatePipelineRunStatus({
+              member,
+              runId,
+              status: PIPELINE_RUN_STATUS.BLOCKED,
+            });
+            return {
+              ok: false,
+              code: 400,
+              error: `${blockedInstances.length} spawned instance(s) are blocked`,
+            };
+          }
+
+          console.log(
+            `[executePipelineDAG] All ${spawnedInstances.length} spawned instance(s) completed successfully`,
+          );
+        }
+
+        console.log(
+          `[executePipelineDAG] Level ${level} completed successfully`,
+        );
+      } catch (error) {
+        console.error(`[executePipelineDAG] Level ${level} failed:`, error);
+        await updatePipelineRunStatus({
+          member,
+          runId,
+          status: PIPELINE_RUN_STATUS.FAILED,
+          errorSummary:
+            error instanceof Error ? error.message : "Level execution failed",
+        });
+        return {
+          ok: false,
+          code: 500,
+          error: `Level ${level} execution failed`,
+        };
+      }
+    }
+
+    // Update run status to completed
+    await updatePipelineRunStatus({
+      member,
+      runId,
+      status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+    });
+
+    return {
+      ok: true,
+      code: 200,
+      data: { runId, status: PIPELINE_RUN_STATUS.COMPLETED },
+    };
+  } catch (error) {
+    console.error("[executePipelineDAG] Error:", error);
+    await updatePipelineRunStatus({
+      member,
+      runId,
+      status: PIPELINE_RUN_STATUS.FAILED,
+      errorSummary: error instanceof Error ? error.message : "Unknown error",
+    });
+    return {
+      ok: false,
+      code: 500,
+      error: "Pipeline execution failed",
+    };
+  }
+}
+
+/**
+ * Dispatcher that routes step execution based on role.
+ * Handles backwards compatibility for steps without a role.
+ */
+async function executeStepByRole({
+  member,
+  runId,
+  stepId,
+  config,
+}: {
+  member: Member;
+  runId: string;
+  stepId: string;
+  config: {
+    model: ReturnType<typeof createLanguageModel>;
+    hatchet?: HatchetConfig;
+  };
+}): Promise<void> {
+  // Get step definition
+  const stepResult = await getPipelineStepById({ member, stepId });
+
+  if (!stepResult.ok) {
+    throw new Error(`Step ${stepId} not found`);
+  }
+
+  const step = stepResult.data;
+
+  // Get the existing run step (created when the run was initialized)
+  const existingRunStep = await db.query.pipelineRunStep.findFirst({
+    where: and(
+      eq(schema.pipelineRunStep.runId, runId),
+      eq(schema.pipelineRunStep.stepId, stepId),
+      or(
+        eq(schema.pipelineRunStep.spawnType, "normal"),
+        eq(schema.pipelineRunStep.spawnType, "template"),
+      ),
+    ),
+  });
+
+  if (!existingRunStep) {
+    throw new Error(`Run step not found for step ${stepId}`);
+  }
+
+  // If it's a template step, check if we should spawn instances now
+  if (existingRunStep.spawnType === "template") {
+    console.log(
+      `[executeStepByRole] Template step ${stepId} encountered - checking if ready to spawn instances`,
+    );
+
+    // Check if parent dependencies are satisfied
+    const parentStepRunsResult = await getParentStepRuns({
+      member,
+      runStepId: existingRunStep.id,
+    });
+
+    if (!parentStepRunsResult.ok) {
+      console.error(
+        `[executeStepByRole] Failed to get parent steps for template ${stepId}`,
+      );
+      return;
+    }
+
+    const parentStepRuns = parentStepRunsResult.data;
+    const incompleteParents = parentStepRuns.filter(
+      (p) => p.status !== PIPELINE_RUN_STEP_STATUS.COMPLETED,
+    );
+
+    if (incompleteParents.length > 0) {
+      console.log(
+        `[executeStepByRole] Template ${stepId} not ready - ${incompleteParents.length} parent(s) not completed`,
+      );
+      return;
+    }
+
+    // All parents completed - try to spawn instances
+    console.log(
+      `[executeStepByRole] All parents completed for template ${stepId} - attempting to spawn instances`,
+    );
+
+    await spawnInstancesForTemplate({
+      member,
+      runId,
+      runStep: existingRunStep,
+      step,
+    });
+
+    // Mark template as spawned
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: existingRunStep.id,
+      status: "spawned",
+    });
+
+    console.log(`[executeStepByRole] Template ${stepId} marked as spawned`);
+    return;
+  }
+
+  const runStep = existingRunStep;
+
+  // Check for unresolved blockers before executing
+  const blockers = await db.query.pipelineRunStepBlocker.findMany({
+    where: and(
+      eq(schema.pipelineRunStepBlocker.runStepId, runStep.id),
+      eq(schema.pipelineRunStepBlocker.resolved, false),
+    ),
+  });
+
+  if (blockers.length > 0) {
+    console.log(
+      `[executeStepByRole] Step ${stepId} has ${blockers.length} unresolved blocker(s), marking as blocked`,
+    );
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.BLOCKED,
+    });
+    // Don't throw - just skip this step for now
+    return;
+  }
+
+  try {
+    // Dispatch based on role (null role = legacy analysis step)
+    switch (step.role) {
+      case "extraction":
+        await executeExtractionStep({ member, runStep, step, config });
+        break;
+      case "integration":
+        await executeIntegrationStep({ member, runStep, step, config });
+        break;
+      case "analysis":
+        await executeAnalysisStep({ member, runStep, step, config });
+        break;
+      case "manual":
+        await executeManualStep({ member, runStep, step });
+        break;
+      default:
+        throw new Error(`Unknown step role: ${step.role}`);
+    }
+
+    // After successful completion:
+    // 1. Check for multi-group spawning (do this BEFORE marking complete)
+    await checkAndSpawnMultiGroups({ member, runStep, step });
+
+    // 2. Resolve this step's blockers and cascade to children
+    // This will mark parent_incomplete blockers on child steps as resolved
+    await resolveStepBlockersOnCompletion({
+      member,
+      runStepId: runStep.id,
+    });
+
+    // 3. Verify output submission was created (critical for downstream steps)
+    const outputMapping =
+      await db.query.pipelineRunStepOutputSubmission.findFirst({
+        where: eq(schema.pipelineRunStepOutputSubmission.runStepId, runStep.id),
+      });
+
+    if (
+      step.role !== "manual" &&
+      step.role !== "integration" &&
+      !outputMapping
+    ) {
+      console.error(
+        `[executeStepByRole] Step ${stepId} completed but no output submission was created!`,
+      );
+      throw new Error("Step completed without creating output submission");
+    }
+
+    console.log(
+      `[executeStepByRole] Step ${stepId} completed successfully with output submission: ${outputMapping?.submissionId ?? "N/A"}`,
+    );
+  } catch (error) {
+    console.error(`[executeStepByRole] Step ${stepId} failed:`, error);
+    throw error;
+  }
+}
+
+/**
+ * Execute a spawned instance step
+ * Similar to executeStepByRole but operates on a specific runStepId (instance)
+ */
+async function executeInstanceStep({
+  member,
+  runId,
+  runStepId,
+  stepId,
+  config,
+}: {
+  member: Member;
+  runId: string;
+  runStepId: string;
+  stepId: string;
+  config: {
+    model: ReturnType<typeof createLanguageModel>;
+    hatchet?: HatchetConfig;
+  };
+}): Promise<void> {
+  // Get step definition
+  const stepResult = await getPipelineStepById({ member, stepId });
+
+  if (!stepResult.ok) {
+    throw new Error(`Step ${stepId} not found`);
+  }
+
+  const step = stepResult.data;
+
+  // Get the specific instance run step
+  const runStep = await db.query.pipelineRunStep.findFirst({
+    where: eq(schema.pipelineRunStep.id, runStepId),
+  });
+
+  if (!runStep) {
+    throw new Error(`Run step ${runStepId} not found`);
+  }
+
+  if (runStep.spawnType !== "instance") {
+    throw new Error(`Run step ${runStepId} is not an instance`);
+  }
+
+  // Check for unresolved blockers before executing
+  const blockers = await db.query.pipelineRunStepBlocker.findMany({
+    where: and(
+      eq(schema.pipelineRunStepBlocker.runStepId, runStep.id),
+      eq(schema.pipelineRunStepBlocker.resolved, false),
+    ),
+  });
+
+  if (blockers.length > 0) {
+    console.log(
+      `[executeInstanceStep] Instance ${runStepId} has ${blockers.length} unresolved blocker(s), marking as blocked`,
+    );
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.BLOCKED,
+    });
+    return;
+  }
+
+  try {
+    // Only analysis steps can have instances
+    if (step.role !== "analysis" && step.role !== null) {
+      throw new Error(`Step role ${step.role} cannot have instances`);
+    }
+
+    await executeAnalysisStep({ member, runStep, step, config });
+
+    // Resolve blockers after completion
+    await resolveStepBlockersOnCompletion({
+      member,
+      runStepId: runStep.id,
+    });
+
+    console.log(
+      `[executeInstanceStep] Instance ${runStepId} completed successfully`,
+    );
+  } catch (error) {
+    console.error(`[executeInstanceStep] Instance ${runStepId} failed:`, error);
+    throw error;
+  }
+}
+
+/**
+ * Execute an extraction step (document extraction)
+ * Triggers async Hatchet workflow for each document via pushDoc()
+ */
+async function executeExtractionStep({
+  member,
+  runStep,
+  step,
+  config,
+}: {
+  member: Member;
+  runStep: PipelineRunStep;
+  step: typeof schema.pipelineStep.$inferSelect & {
+    outputForms: Array<{
+      id: string;
+      stepId: string;
+      formId: string;
+      createdAt: Date;
+    }>;
+  };
+  config: {
+    model: ReturnType<typeof createLanguageModel>;
+    hatchet?: HatchetConfig;
+  };
+}): Promise<void> {
+  try {
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+    });
+
+    // 1. Get documents linked to this step run
+    const documentsResult = await getStepDocuments({
+      member,
+      runStepId: runStep.id,
+    });
+
+    if (!documentsResult.ok) {
+      throw new Error("Failed to load step documents");
+    }
+
+    if (documentsResult.data.length === 0) {
+      throw new Error("No documents linked to extraction step");
+    }
+
+    // 2. Get output form
+    const outputFormId = step.outputForms[0]?.formId;
+    if (!outputFormId) {
+      throw new Error("Extraction step missing output form");
+    }
+
+    // 3. Get pipeline run to access dealId
+    const pipelineRun = await db.query.pipelineRun.findFirst({
+      where: eq(schema.pipelineRun.id, runStep.runId),
+    });
+    if (!pipelineRun) {
+      throw new Error("Pipeline run not found");
+    }
+
+    // 4. Create output submission for extracted data
+    const outputSubmission = await createSubmissionWithConversation({
+      member,
+      data: {
+        formId: outputFormId,
+        dealId: pipelineRun.dealId,
+        name:
+          runStep.spawnType === "instance"
+            ? `${step.name} - Instance ${runStep.stepRunIndex! + 1}`
+            : `${step.name} output`,
+        status: "draft",
+        type: "extraction",
+      },
+    });
+
+    await syncSubmissionToForm({
+      member,
+      submissionId: outputSubmission.id,
+    });
+
+    // 5. For each document, create documentExtraction and trigger Hatchet workflow
+    if (!config.hatchet) {
+      throw new Error("Hatchet config required for extraction steps");
+    }
+
+    const hatchetClient = new HatchetClientMgr(config.hatchet);
+    const extractionIds: string[] = [];
+
+    for (const document of documentsResult.data) {
+      // Create documentExtraction record
+      const [documentExtraction] = await db
+        .insert(schema.documentExtraction)
+        .values({
+          documentId: document.id,
+          submissionId: outputSubmission.id,
+          status: "pending",
+          extractionModel: "general",
+          extractionTrigger: "api",
+        })
+        .returning({ id: schema.documentExtraction.id });
+
+      if (!documentExtraction) {
+        throw new Error(
+          `Failed to create documentExtraction for document ${document.id}`,
+        );
+      }
+
+      extractionIds.push(documentExtraction.id);
+
+      // Trigger Hatchet extraction workflow
+      await hatchetClient.pushDoc(
+        documentExtraction.id,
+        "general",
+        step.prompt || undefined,
+        "medium",
+      );
+
+      console.log(
+        `[executeExtractionStep] Triggered extraction for document: ${document.name} (extractionId: ${documentExtraction.id})`,
+      );
+    }
+
+    // 6. Link output submission to run step
+    await db.insert(schema.pipelineRunStepOutputSubmission).values({
+      runStepId: runStep.id,
+      submissionId: outputSubmission.id,
+    });
+
+    console.log(
+      `[executeExtractionStep] Waiting for ${extractionIds.length} extraction(s) to complete...`,
+    );
+
+    // 7. Poll for extraction completion
+    const MAX_WAIT_TIME_MS = 30 * 60 * 1000; // 30 minutes
+    const POLL_INTERVAL_MS = 5000; // 5 seconds
+    const startTime = Date.now();
+
+    while (true) {
+      // Check if we've exceeded max wait time
+      if (Date.now() - startTime > MAX_WAIT_TIME_MS) {
+        throw new Error(
+          `Extraction timed out after ${MAX_WAIT_TIME_MS / 1000} seconds`,
+        );
+      }
+
+      // Check status of all extractions
+      const extractions = await db.query.documentExtraction.findMany({
+        where: inArray(schema.documentExtraction.id, extractionIds),
+        columns: { id: true, status: true },
+      });
+
+      const allDone = extractions.every((e) => e.status === "done");
+      const anyFailed = extractions.some((e) => e.status === "failed");
+
+      if (anyFailed) {
+        const failedIds = extractions
+          .filter((e) => e.status === "failed")
+          .map((e) => e.id);
+        throw new Error(`Extraction failed for: ${failedIds.join(", ")}`);
+      }
+
+      if (allDone) {
+        console.log(
+          `[executeExtractionStep] All ${extractionIds.length} extraction(s) completed successfully`,
+        );
+        break;
+      }
+
+      // Wait before next poll
+      await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+    }
+
+    // 8. Mark step as completed
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+    });
+
+    console.log(
+      `[executeExtractionStep] Completed extraction step with ${documentsResult.data.length} document(s), output submission: ${outputSubmission.id}`,
+    );
+  } catch (error) {
+    console.error("[executeExtractionStep] Error:", error);
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STATUS.FAILED,
+      error: error instanceof Error ? error.message : "Unknown error",
+    });
+    throw error;
+  }
+}
+
+/**
+ * Execute an integration step (API call)
+ */
+async function executeIntegrationStep({
+  member,
+  runStep,
+  step,
+  config,
+}: {
+  member: Member;
+  runStep: PipelineRunStep;
+  step: typeof schema.pipelineStep.$inferSelect & {
+    outputForms: Array<{
+      id: string;
+      stepId: string;
+      formId: string;
+      createdAt: Date;
+    }>;
+  };
+  config: { model: ReturnType<typeof createLanguageModel> };
+}): Promise<void> {
+  try {
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+    });
+
+    // Get integration config from separate table
+    const integrationConfigResult = await getStepIntegrationConfig({
+      member,
+      stepId: step.id,
+    });
+
+    if (!integrationConfigResult.ok || !integrationConfigResult.data) {
+      throw new Error("Integration step missing config");
+    }
+
+    const integrationConfig = integrationConfigResult.data;
+
+    // Get parent step outputs to build request
+    const parentStepRunsResult = await getParentStepRuns({
+      member,
+      runStepId: runStep.id,
+    });
+
+    if (!parentStepRunsResult.ok) {
+      throw new Error("Failed to load parent step outputs");
+    }
+
+    const parentStepRuns = parentStepRunsResult.data;
+
+    // Load output submissions for each parent step run
+    const parentSubmissions = await Promise.all(
+      parentStepRuns.map(async (p) => {
+        const outputMapping =
+          await db.query.pipelineRunStepOutputSubmission.findFirst({
+            where: eq(schema.pipelineRunStepOutputSubmission.runStepId, p.id),
+          });
+        if (!outputMapping) return null;
+        return getDenormSubmission({
+          member,
+          submissionId: outputMapping.submissionId,
+          withCitations: true,
+          withTransformations: true,
+        });
+      }),
+    );
+
+    // Filter out nulls
+    const validParentSubmissions = parentSubmissions.filter(
+      (s): s is NonNullable<typeof s> => s !== null,
+    );
+
+    // Build request from template
+    const request = await buildHttpRequest({
+      config: {
+        method: integrationConfig.method as "GET" | "POST" | "PUT" | "DELETE",
+        urlTemplate: integrationConfig.urlTemplate,
+        headers: integrationConfig.headers,
+        bodyTemplate: integrationConfig.bodyTemplate ?? undefined,
+      },
+      parentData: validParentSubmissions,
+    });
+
+    // Make API call
+    const startTime = Date.now();
+    let statusCode: number | undefined;
+    let responseBody: unknown;
+    let error: string | undefined;
+
+    try {
+      const response = await fetch(request.url, {
+        method: request.method,
+        headers: request.headers,
+        body: request.body,
+      });
+
+      statusCode = response.status;
+      responseBody = await response.json();
+    } catch (err) {
+      error = err instanceof Error ? err.message : "Request failed";
+    }
+
+    const durationMs = Date.now() - startTime;
+
+    // Log request call
+    await logRequestCall({
+      member,
+      runStepId: runStep.id,
+      method: request.method,
+      url: request.url,
+      requestHeaders: request.headers,
+      requestBody: request.body,
+      statusCode,
+      responseBody,
+      durationMs,
+      error,
+    });
+
+    if (error || !statusCode || statusCode >= 400) {
+      throw new Error(error || `HTTP ${statusCode}`);
+    }
+
+    // Parse response into output form if configured
+    if (step.outputForms.length > 0) {
+      // TODO: Parse responseBody into form structure and store
+      // For now, mark as completed without storing output
+    }
+
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+    });
+  } catch (error) {
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STATUS.FAILED,
+      error: error instanceof Error ? error.message : "Integration step failed",
+    });
+    throw error;
+  }
+}
+
+/**
+ * Execute an analysis/transformation step (LLM-based)
+ * Enhanced to support multi-parent inputs
+ */
+async function executeAnalysisStep({
+  member,
+  runStep,
+  step,
+  config,
+}: {
+  member: Member;
+  runStep: PipelineRunStep;
+  step: typeof schema.pipelineStep.$inferSelect & {
+    outputForms: Array<{
+      id: string;
+      stepId: string;
+      formId: string;
+      createdAt: Date;
+    }>;
+  };
+  config: { model: ReturnType<typeof createLanguageModel> };
+}): Promise<void> {
+  try {
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+    });
+
+    // Validate output form exists
+    if (step.outputForms.length === 0) {
+      throw new Error("Analysis step missing output form");
+    }
+
+    // Get parent step runs with dependency configurations
+    const parentStepRunsResult = await getParentStepRunsWithDependencies({
+      member,
+      runStepId: runStep.id,
+    });
+
+    if (!parentStepRunsResult.ok) {
+      throw new Error("Failed to load parent step runs");
+    }
+
+    const parentStepRuns = parentStepRunsResult.data;
+
+    // Load output submissions for each parent step run
+    const parentSubmissions = await Promise.all(
+      parentStepRuns.map(
+        async (p: {
+          id: string;
+          stepId: string | null;
+          status: string;
+          formIdMapping: Record<string, string[]> | null;
+          multiFormGroupId: string | null;
+        }) => {
+          const outputMapping =
+            await db.query.pipelineRunStepOutputSubmission.findFirst({
+              where: eq(schema.pipelineRunStepOutputSubmission.runStepId, p.id),
+            });
+          if (!outputMapping) return null;
+
+          const submission = await getDenormSubmission({
+            member,
+            submissionId: outputMapping.submissionId,
+            withCitations: true,
+            withTransformations: true,
+          });
+
+          if (!submission) return null;
+
+          // Apply formIdMapping filter if specified
+          if (p.formIdMapping && Object.keys(p.formIdMapping).length > 0) {
+            // Filter submission groups based on formIdMapping
+            // formIdMapping is { [formId]: [groupId1, groupId2, ...] }
+            const allowedGroupIds = new Set<string>();
+            for (const groupIds of Object.values(p.formIdMapping)) {
+              (groupIds as string[]).forEach((gid: string) =>
+                allowedGroupIds.add(gid),
+              );
+            }
+
+            // Filter groups to only include those in formIdMapping
+            const filteredGroups = submission.groups.filter((g) =>
+              allowedGroupIds.has(g.formGroupId),
+            );
+
+            return { ...submission, groups: filteredGroups };
+          }
+
+          return submission;
+        },
+      ),
+    );
+
+    // Filter out nulls
+    const validParentSubmissions = parentSubmissions.filter(
+      (s): s is NonNullable<typeof s> => s !== null,
+    );
+
+    // Check if this is a multi step - if so, filter parent data to specific group
+    const groupResult = await getGroupForStepRun({
+      member,
+      runStepId: runStep.id,
+    });
+
+    let filteredParentSubmissions = validParentSubmissions;
+    if (groupResult.ok) {
+      // This is a multi step, filter to the specific group
+      const targetGroup = groupResult.data;
+
+      // Build a map of which parent steps have the multiFormGroupId
+      const parentStepsWithMultiGroup = new Set<string>();
+      for (const parent of parentStepRuns) {
+        if (parent.multiFormGroupId === targetGroup.formGroupId) {
+          parentStepsWithMultiGroup.add(parent.id);
+        }
+      }
+
+      // Filter each parent submission to only include the specific group instance
+      // BUT only if that parent has the multiFormGroupId configured
+      filteredParentSubmissions = validParentSubmissions.map(
+        (submission, idx) => {
+          const parentStepRun = parentStepRuns[idx];
+          if (!parentStepRun) return submission;
+
+          // If this parent doesn't have the multiFormGroupId, return full data
+          if (!parentStepsWithMultiGroup.has(parentStepRun.id)) {
+            return submission;
+          }
+
+          // This parent has the multiFormGroupId, so filter to the specific group
+          const matchingGroup = submission.groups.find(
+            (g) => g.formGroupId === targetGroup.formGroupId,
+          );
+
+          if (!matchingGroup) {
+            // No matching group, return submission with empty groups
+            return { ...submission, groups: [] };
+          }
+
+          // Return submission with only the matching group
+          return { ...submission, groups: [matchingGroup] };
+        },
+      );
+    }
+
+    // Create output submission
+    const sourceSubmission = validParentSubmissions[0];
+    if (!sourceSubmission) {
+      throw new Error("No parent submissions found");
+    }
+
+    // Get full submission to access dealId
+    const fullSubmissionResult = await getRuntimeSubmission({
+      member,
+      submissionId: sourceSubmission.id,
+    });
+
+    if (!fullSubmissionResult.ok) {
+      throw new Error("Failed to load source submission");
+    }
+
+    const outputFormId = step.outputForms[0]?.formId;
+    if (!outputFormId) {
+      throw new Error("Analysis step missing output form");
+    }
+
+    const outputSubmission = await createSubmissionWithConversation({
+      member,
+      data: {
+        formId: outputFormId,
+        dealId: fullSubmissionResult.data.dealId,
+        name:
+          runStep.spawnType === "instance"
+            ? `${step.name} - Instance ${runStep.stepRunIndex! + 1}`
+            : `${step.name} output`,
+        status: "draft",
+        type: "transformation",
+      },
+    });
+
+    await syncSubmissionToForm({
+      member,
+      submissionId: outputSubmission.id,
+    });
+
+    // Get output form groups
+    const outputSubmissionWithData = await getSubmissionWithData({
+      member,
+      submissionId: outputSubmission.id,
+      typeFilter: "transformation",
+    });
+
+    if (!outputSubmissionWithData?.form?.groups) {
+      throw new Error("Failed to load output form");
+    }
+
+    const outputFormGroups = outputSubmissionWithData.form.groups;
+
+    // If this is a spawned instance, extract the spawning group data
+    let spawningGroupData: {
+      groupName: string;
+      data: DenormSubmission["groups"][0];
+    } | null = null;
+
+    if (groupResult.ok) {
+      const targetGroup = groupResult.data;
+      // Find the spawning group in the filtered parent submissions
+      for (const submission of filteredParentSubmissions) {
+        const matchingGroup = submission.groups.find(
+          (g) => g.formGroupId === targetGroup.formGroupId,
+        );
+        if (matchingGroup) {
+          spawningGroupData = {
+            groupName: matchingGroup.name,
+            data: matchingGroup,
+          };
+          break;
+        }
+      }
+    }
+
+    // Run the LLM-based analysis
+    const stepResult = await runStepExecute({
+      member,
+      runId: runStep.runId,
+      runStepId: runStep.id,
+      step: {
+        id: step.id,
+        description: step.description,
+        prompt: step.prompt,
+        toolsAllowed: step.toolsAllowed,
+        outputFormIds: step.outputForms.map((of) => of.formId),
+        // Legacy fields - no longer used in new architecture but required by function signature
+        stepOrder: 0,
+        parentStepId: null,
+      },
+      inputSubmissions: filteredParentSubmissions,
+      resourceSubmissions: [], // TODO: Load pipeline resources
+      outputFormGroups,
+      conversationId: outputSubmission.conversationId,
+      spawningGroupData,
+      config,
+    });
+
+    // Store output
+    if (stepResult.output?.data) {
+      const storeResult = await storePipelineOutputData({
+        member,
+        outputSubmissionId: outputSubmission.id,
+        outputData: stepResult.output.data,
+      });
+
+      if (!storeResult.ok) {
+        throw new Error("Failed to store pipeline output data");
+      }
+    }
+
+    // Link output submission to run step
+    await db.insert(schema.pipelineRunStepOutputSubmission).values({
+      runStepId: runStep.id,
+      submissionId: outputSubmission.id,
+    });
+
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: stepResult.status,
+      error: stepResult.error,
+    });
+
+    if (stepResult.status === "failed") {
+      throw new Error(stepResult.error || "Analysis step failed");
+    }
+  } catch (error) {
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STATUS.FAILED,
+      error: error instanceof Error ? error.message : "Analysis step failed",
+    });
+    throw error;
+  }
+}
+
+/**
+ * Execute a manual step (waits for human input)
+ */
+async function executeManualStep({
+  member,
+  runStep,
+  step,
+}: {
+  member: Member;
+  runStep: PipelineRunStep;
+  step: typeof schema.pipelineStep.$inferSelect & {
+    outputForms: Array<{
+      id: string;
+      stepId: string;
+      formId: string;
+      createdAt: Date;
+    }>;
+  };
+}): Promise<void> {
+  try {
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+    });
+
+    // Check if manual output submission has been provided
+    const outputMapping =
+      await db.query.pipelineRunStepOutputSubmission.findFirst({
+        where: eq(schema.pipelineRunStepOutputSubmission.runStepId, runStep.id),
+      });
+
+    if (!outputMapping) {
+      // No output submission yet - create a blocker
+      console.log(
+        `[executeManualStep] Step ${step.name} blocked - waiting for manual output submission`,
+      );
+
+      // Check if blocker already exists
+      const existingBlocker = await db.query.pipelineRunStepBlocker.findFirst({
+        where: and(
+          eq(schema.pipelineRunStepBlocker.runStepId, runStep.id),
+          eq(schema.pipelineRunStepBlocker.blockerType, "missing_manual_input"),
+          eq(schema.pipelineRunStepBlocker.resolved, false),
+        ),
+      });
+
+      if (!existingBlocker) {
+        await db.insert(schema.pipelineRunStepBlocker).values({
+          runStepId: runStep.id,
+          blockerType: "missing_manual_input",
+          blockerDetails: {
+            type: "missing_manual_input",
+            stepName: step.name,
+            stepDescription: step.description,
+          },
+          resolved: false,
+        });
+      }
+
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId: runStep.id,
+        status: PIPELINE_RUN_STEP_STATUS.BLOCKED,
+      });
+
+      return;
+    }
+
+    // Output submission exists - mark as completed
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+    });
+
+    console.log(
+      `[executeManualStep] Step ${step.name} completed - manual output submission provided`,
+    );
+  } catch (error) {
+    console.error("[executeManualStep] Error:", error);
+    await updatePipelineRunStepStatus({
+      member,
+      runStepId: runStep.id,
+      status: PIPELINE_RUN_STATUS.FAILED,
+      error: error instanceof Error ? error.message : "Manual step failed",
+    });
+    throw error;
+  }
+}
+
+/**
+ * Spawn instances for a template step based on parent multi-group output
+ */
+async function spawnInstancesForTemplate({
+  member,
+  runId,
+  runStep,
+  step,
+}: {
+  member: Member;
+  runId: string;
+  runStep: PipelineRunStep;
+  step: typeof schema.pipelineStep.$inferSelect & {
+    outputForms: Array<{
+      id: string;
+      stepId: string;
+      formId: string;
+      createdAt: Date;
+    }>;
+  };
+}): Promise<void> {
+  console.log(
+    `[spawnInstancesForTemplate] Starting spawn for template step ${step.id}`,
+  );
+
+  // Get parent step runs with dependency info
+  const parentStepRunsResult = await getParentStepRunsWithDependencies({
+    member,
+    runStepId: runStep.id,
+  });
+
+  if (!parentStepRunsResult.ok) {
+    console.error(
+      `[spawnInstancesForTemplate] Failed to get parent dependencies for step ${step.id}`,
+    );
+    return;
+  }
+
+  const parentStepRuns = parentStepRunsResult.data;
+  console.log(
+    `[spawnInstancesForTemplate] Found ${parentStepRuns.length} parent step(s)`,
+  );
+
+  // Find parent with multiFormGroupId
+  const multiParent = parentStepRuns.find((p) => p.multiFormGroupId !== null);
+
+  if (!multiParent) {
+    console.warn(
+      `[spawnInstancesForTemplate] No multi-group parent found for template step ${step.id}`,
+    );
+    return;
+  }
+
+  console.log(
+    `[spawnInstancesForTemplate] Found multi-group parent with multiFormGroupId: ${multiParent.multiFormGroupId}`,
+  );
+
+  // Get parent output submission
+  const parentOutputMapping =
+    await db.query.pipelineRunStepOutputSubmission.findFirst({
+      where: eq(
+        schema.pipelineRunStepOutputSubmission.runStepId,
+        multiParent.id,
+      ),
+    });
+
+  if (!parentOutputMapping) {
+    console.error(
+      `[spawnInstancesForTemplate] No output submission found for parent step ${multiParent.id}`,
+    );
+    return;
+  }
+
+  console.log(
+    `[spawnInstancesForTemplate] Parent output submission: ${parentOutputMapping.submissionId}`,
+  );
+
+  // Get submission groups using DAL function
+  const groupsResult = await getSubmissionGroups({
+    member,
+    submissionId: parentOutputMapping.submissionId,
+    formGroupId: multiParent.multiFormGroupId!,
+  });
+
+  if (!groupsResult.ok || groupsResult.data.length === 0) {
+    console.log(
+      `[spawnInstancesForTemplate] No groups found for formGroupId ${multiParent.multiFormGroupId}`,
+    );
+    return;
+  }
+
+  const groups = groupsResult.data;
+  console.log(
+    `[spawnInstancesForTemplate] Spawning ${groups.length} instance(s) for step ${step.id}`,
+  );
+
+  // Create instance run steps for each group
+  for (const [index, group] of groups.entries()) {
+    console.log(
+      `[spawnInstancesForTemplate] Creating instance ${index + 1}/${groups.length} for group ${group.id}`,
+    );
+
+    await createPipelineRunStep({
+      member,
+      runId,
+      stepId: step.id,
+      spawnType: "instance",
+      submissionGroupId: group.id,
+      stepRunIndex: index,
+    });
+  }
+
+  console.log(
+    `[spawnInstancesForTemplate] Successfully spawned ${groups.length} instance(s) for step ${step.id}`,
+  );
+}
+
+/**
+ * Check if step output has multiple groups and spawn instance run steps for child steps
+ */
+async function checkAndSpawnMultiGroups({
+  member,
+  runStep,
+  step,
+}: {
+  member: Member;
+  runStep: PipelineRunStep;
+  step: typeof schema.pipelineStep.$inferSelect;
+}): Promise<void> {
+  try {
+    // Only extraction and analysis steps can produce multi-group outputs
+    if (
+      step.role !== "extraction" &&
+      step.role !== "analysis" &&
+      step.role !== null
+    ) {
+      return;
+    }
+
+    // Get output submission if exists
+    const outputMapping =
+      await db.query.pipelineRunStepOutputSubmission.findFirst({
+        where: eq(schema.pipelineRunStepOutputSubmission.runStepId, runStep.id),
+      });
+
+    if (!outputMapping) {
+      return;
+    }
+
+    // Get child steps that depend on this step with multiFormGroupId
+    const childDepsResult = await getStepDependenciesByParentStepId({
+      member,
+      parentStepId: step.id,
+    });
+
+    if (!childDepsResult.ok || childDepsResult.data.length === 0) {
+      return;
+    }
+
+    const multiDeps = childDepsResult.data.filter((d) => d.multiFormGroupId);
+
+    if (multiDeps.length === 0) {
+      return;
+    }
+
+    console.log(
+      `[checkAndSpawnMultiGroups] Found ${multiDeps.length} multi-dependency child step(s)`,
+    );
+
+    for (const dep of multiDeps) {
+      // Find the template run step that was created at run creation
+      const templateRunStep = await db.query.pipelineRunStep.findFirst({
+        where: and(
+          eq(schema.pipelineRunStep.runId, runStep.runId),
+          eq(schema.pipelineRunStep.stepId, dep.childStepId),
+          eq(schema.pipelineRunStep.spawnType, "template"),
+        ),
+      });
+
+      if (!templateRunStep) {
+        console.warn(
+          `[checkAndSpawnMultiGroups] Template run step not found for child step ${dep.childStepId}`,
+        );
+        continue;
+      }
+
+      // Get submission groups using new DAL function
+      const groupsResult = await getSubmissionGroups({
+        member,
+        submissionId: outputMapping.submissionId,
+        formGroupId: dep.multiFormGroupId!,
+      });
+
+      if (!groupsResult.ok || groupsResult.data.length === 0) {
+        console.log(
+          `[checkAndSpawnMultiGroups] No groups found for formGroupId ${dep.multiFormGroupId}`,
+        );
+        continue;
+      }
+
+      const groups = groupsResult.data;
+      console.log(
+        `[checkAndSpawnMultiGroups] Spawning ${groups.length} instance(s) for step ${dep.childStepId}`,
+      );
+
+      // Create instance run steps for each group
+      for (const [index, group] of groups.entries()) {
+        await createPipelineRunStep({
+          member,
+          runId: runStep.runId,
+          stepId: dep.childStepId,
+          spawnType: "instance",
+          submissionGroupId: group.id,
+          stepRunIndex: index,
+        });
+      }
+
+      // Mark template as spawned
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId: templateRunStep.id,
+        status: "spawned",
+      });
+
+      console.log(
+        `[checkAndSpawnMultiGroups] Successfully spawned ${groups.length} instance(s) for step ${dep.childStepId}`,
+      );
+    }
+  } catch (error) {
+    console.error("[checkAndSpawnMultiGroups] Error:", error);
+    // Don't throw - this is non-critical, log and continue
+  }
+}
+
+/**
+ * Build HTTP request from integration config template with variable substitution
+ */
+async function buildHttpRequest({
+  config,
+  parentData,
+}: {
+  config: {
+    method: "GET" | "POST" | "PUT" | "DELETE";
+    urlTemplate: string;
+    headers: Record<string, string>;
+    bodyTemplate?: string;
+  };
+  parentData: DenormSubmission[];
+}): Promise<{
+  method: string;
+  url: string;
+  headers: Record<string, string>;
+  body?: string;
+}> {
+  // Build variable map from parent submissions
+  const variables = new Map<string, string>();
+
+  for (const submission of parentData) {
+    // Add form-level variables
+    variables.set(`form.name`, submission.formName);
+
+    // Add field-level variables from groups
+    for (const group of submission.groups) {
+      for (const item of group.items) {
+        // Variable pattern: groupName.fieldName
+        const key = `${group.name}.${item.name}`;
+        const value = String(item.value ?? "");
+        variables.set(key, value);
+      }
+
+      // Also add derived fields
+      for (const derived of group.derived) {
+        const key = `${group.name}.${derived.name}`;
+        const value = String(derived.value ?? "");
+        variables.set(key, value);
+      }
+    }
+
+    // Add table row variables (use first row for simplicity)
+    for (const table of submission.tables) {
+      const firstRow = table.rows[0];
+      if (firstRow) {
+        for (const item of firstRow.items) {
+          const key = `${table.name}.${item.name}`;
+          const value = String(item.value ?? "");
+          variables.set(key, value);
+        }
+      }
+    }
+  }
+
+  // Substitute variables in URL template
+  // Pattern: {{variableName}} or {{groupName.fieldName}}
+  const substituteVariables = (template: string): string => {
+    return template.replace(/\{\{([^}]+)\}\}/g, (match, varName) => {
+      const trimmedName = varName.trim();
+      const value = variables.get(trimmedName);
+
+      if (value !== undefined) {
+        return encodeURIComponent(value);
+      }
+
+      console.warn(`[buildHttpRequest] Variable not found: ${trimmedName}`);
+      return match; // Keep original if not found
+    });
+  };
+
+  const url = substituteVariables(config.urlTemplate);
+
+  // Substitute in headers
+  const headers: Record<string, string> = {};
+  for (const [key, value] of Object.entries(config.headers)) {
+    headers[key] = substituteVariables(value);
+  }
+
+  // Substitute in body
+  const body = config.bodyTemplate
+    ? substituteVariables(config.bodyTemplate)
+    : undefined;
+
+  return {
+    method: config.method,
+    url,
+    headers,
+    body,
+  };
+}
+
+// ============================================================================
+// Single Step Execution (Internal)
+// ============================================================================
+
+type ExecuteSingleStepResult =
+  | {
+      ok: true;
+      data: { outputSubmissionIds: string[]; runSteps: PipelineRunStep[] };
+    }
+  | { ok: false; error: string };
+
+export async function executeSingleStep({
+  member,
+  runId,
+  pipelineId,
+  step,
+  inputSubmissionId,
+  allInputSubmissionIds,
+  config,
+}: {
+  member: Member;
+  runId: string;
+  pipelineId: string;
+  step: {
+    id: string;
+    stepOrder: number;
+    type: "single" | "multi";
+    description: string;
+    prompt: string;
+    toolsAllowed: string[] | null;
+    parentStepId: string | null;
+    outputFormIds: string[];
+  };
+  inputSubmissionId: string;
+  allInputSubmissionIds?: string[];
+  config: {
+    model: ReturnType<typeof createLanguageModel>;
+  };
+}): Promise<ExecuteSingleStepResult> {
+  console.log("[executeSingleStep] Starting step execution:", {
+    stepId: step.id,
+    stepOrder: step.stepOrder,
+    stepType: step.type,
+    inputSubmissionId,
+    allInputSubmissionIds,
+    allInputCount: allInputSubmissionIds?.length,
+  });
+
+  // 1. Validate step has output form
+  if (!step.outputFormIds || step.outputFormIds.length === 0) {
+    return { ok: false, error: `Step ${step.stepOrder} has no output form` };
+  }
+
+  // 2. Validate input submission exists
+  console.log(
+    "[executeSingleStep] Validating primary input submission:",
+    inputSubmissionId,
+  );
+  const inputSubmissionResult = await getRuntimeSubmission({
+    member,
+    submissionId: inputSubmissionId,
+  });
+  if (!inputSubmissionResult.ok) {
+    console.error("[executeSingleStep] Primary input submission not found:", {
+      inputSubmissionId,
+      organizationId: member.organizationId,
+    });
+    return { ok: false, error: "Input submission not found" };
+  }
+  console.log("[executeSingleStep] Primary input submission found:", {
+    id: inputSubmissionResult.data.id,
+    name: inputSubmissionResult.data.name,
+    formId: inputSubmissionResult.data.formId,
+  });
+  const sourceSubmission = inputSubmissionResult.data;
+
+  // 3. For multi-step, get the input submission data to find multi groups
+  let multiGroupEntries: Array<{ order: number; data: unknown }> = [];
+  if (step.type === "multi") {
+    const inputSubmission = await getDenormSubmission({
+      member,
+      submissionId: inputSubmissionId,
+      withCitations: true,
+      withTransformations: true,
+    });
+
+    // Find the first multi group in tables
+    const firstMultiTable = inputSubmission.tables[0];
+    if (!firstMultiTable || firstMultiTable.rows.length === 0) {
+      return {
+        ok: false,
+        error:
+          "Multi-step requires input with at least one multi group with entries",
+      };
+    }
+
+    // Create an entry for each row in the multi group
+    multiGroupEntries = firstMultiTable.rows.map((row, idx) => ({
+      order: idx,
+      data: row,
+    }));
+  }
+
+  // 4. Determine how many output submissions to create
+  const outputCount = step.type === "multi" ? multiGroupEntries.length : 1;
+  const outputSubmissions: Array<{
+    id: string;
+    dealId: string;
+    conversationId: string;
+  }> = [];
+  const runSteps: PipelineRunStep[] = [];
+
+  // 5. Create output submissions and run steps
+  for (let i = 0; i < outputCount; i++) {
+    const outputFormId = step.outputFormIds[0];
+    if (!outputFormId) {
+      return { ok: false, error: `Step ${step.stepOrder} has no output form` };
+    }
+
+    const outputSubmission = await createSubmissionWithConversation({
+      member,
+      data: {
+        formId: outputFormId,
+        dealId: sourceSubmission.dealId,
+        name:
+          step.type === "multi"
+            ? `Step ${step.stepOrder} - Entry ${i + 1}`
+            : `Step ${step.stepOrder}`,
+        status: "draft",
+        type: "transformation",
+      },
+    });
+
+    // Sync submission to form to create groups and items
+    await syncSubmissionToForm({
+      member,
+      submissionId: outputSubmission.id,
+    });
+
+    outputSubmissions.push(outputSubmission);
+
+    // Create run step in DB
+    const runStepResult = await createPipelineRunStep({
+      member,
+      runId,
+      stepId: step.id,
+    });
+    if (!runStepResult.ok) {
+      return { ok: false, error: "Failed to create run step" };
+    }
+    const runStep = runStepResult.data;
+    runSteps.push(runStep);
+
+    // Link output submission to run step
+    await db.insert(schema.pipelineRunStepOutputSubmission).values({
+      runStepId: runStep.id,
+      submissionId: outputSubmission.id,
+    });
+  }
+
+  // 6. Execute each output submission in parallel (with concurrency limit of 10)
+  const CONCURRENCY_LIMIT = 10;
+  const executeSubmission = async (
+    outputSubmission: (typeof outputSubmissions)[number],
+    i: number,
+  ) => {
+    const runStep = runSteps[i];
+
+    if (!outputSubmission || !runStep) {
+      throw new Error("Internal error: missing output submission or run step");
+    }
+
+    try {
+      // Update status to running
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId: runStep.id,
+        status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+      });
+
+      // Get input submission data with citations
+      // For single-step after multi-step: load ALL input submissions
+      const inputSubmissionsToLoad = allInputSubmissionIds || [
+        inputSubmissionId,
+      ];
+      console.log(
+        "[executeSingleStep] Loading input submissions for execution:",
+        {
+          count: inputSubmissionsToLoad.length,
+          ids: inputSubmissionsToLoad,
+        },
+      );
+
+      const inputSubmissions = await Promise.all(
+        inputSubmissionsToLoad.map(async (id) => {
+          console.log("[executeSingleStep] Loading submission:", id);
+          try {
+            const submission = await getDenormSubmission({
+              member,
+              submissionId: id,
+              withCitations: true,
+              withTransformations: true,
+            });
+            console.log("[executeSingleStep] Successfully loaded submission:", {
+              id,
+              name: submission.name,
+            });
+            return submission;
+          } catch (error) {
+            console.error("[executeSingleStep] Failed to load submission:", {
+              id,
+              error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
+          }
+        }),
+      );
+      console.log(
+        "[executeSingleStep] All input submissions loaded successfully",
+      );
+
+      // Get output form groups
+      const outputSubmissionWithData = await getSubmissionWithData({
+        member,
+        submissionId: outputSubmission.id,
+        typeFilter: "transformation",
+      });
+      if (!outputSubmissionWithData?.form?.groups) {
+        throw new Error("Failed to load output form");
+      }
+      const outputFormGroups = outputSubmissionWithData.form.groups;
+
+      // Run the step execution
+      const stepResult = await runStepExecute({
+        member,
+        runId,
+        runStepId: runStep.id,
+        step,
+        inputSubmissions,
+        resourceSubmissions: [],
+        outputFormGroups,
+        conversationId: outputSubmission.conversationId,
+        config,
+      });
+
+      // Write output using storePipelineOutputData
+      if (stepResult.output?.data) {
+        const storeResult = await storePipelineOutputData({
+          member,
+          outputSubmissionId: outputSubmission.id,
+          outputData: stepResult.output.data,
+        });
+        if (!storeResult.ok) {
+          throw new Error("Failed to store pipeline output data");
+        }
+      }
+
+      // Update step status
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId: runStep.id,
+        status: stepResult.status,
+        error: stepResult.error,
+      });
+
+      if (stepResult.status === "failed") {
+        throw new Error(stepResult.error ?? "Step failed");
+      }
+
+      return outputSubmission.id;
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : "Step execution failed";
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId: runStep.id,
+        status: PIPELINE_RUN_STATUS.FAILED,
+        error: errorMessage,
+      });
+      throw error;
+    }
+  };
+
+  // Execute in batches with concurrency limit
+  const allOutputSubmissionIds: string[] = [];
+  try {
+    for (let i = 0; i < outputSubmissions.length; i += CONCURRENCY_LIMIT) {
+      const batch = outputSubmissions.slice(i, i + CONCURRENCY_LIMIT);
+      const batchPromises = batch.map((outputSubmission, batchIdx) =>
+        executeSubmission(outputSubmission, i + batchIdx),
+      );
+      const batchResults = await Promise.all(batchPromises);
+      allOutputSubmissionIds.push(...batchResults);
+    }
+
+    return {
+      ok: true,
+      data: { outputSubmissionIds: allOutputSubmissionIds, runSteps },
+    };
+  } catch (error) {
+    const errorMessage =
+      error instanceof Error ? error.message : "Step execution failed";
+    return { ok: false, error: errorMessage };
+  }
+}
+
+// ============================================================================
+// Step Executor
+// ============================================================================
+
+function buildStepResponseSchema(outputFormGroups: FormGroupHydrated[]) {
+  const extractionSchema = outputFormGroups.reduce((acc, group) => {
+    const groupSchema = buildExtractionSchemaFromGroup(group);
+    return z.object({
+      ...acc.shape,
+      ...groupSchema.shape,
+    });
+  }, z.object({}));
+
+  return z.object({
+    reasoning: z.string().nullable(),
+    data: extractionSchema.nullable(),
+  });
+}
+
+async function runStepExecute({
+  member,
+  runId,
+  runStepId,
+  step,
+  inputSubmissions,
+  resourceSubmissions,
+  outputFormGroups,
+  conversationId,
+  spawningGroupData,
+  config,
+}: {
+  member: Member;
+  runId: string;
+  runStepId: string;
+  step: {
+    id: string;
+    stepOrder: number;
+    description: string;
+    prompt: string;
+    toolsAllowed: string[] | null;
+    parentStepId: string | null;
+    outputFormIds: string[];
+  };
+  inputSubmissions: DenormSubmission[];
+  resourceSubmissions: DenormSubmission[];
+  outputFormGroups: FormGroupHydrated[];
+  conversationId: string;
+  spawningGroupData?: {
+    groupName: string;
+    data: DenormSubmission["groups"][0];
+  } | null;
+  config: {
+    model: ReturnType<typeof createLanguageModel>;
+  };
+}): Promise<StepExecutionResult> {
+  const { model } = config;
+  try {
+    // Serialize input data and resources
+    // If multiple input submissions provided (parent was multi-step), serialize all of them
+    const serializedInput = inputSubmissions
+      .map(
+        (sub, idx) => `=== Input ${idx + 1} ===\n${serializeSubmission(sub)}`,
+      )
+      .join("\n\n");
+    const serializedResources = serializePipelineResources(resourceSubmissions);
+
+    // Build output schema
+    const workflowStepResponseSchema =
+      buildStepResponseSchema(outputFormGroups);
+    const strictSchema = zodSchema(workflowStepResponseSchema);
+    addAdditionalPropertiesToJsonSchema(strictSchema.jsonSchema as JSONSchema7);
+
+    // Build instructions (system context only)
+    const instructionsParts = [
+      "You are executing one step in a deterministic data workflow.",
+      "Analyze the input data and produce the required output.",
+      "",
+      `Step: ${step.stepOrder}`,
+      `Description: ${step.description}`,
+      "",
+      "IMPORTANT: For each primitive field (string, number, boolean), output:",
+      '  "fieldName": { "value": <extracted_value>, "refId": "<element_id>" }',
+      "",
+      "When a value comes from multiple elements in the input data or pipeline resources, list all element IDs separated by commas in the refId field.",
+      "EVERY extracted value MUST include a refId pointing to the source element(s) from the input data or pipeline resources.",
+    ];
+
+    if (spawningGroupData) {
+      instructionsParts.push(
+        "",
+        "NOTE: This is a spawned instance processing a specific row/entry from a multi-group table.",
+        `The TARGET ROW section below contains the specific ${spawningGroupData.groupName} entry this instance is processing.`,
+        "Use data from the TARGET ROW as the primary context, and reference INPUT DATA for additional context as needed.",
+      );
+    }
+
+    instructionsParts.push("", "===== STEP INSTRUCTIONS =====", step.prompt);
+
+    const instructions = instructionsParts.join("\n");
+
+    console.log("[runStepExecute] Building agent for step:", {
+      runId,
+      runStepId,
+      stepOrder: step.stepOrder,
+      stepDescription: step.description,
+      promptLength: step.prompt.length,
+      promptPreview: step.prompt.substring(0, 200),
+      hasSpawningGroup: !!spawningGroupData,
+    });
+
+    // Build messages with input data and resources
+    const messages = [
+      {
+        role: "user" as const,
+        content: `===== INPUT DATA =====\n${serializedInput}`,
+      },
+    ];
+
+    if (spawningGroupData) {
+      // Serialize the spawning group data
+      const spawningGroupHtml: string[] = [];
+      spawningGroupHtml.push(`<h3>${spawningGroupData.groupName}</h3>`);
+      spawningGroupHtml.push("<dl>");
+
+      for (const item of spawningGroupData.data.items) {
+        spawningGroupHtml.push(`<dt>${item.name}</dt>`);
+        spawningGroupHtml.push(
+          `<dd id="${item.id}">${String(item.value ?? "")}</dd>`,
+        );
+      }
+
+      for (const derived of spawningGroupData.data.derived) {
+        spawningGroupHtml.push(`<dt>${derived.name} (derived)</dt>`);
+        spawningGroupHtml.push(
+          `<dd id="${derived.id}">${String(derived.value ?? "")}</dd>`,
+        );
+      }
+
+      spawningGroupHtml.push("</dl>");
+
+      messages.push({
+        role: "user" as const,
+        content: `===== TARGET ROW (Spawning Group) =====\n${spawningGroupHtml.join("\n")}`,
+      });
+    }
+
+    if (resourceSubmissions.length > 0) {
+      messages.push({
+        role: "user" as const,
+        content: `===== PIPELINE RESOURCES =====\n${serializedResources}`,
+      });
+    }
+
+    messages.push({
+      role: "user" as const,
+      content: "Execute the step as instructed.",
+    });
+
+    // Load pipeline-specific tools (not agent tools)
+    const tools = {
+      createArtifact: createPipelineArtifact({
+        member,
+        runId,
+        runStepId,
+      }),
+    };
+
+    console.log("[runStepExecute] Tools available:", {
+      runId,
+      runStepId,
+      toolNames: Object.keys(tools),
+      toolCount: Object.keys(tools).length,
+    });
+
+    console.log("[runStepExecute] Full instructions being sent to agent:", {
+      runId,
+      runStepId,
+      instructions,
+    });
+
+    // Create agent with structured output
+    const agent = new ToolLoopAgent({
+      model,
+      instructions,
+      tools,
+      stopWhen: stepCountIs(10),
+      output: Output.object({
+        schema: jsonSchema(strictSchema.jsonSchema),
+      }),
+    });
+
+    // Execute agent
+    console.log("[runStepExecute] Starting agent execution...", {
+      runId,
+      runStepId,
+    });
+
+    const result = await agent.generate({ messages });
+
+    console.log("[runStepExecute] Agent execution completed:", {
+      runId,
+      runStepId,
+      hasOutput: !!result.output,
+      stepCount: result.steps?.length ?? 0,
+    });
+
+    const parsedOutput = workflowStepResponseSchema.parse(result.output);
+
+    return {
+      status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      reasoning: parsedOutput.reasoning ?? undefined,
+      output: { data: parsedOutput.data },
+    };
+  } catch (error) {
+    console.error("[runStepExecute] Step execution failed:", {
+      runId,
+      runStepId,
+      error: error instanceof Error ? error.message : "Unknown error",
+      stack: error instanceof Error ? error.stack : undefined,
+    });
+
+    return {
+      status: PIPELINE_RUN_STATUS.FAILED,
+      error: error instanceof Error ? error.message : "Step execution failed",
+      output: { data: null },
+    };
+  }
+}
+
+// ============================================================================
+// Replay Functions
+// ============================================================================
+
+export async function replayPipelineRun({
+  member,
+  runId,
+  config,
+}: {
+  member: Member;
+  runId: string;
+  config: {
+    model: ReturnType<typeof createLanguageModel>;
+  };
+}): Promise<DalResult<unknown>> {
+  // TODO: Implement replay functionality
+  // Steps needed:
+  // 1. Get run timeline with getRunTimeline
+  // 2. Extract input submission IDs from resources
+  // 3. Call executePipeline with same inputs
+  return { ok: false, error: "not-found" };
+}
+
+export async function replayPipelineRunStep({
+  member,
+  runStepId,
+  config,
+}: {
+  member: Member;
+  runStepId: string;
+  config: {
+    model: ReturnType<typeof createLanguageModel>;
+  };
+}): Promise<DalResult<unknown>> {
+  // TODO: Implement step replay functionality
+  // Steps needed:
+  // 1. Get step details with getPipelineRunStepWithDetails
+  // 2. Find the corresponding pipeline step
+  // 3. Determine input (parent step output or run input)
+  // 4. Either make executeSingleStep public or create wrapper
+  return { ok: false, error: "not-found" };
+}