sea-dev 1.0.0

package/apps/workers/src/workflows/pipeline-steps.ts

@@ -0,0 +1,1393 @@
+import { getWorkerConfig } from "@/lib/config";
+import { createLanguageModel } from "@sea/ai/models";
+import { HatchetClientMgr } from "@sea/clients/hatchet";
+import { createPipelineArtifact } from "@sea/core";
+import { addAdditionalPropertiesToJsonSchema } from "@sea/core/data-extraction/structuring/custom";
+import { getForm } from "@sea/dal/form";
+import {
+  createPipelineRunStep,
+  getGroupForStepRun,
+  getParentStepRuns,
+  getParentStepRunsWithDependencies,
+  getPipelineStepById,
+  getStepDocuments,
+  getStepIntegrationConfig,
+  getSubmissionGroups,
+  logRequestCall,
+  resolveStepBlockersOnCompletion,
+  storePipelineOutputData,
+  updatePipelineRunStepStatus,
+} from "@sea/dal/pipeline";
+import type { DenormGroup, DenormSubmission, DenormTable } from "@sea/dal/submission";
+import {
+  createSubmissionWithConversation,
+  getDenormSubmission,
+  getDenormSubmissions,
+  syncSubmissionToForm,
+} from "@sea/dal/submission";
+import { db, PIPELINE_RUN_STEP_STATUS, schema } from "@sea/db";
+import type { FormGroupHydrated, FormWithGroups, Member } from "@sea/db/types";
+import { invariant } from "@sea/util/invariant";
+import type { LanguageModel } from "ai";
+import { jsonSchema, Output, stepCountIs, ToolLoopAgent, zodSchema, type JSONSchema7 } from "ai";
+import { and, eq, inArray } from "drizzle-orm";
+import pLimit from "p-limit";
+import * as z from "zod/v4";
+import { extractDataUnified } from "./extraction";
+
+const config = getWorkerConfig();
+const hatchet = new HatchetClientMgr(config.hatchet).getHatchet();
+
+// ============================================================================
+// Input/Output Types
+// ============================================================================
+
+// Use type instead of interface to allow index signature
+export type StepExecutionInput = {
+  memberId: string;
+  runId: string;
+  runStepId: string;
+  stepId: string;
+  [key: string]: string; // Index signature for JsonObject compatibility
+};
+
+export type StepExecutionOutput = {
+  status: string;
+  outputSubmissionId?: string;
+  error?: string;
+  [key: string]: string | undefined; // Index signature for JsonObject compatibility
+};
+
+// ============================================================================
+// Template Step Workflow - Spawns Multiple Instances
+// ============================================================================
+
+export type TemplateStepInput = StepExecutionInput & {
+  stepRole: "extraction" | "integration" | "analysis" | "manual";
+};
+
+export const templateStepWorkflow = hatchet.workflow<TemplateStepInput>({
+  name: "template-step",
+});
+
+// ============================================================================
+// Helper Functions
+// ============================================================================
+
+function serializeGroup(group: DenormGroup): string {
+  const htmlParts: string[] = [];
+  htmlParts.push(`<h2>${group.name}</h2>`);
+  htmlParts.push("<dl>");
+
+  for (const item of group.items) {
+    htmlParts.push(`<dt>${item.name}</dt>`);
+    htmlParts.push(`<dd id="${item.id}">${String(item.value ?? "")}</dd>`);
+  }
+  for (const derived of group.derived) {
+    htmlParts.push(`<dt>${derived.name} (derived)</dt>`);
+    htmlParts.push(`<dd id="${derived.id}">${String(derived.value ?? "")}</dd>`);
+  }
+  htmlParts.push("</dl>");
+
+  return htmlParts.join("\n");
+}
+
+function serializeTable(table: DenormTable): string {
+  const htmlParts: string[] = [];
+
+  htmlParts.push(`<h2>${table.name}</h2>`);
+  htmlParts.push("<table>");
+
+  // Table header
+  htmlParts.push("<thead><tr>");
+  for (const col of table.columns) {
+    htmlParts.push(`<th>${col}</th>`);
+  }
+  htmlParts.push("</tr></thead>");
+
+  // Table body
+  htmlParts.push("<tbody>");
+  for (const row of table.rows) {
+    htmlParts.push("<tr>");
+
+    // Regular items
+    for (const item of row.items) {
+      htmlParts.push(`<td id="${item.id}">${String(item.value ?? "")}</td>`);
+    }
+
+    // Derived items
+    for (const derived of row.derived) {
+      htmlParts.push(`<td id="${derived.id}">${String(derived.value ?? "")}</td>`);
+    }
+
+    htmlParts.push("</tr>");
+  }
+  htmlParts.push("</tbody>");
+  htmlParts.push("</table>");
+
+  return htmlParts.join("\n");
+}
+
+/**
+ * Serialize submission to HTML for LLM consumption
+ */
+function serializeSubmission(submission: DenormSubmission): string {
+  const htmlParts: string[] = [];
+
+  // Add submission header
+  htmlParts.push(`<section>`);
+  htmlParts.push(`<h1>${submission.formName}</h1>`);
+  if (submission.name) {
+    htmlParts.push(`<p><strong>Submission:</strong> ${submission.name}</p>`);
+  }
+
+  // Serialize single groups (definition lists)
+  for (const group of submission.groups) {
+    htmlParts.push(serializeGroup(group));
+  }
+
+  // Serialize tables (multi groups)
+  for (const table of submission.tables) {
+    htmlParts.push(serializeTable(table));
+  }
+  htmlParts.push("</section>");
+
+  return htmlParts.join("\n");
+}
+
+/**
+ * Build extraction schema for a group with reasoning field for pipeline analysis
+ */
+function buildAnalysisSchemaFromGroup(
+  group: FormGroupHydrated,
+): z.ZodObject<Record<string, z.ZodType>> {
+  const groupFields: Record<string, z.ZodType> = {};
+
+  for (const field of group.fields) {
+    switch (field.type) {
+      case "string":
+      case "string-long":
+        groupFields[field.name] = z
+          .object({
+            value: z.string(),
+            refId: z.string(),
+            reasoning: z
+              .string()
+              .describe("Explanation of how this value was derived from the input data"),
+          })
+          .nullable()
+          .describe(field.hint);
+        break;
+      case "number":
+      case "money":
+        groupFields[field.name] = z
+          .object({
+            value: z.number(),
+            refId: z.string(),
+            reasoning: z
+              .string()
+              .describe("Explanation of how this value was derived from the input data"),
+          })
+          .nullable()
+          .describe(field.hint);
+        break;
+      case "boolean":
+        groupFields[field.name] = z
+          .object({
+            value: z.boolean(),
+            refId: z.string(),
+            reasoning: z
+              .string()
+              .describe("Explanation of how this value was derived from the input data"),
+          })
+          .nullable()
+          .describe(field.hint);
+        break;
+      case "date":
+        groupFields[field.name] = z
+          .object({
+            value: z.iso.date(),
+            refId: z.string(),
+            reasoning: z
+              .string()
+              .describe("Explanation of how this value was derived from the input data"),
+          })
+          .nullable()
+          .describe(field.hint);
+        break;
+      case "email":
+        groupFields[field.name] = z
+          .object({
+            value: z.email(),
+            refId: z.string(),
+            reasoning: z
+              .string()
+              .describe("Explanation of how this value was derived from the input data"),
+          })
+          .nullable()
+          .describe(field.hint);
+        break;
+      case "url":
+        groupFields[field.name] = z
+          .object({
+            value: z.string().describe("Valid URL"),
+            refId: z.string(),
+            reasoning: z
+              .string()
+              .describe("Explanation of how this value was derived from the input data"),
+          })
+          .nullable()
+          .describe(field.hint);
+        break;
+      case "enum":
+        if (field.options && field.options.length > 0) {
+          groupFields[field.name] = z
+            .object({
+              value: z.enum(field.options as [string, ...string[]]),
+              refId: z.string(),
+              reasoning: z
+                .string()
+                .describe("Explanation of how this value was derived from the input data"),
+            })
+            .nullable()
+            .describe(`${field.hint}. Must be one of: ${field.options.join(", ")}`);
+        } else {
+          groupFields[field.name] = z
+            .object({
+              value: z.string(),
+              refId: z.string(),
+              reasoning: z
+                .string()
+                .describe("Explanation of how this value was derived from the input data"),
+            })
+            .nullable()
+            .describe(field.hint);
+        }
+        break;
+      default:
+        groupFields[field.name] = z
+          .object({
+            value: z.unknown(),
+            refId: z.string(),
+            reasoning: z
+              .string()
+              .describe("Explanation of how this value was derived from the input data"),
+          })
+          .nullable()
+          .describe(field.hint);
+        break;
+    }
+  }
+
+  return group.multi
+    ? z.object({
+        [group.name]: z.array(z.object(groupFields)),
+      })
+    : z.object({
+        [group.name]: z.object(groupFields),
+      });
+}
+
+/**
+ * Build response schema from output form groups for pipeline analysis
+ */
+function buildStepResponseSchema(outputFormGroups: FormGroupHydrated[]) {
+  const extractionSchema = outputFormGroups.reduce((acc, group) => {
+    const groupSchema = buildAnalysisSchemaFromGroup(group);
+    return z.object({
+      ...acc.shape,
+      ...groupSchema.shape,
+    });
+  }, z.object({}));
+
+  return z.object({
+    reasoning: z.string().nullable(),
+    data: extractionSchema.nullable(),
+  });
+}
+
+/**
+ * Build HTTP request from integration config template
+ */
+async function buildHttpRequest({
+  config: reqConfig,
+  parentData,
+}: {
+  config: {
+    method: "GET" | "POST" | "PUT" | "DELETE";
+    urlTemplate: string;
+    headers: Record<string, string>;
+    bodyTemplate?: string;
+  };
+  parentData: DenormSubmission[];
+}): Promise<{
+  method: string;
+  url: string;
+  headers: Record<string, string>;
+  body?: string;
+}> {
+  // Build variable map from parent submissions
+  const variables = new Map<string, string>();
+
+  for (const submission of parentData) {
+    // Add form-level variables
+    variables.set(`form.name`, submission.formName);
+
+    // Add field-level variables from groups
+    for (const group of submission.groups) {
+      for (const item of group.items) {
+        const key = `${group.name}.${item.name}`;
+        const value = String(item.value ?? "");
+        variables.set(key, value);
+      }
+
+      for (const derived of group.derived) {
+        const key = `${group.name}.${derived.name}`;
+        const value = String(derived.value ?? "");
+        variables.set(key, value);
+      }
+    }
+
+    // Add table row variables (use first row)
+    for (const table of submission.tables) {
+      const firstRow = table.rows[0];
+      if (firstRow) {
+        for (const item of firstRow.items) {
+          const key = `${table.name}.${item.name}`;
+          const value = String(item.value ?? "");
+          variables.set(key, value);
+        }
+      }
+    }
+  }
+
+  // Substitute variables in templates
+  const substituteVariables = (template: string): string => {
+    return template.replace(/\{\{([^}]+)\}\}/g, (match, varName) => {
+      const trimmedName = varName.trim();
+      const value = variables.get(trimmedName);
+
+      if (value !== undefined) {
+        return encodeURIComponent(value);
+      }
+
+      console.warn(`[buildHttpRequest] Variable not found: ${trimmedName}`);
+      return match;
+    });
+  };
+
+  const url = substituteVariables(reqConfig.urlTemplate);
+
+  const headers: Record<string, string> = {};
+  for (const [key, value] of Object.entries(reqConfig.headers)) {
+    headers[key] = substituteVariables(value);
+  }
+
+  const body = reqConfig.bodyTemplate ? substituteVariables(reqConfig.bodyTemplate) : undefined;
+
+  return {
+    method: reqConfig.method,
+    url,
+    headers,
+    body,
+  };
+}
+
+// ============================================================================
+// LLM Analysis Helper
+// ============================================================================
+
+async function runAnalysisWithLLM({
+  member,
+  runId,
+  runStepId,
+  step,
+  inputSubmissions,
+  outputForm,
+  spawningGroup,
+  model,
+}: {
+  member: Member;
+  runId: string;
+  runStepId: string;
+  step: {
+    id: string;
+    description: string;
+    prompt: string;
+    toolsAllowed: string[] | null;
+  };
+  inputSubmissions: DenormSubmission[];
+  outputForm: FormWithGroups;
+  spawningGroup?: DenormGroup | null;
+  model: LanguageModel;
+}) {
+  try {
+    // Build output schema
+    const workflowStepResponseSchema = buildStepResponseSchema(outputForm.groups);
+    const strictSchema = zodSchema(workflowStepResponseSchema);
+    addAdditionalPropertiesToJsonSchema(strictSchema.jsonSchema as JSONSchema7);
+
+    // Build instructions
+    const instructionsParts = [
+      `
+<role>You are a precise data transformation specialist executing one step in a data workflow.</role>
+
+<task>
+- Your broad task is ${step.description}.
+- The user will specify the exact instructions on how to perform the data transformations for this step in the <step_instructions> section.
+</task>
+
+<input_format>
+- HTML-formatted submission data with multiple sections
+- Each field has an id attribute for reference (e.g., id="field-123")
+- Sections may include definition lists (<dl>) and tables (<table>)
+- Multiple input submissions may be provided
+</input_format>
+
+<transformation_rules>
+1. <completeness>Extract ALL fields defined in the output schema</completeness>
+2. <output_format>For each primitive field (string, number, boolean), output:
+   "fieldName": { "value": <transformed_value>, "refId": "<element_id>", "reasoning": "<explanation>" }</output_format>
+3. <multiple_references>When a value is derived from multiple input elements, list all element IDs separated by commas in the refId field</multiple_references>
+4. <reasoning_requirement>
+   - The reasoning field MUST explain HOW the value was derived from the input data.
+   - When referring to specific elements from input data, refer to them by the corresponding header trace under which they reside
+   </reasoning_requirement>
+5. <reasoning_examples>
+   - "Calculated by summing TransactionAmount fields from rows 1-5"
+   - "Extracted from BankName field in section 'Bank Statement'"
+   - "Average of Balance values across all input data"
+   - "Converted from USD to GBP using exchange rate in row 3"</reasoning_examples>
+6. <source_fidelity>ONLY use values present in or reasonably inferred from the input data - NO external knowledge</source_fidelity>
+7. <reference_requirement>EVERY extracted value MUST include a refId pointing to the source element(s) and a reasoning explaining the transformation</reference_requirement>
+</transformation_rules>
+
+<output_requirements>
+- Return a JSON object conforming to the provided schema
+- Validate output matches schema structure exactly
+- All required fields must be present
+- All values must include refId and reasoning
+</output_requirements>`,
+    ];
+
+    instructionsParts.push("<step_instructions>", step.prompt, "</step_instructions>");
+    const instructions = instructionsParts.join("\n");
+
+    // Build messages
+    const serializedInput = inputSubmissions.map((sub) => serializeSubmission(sub));
+    const messages = [
+      {
+        role: "user" as const,
+        content: `===== INPUT DATA =====\n${serializedInput}`,
+      },
+    ];
+    if (spawningGroup) {
+      messages.push({
+        role: "user" as const,
+        content: ["<main_input>", serializeGroup(spawningGroup), "</main_input>"].join("\n"),
+      });
+    }
+
+    // Load pipeline-specific tools
+    const tools = {
+      createArtifact: createPipelineArtifact({
+        member,
+        runId,
+        runStepId,
+      }),
+    };
+
+    // Create agent with structured output
+    const agent = new ToolLoopAgent({
+      model,
+      instructions,
+      tools,
+      stopWhen: stepCountIs(10),
+      output: Output.object({
+        schema: jsonSchema(strictSchema.jsonSchema),
+      }),
+    });
+
+    const result = await agent.generate({ messages });
+    const parsedOutput = workflowStepResponseSchema.parse(result.output);
+
+    return {
+      status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      data: parsedOutput.data,
+    };
+  } catch (error) {
+    return {
+      status: PIPELINE_RUN_STEP_STATUS.FAILED,
+      data: null,
+      error: error instanceof Error ? error.message : "Analysis failed",
+    };
+  }
+}
+
+// ============================================================================
+// Extraction Step Task
+// ============================================================================
+
+export const executeExtractionStepTask = hatchet.task<StepExecutionInput, StepExecutionOutput>({
+  name: "execute-extraction-step",
+  timeout: "30m", // Extraction can take a while
+  fn: async (input) => {
+    console.log("[executeExtractionStepTask] Starting...");
+    const { memberId, runStepId, stepId } = input;
+
+    try {
+      // Get member
+      const member = await db.query.member.findFirst({
+        where: eq(schema.member.id, memberId),
+      });
+      invariant(member, "Member not found");
+
+      // Get run step
+      const runStep = await db.query.pipelineRunStep.findFirst({
+        where: eq(schema.pipelineRunStep.id, runStepId),
+      });
+      invariant(runStep, "Run step not found");
+
+      // Get step definition
+      const stepResult = await getPipelineStepById({ member, stepId });
+      if (!stepResult.ok) {
+        throw new Error(`Step ${stepId} not found`);
+      }
+      const step = stepResult.data;
+
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+      });
+
+      // Get documents linked to this step run
+      const documentsResult = await getStepDocuments({
+        member,
+        runStepId,
+      });
+
+      if (!documentsResult.ok) {
+        throw new Error("Failed to load step documents");
+      }
+
+      if (documentsResult.data.length === 0) {
+        throw new Error("No documents linked to extraction step");
+      }
+
+      // Validate we have output forms
+      if (step.outputForms.length === 0) {
+        throw new Error("Extraction step missing output form");
+      }
+
+      // Get pipeline run to access dealId
+      const pipelineRun = await db.query.pipelineRun.findFirst({
+        where: eq(schema.pipelineRun.id, runStep.runId),
+      });
+      if (!pipelineRun) {
+        throw new Error("Pipeline run not found");
+      }
+
+      // Process each output form
+      const outputSubmissionIds: string[] = [];
+
+      // Fetch all form names upfront
+      const formIds = step.outputForms.map((f) => f.formId);
+      const forms = await db.query.form.findMany({
+        where: (fields, { inArray }) => inArray(fields.id, formIds),
+      });
+      const formIdToName = new Map(forms.map((f) => [f.id, f.name]));
+
+      for (const [formIndex, outputFormConfig] of step.outputForms.entries()) {
+        const outputFormId = outputFormConfig.formId;
+        const formName = formIdToName.get(outputFormId);
+
+        console.log(
+          `[executeExtractionStepTask] Processing output form ${formIndex + 1}/${step.outputForms.length}: ${outputFormId} (${formName})`,
+        );
+
+        // Build submission name
+        let submissionName: string;
+        if (runStep.spawnType === "instance") {
+          if (step.outputForms.length > 1) {
+            submissionName = `${step.name} - Instance ${runStep.stepRunIndex! + 1} - ${formName ?? `Form ${formIndex + 1}`}`;
+          } else {
+            submissionName = `${step.name} - Instance ${runStep.stepRunIndex! + 1}`;
+          }
+        } else {
+          if (step.outputForms.length > 1) {
+            submissionName = `${step.name} - ${formName ?? `Form ${formIndex + 1}`}`;
+          } else {
+            submissionName = `${step.name} output`;
+          }
+        }
+
+        // Create output submission for extracted data
+        const outputSubmission = await createSubmissionWithConversation({
+          member,
+          data: {
+            formId: outputFormId,
+            dealId: pipelineRun.dealId,
+            name: submissionName,
+            status: "draft",
+            type: "extraction",
+          },
+        });
+
+        await syncSubmissionToForm({
+          member,
+          submissionId: outputSubmission.id,
+        });
+
+        // For each document, create documentExtraction and trigger extraction workflow directly
+        const extractionPromises = documentsResult.data.map(async (document) => {
+          // Create documentExtraction record
+          const [documentExtraction] = await db
+            .insert(schema.documentExtraction)
+            .values({
+              documentId: document.id,
+              submissionId: outputSubmission.id,
+              status: "pending",
+              extractionModel: "smart",
+              extractionTrigger: "api",
+            })
+            .returning({ id: schema.documentExtraction.id });
+
+          invariant(
+            documentExtraction,
+            `Failed to create documentExtraction for document ${document.id}`,
+          );
+
+          console.log(
+            `[executeExtractionStepTask] Calling extraction workflow for form ${formIndex + 1}, document: ${document.name} (extractionId: ${documentExtraction.id})`,
+          );
+
+          // Call extraction workflow directly (no event, no polling!)
+          await extractDataUnified.run({
+            documentExtractionId: documentExtraction.id,
+            options: {
+              extractionModel: "general",
+              reasoningEffort: "medium",
+            },
+            userPrompt: step.prompt || undefined,
+          });
+
+          console.log(
+            `[executeExtractionStepTask] Extraction completed for form ${formIndex + 1}, document: ${document.name}`,
+          );
+
+          return documentExtraction.id;
+        });
+
+        // Wait for all extractions to complete for this form
+        await Promise.all(extractionPromises);
+
+        // Link output submission to run step
+        await db.insert(schema.pipelineRunStepOutputSubmission).values({
+          runStepId,
+          submissionId: outputSubmission.id,
+        });
+
+        outputSubmissionIds.push(outputSubmission.id);
+      }
+
+      // Mark step as completed
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      });
+
+      // Resolve blockers now that extraction is complete
+      await resolveStepBlockersOnCompletion({
+        member,
+        runStepId,
+      });
+
+      console.log(
+        `[executeExtractionStepTask] Completed extraction step with ${documentsResult.data.length} document(s) across ${step.outputForms.length} form(s), output submissions: ${outputSubmissionIds.join(", ")}`,
+      );
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+        outputSubmissionId: outputSubmissionIds[0], // Keep first for backwards compatibility
+      };
+    } catch (error) {
+      console.error("[executeExtractionStepTask] Error:", error);
+      await updatePipelineRunStepStatus({
+        member: { id: memberId } as Member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Unknown error",
+      });
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Extraction step failed",
+      };
+    }
+  },
+});
+
+// ============================================================================
+// Integration Step Task
+// ============================================================================
+
+export const executeIntegrationStepTask = hatchet.task<StepExecutionInput, StepExecutionOutput>({
+  name: "execute-integration-step",
+  timeout: "10m",
+  fn: async (input) => {
+    const { memberId, runStepId, stepId } = input;
+
+    try {
+      const member = await db.query.member.findFirst({
+        where: eq(schema.member.id, memberId),
+      });
+      invariant(member, "Member not found");
+
+      const runStep = await db.query.pipelineRunStep.findFirst({
+        where: eq(schema.pipelineRunStep.id, runStepId),
+      });
+      invariant(runStep, "Run step not found");
+
+      const stepResult = await getPipelineStepById({ member, stepId });
+      if (!stepResult.ok) {
+        throw new Error(`Step ${stepId} not found`);
+      }
+      const step = stepResult.data;
+
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+      });
+
+      // Get integration config
+      const integrationConfigResult = await getStepIntegrationConfig({
+        member,
+        stepId,
+      });
+
+      if (!integrationConfigResult.ok || !integrationConfigResult.data) {
+        throw new Error("Integration step missing config");
+      }
+
+      const integrationConfig = integrationConfigResult.data;
+
+      // Get parent step outputs
+      const parentStepRunsResult = await getParentStepRuns({
+        member,
+        runStepId,
+      });
+
+      if (!parentStepRunsResult.ok) {
+        throw new Error("Failed to load parent step outputs");
+      }
+
+      const parentStepRuns = parentStepRunsResult.data;
+
+      // Load output submissions for each parent step run
+      const parentSubmissions = await Promise.all(
+        parentStepRuns.map(async (p) => {
+          const outputMapping = await db.query.pipelineRunStepOutputSubmission.findFirst({
+            where: eq(schema.pipelineRunStepOutputSubmission.runStepId, p.id),
+          });
+          if (!outputMapping) return null;
+          return getDenormSubmission({
+            member,
+            submissionId: outputMapping.submissionId,
+            withCitations: true,
+            withTransformations: true,
+          });
+        }),
+      );
+
+      const validParentSubmissions = parentSubmissions.filter(
+        (s): s is NonNullable<typeof s> => s !== null,
+      );
+
+      // Build request from template
+      const request = await buildHttpRequest({
+        config: {
+          method: integrationConfig.method as "GET" | "POST" | "PUT" | "DELETE",
+          urlTemplate: integrationConfig.urlTemplate,
+          headers: integrationConfig.headers,
+          bodyTemplate: integrationConfig.bodyTemplate ?? undefined,
+        },
+        parentData: validParentSubmissions,
+      });
+
+      // Make API call
+      const startTime = Date.now();
+      let statusCode: number | undefined;
+      let responseBody: unknown;
+      let error: string | undefined;
+
+      try {
+        const response = await fetch(request.url, {
+          method: request.method,
+          headers: request.headers,
+          body: request.body,
+        });
+
+        statusCode = response.status;
+        responseBody = await response.json();
+      } catch (err) {
+        error = err instanceof Error ? err.message : "Request failed";
+      }
+
+      const durationMs = Date.now() - startTime;
+
+      // Log request call
+      await logRequestCall({
+        member,
+        runStepId,
+        method: request.method,
+        url: request.url,
+        requestHeaders: request.headers,
+        requestBody: request.body,
+        statusCode,
+        responseBody,
+        durationMs,
+        error,
+      });
+
+      if (error || !statusCode || statusCode >= 400) {
+        throw new Error(error || `HTTP ${statusCode}`);
+      }
+
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      });
+
+      // Resolve blockers now that integration is complete
+      await resolveStepBlockersOnCompletion({
+        member,
+        runStepId,
+      });
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      };
+    } catch (error) {
+      await updatePipelineRunStepStatus({
+        member: { id: memberId } as Member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Integration step failed",
+      });
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Integration step failed",
+      };
+    }
+  },
+});
+
+// ============================================================================
+// Analysis Step Task
+// ============================================================================
+
+export const executeAnalysisStepTask = hatchet.task<StepExecutionInput, StepExecutionOutput>({
+  name: "execute-analysis-step",
+  timeout: "20m", // Analysis with LLM can take time
+  fn: async (input) => {
+    const { memberId, runStepId } = input;
+
+    try {
+      const member = await db.query.member.findFirst({
+        where: eq(schema.member.id, memberId),
+      });
+      invariant(member, "Member not found");
+
+      const runStep = await db.query.pipelineRunStep.findFirst({
+        with: { step: { with: { outputForms: true } }, run: true },
+        where: eq(schema.pipelineRunStep.id, runStepId),
+      });
+      invariant(runStep, "Run step not found");
+
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+      });
+
+      // Get parent step runs with dependency configurations
+      const parentStepRunsResult = await getParentStepRunsWithDependencies({
+        member,
+        runStepId,
+      });
+
+      if (!parentStepRunsResult.ok) {
+        throw new Error("Failed to load parent step runs");
+      }
+      const parentStepRuns = parentStepRunsResult.data;
+
+      // Load output submissions for each parent step run
+      const parentSubmissionIds = (
+        await db
+          .select()
+          .from(schema.pipelineRunStepOutputSubmission)
+          .where(
+            inArray(
+              schema.pipelineRunStepOutputSubmission.runStepId,
+              parentStepRuns.map((p) => p.id),
+            ),
+          )
+      ).map((row) => row.submissionId);
+      const parentSubmissions = await getDenormSubmissions({
+        member,
+        submissionIds: parentSubmissionIds,
+        withCitations: true,
+        withTransformations: true,
+      });
+
+      // Check if this is a multi step - if so filter input submissions accordingly
+      const multiGroupResult = await getGroupForStepRun({
+        member,
+        runStepId,
+      });
+      const spawningGroup = multiGroupResult.ok ? multiGroupResult.data : null;
+      const inputSubmissions = parentSubmissions.filter(({ tables }) => {
+        const hasSpawningGroupTable = tables.some(
+          ({ formGroupId }) => formGroupId === spawningGroup?.formGroupId,
+        );
+        return !hasSpawningGroupTable;
+      });
+
+      // Create model instance
+      const model = createLanguageModel({
+        gatewayApiKey: config.vercel.aiGatewayApiKey,
+        modelId: "openai/gpt-5.2",
+      });
+
+      // Create output submission for each output form
+      for (const { formId } of runStep.step.outputForms) {
+        const outputSubmission = await createSubmissionWithConversation({
+          member,
+          data: {
+            formId,
+            dealId: runStep.run.dealId,
+            name:
+              runStep.spawnType === "instance"
+                ? `${runStep.step.name} - Instance ${runStep.stepRunIndex! + 1}`
+                : `${runStep.step.name} output`,
+            status: "draft",
+            type: "transformation",
+          },
+        });
+        await syncSubmissionToForm({
+          member,
+          submissionId: outputSubmission.id,
+        });
+
+        // Run LLM-based analysis
+        const outputForm = await getForm({ member, formId });
+        const analysisResult = await runAnalysisWithLLM({
+          member,
+          runId: runStep.runId,
+          runStepId,
+          step: runStep.step,
+          inputSubmissions,
+          outputForm,
+          spawningGroup,
+          model,
+        });
+
+        // Store output
+        if (analysisResult.data) {
+          const storeResult = await storePipelineOutputData({
+            member,
+            outputSubmissionId: outputSubmission.id,
+            outputData: analysisResult.data as Record<
+              string,
+              | Record<string, { value: string | number | null; refId: string } | null>
+              | Record<string, { value: string | number | null; refId: string } | null>[]
+              | null
+            >,
+          });
+
+          if (!storeResult.ok) {
+            throw new Error("Failed to store pipeline output data");
+          }
+        }
+
+        // Link output submission to run step
+        await db.insert(schema.pipelineRunStepOutputSubmission).values({
+          runStepId,
+          submissionId: outputSubmission.id,
+        });
+
+        await updatePipelineRunStepStatus({
+          member,
+          runStepId,
+          status: analysisResult.status,
+          error: analysisResult.error,
+        });
+
+        if (analysisResult.status === "failed") {
+          throw new Error(analysisResult.error || "Analysis step failed");
+        }
+      }
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      };
+    } catch (error) {
+      await updatePipelineRunStepStatus({
+        member: { id: memberId } as Member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Analysis step failed",
+      });
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Analysis step failed",
+      };
+    }
+  },
+});
+
+// ============================================================================
+// Manual Step Task
+// ============================================================================
+
+export const executeManualStepTask = hatchet.task<StepExecutionInput, StepExecutionOutput>({
+  name: "execute-manual-step",
+  timeout: "5m",
+  fn: async (input) => {
+    const { memberId, runStepId, stepId } = input;
+
+    try {
+      const member = await db.query.member.findFirst({
+        where: eq(schema.member.id, memberId),
+      });
+      invariant(member, "Member not found");
+      const runStep = await db.query.pipelineRunStep.findFirst({
+        where: eq(schema.pipelineRunStep.id, runStepId),
+      });
+      invariant(runStep, "Run step not found");
+
+      const stepResult = await getPipelineStepById({ member, stepId });
+      if (!stepResult.ok) {
+        throw new Error(`Step ${stepId} not found`);
+      }
+      const step = stepResult.data;
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+      });
+
+      // Check if manual output submission has been provided
+      const outputMapping = await db.query.pipelineRunStepOutputSubmission.findFirst({
+        where: eq(schema.pipelineRunStepOutputSubmission.runStepId, runStepId),
+      });
+
+      if (!outputMapping) {
+        // No output submission yet - create a blocker and mark as blocked
+        console.log(
+          `[executeManualStepTask] Step ${step.name} blocked - waiting for manual output submission`,
+        );
+
+        // Check if blocker already exists
+        const existingBlocker = await db.query.pipelineRunStepBlocker.findFirst({
+          where: and(
+            eq(schema.pipelineRunStepBlocker.runStepId, runStepId),
+            eq(schema.pipelineRunStepBlocker.blockerType, "missing_manual_input"),
+            eq(schema.pipelineRunStepBlocker.resolved, false),
+          ),
+        });
+
+        if (!existingBlocker) {
+          await db.insert(schema.pipelineRunStepBlocker).values({
+            runStepId,
+            blockerType: "missing_manual_input",
+            blockerDetails: {
+              type: "missing_manual_input",
+              stepName: step.name,
+              stepDescription: step.description,
+            },
+            resolved: false,
+          });
+        }
+
+        await updatePipelineRunStepStatus({
+          member,
+          runStepId,
+          status: PIPELINE_RUN_STEP_STATUS.BLOCKED,
+        });
+
+        return {
+          status: PIPELINE_RUN_STEP_STATUS.BLOCKED,
+        };
+      }
+
+      // Output submission exists - all dependencies satisfied, mark as completed
+      const result = await updatePipelineRunStepStatus({
+        member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      });
+
+      if (!result.ok) {
+        throw new Error("Failed to update pipeline run step status");
+      }
+
+      console.log(
+        `[executeManualStepTask] Step ${step.name} completed - manual output submission provided`,
+      );
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      };
+    } catch (error) {
+      await updatePipelineRunStepStatus({
+        member: { id: memberId } as Member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Manual step failed",
+      });
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Manual step failed",
+      };
+    }
+  },
+});
+
+// ============================================================================
+// Template Step Task - Dynamic Instance Spawning
+// ============================================================================
+export const executeTemplateStepTask = hatchet.task<StepExecutionInput, StepExecutionOutput>({
+  name: "execute-template-step",
+  timeout: "60m", // Templates can spawn many children
+  fn: async (input) => {
+    console.log("[executeTemplateStepTask] Starting template execution...");
+    const { memberId, runStepId, stepId, stepRole } = input;
+
+    try {
+      // Get member
+      const member = await db.query.member.findFirst({
+        where: eq(schema.member.id, memberId),
+      });
+      invariant(member, "Member not found");
+
+      // Get template run step
+      const runStep = await db.query.pipelineRunStep.findFirst({
+        where: eq(schema.pipelineRunStep.id, runStepId),
+      });
+      invariant(runStep, "Template run step not found");
+      invariant(runStep.spawnType === "template", "Run step must be template type");
+
+      // Get step definition
+      const stepResult = await getPipelineStepById({ member, stepId });
+      if (!stepResult.ok) {
+        throw new Error(`Step ${stepId} not found`);
+      }
+      const step = stepResult.data;
+
+      await updatePipelineRunStepStatus({
+        member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.RUNNING,
+      });
+
+      // Get parent step runs with dependency config
+      const parentStepRunsResult = await getParentStepRunsWithDependencies({
+        member,
+        runStepId,
+      });
+
+      if (!parentStepRunsResult.ok) {
+        throw new Error("Failed to load parent step runs");
+      }
+      const parentStepRuns = parentStepRunsResult.data;
+
+      // Find dependencies with multiFormGroupId (the spawning trigger)
+      const multiDeps = parentStepRuns.filter((p) => p.multiFormGroupId !== null);
+      if (!multiDeps?.length) return { status: PIPELINE_RUN_STEP_STATUS.COMPLETED };
+      if (multiDeps.length > 1)
+        throw new Error("Multiple multi-group parents found, not supported");
+
+      // Get the unique
+      const multiDep = multiDeps[0];
+      invariant(multiDep, "Multi dependency not found");
+
+      // Get parent's output submission
+      const parentOutputMapping = await db.query.pipelineRunStepOutputSubmission.findFirst({
+        where: eq(schema.pipelineRunStepOutputSubmission.runStepId, multiDep.id),
+      });
+      if (!parentOutputMapping) {
+        throw new Error("Parent step has no output submission");
+      }
+
+      // Get submission groups to spawn instances
+      const groupsResult = await getSubmissionGroups({
+        member,
+        submissionId: parentOutputMapping.submissionId,
+        formGroupId: multiDep.multiFormGroupId!,
+      });
+      if (!groupsResult.ok) {
+        throw new Error("Failed to load submission groups");
+      }
+
+      const groups = groupsResult.data;
+      console.log(
+        `[executeTemplateStepTask] Found ${groups.length} group(s) to spawn for step ${step.name}`,
+      );
+
+      if (groups.length === 0) {
+        console.log("[executeTemplateStepTask] No groups to process, marking template as complete");
+        await updatePipelineRunStepStatus({
+          member,
+          runStepId,
+          status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+        });
+        return { status: PIPELINE_RUN_STEP_STATUS.COMPLETED };
+      }
+
+      // Create instance runSteps for each group
+      const instanceRunSteps = [];
+
+      for (let i = 0; i < groups.length; i++) {
+        const group = groups[i];
+        invariant(group, `Group at index ${i} is undefined`);
+
+        console.log(
+          `[executeTemplateStepTask] Creating instance ${i + 1}/${groups.length} for group ${group.id}`,
+        );
+
+        const instanceResult = await createPipelineRunStep({
+          member,
+          runId: runStep.runId,
+          stepId,
+          spawnType: "instance",
+          submissionGroupId: group.id,
+          stepRunIndex: i,
+        });
+
+        if (!instanceResult.ok) {
+          throw new Error(`Failed to create instance runStep for group ${group.id}`);
+        }
+
+        instanceRunSteps.push(instanceResult.data);
+      }
+
+      console.log(
+        `[executeTemplateStepTask] Created ${instanceRunSteps.length} instance runStep(s), spawning workflows...`,
+      );
+
+      // Spawn child workflows with concurrency limit (max 10 parallel)
+      const limit = pLimit(10);
+      const childPromises = instanceRunSteps.map((instanceRunStep) =>
+        limit(async () => {
+          const instanceInput: StepExecutionInput = {
+            memberId,
+            runId: runStep.runId,
+            runStepId: instanceRunStep.id,
+            stepId,
+          };
+
+          console.log(
+            `[executeTemplateStepTask] Spawning instance workflow: ${stepRole}-step (runStepId: ${instanceRunStep.id})`,
+          );
+
+          // Execute based on step role
+          switch (stepRole) {
+            case "extraction": {
+              const result = await executeExtractionStepTask.run(instanceInput);
+              return result;
+            }
+            case "integration": {
+              const result = await executeIntegrationStepTask.run(instanceInput);
+              return result;
+            }
+            case "analysis": {
+              const result = await executeAnalysisStepTask.run(instanceInput);
+              return result;
+            }
+            case "manual": {
+              const result = await executeManualStepTask.run(instanceInput);
+              return result;
+            }
+            default:
+              throw new Error(`Unknown step role: ${stepRole}`);
+          }
+        }),
+      );
+
+      // Wait for all instances to complete (durable wait)
+      console.log(
+        `[executeTemplateStepTask] Waiting for ${childPromises.length} instance(s) to complete with max 10 concurrent executions...`,
+      );
+      const instanceResults = await Promise.all(childPromises);
+
+      // Check instance results
+      const failedInstances = instanceResults.filter(
+        (r) => r && r.status === PIPELINE_RUN_STEP_STATUS.FAILED,
+      );
+      const completedInstances = instanceResults.filter(
+        (r) => r && r.status === PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      );
+
+      console.log(
+        `[executeTemplateStepTask] Instance results: ${completedInstances.length} completed, ${failedInstances.length} failed`,
+      );
+
+      // Resolve child step blockers now that template is complete
+      await resolveStepBlockersOnCompletion({
+        member,
+        runStepId,
+      });
+
+      // Mark template as complete if all instances succeeded
+      if (failedInstances.length === 0) {
+        await updatePipelineRunStepStatus({
+          member,
+          runStepId,
+          status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+        });
+
+        console.log(
+          `[executeTemplateStepTask] Template step completed successfully with ${instanceResults.length} instance(s)`,
+        );
+
+        return {
+          status: PIPELINE_RUN_STEP_STATUS.COMPLETED,
+        };
+      } else {
+        // Some instances failed
+        await updatePipelineRunStepStatus({
+          member,
+          runStepId,
+          status: PIPELINE_RUN_STEP_STATUS.FAILED,
+          error: `${failedInstances.length} instance(s) failed`,
+        });
+
+        return {
+          status: PIPELINE_RUN_STEP_STATUS.FAILED,
+          error: `${failedInstances.length} of ${instanceResults.length} instance(s) failed`,
+        };
+      }
+    } catch (error) {
+      console.error("[executeTemplateStepTask] Error:", error);
+      await updatePipelineRunStepStatus({
+        member: { id: memberId } as Member,
+        runStepId,
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Template step failed",
+      });
+
+      return {
+        status: PIPELINE_RUN_STEP_STATUS.FAILED,
+        error: error instanceof Error ? error.message : "Template step failed",
+      };
+    }
+  },
+});