sea-dev 1.0.0

package/libs/dal/src/pipeline.ts

@@ -0,0 +1,4222 @@
+import { db, PIPELINE_RUN_STATUS, PIPELINE_RUN_STEP_STATUS, schema } from "@sea/db";
+import type {
+  PipelineRunStepBlockerDetails,
+  PipelineRunStepBlockerType,
+  SubmissionItemValue,
+} from "@sea/db/schema";
+import type {
+  Artifact,
+  FormGroupHydrated,
+  InsertDataTransformation,
+  InsertSubmissionGroup,
+  InsertSubmissionItem,
+  InsertSubmissionItemVersion,
+  Member,
+  Pipeline,
+  PipelineRun,
+  PipelineRunStep,
+  PipelineRunTimeline,
+  PipelineStep,
+  PipelineStepIntegrationConfig,
+  PipelineWithSteps,
+  Submission,
+  SubmissionGroup,
+  ToolCall,
+} from "@sea/db/types";
+import type { OutputData, OutputFieldTarget, OutputGroupData } from "@sea/schemas/core/pipeline";
+import { formatISODate } from "@sea/util/fmt";
+import { nanoid } from "@sea/util/id";
+import { invariant } from "@sea/util/invariant";
+import { and, asc, count, desc, eq, inArray, isNull, or, sql } from "drizzle-orm";
+import type { DalResult } from "./result";
+import { internalErrorResult, notFoundResult, okResult } from "./result";
+
+function normalizeSteps<T extends { toolsAllowed: string[] | null }>(
+  steps: T[],
+): (Omit<T, "toolsAllowed"> & { toolsAllowed: string[] })[] {
+  return steps.map((step) => ({
+    ...step,
+    toolsAllowed: step.toolsAllowed ?? [],
+  }));
+}
+
+function normalizeSubmissionItemValue(value: unknown): SubmissionItemValue | null {
+  if (value == null) return null;
+  if (typeof value === "string" || typeof value === "number") return value;
+  return String(value);
+}
+
+async function getPipelineRun({
+  member,
+  runId,
+}: {
+  member: Member;
+  runId: string;
+}): Promise<DalResult<PipelineRun>> {
+  try {
+    const run = await db.query.pipelineRun.findFirst({
+      where: and(
+        eq(schema.pipelineRun.id, runId),
+        eq(schema.pipelineRun.organizationId, member.organizationId),
+      ),
+    });
+    if (!run) return notFoundResult;
+    return okResult(run);
+  } catch {
+    return notFoundResult;
+  }
+}
+
+async function getPipelineRunStep({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}): Promise<DalResult<PipelineRunStep>> {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: { run: true },
+    });
+    if (!runStep || runStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+    return okResult(runStep);
+  } catch (err) {
+    console.error("[getPipelineRunStep] Exception", err);
+    return notFoundResult;
+  }
+}
+
+export async function createPipelineRun({
+  member,
+  pipelineId,
+  dealId,
+  triggerType = "manual",
+}: {
+  member: Member;
+  pipelineId: string;
+  dealId: string;
+  triggerType?: string;
+}): Promise<DalResult<PipelineRun>> {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+      with: {
+        steps: {
+          with: {
+            parentDependencies: true,
+          },
+        },
+      },
+    });
+    if (!pipeline) return notFoundResult;
+
+    const [createdRun] = await db
+      .insert(schema.pipelineRun)
+      .values({
+        organizationId: member.organizationId,
+        pipelineId,
+        dealId,
+        status: "pending",
+        triggerType,
+      })
+      .returning();
+
+    if (!createdRun) return notFoundResult;
+
+    // Create run steps for all pipeline steps
+    for (const step of pipeline.steps) {
+      // Check if this step has any parent dependency with multiFormGroupId
+      // If so, this step should be a template that spawns instances per group
+      const shouldBeTemplate = step.parentDependencies.some((d) => d.multiFormGroupId !== null);
+
+      // Create run step - mark as template if it will spawn instances, otherwise normal
+      await db.insert(schema.pipelineRunStep).values({
+        runId: createdRun.id,
+        stepId: step.id,
+        spawnType: shouldBeTemplate ? "template" : "normal",
+        status: "pending",
+      });
+    }
+
+    return okResult(createdRun);
+  } catch (error) {
+    console.error("[createPipelineRun] Error:", error);
+    return notFoundResult;
+  }
+}
+
+export async function updatePipelineRunStatus({
+  member,
+  runId,
+  status,
+  errorSummary,
+}: {
+  member: Member;
+  runId: string;
+  status: string;
+  errorSummary?: string;
+}): Promise<DalResult<PipelineRun>> {
+  const runResult = await getPipelineRun({ member, runId });
+  if (!runResult.ok) return runResult;
+  const sets: Record<string, unknown> = { status };
+  if (errorSummary !== undefined) sets.errorSummary = errorSummary;
+  if (status === PIPELINE_RUN_STATUS.RUNNING) sets.startedAt = new Date();
+  if (
+    status === PIPELINE_RUN_STATUS.COMPLETED ||
+    status === PIPELINE_RUN_STATUS.FAILED ||
+    status === "cancelled"
+  ) {
+    sets.completedAt = new Date();
+  }
+
+  try {
+    const [run] = await db
+      .update(schema.pipelineRun)
+      .set(sets)
+      .where(eq(schema.pipelineRun.id, runId))
+      .returning();
+    if (!run) return notFoundResult;
+    return okResult(run);
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function getPipelines({
+  member,
+  limit,
+}: {
+  member: Member;
+  limit?: number;
+}): Promise<PipelineWithSteps[]> {
+  const pipelines = await db.query.pipeline.findMany({
+    where: and(
+      eq(schema.pipeline.organizationId, member.organizationId),
+      isNull(schema.pipeline.deletedAt),
+      eq(schema.pipeline.isActive, true),
+    ),
+    orderBy: desc(schema.pipeline.updatedAt),
+    limit,
+    with: {
+      steps: {
+        with: {
+          outputForms: true,
+          parentDependencies: true,
+        },
+      },
+    },
+  });
+  return pipelines.map((pipeline) => ({
+    ...pipeline,
+    steps: normalizeSteps(pipeline.steps),
+  }));
+}
+
+export async function getPipeline({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}): Promise<DalResult<Pipeline>> {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+    });
+    if (!pipeline) return notFoundResult;
+    return okResult(pipeline);
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function getPipelineWithSteps({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}): Promise<DalResult<PipelineWithSteps>> {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+        eq(schema.pipeline.isActive, true),
+      ),
+      with: {
+        steps: {
+          with: {
+            outputForms: true,
+            parentDependencies: true,
+          },
+        },
+      },
+    });
+    if (!pipeline) return notFoundResult;
+    return okResult({
+      ...pipeline,
+      steps: normalizeSteps(pipeline.steps),
+    });
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function createPipeline({
+  member,
+  name = "Untitled Pipeline",
+  description = "",
+}: {
+  member: Member;
+  name?: string;
+  description?: string;
+}): Promise<DalResult<PipelineWithSteps>> {
+  try {
+    const [pipeline] = await db
+      .insert(schema.pipeline)
+      .values({
+        organizationId: member.organizationId,
+        name,
+        description,
+        isActive: true,
+        version: "1",
+        metadata: {},
+      })
+      .returning();
+    if (!pipeline) return notFoundResult;
+
+    return okResult({ ...pipeline, steps: [] });
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function createPipelineRunStep({
+  member,
+  runId,
+  stepId,
+  spawnType = "normal",
+  submissionGroupId,
+  stepRunIndex,
+}: {
+  member: Member;
+  runId: string;
+  stepId: string;
+  spawnType?: "normal" | "template" | "instance";
+  submissionGroupId?: string;
+  stepRunIndex?: number;
+}): Promise<DalResult<PipelineRunStep>> {
+  const runResult = await getPipelineRun({ member, runId });
+  if (!runResult.ok) return runResult;
+
+  try {
+    const [runStep] = await db
+      .insert(schema.pipelineRunStep)
+      .values({
+        runId,
+        stepId,
+        spawnType,
+        submissionGroupId: submissionGroupId ?? null,
+        stepRunIndex: stepRunIndex ?? null,
+        status: "pending",
+      })
+      .returning();
+    if (!runStep) return notFoundResult;
+    return okResult(runStep);
+  } catch (error) {
+    console.error("[createPipelineRunStep] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Get all output submissions from a parent step's run steps
+ */
+export async function getParentStepOutputs({
+  member,
+  runId,
+  parentStepId,
+}: {
+  member: Member;
+  runId: string;
+  parentStepId: string;
+}) {
+  const runResult = await getPipelineRun({ member, runId });
+  if (!runResult.ok) return runResult;
+
+  try {
+    // Get run steps for the parent step
+    const runSteps = await db.query.pipelineRunStep.findMany({
+      where: and(
+        eq(schema.pipelineRunStep.runId, runId),
+        eq(schema.pipelineRunStep.stepId, parentStepId),
+      ),
+    });
+
+    if (runSteps.length === 0) {
+      return okResult([]);
+    }
+
+    // Get output submissions from the junction table
+    const runStepIds = runSteps.map((rs) => rs.id);
+    const outputMappings = await db.query.pipelineRunStepOutputSubmission.findMany({
+      with: {
+        submission: {
+          with: {
+            form: true,
+            groups: { with: { formGroup: { with: { fields: true, derived: true } }, items: true } },
+          },
+        },
+      },
+      where: inArray(schema.pipelineRunStepOutputSubmission.runStepId, runStepIds),
+    });
+
+    const outputSubmissions = outputMappings.map((m) => m.submission);
+
+    return okResult(outputSubmissions);
+  } catch (error) {
+    console.error("[getParentStepOutputs] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+export async function updatePipelineRunStepStatus({
+  member,
+  runStepId,
+  status,
+  error,
+}: {
+  member: Member;
+  runStepId: string;
+  status: string;
+  error?: string;
+}): Promise<DalResult<PipelineRunStep>> {
+  const runStepResult = await getPipelineRunStep({ member, runStepId });
+  if (!runStepResult.ok) return runStepResult;
+
+  const now = new Date();
+  const sets: Record<string, unknown> = { status };
+  if (error !== undefined) sets.error = error;
+  if (status === PIPELINE_RUN_STEP_STATUS.RUNNING) sets.startedAt = now;
+  if (
+    status === PIPELINE_RUN_STEP_STATUS.COMPLETED ||
+    status === PIPELINE_RUN_STEP_STATUS.FAILED ||
+    status === "skipped"
+  ) {
+    sets.completedAt = now;
+  }
+
+  try {
+    const [updated] = await db
+      .update(schema.pipelineRunStep)
+      .set(sets)
+      .where(eq(schema.pipelineRunStep.id, runStepId))
+      .returning();
+    if (!updated) return notFoundResult;
+    return okResult(updated);
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function logToolCall({
+  member,
+  runStepId,
+  toolId,
+  args,
+  result,
+  status = PIPELINE_RUN_STEP_STATUS.COMPLETED,
+  latencyMs,
+}: {
+  member: Member;
+  runStepId: string;
+  toolId: string;
+  args: object;
+  result: object;
+  status?: string;
+  latencyMs?: number;
+}): Promise<DalResult<ToolCall>> {
+  const runStepResult = await getPipelineRunStep({ member, runStepId });
+  if (!runStepResult.ok) return runStepResult;
+
+  try {
+    const [toolCall] = await db
+      .insert(schema.toolCall)
+      .values({
+        toolId,
+        args,
+        result,
+        status,
+        latencyMs,
+      })
+      .returning();
+    if (!toolCall) return notFoundResult;
+    return okResult(toolCall);
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function getOutputFieldTargets({
+  member,
+  submissionId,
+}: {
+  member: Member;
+  submissionId: string;
+}): Promise<DalResult<OutputFieldTarget[]>> {
+  try {
+    const submission = await db.query.submission.findFirst({
+      where: and(
+        eq(schema.submission.id, submissionId),
+        eq(schema.submission.organizationId, member.organizationId),
+        isNull(schema.submission.deletedAt),
+      ),
+    });
+    if (!submission) return notFoundResult;
+
+    const targets = await db
+      .select({
+        formFieldId: schema.formField.id,
+        fieldName: schema.formField.name,
+        groupName: schema.formGroup.name,
+        formGroupId: schema.formGroup.id,
+        multi: schema.formGroup.multi,
+      })
+      .from(schema.formField)
+      .innerJoin(schema.formGroup, eq(schema.formGroup.id, schema.formField.formGroupId))
+      .where(eq(schema.formGroup.formId, submission.formId));
+
+    return okResult(
+      targets.map((target) => ({
+        ...target,
+        defaultGroupOrder: 0,
+      })),
+    );
+  } catch {
+    return notFoundResult;
+  }
+}
+/**
+ * Stores extracted data to the submission by creating submission groups, items, and citations.
+ * Handles both single and multi-instance form groups, validating and linking all data to source references.
+ *
+ * @param params - Configuration object
+ * @param params.submissionId - Target submission ID
+ * @param params.documentExtractionId - Source document extraction ID
+ * @param params.formGroupId - Form group being processed
+ * @param params.data - Structured extracted data
+ * @param params.referenceMap - Maps reference IDs to document locations
+ * @param params.validationData - Optional validation results for marking invalid groups
+ */
+export async function storePipelineOutputGroupData({
+  submissionId,
+  formGroupId,
+  data,
+}: {
+  submissionId: string;
+  formGroupId: string;
+  data: OutputGroupData;
+}) {
+  if (data === null) {
+    console.warn(`Deserialization aborted: data is null`);
+    return;
+  }
+
+  const submissionGroups: InsertSubmissionGroup[] = [];
+  const submissionItems: InsertSubmissionItem[] = [];
+  const itemVersions: InsertSubmissionItemVersion[] = [];
+  const dataTransformations: InsertDataTransformation[] = [];
+  const versionToSourceItemIds = new Map<number, string[]>();
+  const versionToReasoning = new Map<number, string>();
+
+  const formGroup = await db.query.formGroup.findFirst({
+    where: eq(schema.formGroup.id, formGroupId),
+    with: {
+      fields: true,
+      derived: true,
+      submissionGroups: {
+        where: eq(schema.submissionGroup.submissionId, submissionId),
+        with: { items: true },
+        orderBy: asc(schema.submissionGroup.order),
+      },
+    },
+  });
+  invariant(formGroup, `No form group found for id ${formGroupId}`);
+
+  if (formGroup.multi) {
+    if (typeof data !== "object" || !Array.isArray(data)) {
+      console.warn(`Form group marked as multi, but data not an array`);
+      return;
+    }
+    const orderOffset =
+      formGroup.submissionGroups.length > 0
+        ? Math.max(...formGroup.submissionGroups.map((g) => g.order)) + 1
+        : 0;
+
+    for (const [order, groupElement] of data.entries()) {
+      const newSubmissionGroup: InsertSubmissionGroup = {
+        id: nanoid(),
+        submissionId,
+        formGroupId: formGroup.id,
+        order: order + orderOffset,
+      };
+      submissionGroups.push(newSubmissionGroup);
+
+      const newSubmissionItems = formGroup.fields.map((field) => ({
+        id: nanoid(),
+        submissionGroupId: newSubmissionGroup.id!,
+        formFieldId: field.id,
+      }));
+      submissionItems.push(...newSubmissionItems);
+
+      const result = await processGroupData(
+        formGroup,
+        newSubmissionItems,
+        itemVersions,
+        dataTransformations,
+        groupElement,
+        order,
+      );
+      // Merge the maps from this iteration
+      result.versionToSourceItemIds.forEach((value, key) => versionToSourceItemIds.set(key, value));
+      result.versionToReasoning.forEach((value, key) => versionToReasoning.set(key, value));
+    }
+  } else {
+    const submissionGroup = formGroup.submissionGroups[0];
+    invariant(submissionGroup);
+
+    const result = await processGroupData(
+      formGroup,
+      submissionGroup.items,
+      itemVersions,
+      dataTransformations,
+      data,
+    );
+    // Merge the maps from the single call
+    result.versionToSourceItemIds.forEach((value, key) => versionToSourceItemIds.set(key, value));
+    result.versionToReasoning.forEach((value, key) => versionToReasoning.set(key, value));
+  }
+
+  await db.transaction(async (tx) => {
+    if (submissionGroups.length > 0)
+      await tx.insert(schema.submissionGroup).values(submissionGroups);
+    if (submissionItems.length > 0) await tx.insert(schema.submissionItem).values(submissionItems);
+
+    // Use the collected mappings from processGroupData
+
+    if (itemVersions.length > 0 && versionToSourceItemIds) {
+      for (let index = 0; index < itemVersions.length; index++) {
+        const itemVersion = itemVersions[index];
+        if (!itemVersion) continue;
+
+        const sourceItemIds = versionToSourceItemIds.get(index);
+
+        if (!sourceItemIds || sourceItemIds.length === 0) {
+          // No transformation needed, insert version as-is
+          await tx.insert(schema.submissionItemVersion).values([itemVersion]);
+          continue;
+        }
+
+        // Fetch the latest version IDs for each source submission item
+        const sourceVersions = await tx
+          .select({
+            id: schema.submissionItemVersion.id,
+            submissionItemId: schema.submissionItemVersion.submissionItemId,
+          })
+          .from(schema.submissionItemVersion)
+          .where(
+            and(
+              inArray(schema.submissionItemVersion.submissionItemId, sourceItemIds),
+              eq(schema.submissionItemVersion.selected, true),
+            ),
+          );
+
+        const sourceVersionIds = sourceVersions.map((v) => v.id);
+
+        if (sourceVersionIds.length === 0) {
+          // No source versions found, insert without transformation
+          await tx.insert(schema.submissionItemVersion).values([itemVersion]);
+          continue;
+        }
+
+        // Get reasoning for this field (if available)
+        const reasoning = versionToReasoning?.get(index);
+
+        // Create dataTransformation record
+        const transformation: InsertDataTransformation = {
+          id: nanoid(),
+          sourceItemVersionIds: sourceVersionIds,
+          description: reasoning || null,
+        };
+
+        await tx.insert(schema.dataTransformation).values([transformation]);
+
+        // Update item version to reference this transformation
+        const versionWithTransformation: typeof itemVersion = {
+          ...itemVersion,
+          sourceId: transformation.id!,
+        };
+
+        await tx.insert(schema.submissionItemVersion).values([versionWithTransformation]);
+      }
+    } else if (itemVersions.length > 0) {
+      // No transformations needed, insert all versions
+      await tx.insert(schema.submissionItemVersion).values(itemVersions);
+    }
+  });
+}
+
+/**
+ * Processes pipeline output data for a single form group and creates submission items with data transformations.
+ * Creates dataTransformation records linking output items to their source items from parent steps.
+ *
+ * @param formGroup - Form group definition with field metadata
+ * @param submissionItems - Array to populate with submission items
+ * @param itemVersions - Array to populate with submission item versions
+ * @param dataTransformations - Array to populate with data transformations
+ * @param groupData - Pipeline output data for this group
+ * @param order - Optional order index for multi-instance groups
+ */
+export async function processGroupData(
+  formGroup: FormGroupHydrated,
+  submissionItems: InsertSubmissionItem[],
+  itemVersions: InsertSubmissionItemVersion[],
+  dataTransformations: InsertDataTransformation[],
+  groupData: OutputGroupData,
+  order: number | null = null,
+): Promise<{
+  versionToSourceItemIds: Map<number, string[]>;
+  versionToReasoning: Map<number, string>;
+}> {
+  const versionToSourceItemIds = new Map<number, string[]>();
+  const versionToReasoning = new Map<number, string>();
+
+  if (!groupData || typeof groupData !== "object" || Array.isArray(groupData)) {
+    console.warn(
+      `Single group element ${formGroup.name} is missing or data is not an object, skipping...`,
+    );
+    return { versionToSourceItemIds, versionToReasoning };
+  }
+
+  // Process each field and track source item IDs and reasoning
+
+  for (const field of formGroup.fields) {
+    const submissionItem = submissionItems.find((item) => item.formFieldId === field.id);
+    const fieldData = groupData[field.name];
+    if (!fieldData || !submissionItem) continue;
+
+    const { value, refId, reasoning } = fieldData;
+    const sourceItemIds = refId.split(",").filter((id: string) => id.trim());
+
+    // Debug: Log only when reasoning is missing
+    if (!reasoning) {
+      console.warn(`[processGroupData] Field '${field.name}' missing reasoning`);
+    }
+
+    // Create item version with transformation source type
+    const newItemVersion: InsertSubmissionItemVersion = {
+      submissionItemId: submissionItem.id!,
+      sourceType: "transformation" as const,
+      selected: true,
+      sourceId: null,
+      value: normalizeSubmissionItemValue(value),
+    };
+
+    const versionIndex = itemVersions.length;
+    itemVersions.push(newItemVersion);
+
+    // Track source item IDs and reasoning for this version to create transformation later
+    versionToSourceItemIds.set(versionIndex, sourceItemIds);
+    if (reasoning && typeof reasoning === "string") {
+      versionToReasoning.set(versionIndex, reasoning);
+    }
+  }
+
+  return { versionToSourceItemIds, versionToReasoning };
+}
+
+/**
+ * Parses and normalizes field values into the submission item value format.
+ * Dates are converted to ISO format strings, booleans to numbers (0/1), and other values are passed through.
+ *
+ * @param value - Raw value from extraction (string, number, boolean, or Date)
+ * @returns Normalized value suitable for storing in submission items
+ */
+function parseItemValue(value: string | number | boolean | Date): SubmissionItemValue {
+  if (value instanceof Date) {
+    return formatISODate(value);
+  }
+  if (typeof value === "boolean") {
+    return Number(value);
+  }
+
+  return value;
+}
+
+export async function storePipelineOutputData({
+  member,
+  outputSubmissionId,
+  outputData,
+}: {
+  member: Member;
+  outputSubmissionId: string;
+  outputData: OutputData;
+}): Promise<DalResult<{ success: true }>> {
+  try {
+    // Fetch the submission with its form and groups
+    const submission = await db.query.submission.findFirst({
+      where: and(
+        eq(schema.submission.id, outputSubmissionId),
+        eq(schema.submission.organizationId, member.organizationId),
+        isNull(schema.submission.deletedAt),
+      ),
+      with: {
+        form: {
+          with: {
+            groups: {
+              with: {
+                fields: true,
+              },
+              orderBy: asc(schema.formGroup.order),
+            },
+          },
+        },
+      },
+    });
+
+    if (!submission) return notFoundResult;
+
+    // Process each group in the output data
+    for (const group of submission.form.groups) {
+      const groupData = outputData?.[group.name];
+      if (groupData) {
+        await storePipelineOutputGroupData({
+          submissionId: outputSubmissionId,
+          formGroupId: group.id,
+          data: groupData,
+        });
+      }
+    }
+
+    return okResult({ success: true });
+  } catch (error) {
+    console.error("[storePipelineOutputData] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get runtime submission (allows both extraction and transformation types).
+ * Used in pipeline execution to access both input and intermediate submissions.
+ */
+export async function getRuntimeSubmission({
+  member,
+  submissionId,
+}: {
+  member: Member;
+  submissionId: string;
+}): Promise<DalResult<Submission>> {
+  try {
+    const submission = await db.query.submission.findFirst({
+      where: and(
+        eq(schema.submission.id, submissionId),
+        eq(schema.submission.organizationId, member.organizationId),
+        isNull(schema.submission.deletedAt),
+      ),
+    });
+    if (!submission) return notFoundResult;
+    return okResult(submission);
+  } catch (error) {
+    console.error("[getRuntimeSubmission] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Get runtime submission with denormalized data (groups, items, etc).
+ * Used in multi-group spawning and data filtering logic.
+ */
+export async function getRuntimeSubmissionWithData({
+  member,
+  submissionId,
+}: {
+  member: Member;
+  submissionId: string;
+}): Promise<
+  DalResult<
+    Submission & {
+      groups: Array<{
+        id: string;
+        formGroup: {
+          id: string;
+          name: string;
+          multi: boolean;
+        };
+        items: Array<{
+          id: string;
+          formFieldId: string;
+        }>;
+      }>;
+    }
+  >
+> {
+  try {
+    const submission = await db.query.submission.findFirst({
+      where: and(
+        eq(schema.submission.id, submissionId),
+        eq(schema.submission.organizationId, member.organizationId),
+        isNull(schema.submission.deletedAt),
+      ),
+      with: {
+        groups: {
+          orderBy: asc(schema.submissionGroup.order),
+          with: {
+            formGroup: true,
+            items: true,
+          },
+        },
+      },
+    });
+    if (!submission) return notFoundResult;
+    return okResult(submission);
+  } catch (error) {
+    console.error("[getRuntimeSubmissionWithData] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+export async function getPipelineRunWithSteps({
+  member,
+  runId,
+}: {
+  member: Member;
+  runId: string;
+}) {
+  try {
+    const run = await db.query.pipelineRun.findFirst({
+      where: and(
+        eq(schema.pipelineRun.id, runId),
+        eq(schema.pipelineRun.organizationId, member.organizationId),
+      ),
+      with: {
+        pipeline: {
+          with: {
+            steps: {
+              orderBy: asc(schema.pipelineStep.createdAt),
+            },
+          },
+        },
+      },
+    });
+
+    if (!run) return notFoundResult;
+    return okResult(run);
+  } catch (error) {
+    console.error("[getPipelineRunWithSteps] Error:", error);
+    return notFoundResult;
+  }
+}
+
+export async function getPipelineRuns({
+  member,
+  pipelineId,
+  limit = 50,
+}: {
+  member: Member;
+  pipelineId: string;
+  limit?: number;
+}): Promise<DalResult<PipelineRun[]>> {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+    });
+    if (!pipeline) return notFoundResult;
+
+    const runs = await db.query.pipelineRun.findMany({
+      where: and(
+        eq(schema.pipelineRun.organizationId, member.organizationId),
+        eq(schema.pipelineRun.pipelineId, pipelineId),
+      ),
+      orderBy: desc(schema.pipelineRun.createdAt),
+      limit,
+      with: {
+        steps: {
+          orderBy: asc(schema.pipelineRunStep.createdAt),
+        },
+      },
+    });
+    return okResult(runs);
+  } catch {
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get all output submissions for a pipeline run, grouped by run step ID
+ */
+export async function getRunOutputSubmissions({
+  member,
+  runId,
+}: {
+  member: Member;
+  runId: string;
+}): Promise<DalResult<Map<string, string[]>>> {
+  const runResult = await getPipelineRun({ member, runId });
+  if (!runResult.ok) return runResult;
+
+  try {
+    // Get all run steps for this run
+    const runSteps = await db.query.pipelineRunStep.findMany({
+      where: eq(schema.pipelineRunStep.runId, runId),
+    });
+
+    if (runSteps.length === 0) {
+      return okResult(new Map());
+    }
+
+    // Get all output submissions
+    const runStepIds = runSteps.map((rs) => rs.id);
+    const outputMappings = await db.query.pipelineRunStepOutputSubmission.findMany({
+      where: inArray(schema.pipelineRunStepOutputSubmission.runStepId, runStepIds),
+    });
+
+    // Group by run step ID
+    const submissionsByRunStep = new Map<string, string[]>();
+    for (const mapping of outputMappings) {
+      const existing = submissionsByRunStep.get(mapping.runStepId) || [];
+      existing.push(mapping.submissionId);
+      submissionsByRunStep.set(mapping.runStepId, existing);
+    }
+
+    return okResult(submissionsByRunStep);
+  } catch (error) {
+    console.error("[getRunOutputSubmissions] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+export async function getRunTimeline({
+  member,
+  runId,
+}: {
+  member: Member;
+  runId: string;
+}): Promise<DalResult<PipelineRunTimeline>> {
+  try {
+    const run = await db.query.pipelineRun.findFirst({
+      where: and(
+        eq(schema.pipelineRun.id, runId),
+        eq(schema.pipelineRun.organizationId, member.organizationId),
+      ),
+      with: {
+        steps: {
+          orderBy: asc(schema.pipelineRunStep.createdAt),
+          // Filter out template steps - they're just placeholders for spawning instances
+          // Only show normal steps and actual instances
+          where: (step, { ne }) => ne(step.spawnType, "template"),
+        },
+        pipeline: true,
+      },
+    });
+    if (!run) return notFoundResult;
+
+    const stepIds = run.steps.map((s) => s.id);
+    if (stepIds.length === 0) {
+      // No steps, return early with empty arrays
+      return okResult({
+        ...run,
+        steps: [],
+        artifacts: [],
+      });
+    }
+
+    // Load tool calls for all steps
+    const toolCallMappings =
+      run.steps.length > 0
+        ? await db.query.pipelineRunStepToolCall.findMany({
+            where: inArray(
+              schema.pipelineRunStepToolCall.runStepId,
+              run.steps.map((s) => s.id),
+            ),
+            with: {
+              toolCall: true,
+            },
+          })
+        : [];
+
+    // Load artifacts for all steps
+    const stepArtifactMappings =
+      run.steps.length > 0
+        ? await db.query.pipelineRunArtifact.findMany({
+            where: and(
+              eq(schema.pipelineRunArtifact.runId, runId),
+              inArray(
+                schema.pipelineRunArtifact.runStepId,
+                run.steps.map((s) => s.id),
+              ),
+            ),
+            with: {
+              artifact: true,
+            },
+          })
+        : [];
+
+    // Load run-level artifacts
+    const runArtifactMappings = await db.query.pipelineRunArtifact.findMany({
+      where: and(
+        eq(schema.pipelineRunArtifact.runId, runId),
+        isNull(schema.pipelineRunArtifact.runStepId),
+      ),
+      with: {
+        artifact: true,
+      },
+    });
+
+    // Group by step
+    const toolCallsByStep = new Map<string, typeof toolCallMappings>();
+    for (const mapping of toolCallMappings) {
+      const existing = toolCallsByStep.get(mapping.runStepId) || [];
+      existing.push(mapping);
+      toolCallsByStep.set(mapping.runStepId, existing);
+    }
+
+    const artifactsByStep = new Map<string, typeof stepArtifactMappings>();
+    for (const mapping of stepArtifactMappings) {
+      if (mapping.runStepId) {
+        const existing = artifactsByStep.get(mapping.runStepId) || [];
+        existing.push(mapping);
+        artifactsByStep.set(mapping.runStepId, existing);
+      }
+    }
+
+    // Collect all artifacts (both run-level and step-level)
+    const allArtifacts = [
+      ...runArtifactMappings.map((m) => m.artifact),
+      ...stepArtifactMappings.map((m) => m.artifact),
+    ];
+
+    // Transform the data to match the expected type
+    const transformedRun: PipelineRunTimeline = {
+      ...run,
+      steps: run.steps.map((step) => ({
+        ...step,
+        toolCalls: (toolCallsByStep.get(step.id) || []).map((m) => m.toolCall),
+        artifacts: (artifactsByStep.get(step.id) || []).map((m) => m.artifact),
+      })),
+      artifacts: allArtifacts,
+    };
+
+    return okResult(transformedRun);
+  } catch (error) {
+    console.error("getRunTimeline error:", error);
+    return notFoundResult;
+  }
+}
+
+export async function getPipelineDetail({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}) {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+      with: {
+        steps: {
+          with: {
+            outputForms: true,
+            integrationConfig: true,
+            parentDependencies: {
+              with: {
+                parentStep: true,
+              },
+            },
+          },
+        },
+      },
+    });
+    if (!pipeline) return notFoundResult;
+
+    // Get execution levels for all steps
+    const levelsResult = await getPipelineStepsWithLevels({ member, pipelineId });
+    const levelsByStepId = new Map<string, number>();
+    if (levelsResult.ok) {
+      for (const step of levelsResult.data) {
+        levelsByStepId.set(step.id, step.executionLevel);
+      }
+    }
+
+    // Sort steps by execution level, then by createdAt
+    const sortedSteps = [...pipeline.steps].sort((a, b) => {
+      const levelA = levelsByStepId.get(a.id) ?? 0;
+      const levelB = levelsByStepId.get(b.id) ?? 0;
+      if (levelA !== levelB) return levelA - levelB;
+      return new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime();
+    });
+
+    const pipelineWithNormalizedSteps = {
+      ...pipeline,
+      steps: normalizeSteps(sortedSteps),
+    };
+
+    return okResult(pipelineWithNormalizedSteps);
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function deletePipeline({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}): Promise<DalResult<{ success: true }>> {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+    });
+    if (!pipeline) return notFoundResult;
+
+    await db
+      .update(schema.pipeline)
+      .set({ deletedAt: new Date() })
+      .where(eq(schema.pipeline.id, pipelineId));
+
+    return okResult({ success: true });
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function togglePipelineActive({
+  member,
+  pipelineId,
+  isActive,
+}: {
+  member: Member;
+  pipelineId: string;
+  isActive: boolean;
+}): Promise<DalResult<{ success: true }>> {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+    });
+    if (!pipeline) return notFoundResult;
+
+    await db.update(schema.pipeline).set({ isActive }).where(eq(schema.pipeline.id, pipelineId));
+
+    return okResult({ success: true });
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function updatePipelineMetadata({
+  member,
+  pipelineId,
+  name,
+  description,
+}: {
+  member: Member;
+  pipelineId: string;
+  name: string;
+  description: string;
+}): Promise<DalResult<{ success: true }>> {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+    });
+    if (!pipeline) return notFoundResult;
+
+    await db
+      .update(schema.pipeline)
+      .set({ name, description })
+      .where(eq(schema.pipeline.id, pipelineId));
+
+    return okResult({ success: true });
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function createPipelineStep({
+  member,
+  pipelineId,
+  name,
+  description,
+  prompt,
+  type,
+  role,
+  outputFormIds,
+  toolsAllowed,
+  integrationConfig,
+}: {
+  member: Member;
+  pipelineId: string;
+  name: string;
+  description: string;
+  prompt?: string;
+  type: "single" | "multi";
+  role: "extraction" | "integration" | "analysis" | null;
+  outputFormIds?: string[];
+  toolsAllowed?: string[];
+  integrationConfig?: {
+    method: "GET" | "POST" | "PUT" | "DELETE";
+    urlTemplate: string;
+    headers: Record<string, string>;
+    bodyTemplate?: string;
+  };
+}): Promise<DalResult<PipelineStep>> {
+  try {
+    // Verify pipeline exists and member has access
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+    });
+    if (!pipeline) return notFoundResult;
+
+    const [step] = await db
+      .insert(schema.pipelineStep)
+      .values({
+        pipelineId,
+        name,
+        description,
+        prompt: prompt ?? "",
+        type,
+        role,
+        toolsAllowed: toolsAllowed ?? [],
+        // integrationConfig deliberately NOT written - use pipelineStepIntegrationConfig table instead
+        retryPolicy: {},
+      })
+      .returning();
+
+    if (!step) return notFoundResult;
+
+    // Insert into pipeline_step_output_form table
+    if (outputFormIds && outputFormIds.length > 0) {
+      await db.insert(schema.pipelineStepOutputForm).values(
+        outputFormIds.map((formId) => ({
+          stepId: step.id,
+          formId,
+        })),
+      );
+    }
+
+    // Insert into pipeline_step_integration_config table if provided
+    if (integrationConfig && role === "integration") {
+      await db.insert(schema.pipelineStepIntegrationConfig).values({
+        id: nanoid(),
+        stepId: step.id,
+        method: integrationConfig.method,
+        urlTemplate: integrationConfig.urlTemplate,
+        headers: integrationConfig.headers,
+        bodyTemplate: integrationConfig.bodyTemplate ?? null,
+      });
+    }
+
+    return okResult(step);
+  } catch (error) {
+    console.error("[createPipelineStep] Error:", error);
+    return notFoundResult;
+  }
+}
+
+export async function updatePipelineStepConfig({
+  member,
+  stepId,
+  name,
+  description,
+  prompt,
+  type,
+  role,
+  outputFormIds,
+  toolsAllowed,
+  integrationConfig,
+}: {
+  member: Member;
+  stepId: string;
+  name?: string;
+  description?: string;
+  prompt?: string;
+  type?: "single" | "multi";
+  role?: "extraction" | "integration" | "analysis" | null;
+  outputFormIds?: string[];
+  toolsAllowed?: string[];
+  integrationConfig?: {
+    method: "GET" | "POST" | "PUT" | "DELETE";
+    urlTemplate: string;
+    headers: Record<string, string>;
+    bodyTemplate?: string;
+  } | null;
+}): Promise<DalResult<{ success: true }>> {
+  try {
+    const step = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, stepId),
+      with: {
+        pipeline: true,
+      },
+    });
+    if (!step || step.pipeline.organizationId !== member.organizationId) return notFoundResult;
+
+    // Build update object with only provided fields
+    const updateData: Record<string, unknown> = {};
+    if (name !== undefined) updateData.name = name;
+    if (description !== undefined) updateData.description = description;
+    if (prompt !== undefined) updateData.prompt = prompt;
+    if (type !== undefined) updateData.type = type;
+    if (role !== undefined) updateData.role = role;
+    if (toolsAllowed !== undefined) updateData.toolsAllowed = toolsAllowed;
+    // integrationConfig deliberately NOT written - use pipelineStepIntegrationConfig table instead
+
+    await db.update(schema.pipelineStep).set(updateData).where(eq(schema.pipelineStep.id, stepId));
+
+    // Update pipeline_step_output_form table
+    if (outputFormIds !== undefined) {
+      // Delete existing output form associations
+      await db
+        .delete(schema.pipelineStepOutputForm)
+        .where(eq(schema.pipelineStepOutputForm.stepId, stepId));
+
+      // Insert new output form associations
+      if (outputFormIds.length > 0) {
+        await db.insert(schema.pipelineStepOutputForm).values(
+          outputFormIds.map((formId) => ({
+            stepId,
+            formId,
+          })),
+        );
+      }
+    }
+
+    // Update pipeline_step_integration_config table
+    if (integrationConfig !== undefined) {
+      if (integrationConfig === null) {
+        // Delete config if explicitly set to null
+        await db
+          .delete(schema.pipelineStepIntegrationConfig)
+          .where(eq(schema.pipelineStepIntegrationConfig.stepId, stepId));
+      } else {
+        // Upsert integration config via upsertStepIntegrationConfig
+        const upsertResult = await upsertStepIntegrationConfig({
+          member,
+          stepId,
+          config: integrationConfig,
+        });
+        if (!upsertResult.ok) return upsertResult;
+      }
+    }
+
+    return okResult({ success: true });
+  } catch (error) {
+    console.error("[updatePipelineStepConfig] Error:", error);
+    return notFoundResult;
+  }
+}
+
+export async function getPipelineStepById({
+  member,
+  stepId,
+}: {
+  member: Member;
+  stepId: string;
+}): Promise<
+  DalResult<
+    PipelineStep & {
+      pipeline: Pipeline;
+      outputForms: Array<{ id: string; stepId: string; formId: string; createdAt: Date }>;
+      integrationConfig: PipelineStepIntegrationConfig | null;
+    }
+  >
+> {
+  try {
+    const step = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, stepId),
+      with: {
+        pipeline: true,
+        outputForms: true,
+        integrationConfig: true,
+      },
+    });
+    if (!step || step.pipeline.organizationId !== member.organizationId) return notFoundResult;
+    return okResult(step);
+  } catch (error) {
+    console.error("[getPipelineStepById] Error:", error);
+    return notFoundResult;
+  }
+}
+
+export async function deletePipelineStep({
+  member,
+  stepId,
+}: {
+  member: Member;
+  stepId: string;
+}): Promise<DalResult<{ success: true }>> {
+  try {
+    const step = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, stepId),
+      with: {
+        pipeline: true,
+      },
+    });
+    if (!step || step.pipeline.organizationId !== member.organizationId) return notFoundResult;
+
+    await db.delete(schema.pipelineStep).where(eq(schema.pipelineStep.id, stepId));
+
+    return okResult({ success: true });
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function getPipelineRunStepWithDetails({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}): Promise<
+  DalResult<{
+    step: PipelineRunStep;
+    run: PipelineRun & {
+      pipeline: PipelineWithSteps | null;
+      steps: PipelineRunStep[];
+      resources: { submissionIds: string[]; documentIds: string[] };
+    };
+  }>
+> {
+  try {
+    const step = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: {
+        run: {
+          with: {
+            pipeline: {
+              with: {
+                steps: {
+                  orderBy: asc(schema.pipelineStep.createdAt),
+                  with: {
+                    outputForms: true,
+                    parentDependencies: true,
+                  },
+                },
+              },
+            },
+            steps: {
+              orderBy: asc(schema.pipelineRunStep.createdAt),
+            },
+          },
+        },
+      },
+    });
+
+    if (!step) return notFoundResult;
+    if (step.run.organizationId !== member.organizationId) return notFoundResult;
+
+    // TODO: Get run submissions from appropriate table once schema is finalized
+    const submissionIds: string[] = [];
+
+    return okResult({
+      step,
+      run: {
+        ...step.run,
+        pipeline: step.run.pipeline
+          ? {
+              ...step.run.pipeline,
+              steps: normalizeSteps(step.run.pipeline.steps),
+            }
+          : null,
+        resources: { submissionIds, documentIds: [] },
+      },
+    });
+  } catch {
+    return notFoundResult;
+  }
+}
+
+export async function getSubmissionsGroupedByForm({
+  member,
+  formIds,
+  limit = 50,
+}: {
+  member: Member;
+  formIds?: string[];
+  limit?: number;
+}): Promise<
+  DalResult<
+    Array<{
+      formId: string;
+      formName: string;
+      submissions: Array<{
+        id: string;
+        name: string;
+        createdAt: Date;
+        updatedAt: Date;
+        status: string;
+      }>;
+    }>
+  >
+> {
+  try {
+    const whereConditions = [
+      eq(schema.submission.organizationId, member.organizationId),
+      eq(schema.submission.type, "extraction"),
+      isNull(schema.submission.deletedAt),
+    ];
+
+    if (formIds && formIds.length > 0) {
+      whereConditions.push(inArray(schema.submission.formId, formIds));
+    }
+
+    const submissions = await db.query.submission.findMany({
+      where: and(...whereConditions),
+      orderBy: desc(schema.submission.updatedAt),
+      limit,
+      with: {
+        form: true,
+      },
+    });
+
+    // Group submissions by form
+    const groupedMap = new Map<
+      string,
+      {
+        formId: string;
+        formName: string;
+        submissions: Array<{
+          id: string;
+          name: string;
+          createdAt: Date;
+          updatedAt: Date;
+          status: string;
+        }>;
+      }
+    >();
+
+    for (const submission of submissions) {
+      const formId = submission.formId;
+      const formName = submission.form.name;
+
+      if (!groupedMap.has(formId)) {
+        groupedMap.set(formId, {
+          formId,
+          formName,
+          submissions: [],
+        });
+      }
+
+      groupedMap.get(formId)!.submissions.push({
+        id: submission.id,
+        name: submission.name || "Untitled",
+        createdAt: submission.createdAt,
+        updatedAt: submission.updatedAt,
+        status: submission.status,
+      });
+    }
+
+    return okResult(Array.from(groupedMap.values()));
+  } catch {
+    return notFoundResult;
+  }
+}
+
+// Placeholder functions for pipeline execution
+export async function createTransformation({
+  member,
+  runStepId,
+  kind,
+  expression,
+  metadata = {},
+}: {
+  member: Member;
+  runStepId: string;
+  kind: string;
+  expression?: string;
+  metadata?: object;
+}): Promise<DalResult<{ id: string }>> {
+  // TODO: Implement transformation creation
+  return notFoundResult;
+}
+
+export async function upsertTransformationOutputVersions({
+  member,
+  outputSubmissionId,
+  transformationId,
+  outputData,
+}: {
+  member: Member;
+  outputSubmissionId: string;
+  transformationId: string;
+  outputData: OutputData;
+}): Promise<DalResult<{ outputVersionIds: string[]; inputVersionIds: string[] }>> {
+  // Use the renamed function
+  const result = await storePipelineOutputData({
+    member,
+    outputSubmissionId,
+    outputData,
+  });
+
+  if (!result.ok) return notFoundResult;
+
+  // TODO: Return actual version IDs once we track them
+  return okResult({ outputVersionIds: [], inputVersionIds: [] });
+}
+
+export async function createPipelineArtifact({
+  member,
+  runId,
+  runStepId,
+  type = "markdown",
+  title,
+  content,
+  metadata = {},
+}: {
+  member: Member;
+  runId: string;
+  runStepId: string;
+  type?: string;
+  title: string;
+  content: object;
+  metadata?: object;
+}): Promise<DalResult<Artifact>> {
+  try {
+    // Verify run belongs to member's organization
+    const run = await db.query.pipelineRun.findFirst({
+      where: and(
+        eq(schema.pipelineRun.id, runId),
+        eq(schema.pipelineRun.organizationId, member.organizationId),
+      ),
+    });
+    if (!run) return notFoundResult;
+
+    // Verify run step exists and belongs to the run
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: and(eq(schema.pipelineRunStep.id, runStepId), eq(schema.pipelineRunStep.runId, runId)),
+    });
+    if (!runStep) return notFoundResult;
+
+    // Create the artifact
+    const [artifact] = await db
+      .insert(schema.artifact)
+      .values({
+        type,
+        title,
+        content,
+        metadata,
+      })
+      .returning();
+
+    if (!artifact) return notFoundResult;
+
+    // Link artifact to pipeline run and step
+    await db.insert(schema.pipelineRunArtifact).values({
+      runId,
+      artifactId: artifact.id,
+      runStepId,
+    });
+
+    return okResult(artifact);
+  } catch (error) {
+    console.error("[createPipelineArtifact] Error:", error);
+    return notFoundResult;
+  }
+}
+
+export async function logPipelineToolCall({
+  member,
+  runStepId,
+  toolName,
+  args,
+  result,
+  status = PIPELINE_RUN_STEP_STATUS.COMPLETED,
+  latencyMs,
+}: {
+  member: Member;
+  runStepId: string;
+  toolName: string;
+  args: object;
+  result: object;
+  status?: string;
+  latencyMs?: number;
+}): Promise<DalResult<ToolCall>> {
+  // TODO: Implement tool call logging
+  return notFoundResult;
+}
+
+// *****************************************
+// DAG Pipeline Execution Levels (Recursive CTE)
+// *****************************************
+
+/**
+ * Get pipeline steps with their execution levels using recursive CTE.
+ * Level 0 = root steps (no dependencies)
+ * Level 1 = depend on level 0
+ * Level N = depend on level N-1
+ *
+ * This enables proper topological ordering for DAG execution.
+ */
+export async function getPipelineStepsWithLevels({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}) {
+  try {
+    // Use raw SQL for recursive CTE since Drizzle's CTE builder has limitations with recursive joins
+    // Validate pipeline ownership first
+    const result = await db.execute<{
+      id: string;
+      name: string;
+      created_at: Date;
+      role: string | null;
+      type: string;
+      execution_level: number;
+    }>(sql`
+      WITH RECURSIVE step_levels AS (
+        -- Base case: steps with no dependencies (root steps at level 0)
+        SELECT
+          ps.id,
+          ps.name,
+          ps.created_at,
+          ps.role,
+          ps.type,
+          0 as level
+        FROM pipeline_step ps
+        INNER JOIN pipeline p ON ps.pipeline_id = p.id
+        LEFT JOIN pipeline_step_dependency psd ON ps.id = psd.child_step_id
+        WHERE ps.pipeline_id = ${pipelineId}
+          AND p.organization_id = ${member.organizationId}
+          AND p.deleted_at IS NULL
+          AND psd.parent_step_id IS NULL
+
+        UNION ALL
+
+        -- Recursive case: steps that depend on steps already in step_levels
+        SELECT
+          ps.id,
+          ps.name,
+          ps.created_at,
+          ps.role,
+          ps.type,
+          sl.level + 1
+        FROM pipeline_step ps
+        INNER JOIN pipeline_step_dependency psd ON ps.id = psd.child_step_id
+        INNER JOIN step_levels sl ON psd.parent_step_id = sl.id
+        WHERE ps.pipeline_id = ${pipelineId}
+      )
+      -- Get max level per step (in case multiple paths exist)
+      SELECT
+        id,
+        name,
+        created_at,
+        role,
+        type,
+        MAX(level) as execution_level
+      FROM step_levels
+      GROUP BY id, name, created_at, role, type
+      ORDER BY execution_level, created_at
+    `);
+
+    return okResult(
+      result.rows.map((row) => ({
+        id: row.id,
+        name: row.name,
+        createdAt: row.created_at,
+        role: row.role,
+        type: row.type,
+        executionLevel: row.execution_level,
+      })),
+    );
+  } catch (error) {
+    console.error("[getPipelineStepsWithLevels] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get unblocked steps for a pipeline run that are ready to execute.
+ * Uses recursive CTE + SELECT FOR UPDATE SKIP LOCKED for safe concurrent worker access.
+ *
+ * A step is "unblocked" if:
+ * - It has no unresolved blockers
+ * - All parent step runs are completed
+ * - It's currently in 'pending' status
+ */
+export async function getUnblockedStepsForRun({
+  member,
+  runId,
+  limit = 10,
+}: {
+  member: Member;
+  runId: string;
+  limit?: number;
+}) {
+  try {
+    const run = await db.query.pipelineRun.findFirst({
+      where: and(
+        eq(schema.pipelineRun.id, runId),
+        eq(schema.pipelineRun.organizationId, member.organizationId),
+      ),
+    });
+    if (!run) return notFoundResult;
+
+    // Get execution levels first to understand ordering
+    const levelsResult = await getPipelineStepsWithLevels({
+      member,
+      pipelineId: run.pipelineId,
+    });
+    if (!levelsResult.ok) return levelsResult;
+
+    // Find step runs that are ready to execute
+    const readyStepRuns = await db.transaction(async (tx) => {
+      return await tx
+        .select({
+          id: schema.pipelineRunStep.id,
+          runId: schema.pipelineRunStep.runId,
+          stepId: schema.pipelineRunStep.stepId,
+          status: schema.pipelineRunStep.status,
+        })
+        .from(schema.pipelineRunStep)
+        .where(
+          and(
+            eq(schema.pipelineRunStep.runId, runId),
+            eq(schema.pipelineRunStep.status, "pending"),
+            // Check no unresolved blockers exist
+            sql`NOT EXISTS (
+              SELECT 1 FROM ${schema.pipelineRunStepBlocker}
+              WHERE ${schema.pipelineRunStepBlocker.runStepId} = ${schema.pipelineRunStep.id}
+              AND ${schema.pipelineRunStepBlocker.resolved} = false
+            )`,
+            // Check all parent step runs are completed
+            sql`NOT EXISTS (
+              SELECT 1 FROM ${schema.pipelineStepDependency} dep
+              INNER JOIN ${schema.pipelineRunStep} parent_run_step
+                ON parent_run_step.step_id = dep.parent_step_id
+                AND parent_run_step.run_id = ${schema.pipelineRunStep.runId}
+              WHERE dep.child_step_id = ${schema.pipelineRunStep.stepId}
+              AND parent_run_step.status != 'completed'
+            )`,
+          ),
+        )
+        .limit(limit)
+        .for("update", { skipLocked: true });
+    });
+
+    return okResult(readyStepRuns);
+  } catch (error) {
+    console.error("[getUnblockedStepsForRun] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get all parent step runs for a given step run (via dependencies)
+ */
+export async function getParentStepRuns({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: { run: true },
+    });
+    if (!runStep || runStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Get parent steps via dependency table
+    const parentStepRuns = await db
+      .select({
+        id: schema.pipelineRunStep.id,
+        stepId: schema.pipelineRunStep.stepId,
+        status: schema.pipelineRunStep.status,
+      })
+      .from(schema.pipelineStepDependency)
+      .innerJoin(
+        schema.pipelineRunStep,
+        and(
+          eq(schema.pipelineStepDependency.parentStepId, schema.pipelineRunStep.stepId),
+          eq(schema.pipelineRunStep.runId, runStep.runId),
+        ),
+      )
+      .where(eq(schema.pipelineStepDependency.childStepId, runStep.stepId!));
+
+    return okResult(parentStepRuns);
+  } catch (error) {
+    console.error("[getParentStepRuns] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get parent step runs with their dependency configurations (including formIdMapping)
+ */
+export async function getParentStepRunsWithDependencies({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: { run: true },
+    });
+    if (!runStep || runStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Get parent steps with dependency config
+    const parentStepRuns = await db
+      .select({
+        id: schema.pipelineRunStep.id,
+        stepId: schema.pipelineRunStep.stepId,
+        status: schema.pipelineRunStep.status,
+        formIdMapping: schema.pipelineStepDependency.formIdMapping,
+        multiFormGroupId: schema.pipelineStepDependency.multiFormGroupId,
+      })
+      .from(schema.pipelineStepDependency)
+      .innerJoin(
+        schema.pipelineRunStep,
+        and(
+          eq(schema.pipelineStepDependency.parentStepId, schema.pipelineRunStep.stepId),
+          eq(schema.pipelineRunStep.runId, runStep.runId),
+        ),
+      )
+      .where(eq(schema.pipelineStepDependency.childStepId, runStep.stepId!));
+
+    return okResult(parentStepRuns);
+  } catch (error) {
+    console.error("[getParentStepRunsWithDependencies] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get all child step runs for a given step run (via dependencies)
+ */
+export async function getChildStepRuns({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: { run: true },
+    });
+    if (!runStep || runStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Get child steps via dependency table
+    const childStepRuns = await db
+      .select({
+        id: schema.pipelineRunStep.id,
+        stepId: schema.pipelineRunStep.stepId,
+        status: schema.pipelineRunStep.status,
+      })
+      .from(schema.pipelineStepDependency)
+      .innerJoin(
+        schema.pipelineRunStep,
+        and(
+          eq(schema.pipelineStepDependency.childStepId, schema.pipelineRunStep.stepId),
+          eq(schema.pipelineRunStep.runId, runStep.runId),
+        ),
+      )
+      .where(eq(schema.pipelineStepDependency.parentStepId, runStep.stepId!));
+
+    return okResult(childStepRuns);
+  } catch (error) {
+    console.error("[getChildStepRuns] Error:", error);
+    return notFoundResult;
+  }
+}
+
+// *****************************************
+// Multi-Group → Step Run Linkage
+// *****************************************
+
+/**
+ * Get submission groups for a specific formGroupId.
+ * Used to fetch groups from parent output when spawning multi-instance steps.
+ */
+export async function getSubmissionGroups({
+  member,
+  submissionId,
+  formGroupId,
+}: {
+  member: Member;
+  submissionId: string;
+  formGroupId: string;
+}): Promise<DalResult<SubmissionGroup[]>> {
+  try {
+    const submission = await db.query.submission.findFirst({
+      where: and(
+        eq(schema.submission.id, submissionId),
+        eq(schema.submission.organizationId, member.organizationId),
+        isNull(schema.submission.deletedAt),
+      ),
+    });
+    if (!submission) return notFoundResult;
+
+    const groups = await db.query.submissionGroup.findMany({
+      where: and(
+        eq(schema.submissionGroup.submissionId, submissionId),
+        eq(schema.submissionGroup.formGroupId, formGroupId),
+      ),
+      orderBy: asc(schema.submissionGroup.order),
+    });
+
+    return okResult(groups);
+  } catch (error) {
+    console.error("[getSubmissionGroups] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Create child step runs for each group produced by a multi-type parent step.
+ * Links submission groups to their triggered step runs.
+ *
+ * Example: Extraction produces 3 month groups → spawns 3 child analysis runs
+ *
+ * @deprecated This function will be removed. Multi-instance spawning now uses template run steps.
+ */
+export async function createChildRunsFromGroups({
+  member,
+  parentRunStepId,
+  submissionId,
+  childStepId,
+}: {
+  member: Member;
+  parentRunStepId: string;
+  submissionId: string;
+  childStepId: string;
+}) {
+  try {
+    const parentRunStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, parentRunStepId),
+      with: {
+        run: true,
+        step: true,
+      },
+    });
+
+    if (!parentRunStep || parentRunStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Get dependency config to find which formGroup to spawn for
+    const dependency = await db.query.pipelineStepDependency.findFirst({
+      where: eq(schema.pipelineStepDependency.childStepId, childStepId),
+    });
+
+    if (!dependency?.multiFormGroupId) {
+      return okResult({ childStepRuns: [] }); // Not a multi step, no spawning
+    }
+
+    // Get all groups matching the configured formGroup
+    const groups = await db.query.submissionGroup.findMany({
+      where: and(
+        eq(schema.submissionGroup.submissionId, submissionId),
+        eq(schema.submissionGroup.formGroupId, dependency.multiFormGroupId),
+      ),
+      orderBy: (groups, { asc }) => [asc(groups.order)],
+    });
+
+    if (groups.length === 0) {
+      return okResult({ childStepRuns: [] });
+    }
+
+    // Create a child step run for each group
+    const childStepRuns = [];
+
+    for (const group of groups) {
+      // Create child step run
+      const [childStepRun] = await db
+        .insert(schema.pipelineRunStep)
+        .values({
+          runId: parentRunStep.runId,
+          stepId: childStepId,
+          status: "pending",
+        })
+        .returning();
+
+      if (!childStepRun) continue;
+
+      childStepRuns.push(childStepRun);
+
+      // Note: Input data filtering will be handled via formIdMapping at execution time.
+      // Each child will receive filtered parent data based on the specific group instance.
+    }
+
+    return okResult({ childStepRuns });
+  } catch (error) {
+    console.error("[createChildRunsFromGroups] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get the submission group that triggered a specific step run
+ */
+export async function getGroupForStepRun({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: { run: true, step: true },
+    });
+
+    if (
+      !runStep ||
+      runStep.run.organizationId !== member.organizationId ||
+      !runStep.submissionGroupId
+    ) {
+      console.error("[getGroupForStepRun] No step associated with run step:", runStepId);
+      return notFoundResult; // No step associated
+    }
+
+    // Find the unique submissionGroup in the output that matches the formGroupId
+    const group = await db.query.submissionGroup.findFirst({
+      with: {
+        formGroup: true,
+        items: {
+          with: {
+            formField: true,
+            versions: {
+              orderBy: [
+                desc(schema.submissionItemVersion.selected),
+                desc(schema.submissionItemVersion.updatedAt),
+              ],
+              with: { documentSource: true },
+              limit: 1,
+            },
+          },
+        },
+        derived: {
+          with: {
+            formDerived: true,
+          },
+        },
+      },
+      where: eq(schema.submissionGroup.id, runStep.submissionGroupId),
+    });
+    if (!group) {
+      return notFoundResult;
+    }
+
+    const { formGroup, items, derived, ...groupData } = group;
+    const groupDenorm = {
+      ...groupData,
+      items: items.map(({ formField, versions, ...itemData }) => ({
+        ...itemData,
+        name: formField.name,
+        type: formField.type,
+        hint: formField.hint,
+        value: versions.at(0)?.value ?? null,
+      })),
+      derived: derived.map(({ formDerived, ...derivedData }) => ({
+        ...derivedData,
+        name: formDerived.name,
+        prompt: formDerived.prompt,
+      })),
+      name: formGroup.name,
+      hint: formGroup.hint,
+      multi: formGroup.multi,
+    };
+
+    return okResult(groupDenorm);
+  } catch (error) {
+    console.error("[getGroupForStepRun] Error:", error);
+    return notFoundResult;
+  }
+}
+
+// *****************************************
+// Data Sources: Request Calls & Manual Inputs
+// *****************************************
+
+/**
+ * Log an API request call made by an integration step
+ */
+export async function logRequestCall({
+  member,
+  runStepId,
+  method,
+  url,
+  requestHeaders = {},
+  requestBody,
+  statusCode,
+  responseHeaders,
+  responseBody,
+  durationMs,
+  error,
+  submissionItemVersionId,
+}: {
+  member: Member;
+  runStepId: string;
+  method: string;
+  url: string;
+  requestHeaders?: Record<string, string>;
+  requestBody?: unknown;
+  statusCode?: number;
+  responseHeaders?: Record<string, string>;
+  responseBody?: unknown;
+  durationMs?: number;
+  error?: string;
+  submissionItemVersionId?: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: { run: true },
+    });
+
+    if (!runStep || runStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    const [requestCall] = await db
+      .insert(schema.pipelineRunIntegrationRequest)
+      .values({
+        runStepId,
+        submissionItemVersionId,
+        method,
+        url,
+        requestHeaders,
+        requestBody,
+        statusCode,
+        responseHeaders,
+        responseBody,
+        durationMs,
+        error,
+      })
+      .returning();
+
+    if (!requestCall) return notFoundResult;
+    return okResult(requestCall);
+  } catch (error) {
+    console.error("[logRequestCall] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get all request calls made by a step run
+ */
+export async function getRequestCallsForStepRun({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: { run: true },
+    });
+
+    if (!runStep || runStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    const requestCalls = await db.query.pipelineRunIntegrationRequest.findMany({
+      where: eq(schema.pipelineRunIntegrationRequest.runStepId, runStepId),
+      orderBy: (calls, { desc }) => [desc(calls.createdAt)],
+    });
+
+    return okResult(requestCalls);
+  } catch (error) {
+    console.error("[getRequestCallsForStepRun] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get a specific request call by ID
+ */
+export async function getRequestCall({
+  member,
+  requestCallId,
+}: {
+  member: Member;
+  requestCallId: string;
+}) {
+  try {
+    const requestCall = await db.query.pipelineRunIntegrationRequest.findFirst({
+      where: eq(schema.pipelineRunIntegrationRequest.id, requestCallId),
+      with: {
+        runStep: {
+          with: {
+            run: true,
+            step: true,
+          },
+        },
+      },
+    });
+
+    if (!requestCall || requestCall.runStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    return okResult(requestCall);
+  } catch (error) {
+    console.error("[getRequestCall] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Remove a manual input from a step run
+ */
+export async function removeManualInput({
+  member,
+  manualInputId,
+}: {
+  member: Member;
+  manualInputId: string;
+}) {
+  try {
+    // TODO: Implement manual input table
+    // const manualInput = await db.query.pipelineRunStepManualInput.findFirst({
+    //   where: eq(schema.pipelineRunStepManualInput.id, manualInputId),
+    //   with: {
+    //     runStep: {
+    //       with: { run: true },
+    //     },
+    //   },
+    // });
+
+    // if (!manualInput || manualInput.runStep.run.organizationId !== member.organizationId) {
+    //   return notFoundResult;
+    // }
+
+    // await db
+    //   .delete(schema.pipelineRunStepManualInput)
+    //   .where(eq(schema.pipelineRunStepManualInput.id, manualInputId));
+
+    // return okResult({ success: true });
+    return notFoundResult;
+  } catch (error) {
+    console.error("[removeManualInput] Error:", error);
+    return notFoundResult;
+  }
+}
+
+// *****************************************
+// Step Dependency Management
+// *****************************************
+
+/**
+ * Add a parent dependency to a step
+ */
+export async function addStepDependency({
+  member,
+  childStepId,
+  parentStepId,
+  multiFormGroupId,
+}: {
+  member: Member;
+  childStepId: string;
+  parentStepId: string;
+  multiFormGroupId?: string | null;
+}) {
+  try {
+    // Verify both steps exist and belong to the member's organization
+    const childStep = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, childStepId),
+      with: { pipeline: true },
+    });
+
+    if (!childStep || childStep.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    const parentStep = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, parentStepId),
+      with: { pipeline: true },
+    });
+
+    if (!parentStep || parentStep.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Check if they're in the same pipeline
+    if (childStep.pipelineId !== parentStep.pipelineId) {
+      return {
+        ok: false,
+        error: "validation-error",
+        message: "Both steps must be in the same pipeline",
+      };
+    }
+
+    // Check if dependency already exists
+    const existing = await db.query.pipelineStepDependency.findFirst({
+      where: and(
+        eq(schema.pipelineStepDependency.childStepId, childStepId),
+        eq(schema.pipelineStepDependency.parentStepId, parentStepId),
+      ),
+    });
+
+    if (existing) {
+      return {
+        ok: false,
+        error: "validation-error",
+        message: "Dependency already exists",
+      };
+    }
+
+    // Create the dependency
+    const [dependency] = await db
+      .insert(schema.pipelineStepDependency)
+      .values({
+        id: nanoid(),
+        childStepId,
+        parentStepId,
+        multiFormGroupId: multiFormGroupId ?? null,
+      })
+      .returning();
+
+    if (!dependency) return internalErrorResult;
+
+    return okResult({
+      id: dependency.id,
+      childStepId: dependency.childStepId,
+      parentStepId: dependency.parentStepId,
+      multiFormGroupId: dependency.multiFormGroupId,
+    });
+  } catch (error) {
+    console.error("[addStepDependency] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Remove a step dependency
+ */
+export async function removeStepDependency({
+  member,
+  dependencyId,
+}: {
+  member: Member;
+  dependencyId: string;
+}) {
+  try {
+    const dependency = await db.query.pipelineStepDependency.findFirst({
+      where: eq(schema.pipelineStepDependency.id, dependencyId),
+      with: {
+        childStep: {
+          with: { pipeline: true },
+        },
+      },
+    });
+
+    if (!dependency || dependency.childStep.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    await db
+      .delete(schema.pipelineStepDependency)
+      .where(eq(schema.pipelineStepDependency.id, dependencyId));
+
+    return okResult({ success: true });
+  } catch (error) {
+    console.error("[removeStepDependency] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+export async function getStepDependenciesByParentStepId({
+  member,
+  parentStepId,
+}: {
+  member: Member;
+  parentStepId: string;
+}) {
+  try {
+    const parentStep = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, parentStepId),
+      with: { pipeline: true },
+    });
+
+    if (!parentStep || parentStep.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    const dependencies = await db.query.pipelineStepDependency.findMany({
+      where: eq(schema.pipelineStepDependency.parentStepId, parentStepId),
+      with: {
+        childStep: true,
+      },
+    });
+
+    return okResult(dependencies);
+  } catch (error) {
+    console.error("[getStepDependenciesByParentStepId] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+// *****************************************
+// Document Management (Extraction Steps)
+// *****************************************
+
+/**
+ * Get documents linked to extraction step run
+ */
+export async function getStepDocuments({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: {
+        run: {
+          with: { pipeline: true },
+        },
+        documents: true,
+      },
+    });
+
+    if (!runStep || runStep.run.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Load full document records
+    const documentIds = runStep.documents.map((d) => d.documentId);
+    if (documentIds.length === 0) {
+      return okResult([]);
+    }
+
+    const documents = await db.query.document.findMany({
+      where: inArray(schema.document.id, documentIds),
+    });
+
+    return okResult(documents);
+  } catch (error) {
+    console.error("[getStepDocuments] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Link document to extraction step run
+ */
+export async function addStepDocument({
+  member,
+  runStepId,
+  documentId,
+}: {
+  member: Member;
+  runStepId: string;
+  documentId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: {
+        run: {
+          with: { pipeline: true },
+        },
+      },
+    });
+
+    if (!runStep || runStep.run.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Verify document exists and belongs to organization
+    const document = await db.query.document.findFirst({
+      where: eq(schema.document.id, documentId),
+      with: {
+        deal: true,
+      },
+    });
+
+    if (!document || document.deal.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Check if already linked
+    const existing = await db.query.pipelineRunStepDocument.findFirst({
+      where: and(
+        eq(schema.pipelineRunStepDocument.runStepId, runStepId),
+        eq(schema.pipelineRunStepDocument.documentId, documentId),
+      ),
+    });
+
+    if (existing) {
+      return okResult({ success: true }); // Already linked, no-op
+    }
+
+    await db.insert(schema.pipelineRunStepDocument).values({
+      id: nanoid(),
+      runStepId,
+      documentId,
+    });
+
+    return okResult({ success: true });
+  } catch (error) {
+    console.error("[addStepDocument] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Unlink document from extraction step run
+ */
+export async function removeStepDocument({
+  member,
+  runStepId,
+  documentId,
+}: {
+  member: Member;
+  runStepId: string;
+  documentId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: {
+        run: {
+          with: { pipeline: true },
+        },
+      },
+    });
+
+    if (!runStep || runStep.run.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    await db
+      .delete(schema.pipelineRunStepDocument)
+      .where(
+        and(
+          eq(schema.pipelineRunStepDocument.runStepId, runStepId),
+          eq(schema.pipelineRunStepDocument.documentId, documentId),
+        ),
+      );
+
+    return okResult({ success: true });
+  } catch (error) {
+    console.error("[removeStepDocument] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+// *****************************************
+// Integration Config Management
+// *****************************************
+
+/**
+ * Get integration config for step
+ */
+export async function getStepIntegrationConfig({
+  member,
+  stepId,
+}: {
+  member: Member;
+  stepId: string;
+}): Promise<DalResult<PipelineStepIntegrationConfig | null>> {
+  try {
+    const step = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, stepId),
+      with: {
+        pipeline: true,
+        integrationConfig: true,
+      },
+    });
+
+    if (!step || step.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    return okResult(step.integrationConfig);
+  } catch (error) {
+    console.error("[getStepIntegrationConfig] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Create/update integration config for step
+ */
+export async function upsertStepIntegrationConfig({
+  member,
+  stepId,
+  config,
+}: {
+  member: Member;
+  stepId: string;
+  config: {
+    method: string;
+    urlTemplate: string;
+    headers: Record<string, string>;
+    bodyTemplate?: string;
+  };
+}): Promise<DalResult<PipelineStepIntegrationConfig>> {
+  try {
+    const step = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, stepId),
+      with: {
+        pipeline: true,
+        integrationConfig: true,
+      },
+    });
+
+    if (!step || step.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    if (step.integrationConfig) {
+      // Update existing
+      const [updated] = await db
+        .update(schema.pipelineStepIntegrationConfig)
+        .set({
+          method: config.method,
+          urlTemplate: config.urlTemplate,
+          headers: config.headers,
+          bodyTemplate: config.bodyTemplate,
+          updatedAt: new Date(),
+        })
+        .where(eq(schema.pipelineStepIntegrationConfig.stepId, stepId))
+        .returning();
+
+      if (!updated) {
+        return internalErrorResult;
+      }
+
+      return okResult(updated);
+    } else {
+      // Create new
+      const [created] = await db
+        .insert(schema.pipelineStepIntegrationConfig)
+        .values({
+          id: nanoid(),
+          stepId,
+          method: config.method,
+          urlTemplate: config.urlTemplate,
+          headers: config.headers,
+          bodyTemplate: config.bodyTemplate,
+        })
+        .returning();
+
+      if (!created) {
+        return internalErrorResult;
+      }
+
+      return okResult(created);
+    }
+  } catch (error) {
+    console.error("[upsertStepIntegrationConfig] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Delete integration config
+ */
+export async function deleteStepIntegrationConfig({
+  member,
+  stepId,
+}: {
+  member: Member;
+  stepId: string;
+}): Promise<DalResult<{ success: true }>> {
+  try {
+    const step = await db.query.pipelineStep.findFirst({
+      where: eq(schema.pipelineStep.id, stepId),
+      with: {
+        pipeline: true,
+      },
+    });
+
+    if (!step || step.pipeline.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    await db
+      .delete(schema.pipelineStepIntegrationConfig)
+      .where(eq(schema.pipelineStepIntegrationConfig.stepId, stepId));
+
+    return okResult({ success: true });
+  } catch (error) {
+    console.error("[deleteStepIntegrationConfig] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+// *****************************************
+// Recursive Citation Chain
+// *****************************************
+
+export type CitationNode =
+  | {
+      type: "document";
+      documentCitationId: string;
+      documentExtractionId: string;
+      documentId: string;
+      documentName: string;
+      path: string;
+      text: string;
+      sourceRef: string;
+    }
+  | {
+      type: "integration";
+      requestCallId: string;
+      method: string;
+      url: string;
+      statusCode: number | null;
+      durationMs: number | null;
+    }
+  | {
+      type: "transformation";
+      submissionItemVersionId: string;
+      sourceItemVersionIds: string[];
+      children: CitationNode[];
+    }
+  | {
+      type: "manual";
+      submissionItemVersionId: string;
+      value: string | number | null;
+    };
+
+/**
+ * Recursively walk the citation chain for a submission item version.
+ * Returns a tree showing all source data until reaching leaf nodes (documents or API calls).
+ *
+ * Example output:
+ * {
+ *   type: "transformation",
+ *   children: [
+ *     { type: "document", documentName: "Q1 Report.pdf", page: 3 },
+ *     { type: "integration", url: "GET /financials/q2", statusCode: 200 }
+ *   ]
+ * }
+ */
+export async function getCitationChain({
+  member,
+  submissionItemVersionId,
+}: {
+  member: Member;
+  submissionItemVersionId: string;
+}): Promise<DalResult<CitationNode>> {
+  try {
+    // First get the version, then check org access via joins
+    const version = await db.query.submissionItemVersion.findFirst({
+      where: eq(schema.submissionItemVersion.id, submissionItemVersionId),
+    });
+
+    if (!version) return notFoundResult;
+
+    // Verify organization access
+    const item = await db.query.submissionItem.findFirst({
+      where: eq(schema.submissionItem.id, version.submissionItemId),
+    });
+    if (!item) return notFoundResult;
+
+    const group = await db.query.submissionGroup.findFirst({
+      where: eq(schema.submissionGroup.id, item.submissionGroupId),
+    });
+    if (!group) return notFoundResult;
+
+    const submission = await db.query.submission.findFirst({
+      where: eq(schema.submission.id, group.submissionId),
+    });
+    if (!submission || submission.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    const node = await buildCitationNode(version);
+    return okResult(node);
+  } catch (error) {
+    console.error("[getCitationChain] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Recursive helper to build a citation node tree
+ */
+async function buildCitationNode(
+  version: typeof schema.submissionItemVersion.$inferSelect,
+): Promise<CitationNode> {
+  switch (version.sourceType) {
+    case "document": {
+      if (!version.sourceId) {
+        throw new Error("Document source missing sourceId");
+      }
+
+      const citation = await db.query.documentCitation.findFirst({
+        where: eq(schema.documentCitation.id, version.sourceId),
+      });
+
+      if (!citation) {
+        throw new Error(`Document citation ${version.sourceId} not found`);
+      }
+
+      const extraction = await db.query.documentExtraction.findFirst({
+        where: eq(schema.documentExtraction.id, citation.documentExtractionId),
+      });
+
+      const document = extraction
+        ? await db.query.document.findFirst({
+            where: eq(schema.document.id, extraction.documentId),
+          })
+        : null;
+
+      return {
+        type: "document",
+        documentCitationId: citation.id,
+        documentExtractionId: citation.documentExtractionId,
+        documentId: document?.id || "",
+        documentName: document?.name || "Unknown Document",
+        path: citation.path,
+        text: citation.text,
+        sourceRef: citation.sourceRef,
+      };
+    }
+
+    case "integration": {
+      if (!version.sourceId) {
+        throw new Error("Integration source missing sourceId");
+      }
+
+      const requestCall = await db.query.pipelineRunIntegrationRequest.findFirst({
+        where: eq(schema.pipelineRunIntegrationRequest.id, version.sourceId),
+      });
+
+      if (!requestCall) {
+        throw new Error(`Request call ${version.sourceId} not found`);
+      }
+
+      return {
+        type: "integration",
+        requestCallId: requestCall.id,
+        method: requestCall.method,
+        url: requestCall.url,
+        statusCode: requestCall.statusCode,
+        durationMs: requestCall.durationMs,
+      };
+    }
+
+    case "transformation": {
+      // Get the transformation that produced this version using sourceId
+      if (!version.sourceId) {
+        return {
+          type: "transformation",
+          submissionItemVersionId: version.id,
+          sourceItemVersionIds: [],
+          children: [],
+        };
+      }
+
+      const transformation = await db.query.dataTransformation.findFirst({
+        where: eq(schema.dataTransformation.id, version.sourceId),
+      });
+
+      if (!transformation || !transformation.sourceItemVersionIds) {
+        return {
+          type: "transformation",
+          submissionItemVersionId: version.id,
+          sourceItemVersionIds: [],
+          children: [],
+        };
+      }
+
+      // Recursively get source versions
+      const sourceVersions = await db.query.submissionItemVersion.findMany({
+        where: sql`${schema.submissionItemVersion.id} = ANY(ARRAY[${sql.join(
+          transformation.sourceItemVersionIds.map((id) => sql`${id}`),
+          sql`, `,
+        )}]::text[])`,
+      });
+
+      const children = await Promise.all(sourceVersions.map((sv) => buildCitationNode(sv)));
+
+      return {
+        type: "transformation",
+        submissionItemVersionId: version.id,
+        sourceItemVersionIds: transformation.sourceItemVersionIds,
+        children,
+      };
+    }
+
+    case "manual": {
+      return {
+        type: "manual",
+        submissionItemVersionId: version.id,
+        value: version.value,
+      };
+    }
+
+    case "merged":
+    case "message":
+    default: {
+      // For other types, return as transformation with no children for now
+      return {
+        type: "transformation",
+        submissionItemVersionId: version.id,
+        sourceItemVersionIds: [],
+        children: [],
+      };
+    }
+  }
+}
+
+/**
+ * Get all leaf citations (documents and API calls) for a submission item version.
+ * Flattens the recursive tree into a list of leaf nodes.
+ */
+export async function getLeafCitations({
+  member,
+  submissionItemVersionId,
+}: {
+  member: Member;
+  submissionItemVersionId: string;
+}) {
+  try {
+    const chainResult = await getCitationChain({ member, submissionItemVersionId });
+    if (!chainResult.ok) return chainResult;
+
+    const leaves: CitationNode[] = [];
+
+    function collectLeaves(node: CitationNode) {
+      if (node.type === "document" || node.type === "integration" || node.type === "manual") {
+        leaves.push(node);
+      } else if (node.type === "transformation") {
+        for (const child of node.children) {
+          collectLeaves(child);
+        }
+      }
+    }
+
+    collectLeaves(chainResult.data);
+    return okResult(leaves);
+  } catch (error) {
+    console.error("[getLeafCitations] Error:", error);
+    return notFoundResult;
+  }
+}
+
+// ========================================================================================
+// Blocker Management
+// ========================================================================================
+
+/**
+ * Create a pipeline run with pre-flight validation and blocker creation.
+ * Checks all steps for potential blockers before execution begins.
+ */
+export async function createPipelineRunWithBlockers({
+  member,
+  pipelineId,
+  dealId,
+  inputDocumentIds = [],
+  inputSubmissionIds = [],
+  triggerType = "manual",
+}: {
+  member: Member;
+  pipelineId: string;
+  dealId: string;
+  inputDocumentIds?: string[];
+  inputSubmissionIds?: string[];
+  triggerType?: string;
+}): Promise<
+  DalResult<{ runId: string; blockers: Array<{ stepId: string; type: string; details: object }> }>
+> {
+  try {
+    // First create the run using existing function
+    const runResult = await createPipelineRun({
+      member,
+      pipelineId,
+      dealId,
+      triggerType,
+    });
+
+    if (!runResult.ok) return runResult;
+    const run = runResult.data;
+
+    // Get pipeline with steps
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+      with: {
+        steps: {
+          orderBy: asc(schema.pipelineStep.createdAt),
+          with: {
+            outputForms: true,
+            integrationConfig: true,
+          },
+        },
+      },
+    });
+
+    if (!pipeline) return notFoundResult;
+
+    const blockers: Array<{ stepId: string; type: string; details: object }> = [];
+
+    // Get all dependencies for all steps
+    const allDependencies = await db.query.pipelineStepDependency.findMany({
+      where: inArray(
+        schema.pipelineStepDependency.childStepId,
+        pipeline.steps.map((s) => s.id),
+      ),
+    });
+
+    // Group dependencies by child step
+    const dependenciesByStep = new Map<string, typeof allDependencies>();
+    for (const dep of allDependencies) {
+      const existing = dependenciesByStep.get(dep.childStepId) || [];
+      existing.push(dep);
+      dependenciesByStep.set(dep.childStepId, existing);
+    }
+
+    // Create run steps for all pipeline steps
+    const runSteps = await Promise.all(
+      pipeline.steps.map(async (step) => {
+        const runStepResult = await createPipelineRunStep({
+          member,
+          runId: run.id,
+          stepId: step.id,
+        });
+        return runStepResult.ok ? runStepResult.data : null;
+      }),
+    );
+
+    // Check each step for blockers
+    for (let i = 0; i < pipeline.steps.length; i++) {
+      const step = pipeline.steps[i];
+      const runStep = runSteps[i];
+
+      if (!runStep || !step) continue;
+
+      // Check extraction steps have input documents
+      if (step.role === "extraction") {
+        if (inputDocumentIds.length === 0 && inputSubmissionIds.length === 0) {
+          await createStepRunBlocker({
+            runStepId: runStep.id,
+            blockerType: "missing_upload",
+            blockerDetails: {
+              type: "missing_upload",
+              message: "No input documents or submissions provided",
+            },
+          });
+          blockers.push({
+            stepId: step.id,
+            type: "missing_upload",
+            details: { message: "No input documents or submissions provided" },
+          });
+        }
+      }
+
+      // Check integration steps have config
+      if (step.role === "integration" && !step.integrationConfig) {
+        await createStepRunBlocker({
+          runStepId: runStep.id,
+          blockerType: "missing_config",
+          blockerDetails: {
+            type: "missing_config",
+            message: "Integration step missing configuration",
+          },
+        });
+        blockers.push({
+          stepId: step.id,
+          type: "missing_config",
+          details: { message: "Integration step missing configuration" },
+        });
+      }
+
+      // Check analysis steps have output form
+      if ((step.role === "analysis" || step.role === null) && step.outputForms.length === 0) {
+        await createStepRunBlocker({
+          runStepId: runStep.id,
+          blockerType: "missing_config",
+          blockerDetails: { type: "missing_config", message: "Analysis step missing output form" },
+        });
+        blockers.push({
+          stepId: step.id,
+          type: "missing_config",
+          details: { message: "Analysis step missing output form" },
+        });
+      }
+
+      // Check manual steps have output forms and create blockers for each required form
+      if (step.role === "manual") {
+        if (step.outputForms.length === 0) {
+          await createStepRunBlocker({
+            runStepId: runStep.id,
+            blockerType: "missing_config",
+            blockerDetails: {
+              type: "missing_config",
+              message: "Manual step missing output form configuration",
+            },
+          });
+          blockers.push({
+            stepId: step.id,
+            type: "missing_config",
+            details: { message: "Manual step missing output form configuration" },
+          });
+        } else {
+          // Create a blocker for each required output form
+          const forms = await db.query.form.findMany({
+            where: inArray(
+              schema.form.id,
+              step.outputForms.map((f) => f.formId),
+            ),
+          });
+          const formIdToName = new Map(forms.map((f) => [f.id, f.name]));
+
+          for (const outputForm of step.outputForms) {
+            await createStepRunBlocker({
+              runStepId: runStep.id,
+              blockerType: "missing_manual_input",
+              blockerDetails: {
+                type: "missing_manual_input",
+                stepName: step.name,
+                stepDescription: step.description,
+                requiredFormId: outputForm.formId,
+                requiredFormName: formIdToName.get(outputForm.formId) || outputForm.formId,
+              },
+            });
+            blockers.push({
+              stepId: step.id,
+              type: "missing_manual_input",
+              details: {
+                stepName: step.name,
+                stepDescription: step.description,
+                requiredFormId: outputForm.formId,
+                requiredFormName: formIdToName.get(outputForm.formId) || outputForm.formId,
+              },
+            });
+          }
+        }
+      }
+
+      // Mark steps with dependencies as blocked by parent_incomplete
+      const stepDependencies = dependenciesByStep.get(step.id) || [];
+      if (stepDependencies.length > 0) {
+        await createStepRunBlocker({
+          runStepId: runStep.id,
+          blockerType: "parent_incomplete",
+          blockerDetails: {
+            type: "parent_incomplete",
+            parentStepIds: stepDependencies.map((d) => d.parentStepId),
+            message: "Waiting for parent steps to complete",
+          },
+        });
+        blockers.push({
+          stepId: step.id,
+          type: "parent_incomplete",
+          details: {
+            type: "parent_incomplete",
+            parentStepIds: stepDependencies.map((d) => d.parentStepId),
+            message: "Waiting for parent steps to complete",
+          },
+        });
+      }
+    }
+
+    return okResult({ runId: run.id, blockers });
+  } catch (error) {
+    console.error("[createPipelineRunWithBlockers] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Sync blockers for an existing pipeline run
+ * Removes outdated blockers and creates new ones based on current state
+ */
+export async function syncPipelineRunBlockers({
+  member,
+  runId,
+}: {
+  member: Member;
+  runId: string;
+}) {
+  try {
+    // Get the run with pipeline and steps
+    const run = await db.query.pipelineRun.findFirst({
+      where: and(
+        eq(schema.pipelineRun.id, runId),
+        eq(schema.pipelineRun.organizationId, member.organizationId),
+      ),
+      with: {
+        pipeline: {
+          with: {
+            steps: {
+              orderBy: asc(schema.pipelineStep.createdAt),
+              with: {
+                outputForms: true,
+                integrationConfig: true,
+              },
+            },
+          },
+        },
+        steps: {
+          with: {
+            documents: true,
+          },
+        },
+      },
+    });
+
+    if (!run) return notFoundResult;
+
+    // Get all dependencies for pipeline steps
+    const allDependencies = await db.query.pipelineStepDependency.findMany({
+      where: inArray(
+        schema.pipelineStepDependency.childStepId,
+        run.pipeline.steps.map((s) => s.id),
+      ),
+    });
+
+    // Group dependencies by child step
+    const dependenciesByStep = new Map<string, typeof allDependencies>();
+    for (const dep of allDependencies) {
+      const existing = dependenciesByStep.get(dep.childStepId) || [];
+      existing.push(dep);
+      dependenciesByStep.set(dep.childStepId, existing);
+    }
+
+    // TODO: Get existing run submissions once schema is finalized
+    const hasInputs = false;
+    let syncedCount = 0;
+
+    // Delete all existing unresolved blockers (we'll recreate them if still needed)
+    await db.delete(schema.pipelineRunStepBlocker).where(
+      and(
+        inArray(
+          schema.pipelineRunStepBlocker.runStepId,
+          run.steps.map((s) => s.id),
+        ),
+        eq(schema.pipelineRunStepBlocker.resolved, false),
+      ),
+    );
+
+    // Check each run step for blockers
+    for (const runStep of run.steps) {
+      const step = run.pipeline.steps.find((s) => s.id === runStep.stepId);
+      if (!step) continue;
+
+      // Skip completed or failed steps
+      if (
+        runStep.status === PIPELINE_RUN_STEP_STATUS.COMPLETED ||
+        runStep.status === PIPELINE_RUN_STEP_STATUS.FAILED
+      ) {
+        continue;
+      }
+
+      // Check extraction steps have input documents
+      if (step.role === "extraction" && runStep.documents.length === 0) {
+        await createStepRunBlocker({
+          runStepId: runStep.id,
+          blockerType: "missing_upload",
+          blockerDetails: {
+            type: "missing_upload",
+            message: "No documents linked to extraction step",
+          },
+        });
+        syncedCount++;
+      }
+
+      // Check integration steps have config
+      if (step.role === "integration" && !step.integrationConfig) {
+        await createStepRunBlocker({
+          runStepId: runStep.id,
+          blockerType: "missing_config",
+          blockerDetails: {
+            type: "missing_config",
+            message: "Integration step missing configuration",
+          },
+        });
+        syncedCount++;
+      }
+
+      // Check analysis steps have output form
+      if ((step.role === "analysis" || step.role === null) && step.outputForms.length === 0) {
+        await createStepRunBlocker({
+          runStepId: runStep.id,
+          blockerType: "missing_config",
+          blockerDetails: { type: "missing_config", message: "Analysis step missing output form" },
+        });
+        syncedCount++;
+      }
+
+      // Check manual steps have output submissions for all required forms
+      if (step.role === "manual") {
+        if (step.outputForms.length === 0) {
+          await createStepRunBlocker({
+            runStepId: runStep.id,
+            blockerType: "missing_config",
+            blockerDetails: {
+              type: "missing_config",
+              message: "Manual step missing output form configuration",
+            },
+          });
+          syncedCount++;
+        } else {
+          // Get existing output submissions for this run step
+          const outputSubmissions = await db.query.pipelineRunStepOutputSubmission.findMany({
+            where: eq(schema.pipelineRunStepOutputSubmission.runStepId, runStep.id),
+          });
+
+          // Load submission forms for all outputs
+          const outputSubmissionDetails = await Promise.all(
+            outputSubmissions.map(async (output) => {
+              const submission = await db.query.submission.findFirst({
+                where: eq(schema.submission.id, output.submissionId),
+              });
+              return { ...output, formId: submission?.formId };
+            }),
+          );
+
+          // Load form names
+          const forms = await db.query.form.findMany({
+            where: inArray(
+              schema.form.id,
+              step.outputForms.map((f) => f.formId),
+            ),
+          });
+          const formIdToName = new Map(forms.map((f) => [f.id, f.name]));
+
+          // Create a blocker for each required form that doesn't have a submission
+          for (const outputForm of step.outputForms) {
+            const hasSubmissionForForm = outputSubmissionDetails.some(
+              (output) => output.formId === outputForm.formId,
+            );
+
+            if (!hasSubmissionForForm) {
+              await createStepRunBlocker({
+                runStepId: runStep.id,
+                blockerType: "missing_manual_input",
+                blockerDetails: {
+                  type: "missing_manual_input",
+                  stepName: step.name,
+                  stepDescription: step.description,
+                  requiredFormId: outputForm.formId,
+                  requiredFormName: formIdToName.get(outputForm.formId) || outputForm.formId,
+                },
+              });
+              syncedCount++;
+            }
+          }
+        }
+      }
+
+      // Mark steps with dependencies as blocked by parent_incomplete
+      const stepDependencies = dependenciesByStep.get(step.id) || [];
+      if (stepDependencies.length > 0) {
+        // Check if any parent steps are incomplete
+        const parentRunSteps = run.steps.filter((rs) =>
+          stepDependencies.some((d) => d.parentStepId === rs.stepId),
+        );
+        const hasIncompleteParents = parentRunSteps.some(
+          (ps) =>
+            ps.status !== PIPELINE_RUN_STEP_STATUS.COMPLETED &&
+            ps.status !== PIPELINE_RUN_STEP_STATUS.FAILED,
+        );
+
+        if (hasIncompleteParents) {
+          await createStepRunBlocker({
+            runStepId: runStep.id,
+            blockerType: "parent_incomplete",
+            blockerDetails: {
+              type: "parent_incomplete",
+              parentStepIds: stepDependencies.map((d) => d.parentStepId),
+              message: "Waiting for parent steps to complete",
+            },
+          });
+          syncedCount++;
+        }
+      }
+    }
+
+    return okResult({ blockersSynced: syncedCount });
+  } catch (error) {
+    console.error("[syncPipelineRunBlockers] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Resolve blockers for a step when it completes successfully
+ */
+export async function resolveStepBlockersOnCompletion({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}) {
+  try {
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: {
+        run: true,
+        step: true,
+      },
+    });
+
+    if (!runStep || runStep.run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    let resolvedCount = 0;
+
+    console.log(`[resolveStepBlockersOnCompletion] Starting for runStepId: ${runStepId}`);
+    console.log(`[resolveStepBlockersOnCompletion] Run step details:`, {
+      runStepId: runStep.id,
+      stepId: runStep.stepId,
+      status: runStep.status,
+    });
+
+    // 1. Resolve this step's own blockers
+    const ownBlockers = await db
+      .update(schema.pipelineRunStepBlocker)
+      .set({
+        resolved: true,
+        resolvedAt: new Date(),
+      })
+      .where(
+        and(
+          eq(schema.pipelineRunStepBlocker.runStepId, runStepId),
+          eq(schema.pipelineRunStepBlocker.resolved, false),
+        ),
+      )
+      .returning({ id: schema.pipelineRunStepBlocker.id });
+
+    resolvedCount += ownBlockers.length;
+    console.log(`[resolveStepBlockersOnCompletion] Resolved ${ownBlockers.length} own blocker(s)`);
+
+    // 2. Find child steps and check if their parent_incomplete blockers can be resolved
+    console.log(
+      `[resolveStepBlockersOnCompletion] Looking for child steps with parentStepId: ${runStep.stepId}`,
+    );
+
+    const childStepRuns = await db
+      .select({
+        id: schema.pipelineRunStep.id,
+        stepId: schema.pipelineRunStep.stepId,
+      })
+      .from(schema.pipelineStepDependency)
+      .innerJoin(
+        schema.pipelineRunStep,
+        and(
+          eq(schema.pipelineStepDependency.childStepId, schema.pipelineRunStep.stepId),
+          eq(schema.pipelineRunStep.runId, runStep.runId),
+        ),
+      )
+      .where(eq(schema.pipelineStepDependency.parentStepId, runStep.stepId!));
+
+    console.log(
+      `[resolveStepBlockersOnCompletion] Found ${childStepRuns.length} child run step(s)`,
+    );
+    if (childStepRuns.length > 0) {
+      console.log(
+        `[resolveStepBlockersOnCompletion] Child run steps:`,
+        childStepRuns.map((c) => ({ id: c.id, stepId: c.stepId })),
+      );
+    }
+
+    for (const childStepRun of childStepRuns) {
+      console.log(
+        `[resolveStepBlockersOnCompletion] Checking child run step: ${childStepRun.id} (stepId: ${childStepRun.stepId})`,
+      );
+
+      // Check if all parents of this child are completed
+      const parentStepRuns = await db
+        .select({
+          id: schema.pipelineRunStep.id,
+          status: schema.pipelineRunStep.status,
+        })
+        .from(schema.pipelineStepDependency)
+        .innerJoin(
+          schema.pipelineRunStep,
+          and(
+            eq(schema.pipelineStepDependency.parentStepId, schema.pipelineRunStep.stepId),
+            eq(schema.pipelineRunStep.runId, runStep.runId),
+          ),
+        )
+        .where(eq(schema.pipelineStepDependency.childStepId, childStepRun.stepId!));
+
+      console.log(
+        `[resolveStepBlockersOnCompletion] Found ${parentStepRuns.length} parent run step(s) for child ${childStepRun.id}`,
+      );
+      console.log(
+        `[resolveStepBlockersOnCompletion] Parent statuses:`,
+        parentStepRuns.map((p) => ({ id: p.id, status: p.status })),
+      );
+
+      const allParentsComplete = parentStepRuns.every(
+        (p) => p.status === PIPELINE_RUN_STEP_STATUS.COMPLETED,
+      );
+
+      console.log(`[resolveStepBlockersOnCompletion] All parents complete? ${allParentsComplete}`);
+
+      if (allParentsComplete) {
+        // Resolve parent_incomplete blockers for this child
+        console.log(
+          `[resolveStepBlockersOnCompletion] Resolving parent_incomplete blockers for child ${childStepRun.id}`,
+        );
+        const childBlockers = await db
+          .update(schema.pipelineRunStepBlocker)
+          .set({
+            resolved: true,
+            resolvedAt: new Date(),
+          })
+          .where(
+            and(
+              eq(schema.pipelineRunStepBlocker.runStepId, childStepRun.id),
+              eq(schema.pipelineRunStepBlocker.blockerType, "parent_incomplete"),
+              eq(schema.pipelineRunStepBlocker.resolved, false),
+            ),
+          )
+          .returning({ id: schema.pipelineRunStepBlocker.id });
+
+        console.log(
+          `[resolveStepBlockersOnCompletion] Resolved ${childBlockers.length} blocker(s) for child ${childStepRun.id}`,
+        );
+        resolvedCount += childBlockers.length;
+      }
+    }
+
+    console.log(`[resolveStepBlockersOnCompletion] Total resolved: ${resolvedCount}`);
+    return okResult({ resolvedCount });
+  } catch (error) {
+    console.error("[resolveStepBlockersOnCompletion] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Create a blocker for a pipeline run step
+ */
+export async function createStepRunBlocker<T extends PipelineRunStepBlockerType>(params: {
+  runStepId: string;
+  blockerType: T;
+  blockerDetails: Extract<PipelineRunStepBlockerDetails, { type: T }>;
+}) {
+  try {
+    const [blocker] = await db
+      .insert(schema.pipelineRunStepBlocker)
+      .values({
+        runStepId: params.runStepId,
+        blockerType: params.blockerType,
+        blockerDetails: params.blockerDetails,
+      })
+      .returning({ id: schema.pipelineRunStepBlocker.id });
+
+    if (!blocker) return notFoundResult;
+    return okResult(blocker);
+  } catch (error) {
+    console.error("[createStepRunBlocker] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Resolve a blocker by ID
+ */
+export async function resolveBlocker(blockerId: string) {
+  try {
+    const [updated] = await db
+      .update(schema.pipelineRunStepBlocker)
+      .set({
+        resolved: true,
+        resolvedAt: new Date(),
+      })
+      .where(eq(schema.pipelineRunStepBlocker.id, blockerId))
+      .returning({ id: schema.pipelineRunStepBlocker.id });
+
+    if (!updated) return notFoundResult;
+    return okResult({ success: true });
+  } catch (error) {
+    console.error("[resolveBlocker] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get all blockers for a step run (optionally filter by resolved status)
+ */
+export async function getStepRunBlockers(params: { runStepId: string; resolved?: boolean }) {
+  try {
+    const conditions = [eq(schema.pipelineRunStepBlocker.runStepId, params.runStepId)];
+
+    if (params.resolved !== undefined) {
+      conditions.push(eq(schema.pipelineRunStepBlocker.resolved, params.resolved));
+    }
+
+    const blockers = await db
+      .select({
+        id: schema.pipelineRunStepBlocker.id,
+        blockerType: schema.pipelineRunStepBlocker.blockerType,
+        blockerDetails: schema.pipelineRunStepBlocker.blockerDetails,
+        resolved: schema.pipelineRunStepBlocker.resolved,
+        resolvedAt: schema.pipelineRunStepBlocker.resolvedAt,
+        createdAt: schema.pipelineRunStepBlocker.createdAt,
+      })
+      .from(schema.pipelineRunStepBlocker)
+      .where(and(...conditions));
+
+    return okResult(blockers);
+  } catch (error) {
+    console.error("[getStepRunBlockers] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Check if a step run has any unresolved blockers
+ */
+export async function hasUnresolvedBlockers(runStepId: string) {
+  try {
+    const [result] = await db
+      .select({ count: count() })
+      .from(schema.pipelineRunStepBlocker)
+      .where(
+        and(
+          eq(schema.pipelineRunStepBlocker.runStepId, runStepId),
+          eq(schema.pipelineRunStepBlocker.resolved, false),
+        ),
+      );
+
+    return okResult((result?.count ?? 0) > 0);
+  } catch (error) {
+    console.error("[hasUnresolvedBlockers] Error:", error);
+    return notFoundResult;
+  }
+}
+
+/**
+ * Get all blockers for all steps in a pipeline run, grouped by step.
+ * Only returns level 0 blockers (user-actionable) - filters out parent_incomplete
+ * since those are automatically resolved when dependencies complete.
+ */
+export async function getAllBlockersForRun({ member, runId }: { member: Member; runId: string }) {
+  try {
+    // Verify run belongs to member's organization
+    const run = await db.query.pipelineRun.findFirst({
+      where: eq(schema.pipelineRun.id, runId),
+    });
+
+    if (!run || run.organizationId !== member.organizationId) {
+      return notFoundResult;
+    }
+
+    // Get all run steps with their blockers (exclude templates - they're just placeholders)
+    const runSteps = await db.query.pipelineRunStep.findMany({
+      where: and(
+        eq(schema.pipelineRunStep.runId, runId),
+        // Filter out template steps - only show normal and instance steps
+        or(
+          eq(schema.pipelineRunStep.spawnType, "normal"),
+          eq(schema.pipelineRunStep.spawnType, "instance"),
+        ),
+      ),
+      with: {
+        step: true,
+        blockers: {
+          orderBy: [desc(schema.pipelineRunStepBlocker.createdAt)],
+        },
+      },
+      orderBy: [asc(schema.pipelineRunStep.createdAt)],
+    });
+
+    const result = runSteps.map((runStep) => ({
+      runStepId: runStep.id,
+      stepName: runStep.step?.name ?? "Unknown Step",
+      stepOrder: runStep.stepRunIndex ?? 0,
+      // Filter out parent_incomplete blockers - they're not user-actionable
+      // and are automatically resolved when dependencies complete
+      blockers: runStep.blockers
+        .filter((blocker) => blocker.blockerType !== "parent_incomplete")
+        .map((blocker) => ({
+          id: blocker.id,
+          blockerType: blocker.blockerType,
+          blockerDetails: blocker.blockerDetails,
+          resolved: blocker.resolved,
+          resolvedAt: blocker.resolvedAt,
+          createdAt: blocker.createdAt,
+        })),
+    }));
+
+    return okResult(result);
+  } catch (error) {
+    console.error("[getAllBlockersForRun] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Get a pipeline by ID with basic fields (for verification)
+ */
+export async function getPipelineById({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}) {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+      columns: {
+        id: true,
+        name: true,
+        description: true,
+        organizationId: true,
+      },
+    });
+
+    if (!pipeline) {
+      return notFoundResult;
+    }
+
+    return okResult(pipeline);
+  } catch (error) {
+    console.error("[getPipelineById] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Get a pipeline with its input forms
+ */
+export async function getPipelineWithInputForms({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}) {
+  try {
+    const pipeline = await db.query.pipeline.findFirst({
+      where: and(
+        eq(schema.pipeline.id, pipelineId),
+        eq(schema.pipeline.organizationId, member.organizationId),
+        isNull(schema.pipeline.deletedAt),
+      ),
+      columns: {
+        id: true,
+        name: true,
+      },
+    });
+
+    if (!pipeline) {
+      return notFoundResult;
+    }
+
+    return okResult(pipeline);
+  } catch (error) {
+    console.error("[getPipelineWithInputForms] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Create an empty pipeline with basic metadata
+ */
+export async function createEmptyPipeline({
+  member,
+  name = "Untitled Pipeline",
+  description = "",
+}: {
+  member: Member;
+  name?: string;
+  description?: string;
+}): Promise<
+  DalResult<{
+    id: string;
+    name: string;
+  }>
+> {
+  try {
+    const [pipeline] = await db
+      .insert(schema.pipeline)
+      .values({
+        organizationId: member.organizationId,
+        name,
+        description,
+        isActive: true,
+        version: "1",
+        metadata: {},
+      })
+      .returning({ id: schema.pipeline.id, name: schema.pipeline.name });
+    invariant(pipeline);
+
+    return okResult(pipeline);
+  } catch (error) {
+    console.error("[createEmptyPipeline] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Create a complete pipeline with first step and conversation (transaction)
+ */
+export async function createPipelineWithStepAndConversation({
+  member,
+  name,
+  description,
+  stepPrompt,
+}: {
+  member: Member;
+  name: string;
+  description: string;
+  stepPrompt: string;
+}) {
+  try {
+    const created = await db.transaction(async (tx) => {
+      const [pipeline] = await tx
+        .insert(schema.pipeline)
+        .values({
+          organizationId: member.organizationId,
+          name,
+          description,
+          metadata: { createdBy: "pipeline_ui" },
+        })
+        .returning();
+      invariant(pipeline);
+
+      // Create conversation for this pipeline
+      const [conversation] = await tx
+        .insert(schema.conversation)
+        .values({
+          organizationId: member.organizationId,
+          agent: true,
+          name: `Pipeline: ${name}`,
+          model: "openai/gpt-5.2",
+          memory: {},
+        })
+        .returning();
+      invariant(conversation);
+
+      // Link conversation to pipeline via conversation_resource
+      await tx.insert(schema.conversationResource).values({
+        conversationId: conversation.id,
+        resourceType: "pipeline",
+        resourceId: pipeline.id,
+        source: "system",
+      });
+
+      // TODO: Create pipeline-form mapping once pipelineForm table is created
+
+      const [step] = await tx
+        .insert(schema.pipelineStep)
+        .values({
+          pipelineId: pipeline.id,
+          name: "Step 1",
+          description,
+          prompt: stepPrompt,
+          toolsAllowed: [],
+          retryPolicy: {},
+        })
+        .returning();
+      invariant(step);
+
+      return pipeline;
+    });
+
+    return okResult({
+      id: created.id,
+      name: created.name,
+      description: created.description,
+    });
+  } catch (error) {
+    console.error("[createPipelineWithStepAndConversation] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+// *****************************************
+// Blocker Resolution Functions
+// *****************************************
+
+/**
+ * Link documents to a pipeline run step.
+ * Used when resolving missing_upload blockers.
+ */
+export async function linkDocumentsToRunStep({
+  member,
+  runStepId,
+  documentIds,
+}: {
+  member: Member;
+  runStepId: string;
+  documentIds: string[];
+}): Promise<DalResult<void>> {
+  try {
+    // Verify the run step exists and belongs to the member's org
+    const runStepResult = await getPipelineRunStep({ member, runStepId });
+    if (!runStepResult.ok) return runStepResult;
+
+    // Insert documents into pipelineRunStepDocument
+    if (documentIds.length > 0) {
+      await db.insert(schema.pipelineRunStepDocument).values(
+        documentIds.map((documentId) => ({
+          runStepId,
+          documentId,
+        })),
+      );
+    }
+
+    return okResult(undefined);
+  } catch (error) {
+    console.error("[linkDocumentsToRunStep] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Link manual input submissions to a pipeline run step as output submissions.
+ * Used when resolving missing_manual_input blockers.
+ */
+export async function linkManualInputsToRunStep({
+  member,
+  runStepId,
+  submissionIds,
+  addedBy,
+  reason,
+}: {
+  member: Member;
+  runStepId: string;
+  submissionIds: string[];
+  addedBy: string;
+  reason?: string;
+}): Promise<DalResult<void>> {
+  try {
+    // Verify the run step exists and belongs to the member's org
+    const runStepResult = await getPipelineRunStep({ member, runStepId });
+    if (!runStepResult.ok) return runStepResult;
+
+    // Insert submissions into pipelineRunStepOutputSubmission
+    if (submissionIds.length > 0) {
+      // Check for existing entries to avoid duplicates
+      const existing = await db.query.pipelineRunStepOutputSubmission.findMany({
+        where: and(
+          eq(schema.pipelineRunStepOutputSubmission.runStepId, runStepId),
+          inArray(schema.pipelineRunStepOutputSubmission.submissionId, submissionIds),
+        ),
+      });
+
+      const existingSubmissionIds = new Set(existing.map((e) => e.submissionId));
+      const newSubmissionIds = submissionIds.filter((id) => !existingSubmissionIds.has(id));
+
+      if (newSubmissionIds.length > 0) {
+        await db.insert(schema.pipelineRunStepOutputSubmission).values(
+          newSubmissionIds.map((submissionId) => ({
+            runStepId,
+            submissionId,
+          })),
+        );
+        console.log(
+          `[linkManualInputsToRunStep] Added ${newSubmissionIds.length} output submission(s) to step ${runStepId}`,
+        );
+      }
+    }
+
+    return okResult(undefined);
+  } catch (error) {
+    console.error("[linkManualInputsToRunStep] Error:", error);
+    return internalErrorResult;
+  }
+}
+
+/**
+ * Retry a pipeline run step by marking it as pending.
+ * This allows the workflow engine to pick it up again.
+ */
+export async function retryRunStep({
+  member,
+  runStepId,
+}: {
+  member: Member;
+  runStepId: string;
+}): Promise<DalResult<void>> {
+  try {
+    // Verify the run step exists and belongs to the member's org
+    const runStepResult = await getPipelineRunStep({ member, runStepId });
+    if (!runStepResult.ok) return runStepResult;
+
+    // Update the run step status to pending
+    await db
+      .update(schema.pipelineRunStep)
+      .set({
+        status: PIPELINE_RUN_STEP_STATUS.PENDING,
+      })
+      .where(eq(schema.pipelineRunStep.id, runStepId));
+
+    return okResult(undefined);
+  } catch (error) {
+    console.error("[retryRunStep] Error:", error);
+    return internalErrorResult;
+  }
+}