sea-dev 1.0.0

package/libs/core/src/pipelines/display.ts

@@ -0,0 +1,678 @@
+import type { Config } from "@sea/config";
+import { AzureClient } from "@sea/clients/azure";
+import { GCPClient } from "@sea/clients/gcp";
+import { HatchetClientMgr } from "@sea/clients/hatchet";
+import {
+  createConversationForPipeline,
+  findConversationResourceByPipeline,
+  getConversation,
+} from "@sea/dal/conversation";
+import { getDeals } from "@sea/dal/deal";
+import {
+  getDocumentExtraction,
+  updateDocumentAfterUpload,
+} from "@sea/dal/document";
+import {
+  getFormsForOrganization,
+  getFormsByIds,
+  getForms,
+} from "@sea/dal/form";
+import {
+  addStepDependency as dalAddStepDependency,
+  createEmptyPipeline as dalCreateEmptyPipeline,
+  createPipelineStep as dalCreatePipelineStep,
+  createPipelineWithStepAndConversation as dalCreatePipelineWithStepAndConversation,
+  deletePipeline as dalDeletePipeline,
+  deletePipelineStep as dalDeletePipelineStep,
+  getAllBlockersForRun as dalGetAllBlockersForRun,
+  getCitationChain as dalGetCitationChain,
+  getPipelineById,
+  getPipelineDetail as dalGetPipelineDetail,
+  getPipelineRuns as dalGetPipelineRuns,
+  getSubmissionsGroupedByForm,
+  removeStepDependency as dalRemoveStepDependency,
+  updatePipelineMetadata,
+  updatePipelineStepConfig as dalUpdatePipelineStepConfig,
+} from "@sea/dal/pipeline";
+import { getSubmissionsWithExtractions } from "@sea/dal/submission";
+import type { ConversationHydrated, Member } from "@sea/db/types";
+import type { CoreResult } from "@sea/schemas";
+import { db, schema } from "@sea/db";
+import { asc, eq } from "drizzle-orm";
+
+// Note: getPipelineDetail is in DAL, not needed here. Use DAL version directly.
+
+export async function getPipelineData({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}) {
+  const [
+    pipelineResult,
+    runsResult,
+    dealsResult,
+    formsResultBasic,
+    submissionsResult,
+  ] = await Promise.all([
+    dalGetPipelineDetail({ member, pipelineId }),
+    dalGetPipelineRuns({ member, pipelineId }),
+    getDeals({ member, limit: 100 }),
+    getForms({ member }),
+    getSubmissionsGroupedByForm({ member, limit: 100 }),
+  ]);
+  if (!pipelineResult.ok || !runsResult.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+
+  // Enhance forms with groups for pipeline step configuration
+  const formsResult = await Promise.all(
+    (formsResultBasic || []).map(async (f) => {
+      const groups = await db.query.formGroup.findMany({
+        where: eq(schema.formGroup.formId, f.id),
+        columns: { id: true, name: true, multi: true },
+        orderBy: asc(schema.formGroup.order),
+      });
+      return { ...f, groups };
+    }),
+  );
+
+  // Load the conversation with messages via DAL
+  const conversationLoadResult = await getOrCreatePipelineConversation({
+    member,
+    pipelineId,
+  });
+
+  if (!conversationLoadResult.ok) {
+    return { ok: false as const, code: 404, error: "not-found" as const };
+  }
+
+  return {
+    ok: true,
+    code: 200,
+    data: {
+      pipeline: pipelineResult.data,
+      conversation: conversationLoadResult.data,
+      runs: runsResult.data,
+      deals: dealsResult || [],
+      forms: formsResult || [],
+      submissionsGroupedByForm: submissionsResult.ok
+        ? submissionsResult.data
+        : [],
+    },
+  };
+}
+
+export async function deletePipeline({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}) {
+  const result = await dalDeletePipeline({ member, pipelineId });
+  if (!result.ok) {
+    return { ok: false, code: 400, error: "Failed to delete pipeline" };
+  }
+
+  return { ok: true, code: 200, data: { success: true } };
+}
+
+export async function createEmptyPipeline({ member }: { member: Member }) {
+  // Create pipeline with basic fields
+  const pipelineResult = await dalCreateEmptyPipeline({ member });
+
+  if (!pipelineResult.ok) {
+    return { ok: false, code: 500, error: "Failed to create pipeline" };
+  }
+
+  const pipeline = pipelineResult.data;
+
+  // Create conversation and link it to the pipeline
+  const convResult = await createConversationForPipeline({
+    member,
+    pipelineId: pipeline.id,
+    pipelineName: pipeline.name,
+  });
+
+  if (!convResult.ok) {
+    return { ok: false, code: 500, error: "Failed to create conversation" };
+  }
+
+  return { ok: true, code: 200, data: { id: pipeline.id } };
+}
+
+export async function createPipelineStep({
+  member,
+  pipelineId,
+  name,
+  description,
+  prompt,
+  type,
+  role,
+  stepOrder,
+  outputFormIds,
+  toolsAllowed,
+  integrationConfig,
+}: {
+  member: Member;
+  pipelineId: string;
+  name?: string;
+  description?: string;
+  prompt?: string;
+  type?: "single" | "multi";
+  role?: "extraction" | "integration" | "analysis" | null;
+  stepOrder?: number;
+  outputFormIds?: string[];
+  toolsAllowed?: string[];
+  integrationConfig?: {
+    method: "GET" | "POST" | "PUT" | "DELETE";
+    urlTemplate: string;
+    headers: Record<string, string>;
+    bodyTemplate?: string;
+  };
+}) {
+  const result = await dalCreatePipelineStep({
+    member,
+    pipelineId,
+    name: name ?? "Untitled Step",
+    description: description ?? "",
+    prompt,
+    type: type ?? "single",
+    role: role ?? null,
+    outputFormIds,
+    toolsAllowed,
+    integrationConfig,
+  });
+  if (!result.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+  return { ok: true, code: 200, data: { id: result.data.id } };
+}
+
+export async function deletePipelineStep({
+  member,
+  stepId,
+}: {
+  member: Member;
+  stepId: string;
+}) {
+  const result = await dalDeletePipelineStep({ member, stepId });
+  if (!result.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+  return { ok: true, code: 200, data: result.data };
+}
+
+export async function createPipelineConversation({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}) {
+  const pipelineResult = await getPipelineById({ member, pipelineId });
+
+  if (!pipelineResult.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+
+  const convResult = await createConversationForPipeline({
+    member,
+    pipelineId,
+    pipelineName: pipelineResult.data.name,
+  });
+
+  if (!convResult.ok) {
+    return { ok: false, code: 500, error: "Failed to create conversation" };
+  }
+
+  return { ok: true, code: 200, data: { id: convResult.data.conversationId } };
+}
+
+export async function getOrCreatePipelineConversation({
+  member,
+  pipelineId,
+}: {
+  member: Member;
+  pipelineId: string;
+}): Promise<CoreResult<ConversationHydrated>> {
+  // Verify pipeline exists
+  const pipelineResult = await getPipelineById({ member, pipelineId });
+
+  if (!pipelineResult.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+
+  // Check if conversation already exists for this pipeline
+  const existingResult = await findConversationResourceByPipeline({
+    member,
+    pipelineId,
+  });
+  if (!existingResult.ok) {
+    return { ok: false, code: 500, error: "Failed to check conversation" };
+  }
+
+  if (existingResult.data) {
+    const conversation = await getConversation({
+      member,
+      conversationId: existingResult.data.conversationId,
+    });
+    if (!conversation.ok) {
+      return { ok: false, code: 500, error: "Failed to get conversation" };
+    }
+    return {
+      ok: true,
+      code: 200,
+      data: conversation.data,
+    };
+  }
+
+  // Create new conversation if none exists
+  const convResult = await createConversationForPipeline({
+    member,
+    pipelineId,
+    pipelineName: pipelineResult.data.name,
+  });
+
+  if (!convResult.ok) {
+    return { ok: false, code: 500, error: "Failed to create conversation" };
+  }
+  const conversation = await getConversation({
+    member,
+    conversationId: convResult.data.conversationId,
+  });
+  if (!conversation.ok) {
+    return { ok: false, code: 500, error: "Failed to get conversation" };
+  }
+  return {
+    ok: true,
+    code: 200,
+    data: conversation.data,
+  };
+}
+
+export async function getNewPipelineData({ member }: { member: Member }) {
+  const formsResult = await getFormsForOrganization({ member, limit: 100 });
+
+  return { ok: true, code: 200, data: { forms: formsResult.data } };
+}
+
+export async function createPipeline({
+  member,
+  data,
+}: {
+  member: Member;
+  data: {
+    name: string;
+    description: string;
+    conversationId: string;
+    inputFormId: string;
+    outputFormId: string;
+    stepPrompt: string;
+  };
+}) {
+  // Verify both forms exist
+  const formsResult = await getFormsByIds({
+    member,
+    formIds: [data.inputFormId, data.outputFormId],
+  });
+
+  if (!formsResult.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+
+  // Create pipeline with step and conversation using DAL transaction
+  const result = await dalCreatePipelineWithStepAndConversation({
+    member,
+    name: data.name,
+    description: data.description,
+    stepPrompt: data.stepPrompt,
+  });
+
+  if (!result.ok) {
+    return { ok: false, code: 500, error: "Failed to create pipeline" };
+  }
+
+  return {
+    ok: true,
+    code: 200,
+    data: result.data,
+  };
+}
+
+export async function uploadPipelineDocument({
+  member,
+  file,
+  documentExtractionId,
+  config,
+}: {
+  member: Member;
+  file: { arrayBuffer: ArrayBuffer; type: string };
+  documentExtractionId: string;
+  config: Config;
+}) {
+  const { document, ...documentExtraction } = await getDocumentExtraction({
+    member,
+    documentExtractionId,
+  });
+
+  const { blobKey, storageProvider } = document;
+  const client =
+    storageProvider === "azure"
+      ? new AzureClient(config.azure)
+      : new GCPClient(config.gcp);
+  const buffer = Buffer.from(file.arrayBuffer);
+  await client.uploadFile(buffer, blobKey, file.type);
+  const { url, expiry } = await client.getSignedUrl(blobKey);
+
+  await updateDocumentAfterUpload({
+    member,
+    documentId: document.id,
+    blobUrl: url,
+    blobUrlExpiry: expiry,
+  });
+
+  // Trigger extraction via Hatchet
+  const hatchetClient = new HatchetClientMgr(config.hatchet);
+  await hatchetClient.pushDoc(
+    documentExtractionId,
+    documentExtraction.extractionModel,
+    undefined, // userPrompt
+    undefined, // reasoningEffort
+  );
+
+  return { ok: true, code: 200, data: { success: true } };
+}
+
+export async function checkSubmissionExtractionsComplete({
+  member,
+  submissionIds,
+}: {
+  member: Member;
+  submissionIds: string[];
+}) {
+  const submissions = await getSubmissionsWithExtractions({
+    member,
+    submissionIds,
+  });
+
+  // For each submission, check if all extractions are complete (done or failed)
+  const results = submissions.map((sub) => {
+    const hasExtractions = sub.extractions.length > 0;
+    const pendingExtractions = sub.extractions.filter(
+      (ext) => ext.status === "pending" || ext.status === "processing",
+    );
+    const isComplete = hasExtractions ? pendingExtractions.length === 0 : true;
+
+    return {
+      submissionId: sub.id,
+      isComplete,
+      totalExtractions: sub.extractions.length,
+      pendingCount: pendingExtractions.length,
+      pendingIds: pendingExtractions.map((e) => e.id),
+    };
+  });
+
+  return {
+    ok: true,
+    code: 200,
+    data: {
+      allComplete: results.every((r) => r.isComplete),
+      submissions: results,
+    },
+  };
+}
+
+export async function updatePipeline({
+  member,
+  pipelineId,
+  name,
+  description,
+}: {
+  member: Member;
+  pipelineId: string;
+  name: string;
+  description: string;
+}) {
+  const result = await updatePipelineMetadata({
+    member,
+    pipelineId,
+    name,
+    description,
+  });
+  if (!result.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+  return { ok: true, code: 200, data: result.data };
+}
+
+export async function updatePipelineStep({
+  member,
+  stepId,
+  name,
+  description,
+  prompt,
+  type,
+  role,
+  outputFormIds,
+  toolsAllowed,
+  integrationConfig,
+}: {
+  member: Member;
+  stepId: string;
+  name?: string;
+  description?: string;
+  prompt?: string;
+  type?: "single" | "multi";
+  role?: "extraction" | "integration" | "analysis" | null;
+  outputFormIds?: string[];
+  toolsAllowed?: string[];
+  integrationConfig?: {
+    method: "GET" | "POST" | "PUT" | "DELETE";
+    urlTemplate: string;
+    headers: Record<string, string>;
+    bodyTemplate?: string;
+  } | null;
+}) {
+  const result = await dalUpdatePipelineStepConfig({
+    member,
+    stepId,
+    ...(name !== undefined && { name }),
+    ...(description !== undefined && { description }),
+    ...(prompt !== undefined && { prompt }),
+    ...(type !== undefined && { type }),
+    ...(role !== undefined && { role }),
+    ...(outputFormIds !== undefined && { outputFormIds }),
+    ...(toolsAllowed !== undefined && { toolsAllowed }),
+    ...(integrationConfig !== undefined && { integrationConfig }),
+  });
+  if (!result.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+  return { ok: true, code: 200, data: { success: true as const } };
+}
+
+// Step Dependency Management
+export async function addStepDependency({
+  member,
+  childStepId,
+  parentStepId,
+  multiFormGroupId,
+}: {
+  member: Member;
+  childStepId: string;
+  parentStepId: string;
+  multiFormGroupId?: string | null;
+}) {
+  const result = await dalAddStepDependency({
+    member,
+    childStepId,
+    parentStepId,
+    multiFormGroupId,
+  });
+  if (!result.ok) {
+    if ("message" in result && result.error === "validation-error") {
+      return { ok: false, code: 400, error: "validation-error" };
+    }
+    return { ok: false, code: 404, error: "not-found" };
+  }
+  // TypeScript doesn't narrow the union type properly, so we assert
+  const successResult = result as {
+    ok: true;
+    data: {
+      id: string;
+      childStepId: string;
+      parentStepId: string;
+      multiFormGroupId: string | null;
+    };
+  };
+  return { ok: true, code: 200, data: successResult.data };
+}
+
+export async function removeStepDependency({
+  member,
+  dependencyId,
+}: {
+  member: Member;
+  dependencyId: string;
+}) {
+  const result = await dalRemoveStepDependency({ member, dependencyId });
+  if (!result.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+  return { ok: true, code: 200, data: { success: true as const } };
+}
+
+// Manual Input Management
+// Manual steps use output submissions directly - link a submission to resolve the blocker
+export async function addManualInput({
+  member,
+  runStepId,
+  submissionId,
+}: {
+  member: Member;
+  runStepId: string;
+  submissionId: string;
+}) {
+  // Link submission as output for the manual step
+  const { db, schema } = await import("@sea/db");
+  const { eq, and } = await import("drizzle-orm");
+
+  try {
+    // Verify run step exists and belongs to member's org
+    const runStep = await db.query.pipelineRunStep.findFirst({
+      where: eq(schema.pipelineRunStep.id, runStepId),
+      with: { run: true },
+    });
+
+    if (!runStep || runStep.run.organizationId !== member.organizationId) {
+      return { ok: false, code: 404, error: "not-found" };
+    }
+
+    // Verify submission exists and belongs to org
+    const submission = await db.query.submission.findFirst({
+      where: and(
+        eq(schema.submission.id, submissionId),
+        eq(schema.submission.organizationId, member.organizationId),
+      ),
+    });
+
+    if (!submission) {
+      return { ok: false, code: 404, error: "not-found" };
+    }
+
+    // Link submission as output
+    await db.insert(schema.pipelineRunStepOutputSubmission).values({
+      runStepId,
+      submissionId,
+    });
+
+    // Resolve missing_manual_input blocker if it exists
+    await db
+      .update(schema.pipelineRunStepBlocker)
+      .set({ resolved: true, resolvedAt: new Date() })
+      .where(
+        and(
+          eq(schema.pipelineRunStepBlocker.runStepId, runStepId),
+          eq(schema.pipelineRunStepBlocker.blockerType, "missing_manual_input"),
+          eq(schema.pipelineRunStepBlocker.resolved, false),
+        ),
+      );
+
+    return { ok: true, code: 200, data: { runStepId, submissionId } };
+  } catch (error) {
+    console.error("[addManualInput] Error:", error);
+    return { ok: false, code: 500, error: "internal-error" };
+  }
+}
+
+export async function removeManualInput({
+  member,
+  manualInputId,
+}: {
+  member: Member;
+  manualInputId: string;
+}) {
+  // manualInputId is actually the output submission mapping ID
+  const { db, schema } = await import("@sea/db");
+  const { eq, and } = await import("drizzle-orm");
+
+  try {
+    const mapping = await db.query.pipelineRunStepOutputSubmission.findFirst({
+      where: eq(schema.pipelineRunStepOutputSubmission.id, manualInputId),
+      with: {
+        runStep: {
+          with: { run: true },
+        },
+      },
+    });
+
+    if (
+      !mapping ||
+      mapping.runStep.run.organizationId !== member.organizationId
+    ) {
+      return { ok: false, code: 404, error: "not-found" };
+    }
+
+    await db
+      .delete(schema.pipelineRunStepOutputSubmission)
+      .where(eq(schema.pipelineRunStepOutputSubmission.id, manualInputId));
+
+    return { ok: true, code: 200, data: { success: true as const } };
+  } catch (error) {
+    console.error("[removeManualInput] Error:", error);
+    return { ok: false, code: 500, error: "internal-error" };
+  }
+}
+
+// Blocker Management
+export async function getAllBlockersForRun({
+  member,
+  runId,
+}: {
+  member: Member;
+  runId: string;
+}) {
+  const result = await dalGetAllBlockersForRun({ member, runId });
+  if (!result.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+  return { ok: true, code: 200, data: result.data };
+}
+
+// Citation Chain
+export async function getCitationChain({
+  member,
+  submissionItemVersionId,
+}: {
+  member: Member;
+  submissionItemVersionId: string;
+}) {
+  const result = await dalGetCitationChain({ member, submissionItemVersionId });
+  if (!result.ok) {
+    return { ok: false, code: 404, error: "not-found" };
+  }
+  return { ok: true, code: 200, data: result.data };
+}