sea-dev 1.0.0

package/apps/main/src/routes/_authed/_app/deals/$dealId/-fns.ts

@@ -0,0 +1,653 @@
+import { getAppConfig } from "@/lib/config";
+import { catchNotFound } from "@/lib/not-found";
+import * as zfd from "@/lib/zfd";
+import { membershipMiddleware, requestLogMiddleware } from "@/middleware";
+import { AzureClient, GCPClient, HatchetClientMgr } from "@sea/clients";
+import { calcDerived, getChatModel, summarizeSubmission } from "@sea/core";
+import { ReasoningEffortSchema } from "@sea/core/data-extraction/models";
+import { getExtractionSetup } from "@sea/core/data-extraction/utils";
+import { checkCanExtract } from "@sea/dal/billing";
+import { getDocument } from "@sea/dal/document";
+import {
+  type DenormSubmission,
+  getDenormSubmission,
+  getSubmission,
+  getSubmissionWithData,
+  resetSubmission,
+  syncSubmissionToForm,
+} from "@sea/dal/submission";
+import { db, schema } from "@sea/db";
+import type { ChatModel } from "@sea/db/schema";
+import { chatModelSchema } from "@sea/db/schema";
+import type { DocumentCitation } from "@sea/db/types";
+import type { ApiReconciliation } from "@sea/schemas/core/submission";
+import { convertItemsToRecord, groupBy } from "@sea/util/data-transform";
+import { nanoid } from "@sea/util/id";
+import { invariant } from "@sea/util/invariant";
+import { notFound, redirect } from "@tanstack/react-router";
+import { createServerFn } from "@tanstack/react-start";
+import { isFuture, subMinutes } from "date-fns";
+import { and, desc, eq, inArray, isNull } from "drizzle-orm";
+
+const config = getAppConfig();
+
+const hasMeaningfulValue = (value: unknown) => {
+  if (value === null || value === undefined) {
+    return false;
+  }
+  if (typeof value === "string") {
+    return value.trim().length > 0;
+  }
+  return true;
+};
+
+const hasSubmissionInfo = (submission: DenormSubmission) => {
+  const groupHasValue = submission.groups.some(
+    (group) =>
+      group.items.some((item) => hasMeaningfulValue(item.value)) ||
+      group.derived.some((item) => hasMeaningfulValue(item.value)),
+  );
+  if (groupHasValue) {
+    return true;
+  }
+
+  return submission.tables.some((table) =>
+    table.rows.some(
+      (row) =>
+        row.items.some((item) => hasMeaningfulValue(item.value)) ||
+        row.derived.some((item) => hasMeaningfulValue(item.value)),
+    ),
+  );
+};
+
+export const getSubmissionFn = createServerFn({ method: "GET" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId } }) => {
+    const check = await getSubmission({ member, submissionId, typeFilter: "all" });
+    if (!check) {
+      throw catchNotFound({ submissionId });
+    }
+
+    await syncSubmissionToForm({ member, submissionId });
+    const submission = await getSubmissionWithData({ member, submissionId, typeFilter: "all" });
+    invariant(submission);
+
+    const groups = submission.groups.map((group) => {
+      const items = group.items.map((item) => ({
+        ...item,
+        selectedVersion: item.versions[0] ?? null,
+      }));
+      return {
+        ...group,
+        items: [...items].sort((a, b) => a.formField.order - b.formField.order),
+        itemsMap: convertItemsToRecord(items, "formFieldId"),
+        derivedMap: convertItemsToRecord(group.derived, "formDerivedId"),
+      };
+    });
+    const groupObj = groupBy(groups, "formGroupId");
+    const grouped = submission.form.groups.map((formGroup) => {
+      const groups = groupObj[formGroup.id] ?? [];
+      return {
+        formGroup,
+        groups,
+      };
+    });
+
+    // TODO: add reference resolution by item to make sure we have valid citations
+    // for current data and are able to resolve conflicts
+
+    // For transformation submissions, load documents from the deal instead of just the submission
+    let extractionsToUse = submission.extractions;
+    if (submission.type === "transformation") {
+      const dealDocuments = await db.query.document.findMany({
+        where: and(
+          eq(schema.document.dealId, submission.dealId),
+          isNull(schema.document.deletedAt),
+        ),
+        with: {
+          extractions: {
+            where: isNull(schema.documentExtraction.deletedAt),
+            with: {
+              citations: true,
+            },
+          },
+        },
+      });
+      // Flatten extractions from all documents
+      const dealExtractions = dealDocuments.flatMap((doc) =>
+        doc.extractions.map((extraction) => ({
+          ...extraction,
+          document: doc,
+        })),
+      );
+      extractionsToUse = dealExtractions;
+    }
+
+    const extractionsById = Object.fromEntries(
+      extractionsToUse.map((e) => [
+        e.id,
+        {
+          documentId: e.document.id,
+          documentExtractionId: e.id,
+          url: e.document.blobUrl,
+          urlExpiry: e.document.blobUrlExpiry,
+          uploaded: e.document.uploaded,
+          name: e.document.name,
+          contentType: e.document.contentType,
+          status: e.status,
+          displayType: e.document.displayType,
+          extractionModel: e.extractionModel,
+          extractionTrigger: e.extractionTrigger,
+          extractedData: e.data,
+          reconciliationData: e.reconciliation as ApiReconciliation,
+          citations: [] as DocumentCitation[],
+          composite: e.document.composite,
+          parentId: e.document.parentId,
+          classificationLabel: e.document.classificationLabel,
+        },
+      ]),
+    );
+    groups.forEach((group) => {
+      group.items.forEach((item) => {
+        const citation = item.versions[0]?.documentSource;
+        if (!citation) return;
+        const extraction = extractionsById[citation.documentExtractionId];
+        if (!extraction) return;
+        extraction.citations.push(citation);
+      });
+    });
+    const extractionInfo = Object.values(extractionsById);
+
+    const { data: chatModelRaw } = chatModelSchema.safeParse(submission.conversation.model);
+    const chatModel: ChatModel = chatModelRaw ?? "gpt-4.1";
+    const messageDraft = submission.conversation.messages.find(
+      (m) => m.status === "draft",
+    )?.content;
+
+    const canExtract = await checkCanExtract({ member });
+
+    const res = {
+      submission: {
+        ...submission,
+        conversation: {
+          ...submission.conversation,
+          messages: submission.conversation.messages.filter((m) => m.status === "sent"),
+        },
+        groups,
+      },
+      grouped,
+      extractionInfo,
+      chatModel,
+      messageDraft,
+      canExtract,
+    };
+
+    return res;
+  });
+
+export const renameSubmissionFn = createServerFn()
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string; name: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId, name } }) => {
+    await db
+      .update(schema.submission)
+      .set({ name })
+      .where(
+        and(
+          eq(schema.submission.organizationId, member.organizationId),
+          eq(schema.submission.id, submissionId),
+        ),
+      );
+  });
+
+export const checkExtractionStatusesFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string; extractionsIds: string[] }) => data)
+  .handler(async ({ context: { member }, data: { submissionId, extractionsIds } }) => {
+    const submission = await db.query.submission.findFirst({
+      where: and(
+        eq(schema.submission.organizationId, member.organizationId),
+        eq(schema.submission.id, submissionId),
+        isNull(schema.submission.deletedAt),
+      ),
+      with: {
+        extractions: {
+          where: and(
+            inArray(schema.documentExtraction.status, ["done", "failed"]),
+            inArray(schema.documentExtraction.id, extractionsIds),
+            isNull(schema.documentExtraction.deletedAt),
+          ),
+        },
+        conversation: {
+          with: {
+            messages: {
+              orderBy: desc(schema.message.updatedAt),
+              limit: 1,
+            },
+          },
+        },
+      },
+    });
+    invariant(submission);
+    const { extractions } = submission;
+    const latestMessage = submission.conversation.messages[0];
+    return {
+      extractions,
+      latestMessage: latestMessage
+        ? { id: latestMessage.id, updatedAt: latestMessage.updatedAt }
+        : null,
+    };
+  });
+
+export const updateUpdatedAtFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId } }) => {
+    await db
+      .update(schema.submission)
+      .set({ updatedAt: new Date() })
+      .where(
+        and(
+          eq(schema.submission.id, submissionId),
+          eq(schema.submission.organizationId, member.organizationId),
+        ),
+      );
+  });
+
+export const reloadFormFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId } }) => {
+    await syncSubmissionToForm({ member, submissionId });
+  });
+
+export const resetSubmissionFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId } }) => {
+    await resetSubmission({ member, submissionId });
+  });
+
+export const toggleFinalFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId } }) => {
+    const submission = await getSubmission({ member, submissionId });
+    const status = submission.status === "final" ? "draft" : "final";
+    await db
+      .update(schema.submission)
+      .set({ status })
+      .where(eq(schema.submission.id, submission.id));
+
+    if (status === "final") {
+      const config = getAppConfig();
+      const model = getChatModel({
+        aiGatewayApiKey: config.vercel.aiGatewayApiKey,
+        modelId: "gpt-4o",
+      });
+      const derivations = await calcDerived(model, member, submissionId);
+      await db.transaction(async (tx) => {
+        await Promise.all(
+          derivations.map(async ({ id, text }) => {
+            await tx
+              .update(schema.submissionDerived)
+              .set({ value: text })
+              .where(eq(schema.submissionDerived.id, id));
+          }),
+        );
+      });
+      return { numDerived: derivations.length };
+    }
+    return { numDerived: 0 };
+  });
+
+export const deleteSubmissionFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId } }) => {
+    const [submission] = await db
+      .select({
+        id: schema.submission.id,
+        dealId: schema.submission.dealId,
+      })
+      .from(schema.submission)
+      .innerJoin(schema.deal, eq(schema.deal.id, schema.submission.dealId))
+      .where(
+        and(
+          eq(schema.deal.organizationId, member.organizationId),
+          eq(schema.submission.id, submissionId),
+        ),
+      );
+    invariant(submission);
+    const { dealId } = submission;
+
+    await db
+      .update(schema.submission)
+      .set({ deletedAt: new Date() })
+      .where(eq(schema.submission.id, submission.id))
+      .returning();
+
+    throw redirect({
+      to: "/deals/$dealId",
+      params: {
+        dealId,
+      },
+    });
+  });
+
+export const storeMessageDraftFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { conversationId: string; content: string }) => data)
+  .handler(async ({ context: { member }, data: { conversationId, content } }) => {
+    const conversation = await db.query.conversation.findFirst({
+      where: and(
+        eq(schema.conversation.organizationId, member.organizationId),
+        eq(schema.conversation.id, conversationId),
+        isNull(schema.conversation.deletedAt),
+      ),
+    });
+    invariant(conversation);
+    await db
+      .insert(schema.message)
+      .values({
+        id: nanoid(),
+        conversationId,
+        role: "user",
+        status: "draft",
+        content,
+      })
+      .onConflictDoUpdate({
+        target: [schema.message.conversationId, schema.message.status],
+        targetWhere: eq(schema.message.status, "draft"),
+        set: { content },
+      });
+  });
+
+export const insertDocumentFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator(
+    (data: {
+      submissionId: string;
+      model: string;
+      files: { id: string; name: string; type: string }[];
+      composite?: boolean;
+    }) => {
+      console.log("[insertDocumentFn validator] Received data:", data);
+      return {
+        ...data,
+        composite: data.composite ?? false,
+      };
+    },
+  )
+  .handler(async ({ context: { member }, data: { submissionId, model, files, composite } }) => {
+    const submission = await getSubmission({ member, submissionId });
+    const documents = await Promise.all(
+      files.map(async (file) => {
+        const { name, type: contentType, id } = file;
+        // TODO: remove this nonsense once we merge all the models we use
+        const { storageProvider, displayType } = getExtractionSetup(config);
+        return db.transaction(async (tx) => {
+          const blobKey = nanoid() + "-" + name;
+          const [insertedDocument] = await tx
+            .insert(schema.document)
+            .values({
+              id,
+              dealId: submission.dealId,
+              name,
+              contentType,
+              storageProvider,
+              displayType,
+              blobKey,
+              composite,
+            })
+            .returning();
+
+          invariant(insertedDocument);
+          const { id: documentId } = insertedDocument;
+
+          const [documentExtraction] = await tx
+            .insert(schema.documentExtraction)
+            .values({
+              documentId,
+              submissionId: submission.id,
+              status: "pending",
+              extractionModel: model,
+            })
+            .returning({ id: schema.documentExtraction.id });
+          invariant(documentExtraction);
+
+          return { documentId, extractionId: documentExtraction.id, blobKey };
+        });
+      }),
+    );
+    return documents;
+  });
+
+const uploadDocumentSchema = zfd.formData({
+  documentExtractionId: zfd.text(),
+  userPrompt: zfd.text().optional(),
+  reasoningEffort: ReasoningEffortSchema.optional(),
+  file: zfd.file(),
+});
+
+export const uploadDocumentFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware, requestLogMiddleware])
+  .validator((formData: FormData) => uploadDocumentSchema.parse(formData))
+  .handler(
+    async ({
+      context: { member, reqLogger },
+      data: { file, documentExtractionId, userPrompt, reasoningEffort },
+    }) => {
+      const documentExtraction = await db.query.documentExtraction.findFirst({
+        where: eq(schema.documentExtraction.id, documentExtractionId),
+        with: { document: true },
+      });
+      invariant(documentExtraction);
+      invariant(documentExtraction.document);
+
+      const { blobKey, storageProvider } = documentExtraction.document;
+      const client =
+        storageProvider === "azure" ? new AzureClient(config.azure) : new GCPClient(config.gcp);
+
+      const buffer = Buffer.from(await file.arrayBuffer());
+      await client.uploadFile(buffer, blobKey, file.type);
+      const { url, expiry } = await client.getSignedUrl(blobKey);
+
+      await db.transaction(async (tx) => {
+        await tx
+          .update(schema.document)
+          .set({ uploaded: true, blobUrl: url, blobUrlExpiry: expiry })
+          .where(eq(schema.document.id, documentExtraction.documentId));
+      });
+
+      // run extraction
+      reqLogger.info(
+        {
+          memberId: member.id,
+          submissionId: documentExtraction.submissionId,
+          documentId: documentExtraction.documentId,
+        },
+        "extraction: start",
+      );
+
+      const hatchetClient = new HatchetClientMgr(config.hatchet);
+      await hatchetClient.pushDoc(
+        documentExtraction.id,
+        documentExtraction.extractionModel,
+        userPrompt,
+        reasoningEffort,
+      );
+      reqLogger.info({}, "extraction: pushed to hatchet");
+    },
+  );
+
+export const getDenormSubmissionFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId } }) => {
+    const denormSubmission = await getDenormSubmission({ member, submissionId });
+    return denormSubmission;
+  });
+
+export const summarizeSubmissionFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { submissionId: string }) => data)
+  .handler(async ({ context: { member }, data: { submissionId } }) => {
+    const denormSubmission = await getDenormSubmission({ member, submissionId });
+
+    const where = and(
+      eq(schema.submission.organizationId, member.organizationId),
+      eq(schema.submission.id, submissionId),
+    );
+
+    if (!hasSubmissionInfo(denormSubmission)) {
+      await db
+        .update(schema.submission)
+        .set({
+          summary: null,
+          summarizedAt: new Date(),
+        })
+        .where(where);
+      return null;
+    }
+
+    const submissionHydrated = await db.query.submission.findFirst({
+      where,
+      with: { deal: true },
+    });
+    invariant(submissionHydrated);
+
+    const context = `Deal name: ${submissionHydrated.deal.name}`;
+    const config = getAppConfig();
+    const model = getChatModel({
+      aiGatewayApiKey: config.vercel.aiGatewayApiKey,
+      modelId: "openai/gpt-4.1",
+    });
+    const text = await summarizeSubmission(model, denormSubmission, context);
+    const trimmed = text.trim();
+    const summary = trimmed.length > 0 ? trimmed : null;
+    await db
+      .update(schema.submission)
+      .set({
+        summary,
+        summarizedAt: new Date(),
+      })
+      .where(where);
+    return summary;
+  });
+
+export const deleteDocumentFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { documentId: string }) => data)
+  .handler(async ({ context: { member }, data: { documentId } }) => {
+    const document = await db.query.document.findFirst({
+      where: eq(schema.document.id, documentId),
+      with: { deal: true },
+    });
+    invariant(document);
+    if (document.deal.organizationId !== member.organizationId) {
+      throw notFound();
+    }
+
+    await db.transaction(async (tx) => {
+      await tx
+        .update(schema.document)
+        .set({ deletedAt: new Date() })
+        .where(eq(schema.document.id, document.id));
+      await tx
+        .update(schema.documentExtraction)
+        .set({ deletedAt: new Date() })
+        .where(eq(schema.documentExtraction.documentId, document.id));
+    });
+  });
+
+export const deleteDocumentExtractionFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { documentExtractionId: string }) => data)
+  .handler(async ({ context: { member }, data: { documentExtractionId } }) => {
+    const documentExtraction = await db.query.documentExtraction.findFirst({
+      where: eq(schema.document.id, documentExtractionId),
+      with: { submission: true },
+    });
+    invariant(documentExtraction);
+    if (documentExtraction.submission.organizationId !== member.organizationId) {
+      throw notFound();
+    }
+
+    await db
+      .update(schema.documentExtraction)
+      .set({ deletedAt: new Date() })
+      .where(eq(schema.documentExtraction.id, documentExtraction.id));
+  });
+
+export const refreshUrlFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { documentId: string }) => data)
+  .handler(async ({ context: { member }, data: { documentId } }) => {
+    // Validate access
+    const document = await getDocument({ member, documentId });
+    if (document.storageProvider === "gcp") {
+      const gcpClient = new GCPClient(config.gcp);
+      const { url, expiry } = await gcpClient.getSignedUrl(document.blobKey);
+      await db
+        .update(schema.document)
+        .set({ blobUrlExpiry: expiry, blobUrl: url })
+        .where(eq(schema.document.id, document.id));
+    } else {
+      const azureClient = new AzureClient(config.azure);
+      const { url, expiry } = await azureClient.getSignedUrl(document.blobKey);
+      await db
+        .update(schema.document)
+        .set({ blobUrlExpiry: expiry, blobUrl: url })
+        .where(eq(schema.document.id, document.id));
+    }
+  });
+
+export const getDocumentForTooltipFn = createServerFn({ method: "POST" })
+  .middleware([membershipMiddleware])
+  .validator((data: { documentExtractionId: string }) => data)
+  .handler(async ({ context: { member }, data: { documentExtractionId } }) => {
+    const [document] = await db
+      .select({
+        name: schema.document.name,
+        createdAt: schema.document.createdAt,
+      })
+      .from(schema.documentExtraction)
+      .innerJoin(schema.document, eq(schema.document.id, schema.documentExtraction.documentId))
+      .innerJoin(schema.deal, eq(schema.deal.id, schema.document.dealId))
+      .where(
+        and(
+          eq(schema.deal.organizationId, member.organizationId),
+          isNull(schema.document.deletedAt),
+          eq(schema.documentExtraction.id, documentExtractionId),
+        ),
+      );
+    if (document === undefined) {
+      throw notFound();
+    }
+
+    return document;
+  });
+
+export const downloadDocumentFn = createServerFn({ method: "GET" })
+  .middleware([membershipMiddleware])
+  .validator((data: { documentId: string }) => data)
+  .handler(async ({ context: { member }, data: { documentId } }) => {
+    const { name, blobKey, blobUrl, blobUrlExpiry } = await getDocument({ member, documentId });
+
+    invariant(blobUrl, "blobUrl not set");
+    invariant(blobUrlExpiry, "blobUrlExpiry not set");
+
+    if (isFuture(subMinutes(blobUrlExpiry, 5))) {
+      return { name, blobUrl };
+    }
+
+    const storageClient = new AzureClient(config.azure);
+    const { url, expiry } = await storageClient.getSignedUrl(blobKey);
+    await db
+      .update(schema.document)
+      .set({ blobUrlExpiry: expiry, blobUrl: url })
+      .where(eq(schema.document.id, documentId));
+    return { name, blobUrl: url };
+  });