sea-dev 1.0.0

package/apps/api/src/routes/v2/submissions/handlers.ts

@@ -0,0 +1,555 @@
+import { captureEvent } from "@/clients/posthog";
+import type { Channel } from "@/lib/channel";
+import { getApiConfig } from "@/lib/config";
+import { AzureClient } from "@sea/clients/azure";
+import { GCPClient } from "@sea/clients/gcp";
+import { HatchetClientMgr } from "@sea/clients/hatchet";
+import type { Config } from "@sea/config";
+import { getExtractionSetup } from "@sea/core/data-extraction/utils";
+import { checkCanExtract } from "@sea/dal/billing";
+import { createDeal, getDealByRefId } from "@sea/dal/deal";
+import { getDocumentExtraction } from "@sea/dal/document";
+import { createCorrection, getCorrectionsForItem, markAddressed } from "@sea/dal/feedback";
+import type { MemberHydrated } from "@sea/dal/member";
+import {
+  addSubmissionGroupRow,
+  createSubmissionWithConversation,
+  getSubmission,
+  getSubmissions,
+  getSubmissionWithData,
+  getSubmissionWithExtractions,
+  syncSubmissionToForm,
+  updateSubmission,
+  updateSubmissionGroup,
+  updateSubmissionValue,
+} from "@sea/dal/submission";
+import { db, schema } from "@sea/db";
+import type { Document, DocumentCitation } from "@sea/db/types";
+import type {
+  ApiFormGroup,
+  ApiSubmissionDocument,
+  ApiSubmissionGroup,
+  ApiSubmissionItem,
+} from "@sea/schemas/core/submission";
+import { nanoid } from "@sea/util/id";
+import { invariant, InvariantError } from "@sea/util/invariant";
+import { differenceInSeconds, endOfMonth } from "date-fns";
+import { eq } from "drizzle-orm";
+import type { FastifyReply } from "fastify";
+import type {
+  DocumentExtractionDataQuery,
+  ExtractDataBody,
+  GetSubmissionParams,
+  GetSubmissionResponse,
+  GetSubmissionStatusParams,
+  GetSubmissionStatusResponse,
+  ListSubmissionsQuery,
+  ListSubmissionsResponse,
+  SubmissionCreate,
+} from "./schemas";
+
+const config = getApiConfig();
+
+// TODO: Move this process to a shared location
+async function checkAndRefreshUrl(document: Document) {
+  try {
+    if (!document.blobUrlExpiry) return;
+    const pgDate = new Date(document.blobUrlExpiry);
+    if (pgDate.getTime() > Date.now()) return;
+
+    if (document.storageProvider === "gcp") {
+      const gcpClient = new GCPClient(config.gcp);
+      const { url, expiry } = await gcpClient.getSignedUrl(document.blobKey);
+      const [updatedDocument] = await db
+        .update(schema.document)
+        .set({ blobUrlExpiry: expiry, blobUrl: url })
+        .where(eq(schema.document.id, document.id))
+        .returning();
+
+      if (!updatedDocument) return;
+      document.blobUrl = updatedDocument.blobUrl;
+      document.blobUrlExpiry = updatedDocument.blobUrlExpiry;
+    } else {
+      const azureClient = new AzureClient(config.azure);
+      const { url, expiry } = await azureClient.getSignedUrl(document.blobKey);
+      const [updatedDocument] = await db
+        .update(schema.document)
+        .set({ blobUrlExpiry: expiry, blobUrl: url })
+        .where(eq(schema.document.id, document.id))
+        .returning();
+
+      if (!updatedDocument) return;
+      document.blobUrl = updatedDocument.blobUrl;
+      document.blobUrlExpiry = updatedDocument.blobUrlExpiry;
+    }
+  } catch (error) {
+    console.error(error);
+  }
+}
+
+async function getOrCreateDeal(member: MemberHydrated, dealRefId: string) {
+  const deal = await getDealByRefId({ member, dealRefId });
+  if (deal) return deal;
+  const newDeal = await createDeal({
+    member,
+    name: `New Deal ${dealRefId.substring(0, 4)}`,
+    refId: dealRefId,
+    nameAutoGenerated: true,
+  });
+  return newDeal;
+}
+
+export async function createSubmissionHandler(
+  { formId: formId, dealId, dealRefId }: SubmissionCreate,
+  member: MemberHydrated,
+  reply: FastifyReply,
+  channel: Channel,
+) {
+  try {
+    if (dealId && dealRefId) {
+      return reply.code(400).send({
+        description: "dealId and dealRefId cannot be provided together",
+      });
+    }
+    if (dealRefId) {
+      const deal = await getOrCreateDeal(member, dealRefId);
+      if (!deal) {
+        return reply.code(404).send({
+          description: "Deal not found and could not be created",
+        });
+      }
+      const submission = await createSubmissionWithConversation({
+        member,
+        data: { formId, dealId: deal.id, status: "draft" },
+      });
+      await syncSubmissionToForm({ member, submissionId: submission.id });
+      captureEvent({
+        event: "submission_created",
+        distinctId: member.organizationId,
+        properties: {
+          submission_id: submission.id,
+          form_id: submission.formId,
+          deal_id: submission.dealId,
+          organization_id: member.organizationId,
+          user_id: member.user.id,
+          channel,
+        },
+        groups: { company: member.organizationId },
+      });
+      return submission;
+    }
+    if (!dealId) {
+      return reply.code(400).send({
+        description: "dealId or dealRefId is required",
+      });
+    }
+    const submission = await createSubmissionWithConversation({
+      member,
+      data: { formId, dealId, status: "draft" },
+    });
+    await syncSubmissionToForm({ member, submissionId: submission.id });
+    captureEvent({
+      event: "submission_created",
+      distinctId: member.organizationId,
+      properties: {
+        submission_id: submission.id,
+        form_id: submission.formId,
+        deal_id: submission.dealId,
+        organization_id: member.organizationId,
+        user_id: member.user.id,
+        channel,
+      },
+      groups: { company: member.organizationId },
+    });
+    return submission;
+  } catch (error) {
+    console.error(error);
+    return reply.code(500).send({ description: "Failed to create submission" });
+  }
+}
+
+export async function checkStatusHandler(
+  { submissionId }: GetSubmissionStatusParams,
+  member: MemberHydrated,
+  reply: FastifyReply,
+): Promise<GetSubmissionStatusResponse> {
+  if (!submissionId) {
+    return reply.code(400).send({
+      description: "submission_id must be provided",
+    });
+  }
+  const submission = await getSubmissionWithExtractions({
+    member,
+    submissionId,
+  });
+  if (!submission)
+    return reply.code(404).send({
+      description: "Submission not found or you don't have permission to access it",
+    });
+
+  const res = submission.extractions.every((e) => e.status === "done" || e.status === "failed")
+    ? "done"
+    : "pending";
+  return { status: res };
+}
+
+export async function listSubmissionsHandler(
+  { formId, dealId, afterDate }: ListSubmissionsQuery,
+  member: MemberHydrated,
+  reply: FastifyReply,
+): Promise<ListSubmissionsResponse[]> {
+  try {
+    const submissions = await getSubmissions({ member, formId, dealId, afterDate });
+    return submissions;
+  } catch (error) {
+    console.error(error);
+    return reply.code(500).send({ description: "Failed to get submissions" });
+  }
+}
+
+export async function getSubmissionDataHandler(
+  { submissionId }: GetSubmissionParams,
+  member: MemberHydrated,
+  reply: FastifyReply,
+): Promise<GetSubmissionResponse> {
+  const submission = await getSubmissionWithData({ member, submissionId });
+  if (!submission) {
+    return reply.code(404).send({
+      description:
+        "The submission you requested does not exist or you don't have permission to access it",
+    });
+  }
+
+  // Refresh URLs for all documents in the requested submission
+  await Promise.allSettled(submission.extractions.map((e) => checkAndRefreshUrl(e.document)));
+
+  const formGroups = submission.form.groups satisfies ApiFormGroup[];
+  const groups = submission.groups.map((group) => {
+    const items = group.items.map((item) => {
+      const version = item.versions[0];
+      return {
+        id: item.id,
+        formField: {
+          id: item.formField.id,
+          name: item.formField.name,
+          hint: item.formField.hint,
+          order: item.formField.order,
+        },
+        value: version?.value ?? null,
+        sourceId: version?.sourceId ?? null,
+        citation: version?.documentSource ?? null,
+        documentCitationId: version?.documentSource?.id ?? null,
+        documentExtractionId: version?.documentSource?.documentExtractionId ?? null,
+      } satisfies ApiSubmissionItem & { citation: DocumentCitation | null };
+    });
+    return {
+      id: group.id,
+      order: group.order,
+      formGroupId: group.formGroupId,
+      items: [...items].sort((a, b) => a.formField.order - b.formField.order),
+    } satisfies ApiSubmissionGroup;
+  });
+
+  // Generate extracted pdfs
+  const documents = submission.extractions.map((e) => {
+    const citations = groups
+      .flatMap((group) => group.items)
+      .filter((item) => item.documentExtractionId === e.id && item.documentCitationId)
+      .map((item) => item.citation!);
+
+    return {
+      documentId: e.document.id,
+      documentExtractionId: e.id,
+      url: e.document.blobUrl,
+      urlExpiry: e.document.blobUrlExpiry,
+      name: e.document.name,
+      status: e.status,
+      extractionModel: e.extractionModel,
+      extractedData: e.data as ApiSubmissionDocument["extractedData"],
+      citations,
+      reconciliation: e.reconciliation as ApiSubmissionDocument["reconciliation"],
+    } satisfies ApiSubmissionDocument;
+  });
+
+  const { name, formId, dealId, status } = submission;
+
+  return { submissionId, name, formId, dealId, status, formGroups, groups, documents };
+}
+
+export async function getDocumentExtractionHandler(
+  { documentExtractionId }: DocumentExtractionDataQuery,
+  member: MemberHydrated,
+  reply: FastifyReply,
+) {
+  try {
+    const extraction = await getDocumentExtraction({ member, documentExtractionId });
+    return extraction;
+  } catch (error) {
+    console.error(error);
+    return reply.code(404).send({
+      description: "Document extraction not found or you don't have permission to access it",
+    });
+  }
+}
+
+async function uploadFile(
+  file: File,
+  config: Config,
+  submissionId: string,
+  dealId: string,
+  model: string,
+  userPrompt?: string,
+) {
+  const blobName = `${nanoid()}-${file.name}`;
+
+  // TODO: remove this nonsense once we merge all the models we use
+  const { client, displayType, storageProvider } = getExtractionSetup(config);
+  await client.uploadFile(Buffer.from(await file.arrayBuffer()), blobName, file.type);
+  const { url, expiry } = await client.getSignedUrl(blobName);
+  const [insertedDocument] = await db
+    .insert(schema.document)
+    .values({
+      dealId: dealId,
+      name: file.name,
+      contentType: file.type,
+      blobKey: blobName,
+      blobUrl: url,
+      blobUrlExpiry: expiry,
+      displayType,
+      storageProvider,
+      uploaded: true,
+    })
+    .returning({ id: schema.document.id });
+
+  invariant(insertedDocument);
+  const [documentExtraction] = await db
+    .insert(schema.documentExtraction)
+    .values({
+      documentId: insertedDocument.id,
+      submissionId,
+      status: "pending",
+      extractionModel: model,
+      extractionTrigger: "api",
+    })
+    .returning({ id: schema.documentExtraction.id });
+
+  invariant(documentExtraction);
+  const hatchetClient = new HatchetClientMgr(config.hatchet);
+  await hatchetClient.pushDoc(documentExtraction.id, model, userPrompt ?? "");
+
+  return { documentId: insertedDocument.id, extractionId: documentExtraction.id };
+}
+
+export async function extractDataHandler(
+  { files, model, userPrompt, submissionId }: ExtractDataBody & { submissionId: string },
+  member: MemberHydrated,
+  apiKeyId: string,
+  reply: FastifyReply,
+  channel: Channel,
+) {
+  const canExtract = await checkCanExtract({ member });
+  if (!canExtract) {
+    const secondsUntilEndOfMonth = differenceInSeconds(endOfMonth(new Date()), new Date());
+    return reply.code(429).header("Retry-After", String(secondsUntilEndOfMonth)).send({
+      statusCode: 429,
+      message:
+        "Out of extractions for the month. Please upgrade your plan at app.sea.dev or get in touch: matt@sea.dev",
+    });
+  }
+  const submission = await getSubmission({ member, submissionId });
+  if (!submission) {
+    return reply.code(400).send({
+      description: "Failed to get submission",
+    });
+  }
+
+  const filesArray = Array.isArray(files) ? files : [files];
+  const results = await Promise.allSettled(
+    filesArray.map(async (file) => {
+      return uploadFile(file, config, submissionId, submission.dealId, model, userPrompt);
+    }),
+  );
+  const failed = results.filter((r) => r.status === "rejected");
+  const extractions = results
+    .filter((result) => result.status === "fulfilled")
+    .map((result) => result.value);
+
+  captureEvent({
+    event: "submission_documents_uploaded",
+    distinctId: member.organizationId,
+    properties: {
+      submission_id: submission.id,
+      organization_id: member.organizationId,
+      user_id: member.user.id,
+      model,
+      documents_count: filesArray.length,
+      failures: failed.length,
+      channel,
+    },
+    groups: { company: member.organizationId },
+  });
+
+  // Special event for specific API key
+  if (apiKeyId === "lw88ttoldghpqz833w5m8" || apiKeyId === "ebi2obyph35hgkjztudis") {
+    captureEvent({
+      event: "special_api_upload",
+      distinctId: member.organizationId,
+      properties: {
+        submission_id: submission.id,
+        deal_id: submission.dealId,
+        user_name: member.user.name,
+        model,
+        document_names: filesArray.map((file) => file.name),
+        failures: failed.length,
+        channel,
+      },
+      groups: { company: member.organizationId },
+    });
+  }
+
+  return { submissionId: submission.id, extractions, failures: failed.length };
+}
+
+export async function submissionItemUpdate(
+  {
+    value,
+    submissionId,
+    itemId: submissionItemId,
+  }: { value: string | number; submissionId: string; itemId: string },
+  member: MemberHydrated,
+  reply: FastifyReply,
+) {
+  try {
+    if (!submissionId) {
+      reply.code(400).send("submissionItemId is required");
+    }
+    if (!submissionItemId) {
+      reply.code(400).send("submissionItemId is required");
+    }
+    await updateSubmissionValue({ member, submissionItemId, value });
+  } catch (error) {
+    console.error(error);
+    if (error instanceof InvariantError) {
+      return reply.code(404).send({ description: error.message });
+    }
+    return reply.code(500).send({ description: "Failed to update form submission item" });
+  }
+}
+
+export async function createFieldCorrectionHandler(
+  {
+    itemId: submissionItemId,
+    extractedValue,
+    correctedValue,
+    feedback,
+  }: {
+    itemId: string;
+    extractedValue: string | number | null;
+    correctedValue: string | null;
+    feedback: string | null;
+  },
+  member: MemberHydrated,
+  reply: FastifyReply,
+) {
+  try {
+    const correction = await createCorrection({
+      member,
+      submissionItemId,
+      extractedValue,
+      correctedValue,
+      feedback,
+    });
+    return correction;
+  } catch (error) {
+    console.error(error);
+    if (error instanceof InvariantError) {
+      return reply.code(404).send({ description: error.message });
+    }
+    return reply.code(500).send({ description: "Failed to update field correction" });
+  }
+}
+
+export async function getFieldCorrectionsHandler(
+  { itemId: submissionItemId }: { itemId: string },
+  member: MemberHydrated,
+  reply: FastifyReply,
+) {
+  try {
+    const corrections = await getCorrectionsForItem({
+      member,
+      submissionItemId,
+    });
+    return corrections;
+  } catch (error) {
+    console.error(error);
+    if (error instanceof InvariantError) {
+      return reply.code(404).send({ description: error.message });
+    }
+    return reply.code(500).send({ description: "Failed to get field correction" });
+  }
+}
+
+export async function updateSubmissionHandler(
+  { submissionId, ...updateData }: { submissionId: string; name?: string; summary?: string },
+  member: MemberHydrated,
+  reply: FastifyReply,
+) {
+  try {
+    const submission = await updateSubmission({ member, submissionId, ...updateData });
+    return submission;
+  } catch (error) {
+    console.error(error);
+    return reply.code(404).send({ description: "Submission not found" });
+  }
+}
+
+export async function markCorrectionAddressedHandler(
+  { correctionId }: { correctionId: string },
+  member: MemberHydrated,
+  reply: FastifyReply,
+) {
+  try {
+    await markAddressed({ member, correctionId });
+    return { success: true };
+  } catch (error) {
+    console.error(error);
+    return reply.code(404).send({ description: "Correction not found" });
+  }
+}
+
+export async function createSubmissionGroupHandler(
+  { submissionId, formGroupId }: { submissionId: string; formGroupId: string },
+  member: MemberHydrated,
+  reply: FastifyReply,
+) {
+  try {
+    const submissionGroup = await addSubmissionGroupRow({ member, submissionId, formGroupId });
+    return submissionGroup;
+  } catch (error) {
+    console.error(error);
+    return reply.code(400).send({ description: "Failed to create submission group" });
+  }
+}
+
+export async function updateSubmissionGroupHandler(
+  {
+    submissionId,
+    groupId,
+    ...updateData
+  }: { submissionId: string; groupId: string; isValid?: boolean; order?: number },
+  member: MemberHydrated,
+  reply: FastifyReply,
+) {
+  try {
+    const submissionGroup = await updateSubmissionGroup({
+      member,
+      submissionId,
+      submissionGroupId: groupId,
+      ...updateData,
+    });
+    return submissionGroup;
+  } catch (error) {
+    console.error(error);
+    return reply.code(404).send({ description: "Submission group not found" });
+  }
+}