sea-dev 1.0.0

package/libs/dal/src/submission.ts

@@ -0,0 +1,1396 @@
+import { db, schema } from "@sea/db";
+import type { SubmissionItemValue } from "@sea/db/schema";
+import type { Member } from "@sea/db/types";
+import { invariant, InvariantError } from "@sea/util/invariant";
+import { and, asc, desc, eq, gt, inArray, isNull } from "drizzle-orm";
+import { getForm } from "./form";
+
+interface DenormItem {
+  id: string;
+  name: string;
+  type:
+    | "string"
+    | "number"
+    | "boolean"
+    | "date"
+    | "email"
+    | "string-long"
+    | "money"
+    | "url"
+    | "enum";
+  hint: string;
+  value: SubmissionItemValue | null;
+  citationId?: string | null;
+}
+
+interface DenormDerived {
+  id: string;
+  name: string;
+  prompt: string;
+  value: string | null;
+}
+
+export interface DenormGroup {
+  id: string;
+  formGroupId: string;
+  name: string;
+  multi: boolean;
+  hint: string;
+  items: DenormItem[];
+  derived: DenormDerived[];
+}
+
+export interface DenormTable {
+  formGroupId: string;
+  name: string;
+  multi: boolean;
+  hint: string;
+  columns: string[];
+  rows: {
+    items: DenormItem[];
+    derived: DenormDerived[];
+  }[];
+}
+
+export type DenormSubmission = {
+  id: string;
+  name: string | null;
+  summary: string | null;
+  formName: string;
+  hint: string;
+  groups: DenormGroup[];
+  tables: DenormTable[];
+};
+
+export async function getSubmission({
+  member,
+  submissionId,
+  typeFilter = "extraction",
+}: {
+  member: Member;
+  submissionId: string;
+  typeFilter?: "extraction" | "transformation" | "all";
+}) {
+  const conditions = [
+    eq(schema.submission.organizationId, member.organizationId),
+    eq(schema.submission.id, submissionId),
+    isNull(schema.submission.deletedAt),
+  ];
+
+  // Add type filter unless "all" is specified
+  if (typeFilter !== "all") {
+    conditions.push(eq(schema.submission.type, typeFilter));
+  }
+
+  const submission = await db.query.submission.findFirst({
+    where: and(...conditions),
+  });
+  invariant(submission, `no submission found for ${submissionId}`);
+  return submission;
+}
+
+export async function getSubmissionGroup({
+  member,
+  submissionId,
+  submissionGroupId,
+}: {
+  member: Member;
+  submissionId: string;
+  submissionGroupId: string;
+}) {
+  const submission = await db.query.submission.findFirst({
+    where: and(
+      eq(schema.submission.organizationId, member.organizationId),
+      eq(schema.submission.id, submissionId),
+      eq(schema.submission.type, "extraction"),
+      isNull(schema.submission.deletedAt),
+    ),
+    with: {
+      groups: {
+        where: eq(schema.submissionGroup.id, submissionGroupId),
+        orderBy: asc(schema.submissionGroup.order),
+      },
+    },
+  });
+  invariant(submission, `no submission found for ${submissionId}`);
+  const [group] = submission.groups;
+  invariant(group, `no submissionGroup found for ${submissionGroupId}`);
+
+  return group;
+}
+
+export async function getDenormSubmission({
+  member,
+  submissionId,
+  withCitations = false,
+  withTransformations = false,
+}: {
+  member: Member;
+  submissionId: string;
+  withCitations?: boolean;
+  withTransformations?: boolean;
+}): Promise<DenormSubmission> {
+  const typeFilter = withTransformations ? undefined : eq(schema.submission.type, "extraction");
+  const baseQuery = {
+    where: and(
+      eq(schema.submission.organizationId, member.organizationId),
+      eq(schema.submission.id, submissionId),
+      typeFilter,
+      isNull(schema.submission.deletedAt),
+    ),
+  };
+
+  const submission = withCitations
+    ? await db.query.submission.findFirst({
+        ...baseQuery,
+        with: {
+          form: true,
+          groups: {
+            orderBy: asc(schema.submissionGroup.order),
+            with: {
+              formGroup: true,
+              derived: {
+                with: {
+                  formDerived: true,
+                },
+              },
+              items: {
+                with: {
+                  formField: true,
+                  versions: {
+                    orderBy: [
+                      desc(schema.submissionItemVersion.selected),
+                      desc(schema.submissionItemVersion.updatedAt),
+                    ],
+                    with: { documentSource: true },
+                    limit: 1,
+                  },
+                },
+              },
+            },
+          },
+        },
+      })
+    : await db.query.submission.findFirst({
+        ...baseQuery,
+        with: {
+          form: true,
+          groups: {
+            orderBy: asc(schema.submissionGroup.order),
+            with: {
+              formGroup: true,
+              derived: {
+                with: {
+                  formDerived: true,
+                },
+              },
+              items: {
+                with: {
+                  formField: true,
+                  versions: {
+                    orderBy: [
+                      desc(schema.submissionItemVersion.selected),
+                      desc(schema.submissionItemVersion.updatedAt),
+                    ],
+                    limit: 1,
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+  invariant(submission, `no submission found for ${submissionId}`);
+
+  const denorm = {
+    id: submission.id,
+    name: submission.name,
+    summary: submission.summary,
+    formName: submission.form.name,
+    hint: submission.form.hint,
+    groups: submission.groups.map((group) => ({
+      id: group.id,
+      formGroupId: group.formGroup.id,
+      name: group.formGroup.name,
+      multi: group.formGroup.multi,
+      hint: group.formGroup.hint,
+      order: group.formGroup.order,
+      items: group.items
+        .map((item) => {
+          const version = item.versions.at(0);
+          return {
+            id: item.id,
+            name: item.formField.name,
+            type: item.formField.type,
+            hint: item.formField.hint,
+            order: item.formField.order,
+            value: version?.value ?? null,
+            citationId:
+              withCitations && version && "documentSource" in version
+                ? (version.documentSource?.id ?? null)
+                : undefined,
+          };
+        })
+        .sort((a, b) => a.order - b.order),
+      derived: group.derived.map((item) => ({
+        id: item.id,
+        name: item.formDerived.name,
+        prompt: item.formDerived.prompt,
+        value: item.value,
+      })),
+    })),
+  };
+
+  const processedDenorm = {
+    ...denorm,
+    groups: denorm.groups.filter((g) => !g.multi).sort((a, b) => a.order - b.order),
+    tables: denorm.groups
+      .filter((g) => g.multi === true)
+      .sort((a, b) => a.order - b.order)
+      .reduce((acc, group) => {
+        const existing = acc.find((t) => t.formGroupId === group.formGroupId);
+        if (existing) {
+          existing.rows.push({ items: group.items, derived: group.derived });
+        } else {
+          acc.push({
+            ...group,
+            columns: [...group.items.map((i) => i.name), ...group.derived.map((d) => d.name)],
+            rows: [{ items: group.items, derived: group.derived }],
+          });
+        }
+        return acc;
+      }, [] as DenormTable[]),
+  };
+
+  return processedDenorm;
+}
+
+export async function getDenormSubmissions({
+  member,
+  submissionIds,
+  withCitations = false,
+  withTransformations = false,
+}: {
+  member: Member;
+  submissionIds: string[];
+  withCitations?: boolean;
+  withTransformations?: boolean;
+}): Promise<DenormSubmission[]> {
+  if (submissionIds.length === 0) {
+    return [];
+  }
+
+  const typeFilter = withTransformations ? undefined : eq(schema.submission.type, "extraction");
+  const baseQuery = {
+    where: and(
+      eq(schema.submission.organizationId, member.organizationId),
+      inArray(schema.submission.id, submissionIds),
+      typeFilter,
+      isNull(schema.submission.deletedAt),
+    ),
+  };
+
+  const submissions = withCitations
+    ? await db.query.submission.findMany({
+        ...baseQuery,
+        with: {
+          form: true,
+          groups: {
+            orderBy: asc(schema.submissionGroup.order),
+            with: {
+              formGroup: true,
+              derived: {
+                with: {
+                  formDerived: true,
+                },
+              },
+              items: {
+                with: {
+                  formField: true,
+                  versions: {
+                    orderBy: [
+                      desc(schema.submissionItemVersion.selected),
+                      desc(schema.submissionItemVersion.updatedAt),
+                    ],
+                    with: { documentSource: true },
+                    limit: 1,
+                  },
+                },
+              },
+            },
+          },
+        },
+      })
+    : await db.query.submission.findMany({
+        ...baseQuery,
+        with: {
+          form: true,
+          groups: {
+            orderBy: asc(schema.submissionGroup.order),
+            with: {
+              formGroup: true,
+              derived: {
+                with: {
+                  formDerived: true,
+                },
+              },
+              items: {
+                with: {
+                  formField: true,
+                  versions: {
+                    orderBy: [
+                      desc(schema.submissionItemVersion.selected),
+                      desc(schema.submissionItemVersion.updatedAt),
+                    ],
+                    limit: 1,
+                  },
+                },
+              },
+            },
+          },
+        },
+      });
+
+  return submissions.map((submission) => {
+    const denorm = {
+      id: submission.id,
+      name: submission.name,
+      summary: submission.summary,
+      formName: submission.form.name,
+      hint: submission.form.hint,
+      groups: submission.groups.map((group) => ({
+        id: group.id,
+        formGroupId: group.formGroup.id,
+        name: group.formGroup.name,
+        multi: group.formGroup.multi,
+        hint: group.formGroup.hint,
+        order: group.formGroup.order,
+        items: group.items
+          .map((item) => {
+            const version = item.versions.at(0);
+            return {
+              id: item.id,
+              name: item.formField.name,
+              type: item.formField.type,
+              hint: item.formField.hint,
+              order: item.formField.order,
+              value: version?.value ?? null,
+              citationId:
+                withCitations && version && "documentSource" in version
+                  ? (version.documentSource?.id ?? null)
+                  : undefined,
+            };
+          })
+          .sort((a, b) => a.order - b.order),
+        derived: group.derived.map((item) => ({
+          id: item.id,
+          name: item.formDerived.name,
+          prompt: item.formDerived.prompt,
+          value: item.value,
+        })),
+      })),
+    };
+
+    const processedDenorm = {
+      ...denorm,
+      groups: denorm.groups.filter((g) => !g.multi).sort((a, b) => a.order - b.order),
+      tables: denorm.groups
+        .filter((g) => g.multi === true)
+        .sort((a, b) => a.order - b.order)
+        .reduce((acc, group) => {
+          const existing = acc.find((t) => t.formGroupId === group.formGroupId);
+          if (existing) {
+            existing.rows.push({ items: group.items, derived: group.derived });
+          } else {
+            acc.push({
+              ...group,
+              columns: [...group.items.map((i) => i.name), ...group.derived.map((d) => d.name)],
+              rows: [{ items: group.items, derived: group.derived }],
+            });
+          }
+          return acc;
+        }, [] as DenormTable[]),
+    };
+
+    return processedDenorm;
+  });
+}
+
+export async function getSubmissionWithData({
+  member,
+  submissionId,
+  typeFilter = "extraction",
+}: {
+  member: Member;
+  submissionId: string;
+  typeFilter?: "extraction" | "transformation" | "all";
+}) {
+  const conditions = [
+    eq(schema.submission.organizationId, member.organizationId),
+    eq(schema.submission.id, submissionId),
+    isNull(schema.submission.deletedAt),
+  ];
+
+  // Add type filter unless "all" is specified
+  if (typeFilter !== "all") {
+    conditions.push(eq(schema.submission.type, typeFilter));
+  }
+
+  const submission = await db.query.submission.findFirst({
+    where: and(...conditions),
+    with: {
+      extractions: {
+        with: { document: true, citations: true },
+        where: isNull(schema.documentExtraction.deletedAt),
+      },
+      deal: true,
+      form: {
+        with: {
+          workflow: true,
+          groups: {
+            with: {
+              fields: { orderBy: asc(schema.formField.order) },
+              derived: true,
+            },
+            orderBy: asc(schema.formGroup.order),
+          },
+        },
+      },
+      conversation: {
+        with: {
+          messages: {
+            orderBy: asc(schema.message.createdAt),
+          },
+        },
+      },
+      groups: {
+        with: {
+          derived: {
+            with: { formDerived: true },
+          },
+          items: {
+            with: {
+              formField: true,
+              versions: {
+                orderBy: [
+                  desc(schema.submissionItemVersion.selected),
+                  desc(schema.submissionItemVersion.updatedAt),
+                ],
+                with: { documentSource: true, messageSource: true },
+                limit: 1,
+              },
+            },
+          },
+        },
+        orderBy: [schema.submissionGroup.order],
+      },
+    },
+  });
+  return submission;
+}
+
+export async function getSubmissionWithExtractions({
+  member,
+  submissionId,
+  typeFilter = "extraction",
+}: {
+  member: Member;
+  submissionId: string;
+  typeFilter?: "extraction" | "transformation" | "all";
+}) {
+  const conditions = [
+    eq(schema.submission.organizationId, member.organizationId),
+    eq(schema.submission.id, submissionId),
+    isNull(schema.submission.deletedAt),
+  ];
+
+  // Add type filter unless "all" is specified
+  if (typeFilter !== "all") {
+    conditions.push(eq(schema.submission.type, typeFilter));
+  }
+
+  const submission = await db.query.submission.findFirst({
+    with: {
+      extractions: {
+        where: isNull(schema.documentExtraction.deletedAt),
+      },
+    },
+    where: and(...conditions),
+  });
+  invariant(submission);
+
+  return submission;
+}
+
+export async function getSubmissionWithFormAndExtractions({
+  member,
+  submissionId,
+}: {
+  member: Member;
+  submissionId: string;
+}) {
+  const submission = await db.query.submission.findFirst({
+    where: and(
+      eq(schema.submission.organizationId, member.organizationId),
+      eq(schema.submission.id, submissionId),
+      isNull(schema.submission.deletedAt),
+    ),
+    with: {
+      form: {
+        with: {
+          groups: {
+            with: {
+              fields: { orderBy: asc(schema.formField.order) },
+              derived: { orderBy: asc(schema.formDerived.name) },
+            },
+            orderBy: asc(schema.formGroup.order),
+          },
+        },
+      },
+      groups: {
+        with: {
+          derived: {
+            with: { formDerived: true },
+          },
+          items: {
+            with: {
+              formField: true,
+              versions: {
+                orderBy: [
+                  desc(schema.submissionItemVersion.selected),
+                  desc(schema.submissionItemVersion.updatedAt),
+                ],
+                with: {
+                  documentSource: true,
+                  messageSource: true,
+                },
+                limit: 1,
+              },
+            },
+          },
+        },
+        orderBy: asc(schema.submissionGroup.order),
+      },
+      extractions: {
+        with: {
+          document: true,
+          citations: true,
+        },
+      },
+    },
+  });
+
+  if (!submission) return null;
+
+  // Check if any item versions have transformation sources
+  // This can happen in any submission type (extraction or transformation)
+  const transformationItems: Array<{ itemId: string; versionId: string; sourceId: string }> = [];
+  for (const group of submission.groups) {
+    for (const item of group.items) {
+      const version = item.versions[0];
+      if (version?.sourceType === "transformation" && version.sourceId) {
+        transformationItems.push({
+          itemId: item.id,
+          versionId: version.id,
+          sourceId: version.sourceId,
+        });
+      }
+    }
+  }
+
+  const hasTransformationSources = transformationItems.length > 0;
+
+  console.log(
+    `[getSubmissionWithFormAndExtractions] Submission ${submissionId}: found ${transformationItems.length} items with transformation sources`,
+  );
+
+  if (hasTransformationSources) {
+    console.log(
+      `[getSubmissionWithFormAndExtractions] Loading transformation citations for ${transformationItems.length} items`,
+    );
+
+    // Collect initial transformation sourceIds from item versions
+    const initialTransformationIds: string[] = [];
+    for (const group of submission.groups) {
+      for (const item of group.items) {
+        const version = item.versions[0];
+        if (version?.sourceType === "transformation" && version.sourceId) {
+          initialTransformationIds.push(version.sourceId);
+        }
+      }
+    }
+
+    // Iteratively load all transformations and source versions to arbitrary depth
+    const allDataTransformations: Awaited<ReturnType<typeof db.query.dataTransformation.findMany>> =
+      [];
+    const allSourceVersions: Awaited<
+      ReturnType<
+        typeof db.query.submissionItemVersion.findMany<{
+          with: {
+            documentSource: true;
+            submissionItem: {
+              with: {
+                formField: true;
+                group: {
+                  with: {
+                    formGroup: true;
+                    submission: {
+                      with: {
+                        form: true;
+                      };
+                    };
+                  };
+                };
+              };
+            };
+          };
+        }>
+      >
+    > = [];
+
+    let transformationIdsToLoad = initialTransformationIds;
+    const loadedTransformationIds = new Set<string>();
+
+    while (transformationIdsToLoad.length > 0) {
+      // Load transformations
+      const transformations = await db.query.dataTransformation.findMany({
+        where: inArray(schema.dataTransformation.id, transformationIdsToLoad),
+      });
+
+      allDataTransformations.push(...transformations);
+      transformationIdsToLoad.forEach((id) => loadedTransformationIds.add(id));
+
+      // Get all source version IDs from these transformations
+      const sourceVersionIds = transformations.flatMap((dt) => dt.sourceItemVersionIds ?? []);
+
+      // Load source versions with full context
+      if (sourceVersionIds.length > 0) {
+        const sourceVersions = await db.query.submissionItemVersion.findMany({
+          where: inArray(schema.submissionItemVersion.id, sourceVersionIds),
+          with: {
+            documentSource: true,
+            submissionItem: {
+              with: {
+                formField: true,
+                group: {
+                  with: {
+                    formGroup: true,
+                    submission: {
+                      with: {
+                        form: true,
+                      },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        });
+
+        allSourceVersions.push(...sourceVersions);
+
+        // Check for more nested transformations
+        transformationIdsToLoad = sourceVersions
+          .filter((v) => v.sourceType === "transformation" && v.sourceId)
+          .map((v) => v.sourceId!)
+          .filter((id) => !loadedTransformationIds.has(id));
+      } else {
+        transformationIdsToLoad = [];
+      }
+    }
+
+    // Build a map of version ID -> full version with item context
+    const versionIdToVersion = new Map(allSourceVersions.map((version) => [version.id, version]));
+
+    // Build a map of transformation ID -> source versions (with full item context)
+    const transformationToSourceVersions = new Map<string, typeof allSourceVersions>();
+    // Build a map of transformation ID -> description
+    const transformationToDescription = new Map<string, string | null>();
+    for (const dt of allDataTransformations) {
+      const versions: typeof allSourceVersions = [];
+      const missingVersionIds: string[] = [];
+      for (const versionId of dt.sourceItemVersionIds ?? []) {
+        const version = versionIdToVersion.get(versionId);
+        if (version) {
+          versions.push(version);
+        } else {
+          missingVersionIds.push(versionId);
+        }
+      }
+      transformationToSourceVersions.set(dt.id, versions);
+      transformationToDescription.set(dt.id, dt.description ?? null);
+
+      // Debug: Log transformation descriptions
+      if (dt.description) {
+        console.log(
+          `[getSubmissionWithFormAndExtractions] Transformation ${dt.id} has description: "${dt.description}"`,
+        );
+      } else {
+        console.log(
+          `[getSubmissionWithFormAndExtractions] Transformation ${dt.id} has NO description`,
+        );
+      }
+
+      if (missingVersionIds.length > 0) {
+        console.warn(
+          `[getSubmissionWithFormAndExtractions] Transformation ${dt.id} references ${missingVersionIds.length} missing version IDs: ${missingVersionIds.join(", ")}`,
+        );
+      }
+      if (versions.length === 0 && (dt.sourceItemVersionIds?.length ?? 0) > 0) {
+        console.warn(
+          `[getSubmissionWithFormAndExtractions] Transformation ${dt.id} has sourceItemVersionIds but NO versions were found!`,
+        );
+      }
+    }
+
+    // Helper function to recursively attach transformationSource to versions
+    const attachTransformationSource = (
+      version: (typeof allSourceVersions)[0],
+    ): (typeof allSourceVersions)[0] => {
+      if (version.sourceType === "transformation" && version.sourceId) {
+        const nestedSourceVersions = transformationToSourceVersions.get(version.sourceId) ?? [];
+        const transformationDescription = transformationToDescription.get(version.sourceId) ?? null;
+        // Recursively attach transformationSource to nested versions
+        const enrichedSourceVersions: typeof allSourceVersions = nestedSourceVersions.map((sv) =>
+          attachTransformationSource(sv),
+        );
+        return {
+          ...version,
+          transformationSource: enrichedSourceVersions,
+          transformationDescription,
+        } as typeof version & {
+          transformationSource: typeof enrichedSourceVersions;
+          transformationDescription: string | null;
+        };
+      }
+      return version;
+    };
+
+    // Transform groups and attach source item versions to transformations
+    const groups = submission.groups.map((group) => {
+      const items = group.items.map((item) => {
+        const version = item.versions[0];
+        let selectedVersion = version ?? null;
+
+        // If this version is from a transformation, attach the source item versions recursively
+        if (version?.sourceType === "transformation" && version.sourceId) {
+          const sourceVersions = transformationToSourceVersions.get(version.sourceId) ?? [];
+
+          if (sourceVersions.length === 0) {
+            console.warn(
+              `[getSubmissionWithFormAndExtractions] WARNING: Item ${item.id} (field: ${item.formField.name}) has sourceType=transformation with sourceId=${version.sourceId} but NO source versions found!`,
+            );
+          }
+
+          // Recursively attach transformationSource to nested versions
+          const enrichedSourceVersions = sourceVersions.map((sv) => attachTransformationSource(sv));
+
+          // Get the description for this transformation
+          const description = transformationToDescription.get(version.sourceId) ?? null;
+
+          selectedVersion = {
+            ...version,
+            transformationSource: enrichedSourceVersions,
+            transformationDescription: description,
+          } as typeof selectedVersion & {
+            transformationSource: typeof enrichedSourceVersions;
+            transformationDescription: string | null;
+          };
+        }
+
+        return {
+          ...item,
+          selectedVersion,
+        };
+      });
+      return {
+        ...group,
+        items: [...items].sort((a, b) => a.formField.order - b.formField.order),
+      };
+    });
+
+    return {
+      ...submission,
+      groups,
+    };
+  }
+
+  // No transformation sources, use the standard logic
+  const groups = submission.groups.map((group) => {
+    const items = group.items.map((item) => ({
+      ...item,
+      selectedVersion: item.versions[0] ?? null,
+    }));
+    return {
+      ...group,
+      items: [...items].sort((a, b) => a.formField.order - b.formField.order),
+    };
+  });
+
+  return {
+    ...submission,
+    groups,
+  };
+}
+
+export async function getSubmissions({
+  member,
+  formId,
+  dealId,
+  afterDate,
+  typeFilter = "extraction",
+}: {
+  member: Member;
+  formId?: string;
+  dealId?: string;
+  afterDate?: string;
+  typeFilter?: "extraction" | "transformation" | "all";
+}) {
+  const conditions = [
+    eq(schema.submission.organizationId, member.organizationId),
+    isNull(schema.submission.deletedAt),
+  ];
+
+  // Add type filter unless "all" is specified
+  if (typeFilter !== "all") {
+    conditions.push(eq(schema.submission.type, typeFilter));
+  }
+
+  if (formId) {
+    conditions.push(eq(schema.submission.formId, formId));
+  }
+  if (dealId) {
+    conditions.push(eq(schema.submission.dealId, dealId));
+  }
+  if (afterDate) {
+    conditions.push(gt(schema.submission.createdAt, new Date(afterDate)));
+  }
+  const submissions = await db.query.submission.findMany({
+    where: and(...conditions),
+    with: { form: { with: { workflow: true } } },
+  });
+  return submissions;
+}
+
+export async function syncSubmissionToForm({
+  member,
+  submissionId,
+}: {
+  member: Member;
+  submissionId: string;
+}) {
+  const submission = await getSubmission({ member, submissionId, typeFilter: "all" });
+  await db.transaction(async (tx) => {
+    // insert all missing non-multi groups, if any
+    const missingGroupIds = await tx
+      .select({ id: schema.formGroup.id })
+      .from(schema.formGroup)
+      .leftJoin(
+        schema.submissionGroup,
+        and(
+          eq(schema.submissionGroup.formGroupId, schema.formGroup.id),
+          eq(schema.submissionGroup.submissionId, submissionId),
+        ),
+      )
+      .where(
+        and(
+          eq(schema.formGroup.formId, submission.formId),
+          eq(schema.formGroup.multi, false),
+          isNull(schema.submissionGroup.id),
+        ),
+      );
+
+    if (missingGroupIds.length > 0) {
+      await tx.insert(schema.submissionGroup).values(
+        missingGroupIds.map(({ id }) => ({
+          submissionId,
+          formGroupId: id,
+          order: 0,
+        })),
+      );
+    }
+
+    // insert all missing submissionItems, if any
+    const missingItems = await tx
+      .select({
+        submissionGroupId: schema.submissionGroup.id,
+        formFieldId: schema.formField.id,
+      })
+      .from(schema.submissionGroup)
+      .innerJoin(
+        schema.formField,
+        eq(schema.formField.formGroupId, schema.submissionGroup.formGroupId),
+      )
+      .leftJoin(
+        schema.submissionItem,
+        and(
+          eq(schema.submissionItem.submissionGroupId, schema.submissionGroup.id),
+          eq(schema.submissionItem.formFieldId, schema.formField.id),
+        ),
+      )
+      .where(
+        and(
+          eq(schema.submissionGroup.submissionId, submissionId),
+          isNull(schema.submissionItem.id), // doesn't exist yet
+        ),
+      );
+
+    if (missingItems.length > 0) {
+      await tx.insert(schema.submissionItem).values(missingItems);
+    }
+
+    // insert all missing submissionDeriveds, if any
+    const missingDeriveds = await tx
+      .select({
+        submissionGroupId: schema.submissionGroup.id,
+        formDerivedId: schema.formDerived.id,
+      })
+      .from(schema.submissionGroup)
+      .innerJoin(
+        schema.formDerived,
+        eq(schema.formDerived.formGroupId, schema.submissionGroup.formGroupId),
+      )
+      .leftJoin(
+        schema.submissionDerived,
+        and(
+          eq(schema.submissionDerived.submissionGroupId, schema.submissionGroup.id),
+          eq(schema.submissionDerived.formDerivedId, schema.formDerived.id),
+        ),
+      )
+      .where(
+        and(
+          eq(schema.submissionGroup.submissionId, submissionId),
+          isNull(schema.submissionDerived.id),
+        ),
+      );
+
+    if (missingDeriveds.length > 0) {
+      await tx.insert(schema.submissionDerived).values(missingDeriveds);
+    }
+  });
+}
+
+export async function resetSubmission({
+  member,
+  submissionId,
+}: {
+  member: Member;
+  submissionId: string;
+}) {
+  try {
+    // Validate access
+    const submission = await getSubmission({ member, submissionId });
+
+    await db.transaction(async (tx) => {
+      // Delete all item versions that relate to the current submission
+      await tx
+        .delete(schema.submissionItemVersion)
+        .where(
+          inArray(
+            schema.submissionItemVersion.submissionItemId,
+            tx
+              .select({ id: schema.submissionItem.id })
+              .from(schema.submissionItem)
+              .innerJoin(
+                schema.submissionGroup,
+                eq(schema.submissionItem.submissionGroupId, schema.submissionGroup.id),
+              )
+              .where(eq(schema.submissionGroup.submissionId, submission.id)),
+          ),
+        );
+
+      // Reset derived fields
+      await tx
+        .update(schema.submissionDerived)
+        .set({ value: null })
+        .where(
+          inArray(
+            schema.submissionDerived.id,
+            tx
+              .select({ id: schema.submissionDerived.id })
+              .from(schema.submissionDerived)
+              .innerJoin(
+                schema.submissionGroup,
+                eq(schema.submissionDerived.submissionGroupId, schema.submissionGroup.id),
+              )
+              .where(eq(schema.submissionGroup.submissionId, submission.id)),
+          ),
+        );
+
+      // Delete all multisubmission groups for this submission with org validation
+      await tx.delete(schema.submissionGroup).where(
+        inArray(
+          schema.submissionGroup.id,
+          tx
+            .select({ id: schema.submissionGroup.id })
+            .from(schema.submissionGroup)
+            .innerJoin(
+              schema.submission,
+              and(
+                eq(schema.submissionGroup.submissionId, schema.submission.id),
+                eq(schema.submission.id, submission.id),
+              ),
+            )
+            .innerJoin(
+              schema.formGroup,
+              eq(schema.submissionGroup.formGroupId, schema.formGroup.id),
+            )
+            .where(eq(schema.formGroup.multi, true)),
+        ),
+      );
+    });
+  } catch (error) {
+    console.error("Failed to reset submission");
+    if (!(error instanceof InvariantError)) throw error;
+  }
+}
+
+export async function updateSubmission({
+  member,
+  submissionId,
+  name,
+  summary,
+}: {
+  member: Member;
+  submissionId: string;
+  name?: string;
+  summary?: string;
+}) {
+  // Validate access
+  const submission = await getSubmission({ member, submissionId });
+  const updateData: { name?: string; summary?: string; nameAutoGenerated?: boolean } = {};
+  if (name !== undefined) {
+    updateData.name = name;
+    updateData.nameAutoGenerated = false;
+  }
+  if (summary !== undefined) {
+    updateData.summary = summary;
+  }
+  const [updatedSubmission] = await db
+    .update(schema.submission)
+    .set(updateData)
+    .where(eq(schema.submission.id, submission.id))
+    .returning();
+  invariant(updatedSubmission, `failed to update submission ${submissionId}`);
+
+  return updatedSubmission;
+}
+
+/**
+ * Parses a documentCitation path (e.g. "Transactions[2].Credit" or "Bank Statement.BankName")
+ * and sets the given value at that location in the extraction data JSON blob.
+ * Returns the mutated data object, or the original if the path cannot be resolved.
+ */
+function applyValueToExtractionData(
+  data: Record<string, unknown>,
+  path: string,
+  value: SubmissionItemValue | null,
+): Record<string, unknown> {
+  // Path format: "<groupName>[<index>].<fieldName>" or "<groupName>.<fieldName>"
+  const multiMatch = path.match(/^(.+)\[(\d+)\]\.(.+)$/);
+  const singleMatch = !multiMatch ? path.match(/^(.+)\.(.+)$/) : null;
+
+  if (multiMatch) {
+    const groupName = multiMatch[1]!;
+    const index = parseInt(multiMatch[2]!, 10);
+    const fieldName = multiMatch[3]!;
+    const group = data[groupName];
+    if (Array.isArray(group) && group[index] != null && typeof group[index] === "object") {
+      (group[index] as Record<string, unknown>)[fieldName] = value;
+    }
+  } else if (singleMatch) {
+    const groupName = singleMatch[1]!;
+    const fieldName = singleMatch[2]!;
+    const group = data[groupName];
+    if (group != null && typeof group === "object" && !Array.isArray(group)) {
+      (group as Record<string, unknown>)[fieldName] = value;
+    }
+  }
+
+  return data;
+}
+
+/**
+ * After a manual value update, find the most recent document-sourced version for the item,
+ * follow its citation to the linked documentExtraction, and patch the value in the data JSON
+ * using the citation path as a guide.
+ */
+async function syncValueToDocumentExtraction(
+  submissionItemId: string,
+  value: SubmissionItemValue | null,
+): Promise<void> {
+  // Find the most recent version sourced from a document
+  const documentVersion = await db.query.submissionItemVersion.findFirst({
+    where: and(
+      eq(schema.submissionItemVersion.submissionItemId, submissionItemId),
+      eq(schema.submissionItemVersion.sourceType, "document"),
+    ),
+    orderBy: desc(schema.submissionItemVersion.createdAt),
+  });
+
+  if (!documentVersion?.sourceId) return;
+
+  // Resolve the citation and its parent extraction
+  const citation = await db.query.documentCitation.findFirst({
+    where: eq(schema.documentCitation.id, documentVersion.sourceId),
+  });
+
+  if (!citation) return;
+
+  const extraction = await db.query.documentExtraction.findFirst({
+    where: eq(schema.documentExtraction.id, citation.documentExtractionId),
+  });
+
+  if (!extraction?.data) return;
+
+  const updatedData = applyValueToExtractionData(
+    extraction.data as Record<string, unknown>,
+    citation.path,
+    value,
+  );
+
+  await db
+    .update(schema.documentExtraction)
+    .set({ data: updatedData })
+    .where(eq(schema.documentExtraction.id, extraction.id));
+}
+
+export async function updateSubmissionValue({
+  member,
+  submissionItemId,
+  value,
+}: {
+  member: Member;
+  submissionItemId: string;
+  value: SubmissionItemValue | null;
+}) {
+  const submissionItem = await db.query.submissionItem.findFirst({
+    with: { group: { with: { submission: true } } },
+    where: eq(schema.submissionItem.id, submissionItemId),
+  });
+  invariant(submissionItem, `no formField found for ${submissionItemId}`);
+
+  // Validate access
+  if (submissionItem.group?.submission.organizationId !== member.organizationId) {
+    throw new InvariantError(`no submission found containing submissionItem ${submissionItemId}`);
+  }
+
+  await db.transaction(async (tx) => {
+    await tx
+      .update(schema.submissionItemVersion)
+      .set({ selected: false })
+      .where(eq(schema.submissionItemVersion.submissionItemId, submissionItem.id));
+
+    await tx.insert(schema.submissionItemVersion).values({
+      sourceType: "manual",
+      submissionItemId: submissionItem.id,
+      value,
+      selected: true,
+    });
+  });
+
+  // // Best-effort: patch the linked documentExtraction data JSON to reflect the manual edit.
+  // // Errors here are non-fatal — the item version has already been persisted above.
+  // await syncValueToDocumentExtraction(submissionItemId, value).catch((err) => {
+  //   console.error(
+  //     `[updateSubmissionValue] failed to sync value to documentExtraction for item ${submissionItemId}:`,
+  //     err,
+  //   );
+  // });
+}
+
+export async function addSubmissionGroupRow({
+  member,
+  submissionId,
+  formGroupId,
+}: {
+  member: Member;
+  submissionId: string;
+  formGroupId: string;
+}) {
+  const formGroup = await db.query.formGroup.findFirst({
+    where: and(eq(schema.formGroup.id, formGroupId), eq(schema.formGroup.multi, true)),
+  });
+  invariant(formGroup, `no formGroup found for ${formGroupId}`);
+
+  // Verify the submission exists and belongs to the user
+  await getForm({ member, formId: formGroup.formId });
+  const submission = await getSubmission({ member, submissionId });
+  invariant(submission, `no submission found for ${submissionId}`);
+
+  const submissionGroup = await db.transaction(async (tx) => {
+    const existingRows = await tx
+      .select({ order: schema.submissionGroup.order })
+      .from(schema.submissionGroup)
+      .where(
+        and(
+          eq(schema.submissionGroup.submissionId, submissionId),
+          eq(schema.submissionGroup.formGroupId, formGroup.id),
+        ),
+      )
+      .for("update");
+    const order =
+      existingRows.length > 0 ? Math.max(...existingRows.map((row) => row.order)) + 1 : 0;
+
+    const [submissionGroup] = await tx
+      .insert(schema.submissionGroup)
+      .values({
+        submissionId,
+        formGroupId: formGroup.id,
+        order,
+      })
+      .returning();
+    invariant(submissionGroup);
+    return submissionGroup;
+  });
+  return submissionGroup;
+}
+
+export async function updateSubmissionGroup({
+  member,
+  submissionId,
+  submissionGroupId,
+  isValid,
+  order,
+}: {
+  member: Member;
+  submissionId: string;
+  submissionGroupId: string;
+  isValid?: boolean;
+  order?: number;
+}) {
+  // Verify the submission group belongs to this submission and user has access
+  const [submissionGroup] = await db
+    .select({ id: schema.submissionGroup.id })
+    .from(schema.submissionGroup)
+    .leftJoin(schema.submission, eq(schema.submission.id, schema.submissionGroup.submissionId))
+    .where(
+      and(
+        eq(schema.submissionGroup.id, submissionGroupId),
+        eq(schema.submissionGroup.submissionId, submissionId),
+        eq(schema.submission.organizationId, member.organizationId),
+        eq(schema.submission.type, "extraction"),
+        isNull(schema.submission.deletedAt),
+      ),
+    );
+  invariant(submissionGroup, `no submissionGroup found for ${submissionGroupId}`);
+
+  const updateData: { isValid?: boolean; order?: number } = {};
+  if (isValid !== undefined) updateData.isValid = isValid;
+  if (order !== undefined) updateData.order = order;
+
+  const [updated] = await db
+    .update(schema.submissionGroup)
+    .set(updateData)
+    .where(eq(schema.submissionGroup.id, submissionGroupId))
+    .returning();
+
+  invariant(updated, `failed to update submissionGroup ${submissionGroupId}`);
+  return updated;
+}
+
+export async function createSubmissionWithConversation({
+  member,
+  data,
+}: {
+  member: Member;
+  data: {
+    formId: string;
+    dealId: string;
+    name?: string;
+    status?: "draft" | "final";
+    type?: "extraction" | "transformation";
+  };
+}) {
+  const form = await db.query.form.findFirst({
+    where: and(
+      eq(schema.form.id, data.formId),
+      eq(schema.form.organizationId, member.organizationId),
+      isNull(schema.form.deletedAt),
+    ),
+  });
+  invariant(form);
+
+  const deal = await db.query.deal.findFirst({
+    where: and(
+      eq(schema.deal.organizationId, member.organizationId),
+      eq(schema.deal.id, data.dealId),
+      isNull(schema.deal.deletedAt),
+    ),
+  });
+  invariant(deal);
+
+  const createdSubmission = await db.transaction(async (tx) => {
+    const [conversation] = await tx
+      .insert(schema.conversation)
+      .values({ organizationId: member.organizationId })
+      .returning();
+    invariant(conversation);
+
+    const [createdSubmission] = await tx
+      .insert(schema.submission)
+      .values({
+        organizationId: member.organizationId,
+        formId: data.formId,
+        conversationId: conversation.id,
+        dealId: deal.id,
+        name: data.name,
+        nameAutoGenerated: !data.name,
+        status: data.status,
+        type: data.type,
+      })
+      .returning();
+    invariant(createdSubmission);
+    return createdSubmission;
+  });
+
+  return createdSubmission;
+}
+
+/**
+ * Get submissions with their extractions
+ */
+export async function getSubmissionsWithExtractions({
+  member,
+  submissionIds,
+}: {
+  member: Member;
+  submissionIds: string[];
+}): Promise<
+  Array<{
+    id: string;
+    extractions: Array<{
+      id: string;
+      status: string;
+      deletedAt: Date | null;
+    }>;
+  }>
+> {
+  if (submissionIds.length === 0) {
+    return [];
+  }
+
+  const submissions = await db.query.submission.findMany({
+    where: and(
+      eq(schema.submission.organizationId, member.organizationId),
+      inArray(schema.submission.id, submissionIds),
+      isNull(schema.submission.deletedAt),
+    ),
+    columns: {
+      id: true,
+    },
+    with: {
+      extractions: {
+        where: isNull(schema.documentExtraction.deletedAt),
+        columns: {
+          id: true,
+          status: true,
+          deletedAt: true,
+        },
+      },
+    },
+  });
+
+  return submissions;
+}