sea-dev 1.0.0

package/libs/core/src/chat/tools/helpers.ts

@@ -0,0 +1,91 @@
+import { formFieldTypeArray } from "@sea/db/schema";
+import * as z from "zod/v4";
+
+// Shared schemas - these stay in core because they depend on db/schema
+export const formFieldSchema = z.object({
+  id: z
+    .string()
+    .optional()
+    .describe(
+      "Provide this only for existing items that have an id. Don't generate an id for new items",
+    ),
+  name: z.string().describe("The user-facing name of this item"),
+  type: z.enum(formFieldTypeArray),
+  hint: z
+    .string()
+    .describe(
+      "A hint to another LLM that will need to extract this data from a PDF",
+    ),
+  options: z
+    .array(z.string())
+    .optional()
+    .describe(
+      "For enum/multiple-choice fields only: Array of allowed values. Example: ['Yes', 'No', 'N/A'] or ['Low', 'Medium', 'High']. The extraction LLM will be constrained to select one of these values.",
+    ),
+  order: z
+    .number()
+    .describe("Unique order that defines the items position in forms"),
+});
+
+export const formDerivedSchema = z.object({
+  id: z
+    .string()
+    .optional()
+    .describe(
+      "Provide this only for existing derived fields that have an id. Don't generate an id for new items",
+    ),
+  name: z
+    .string()
+    .describe("The user-facing name of this derived/assessment field"),
+  prompt: z
+    .string()
+    .describe(
+      "The LLM prompt that will be used to generate this field's value after extraction. " +
+        "Example: 'Based on the extracted financial data, assess whether this borrower shows signs of financial distress. Return YES or NO with a brief explanation.'",
+    ),
+});
+
+export const formGroupSchema = z.object({
+  id: z
+    .string()
+    .optional()
+    .describe(
+      "Provide this only for existing items that have an id. Don't generate an id for new groups",
+    ),
+  name: z.string().describe("The user-facing name of this group"),
+  hint: z.string().describe("LLM-facing hint for the group"),
+  multi: z
+    .boolean()
+    .describe(
+      "Whether this group should allow multiple entries, eg data this is repeated for multiple years",
+    ),
+  fields: z.array(formFieldSchema),
+  derived: z
+    .array(formDerivedSchema)
+    .optional()
+    .describe(
+      `Assessment fields that are populated by AI AFTER extraction completes.
+Use these for judgments, risk flags, or calculations based on extracted data.
+Do NOT use for direct extraction from documents.`,
+    ),
+});
+
+export const groupsSchema = z.array(formGroupSchema);
+export type Group = z.infer<typeof formGroupSchema>;
+
+export const jsonPatchOperationSchema = z.union([
+  z.object({ op: z.literal("add"), path: z.string(), value: z.unknown() }),
+  z.object({ op: z.literal("remove"), path: z.string() }),
+  z.object({ op: z.literal("replace"), path: z.string(), value: z.unknown() }),
+  z.object({ op: z.literal("move"), path: z.string(), from: z.string() }),
+  z.object({ op: z.literal("copy"), path: z.string(), from: z.string() }),
+  z.object({ op: z.literal("test"), path: z.string(), value: z.unknown() }),
+]);
+
+// Helper function
+export function returnErr(err: unknown) {
+  return {
+    ok: false,
+    error: err instanceof Error ? err.message : String(err),
+  };
+}

package/libs/core/src/chat/tools/index.ts

@@ -0,0 +1,42 @@
+// Re-export helpers
+export * from "./helpers";
+
+// Re-export form tools
+export * from "./form";
+
+// Re-export submission tools
+export * from "./submission";
+
+// Re-export agent tools
+export * from "./agent";
+
+// Re-export pipeline tools
+export * from "./pipeline-artifact";
+
+// Re-export workflow tools
+export * from "./workflow";
+
+// Re-export search tools
+export * from "./search";
+
+// Re-export report tools
+export * from "./report";
+
+// Re-export credit tools
+export * from "./credit";
+
+// Re-export deep-dive tools
+export * from "./deep-dive";
+
+// Re-export chart tools
+export * from "./chart";
+
+// Re-export agent implementations
+export { createDeepDiveAgent } from "./deep-dive-agent";
+export type { DeepDiveAgentConfig } from "./deep-dive-agent";
+
+export { createCreditAgent } from "./credit-agent";
+export type { CreditAgentConfig } from "./credit-agent";
+
+export { createChartAgent } from "./chart-agent";
+export type { ChartAgentConfig } from "./chart-agent";

package/libs/core/src/chat/tools/pipeline-artifact.ts

@@ -0,0 +1,76 @@
+import type { Member } from "@sea/db/types";
+import { tool } from "ai";
+import * as z from "zod/v4";
+import { returnErr } from "./helpers";
+import { createPipelineArtifact as dalCreatePipelineArtifact } from "@sea/dal/pipeline";
+
+/**
+ * Pipeline-specific artifact creation tool.
+ * Unlike the agent artifact tool, this writes to the artifact database table
+ * and links artifacts to pipeline runs via pipeline_run_artifact.
+ */
+export function createPipelineArtifact({
+  member,
+  runId,
+  runStepId,
+}: {
+  member: Member;
+  runId: string;
+  runStepId: string;
+}) {
+  return tool({
+    description:
+      "Create a markdown report artifact that will be stored in the pipeline run. Use this to generate analysis reports, summaries, or documentation during pipeline execution.",
+    inputSchema: z.object({
+      title: z.string().describe("Short report title"),
+      content: z.string().describe("Markdown content for the report"),
+    }),
+    execute: async ({ title, content }) => {
+      console.log("[createPipelineArtifact] Tool called:", {
+        runId,
+        runStepId,
+        memberId: member.id,
+        title,
+        contentLength: content.length,
+      });
+
+      try {
+        const result = await dalCreatePipelineArtifact({
+          member,
+          runId,
+          runStepId,
+          type: "markdown",
+          title,
+          content: { markdown: content },
+          metadata: {},
+        });
+
+        if (!result.ok) {
+          console.error("[createPipelineArtifact] Failed to create artifact:", {
+            runId,
+            runStepId,
+          });
+          return {
+            ok: false,
+            error: "Failed to create artifact",
+          };
+        }
+
+        console.log("[createPipelineArtifact] Artifact created successfully:", {
+          artifactId: result.data.id,
+          runId,
+          runStepId,
+          title,
+        });
+
+        return {
+          ok: true,
+          data: { artifactId: result.data.id, title },
+        };
+      } catch (err) {
+        console.error("[createPipelineArtifact] Error:", err);
+        return returnErr(err);
+      }
+    },
+  });
+}

package/libs/core/src/chat/tools/report.ts

@@ -0,0 +1,40 @@
+import { getReportData } from "@sea/dal/report";
+import { db, schema } from "@sea/db";
+import type { Member } from "@sea/db/types";
+import { invariant } from "@sea/util/invariant";
+import { tool } from "ai";
+import { and, eq } from "drizzle-orm";
+import * as z from "zod/v4";
+import { returnErr } from "./helpers";
+
+/**
+ * Tool for report chat to fetch current submission data with optional status filter
+ */
+export function fetchReportData({
+  member,
+  conversationId,
+}: {
+  member: Member;
+  conversationId: string;
+}) {
+  return tool({
+    description:
+      "Fetch the current report data. Use this when you need to see the latest submission data, especially after the user has changed filters. The status parameter filters submissions by their status.",
+    inputSchema: z.object({}),
+    execute: async () => {
+      const report = await db.query.report.findFirst({
+        where: and(
+          eq(schema.report.organizationId, member.organizationId),
+          eq(schema.report.conversationId, conversationId),
+        ),
+      });
+      invariant(report);
+      try {
+        return await getReportData({ member, reportId: report.id });
+      } catch (err) {
+        console.error("fetchReportData error:", err);
+        return returnErr(err);
+      }
+    },
+  });
+}

package/libs/core/src/chat/tools/search.ts

@@ -0,0 +1,390 @@
+import { openai } from "@ai-sdk/openai";
+import { getSubmission } from "@sea/dal/submission";
+import { db, schema } from "@sea/db";
+import { memorySchema } from "@sea/db/schema";
+import type { FormField, FormGroup, Member } from "@sea/db/types";
+import { invariant } from "@sea/util/invariant";
+import { embed, tool } from "ai";
+import { and, asc, desc, eq, exists, inArray, isNull, sql } from "drizzle-orm";
+import * as z from "zod/v4";
+import { returnErr } from "./helpers";
+
+export function getSubmissionDocumentList({
+  member,
+  submissionId,
+}: {
+  member: Member;
+  submissionId: string;
+}) {
+  return tool({
+    description:
+      "Get the full list of document names and IDs for the current submission",
+    inputSchema: z.object({}),
+    execute: async () => {
+      try {
+        const submission = await getSubmission({ member, submissionId });
+
+        const extractions = await db.query.documentExtraction.findMany({
+          where: and(eq(schema.documentExtraction.submissionId, submission.id)),
+          with: { document: true },
+        });
+
+        const result = extractions.map((e) => ({
+          extractionId: e.id,
+          documentId: e.document.id,
+          name: e.document.name,
+        }));
+        return {
+          ok: true,
+          data: result,
+        };
+      } catch (err) {
+        return returnErr(err);
+      }
+    },
+  });
+}
+
+export function getSubmissionDocumentData({
+  member,
+  submissionId,
+}: {
+  member: Member;
+  submissionId: string;
+}) {
+  return tool({
+    description: "Get extracted data of a document based on its extractionId",
+    inputSchema: z.object({ extractionId: z.string() }),
+    execute: async ({ extractionId }) => {
+      try {
+        const submission = await getSubmission({ member, submissionId });
+
+        const extraction = await db.query.documentExtraction.findFirst({
+          where: and(
+            eq(schema.documentExtraction.submissionId, submission.id),
+            eq(schema.documentExtraction.id, extractionId),
+          ),
+          with: { document: true },
+        });
+        invariant(extraction, `no extraction for ${extractionId}`);
+
+        const result = {
+          extractionId,
+          data: extraction.data,
+          name: extraction.document.name,
+        };
+        return {
+          ok: true,
+          data: result,
+        };
+      } catch (err) {
+        return returnErr(err);
+      }
+    },
+  });
+}
+
+export function getAgentDocuments({
+  member,
+  conversationId,
+}: {
+  member: Member;
+  conversationId: string;
+}) {
+  return tool({
+    description:
+      "Get documents that have been uploaded to this agent conversation. Use this to see what documents the user has shared.",
+    inputSchema: z.object({}),
+    execute: async () => {
+      try {
+        // First verify the conversation exists and belongs to the member's organization
+        const conversation = await db.query.conversation.findFirst({
+          where: and(
+            eq(schema.conversation.id, conversationId),
+            eq(schema.conversation.organizationId, member.organizationId),
+            isNull(schema.conversation.deletedAt),
+          ),
+        });
+        invariant(conversation);
+
+        const memory = memorySchema.parse(conversation.memory);
+
+        if (memory.dealIds.length === 0) {
+          return {
+            documents: [],
+            summary: {
+              totalDocuments: 0,
+            },
+          };
+        }
+
+        // Get documents for these workflows
+        const documents = await db.query.document.findMany({
+          where: and(
+            inArray(schema.document.dealId, memory.dealIds),
+            isNull(schema.document.deletedAt),
+            // Verify organization access through workflow
+            exists(
+              db
+                .select()
+                .from(schema.deal)
+                .where(
+                  and(
+                    eq(schema.deal.id, schema.document.dealId),
+                    eq(schema.deal.organizationId, member.organizationId),
+                  ),
+                ),
+            ),
+          ),
+          orderBy: desc(schema.document.updatedAt),
+          limit: 20,
+        });
+
+        const data = {
+          documents,
+          summary: {
+            totalDocuments: documents.length,
+          },
+        };
+        return {
+          ok: true,
+          data,
+        };
+      } catch (err) {
+        return returnErr(err);
+      }
+    },
+  });
+}
+
+export function fileSearch({
+  member,
+  conversationId,
+}: {
+  member: Member;
+  conversationId: string;
+}) {
+  return tool({
+    description:
+      "Search for information in uploaded documents using RAG (Retrieval-Augmented Generation). For a specific a field name from the form schema this tool will search the documents for relevant information about that field.",
+    inputSchema: z.object({
+      fieldName: z
+        .string()
+        .describe(
+          "The name of the field from the form schema to search for. Must match a field name in one of the form groups.",
+        ),
+    }),
+    execute: async ({ fieldName }) => {
+      try {
+        console.log("[fileSearch] Starting RAG search for field:", fieldName);
+
+        // 1. Fetch submission with form schema
+        console.log("[fileSearch] Fetching submission and form schema...");
+        const submission = await db.query.submission.findFirst({
+          where: and(
+            eq(schema.submission.organizationId, member.organizationId),
+            eq(schema.submission.conversationId, conversationId),
+            isNull(schema.submission.deletedAt),
+          ),
+          with: {
+            form: {
+              with: {
+                groups: {
+                  with: { fields: { orderBy: asc(schema.formField.order) } },
+                  orderBy: asc(schema.formGroup.order),
+                },
+              },
+            },
+            extractions: { with: { document: true } },
+          },
+        });
+        invariant(submission);
+
+        // extract relevant document IDs to search for
+        const documentIds = submission.extractions.map((e) => e.document.id);
+
+        // Early return if no documents
+        if (documentIds.length === 0) {
+          console.log("[fileSearch] No documents found in submission");
+          return {
+            ok: false,
+            error:
+              "No documents have been uploaded to this submission yet. Please upload documents first.",
+          };
+        }
+
+        // 2. Find the field in the schema
+        let targetField: FormField | null = null;
+        let targetGroup: FormGroup | null = null;
+
+        for (const group of submission.form.groups) {
+          const field = group.fields.find((f) => f.name === fieldName);
+          if (field) {
+            targetField = field;
+            targetGroup = group;
+            break;
+          }
+        }
+
+        if (!targetField || !targetGroup) {
+          console.log("[fileSearch] Field not found in schema:", fieldName);
+          return {
+            ok: false,
+            error: `Field "${fieldName}" not found in form schema. Available fields: ${submission.form.groups.flatMap((g) => g.fields.map((f) => f.name)).join(", ")}`,
+          };
+        }
+
+        console.log(
+          "[fileSearch] Found field:",
+          targetField.name,
+          "in group:",
+          targetGroup.name,
+        );
+
+        // 3. Build search query based on field hint and name
+        const searchQuery = `${targetField.name}: ${targetField.hint}`;
+        console.log("[fileSearch] Search query:", searchQuery);
+
+        // 4. Perform hybrid RAG search (vector + BM25) -> Get Candidates
+        console.log(
+          "[fileSearch] Performing hybrid search (fetching candidates)...",
+        );
+
+        const { embedding } = await embed({
+          model: openai.embedding("text-embedding-3-small"),
+          value: searchQuery,
+        });
+
+        const k = 60; // RRF constant
+
+        type SearchRow = {
+          context_text: string;
+          rrf_score: number;
+          document_id: string;
+          heading_path: string[] | null;
+          matched_text: string;
+          chunk_type: string;
+          filename_match: boolean;
+        };
+
+        const results = await db.execute<SearchRow>(sql`
+          WITH vector_search AS (
+            SELECT
+              id,
+              document_id,
+              ROW_NUMBER() OVER (PARTITION BY document_id ORDER BY embedding <=> ${JSON.stringify(embedding)}::vector) as doc_rank,
+              ROW_NUMBER() OVER (ORDER BY embedding <=> ${JSON.stringify(embedding)}::vector) as global_rank
+            FROM ${schema.documentChunks}
+            WHERE document_id = ANY(ARRAY[${sql.raw(documentIds.map((id) => `'${id}'`).join(","))}])
+          ),
+          text_search AS (
+            SELECT
+              id,
+              document_id,
+              ROW_NUMBER() OVER (PARTITION BY document_id ORDER BY ts_rank(to_tsvector('english', text), plainto_tsquery('english', ${searchQuery})) DESC) as doc_rank,
+              ROW_NUMBER() OVER (ORDER BY ts_rank(to_tsvector('english', text), plainto_tsquery('english', ${searchQuery})) DESC) as global_rank
+            FROM ${schema.documentChunks}
+            WHERE document_id = ANY(ARRAY[${sql.raw(documentIds.map((id) => `'${id}'`).join(","))}])
+              AND to_tsvector('english', ${schema.documentChunks.text}) @@ plainto_tsquery('english', ${searchQuery})
+          ),
+          -- Get top results from each search method per document
+          top_vector AS (
+            SELECT id, global_rank as rank FROM vector_search WHERE doc_rank <= 10
+          ),
+          top_text AS (
+            SELECT id, global_rank as rank FROM text_search WHERE doc_rank <= 10
+          ),
+          combined_ranks AS (
+            SELECT
+              COALESCE(v.id, t.id) as chunk_id,
+              (COALESCE(1.0 / (${k} + v.rank), 0) + COALESCE(1.0 / (${k} + t.rank), 0)) as rrf_score
+            FROM top_vector v
+            FULL OUTER JOIN top_text t ON v.id = t.id
+          ),
+          scored_chunks AS (
+            SELECT
+              c.rrf_score,
+              target.text as matched_text,
+              target.chunk_type,
+              -- If the match is a child, fetch the parent text instead
+              COALESCE(parent.text, target.text) as context_text,
+              COALESCE(parent.heading_path, target.heading_path) as heading_path,
+              target.document_id,
+              target.node_type,
+              -- Check if document name matches any terms in the search query
+              to_tsvector('english', doc.name) @@ plainto_tsquery('english', ${searchQuery}) as filename_match,
+              -- Boost score by 30% if filename matches
+              CASE WHEN to_tsvector('english', doc.name) @@ plainto_tsquery('english', ${searchQuery})
+                THEN c.rrf_score * 1.3
+                ELSE c.rrf_score
+              END as final_score,
+              -- Rank within each document
+              ROW_NUMBER() OVER (
+                PARTITION BY target.document_id
+                ORDER BY CASE WHEN to_tsvector('english', doc.name) @@ plainto_tsquery('english', ${searchQuery})
+                  THEN c.rrf_score * 1.3
+                  ELSE c.rrf_score
+                END DESC
+              ) as doc_rank
+            FROM combined_ranks c
+            JOIN ${schema.documentChunks} target ON c.chunk_id = target.id
+            LEFT JOIN ${schema.documentChunks} parent ON target.parent_id = parent.id
+              AND target.document_id = parent.document_id
+            JOIN ${schema.document} doc ON target.document_id = doc.id
+          )
+          SELECT
+            rrf_score,
+            matched_text,
+            chunk_type,
+            context_text,
+            heading_path,
+            document_id,
+            node_type,
+            filename_match
+          FROM scored_chunks
+          WHERE doc_rank <= 3  -- Get top 3 results from each document
+          ORDER BY final_score DESC
+          LIMIT 15  -- Limit total results (3 documents × 3 results = 9, with buffer)
+        `);
+
+        // Map results to final structure
+        const rows = results.rows;
+        const finalResults = rows.map((row) => ({
+          text: row.context_text, // Use context_text (parent) if available
+          score: row.rrf_score,
+          documentId: row.document_id,
+          headingPath: row.heading_path,
+          matchedText: row.matched_text, // Keep original match for debug/highlighting if needed
+          chunkType: row.chunk_type,
+        }));
+
+        console.log("[fileSearch] Hybrid search completed");
+        console.log(
+          "[fileSearch] Results:",
+          JSON.stringify(finalResults, null, 2),
+        );
+
+        const documents = await db.query.document.findMany({
+          where: inArray(
+            schema.document.id,
+            finalResults.map((r) => r.documentId),
+          ),
+        });
+
+        return finalResults
+          .map((r) => {
+            const doc = documents.find((d) => d.id === r.documentId);
+            const heading = r.headingPath?.length
+              ? r.headingPath.join(" > ")
+              : "";
+            const context = heading ? `[${heading}]\n` : "";
+            return `< doc name = "${doc?.name || "Unknown"}" type = "${r.chunkType}" >\n${context}${r.text} \n </doc>`;
+          })
+          .join("\n\n");
+      } catch (err) {
+        console.error("[fileSearch] Error:", err);
+        return returnErr(err);
+      }
+    },
+  });
+}