sea-dev 1.0.0

package/libs/core/src/chat/feedback.ts

@@ -0,0 +1,293 @@
+/**
+ * Feedback analysis functions using LLM
+ */
+
+import {
+  generateCacheKey,
+  generateCorrectionsHash,
+  getCachedFeedbackAnalysis,
+  setCachedFeedbackAnalysis,
+} from "@sea/cache/feedback";
+import type { SubmissionItemValue } from "@sea/db/schema";
+import type { FieldCorrection, Member } from "@sea/db/types";
+import { generateText, type LanguageModel } from "ai";
+
+// Re-export types and schemas from @sea/schemas
+export type { FeedbackAnalysis } from "@sea/schemas/core/chat";
+export { feedbackAnalysisSchema } from "@sea/schemas/core/chat";
+
+// TODO: reconcile this with the shared library and use that
+type FieldCorrectionWithSubmissionItem = FieldCorrection & {
+  submissionItem?: {
+    formField?: {
+      name: string;
+    };
+  };
+};
+
+export async function analyzeFeedbackForItem(
+  model: LanguageModel,
+  member: Member,
+  corrections: FieldCorrectionWithSubmissionItem[],
+  itemId: string,
+) {
+  // Generate cache key based on corrections content
+  const correctionsForHash = corrections.map((c) => ({
+    id: c.id,
+    extractedValue: c.extractedValue,
+    correctedValue: c.correctedValue,
+  }));
+
+  const correctionsHash = generateCorrectionsHash(correctionsForHash);
+  const cacheKey = generateCacheKey({
+    organizationId: member.organizationId,
+    type: "item",
+    itemId,
+    correctionsHash,
+  });
+
+  // Check memory cache first
+  const cachedResult = getCachedFeedbackAnalysis(cacheKey);
+  if (cachedResult) {
+    return cachedResult;
+  }
+
+  const systemPromptContent = `
+You are an AI extraction analysis expert. Analyze feedback about incorrect AI extractions for a specific submission item.
+
+Your task is to identify what went wrong in the extraction process and provide actionable insights.
+
+Focus on:
+1. Common extraction errors and their root causes
+2. Patterns in what the AI missed or misinterpreted
+3. Specific recommendations to improve extraction accuracy
+4. Field-specific guidance for better extraction hints
+
+Provide your analysis in a structured format that can help improve the AI extraction system.
+  `;
+
+  const systemPrompt = {
+    role: "system" as const,
+    content: systemPromptContent,
+  };
+
+  const correctionsData = corrections.map((correction) => ({
+    extractedValue: correction.extractedValue,
+    correctedValue: correction.correctedValue,
+    feedback: correction.feedback,
+    fieldName: correction.submissionItem?.formField?.name || "unknown",
+  }));
+
+  const messageText = `
+Analyze the following field corrections for submission item ${itemId}:
+
+CORRECTIONS DATA:
+${JSON.stringify(correctionsData, null, 2)}
+
+Please provide:
+1. Analysis of what went wrong in the extractions
+2. Common patterns across these corrections
+3. Specific recommendations for improving extraction accuracy
+4. Actionable insights for the AI system
+  `;
+
+  const messagePrompt = {
+    role: "user" as const,
+    content: messageText,
+  };
+
+  const result = await generateText({
+    model,
+    messages: [systemPrompt, messagePrompt],
+  });
+
+  // Cache the result
+  setCachedFeedbackAnalysis(cacheKey, result.text);
+
+  return result.text;
+}
+
+export async function detectFeedbackPatterns(
+  model: LanguageModel,
+  member: Member,
+  corrections: FieldCorrectionWithSubmissionItem[],
+  formId?: string,
+) {
+  // Generate cache key based on corrections content
+  const correctionsForHash = corrections.map((c) => ({
+    id: c.id,
+    extractedValue: c.extractedValue,
+    correctedValue: c.correctedValue,
+  }));
+
+  const correctionsHash = generateCorrectionsHash(correctionsForHash);
+  const cacheKey = generateCacheKey({
+    organizationId: member.organizationId,
+    type: "pattern",
+    formId: formId,
+    correctionsHash,
+  });
+
+  // Check memory cache first
+  const cachedResult = getCachedFeedbackAnalysis(cacheKey);
+  if (cachedResult) {
+    return cachedResult;
+  }
+
+  const systemPromptContent = `
+You are an AI extraction pattern analysis expert. Analyze feedback data across multiple submission items to identify recurring issues and systemic problems.
+
+Your task is to:
+1. Identify patterns that occur across different submission items
+2. Categorize types of extraction errors (missing data, wrong format, misinterpretation, etc.)
+3. Quantify the frequency and impact of different error types
+4. Provide systemic recommendations for improving the overall extraction system
+
+Focus on cross-cutting issues that affect multiple fields or items, not just isolated problems.
+  `;
+
+  const systemPrompt = {
+    role: "system" as const,
+    content: systemPromptContent,
+  };
+
+  // Group corrections by field and analyze patterns
+  const patternData = corrections.reduce(
+    (acc, correction) => {
+      const fieldName = correction.submissionItem?.formField?.name || "unknown";
+      if (!acc[fieldName]) {
+        acc[fieldName] = [];
+      }
+      acc[fieldName].push({
+        extractedValue: correction.extractedValue,
+        correctedValue: correction.correctedValue,
+        feedback: correction.feedback,
+        itemId: correction.submissionItemId,
+      });
+      return acc;
+    },
+    {} as Record<
+      string,
+      Array<{
+        extractedValue: SubmissionItemValue | null;
+        correctedValue: SubmissionItemValue | null;
+        feedback: string | null;
+        itemId: string;
+      }>
+    >,
+  );
+
+  const messageText = `
+Analyze the following feedback patterns across multiple submission items${formId ? ` for form ${formId}` : ""}:
+
+PATTERN DATA BY FIELD:
+${JSON.stringify(patternData, null, 2)}
+
+Please identify:
+1. Recurring patterns across different submission items
+2. Common types of extraction errors and their frequencies
+3. Fields that consistently have issues
+4. Systemic recommendations for improving extraction accuracy
+5. Priority areas for improvement based on frequency and impact
+  `;
+
+  const messagePrompt = {
+    role: "user" as const,
+    content: messageText,
+  };
+
+  const result = await generateText({
+    model,
+    messages: [systemPrompt, messagePrompt],
+  });
+
+  // Cache the result
+  setCachedFeedbackAnalysis(cacheKey, result.text);
+
+  return result.text;
+}
+
+export async function generateImprovedHint(
+  model: LanguageModel,
+  member: Member,
+  fieldName: string,
+  currentHint: string,
+  corrections: FieldCorrectionWithSubmissionItem[],
+) {
+  // Generate cache key for hint improvement
+  const correctionsForHash = corrections.map((c) => ({
+    id: c.id,
+    extractedValue: c.extractedValue,
+    correctedValue: c.correctedValue,
+  }));
+
+  const correctionsHash = generateCorrectionsHash(correctionsForHash);
+  const cacheKey = generateCacheKey({
+    organizationId: member.organizationId,
+    type: "pattern",
+    itemId: `hint_${fieldName}`,
+    correctionsHash,
+  });
+
+  // Check memory cache first
+  const cachedResult = getCachedFeedbackAnalysis(cacheKey);
+  if (cachedResult) {
+    return { improvedHint: cachedResult };
+  }
+
+  const systemPromptContent = `
+You are an AI extraction hint optimization expert. Your task is to directly improve AI extraction hints based on user corrections.
+
+A hint is a short snippet of text that guides a PDF extraction model on how to find/calculate/extract the required value for a field.
+
+Only incorporate the FEEDBACK if it seems likely to improve the capabilities of general data extraction.
+
+Your goal is to create a precise, actionable hint that will help the AI extract data correctly based on the correction examples.
+
+A few example hints:
+- "Usually found on the first page"
+- "Get this from annual financial statement"
+- "Combine COGS and revenue to get this"
+
+The hint you generate must match this style. Don't give commentary or explain what you did or where you got it from. Just the succinct hint to pass to a model.
+
+Return only the hint.
+`;
+
+  const systemPrompt = {
+    role: "system" as const,
+    content: systemPromptContent,
+  };
+
+  // Create examples from corrections
+  const correctionExamples = corrections.slice(0, 10).map((correction) => ({
+    feedback: correction.feedback,
+  }));
+
+  const messageText = `
+Field Name: ${fieldName}
+Current Hint: "${currentHint || "No hint provided"}"
+
+<feedback>
+${JSON.stringify(correctionExamples, null, 2)}
+</feedback>
+
+<instructions>
+Return only the hint. No commentary or explaining where you got something from. Just the hint ready to be persisted to the DB.
+</instructions>
+`;
+
+  const messagePrompt = {
+    role: "user" as const,
+    content: messageText,
+  };
+
+  const { text: improvedHint } = await generateText({
+    model,
+    messages: [systemPrompt, messagePrompt],
+  });
+
+  setCachedFeedbackAnalysis(cacheKey, improvedHint);
+
+  return { improvedHint };
+}

package/libs/core/src/chat/index.ts

@@ -0,0 +1,6 @@
+export * from "./derived";
+export * from "./feedback";
+export * from "./model";
+export * from "./stream";
+export * from "./summarize";
+export * from "./tools";

package/libs/core/src/chat/model.ts

@@ -0,0 +1,92 @@
+import { createLanguageModel } from "@sea/ai/models";
+import { withTracing } from "@posthog/ai";
+import type { LanguageModel } from "ai";
+import type { PostHog } from "posthog-node";
+
+// Re-export type from @sea/schemas (with PostHog type refinement)
+export type GetChatModelConfig = {
+  aiGatewayApiKey: string;
+  modelId?: string;
+  posthog?: PostHog;
+  posthogDistinctId?: string;
+  posthogTraceId?: string;
+  posthogProperties?: Record<string, unknown>;
+  posthogGroups?: Record<string, string>;
+};
+
+/**
+ * Gets a production-ready chat model with optional PostHog tracing.
+ *
+ * This is the recommended way to get a language model in the app - it handles:
+ * - Provider selection based on model ID
+ * - Optional observability/tracing via PostHog
+ * - Consistent configuration
+ *
+ * @example
+ * ```ts
+ * const model = getChatModel({
+ *   aiGatewayApiKey: config.vercel.aiGatewayApiKey,
+ *   modelId: "claude-sonnet-4-5",
+ *   posthog: posthogClient,
+ *   posthogDistinctId: member.userId,
+ *   posthogGroups: { company: member.organizationId }
+ * });
+ * ```
+ */
+export function getChatModel(config: GetChatModelConfig): LanguageModel {
+  const {
+    aiGatewayApiKey,
+    modelId = "openai/gpt-5.2",
+    posthog,
+    posthogDistinctId,
+    posthogTraceId,
+    posthogProperties,
+    posthogGroups,
+  } = config;
+
+  // Create base model with provider/model format
+  const baseModel = createLanguageModel({
+    gatewayApiKey: aiGatewayApiKey,
+    modelId,
+  });
+
+  // Return traced model if PostHog is provided
+  if (posthog && posthogDistinctId) {
+    // @ts-expect-error - PostHog withTracing types don't match AI SDK types after zod v4 upgrade
+    return withTracing(baseModel, posthog, {
+      posthogDistinctId,
+      posthogTraceId,
+      posthogProperties,
+      posthogGroups,
+      posthogPrivacyMode: false,
+    }) as LanguageModel;
+  }
+
+  return baseModel;
+}
+
+/**
+ * Binds a model to a function that takes a model as its first parameter.
+ * This is useful for creating pre-configured versions of core functions.
+ *
+ * @example
+ * ```ts
+ * const boundSummarize = bindModel(
+ *   { aiGatewayApiKey: config.vercel.aiGatewayApiKey, modelId: "gpt-4o" },
+ *   summarizeSubmission
+ * );
+ *
+ * // Now call without model argument
+ * const summary = await boundSummarize({ denormSubmission, context });
+ * ```
+ */
+export function bindModel<TArgs extends unknown[], TReturn>(
+  config: { aiGatewayApiKey: string; modelId?: string },
+  fn: (model: LanguageModel, ...args: TArgs) => TReturn,
+): (...args: TArgs) => TReturn {
+  const model = getChatModel({
+    aiGatewayApiKey: config.aiGatewayApiKey,
+    modelId: config.modelId,
+  });
+  return (...args: TArgs) => fn(model, ...args);
+}

package/libs/core/src/chat/prepare-tools.ts

@@ -0,0 +1,286 @@
+import { getVercelConfig } from "@sea/config/vercel";
+import { getToolsForConversation, upsertToolConfig } from "@sea/dal/tool";
+import { db, schema } from "@sea/db";
+import type { Member } from "@sea/db/types";
+import { invariant } from "@sea/util/invariant";
+import {
+  tool,
+  type LanguageModel,
+  type ToolSet,
+  type UIMessageStreamWriter,
+} from "ai";
+import { and, eq } from "drizzle-orm";
+import {
+  addSubmissionRow,
+  calculateFinancialMetric,
+  chartAgent,
+  checkTrendConsistency,
+  createArtifact,
+  createForm,
+  createPipelineTool,
+  createWorkflow,
+  deepDiveAgent,
+  fetchCurrentForm,
+  fetchCurrentSubmission,
+  fetchFormAgent,
+  fetchFormEditability,
+  fetchFormEditabilityAgent,
+  fetchPipeline,
+  fetchReportData,
+  fetchSubmissionAgent,
+  fileSearch,
+  getAgentDocuments,
+  getOrganizationData,
+  getSubmissionDocumentList,
+  listForms,
+  listPipelines,
+  modifyPipeline,
+  persistForm,
+  persistFormAgent,
+  populateSubmissionItem,
+  reconcileAddBacks,
+  resetCurrentSubmission,
+  searchDeals,
+  searchDealSubmissions,
+  updateAgentMemory,
+  verifyAccountingPolicy,
+  verifyAccountReconciliation,
+} from "./tools";
+import type { ChatType } from "./types";
+
+export interface PrepareToolsProps {
+  chatType: ChatType;
+  member: Member;
+  conversationId: string;
+  model: LanguageModel;
+  writer?: UIMessageStreamWriter;
+}
+
+/**
+ * Format a toolset with descriptions in a readable format for injection into prompts
+ * @param tools The ToolSet object (from prepareTools) containing tool instances
+ * @returns Formatted string with tool names and descriptions
+ */
+export function listTools(tools: ToolSet): string {
+  return Object.entries(tools)
+    .map(
+      ([name, toolInstance]) =>
+        `- **${name}**: ${toolInstance.description || "No description available"}`,
+    )
+    .join("\n");
+}
+
+/**
+ * Apply database-stored descriptions to tools, falling back to the default description.
+ * Also ensures all tools are registered in the database for future configuration.
+ * @param tools The ToolSet to modify
+ * @param member The member making the request
+ * @param conversationId The conversation ID
+ */
+async function applyToolDescriptions(
+  tools: ToolSet,
+  member: Member,
+  conversationId: string,
+): Promise<ToolSet> {
+  const dbTools = await getToolsForConversation({ member, conversationId });
+
+  const toolsWithDescriptions: ToolSet = {};
+  for (const [toolName, toolInstance] of Object.entries(tools)) {
+    // Create a schema representation for this tool
+    const toolSchema = {
+      toolName,
+      toolSchema: toolInstance.inputSchema,
+      description: toolInstance.description,
+    };
+
+    // Find matching DB tool by toolName
+    const dbTool = dbTools.find((t) => t.toolName === toolName);
+
+    if (!dbTool) {
+      // Register this tool in the database with null description (use default)
+      await upsertToolConfig({
+        member,
+        conversationId,
+        toolName,
+        toolSchema,
+        description: undefined,
+      });
+      // Keep original tool with default description
+      toolsWithDescriptions[toolName] = toolInstance;
+    } else if (dbTool.description) {
+      // Override with database description
+      toolsWithDescriptions[toolName] = tool({
+        ...toolInstance,
+        description: dbTool.description,
+      });
+    } else {
+      // Keep original tool with default description
+      toolsWithDescriptions[toolName] = toolInstance;
+    }
+  }
+
+  return toolsWithDescriptions;
+}
+
+export async function prepareTools(
+  props: PrepareToolsProps,
+): Promise<{ tools: ToolSet; baseTools?: ToolSet; maxSteps: number }> {
+  switch (props.chatType) {
+    case "form": {
+      return await prepareFormTools(props);
+    }
+    case "submission": {
+      return await prepareSubmissionTools(props);
+    }
+    case "agent": {
+      return await prepareAgentTools(props);
+    }
+    case "report": {
+      return await prepareReportTools(props);
+    }
+    case "pipeline": {
+      return await preparePipelineTools(props);
+    }
+    case "default": {
+      return { tools: {} as ToolSet, maxSteps: 1 };
+    }
+    default: {
+      throw new Error(`Unhandled case: ${props.chatType}`);
+    }
+  }
+}
+
+async function prepareFormTools({ member, conversationId }: PrepareToolsProps) {
+  const form = await db.query.form.findFirst({
+    where: and(
+      eq(schema.form.organizationId, member.organizationId),
+      eq(schema.form.conversationId, conversationId),
+    ),
+  });
+  invariant(form);
+  const { id: formId } = form;
+  const tools: ToolSet = {
+    fetchCurrentForm: fetchCurrentForm({ member, formId }),
+    fetchFormEditability: fetchFormEditability({ member, formId }),
+    persistForm: persistForm({ member, formId }),
+  };
+  return { tools, maxSteps: 3 };
+}
+
+async function prepareSubmissionTools({
+  member,
+  conversationId,
+}: PrepareToolsProps) {
+  const submission = await db.query.submission.findFirst({
+    where: and(
+      eq(schema.submission.organizationId, member.organizationId),
+      eq(schema.submission.conversationId, conversationId),
+    ),
+  });
+  invariant(submission);
+  const { id: submissionId, formId } = submission;
+
+  const tools: ToolSet = {
+    fetchCurrentForm: fetchCurrentForm({ member, formId }),
+    fetchCurrentSubmission: fetchCurrentSubmission({ member, submissionId }),
+    resetCurrentSubmission: resetCurrentSubmission({ member, submissionId }),
+    populateSubmissionItem: populateSubmissionItem({ member }),
+    addSubmissionRow: addSubmissionRow({ member, submissionId }),
+    getSubmissionDocumentList: getSubmissionDocumentList({
+      member,
+      submissionId,
+    }),
+    fileSearch: fileSearch({ member, conversationId }),
+  };
+  return { tools, maxSteps: 5 };
+}
+
+async function prepareAgentTools({
+  member,
+  conversationId,
+  model,
+  writer,
+}: PrepareToolsProps) {
+  const vercelConfig = getVercelConfig();
+
+  const baseTools: ToolSet = {
+    // General
+    getOrganizationData: getOrganizationData({ member }),
+    searchDeals: searchDeals({ member }),
+    searchDealSubmissions: searchDealSubmissions({ member }),
+
+    // Agent
+    updateAgentMemory: updateAgentMemory({ member, conversationId }),
+    createArtifact: createArtifact({ member, conversationId }),
+    getAgentDocuments: getAgentDocuments({ member, conversationId }),
+    deepDiveAgent: deepDiveAgent({
+      member,
+      conversationId,
+      vercelConfig,
+      model,
+      writer,
+    }),
+    chartAgent: chartAgent({
+      member,
+      conversationId,
+      vercelConfig,
+      model,
+      writer,
+    }),
+
+    // Forms
+    fetchForm: fetchFormAgent({ member }),
+
+    // Submissions
+    fetchSubmission: fetchSubmissionAgent({ member }),
+    fileSearch: fileSearch({ member, conversationId }),
+
+    // Financial Verification Tools
+    verifyAccountReconciliation: verifyAccountReconciliation({ member }),
+    calculateFinancialMetric: calculateFinancialMetric({ member }),
+    verifyAccountingPolicy: verifyAccountingPolicy({ member }),
+    checkTrendConsistency: checkTrendConsistency({ member }),
+    reconcileAddBacks: reconcileAddBacks({ member }),
+  };
+
+  // Apply database-backed descriptions to agent tools
+  const tools = await applyToolDescriptions(baseTools, member, conversationId);
+
+  return { tools, baseTools, maxSteps: 5 };
+}
+
+async function prepareReportTools({
+  member,
+  conversationId,
+}: PrepareToolsProps) {
+  const tools: ToolSet = {
+    fetchReportData: fetchReportData({ member, conversationId }),
+  };
+  return { tools, maxSteps: 3 };
+}
+
+async function preparePipelineTools({
+  member,
+  conversationId,
+}: PrepareToolsProps) {
+  const tools: ToolSet = {
+    // Form discovery
+    listForms: listForms({ member }),
+
+    // Workflow and form creation
+    createWorkflow: createWorkflow({ member, conversationId }),
+    createForm: createForm({ member, conversationId }),
+
+    // Form editing
+    fetchForm: fetchFormAgent({ member }),
+    fetchFormEditability: fetchFormEditabilityAgent({ member }),
+    persistForm: persistFormAgent({ member }),
+
+    // Pipeline viewing and modification
+    listPipelines: listPipelines({ member }),
+    fetchPipeline: fetchPipeline({ member }),
+    createPipeline: createPipelineTool({ member, conversationId }),
+    modifyPipeline: modifyPipeline({ member }),
+  };
+  return { tools, maxSteps: 5 };
+}