sea-dev 1.0.0

package/libs/ai/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@sea/ai",
+  "private": true,
+  "type": "module",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": "./dist/index.js",
+    "./models": "./dist/models.js",
+    "./rag": "./dist/rag/index.js"
+  },
+  "scripts": {
+    "build": "vite build",
+    "fmt": "prettier --log-level warn --write .",
+    "lint": "eslint --fix . --ext .js,.jsx,.ts,.tsx",
+    "check": "tsc --noEmit",
+    "ci:fmt": "prettier --check .",
+    "ci:lint": "eslint . --ext .js,.jsx,.ts,.tsx",
+    "ci:check": "pnpm run check",
+    "all": "pnpm run fmt && pnpm run lint && pnpm run check"
+  },
+  "dependencies": {
+    "@ai-sdk/openai": "^3.0.1",
+    "@sea/schemas": "workspace:*",
+    "ai": "^6.0.3",
+    "js-tiktoken": "^1.0.21"
+  },
+  "devDependencies": {
+    "@sea/config": "workspace:*",
+    "@sea/lint": "workspace:*",
+    "glob": "^11.0.3",
+    "typescript": "^5.7.2",
+    "vite": "^8.0.1",
+    "vite-plugin-dts": "^4.5.4",
+    "vitest": "^2.1.8"
+  },
+  "engines": {
+    "node": ">=22.0.0 <23.0.0",
+    "pnpm": ">=10.13.0 <11.0.0"
+  }
+}

package/libs/ai/src/index.ts

@@ -0,0 +1,5 @@
+// Export AI infrastructure
+export * from "./models";
+
+// Export RAG subsystem
+export * from "./rag";

package/libs/ai/src/models.ts

@@ -0,0 +1,19 @@
+import { createGateway, type LanguageModel } from "ai";
+
+export type ModelConfig = {
+  modelId: string;
+  gatewayApiKey: string;
+};
+
+/**
+ * Creates a base language model without any observability or tracing.
+ * Uses Vercel AI Gateway for all model access.
+ * This is a pure AI primitive - for production use, wrap with tracing via getChatModel from @sea/core.
+ */
+export function createLanguageModel(config: ModelConfig): LanguageModel {
+  const { gatewayApiKey, modelId } = config;
+  const gateway = createGateway({ apiKey: gatewayApiKey });
+
+  // Use gateway with provider/model format
+  return gateway(modelId);
+}

package/libs/ai/src/rag/index.ts

@@ -0,0 +1 @@
+export * from "./rag";

package/libs/ai/src/rag/rag.test.ts

@@ -0,0 +1,99 @@
+import {
+  Cell,
+  DocumentLayout,
+  Heading,
+  List,
+  ListItem,
+  Paragraph,
+  Row,
+  Section,
+  Table,
+} from "@sea/schemas/core/layout";
+import { Rag } from "./rag";
+
+// Helper to create a simple document layout
+function createSampleDoc(): DocumentLayout {
+  const doc = new DocumentLayout([], {}, {});
+
+  // H1: Documentation
+  const h1 = new Heading("h1", 1, "Documentation");
+
+  // P: Welcome
+  const p1 = new Paragraph("p1", "Welcome to the docs.");
+
+  // Section: Installation
+  const h2_install = new Heading("h2", 2, "Installation");
+  const p_install = new Paragraph("p2", "To install, run npm install.");
+  const section_install = new Section("sec1", [h2_install, p_install], null as any, null as any);
+
+  // Section: Features
+  const h2_features = new Heading("h3", 2, "Features");
+  const list = new List("l1", false, [
+    new ListItem("li1", "Feature A"),
+    new ListItem("li2", "Feature B"),
+  ]);
+
+  // Nested Section: Advanced Usage
+  const h3_advanced = new Heading("h4", 3, "Advanced Usage");
+  const p_advanced = new Paragraph("p3", "For advanced usage, see below.");
+
+  // Table
+  const table = new Table("t1", [
+    new Row("head", [new Cell("c1", "Option"), new Cell("c2", "Description")]),
+    new Row("body", [new Cell("c3", "--verbose"), new Cell("c4", "Run with verbose logging")]),
+  ]);
+
+  const section_advanced = new Section(
+    "sec2",
+    [h3_advanced, p_advanced, table],
+    null as any,
+    null as any,
+  );
+
+  const section_features = new Section(
+    "sec3",
+    [h2_features, list, section_advanced],
+    null as any,
+    null as any,
+  );
+
+  doc.documentLayout = [h1, p1, section_install, section_features];
+  return doc;
+}
+
+async function test() {
+  console.log("Initializing Rag with DocumentLayout...");
+  const doc = createSampleDoc();
+  const rag = new Rag([doc]);
+
+  console.log("\n--- Chunks ---");
+  rag.chunks.forEach((c) => {
+    console.log(`[${c.type.toUpperCase()}] ID: ${c.id}`);
+    console.log(`Metadata:`, c.metadata);
+    console.log(`Text Preview: ${c.text.slice(0, 50).replace(/\n/g, "\\n")}...`);
+    console.log("---");
+  });
+
+  console.log("\n--- Vectorizing ---");
+  try {
+    await rag.vectorize();
+    console.log("Vectorization complete.");
+  } catch (e) {
+    console.log("Vectorization skipped (no API key).", e);
+  }
+
+  // Search test (using BM25 only if vectors fail)
+  console.log("\n--- Search: 'install' ---");
+  try {
+    const results1 = await rag.search("install");
+    results1.forEach((r, i) => {
+      console.log(
+        `${i + 1}. [${r.score?.toFixed(4)}] ${r.text.slice(0, 50).replace(/\n/g, "\\n")}...`,
+      );
+    });
+  } catch (e) {
+    console.log("Search failed (likely no vectors):", e);
+  }
+}
+
+test().catch(console.error);

package/libs/ai/src/rag/rag.ts

@@ -0,0 +1,510 @@
+import { openai } from "@ai-sdk/openai";
+import type {
+  DocumentLayout,
+  Heading,
+  LayoutItemType,
+  List,
+  Paragraph,
+  Section,
+  Table,
+} from "@sea/schemas/core/layout";
+import { embed, embedMany } from "ai";
+import { getEncoding } from "js-tiktoken";
+
+// Helper for error wrapping since the module is missing
+async function wrapAiFunc<T>(fn: () => Promise<T>): Promise<T> {
+  try {
+    return await fn();
+  } catch (error) {
+    console.error("AI Error:", error);
+    throw error;
+  }
+}
+
+// --- Types ---
+
+export interface RagNode {
+  type: "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "p" | "ul" | "ol" | "table" | "code" | "other";
+  text: string;
+  headingPath: string[];
+  id?: string;
+  html?: string; // Store serialized HTML for tables/code if needed
+}
+
+export interface RagChunk {
+  id: string;
+  text: string;
+  type: "parent" | "child";
+  metadata: {
+    headingPath: string[];
+    nodeType?: string;
+    originalId?: string;
+    childIds?: string[]; // For parent chunks
+    parentId?: string; // For child chunks
+  };
+  embedding?: number[];
+  parentId?: string; // Link child to parent
+  termFrequencies?: Map<string, number>; // Pre-calculate for BM25
+}
+
+export interface VectorItem {
+  text: string;
+  embedding: number[];
+  metadata?: unknown;
+  score?: number; // For search results
+}
+
+// --- Constants ---
+
+const PARENT_CHUNK_TOKEN_TARGET = 600; // Target tokens for parent sections
+// const CHILD_CHUNK_TOKEN_TARGET = 150; // Target tokens for child nodes
+const ENCODING = getEncoding("cl100k_base");
+
+// --- Rag Class ---
+
+export class Rag {
+  chunks: RagChunk[] = [];
+  vectors: VectorItem[] | undefined;
+
+  // Simple BM25 Index
+  private docFreqs: Map<string, number> = new Map(); // term -> count of docs containing term
+  private docLengths: Map<string, number> = new Map(); // chunkId -> length in terms
+  private avgDocLength: number = 0;
+  private invertedIndex: Map<string, string[]> = new Map(); // term -> list of chunkIds
+
+  constructor(documents: (DocumentLayout | { layout: DocumentLayout; name: string })[]) {
+    const allNodes: RagNode[] = [];
+
+    for (const item of documents) {
+      let doc: DocumentLayout;
+      let name: string | undefined;
+
+      if ("layout" in item) {
+        doc = item.layout;
+        name = item.name;
+      } else {
+        doc = item;
+      }
+
+      const nodes = this.processLayout(doc, name);
+      allNodes.push(...nodes);
+    }
+
+    this.chunks = this.chunkNodes(allNodes);
+    this.buildBm25Index();
+  }
+
+  private processLayout(doc: DocumentLayout, docName?: string): RagNode[] {
+    const nodes: RagNode[] = [];
+    const headingStack: { level: number; text: string }[] = [];
+
+    this.traverseItems(doc.documentLayout, headingStack, nodes, docName);
+
+    return nodes;
+  }
+
+  private traverseItems(
+    items: LayoutItemType[],
+    headingStack: { level: number; text: string }[],
+    nodes: RagNode[],
+    docName?: string,
+  ) {
+    for (const item of items) {
+      this.processItem(item, headingStack, nodes, docName);
+    }
+  }
+
+  private processItem(
+    item: LayoutItemType,
+    headingStack: { level: number; text: string }[],
+    nodes: RagNode[],
+    docName?: string,
+  ) {
+    switch (item.type) {
+      case "heading": {
+        const heading = item as Heading;
+        const level = heading.level;
+        const text = heading.content.trim();
+
+        if (text) {
+          // Pop headings of same or higher level (lower number)
+          while (headingStack.length > 0 && headingStack[headingStack.length - 1]!.level >= level) {
+            headingStack.pop();
+          }
+          headingStack.push({ level, text });
+
+          nodes.push({
+            type: `h${level}`,
+            text,
+            headingPath: headingStack.map((h) => h.text),
+            id: heading.id,
+          });
+        }
+        break;
+      }
+      case "paragraph": {
+        const p = item as Paragraph;
+        const text = p.content.trim();
+        if (text) {
+          nodes.push({
+            type: "p",
+            text: docName ? `[Document: ${docName}]\n${text}` : text,
+            headingPath: headingStack.map((h) => h.text),
+            id: p.id,
+          });
+        }
+        break;
+      }
+      case "list": {
+        const list = item as List;
+        // Treat list as atomic block
+        // Flatten list items into a single text block
+        const text = list.items.map((li) => `• ${li.content.trim()}`).join("\n");
+
+        if (text) {
+          nodes.push({
+            type: list.ordered ? "ol" : "ul",
+            text,
+            headingPath: headingStack.map((h) => h.text),
+            id: list.id,
+          });
+        }
+        break;
+      }
+      case "table": {
+        const table = item as Table;
+        // Linearize table for search
+        const rows: string[] = [];
+        table.content.forEach((row) => {
+          const cells = row.cells.map((cell) => cell.content.trim());
+          rows.push(cells.join(" | "));
+        });
+        const text = rows.join("\n");
+
+        nodes.push({
+          type: "table",
+          text,
+          headingPath: headingStack.map((h) => h.text),
+          id: table.id,
+          html: table.serialize(), // Keep HTML for rendering if needed
+        });
+        break;
+      }
+      case "section": {
+        const section = item as Section;
+        // Process header if exists
+        if (section.header) {
+          this.traverseItems(section.header.content, headingStack, nodes, docName);
+        }
+
+        // Process content
+        this.traverseItems(section.content, headingStack, nodes, docName);
+
+        // Process footer if exists
+        if (section.footer) {
+          this.traverseItems(section.footer.content, headingStack, nodes, docName);
+        }
+        break;
+      }
+      default:
+        // Ignore unknown types or recurse if it has content
+        break;
+    }
+  }
+
+  private chunkNodes(nodes: RagNode[]): RagChunk[] {
+    const chunks: RagChunk[] = [];
+    const parentGroups: { parentText: string; children: RagNode[] }[] = [];
+
+    // 1. Group nodes into Parent Sections
+    let currentGroupNodes: RagNode[] = [];
+    let currentTokens = 0;
+
+    const flushGroup = () => {
+      if (currentGroupNodes.length === 0) return;
+      parentGroups.push({
+        parentText: this.assembleParentText(currentGroupNodes),
+        children: [...currentGroupNodes],
+      });
+      currentGroupNodes = [];
+      currentTokens = 0;
+    };
+
+    for (const node of nodes) {
+      const nodeTokens = this.countTokens(node.text);
+      const isMajorHeading = node.type === "h1" || node.type === "h2";
+
+      if (
+        (isMajorHeading || currentTokens + nodeTokens > PARENT_CHUNK_TOKEN_TARGET) &&
+        currentGroupNodes.length > 0
+      ) {
+        flushGroup();
+      }
+      currentGroupNodes.push(node);
+      currentTokens += nodeTokens;
+    }
+    flushGroup();
+
+    // 2. Create actual RagChunk objects
+    parentGroups.forEach((group, pIdx) => {
+      const parentId = `parent-${pIdx}`;
+      const childIds: string[] = [];
+
+      // Create IDs for children first so we can link them in parent
+      group.children.forEach((_, cIdx) => childIds.push(`${parentId}-child-${cIdx}`));
+
+      // Push the Parent Chunk
+      chunks.push({
+        id: parentId,
+        text: group.parentText,
+        type: "parent",
+        metadata: {
+          headingPath: group.children[0]?.headingPath || [],
+          childIds,
+        },
+      });
+
+      // Push Child Chunks linked to this Parent
+      group.children.forEach((childNode, cIdx) => {
+        chunks.push({
+          id: childIds[cIdx]!,
+          parentId: parentId, // The link!
+          text: `${childNode.headingPath.join(" > ")}\n${childNode.text}`,
+          type: "child",
+          metadata: {
+            headingPath: childNode.headingPath,
+            nodeType: childNode.type,
+            originalId: childNode.id,
+            parentId,
+          },
+        });
+      });
+    });
+
+    return chunks;
+  }
+
+  private assembleParentText(nodes: RagNode[]): string {
+    const firstNode = nodes[0];
+    const header = firstNode?.headingPath.join(" > ") || "";
+    const body = nodes
+      .map((n) => {
+        if (n.type.startsWith("h")) return `\n# ${n.text}`;
+        return n.text;
+      })
+      .join("\n\n");
+    return `${header}\n\n${body}`;
+  }
+
+  private countTokens(text: string): number {
+    const tokens = ENCODING.encode(text);
+    return tokens.length;
+  }
+
+  async vectorize() {
+    const texts = this.chunks.map((c) => c.text);
+    const vectors = await this.vectorizeData(texts);
+
+    this.chunks.forEach((chunk, i) => {
+      chunk.embedding = vectors[i]!.embedding;
+    });
+
+    this.vectors = this.chunks.map((c) => ({
+      text: c.text,
+      embedding: c.embedding!,
+      metadata: c.metadata,
+    }));
+  }
+
+  private async vectorizeData(values: string[]): Promise<VectorItem[]> {
+    const chunkSize = 10;
+    const results: VectorItem[] = [];
+
+    for (let i = 0; i < values.length; i += chunkSize) {
+      const chunk = values.slice(i, i + chunkSize);
+
+      const { embeddings } = await wrapAiFunc(() =>
+        embedMany({
+          model: openai.embedding("text-embedding-3-small"),
+          values: chunk,
+        }),
+      );
+
+      chunk.forEach((text, idx) => {
+        results.push({
+          text,
+          embedding: embeddings[idx]!,
+        });
+      });
+    }
+
+    return results;
+  }
+
+  // --- BM25 Implementation ---
+
+  private buildBm25Index() {
+    this.docFreqs.clear();
+    this.docLengths.clear();
+    this.invertedIndex.clear();
+    let totalLen = 0;
+
+    this.chunks.forEach((chunk) => {
+      const terms = this.tokenize(chunk.text);
+      this.docLengths.set(chunk.id, terms.length);
+      totalLen += terms.length;
+
+      const uniqueTerms = new Set(terms);
+
+      // Calculate Term Frequency (TF) for this doc
+      const counts = new Map<string, number>();
+      terms.forEach((t) => counts.set(t, (counts.get(t) || 0) + 1));
+      chunk.termFrequencies = counts;
+
+      uniqueTerms.forEach((term) => {
+        this.docFreqs.set(term, (this.docFreqs.get(term) || 0) + 1);
+
+        if (!this.invertedIndex.has(term)) {
+          this.invertedIndex.set(term, []);
+        }
+        this.invertedIndex.get(term)!.push(chunk.id);
+      });
+    });
+
+    this.avgDocLength = totalLen / this.chunks.length;
+  }
+
+  private tokenize(text: string): string[] {
+    return text
+      .toLowerCase()
+      .replace(/[^\w\s]/g, "")
+      .split(/\s+/)
+      .filter((t) => t.length > 0);
+  }
+
+  private searchBm25(query: string, k: number = 1.2, b: number = 0.75): Map<string, number> {
+    const scores = new Map<string, number>();
+    const terms = this.tokenize(query);
+    const N = this.chunks.length;
+
+    terms.forEach((term) => {
+      if (!this.invertedIndex.has(term)) return;
+
+      const docIds = this.invertedIndex.get(term)!;
+      const df = this.docFreqs.get(term) || 0;
+      const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
+
+      docIds.forEach((docId) => {
+        const chunk = this.chunks.find((c) => c.id === docId)!;
+        const docLen = this.docLengths.get(docId)!;
+        const termFreq = chunk.termFrequencies?.get(term) || 0;
+
+        const score =
+          (idf * (termFreq * (k + 1))) /
+          (termFreq + k * (1 - b + b * (docLen / this.avgDocLength)));
+        scores.set(docId, (scores.get(docId) || 0) + score);
+      });
+    });
+
+    return scores;
+  }
+
+  // --- Hybrid Search ---
+
+  async search(query: string, topK: number = 5) {
+    if (!this.vectors) {
+      throw new Error("No vectors");
+    }
+
+    // 1. Dense Search
+    const { embedding: queryEmbedding } = await wrapAiFunc(() =>
+      embed({
+        model: openai.embedding("text-embedding-3-small"),
+        value: query,
+      }),
+    );
+
+    const denseResults = this.chunks
+      .map((chunk) => ({
+        id: chunk.id,
+        score: this.cosineSimilarity(queryEmbedding, chunk.embedding!),
+        chunk,
+      }))
+      .sort((a, b) => b.score - a.score)
+      .slice(0, topK * 2);
+
+    console.log("DENSE: ", denseResults);
+
+    // 2. Sparse Search (BM25)
+    // Note: This relies on in-memory BM25 which we just optimized.
+    const bm25Scores = this.searchBm25(query);
+    const sparseResults = Array.from(bm25Scores.entries())
+      .map(([id, score]) => ({
+        id,
+        score,
+        chunk: this.chunks.find((c) => c.id === id)!,
+      }))
+      .sort((a, b) => b.score - a.score)
+      .slice(0, topK * 2);
+
+    // 3. RRF Fusion
+    const rrfScores = new Map<string, number>();
+    const k = 60;
+
+    [denseResults, sparseResults].forEach((results) => {
+      results.forEach((item, rank) => {
+        const current = rrfScores.get(item.id) || 0;
+        rrfScores.set(item.id, current + 1 / (k + rank + 1));
+      });
+    });
+
+    const finalResults = Array.from(rrfScores.entries())
+      .map(([id, score]) => {
+        const chunk = this.chunks.find((c) => c.id === id)!;
+        return {
+          text: chunk.text,
+          embedding: chunk.embedding!,
+          metadata: chunk.metadata,
+          score,
+          parentId: chunk.parentId,
+          id: chunk.id,
+        };
+      })
+      .sort((a, b) => b.score - a.score);
+
+    // Hydrate parent text if it's a child result (Parent Expansion)
+    const hydratedResults = finalResults.map((result) => {
+      if (result.parentId) {
+        const parent = this.chunks.find((c) => c.id === result.parentId);
+        if (parent) {
+          return {
+            ...result,
+            text: parent.text, // Swap child text for parent text
+            isFromChild: true,
+            originalSnippet: result.text,
+          };
+        }
+      }
+      return result;
+    });
+
+    // Deduplicate based on ID (which might be parent ID after hydration if we were strictly mapping to parents,
+    // but here we just want unique content. If we hydrated, we effectively have the parent content, but distinct child hits.
+    // However, the industry practice for "Parent Document Retrieval" is to return the parent.
+    // If multiple children match the same parent, we don't want the parent returned multiple times.
+    const seen = new Set<string>();
+    const uniqueResults = hydratedResults.filter((item) => {
+      // Use parentId if it was hydrated (effectively it IS the parent now), otherwise use id
+      const effectiveId = item.parentId || item.id;
+      if (seen.has(effectiveId)) return false;
+      seen.add(effectiveId);
+      return true;
+    });
+
+    return uniqueResults.slice(0, topK);
+  }
+
+  private cosineSimilarity(a: number[], b: number[]): number {
+    return a.reduce((sum, val, i) => sum + val * b[i]!, 0);
+  }
+}

package/libs/ai/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "declaration": true,
+    "moduleResolution": "Bundler",
+    "module": "ESNext",
+    "target": "ES2022",
+    "skipLibCheck": true,
+    "strict": true,
+    "strictNullChecks": true,
+    "noUncheckedIndexedAccess": true,
+    "baseUrl": ".",
+    "rootDir": "./src",
+    "outDir": "./dist",
+    "lib": ["ES2022", "DOM"],
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["src/**/*"],
+  "exclude": ["dist"]
+}