veryfront 0.1.47 → 0.1.49

package/src/src/embedding/resolve.ts

@@ -0,0 +1,126 @@
+import type { EmbeddingModel } from "ai";
+import { createError, toError } from "../errors/veryfront-error.js";
+import { createOpenAI } from "@ai-sdk/openai";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { getGoogleGenAIEnvConfig, getOpenAIEnvConfig } from "../config/env.js";
+import { createLocalEmbeddingModel } from "../provider/local/local-embedding-adapter.js";
+
+export type EmbeddingProviderFactory = (modelId: string) => EmbeddingModel;
+
+const providers = new Map<string, EmbeddingProviderFactory>();
+let autoInitialized = false;
+
+/**
+ * Register an embedding provider factory.
+ *
+ * @example
+ * ```ts
+ * registerEmbeddingProvider("openai", (id) =>
+ *   createOpenAI({ apiKey })().embedding(id)
+ * );
+ * ```
+ */
+export function registerEmbeddingProvider(
+  name: string,
+  factory: EmbeddingProviderFactory,
+): void {
+  providers.set(name, factory);
+}
+
+function autoInitializeFromEnv(): void {
+  if (autoInitialized) return;
+  autoInitialized = true;
+
+  if (!providers.has("openai")) {
+    providers.set("openai", (id) => {
+      const config = getOpenAIEnvConfig();
+      if (!config.apiKey) {
+        throw toError(
+          createError({
+            type: "config",
+            message:
+              "OPENAI_API_KEY not set. Set the environment variable or register a custom provider with registerEmbeddingProvider().",
+          }),
+        );
+      }
+      return createOpenAI({ apiKey: config.apiKey, baseURL: config.baseURL }).embedding(id);
+    });
+  }
+
+  if (!providers.has("google")) {
+    providers.set("google", (id) => {
+      const config = getGoogleGenAIEnvConfig();
+      if (!config.apiKey) {
+        throw toError(
+          createError({
+            type: "config",
+            message:
+              "GOOGLE_API_KEY (or GOOGLE_GENERATIVE_AI_API_KEY) not set. Set the environment variable or register a custom provider with registerEmbeddingProvider().",
+          }),
+        );
+      }
+      return createGoogleGenerativeAI({ apiKey: config.apiKey }).textEmbeddingModel(id);
+    });
+  }
+
+  if (!providers.has("local")) {
+    providers.set("local", createLocalEmbeddingModel);
+  }
+}
+
+/**
+ * Resolve a "provider/model" string to an AI SDK EmbeddingModel instance.
+ *
+ * @example
+ * ```ts
+ * const model = resolveEmbeddingModel("openai/text-embedding-3-small");
+ * ```
+ */
+export function resolveEmbeddingModel(modelString: string): EmbeddingModel {
+  autoInitializeFromEnv();
+
+  const slashIndex = modelString.indexOf("/");
+  if (slashIndex === -1) {
+    throw toError(
+      createError({
+        type: "config",
+        message:
+          `Invalid model string: "${modelString}". Expected "provider/model" (e.g. "openai/text-embedding-3-small").`,
+      }),
+    );
+  }
+
+  const providerName = modelString.slice(0, slashIndex);
+  const modelId = modelString.slice(slashIndex + 1);
+
+  if (!providerName || !modelId) {
+    throw toError(
+      createError({
+        type: "config",
+        message:
+          `Invalid model string: "${modelString}". Both provider and model name are required.`,
+      }),
+    );
+  }
+
+  const factory = providers.get(providerName);
+  if (!factory) {
+    const available = [...providers.keys()].join(", ") || "none";
+    throw toError(
+      createError({
+        type: "config",
+        message: `Embedding provider "${providerName}" not registered. Available: ${available}`,
+      }),
+    );
+  }
+
+  return factory(modelId);
+}
+
+/**
+ * Clear all registered embedding providers (for testing).
+ */
+export function clearEmbeddingProviders(): void {
+  providers.clear();
+  autoInitialized = false;
+}

package/src/src/embedding/types.ts

@@ -0,0 +1,103 @@
+export interface EmbeddingConfig {
+  model: string; // "provider/model" format, e.g. "openai/text-embedding-3-small"
+  documentPrefix?: string; // prepended when embedding documents, e.g. "search_document: "
+  queryPrefix?: string; // prepended when embedding queries, e.g. "search_query: "
+  batchSize?: number; // max texts per embedMany API call (default 100)
+}
+
+export interface Embedding {
+  model: string;
+  /** Embed a single text. Applies queryPrefix if configured. */
+  embed(text: string): Promise<number[]>;
+  /** Embed multiple texts. Applies documentPrefix if configured. Batches automatically. */
+  embedMany(texts: string[]): Promise<number[][]>;
+}
+
+export interface ChunkOptions {
+  maxChars?: number; // default 2000 (~512 tokens)
+  overlap?: number; // default 200 chars
+  separators?: string[]; // default ["\n\n", "\n", " ", ""]
+}
+
+export interface VectorStoreConfig {
+  embedder: Embedding;
+}
+
+export interface SearchOptions {
+  topK?: number; // default 5
+  threshold?: number; // minimum similarity score (0-1), discard below
+  filter?: Record<string, unknown>; // metadata exact-match filter
+  strategy?: "dense" | "hybrid" | "mmr"; // default "dense"
+  lambda?: number; // MMR diversity param (0 = max diversity, 1 = max relevance, default 0.5)
+}
+
+export interface SearchResult {
+  text: string;
+  score: number;
+  metadata?: Record<string, unknown>;
+}
+
+export interface VectorStore {
+  add(texts: string[], metadata?: Record<string, unknown>[]): Promise<void>;
+  search(query: string, options?: SearchOptions): Promise<SearchResult[]>;
+  clear(): void;
+  size: number;
+}
+
+// ---------------------------------------------------------------------------
+// Upload Store
+// ---------------------------------------------------------------------------
+
+export interface UploadMeta {
+  id: string;
+  title: string;
+  source: string;
+  type: string;
+  createdAt: number;
+}
+
+export interface StoredChunk {
+  id: string;
+  uploadId: string;
+  text: string;
+  embedding: number[]; // [] = not yet embedded
+  index: number;
+}
+
+export interface UploadStoreData {
+  uploads: UploadMeta[];
+  chunks: StoredChunk[];
+}
+
+export interface UploadStoreConfig {
+  model: string;
+  storagePath?: string; // default "data/index.json"
+  contentDir?: string; // optional auto-index dir
+  contentExtensions?: string[]; // default [".md", ".mdx", ".txt"]
+  chunkOptions?: ChunkOptions;
+  documentPrefix?: string;
+  queryPrefix?: string;
+  batchSize?: number;
+}
+
+export interface UploadSearchResult {
+  text: string;
+  score: number;
+  uploadId: string;
+  title: string;
+  source: string;
+  type: string;
+}
+
+export interface UploadSearchOptions {
+  topK?: number; // default 5
+  threshold?: number; // minimum similarity score
+}
+
+export interface UploadStore {
+  ingest(title: string, text: string, meta?: { source?: string; type?: string }): Promise<string>;
+  search(query: string, options?: UploadSearchOptions): Promise<UploadSearchResult[]>;
+  listUploads(): Promise<UploadMeta[]>;
+  removeUpload(id: string): Promise<void>;
+  indexContentDir(): Promise<void>;
+}

package/src/src/embedding/upload-handler.ts

@@ -0,0 +1,180 @@
+import * as dntShim from "../../_dnt.shims.js";
+import type { UploadStore } from "./types.js";
+import { loadUpload } from "./upload-loader.js";
+
+const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB
+
+const MIME_TO_TYPE: Record<string, string> = {
+  "text/plain": "txt",
+  "text/markdown": "md",
+  "text/mdx": "mdx",
+  "text/csv": "csv",
+  "text/html": "html",
+  "text/xml": "xml",
+  "application/csv": "csv",
+  "application/pdf": "pdf",
+  "application/rtf": "rtf",
+  "application/json": "json",
+  "application/epub+zip": "epub",
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
+  "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx",
+  "application/msword": "doc",
+  "application/vnd.ms-excel": "xls",
+  "application/vnd.ms-powerpoint": "ppt",
+};
+
+const EXT_TO_TYPE: Record<string, string> = {
+  txt: "txt",
+  md: "md",
+  mdx: "mdx",
+  csv: "csv",
+  html: "html",
+  htm: "html",
+  xml: "xml",
+  pdf: "pdf",
+  rtf: "rtf",
+  json: "json",
+  epub: "epub",
+  docx: "docx",
+  xlsx: "xlsx",
+  pptx: "pptx",
+  doc: "doc",
+  xls: "xls",
+  ppt: "ppt",
+};
+
+function inferType(file: dntShim.File): string | null {
+  const fromMime = MIME_TO_TYPE[file.type];
+  if (fromMime) return fromMime;
+  const ext = file.name.split(".").pop()?.toLowerCase();
+  return EXT_TO_TYPE[ext ?? ""] ?? null;
+}
+
+const TYPE_TO_MIME: Record<string, string> = Object.fromEntries(
+  Object.entries(MIME_TO_TYPE).map(([mime, type]) => [type, mime]),
+);
+
+function typeToMime(type: string): string {
+  return TYPE_TO_MIME[type] ?? "text/plain";
+}
+
+interface UploadHandlerConfig {
+  maxFileSize?: number;
+}
+
+/**
+ * Creates HTTP route handlers for upload, listing, and deletion.
+ *
+ * **Important:** These handlers do not include authentication or authorization.
+ * Add your own auth middleware before exposing them in production.
+ *
+ * Returns `{ POST, GET, DELETE }` handlers compatible with file-based routing.
+ * POST accepts multipart form data with a `file` field, extracts text via
+ * `loadUpload`, and ingests into the provided upload store. GET returns
+ * the upload list. DELETE removes an upload by ID from route params.
+ *
+ * @example
+ * ```ts
+ * // app/api/uploads/route.ts
+ * import { createUploadHandler } from "veryfront/embedding";
+ * import { store } from "../../../lib/store.ts";
+ *
+ * export const { POST, GET } = createUploadHandler(store);
+ * ```
+ *
+ * @example
+ * ```ts
+ * // app/api/uploads/[id]/route.ts
+ * import { createUploadHandler } from "veryfront/embedding";
+ * import { store } from "../../../../lib/store.ts";
+ *
+ * export const { DELETE } = createUploadHandler(store);
+ * ```
+ */
+export function createUploadHandler(
+  store: UploadStore,
+  config?: UploadHandlerConfig,
+) {
+  const maxSize = config?.maxFileSize ?? MAX_FILE_SIZE;
+
+  async function POST(request: dntShim.Request): Promise<dntShim.Response> {
+    try {
+      const formData = await request.formData();
+      const file = formData.get("file");
+
+      if (!file || !(file instanceof dntShim.File)) {
+        return dntShim.Response.json({ error: "No file provided" }, { status: 400 });
+      }
+
+      if (file.size > maxSize) {
+        return dntShim.Response.json(
+          { error: `File exceeds ${Math.round(maxSize / 1024 / 1024)} MB limit` },
+          { status: 400 },
+        );
+      }
+
+      const fileType = inferType(file);
+      if (!fileType) {
+        return dntShim.Response.json(
+          { error: `Unsupported file type: ${file.type || file.name}` },
+          { status: 400 },
+        );
+      }
+
+      const buffer = await file.arrayBuffer();
+      const text = await loadUpload(buffer, typeToMime(fileType));
+      if (!text.trim()) {
+        return dntShim.Response.json(
+          { error: "No text could be extracted from file" },
+          { status: 400 },
+        );
+      }
+
+      // Sanitize file name: strip path components, limit length
+      const safeName = file.name.replace(/[/\\]/g, "_").slice(0, 200);
+
+      const id = await store.ingest(safeName, text, {
+        source: `upload:${safeName}`,
+        type: fileType,
+      });
+
+      return dntShim.Response.json({
+        success: true,
+        upload: { id, title: file.name, type: fileType },
+      });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : "Upload failed";
+      return dntShim.Response.json({ error: message }, { status: 500 });
+    }
+  }
+
+  async function GET(): Promise<dntShim.Response> {
+    try {
+      const uploads = await store.listUploads();
+      return dntShim.Response.json({ uploads });
+    } catch (error) {
+      console.error("Upload list failed:", error);
+      return dntShim.Response.json({ error: "Failed to list uploads" }, { status: 500 });
+    }
+  }
+
+  async function DELETE(
+    _request: dntShim.Request,
+    context: { params: Record<string, string> },
+  ): Promise<dntShim.Response> {
+    try {
+      const id = context.params.id;
+      if (!id) {
+        return dntShim.Response.json({ error: "Missing upload ID" }, { status: 400 });
+      }
+      await store.removeUpload(id);
+      return dntShim.Response.json({ success: true });
+    } catch (error) {
+      console.error("Upload delete failed:", error);
+      return dntShim.Response.json({ error: "Delete failed" }, { status: 500 });
+    }
+  }
+
+  return { POST, GET, DELETE };
+}

package/src/src/embedding/upload-loader.ts

@@ -0,0 +1,76 @@
+import { importKreuzberg } from "../platform/compat/opaque-deps.js";
+
+/**
+ * Extracts plain text from various upload formats.
+ *
+ * Text and CSV are handled inline (CSV uses a RAG-optimized format that
+ * denormalizes headers into every row). All other formats (PDF, DOCX, XLSX,
+ * PPTX, HTML, RTF, EPUB, etc.) are delegated to kreuzberg for extraction.
+ *
+ * @example
+ * ```ts
+ * const text = await loadUpload(buffer, "application/pdf");
+ * ```
+ */
+export async function loadUpload(buffer: ArrayBuffer, mimeType: string): Promise<string> {
+  // CSV gets RAG-optimized formatting (headers denormalized into every row)
+  if (mimeType === "text/csv" || mimeType === "application/csv") {
+    return extractCSV(buffer);
+  }
+
+  // Plain text and markdown — no extraction needed
+  if (
+    mimeType === "text/plain" ||
+    mimeType === "text/markdown" ||
+    mimeType === "text/mdx"
+  ) {
+    return new TextDecoder().decode(buffer);
+  }
+
+  // Everything else (PDF, DOCX, XLSX, PPTX, HTML, XML, etc.) → kreuzberg
+  const { extractBytes } = await importKreuzberg();
+  const result = await extractBytes(new Uint8Array(buffer), mimeType);
+  return result.content;
+}
+
+function extractCSV(buffer: ArrayBuffer): string {
+  const text = new TextDecoder().decode(buffer);
+  const lines = text.split("\n").filter((line) => line.trim());
+  if (lines.length < 2) return text;
+
+  const headers = parseCSVLine(lines[0]!);
+  const rows = lines.slice(1).map((line) => {
+    const values = parseCSVLine(line);
+    return headers
+      .map((header, i) => `${header}: ${values[i] ?? ""}`)
+      .join(", ");
+  });
+
+  return rows.join("\n");
+}
+
+function parseCSVLine(line: string): string[] {
+  const values: string[] = [];
+  let current = "";
+  let inQuotes = false;
+
+  for (let i = 0; i < line.length; i++) {
+    const char = line[i]!;
+    if (char === '"') {
+      // RFC 4180: doubled quote inside a quoted field is a literal quote
+      if (inQuotes && i + 1 < line.length && line[i + 1] === '"') {
+        current += '"';
+        i++; // skip the second quote
+      } else {
+        inQuotes = !inQuotes;
+      }
+    } else if (char === "," && !inQuotes) {
+      values.push(current.trim());
+      current = "";
+    } else {
+      current += char;
+    }
+  }
+  values.push(current.trim());
+  return values;
+}