veryfront 0.1.49 → 0.1.51

package/src/src/embedding/chunk.ts

@@ -1,5 +1,11 @@
 import type { ChunkOptions } from "./types.js";
 
+/** Default maximum characters per chunk (~512 tokens). */
+const DEFAULT_CHUNK_MAX_CHARS = 2_000;
+
+/** Default character overlap between consecutive chunks. */
+const DEFAULT_CHUNK_OVERLAP = 200;
+
 /**
  * Splits text into overlapping chunks for embedding.
  *
@@ -16,8 +22,8 @@ import type { ChunkOptions } from "./types.js";
  * ```
  */
 export async function chunk(text: string, options?: ChunkOptions): Promise<string[]> {
-  const maxChars = options?.maxChars ?? 2000;
-  const overlap = options?.overlap ?? 200;
+  const maxChars = options?.maxChars ?? DEFAULT_CHUNK_MAX_CHARS;
+  const overlap = options?.overlap ?? DEFAULT_CHUNK_OVERLAP;
   const separators = options?.separators ?? ["\n\n", "\n", " ", ""];
 
   return splitRecursive(text, separators, maxChars, overlap);

package/src/src/embedding/embedding.ts

@@ -1,6 +1,7 @@
 import { embed, embedMany } from "ai";
 import type { Embedding, EmbeddingConfig } from "./types.js";
 import { resolveEmbeddingModel } from "./resolve.js";
+import { resolveConfiguredEmbeddingModel } from "./model-resolution.js";
 
 const DEFAULT_BATCH_SIZE = 100;
 
@@ -16,7 +17,6 @@ const DEFAULT_BATCH_SIZE = 100;
  * @example
  * ```ts
  * const embedder = embedding({
- *   model: "openai/text-embedding-3-small",
  *   documentPrefix: "search_document: ",  // for Nomic, E5, BGE models
  *   queryPrefix: "search_query: ",
  * });
@@ -25,13 +25,14 @@ const DEFAULT_BATCH_SIZE = 100;
  * ```
  */
 export function embedding(config: EmbeddingConfig): Embedding {
-  const model = resolveEmbeddingModel(config.model);
+  const modelId = resolveConfiguredEmbeddingModel(config.model);
+  const model = resolveEmbeddingModel(modelId);
   const batchSize = config.batchSize ?? DEFAULT_BATCH_SIZE;
   const docPrefix = config.documentPrefix ?? "";
   const queryPrefix = config.queryPrefix ?? "";
 
   return {
-    model: config.model,
+    model: modelId,
 
     async embed(text: string): Promise<number[]> {
       const result = await embed({ model, value: queryPrefix + text });

package/src/src/embedding/index.ts

@@ -8,9 +8,9 @@
  *
  * @example
  * ```ts
- * import { uploadStore, createUploadHandler } from "veryfront/embedding";
+ * import { ragStore, createUploadHandler } from "veryfront/embedding";
  *
- * const store = uploadStore({ model: "openai/text-embedding-3-small" });
+ * const store = ragStore({});
  * export const { POST, GET, DELETE } = createUploadHandler(store);
  * ```
  */
@@ -21,7 +21,7 @@ export { embedding } from "./embedding.js";
 export { chunk } from "./chunk.js";
 export { cosineSimilarity as similarity } from "ai";
 export { vectorStore } from "./vector-store.js";
-export { uploadStore } from "./upload-store.js";
+export { ragStore } from "./rag-store.js";
 export { createUploadHandler } from "./upload-handler.js";
 export { loadUpload } from "./upload-loader.js";
 export { useUploads } from "./react/use-uploads.js";
@@ -35,15 +35,16 @@ export type {
   ChunkOptions,
   Embedding,
   EmbeddingConfig,
+  RagChunk,
+  RagDocumentMeta,
+  RagSearchOptions,
+  RagSearchResult,
+  RagStore,
+  RagStoreBackend,
+  RagStoreConfig,
+  RagStoreData,
   SearchOptions,
   SearchResult,
-  StoredChunk,
-  UploadMeta,
-  UploadSearchOptions,
-  UploadSearchResult,
-  UploadStore,
-  UploadStoreConfig,
-  UploadStoreData,
   VectorStore,
   VectorStoreConfig,
 } from "./types.js";

package/src/src/embedding/model-resolution.ts

@@ -0,0 +1,23 @@
+import {
+  getDefaultVeryfrontCloudEmbeddingModel,
+  isVeryfrontCloudEnabled,
+} from "../platform/cloud/resolver.js";
+import { DEFAULT_LOCAL_EMBEDDING_MODEL } from "../provider/local/model-catalog.js";
+
+export const AUTO_EMBEDDING_MODEL = "auto";
+
+export function normalizeEmbeddingModelConfig(model?: string): string {
+  const normalized = model?.trim();
+  return normalized && normalized.length > 0 ? normalized : AUTO_EMBEDDING_MODEL;
+}
+
+export function resolveConfiguredEmbeddingModel(model?: string): string {
+  const normalized = normalizeEmbeddingModelConfig(model);
+  if (normalized !== AUTO_EMBEDDING_MODEL) {
+    return normalized;
+  }
+
+  return isVeryfrontCloudEnabled()
+    ? getDefaultVeryfrontCloudEmbeddingModel()
+    : `local/${DEFAULT_LOCAL_EMBEDDING_MODEL}`;
+}

package/src/src/embedding/rag-store.ts

@@ -0,0 +1,431 @@
+import * as dntShim from "../../_dnt.shims.js";
+import { cosineSimilarity } from "ai";
+import {
+  isNotFoundError,
+  mkdir,
+  readDir,
+  readTextFile,
+  writeTextFile,
+} from "../platform/compat/fs.js";
+import { dirname, extname, join } from "../platform/compat/path/basic-operations.js";
+import { serverLogger } from "../utils/index.js";
+import { isVeryfrontCloudEnabled } from "../platform/cloud/resolver.js";
+import { getEnv } from "../platform/compat/process.js";
+import { embedding } from "./embedding.js";
+import { chunk } from "./chunk.js";
+import { createVeryfrontCloudRagStore } from "./veryfront-cloud/rag-store.js";
+import { resolveConfiguredEmbeddingModel } from "./model-resolution.js";
+import type {
+  RagChunk,
+  RagDocumentMeta,
+  RagSearchOptions,
+  RagSearchResult,
+  RagStore,
+  RagStoreBackend,
+  RagStoreConfig,
+  RagStoreData,
+} from "./types.js";
+
+// Legacy data shapes used only for migrating old upload-store JSON files.
+interface LegacyStoredChunk {
+  id: string;
+  uploadId: string;
+  text: string;
+  embedding: number[];
+  index: number;
+}
+
+interface LegacyUploadStoreData {
+  uploads: RagDocumentMeta[];
+  chunks: LegacyStoredChunk[];
+}
+import { INVALID_ARGUMENT } from "../errors/index.js";
+
+type ResolvedRagStoreConfig = RagStoreConfig & { model: string };
+
+/** Default number of top results returned by similarity search. */
+const DEFAULT_TOP_K = 5;
+
+/**
+ * Creates a persistent RAG store with lazy embedding and similarity search.
+ *
+ * Combines document ingestion, chunking, embedding, and vector search into
+ * a single factory. Documents are chunked on ingest; embeddings are generated
+ * lazily on the first search call to avoid blocking uploads on AI API calls.
+ *
+ * By default, this uses the local JSON store. When Veryfront Cloud bootstrap
+ * is present, it automatically upgrades to the cloud-backed store unless
+ * explicitly overridden.
+ *
+ * @example
+ * ```ts
+ * import { ragStore } from "veryfront/embedding";
+ *
+ * const store = ragStore({
+ *   storagePath: "data/index.json",
+ *   contentDir: "content",
+ * });
+ *
+ * await store.ingest("My Doc", text, { source: "upload:file.pdf", type: "pdf" });
+ * const results = await store.search("query", { topK: 5, threshold: 0.7 });
+ * ```
+ */
+export function ragStore(config: RagStoreConfig): RagStore {
+  const storeCache = new Map<string, RagStore>();
+
+  function getStore(): RagStore {
+    const resolvedConfig = resolveRagStoreConfig(config);
+    const backend = resolveRagStoreBackend(config);
+    const cacheKey = JSON.stringify({ backend, config: resolvedConfig });
+    const cached = storeCache.get(cacheKey);
+    if (cached) return cached;
+
+    const store = backend === "veryfront-cloud"
+      ? createVeryfrontCloudRagStore(resolvedConfig)
+      : createLocalJsonRagStore(resolvedConfig);
+    storeCache.set(cacheKey, store);
+    return store;
+  }
+
+  return {
+    ingest(title, text, meta) {
+      return getStore().ingest(title, text, meta);
+    },
+    search(query, options) {
+      return getStore().search(query, options);
+    },
+    listDocuments() {
+      return getStore().listDocuments();
+    },
+    removeDocument(id) {
+      return getStore().removeDocument(id);
+    },
+    indexContentDir() {
+      return getStore().indexContentDir();
+    },
+  };
+}
+
+function resolveRagStoreConfig(config: RagStoreConfig): ResolvedRagStoreConfig {
+  return {
+    ...config,
+    model: resolveConfiguredEmbeddingModel(config.model),
+  };
+}
+
+function normalizeRagStoreBackend(
+  value: string | undefined,
+): RagStoreBackend | undefined {
+  const normalized = value?.trim().toLowerCase();
+
+  switch (normalized) {
+    case undefined:
+    case "":
+      return undefined;
+    case "auto":
+      return "auto";
+    case "local":
+    case "local-json":
+      return "local-json";
+    case "cloud":
+    case "veryfront-cloud":
+      return "veryfront-cloud";
+    default:
+      throw INVALID_ARGUMENT.create({
+        detail:
+          `Invalid RAG backend "${value}". Expected "auto", "local-json", or "veryfront-cloud".`,
+      });
+  }
+}
+
+function resolveRagStoreBackend(config: RagStoreConfig): Exclude<RagStoreBackend, "auto"> {
+  const configured = normalizeRagStoreBackend(config.backend);
+  if (configured && configured !== "auto") return configured;
+
+  const envOverride = normalizeRagStoreBackend(getEnv("VERYFRONT_RAG_BACKEND"));
+  if (envOverride && envOverride !== "auto") return envOverride;
+
+  return isVeryfrontCloudEnabled() ? "veryfront-cloud" : "local-json";
+}
+
+function createLocalJsonRagStore(config: ResolvedRagStoreConfig): RagStore {
+  const storagePath = config.storagePath ?? "data/index.json";
+  const contentDir = config.contentDir;
+  const contentExtensions = new Set(config.contentExtensions ?? [".md", ".mdx", ".txt"]);
+  const chunkOptions = config.chunkOptions;
+
+  const MAX_TEXT_LENGTH = 5 * 1024 * 1024; // 5 MB text limit per document
+
+  // Serialize all load→modify→save operations to prevent concurrent overwrites.
+  // NOTE: This is a single-process, single-instance mutex. In multi-instance
+  // deployments, concurrent stores targeting the same file will race.
+  let mutex: Promise<void> = Promise.resolve();
+  function withLock<T>(fn: () => Promise<T>): Promise<T> {
+    const result = mutex.then(fn);
+    mutex = result.then(
+      () => {},
+      (err) => {
+        serverLogger.error("[rag-store] Lock operation failed:", err);
+      },
+    );
+    return result;
+  }
+
+  function createEmbedder() {
+    return embedding({
+      model: config.model,
+      documentPrefix: config.documentPrefix,
+      queryPrefix: config.queryPrefix,
+      batchSize: config.batchSize,
+    });
+  }
+
+  function isLegacyUploadStoreData(value: unknown): value is LegacyUploadStoreData {
+    if (!value || typeof value !== "object") return false;
+    const data = value as { uploads?: unknown; chunks?: unknown };
+    return Array.isArray(data.uploads) && Array.isArray(data.chunks);
+  }
+
+  function migrateLegacyUploadStoreData(data: LegacyUploadStoreData): RagStoreData {
+    return {
+      documents: data.uploads.map((upload) => ({ ...upload })),
+      chunks: data.chunks.map((chunk: LegacyStoredChunk) => ({
+        id: chunk.id,
+        documentId: chunk.uploadId,
+        text: chunk.text,
+        embedding: chunk.embedding,
+        index: chunk.index,
+      })),
+    };
+  }
+
+  async function load(): Promise<RagStoreData> {
+    try {
+      const data = await readTextFile(storagePath);
+      const parsed = JSON.parse(data);
+      if (isLegacyUploadStoreData(parsed)) {
+        return migrateLegacyUploadStoreData(parsed);
+      }
+      if (!parsed || !Array.isArray(parsed.documents) || !Array.isArray(parsed.chunks)) {
+        serverLogger.warn("[rag-store] Corrupted store file, resetting", { storagePath });
+        return { documents: [], chunks: [] };
+      }
+      return parsed as RagStoreData;
+    } catch (err) {
+      // File not found is expected on first run; anything else is worth logging
+      if (isNotFoundError(err)) {
+        return { documents: [], chunks: [] };
+      }
+      serverLogger.warn("[rag-store] Failed to load store, resetting", err);
+      return { documents: [], chunks: [] };
+    }
+  }
+
+  async function save(data: RagStoreData): Promise<void> {
+    const dir = dirname(storagePath);
+    if (dir && dir !== ".") {
+      await mkdir(dir, { recursive: true });
+    }
+    const payload = JSON.stringify(data, null, 2);
+    // Atomic write: write to temp file then rename to prevent corruption on crash
+    const tmpPath = storagePath + ".tmp";
+    await writeTextFile(tmpPath, payload);
+    try {
+      if (typeof dntShim.Deno !== "undefined") {
+        await dntShim.Deno.rename(tmpPath, storagePath);
+      } else {
+        const fs = await import("node:fs/promises");
+        await fs.rename(tmpPath, storagePath);
+      }
+    } catch (_) {
+      // expected: rename not available in all environments, fall back to direct write
+      await writeTextFile(storagePath, payload);
+    }
+  }
+
+  async function ensureEmbeddings(data: RagStoreData): Promise<boolean> {
+    const unembedded = data.chunks.filter((c) => c.embedding.length === 0);
+    if (unembedded.length === 0) return false;
+
+    const embedder = createEmbedder();
+    const embeddings = await embedder.embedMany(unembedded.map((c) => c.text));
+    for (let i = 0; i < unembedded.length; i++) {
+      unembedded[i]!.embedding = embeddings[i]!;
+    }
+    return true;
+  }
+
+  async function listContentFiles(dir: string): Promise<string[]> {
+    const files: string[] = [];
+    try {
+      for await (const entry of readDir(dir)) {
+        const fullPath = join(dir, entry.name);
+        if (entry.isDirectory) {
+          files.push(...(await listContentFiles(fullPath)));
+        } else if (entry.isFile && contentExtensions.has(extname(entry.name))) {
+          files.push(fullPath);
+        }
+      }
+    } catch (_) {
+      // expected: directory may not exist yet
+    }
+    return files;
+  }
+
+  return {
+    async ingest(
+      title: string,
+      text: string,
+      meta?: { source?: string; type?: string },
+    ): Promise<string> {
+      return withLock(async () => {
+        const data = await load();
+        const documentId = dntShim.crypto.randomUUID();
+
+        if (text.length > MAX_TEXT_LENGTH) {
+          throw INVALID_ARGUMENT.create({
+            detail: `Upload text exceeds ${MAX_TEXT_LENGTH / 1024 / 1024} MB limit`,
+          });
+        }
+
+        const chunks = await chunk(text, chunkOptions);
+        if (chunks.length === 0) {
+          throw INVALID_ARGUMENT.create({ detail: "Upload contains no extractable text" });
+        }
+
+        const doc: RagDocumentMeta = {
+          id: documentId,
+          title,
+          source: meta?.source ?? "",
+          type: meta?.type ?? "",
+          createdAt: Date.now(),
+        };
+
+        const chunkRecords: RagChunk[] = chunks.map((chunkText, i) => ({
+          id: dntShim.crypto.randomUUID(),
+          documentId,
+          text: chunkText,
+          embedding: [], // filled lazily on first search
+          index: i,
+        }));
+
+        data.documents.push(doc);
+        data.chunks.push(...chunkRecords);
+        await save(data);
+
+        return documentId;
+      });
+    },
+
+    async search(
+      query: string,
+      options?: RagSearchOptions,
+    ): Promise<RagSearchResult[]> {
+      return withLock(async () => {
+        const data = await load();
+        if (data.chunks.length === 0) return [];
+
+        const updated = await ensureEmbeddings(data);
+        if (updated) await save(data);
+
+        const embedder = createEmbedder();
+        const queryEmbedding = await embedder.embed(query);
+        const topK = options?.topK ?? DEFAULT_TOP_K;
+        const threshold = options?.threshold;
+
+        const docMap = new Map(data.documents.map((d) => [d.id, d]));
+
+        const scored = data.chunks.map((c) => {
+          const doc = docMap.get(c.documentId);
+          return {
+            text: c.text,
+            score: cosineSimilarity(queryEmbedding, c.embedding),
+            documentId: c.documentId,
+            title: doc?.title ?? "Unknown",
+            source: doc?.source ?? "",
+            type: doc?.type ?? "",
+          };
+        });
+
+        scored.sort((a, b) => b.score - a.score);
+
+        let results = scored.slice(0, topK);
+        if (threshold !== undefined) {
+          results = results.filter((r) => r.score >= threshold);
+        }
+        return results;
+      });
+    },
+
+    async listDocuments(): Promise<RagDocumentMeta[]> {
+      return withLock(async () => {
+        const data = await load();
+        return data.documents;
+      });
+    },
+
+    async removeDocument(id: string): Promise<void> {
+      return withLock(async () => {
+        const data = await load();
+        data.documents = data.documents.filter((d) => d.id !== id);
+        data.chunks = data.chunks.filter((c) => c.documentId !== id);
+        await save(data);
+      });
+    },
+
+    async indexContentDir(): Promise<void> {
+      if (!contentDir) return;
+
+      return withLock(async () => {
+        const data = await load();
+        const indexedSources = new Set(data.documents.map((d) => d.source));
+
+        const files = await listContentFiles(contentDir);
+        const newFiles = files.filter((f) => !indexedSources.has(f));
+        if (newFiles.length === 0) return;
+
+        for (const file of newFiles) {
+          const content = await readTextFile(file);
+          if (!content?.trim()) continue;
+          if (content.length > MAX_TEXT_LENGTH) {
+            serverLogger.warn(
+              `[rag-store] Skipping ${file}: exceeds ${
+                MAX_TEXT_LENGTH / 1024 / 1024
+              } MB text limit`,
+            );
+            continue;
+          }
+
+          const title = file.startsWith(contentDir + "/")
+            ? file.slice(contentDir.length + 1).replace(/\.[^.]+$/, "")
+            : file.replace(/\.[^.]+$/, "");
+          const documentId = dntShim.crypto.randomUUID();
+          const type = extname(file).slice(1);
+
+          const chunks = await chunk(content, chunkOptions);
+          if (chunks.length === 0) continue;
+
+          data.documents.push({
+            id: documentId,
+            title,
+            source: file,
+            type,
+            createdAt: Date.now(),
+          });
+
+          data.chunks.push(
+            ...chunks.map((chunkText, i) => ({
+              id: dntShim.crypto.randomUUID(),
+              documentId,
+              text: chunkText,
+              embedding: [] as number[],
+              index: i,
+            })),
+          );
+        }
+
+        await save(data);
+      });
+    },
+  };
+}

package/src/src/embedding/react/use-uploads.ts

@@ -8,7 +8,8 @@ import * as dntShim from "../../../_dnt.shims.js";
 
 
 import { useCallback, useEffect, useRef, useState } from "react";
-import type { UploadMeta } from "../types.js";
+import type { RagDocumentMeta } from "../types.js";
+import { API_ERROR } from "../../errors/index.js";
 
 export interface UseUploadsOptions {
   /** API endpoint, e.g. "/api/uploads" */
@@ -17,7 +18,7 @@ export interface UseUploadsOptions {
 
 export interface UseUploadsResult {
   /** All uploads from the server */
-  uploads: UploadMeta[];
+  uploads: RagDocumentMeta[];
   /** True while an upload is in progress */
   uploading: boolean;
   /** User-facing error message, cleared on next action */
@@ -34,7 +35,7 @@ export interface UseUploadsResult {
  * useUploads hook for managing RAG upload lifecycle.
  */
 export function useUploads(options: UseUploadsOptions): UseUploadsResult {
-  const [uploads, setUploads] = useState<UploadMeta[]>([]);
+  const [uploads, setUploads] = useState<RagDocumentMeta[]>([]);
   const [uploading, setUploading] = useState(false);
   const [error, setError] = useState<string | null>(null);
   const abortControllerRef = useRef<AbortController | null>(null);
@@ -45,7 +46,8 @@ export function useUploads(options: UseUploadsOptions): UseUploadsResult {
       if (!res.ok) return;
       const data = await res.json();
       setUploads(data.uploads ?? []);
-    } catch {
+    } catch (error) {
+      console.debug("useUploads: failed to load uploads", error);
       setError("Failed to load uploads");
     }
   }, [options.api]);
@@ -71,7 +73,7 @@ export function useUploads(options: UseUploadsOptions): UseUploadsResult {
           signal: abortController.signal,
         });
         const data = await res.json();
-        if (!res.ok) throw new Error(data.error ?? "Upload failed");
+        if (!res.ok) throw API_ERROR.create({ detail: data.error ?? "Upload failed" });
         await refresh();
       } catch (err) {
         if (err instanceof Error && err.name === "AbortError") return;
@@ -90,7 +92,8 @@ export function useUploads(options: UseUploadsOptions): UseUploadsResult {
       try {
         await dntShim.fetch(`${options.api}/${id}`, { method: "DELETE" });
         await refresh();
-      } catch {
+      } catch (error) {
+        console.debug("useUploads: failed to delete upload", error);
         setError("Failed to delete upload");
       }
     },

package/src/src/embedding/resolve.ts

@@ -4,8 +4,9 @@ import { createOpenAI } from "@ai-sdk/openai";
 import { createGoogleGenerativeAI } from "@ai-sdk/google";
 import { getGoogleGenAIEnvConfig, getOpenAIEnvConfig } from "../config/env.js";
 import { createLocalEmbeddingModel } from "../provider/local/local-embedding-adapter.js";
+import { createVeryfrontCloudEmbeddingModel } from "./veryfront-cloud/provider.js";
 
-export type EmbeddingProviderFactory = (modelId: string) => EmbeddingModel;
+type EmbeddingProviderFactory = (modelId: string) => EmbeddingModel;
 
 const providers = new Map<string, EmbeddingProviderFactory>();
 let autoInitialized = false;
@@ -66,6 +67,10 @@ function autoInitializeFromEnv(): void {
   if (!providers.has("local")) {
     providers.set("local", createLocalEmbeddingModel);
   }
+
+  if (!providers.has("veryfront-cloud")) {
+    providers.set("veryfront-cloud", createVeryfrontCloudEmbeddingModel);
+  }
 }
 
 /**