veryfront 0.1.47 → 0.1.49

package/src/src/embedding/upload-store.ts

@@ -0,0 +1,299 @@
+import * as dntShim from "../../_dnt.shims.js";
+import { cosineSimilarity } from "ai";
+import {
+  isNotFoundError,
+  mkdir,
+  readDir,
+  readTextFile,
+  writeTextFile,
+} from "../platform/compat/fs.js";
+import { dirname, extname, join } from "../platform/compat/path/basic-operations.js";
+import { embedding } from "./embedding.js";
+import { chunk } from "./chunk.js";
+import type {
+  StoredChunk,
+  UploadMeta,
+  UploadSearchOptions,
+  UploadSearchResult,
+  UploadStore,
+  UploadStoreConfig,
+  UploadStoreData,
+} from "./types.js";
+
+/**
+ * Creates a persistent upload store with lazy embedding and similarity search.
+ *
+ * Combines upload management, chunking, embedding, and vector search into
+ * a single factory. Uploads are chunked on ingest; embeddings are generated
+ * lazily on the first search call to avoid blocking uploads on AI API calls.
+ *
+ * Persistence is JSON-file-based (via the platform fs adapter), making it
+ * suitable for prototypes, templates, and small-to-medium knowledge bases.
+ *
+ * @example
+ * ```ts
+ * import { uploadStore } from "veryfront/embedding";
+ *
+ * const store = uploadStore({
+ *   model: "openai/text-embedding-3-small",
+ *   storagePath: "data/index.json",
+ *   contentDir: "content",
+ * });
+ *
+ * await store.ingest("My Doc", text, { source: "upload:file.pdf", type: "pdf" });
+ * const results = await store.search("query", { topK: 5, threshold: 0.7 });
+ * ```
+ */
+export function uploadStore(config: UploadStoreConfig): UploadStore {
+  const storagePath = config.storagePath ?? "data/index.json";
+  const contentDir = config.contentDir;
+  const contentExtensions = new Set(config.contentExtensions ?? [".md", ".mdx", ".txt"]);
+  const chunkOptions = config.chunkOptions;
+
+  const MAX_TEXT_LENGTH = 5 * 1024 * 1024; // 5 MB text limit per document
+
+  // Serialize all load→modify→save operations to prevent concurrent overwrites.
+  let mutex: Promise<void> = Promise.resolve();
+  function withLock<T>(fn: () => Promise<T>): Promise<T> {
+    const result = mutex.then(fn);
+    mutex = result.then(
+      () => {},
+      (err) => {
+        console.error("[upload-store] Lock operation failed:", err);
+      },
+    );
+    return result;
+  }
+
+  const embedder = embedding({
+    model: config.model,
+    documentPrefix: config.documentPrefix,
+    queryPrefix: config.queryPrefix,
+    batchSize: config.batchSize,
+  });
+
+  async function load(): Promise<UploadStoreData> {
+    try {
+      const data = await readTextFile(storagePath);
+      const parsed = JSON.parse(data);
+      if (!parsed || !Array.isArray(parsed.uploads) || !Array.isArray(parsed.chunks)) {
+        console.warn("[upload-store] Corrupted store file, resetting:", storagePath);
+        return { uploads: [], chunks: [] };
+      }
+      return parsed as UploadStoreData;
+    } catch (err) {
+      // File not found is expected on first run; anything else is worth logging
+      if (isNotFoundError(err)) {
+        return { uploads: [], chunks: [] };
+      }
+      console.warn("[upload-store] Failed to load store, resetting:", err);
+      return { uploads: [], chunks: [] };
+    }
+  }
+
+  async function save(data: UploadStoreData): Promise<void> {
+    const dir = dirname(storagePath);
+    if (dir && dir !== ".") {
+      await mkdir(dir, { recursive: true });
+    }
+    const payload = JSON.stringify(data, null, 2);
+    // Atomic write: write to temp file then rename to prevent corruption on crash
+    const tmpPath = storagePath + ".tmp";
+    await writeTextFile(tmpPath, payload);
+    try {
+      if (typeof dntShim.Deno !== "undefined") {
+        await dntShim.Deno.rename(tmpPath, storagePath);
+      } else {
+        const fs = await import("node:fs/promises");
+        await fs.rename(tmpPath, storagePath);
+      }
+    } catch {
+      // Fallback for environments where rename isn't available
+      await writeTextFile(storagePath, payload);
+    }
+  }
+
+  async function ensureEmbeddings(data: UploadStoreData): Promise<boolean> {
+    const unembedded = data.chunks.filter((c) => c.embedding.length === 0);
+    if (unembedded.length === 0) return false;
+
+    const embeddings = await embedder.embedMany(unembedded.map((c) => c.text));
+    for (let i = 0; i < unembedded.length; i++) {
+      unembedded[i]!.embedding = embeddings[i]!;
+    }
+    return true;
+  }
+
+  async function listContentFiles(dir: string): Promise<string[]> {
+    const files: string[] = [];
+    try {
+      for await (const entry of readDir(dir)) {
+        const fullPath = join(dir, entry.name);
+        if (entry.isDirectory) {
+          files.push(...(await listContentFiles(fullPath)));
+        } else if (entry.isFile && contentExtensions.has(extname(entry.name))) {
+          files.push(fullPath);
+        }
+      }
+    } catch {
+      // directory may not exist yet
+    }
+    return files;
+  }
+
+  return {
+    async ingest(
+      title: string,
+      text: string,
+      meta?: { source?: string; type?: string },
+    ): Promise<string> {
+      return withLock(async () => {
+        const data = await load();
+        const uploadId = dntShim.crypto.randomUUID();
+
+        if (text.length > MAX_TEXT_LENGTH) {
+          throw new Error(
+            `Upload text exceeds ${MAX_TEXT_LENGTH / 1024 / 1024} MB limit`,
+          );
+        }
+
+        const chunks = await chunk(text, chunkOptions);
+        if (chunks.length === 0) {
+          throw new Error("Upload contains no extractable text");
+        }
+
+        const doc: UploadMeta = {
+          id: uploadId,
+          title,
+          source: meta?.source ?? "",
+          type: meta?.type ?? "",
+          createdAt: Date.now(),
+        };
+
+        const chunkRecords: StoredChunk[] = chunks.map((chunkText, i) => ({
+          id: dntShim.crypto.randomUUID(),
+          uploadId,
+          text: chunkText,
+          embedding: [], // filled lazily on first search
+          index: i,
+        }));
+
+        data.uploads.push(doc);
+        data.chunks.push(...chunkRecords);
+        await save(data);
+
+        return uploadId;
+      });
+    },
+
+    async search(
+      query: string,
+      options?: UploadSearchOptions,
+    ): Promise<UploadSearchResult[]> {
+      return withLock(async () => {
+        const data = await load();
+        if (data.chunks.length === 0) return [];
+
+        const updated = await ensureEmbeddings(data);
+        if (updated) await save(data);
+
+        const queryEmbedding = await embedder.embed(query);
+        const topK = options?.topK ?? 5;
+        const threshold = options?.threshold;
+
+        const docMap = new Map(data.uploads.map((d) => [d.id, d]));
+
+        const scored = data.chunks.map((c) => {
+          const doc = docMap.get(c.uploadId);
+          return {
+            text: c.text,
+            score: cosineSimilarity(queryEmbedding, c.embedding),
+            uploadId: c.uploadId,
+            title: doc?.title ?? "Unknown",
+            source: doc?.source ?? "",
+            type: doc?.type ?? "",
+          };
+        });
+
+        scored.sort((a, b) => b.score - a.score);
+
+        let results = scored.slice(0, topK);
+        if (threshold !== undefined) {
+          results = results.filter((r) => r.score >= threshold);
+        }
+        return results;
+      });
+    },
+
+    async listUploads(): Promise<UploadMeta[]> {
+      return withLock(async () => {
+        const data = await load();
+        return data.uploads;
+      });
+    },
+
+    async removeUpload(id: string): Promise<void> {
+      return withLock(async () => {
+        const data = await load();
+        data.uploads = data.uploads.filter((d) => d.id !== id);
+        data.chunks = data.chunks.filter((c) => c.uploadId !== id);
+        await save(data);
+      });
+    },
+
+    async indexContentDir(): Promise<void> {
+      if (!contentDir) return;
+
+      return withLock(async () => {
+        const data = await load();
+        const indexedSources = new Set(data.uploads.map((d) => d.source));
+
+        const files = await listContentFiles(contentDir);
+        const newFiles = files.filter((f) => !indexedSources.has(f));
+        if (newFiles.length === 0) return;
+
+        for (const file of newFiles) {
+          const content = await readTextFile(file);
+          if (!content?.trim()) continue;
+          if (content.length > MAX_TEXT_LENGTH) {
+            console.warn(
+              `[upload-store] Skipping ${file}: exceeds ${
+                MAX_TEXT_LENGTH / 1024 / 1024
+              } MB text limit`,
+            );
+            continue;
+          }
+
+          const title = file.startsWith(contentDir + "/")
+            ? file.slice(contentDir.length + 1).replace(/\.[^.]+$/, "")
+            : file.replace(/\.[^.]+$/, "");
+          const uploadId = dntShim.crypto.randomUUID();
+          const type = extname(file).slice(1);
+
+          const chunks = await chunk(content, chunkOptions);
+          if (chunks.length === 0) continue;
+
+          data.uploads.push({
+            id: uploadId,
+            title,
+            source: file,
+            type,
+            createdAt: Date.now(),
+          });
+
+          data.chunks.push(
+            ...chunks.map((chunkText, i) => ({
+              id: dntShim.crypto.randomUUID(),
+              uploadId,
+              text: chunkText,
+              embedding: [] as number[],
+              index: i,
+            })),
+          );
+        }
+
+        await save(data);
+      });
+    },
+  };
+}

package/src/src/embedding/vector-store.ts

@@ -0,0 +1,282 @@
+import { cosineSimilarity } from "ai";
+import type {
+  Embedding,
+  SearchOptions,
+  SearchResult,
+  VectorStore,
+  VectorStoreConfig,
+} from "./types.js";
+
+interface VectorEntry {
+  text: string;
+  vector: number[];
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Creates an in-memory vector store with integrated embedding and similarity search.
+ *
+ * Supports three search strategies:
+ * - `"dense"` (default) — cosine similarity ranking
+ * - `"mmr"` — Maximum Marginal Relevance for diverse results
+ * - `"hybrid"` — BM25 + dense fusion via Reciprocal Rank Fusion
+ *
+ * @example
+ * ```ts
+ * const store = vectorStore({ embedder });
+ * await store.add(["chunk 1", "chunk 2"]);
+ *
+ * // Basic search
+ * const results = await store.search("query", { topK: 5 });
+ *
+ * // With threshold and metadata filtering
+ * const filtered = await store.search("query", {
+ *   topK: 10,
+ *   threshold: 0.7,
+ *   filter: { source: "upload" },
+ * });
+ *
+ * // MMR for diverse results
+ * const diverse = await store.search("query", { strategy: "mmr", lambda: 0.5 });
+ *
+ * // Hybrid search (BM25 + dense, fused with RRF)
+ * const hybrid = await store.search("query", { strategy: "hybrid" });
+ * ```
+ */
+export function vectorStore(config: VectorStoreConfig): VectorStore {
+  const embedder: Embedding = config.embedder;
+  const entries: VectorEntry[] = [];
+
+  return {
+    async add(texts: string[], metadata?: Record<string, unknown>[]): Promise<void> {
+      const embeddings = await embedder.embedMany(texts);
+      for (let i = 0; i < texts.length; i++) {
+        entries.push({
+          text: texts[i]!,
+          vector: embeddings[i]!,
+          metadata: metadata?.[i],
+        });
+      }
+    },
+
+    async search(query: string, options?: SearchOptions): Promise<SearchResult[]> {
+      if (entries.length === 0) return [];
+
+      const topK = options?.topK ?? 5;
+      const threshold = options?.threshold;
+      const filter = options?.filter;
+      const strategy = options?.strategy ?? "dense";
+
+      // Filter entries by metadata
+      const candidates = filter
+        ? entries.filter((e) => matchesFilter(e.metadata, filter))
+        : entries;
+      if (candidates.length === 0) return [];
+
+      const queryEmbedding = await embedder.embed(query);
+
+      let results: SearchResult[];
+
+      switch (strategy) {
+        case "mmr":
+          results = searchMMR(queryEmbedding, candidates, topK, options?.lambda ?? 0.5);
+          break;
+        case "hybrid":
+          results = searchHybrid(query, queryEmbedding, candidates, topK);
+          break;
+        default:
+          results = searchDense(queryEmbedding, candidates, topK);
+      }
+
+      // Apply score threshold
+      if (threshold !== undefined) {
+        results = results.filter((r) => r.score >= threshold);
+      }
+
+      return results;
+    },
+
+    clear(): void {
+      entries.length = 0;
+    },
+
+    get size(): number {
+      return entries.length;
+    },
+  };
+}
+
+// --- Dense search (cosine similarity) ---
+
+function searchDense(
+  queryVector: number[],
+  candidates: VectorEntry[],
+  topK: number,
+): SearchResult[] {
+  const scored = candidates.map((e) => ({
+    text: e.text,
+    score: cosineSimilarity(queryVector, e.vector),
+    metadata: e.metadata,
+  }));
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, topK);
+}
+
+// --- MMR (Maximum Marginal Relevance) ---
+
+function searchMMR(
+  queryVector: number[],
+  candidates: VectorEntry[],
+  topK: number,
+  lambda: number,
+): SearchResult[] {
+  // Score all candidates against query
+  const scored = candidates.map((e, i) => ({
+    index: i,
+    text: e.text,
+    vector: e.vector,
+    relevance: cosineSimilarity(queryVector, e.vector),
+    metadata: e.metadata,
+  }));
+
+  const selected: SearchResult[] = [];
+  const selectedVectors: number[][] = [];
+  const remaining = new Set(scored.map((_, i) => i));
+
+  for (let k = 0; k < Math.min(topK, scored.length); k++) {
+    let bestIdx = -1;
+    let bestScore = -Infinity;
+
+    for (const i of remaining) {
+      const candidate = scored[i]!;
+      const relevance = candidate.relevance;
+
+      // Max similarity to any already-selected document
+      let maxSim = 0;
+      for (const sv of selectedVectors) {
+        const sim = cosineSimilarity(candidate.vector, sv);
+        if (sim > maxSim) maxSim = sim;
+      }
+
+      const mmrScore = lambda * relevance - (1 - lambda) * maxSim;
+      if (mmrScore > bestScore) {
+        bestScore = mmrScore;
+        bestIdx = i;
+      }
+    }
+
+    if (bestIdx === -1) break;
+
+    const winner = scored[bestIdx]!;
+    remaining.delete(bestIdx);
+    selectedVectors.push(winner.vector);
+    selected.push({
+      text: winner.text,
+      score: winner.relevance, // return the actual relevance score, not MMR score
+      metadata: winner.metadata,
+    });
+  }
+
+  return selected;
+}
+
+// --- Hybrid search (BM25 + dense, fused with RRF) ---
+
+const RRF_K = 60; // standard RRF constant
+const BM25_K1 = 1.2;
+const BM25_B = 0.75;
+
+function searchHybrid(
+  query: string,
+  queryVector: number[],
+  candidates: VectorEntry[],
+  topK: number,
+): SearchResult[] {
+  // Dense ranking
+  const denseScores = candidates.map((e, i) => ({
+    index: i,
+    score: cosineSimilarity(queryVector, e.vector),
+  }));
+  denseScores.sort((a, b) => b.score - a.score);
+
+  // BM25 ranking
+  const bm25Scores = computeBM25(query, candidates);
+  bm25Scores.sort((a, b) => b.score - a.score);
+
+  // Build rank maps (index → rank, 0-based)
+  const denseRank = new Map<number, number>();
+  for (let r = 0; r < denseScores.length; r++) {
+    denseRank.set(denseScores[r]!.index, r);
+  }
+
+  const bm25Rank = new Map<number, number>();
+  for (let r = 0; r < bm25Scores.length; r++) {
+    bm25Rank.set(bm25Scores[r]!.index, r);
+  }
+
+  // Reciprocal Rank Fusion
+  const fused = candidates.map((e, i) => {
+    const dr = denseRank.get(i) ?? candidates.length;
+    const br = bm25Rank.get(i) ?? candidates.length;
+    const rrfScore = 1 / (RRF_K + dr) + 1 / (RRF_K + br);
+    return { text: e.text, score: rrfScore, metadata: e.metadata };
+  });
+
+  fused.sort((a, b) => b.score - a.score);
+  return fused.slice(0, topK);
+}
+
+function computeBM25(
+  query: string,
+  candidates: VectorEntry[],
+): { index: number; score: number }[] {
+  const queryTerms = tokenize(query);
+  if (queryTerms.length === 0) {
+    return candidates.map((_, i) => ({ index: i, score: 0 }));
+  }
+
+  const N = candidates.length;
+  const docTokens = candidates.map((e) => tokenize(e.text));
+  const totalTokens = docTokens.reduce((sum, t) => sum + t.length, 0);
+  const avgDl = totalTokens === 0 ? 1 : totalTokens / N;
+
+  // Document frequency per query term
+  const df = new Map<string, number>();
+  for (const term of queryTerms) {
+    if (df.has(term)) continue;
+    let count = 0;
+    for (const tokens of docTokens) {
+      if (tokens.includes(term)) count++;
+    }
+    df.set(term, count);
+  }
+
+  return candidates.map((_, i) => {
+    const tokens = docTokens[i]!;
+    const dl = tokens.length;
+    let score = 0;
+
+    for (const term of queryTerms) {
+      const n = df.get(term) ?? 0;
+      const idf = Math.log((N - n + 0.5) / (n + 0.5) + 1);
+      const tf = tokens.filter((t) => t === term).length;
+      score += idf * ((tf * (BM25_K1 + 1)) / (tf + BM25_K1 * (1 - BM25_B + BM25_B * dl / avgDl)));
+    }
+
+    return { index: i, score };
+  });
+}
+
+function tokenize(text: string): string[] {
+  return text.toLowerCase().split(/\W+/).filter(Boolean);
+}
+
+// --- Metadata filtering ---
+
+function matchesFilter(
+  metadata: Record<string, unknown> | undefined,
+  filter: Record<string, unknown>,
+): boolean {
+  if (!metadata) return false;
+  return Object.entries(filter).every(([key, value]) => metadata[key] === value);
+}

package/src/src/errors/error-registry.ts

@@ -1,4 +1,4 @@
-import { defineError } from "./types.js";
+import { defineError, type ErrorCategory } from "./types.js";
 
 // =============================================================================
 // CONFIG - Configuration & environment errors
@@ -789,7 +789,7 @@ export function getErrorBySlug(slug: ErrorSlug) {
 /**
  * Get all errors in a category
  */
-export function getErrorsByCategory(category: string) {
+export function getErrorsByCategory(category: ErrorCategory) {
   return Object.values(ERROR_REGISTRY).filter((error) => error.category === category);
 }
 

package/src/src/errors/http-error.ts

@@ -9,9 +9,13 @@
 import * as dntShim from "../../_dnt.shims.js";
 
 
-import { type ErrorCategory, type RFC9457Response, VeryfrontError } from "./types.js";
+import {
+  type ErrorCategory,
+  type RegisteredError,
+  type RFC9457Response,
+  VeryfrontError,
+} from "./types.js";
 import { getErrorMessage } from "./veryfront-error.js";
-import type { RegisteredError } from "./types.js";
 
 /**
  * Content-Type header for RFC 9457 responses
@@ -83,7 +87,7 @@ export function createProblemResponse(params: {
  * Check if an error is a VeryfrontError with slug-based identity
  */
 export function isVeryfrontError(error: unknown): error is VeryfrontError {
-  return error instanceof VeryfrontError && typeof (error as VeryfrontError).slug === "string";
+  return error instanceof VeryfrontError;
 }
 
 /**

package/src/src/errors/middleware/cli-error-boundary.ts

@@ -33,7 +33,20 @@ interface ColorFormatter {
  * Check if output is a TTY (supports colors)
  */
 function isTTY(): boolean {
-  return dntShim.Deno.stdout.isTerminal?.() ?? false;
+  const deno = dntShim.dntGlobalThis as {
+    Deno?: {
+      stdout?: { isTerminal?: () => boolean };
+    };
+    process?: {
+      stdout?: { isTTY?: boolean };
+    };
+  };
+
+  if (typeof deno.Deno?.stdout?.isTerminal === "function") {
+    return deno.Deno.stdout.isTerminal();
+  }
+
+  return deno.process?.stdout?.isTTY ?? false;
 }
 
 /**
@@ -210,5 +223,18 @@ export function cliErrorBoundarySync(
  * Exit the process with a status code
  */
 function exit(code: number): never {
-  dntShim.Deno.exit(code);
+  const runtime = dntShim.dntGlobalThis as {
+    Deno?: { exit?: (code: number) => never };
+    process?: { exit?: (code: number) => never };
+  };
+
+  if (typeof runtime.Deno?.exit === "function") {
+    runtime.Deno.exit(code);
+  }
+
+  if (typeof runtime.process?.exit === "function") {
+    runtime.process.exit(code);
+  }
+
+  throw new Error(`Failed to exit with code ${code}`);
 }

package/src/src/errors/middleware/wrap-unknown.ts

@@ -55,16 +55,6 @@ export function wrapUnknownError(
   });
 }
 
-/**
- * Check if an error is already a VeryfrontError
- *
- * @param error - Any error value
- * @returns True if the error is a VeryfrontError instance
- */
-export function isVeryfrontError(error: unknown): error is VeryfrontError {
-  return error instanceof VeryfrontError;
-}
-
 /**
  * Wrap an error with additional context
  *

package/src/src/errors/veryfront-error.ts

@@ -132,15 +132,6 @@ export function fromError(error: unknown): VeryfrontErrorData | null {
   return context as VeryfrontErrorData;
 }
 
-export function logError(
-  error: VeryfrontErrorData,
-  logger?: { error: (msg: string, ...args: unknown[]) => void },
-): void {
-  const log = logger ?? console;
-  const context = "context" in error ? error.context : undefined;
-  log.error(`[${error.type}] ${error.message}`, context ?? {});
-}
-
 /**
  * Extract error message from any error type
  */