veryfront 0.1.49 → 0.1.51

package/src/src/embedding/veryfront-cloud/rag-store.ts

@@ -0,0 +1,599 @@
+import * as dntShim from "../../../_dnt.shims.js";
+import { readDir, readTextFile } from "../../platform/compat/fs.js";
+import { extname, join } from "../../platform/compat/path/basic-operations.js";
+import { getCurrentRequestContext } from "../../platform/adapters/fs/veryfront/multi-project-adapter.js";
+import { INVALID_ARGUMENT } from "../../errors/index.js";
+import { serverLogger } from "../../utils/index.js";
+import {
+  createVeryfrontCloudFetch,
+  requireVeryfrontCloudBootstrap,
+} from "../../provider/veryfront-cloud/shared.js";
+import { chunk } from "../chunk.js";
+import { embedding } from "../embedding.js";
+import type {
+  RagDocumentMeta,
+  RagSearchOptions,
+  RagSearchResult,
+  RagStore,
+  RagStoreConfig,
+} from "../types.js";
+
+const DEFAULT_TOP_K = 5;
+const MAX_TEXT_LENGTH = 5 * 1024 * 1024; // 5 MB text limit per document
+const MAX_API_CHUNK_BATCH = 500;
+const MAX_API_EMBEDDING_BATCH = 100;
+const MAX_SEARCH_LIMIT = 100;
+const SEARCH_OVERSCAN = 25;
+const DOCUMENTS_DIR = ".veryfront/rag/documents";
+
+type SupportedDimension = 768 | 1024 | 1536 | 3072 | 4096;
+
+interface CloudChunkListResponse {
+  data: Array<{
+    id: string;
+    index: number;
+    content: string;
+    metadata?: Record<string, unknown>;
+  }>;
+  page_info?: {
+    next?: string | null;
+  };
+}
+
+interface CloudUpsertChunksResponse {
+  chunks: Array<{ id: string; index: number }>;
+}
+
+interface CloudSearchResponse {
+  data: Array<{
+    chunk: {
+      file_path: string;
+      content: string;
+      metadata?: Record<string, unknown>;
+    };
+    score: number;
+  }>;
+}
+
+interface CloudRagDocumentResponse {
+  id: string;
+  title: string;
+  source: string;
+  type: string;
+  metadata: Record<string, unknown>;
+  created_at: string;
+  updated_at: string;
+}
+
+interface CloudListRagDocumentsResponse {
+  documents: CloudRagDocumentResponse[];
+}
+
+interface CloudUpsertRagDocumentResponse {
+  document: CloudRagDocumentResponse;
+}
+
+interface CloudStoreContext {
+  apiBaseUrl: string;
+  fetch: typeof dntShim.fetch;
+  projectSlug: string;
+  branch: string;
+}
+
+interface ChunkMutationInput {
+  chunk_index: number;
+  content: string;
+  start_offset: number;
+  end_offset: number;
+  token_count: number;
+  metadata?: Record<string, unknown>;
+}
+
+type ResolvedCloudRagStoreConfig = RagStoreConfig & { model: string };
+
+function buildUrl(base: string, path: string): string {
+  return `${base.replace(/\/+$/, "")}/${path.replace(/^\/+/, "")}`;
+}
+
+function estimateTokenCount(text: string): number {
+  return Math.max(1, Math.ceil(text.length / 4));
+}
+
+function toSupportedDimension(length: number): SupportedDimension {
+  switch (length) {
+    case 768:
+    case 1024:
+    case 1536:
+    case 3072:
+    case 4096:
+      return length;
+    default:
+      throw INVALID_ARGUMENT.create({
+        detail:
+          `Unsupported embedding dimension ${length}. Expected one of 768, 1024, 1536, 3072, 4096.`,
+      });
+  }
+}
+
+function normalizeEmbeddingModelDescriptor(model: string, dimension: SupportedDimension): {
+  name: string;
+  provider: string;
+  dimension: SupportedDimension;
+} {
+  const normalized = model.startsWith("veryfront-cloud/")
+    ? model.slice("veryfront-cloud/".length)
+    : model;
+  const slashIndex = normalized.indexOf("/");
+
+  if (slashIndex === -1) {
+    return {
+      name: normalized,
+      provider: "unknown",
+      dimension,
+    };
+  }
+
+  return {
+    provider: normalized.slice(0, slashIndex),
+    name: normalized.slice(slashIndex + 1),
+    dimension,
+  };
+}
+
+function buildDocumentFilePath(documentId: string, type?: string): string {
+  const extension = type?.trim().toLowerCase().replace(/[^a-z0-9]+/g, "");
+  return `${DOCUMENTS_DIR}/${documentId}.${extension || "txt"}`;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+function toStringValue(value: unknown, fallback = ""): string {
+  return typeof value === "string" ? value : fallback;
+}
+
+function buildDocumentChunks(
+  sourceText: string,
+  chunkTexts: string[],
+  metadata: Record<string, unknown>,
+): ChunkMutationInput[] {
+  let searchStart = 0;
+
+  return chunkTexts.map((content, index) => {
+    const foundAt = sourceText.indexOf(content, searchStart);
+    const startOffset = foundAt >= 0 ? foundAt : searchStart;
+    const endOffset = startOffset + content.length;
+    searchStart = Math.max(startOffset + 1, endOffset);
+
+    return {
+      chunk_index: index,
+      content,
+      start_offset: startOffset,
+      end_offset: endOffset,
+      token_count: estimateTokenCount(content),
+      metadata,
+    };
+  });
+}
+
+async function requestJson<T>(
+  context: CloudStoreContext,
+  path: string,
+  init?: dntShim.RequestInit,
+  options?: { allowNotFound?: boolean },
+): Promise<T | null> {
+  const request = new dntShim.Request(buildUrl(context.apiBaseUrl, path), init);
+  const headers = new dntShim.Headers(request.headers);
+
+  if (!headers.has("Content-Type") && request.method !== "GET" && request.method !== "HEAD") {
+    headers.set("Content-Type", "application/json");
+  }
+
+  const response = await context.fetch(new dntShim.Request(request, { headers }));
+  if (options?.allowNotFound && response.status === 404) {
+    return null;
+  }
+
+  if (!response.ok) {
+    const body = await response.text().catch(() => "");
+    throw INVALID_ARGUMENT.create({
+      detail: `Veryfront Cloud RAG request failed (${response.status} ${response.statusText}): ${
+        body || path
+      }`,
+    });
+  }
+
+  if (response.status === 204) {
+    return null;
+  }
+
+  return await response.json() as T;
+}
+
+function getCloudStoreContext(config: RagStoreConfig): CloudStoreContext {
+  const bootstrap = requireVeryfrontCloudBootstrap();
+  if (!bootstrap.projectSlug) {
+    throw INVALID_ARGUMENT.create({
+      detail:
+        "VERYFRONT_PROJECT_SLUG not set. Set the environment variable or runtime projectSlug before using the veryfront-cloud RAG store.",
+    });
+  }
+
+  return {
+    apiBaseUrl: bootstrap.apiBaseUrl,
+    fetch: createVeryfrontCloudFetch(bootstrap.apiToken),
+    projectSlug: bootstrap.projectSlug,
+    branch: config.branch ?? getCurrentRequestContext()?.branch ?? "main",
+  };
+}
+
+function getFileChunksPath(
+  context: CloudStoreContext,
+  filePath: string,
+): string {
+  return `/projects/${encodeURIComponent(context.projectSlug)}/branches/${
+    encodeURIComponent(context.branch)
+  }/files/${encodeURIComponent(filePath)}/chunks`;
+}
+
+function getSearchPath(context: CloudStoreContext): string {
+  return `/projects/${encodeURIComponent(context.projectSlug)}/branches/${
+    encodeURIComponent(context.branch)
+  }/search`;
+}
+
+function getEmbeddingsPath(context: CloudStoreContext): string {
+  return `/projects/${encodeURIComponent(context.projectSlug)}/embeddings`;
+}
+
+function getRagDocumentsPath(context: CloudStoreContext): string {
+  return `/projects/${encodeURIComponent(context.projectSlug)}/rag/documents`;
+}
+
+async function deleteFileChunks(context: CloudStoreContext, filePath: string): Promise<void> {
+  await requestJson(
+    context,
+    getFileChunksPath(context, filePath),
+    { method: "DELETE" },
+    { allowNotFound: true },
+  );
+}
+
+async function upsertFileChunks(
+  context: CloudStoreContext,
+  filePath: string,
+  chunks: ChunkMutationInput[],
+): Promise<Array<{ id: string; index: number }>> {
+  if (chunks.length === 0) {
+    return [];
+  }
+
+  const results: Array<{ id: string; index: number }> = [];
+  for (let i = 0; i < chunks.length; i += MAX_API_CHUNK_BATCH) {
+    const batch = chunks.slice(i, i + MAX_API_CHUNK_BATCH);
+    const response = await requestJson<CloudUpsertChunksResponse>(
+      context,
+      getFileChunksPath(context, filePath),
+      {
+        method: "POST",
+        body: JSON.stringify({ chunks: batch }),
+      },
+    );
+
+    if (response) {
+      results.push(...response.chunks);
+    }
+  }
+
+  results.sort((a, b) => a.index - b.index);
+  return results;
+}
+
+async function upsertEmbeddings(
+  context: CloudStoreContext,
+  chunkIds: string[],
+  vectors: number[][],
+  model: { name: string; provider: string; dimension: SupportedDimension },
+): Promise<void> {
+  for (let i = 0; i < chunkIds.length; i += MAX_API_EMBEDDING_BATCH) {
+    const batchChunkIds = chunkIds.slice(i, i + MAX_API_EMBEDDING_BATCH);
+    const batchVectors = vectors.slice(i, i + MAX_API_EMBEDDING_BATCH);
+
+    await requestJson(
+      context,
+      getEmbeddingsPath(context),
+      {
+        method: "POST",
+        body: JSON.stringify({
+          chunk_ids: batchChunkIds,
+          vectors: batchVectors,
+          model,
+        }),
+      },
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Server-side RAG document management
+// ---------------------------------------------------------------------------
+
+async function listRagDocuments(
+  context: CloudStoreContext,
+): Promise<RagDocumentMeta[]> {
+  const response = await requestJson<CloudListRagDocumentsResponse>(
+    context,
+    getRagDocumentsPath(context),
+  );
+
+  return (response?.documents ?? []).map((doc) => ({
+    id: doc.id,
+    title: doc.title,
+    source: doc.source,
+    type: doc.type,
+    createdAt: new Date(doc.created_at).getTime(),
+  }));
+}
+
+async function upsertRagDocument(
+  context: CloudStoreContext,
+  document: {
+    id: string;
+    title: string;
+    source?: string;
+    type?: string;
+    metadata?: Record<string, unknown>;
+  },
+): Promise<void> {
+  await requestJson<CloudUpsertRagDocumentResponse>(
+    context,
+    getRagDocumentsPath(context),
+    {
+      method: "POST",
+      body: JSON.stringify({
+        id: document.id,
+        title: document.title,
+        source: document.source ?? "",
+        type: document.type ?? "",
+        metadata: document.metadata,
+      }),
+    },
+  );
+}
+
+async function deleteRagDocument(
+  context: CloudStoreContext,
+  documentId: string,
+): Promise<void> {
+  await requestJson(
+    context,
+    `${getRagDocumentsPath(context)}/${encodeURIComponent(documentId)}`,
+    { method: "DELETE" },
+    { allowNotFound: true },
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Document ingestion (chunks + embeddings + server-side record)
+// ---------------------------------------------------------------------------
+
+async function ingestDocument(
+  context: CloudStoreContext,
+  config: ResolvedCloudRagStoreConfig,
+  title: string,
+  text: string,
+  meta?: { source?: string; type?: string },
+): Promise<string> {
+  if (text.length > MAX_TEXT_LENGTH) {
+    throw INVALID_ARGUMENT.create({
+      detail: `Upload text exceeds ${MAX_TEXT_LENGTH / 1024 / 1024} MB limit`,
+    });
+  }
+
+  const chunkTexts = await chunk(text, config.chunkOptions);
+  if (chunkTexts.length === 0) {
+    throw INVALID_ARGUMENT.create({ detail: "Upload contains no extractable text" });
+  }
+
+  const documentId = dntShim.crypto.randomUUID();
+  const filePath = buildDocumentFilePath(documentId, meta?.type);
+  const embedder = createEmbedder(config);
+  const vectors = await embedder.embedMany(chunkTexts);
+  const dimension = toSupportedDimension(vectors[0]?.length ?? 0);
+  const chunkInputs = buildDocumentChunks(text, chunkTexts, {
+    kind: "rag-document",
+    document_id: documentId,
+    title,
+    source: meta?.source ?? "",
+    type: meta?.type ?? "",
+  });
+
+  try {
+    const createdChunks = await upsertFileChunks(context, filePath, chunkInputs);
+    const chunkIds = createdChunks.map((entry) => entry.id);
+
+    if (chunkIds.length !== vectors.length) {
+      throw INVALID_ARGUMENT.create({
+        detail:
+          `Expected ${vectors.length} chunk IDs from Veryfront Cloud, received ${chunkIds.length}.`,
+      });
+    }
+
+    await upsertEmbeddings(
+      context,
+      chunkIds,
+      vectors,
+      normalizeEmbeddingModelDescriptor(config.model, dimension),
+    );
+  } catch (error) {
+    await deleteFileChunks(context, filePath).catch(() => undefined);
+    throw error;
+  }
+
+  // Record document in server-side store (atomic, no client-side manifest needed)
+  await upsertRagDocument(context, {
+    id: documentId,
+    title,
+    source: meta?.source ?? "",
+    type: meta?.type ?? "",
+    metadata: { filePath },
+  });
+
+  return documentId;
+}
+
+function createEmbedder(config: ResolvedCloudRagStoreConfig) {
+  return embedding({
+    model: config.model,
+    documentPrefix: config.documentPrefix,
+    queryPrefix: config.queryPrefix,
+    batchSize: config.batchSize,
+  });
+}
+
+async function listContentFiles(
+  contentDir: string,
+  contentExtensions: Set<string>,
+): Promise<string[]> {
+  const files: string[] = [];
+
+  try {
+    for await (const entry of readDir(contentDir)) {
+      const fullPath = join(contentDir, entry.name);
+      if (entry.isDirectory) {
+        files.push(...(await listContentFiles(fullPath, contentExtensions)));
+      } else if (entry.isFile && contentExtensions.has(extname(entry.name))) {
+        files.push(fullPath);
+      }
+    }
+  } catch (_) {
+    // expected: directory may not exist yet
+  }
+
+  return files;
+}
+
+export function createVeryfrontCloudRagStore(config: ResolvedCloudRagStoreConfig): RagStore {
+  const contentDir = config.contentDir;
+  const contentExtensions = new Set(config.contentExtensions ?? [".md", ".mdx", ".txt"]);
+
+  return {
+    async ingest(
+      title: string,
+      text: string,
+      meta?: { source?: string; type?: string },
+    ): Promise<string> {
+      const context = getCloudStoreContext(config);
+      return ingestDocument(context, config, title, text, meta);
+    },
+
+    async search(
+      query: string,
+      options?: RagSearchOptions,
+    ): Promise<RagSearchResult[]> {
+      const context = getCloudStoreContext(config);
+      const queryEmbedder = createEmbedder(config);
+      const vector = await queryEmbedder.embed(query);
+      const topK = options?.topK ?? DEFAULT_TOP_K;
+      const limit = Math.min(MAX_SEARCH_LIMIT, topK + SEARCH_OVERSCAN);
+      const dimension = toSupportedDimension(vector.length);
+      const response = await requestJson<CloudSearchResponse>(
+        context,
+        getSearchPath(context),
+        {
+          method: "POST",
+          body: JSON.stringify({
+            vector,
+            dimension,
+            limit,
+            threshold: options?.threshold ?? 0,
+          }),
+        },
+      );
+
+      const results = (response?.data ?? [])
+        .filter((result) =>
+          result.chunk.metadata?.kind !== "rag-manifest" &&
+          result.chunk.metadata?.kind !== "rag-manifest-padding"
+        )
+        .map((result) => {
+          const metadata = isRecord(result.chunk.metadata) ? result.chunk.metadata : {};
+
+          return {
+            text: result.chunk.content,
+            score: result.score,
+            documentId: toStringValue(metadata.document_id, result.chunk.file_path),
+            title: toStringValue(metadata.title, "Unknown"),
+            source: toStringValue(metadata.source, result.chunk.file_path),
+            type: toStringValue(metadata.type, ""),
+          };
+        });
+
+      return results.slice(0, topK);
+    },
+
+    async listDocuments(): Promise<RagDocumentMeta[]> {
+      const context = getCloudStoreContext(config);
+      return listRagDocuments(context);
+    },
+
+    async removeDocument(id: string): Promise<void> {
+      const context = getCloudStoreContext(config);
+
+      // Fetch document metadata to find its filePath for chunk cleanup
+      const documents = await listRagDocuments(context);
+      const target = documents.find((doc) => doc.id === id);
+
+      // Delete server-side document record first (authoritative)
+      await deleteRagDocument(context, id);
+
+      // Best-effort cleanup of file chunks
+      if (target) {
+        const filePath = buildDocumentFilePath(id, target.type);
+        try {
+          await deleteFileChunks(context, filePath);
+        } catch (error) {
+          serverLogger.warn("[rag-store/cloud] Failed to clean up file chunks for document", {
+            id,
+            filePath,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+    },
+
+    async indexContentDir(): Promise<void> {
+      if (!contentDir) return;
+
+      const context = getCloudStoreContext(config);
+      const existingDocuments = await listRagDocuments(context);
+      const indexedSources = new Set(existingDocuments.map((doc) => doc.source));
+      const files = await listContentFiles(contentDir, contentExtensions);
+      const newFiles = files.filter((file) => !indexedSources.has(file));
+
+      for (const file of newFiles) {
+        const content = await readTextFile(file);
+        if (!content?.trim()) continue;
+        if (content.length > MAX_TEXT_LENGTH) {
+          serverLogger.warn(
+            `[rag-store/cloud] Skipping ${file}: exceeds ${
+              MAX_TEXT_LENGTH / 1024 / 1024
+            } MB text limit`,
+          );
+          continue;
+        }
+
+        const title = file.startsWith(contentDir + "/")
+          ? file.slice(contentDir.length + 1).replace(/\.[^.]+$/, "")
+          : file.replace(/\.[^.]+$/, "");
+        const type = extname(file).slice(1);
+
+        await ingestDocument(context, config, title, content, {
+          source: file,
+          type,
+        });
+      }
+    },
+  };
+}

package/src/src/errors/catalog/general-errors.ts

@@ -36,6 +36,17 @@ export const GENERAL_ERROR_CATALOG: PartialErrorCatalog = {
     ],
   ),
 
+  "resource-not-found": createSimpleError(
+    "resource-not-found",
+    "Resource not found",
+    "The requested resource does not exist.",
+    [
+      "Verify the referenced workflow, run, approval, or tool ID",
+      "Check for typos in the resource name",
+      "Confirm the resource has not been deleted",
+    ],
+  ),
+
   "invalid-argument": createSimpleError(
     "invalid-argument",
     "Invalid argument",

package/src/src/errors/error-context.ts

@@ -1,5 +1,8 @@
 /******** Structured error handling with logging for silent failure operations */
 
+// NOTE: serverLogger is imported but only accessed inside function bodies (never at
+// module-eval time). This is safe even with circular deps because by the time any
+// function in this module is called, all modules have finished initializing.
 import { serverLogger } from "../utils/logger/logger.js";
 import { getErrorMessage } from "./veryfront-error.js";
 

package/src/src/errors/error-handlers.ts

@@ -13,8 +13,8 @@ function safeLog(logFn: () => void): void {
   } catch (error) {
     try {
       logger.warn("Logging failed:", error);
-    } catch {
-      // Silently ignore if even warning fails
+    } catch (_) {
+      // expected: last-resort fallback; nothing left to do if logging itself fails
     }
   }
 }
@@ -75,6 +75,7 @@ export async function retryWithBackoff<T>(
         continue;
       }
 
+      // no cleanup needed: one-shot
       await new Promise<void>((resolve) => dntShim.setTimeout(resolve, delay));
       delay = Math.min(delay * 2, maxDelay);
     }

package/src/src/errors/error-registry.ts

@@ -619,6 +619,14 @@ export const FILE_NOT_FOUND = defineError({
   suggestion: "Verify the file path exists",
 });
 
+export const RESOURCE_NOT_FOUND = defineError({
+  slug: "resource-not-found",
+  category: "GENERAL",
+  status: 404,
+  title: "Requested resource not found",
+  suggestion: "Verify the referenced resource ID or name exists",
+});
+
 export const INVALID_ARGUMENT = defineError({
   slug: "invalid-argument",
   category: "GENERAL",
@@ -769,6 +777,7 @@ export const ERROR_REGISTRY = {
   "unknown-error": UNKNOWN_ERROR,
   "permission-denied": PERMISSION_DENIED,
   "file-not-found": FILE_NOT_FOUND,
+  "resource-not-found": RESOURCE_NOT_FOUND,
   "invalid-argument": INVALID_ARGUMENT,
   "timeout-error": TIMEOUT_ERROR,
   "initialization-error": INITIALIZATION_ERROR,

package/src/src/errors/index.ts

@@ -93,6 +93,7 @@ export {
   RELEASE_NOT_FOUND,
   RENDER_ERROR,
   REQUEST_ERROR,
+  RESOURCE_NOT_FOUND,
   // ROUTE
   ROUTE_CONFLICT,
   ROUTE_HANDLER_INVALID,