veryfront 0.1.49 → 0.1.51

package/src/src/embedding/types.ts

@@ -1,5 +1,12 @@
 export interface EmbeddingConfig {
-  model: string; // "provider/model" format, e.g. "openai/text-embedding-3-small"
+  /**
+   * Optional model string in "provider/model" format.
+   *
+   * When omitted or set to `"auto"`, Veryfront chooses the runtime default:
+   * a local embedding model by default, automatically switching to the
+   * Veryfront Cloud embedding default when cloud bootstrap is present.
+   */
+  model?: string;
   documentPrefix?: string; // prepended when embedding documents, e.g. "search_document: "
   queryPrefix?: string; // prepended when embedding queries, e.g. "search_query: "
   batchSize?: number; // max texts per embedMany API call (default 100)
@@ -45,32 +52,37 @@ export interface VectorStore {
 }
 
 // ---------------------------------------------------------------------------
-// Upload Store
+// RAG Store
 // ---------------------------------------------------------------------------
 
-export interface UploadMeta {
+export interface RagDocumentMeta {
   id: string;
   title: string;
   source: string;
   type: string;
   createdAt: number;
+  url?: string;
 }
 
-export interface StoredChunk {
+export interface RagChunk {
   id: string;
-  uploadId: string;
+  documentId: string;
   text: string;
   embedding: number[]; // [] = not yet embedded
   index: number;
 }
 
-export interface UploadStoreData {
-  uploads: UploadMeta[];
-  chunks: StoredChunk[];
+export interface RagStoreData {
+  documents: RagDocumentMeta[];
+  chunks: RagChunk[];
 }
 
-export interface UploadStoreConfig {
-  model: string;
+export type RagStoreBackend = "auto" | "local-json" | "veryfront-cloud";
+
+export interface RagStoreConfig {
+  model?: string;
+  backend?: RagStoreBackend;
+  branch?: string; // optional branch override for cloud-backed stores
   storagePath?: string; // default "data/index.json"
   contentDir?: string; // optional auto-index dir
   contentExtensions?: string[]; // default [".md", ".mdx", ".txt"]
@@ -80,24 +92,24 @@ export interface UploadStoreConfig {
   batchSize?: number;
 }
 
-export interface UploadSearchResult {
+export interface RagSearchResult {
   text: string;
   score: number;
-  uploadId: string;
+  documentId: string;
   title: string;
   source: string;
   type: string;
 }
 
-export interface UploadSearchOptions {
+export interface RagSearchOptions {
   topK?: number; // default 5
   threshold?: number; // minimum similarity score
 }
 
-export interface UploadStore {
+export interface RagStore {
   ingest(title: string, text: string, meta?: { source?: string; type?: string }): Promise<string>;
-  search(query: string, options?: UploadSearchOptions): Promise<UploadSearchResult[]>;
-  listUploads(): Promise<UploadMeta[]>;
-  removeUpload(id: string): Promise<void>;
+  search(query: string, options?: RagSearchOptions): Promise<RagSearchResult[]>;
+  listDocuments(): Promise<RagDocumentMeta[]>;
+  removeDocument(id: string): Promise<void>;
   indexContentDir(): Promise<void>;
 }

package/src/src/embedding/upload-handler.ts

@@ -1,8 +1,13 @@
 import * as dntShim from "../../_dnt.shims.js";
-import type { UploadStore } from "./types.js";
+import { isVeryfrontCloudEnabled } from "../platform/cloud/resolver.js";
+import { VeryfrontCloudBlobStorage } from "../workflow/blob/veryfront-cloud-storage.js";
+import { serverLogger } from "../utils/index.js";
+import type { RagStore } from "./types.js";
 import { loadUpload } from "./upload-loader.js";
 
 const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB
+const MAX_FILE_NAME_LENGTH = 200;
+const CLOUD_UPLOAD_PREFIX = ".veryfront/rag/uploads/";
 
 const MIME_TO_TYPE: Record<string, string> = {
   "text/plain": "txt",
@@ -63,6 +68,47 @@ interface UploadHandlerConfig {
   maxFileSize?: number;
 }
 
+const MAX_CONCURRENT_URL_LOOKUPS = 5;
+
+async function enrichUploadsWithSourceUrls(
+  uploads: Awaited<ReturnType<RagStore["listDocuments"]>>,
+): Promise<Awaited<ReturnType<RagStore["listDocuments"]>>> {
+  const sourceBlobStorage = getSourceBlobStorage();
+  if (!sourceBlobStorage) return uploads;
+
+  const cloudUploads = uploads.filter((u) => u.source.startsWith("upload:"));
+  if (cloudUploads.length === 0) return uploads;
+
+  // Resolve signed URLs with bounded concurrency to avoid thundering herd
+  // against the uploads API when the document list is large.
+  const urlMap = new Map<string, string>();
+  for (let i = 0; i < cloudUploads.length; i += MAX_CONCURRENT_URL_LOOKUPS) {
+    const batch = cloudUploads.slice(i, i + MAX_CONCURRENT_URL_LOOKUPS);
+    const results = await Promise.allSettled(
+      batch.map(async (upload) => {
+        const blob = await sourceBlobStorage.stat(upload.id);
+        if (blob?.url) urlMap.set(upload.id, blob.url);
+      }),
+    );
+    for (const result of results) {
+      if (result.status === "rejected") {
+        serverLogger.warn("Upload source URL lookup failed:", result.reason);
+      }
+    }
+  }
+
+  return uploads.map((upload) => {
+    const url = urlMap.get(upload.id);
+    return url ? { ...upload, url } : upload;
+  });
+}
+
+function getSourceBlobStorage(): VeryfrontCloudBlobStorage | null {
+  return isVeryfrontCloudEnabled()
+    ? new VeryfrontCloudBlobStorage({ prefix: CLOUD_UPLOAD_PREFIX })
+    : null;
+}
+
 /**
  * Creates HTTP route handlers for upload, listing, and deletion.
  *
@@ -71,8 +117,10 @@ interface UploadHandlerConfig {
  *
  * Returns `{ POST, GET, DELETE }` handlers compatible with file-based routing.
  * POST accepts multipart form data with a `file` field, extracts text via
- * `loadUpload`, and ingests into the provided upload store. GET returns
- * the upload list. DELETE removes an upload by ID from route params.
+ * `loadUpload`, and ingests into the provided RAG store. When Veryfront Cloud
+ * bootstrap is present, the original binary is also stored in the project's
+ * uploads store via the cloud adapter. GET returns the upload list. DELETE
+ * removes an upload by ID from route params.
  *
  * @example
  * ```ts
@@ -93,7 +141,7 @@ interface UploadHandlerConfig {
  * ```
  */
 export function createUploadHandler(
-  store: UploadStore,
+  store: RagStore,
   config?: UploadHandlerConfig,
 ) {
   const maxSize = config?.maxFileSize ?? MAX_FILE_SIZE;
@@ -132,13 +180,39 @@ export function createUploadHandler(
       }
 
       // Sanitize file name: strip path components, limit length
-      const safeName = file.name.replace(/[/\\]/g, "_").slice(0, 200);
+      const safeName = file.name.replace(/[/\\]/g, "_").slice(0, MAX_FILE_NAME_LENGTH);
 
       const id = await store.ingest(safeName, text, {
         source: `upload:${safeName}`,
         type: fileType,
       });
 
+      const sourceBlobStorage = getSourceBlobStorage();
+      if (sourceBlobStorage) {
+        try {
+          await sourceBlobStorage.put(file, {
+            id,
+            mimeType: file.type || typeToMime(fileType),
+            metadata: {
+              originalName: safeName,
+              source: `upload:${safeName}`,
+              title: safeName,
+              type: fileType,
+            },
+          });
+        } catch (error) {
+          try {
+            await store.removeDocument(id);
+          } catch (cleanupError) {
+            serverLogger.warn(
+              "Upload rollback failed after source persistence error:",
+              cleanupError,
+            );
+          }
+          throw error;
+        }
+      }
+
       return dntShim.Response.json({
         success: true,
         upload: { id, title: file.name, type: fileType },
@@ -151,10 +225,10 @@ export function createUploadHandler(
 
   async function GET(): Promise<dntShim.Response> {
     try {
-      const uploads = await store.listUploads();
+      const uploads = await enrichUploadsWithSourceUrls(await store.listDocuments());
       return dntShim.Response.json({ uploads });
     } catch (error) {
-      console.error("Upload list failed:", error);
+      serverLogger.error("Upload list failed:", error);
       return dntShim.Response.json({ error: "Failed to list uploads" }, { status: 500 });
     }
   }
@@ -168,10 +242,21 @@ export function createUploadHandler(
       if (!id) {
         return dntShim.Response.json({ error: "Missing upload ID" }, { status: 400 });
       }
-      await store.removeUpload(id);
+
+      await store.removeDocument(id);
+
+      const sourceBlobStorage = getSourceBlobStorage();
+      if (sourceBlobStorage) {
+        try {
+          await sourceBlobStorage.delete(id);
+        } catch (error) {
+          serverLogger.warn("Upload source blob cleanup failed:", error);
+        }
+      }
+
       return dntShim.Response.json({ success: true });
     } catch (error) {
-      console.error("Upload delete failed:", error);
+      serverLogger.error("Upload delete failed:", error);
       return dntShim.Response.json({ error: "Delete failed" }, { status: 500 });
     }
   }

package/src/src/embedding/upload-loader.ts

@@ -38,7 +38,7 @@ function extractCSV(buffer: ArrayBuffer): string {
   const lines = text.split("\n").filter((line) => line.trim());
   if (lines.length < 2) return text;
 
-  const headers = parseCSVLine(lines[0]!);
+  const headers = parseCSVLine(lines[0] ?? "");
   const rows = lines.slice(1).map((line) => {
     const values = parseCSVLine(line);
     return headers

package/src/src/embedding/veryfront-cloud/provider.ts

@@ -0,0 +1,42 @@
+import type { EmbeddingModel } from "ai";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { createOpenAI } from "@ai-sdk/openai";
+import { createError, toError } from "../../errors/veryfront-error.js";
+import {
+  createVeryfrontCloudFetch,
+  getVeryfrontCloudGatewayBaseUrl,
+  parseVeryfrontCloudModelId,
+  requireVeryfrontCloudBootstrap,
+} from "../../provider/veryfront-cloud/shared.js";
+
+export function createVeryfrontCloudEmbeddingModel(modelId: string): EmbeddingModel {
+  const { provider, modelId: upstreamModelId } = parseVeryfrontCloudModelId(modelId, "embedding");
+  const { apiBaseUrl, apiToken } = requireVeryfrontCloudBootstrap();
+  const baseURL = getVeryfrontCloudGatewayBaseUrl(apiBaseUrl, provider);
+  const fetch = createVeryfrontCloudFetch(apiToken);
+
+  switch (provider) {
+    case "openai":
+      return createOpenAI({
+        apiKey: apiToken,
+        baseURL,
+        name: "veryfront-cloud",
+        fetch,
+      }).embedding(upstreamModelId);
+
+    case "google":
+      return createGoogleGenerativeAI({
+        apiKey: apiToken,
+        baseURL,
+        name: "veryfront-cloud",
+        fetch,
+      }).textEmbeddingModel(upstreamModelId);
+  }
+
+  throw toError(
+    createError({
+      type: "config",
+      message: `Embedding provider "${provider}" is not supported for veryfront-cloud.`,
+    }),
+  );
+}