vibefast-cli 0.7.12 → 0.7.14

package/recipes/chatbot/packages/backend/convex/ragKnowledge.ts

@@ -1,714 +0,0 @@
-import { openai } from '@ai-sdk/openai';
-import { getAuthUserId } from '@convex-dev/auth/server';
-import {
-  contentHashFromArrayBuffer,
-  defaultChunker,
-  type Entry,
-  type EntryId,
-  guessMimeTypeFromContents,
-  guessMimeTypeFromExtension,
-  RAG,
-  type SearchEntry,
-  vEntryId,
-} from '@convex-dev/rag';
-import type { PaginationResult, StorageReader } from 'convex/server';
-import { paginationOptsValidator } from 'convex/server';
-import { v } from 'convex/values';
-
-import { components, internal } from './_generated/api';
-import type { DataModel, Doc, Id } from './_generated/dataModel';
-import {
-  type ActionCtx,
-  action,
-  internalMutation,
-  internalQuery,
-  type MutationCtx,
-  mutation,
-  type QueryCtx,
-  query,
-} from './_generated/server';
-
-type Filters = {
-  filename: string;
-  category: string | null;
-};
-
-type Metadata = {
-  storageId: Id<'_storage'>;
-  uploadedBy: Id<'users'>;
-  namespace: string;
-};
-
-const EMBEDDING_DIMENSION = 1536;
-const DEFAULT_CHUNK_CONTEXT = { before: 1, after: 1 };
-const GLOBAL_NAMESPACE = 'global';
-
-const rag = new RAG<Filters, Metadata>(components.rag, {
-  filterNames: ['filename', 'category'],
-  // @ts-expect-error: Convex RAG expects AI SDK v2 embedding interface.
-  textEmbeddingModel: openai.textEmbeddingModel('text-embedding-3-small'),
-  embeddingDimension: EMBEDDING_DIMENSION,
-});
-
-type RagSearchResult = Awaited<ReturnType<typeof rag.search>>;
-type RagGenerateResult = Awaited<ReturnType<typeof rag.generateText>>;
-type RagGenerationContext = RagGenerateResult['context'];
-
-export const addKnowledgeFile = action({
-  args: {
-    globalNamespace: v.boolean(),
-    filename: v.string(),
-    bytes: v.bytes(),
-    mimeType: v.optional(v.string()),
-    category: v.optional(v.string()),
-  },
-  handler: async (
-    ctx,
-    args,
-  ): Promise<{ entryId: EntryId; created: boolean; url: string | null }> => {
-    const userId = await getRequiredUserId(ctx);
-    const namespace = args.globalNamespace ? GLOBAL_NAMESPACE : userId;
-    const contentHash = await contentHashFromArrayBuffer(args.bytes);
-
-    const existing = await rag.findEntryByContentHash(ctx, {
-      contentHash,
-      key: args.filename,
-      namespace,
-    });
-
-    if (existing) {
-      const existingFile: Doc<'knowledgeFiles'> | null =
-        await findKnowledgeFileByEntryId(ctx, existing.entryId);
-      const existingUrl: string | null = existingFile
-        ? await ctx.storage.getUrl(existingFile.storageId)
-        : null;
-      return {
-        entryId: existing.entryId,
-        created: false,
-        url: existingUrl,
-      };
-    }
-
-    const mimeType =
-      args.mimeType ??
-      guessMimeType(args.filename, args.bytes) ??
-      'application/octet-stream';
-
-    const blob = new Blob([args.bytes], { type: mimeType });
-    const storageId = await ctx.storage.store(blob);
-    const text = await extractText(ctx, {
-      filename: args.filename,
-      bytes: args.bytes,
-      mimeType,
-    });
-
-    const { entryId, created } = await rag.add(ctx, {
-      namespace,
-      text,
-      key: args.filename,
-      title: args.filename,
-      filterValues: [
-        { name: 'filename', value: args.filename },
-        { name: 'category', value: args.category ?? null },
-      ],
-      metadata: { storageId, uploadedBy: userId, namespace },
-      contentHash,
-      onComplete: internal.ragKnowledge.recordUploadMetadata,
-    });
-
-    if (!created) {
-      await ctx.storage.delete(storageId);
-      return {
-        entryId,
-        created: false,
-        url: await resolveUrlForEntry(ctx, entryId),
-      };
-    }
-
-    return {
-      entryId,
-      created: true,
-      url: await ctx.storage.getUrl(storageId),
-    };
-  },
-});
-
-export const addKnowledgeFileAsync = action({
-  args: {
-    globalNamespace: v.boolean(),
-    filename: v.string(),
-    storageId: v.id('_storage'),
-    mimeType: v.optional(v.string()),
-    category: v.optional(v.string()),
-  },
-  handler: async (
-    ctx,
-    args,
-  ): Promise<{ entryId: EntryId; created: boolean; url: string | null }> => {
-    const userId = await getRequiredUserId(ctx);
-    const namespace = args.globalNamespace ? GLOBAL_NAMESPACE : userId;
-    const metadata = await ctx.storage.getMetadata(args.storageId);
-
-    const blob = await ctx.storage.get(args.storageId);
-    if (!blob) {
-      throw new Error('Unable to retrieve stored file');
-    }
-
-    const bytes = await blob.arrayBuffer();
-    const contentHash = await contentHashFromArrayBuffer(bytes);
-
-    const existing = await rag.findEntryByContentHash(ctx, {
-      contentHash,
-      key: args.filename,
-      namespace,
-    });
-
-    if (existing) {
-      return {
-        entryId: existing.entryId,
-        created: false,
-        url: await resolveUrlForEntry(ctx, existing.entryId),
-      };
-    }
-
-    const mimeType =
-      args.mimeType ??
-      metadata?.contentType ??
-      guessMimeType(args.filename, bytes) ??
-      'application/octet-stream';
-
-    const { entryId } = await rag.addAsync(ctx, {
-      namespace,
-      key: args.filename,
-      title: args.filename,
-      filterValues: [
-        { name: 'filename', value: args.filename },
-        { name: 'category', value: args.category ?? null },
-      ],
-      metadata: { storageId: args.storageId, uploadedBy: userId, namespace },
-      chunkerAction: internal.ragKnowledge.chunkerAction,
-      contentHash,
-      onComplete: internal.ragKnowledge.recordUploadMetadata,
-    });
-
-    return {
-      entryId,
-      created: true,
-      url: await ctx.storage.getUrl(args.storageId),
-    };
-  },
-});
-
-export const searchKnowledge = action({
-  args: {
-    query: v.string(),
-    globalNamespace: v.boolean(),
-    limit: v.optional(v.number()),
-    chunkContext: v.optional(
-      v.object({ before: v.number(), after: v.number() }),
-    ),
-    filter: v.optional(
-      v.union(
-        v.object({
-          name: v.literal('category'),
-          value: v.union(v.null(), v.string()),
-        }),
-        v.object({
-          name: v.literal('filename'),
-          value: v.string(),
-        }),
-      ),
-    ),
-  },
-  handler: async (
-    ctx,
-    args,
-  ): Promise<RagSearchResult & { files: PublicKnowledgeFile[] }> => {
-    const userId = await getRequiredUserId(ctx);
-    const namespace = args.globalNamespace ? GLOBAL_NAMESPACE : userId;
-    const results = await rag.search(ctx, {
-      namespace,
-      query: args.query,
-      limit: args.limit ?? 10,
-      filters: args.filter ? [args.filter] : undefined,
-      chunkContext: args.chunkContext ?? DEFAULT_CHUNK_CONTEXT,
-    });
-
-    return {
-      ...results,
-      files: await entriesToPublicFiles(ctx, results.entries),
-    };
-  },
-});
-
-export const askKnowledge = action({
-  args: {
-    prompt: v.string(),
-    globalNamespace: v.boolean(),
-    limit: v.optional(v.number()),
-    chunkContext: v.optional(
-      v.object({ before: v.number(), after: v.number() }),
-    ),
-    filter: v.optional(
-      v.union(
-        v.object({
-          name: v.literal('category'),
-          value: v.union(v.null(), v.string()),
-        }),
-        v.object({
-          name: v.literal('filename'),
-          value: v.string(),
-        }),
-      ),
-    ),
-  },
-  handler: async (
-    ctx,
-    args,
-  ): Promise<
-    { answer: string; files: PublicKnowledgeFile[] } & RagGenerationContext
-  > => {
-    const userId = await getRequiredUserId(ctx);
-    const namespace = args.globalNamespace ? GLOBAL_NAMESPACE : userId;
-
-    const { text, context } = await rag.generateText(ctx, {
-      search: {
-        namespace,
-        limit: args.limit ?? 10,
-        filters: args.filter ? [args.filter] : undefined,
-        chunkContext: args.chunkContext ?? DEFAULT_CHUNK_CONTEXT,
-      },
-      prompt: args.prompt,
-      // @ts-expect-error: Convex RAG expects AI SDK v2 language model interface.
-      model: openai.chat('gpt-4o-mini'),
-    });
-
-    return {
-      answer: text,
-      ...context,
-      files: await entriesToPublicFiles(ctx, context.entries),
-    };
-  },
-});
-
-export const listKnowledgeFiles = query({
-  args: {
-    globalNamespace: v.boolean(),
-    category: v.optional(v.string()),
-    paginationOpts: paginationOptsValidator,
-  },
-  handler: async (
-    ctx,
-    args,
-  ): Promise<PaginationResult<PublicKnowledgeFile>> => {
-    const userId = await getRequiredUserId(ctx);
-    const namespace = args.globalNamespace ? GLOBAL_NAMESPACE : userId;
-    const namespaceInfo = await rag.getNamespace(ctx, { namespace });
-
-    if (!namespaceInfo) {
-      return { isDone: true, page: [], continueCursor: '' };
-    }
-
-    const results = await rag.list(ctx, {
-      namespaceId: namespaceInfo.namespaceId,
-      paginationOpts: args.paginationOpts,
-    });
-
-    const entries = args.category
-      ? results.page.filter((entry) =>
-          entry.filterValues.some(
-            (filter) =>
-              filter.name === 'category' && filter.value === args.category,
-          ),
-        )
-      : results.page;
-
-    const pageFiles = (
-      await Promise.all(entries.map((entry) => entryToPublicFile(ctx, entry)))
-    ).filter((file): file is PublicKnowledgeFile => file !== null);
-
-    return {
-      ...results,
-      page: pageFiles,
-    };
-  },
-});
-
-export const listPendingKnowledgeFiles = query({
-  args: {},
-  handler: async (ctx) => {
-    const userId = await getRequiredUserId(ctx);
-    const namespace = await rag.getNamespace(ctx, {
-      namespace: userId,
-    });
-    const globalNamespaceInfo = await rag.getNamespace(ctx, {
-      namespace: GLOBAL_NAMESPACE,
-    });
-
-    const paginationOpts = { numItems: 20, cursor: null };
-
-    const [userResults, globalResults] = await Promise.all([
-      namespace
-        ? rag.list(ctx, {
-            namespaceId: namespace.namespaceId,
-            status: 'pending',
-            paginationOpts,
-          })
-        : null,
-      globalNamespaceInfo
-        ? rag.list(ctx, {
-            namespaceId: globalNamespaceInfo.namespaceId,
-            status: 'pending',
-            paginationOpts,
-          })
-        : null,
-    ]);
-
-    const pendingEntries = [
-      ...(userResults?.page ?? []),
-      ...(globalResults?.page ?? []),
-    ];
-
-    return (
-      await Promise.all(
-        pendingEntries.map((entry) => entryToPublicFile(ctx, entry)),
-      )
-    ).filter((file): file is PublicKnowledgeFile => file !== null);
-  },
-});
-
-export const listKnowledgeChunks = query({
-  args: {
-    entryId: vEntryId,
-    paginationOpts: paginationOptsValidator,
-    order: v.union(v.literal('desc'), v.literal('asc')),
-  },
-  handler: async (ctx, args) => {
-    await getRequiredUserId(ctx);
-    return await rag.listChunks(ctx, {
-      entryId: args.entryId,
-      paginationOpts: args.paginationOpts,
-      order: args.order,
-    });
-  },
-});
-
-export const deleteKnowledgeFile = mutation({
-  args: { entryId: vEntryId },
-  handler: async (ctx, args) => {
-    const userId = await getRequiredUserId(ctx);
-    const fileDoc = await ctx.db
-      .query('knowledgeFiles')
-      .withIndex('by_entryId', (q) => q.eq('entryId', args.entryId))
-      .first();
-
-    if (!fileDoc) {
-      return;
-    }
-
-    if (!fileDoc.global && fileDoc.uploadedBy !== userId) {
-      throw new Error('Not authorized to delete this file');
-    }
-
-    await deleteFileRecord(ctx, args.entryId);
-  },
-});
-
-export const chunkerAction = rag.defineChunkerAction(async (ctx, { entry }) => {
-  if (!entry.metadata?.storageId) {
-    throw new Error('Missing storage metadata for chunking');
-  }
-
-  const fileBlob = await ctx.storage.get(entry.metadata.storageId);
-  if (!fileBlob) {
-    throw new Error('Stored file not found for chunking');
-  }
-
-  const bytes = await fileBlob.arrayBuffer();
-  const mimeType =
-    fileBlob.type ??
-    (await ctx.storage.getMetadata(entry.metadata.storageId))?.contentType ??
-    guessMimeType(entry.key ?? 'uploaded', bytes) ??
-    'application/octet-stream';
-
-  const text = await extractText(ctx, {
-    filename: entry.key ?? 'uploaded',
-    bytes,
-    mimeType,
-  });
-
-  return { chunks: defaultChunker(text) };
-});
-
-export const recordUploadMetadata = rag.defineOnComplete<DataModel>(
-  async (ctx, args) => {
-    const { entry, replacedEntry, namespace, error } = args;
-
-    if (replacedEntry) {
-      await deleteFileRecord(ctx, replacedEntry.entryId);
-    }
-
-    if (!entry.metadata?.storageId || !entry.metadata?.uploadedBy) {
-      return;
-    }
-
-    if (error) {
-      const errorMessage =
-        typeof error === 'string'
-          ? error
-          : ((error as { message?: string })?.message ?? 'Unknown error');
-      await ctx.db.insert('knowledgeFiles', {
-        entryId: entry.entryId,
-        namespace: namespace.namespace,
-        filename: entry.key ?? 'unknown',
-        storageId: entry.metadata.storageId,
-        uploadedBy: entry.metadata.uploadedBy,
-        global: namespace.namespace === GLOBAL_NAMESPACE,
-        category:
-          entry.filterValues.find((f) => f.name === 'category')?.value ??
-          undefined,
-        title: entry.title ?? undefined,
-        status: 'error',
-        lastError: errorMessage,
-        createdAt: Date.now(),
-        updatedAt: Date.now(),
-      });
-      return;
-    }
-
-    if (entry.status !== 'ready') {
-      await upsertKnowledgeFile(ctx, {
-        entry,
-        namespace: namespace.namespace,
-        status: 'pending',
-      });
-      return;
-    }
-
-    await upsertKnowledgeFile(ctx, {
-      entry,
-      namespace: namespace.namespace,
-      status: 'ready',
-    });
-  },
-);
-
-export const deleteReplacedKnowledge = internalMutation({
-  args: { cursor: v.optional(v.string()) },
-  handler: async (ctx, args) => {
-    const entries = await rag.list(ctx, {
-      status: 'replaced',
-      paginationOpts: { cursor: args.cursor ?? null, numItems: 100 },
-    });
-
-    const cutoff = Date.now() - 1000 * 60 * 60 * 24 * 7;
-
-    for (const entry of entries.page) {
-      const replacedAt = (
-        entry as EntryWithMetadata & {
-          replacedAt?: number;
-        }
-      ).replacedAt;
-      if (typeof replacedAt === 'number' && replacedAt >= cutoff) {
-        continue;
-      }
-      await deleteFileRecord(ctx, entry.entryId);
-    }
-
-    if (!entries.isDone) {
-      await ctx.scheduler.runAfter(
-        0,
-        internal.ragKnowledge.deleteReplacedKnowledge,
-        {
-          cursor: entries.continueCursor,
-        },
-      );
-    }
-  },
-});
-
-export const getKnowledgeFileMetadata = internalQuery({
-  args: { entryId: v.string() },
-  handler: async (ctx, args) => {
-    return await ctx.db
-      .query('knowledgeFiles')
-      .withIndex('by_entryId', (q) => q.eq('entryId', args.entryId))
-      .unique();
-  },
-});
-
-type EntryWithMetadata = Entry<Filters, Metadata>;
-
-export type PublicKnowledgeFile = {
-  entryId: EntryId;
-  filename: string;
-  storageId: Id<'_storage'>;
-  global: boolean;
-  category?: string;
-  title?: string;
-  status: EntryWithMetadata['status'];
-  url: string | null;
-};
-
-async function entriesToPublicFiles(
-  ctx: ActionCtx | QueryCtx,
-  entries: SearchEntry<Filters, Metadata>[],
-): Promise<PublicKnowledgeFile[]> {
-  const files = await Promise.all(
-    entries.map((entry) => entryToPublicFile(ctx, entry)),
-  );
-  return files.filter((file): file is PublicKnowledgeFile => file !== null);
-}
-
-async function entryToPublicFile(
-  ctx: { storage: StorageReader },
-  entry: EntryWithMetadata,
-): Promise<PublicKnowledgeFile | null> {
-  if (!entry.metadata?.storageId) {
-    return null;
-  }
-
-  const category =
-    entry.filterValues.find((filter) => filter.name === 'category')?.value ??
-    undefined;
-
-  return {
-    entryId: entry.entryId,
-    filename: entry.key ?? 'unknown',
-    storageId: entry.metadata.storageId,
-    global: entry.metadata.namespace === GLOBAL_NAMESPACE,
-    category,
-    title: entry.title ?? undefined,
-    status: entry.status,
-    url: await ctx.storage.getUrl(entry.metadata.storageId),
-  };
-}
-
-async function getRequiredUserId(ctx: QueryCtx | MutationCtx | ActionCtx) {
-  const userId = await getAuthUserId(ctx);
-  if (!userId) {
-    throw new Error('Unauthorized');
-  }
-  return userId;
-}
-
-async function upsertKnowledgeFile(
-  ctx: MutationCtx,
-  params: {
-    entry: EntryWithMetadata;
-    namespace: string;
-    status: 'pending' | 'ready';
-  },
-) {
-  const doc = await ctx.db
-    .query('knowledgeFiles')
-    .withIndex('by_entryId', (q) => q.eq('entryId', params.entry.entryId))
-    .unique();
-
-  const storageId = params.entry.metadata?.storageId;
-  const uploadedBy = params.entry.metadata?.uploadedBy;
-
-  if (!storageId || !uploadedBy) {
-    return;
-  }
-
-  const payload: Omit<Doc<'knowledgeFiles'>, '_id' | '_creationTime'> = {
-    entryId: params.entry.entryId,
-    namespace: params.namespace,
-    filename: params.entry.key ?? 'unknown',
-    storageId: storageId as Id<'_storage'>,
-    uploadedBy: uploadedBy as Id<'users'>,
-    global: params.namespace === GLOBAL_NAMESPACE,
-    category:
-      params.entry.filterValues.find((filter) => filter.name === 'category')
-        ?.value ?? undefined,
-    title: params.entry.title ?? undefined,
-    status: params.status,
-    lastError: undefined,
-    createdAt: doc?.createdAt ?? Date.now(),
-    updatedAt: Date.now(),
-  };
-
-  if (doc) {
-    await ctx.db.replace(doc._id, payload);
-  } else {
-    await ctx.db.insert('knowledgeFiles', payload);
-  }
-}
-
-async function deleteFileRecord(ctx: MutationCtx, entryId: EntryId) {
-  const doc = await ctx.db
-    .query('knowledgeFiles')
-    .withIndex('by_entryId', (q) => q.eq('entryId', entryId))
-    .unique();
-
-  if (doc) {
-    await ctx.db.delete(doc._id);
-    await ctx.storage.delete(doc.storageId);
-  }
-
-  await rag.delete(ctx, { entryId });
-}
-
-async function extractText(
-  ctx: Pick<ActionCtx, 'storage'>,
-  {
-    filename,
-    bytes,
-    storageId,
-    mimeType,
-  }: {
-    filename: string;
-    bytes?: ArrayBuffer;
-    storageId?: Id<'_storage'>;
-    mimeType?: string;
-  },
-) {
-  let buffer = bytes;
-
-  if (!buffer && storageId) {
-    const blob = await ctx.storage.get(storageId);
-    if (blob) {
-      buffer = await blob.arrayBuffer();
-    }
-  }
-
-  if (!buffer) {
-    throw new Error('Unable to retrieve file contents for ingestion');
-  }
-
-  const detectedMime =
-    mimeType ?? guessMimeType(filename, buffer) ?? 'application/octet-stream';
-
-  if (
-    detectedMime.startsWith('text/') ||
-    detectedMime === 'application/json' ||
-    detectedMime === 'application/xml'
-  ) {
-    return new TextDecoder('utf-8').decode(buffer);
-  }
-
-  throw new Error(
-    `Unsupported file type for ingestion: ${detectedMime} (${filename})`,
-  );
-}
-
-async function resolveUrlForEntry(ctx: ActionCtx, entryId: EntryId) {
-  const fileDoc = await findKnowledgeFileByEntryId(ctx, entryId);
-  return fileDoc ? await ctx.storage.getUrl(fileDoc.storageId) : null;
-}
-
-async function findKnowledgeFileByEntryId(
-  ctx: ActionCtx,
-  entryId: EntryId,
-): Promise<Doc<'knowledgeFiles'> | null> {
-  return await ctx.runQuery(internal.ragKnowledge.getKnowledgeFileMetadata, {
-    entryId,
-  });
-}
-
-function guessMimeType(filename: string, bytes: ArrayBuffer) {
-  return (
-    guessMimeTypeFromExtension(filename) || guessMimeTypeFromContents(bytes)
-  );
-}