unrag 0.2.3 → 0.2.4

package/dist/cli/index.js

@@ -4,7 +4,15 @@
 import { intro, outro as outro3 } from "@clack/prompts";
 
 // cli/commands/init.ts
-import { cancel as cancel2, isCancel as isCancel2, outro, select, text } from "@clack/prompts";
+import {
+  cancel as cancel2,
+  confirm as confirm2,
+  groupMultiselect,
+  isCancel as isCancel2,
+  outro,
+  select,
+  text
+} from "@clack/prompts";
 import path5 from "node:path";
 import { fileURLToPath } from "node:url";
 
@@ -71,8 +79,39 @@ var writeText = async (filePath, content) => {
   await ensureDir(path2.dirname(filePath));
   await writeFile(filePath, content, "utf8");
 };
+var EXTRACTOR_FACTORY = {
+  "pdf-llm": "createPdfLlmExtractor",
+  "pdf-text-layer": "createPdfTextLayerExtractor",
+  "pdf-ocr": "createPdfOcrExtractor",
+  "image-ocr": "createImageOcrExtractor",
+  "image-caption-llm": "createImageCaptionLlmExtractor",
+  "audio-transcribe": "createAudioTranscribeExtractor",
+  "video-transcribe": "createVideoTranscribeExtractor",
+  "video-frames": "createVideoFramesExtractor",
+  "file-text": "createFileTextExtractor",
+  "file-docx": "createFileDocxExtractor",
+  "file-pptx": "createFilePptxExtractor",
+  "file-xlsx": "createFileXlsxExtractor"
+};
+var EXTRACTOR_FLAG_KEYS = {
+  "pdf-text-layer": ["pdf_textLayer"],
+  "pdf-llm": ["pdf_llmExtraction"],
+  "pdf-ocr": ["pdf_ocr"],
+  "image-ocr": ["image_ocr"],
+  "image-caption-llm": ["image_captionLlm"],
+  "audio-transcribe": ["audio_transcription"],
+  "video-transcribe": ["video_transcription"],
+  "video-frames": ["video_frames"],
+  "file-text": ["file_text"],
+  "file-docx": ["file_docx"],
+  "file-pptx": ["file_pptx"],
+  "file-xlsx": ["file_xlsx"]
+};
+var ALL_FLAG_KEYS = Array.from(new Set(Object.values(EXTRACTOR_FLAG_KEYS).flat())).sort();
 var renderUnragConfig = (content, selection) => {
   const installImportBase = `./${selection.installDir.replace(/\\/g, "/")}`;
+  const richMedia = selection.richMedia ?? { enabled: false, extractors: [] };
+  const selectedExtractors = Array.from(new Set(richMedia.extractors ?? [])).sort();
   const baseImports = [
     `import { defineUnragConfig } from "${installImportBase}/core";`
   ];
@@ -88,22 +127,40 @@ var renderUnragConfig = (content, selection) => {
     storeImports.push(`import { createPrismaVectorStore } from "${installImportBase}/store/prisma";`, `import { PrismaClient } from "@prisma/client";`);
     storeCreateLines.push(`  const prisma = (globalThis as any).__unragPrisma ?? new PrismaClient();`, `  (globalThis as any).__unragPrisma = prisma;`, `  const store = createPrismaVectorStore(prisma);`);
   }
-  const importsBlock = [...baseImports, ...storeImports].join(`
+  const extractorImports = [];
+  if (richMedia.enabled && selectedExtractors.length > 0) {
+    for (const ex of selectedExtractors) {
+      const factory = EXTRACTOR_FACTORY[ex];
+      extractorImports.push(`import { ${factory} } from "${installImportBase}/extractors/${ex}";`);
+    }
+  }
+  const importsBlock = [...baseImports, ...storeImports, ...extractorImports].join(`
 `);
   const createEngineBlock = [
     `export function createUnragEngine() {`,
     ...storeCreateLines,
     ``,
     `  return unrag.createEngine({ store });`,
-    `}`,
-    ``,
-    `export async function retrieve(query: string) {`,
-    `  const engine = createUnragEngine();`,
-    `  return engine.retrieve({ query, topK: unrag.defaults.retrieval.topK });`,
     `}`
   ].join(`
 `);
-  return content.replace("// __UNRAG_IMPORTS__", importsBlock).replace("// __UNRAG_CREATE_ENGINE__", createEngineBlock);
+  let out = content.replace("// __UNRAG_IMPORTS__", importsBlock).replace("// __UNRAG_CREATE_ENGINE__", createEngineBlock);
+  out = out.replace('type: "text", // __UNRAG_EMBEDDING_TYPE__', richMedia.enabled ? 'type: "multimodal",' : 'type: "text",').replace('model: "openai/text-embedding-3-small", // __UNRAG_EMBEDDING_MODEL__', richMedia.enabled ? 'model: "cohere/embed-v4.0",' : 'model: "openai/text-embedding-3-small",');
+  const enabledFlagKeys = new Set;
+  if (richMedia.enabled) {
+    for (const ex of selectedExtractors) {
+      for (const k of EXTRACTOR_FLAG_KEYS[ex] ?? []) {
+        enabledFlagKeys.add(k);
+      }
+    }
+  }
+  for (const k of ALL_FLAG_KEYS) {
+    out = out.replace(`enabled: false, // __UNRAG_FLAG_${k}__`, `enabled: ${enabledFlagKeys.has(k) ? "true" : "false"},`);
+  }
+  const extractorLines = richMedia.enabled && selectedExtractors.length > 0 ? selectedExtractors.map((ex) => `      ${EXTRACTOR_FACTORY[ex]}(),`).join(`
+`) : "";
+  out = out.replace("      // __UNRAG_EXTRACTORS__", extractorLines);
+  return out;
 };
 var renderDocs = (content, selection) => {
   const notes = [];
@@ -272,28 +329,34 @@ async function copyExtractorFiles(selection) {
   const destRootAbs = path2.join(installBaseAbs, "extractors", selection.extractor);
   const sharedDestRootAbs = path2.join(installBaseAbs, "extractors", "_shared");
   const nonInteractive = Boolean(selection.yes) || !process.stdin.isTTY;
+  const shouldWrite = async (src, dest) => {
+    if (!await exists(dest))
+      return true;
+    if (nonInteractive)
+      return false;
+    try {
+      const [srcRaw, destRaw] = await Promise.all([readText(src), readText(dest)]);
+      if (srcRaw === destRaw)
+        return false;
+    } catch {}
+    const answer = await confirm({
+      message: `Overwrite ${path2.relative(selection.projectRoot, dest)}?`,
+      initialValue: false
+    });
+    if (isCancel(answer)) {
+      cancel("Cancelled.");
+      return false;
+    }
+    return Boolean(answer);
+  };
   for (const src of extractorFiles) {
     if (!await exists(src)) {
       throw new Error(`Registry file missing: ${src}`);
     }
     const rel = path2.relative(extractorRegistryAbs, src);
     const dest = path2.join(destRootAbs, rel);
-    if (await exists(dest)) {
-      if (nonInteractive) {
-        continue;
-      }
-      const answer = await confirm({
-        message: `Overwrite ${path2.relative(selection.projectRoot, dest)}?`,
-        initialValue: false
-      });
-      if (isCancel(answer)) {
-        cancel("Cancelled.");
-        return;
-      }
-      if (!answer) {
-        continue;
-      }
-    }
+    if (!await shouldWrite(src, dest))
+      continue;
     const raw = await readText(src);
     await writeText(dest, raw);
   }
@@ -303,22 +366,8 @@ async function copyExtractorFiles(selection) {
     }
     const rel = path2.relative(sharedRegistryAbs, src);
     const dest = path2.join(sharedDestRootAbs, rel);
-    if (await exists(dest)) {
-      if (nonInteractive) {
-        continue;
-      }
-      const answer = await confirm({
-        message: `Overwrite ${path2.relative(selection.projectRoot, dest)}?`,
-        initialValue: false
-      });
-      if (isCancel(answer)) {
-        cancel("Cancelled.");
-        return;
-      }
-      if (!answer) {
-        continue;
-      }
-    }
+    if (!await shouldWrite(src, dest))
+      continue;
     const raw = await readText(src);
     await writeText(dest, raw);
   }
@@ -552,9 +601,93 @@ var parseInitArgs = (args) => {
       }
       continue;
     }
+    if (a === "--rich-media") {
+      out.richMedia = true;
+      continue;
+    }
+    if (a === "--no-rich-media") {
+      out.richMedia = false;
+      continue;
+    }
+    if (a === "--extractors") {
+      const v = args[i + 1];
+      if (v) {
+        out.extractors = v.split(",").map((s) => s.trim()).filter(Boolean);
+        i++;
+      }
+      continue;
+    }
   }
   return out;
 };
+var DEFAULT_RICH_MEDIA_EXTRACTORS = ["pdf-text-layer", "file-text"];
+var EXTRACTOR_OPTIONS = [
+  {
+    group: "PDF",
+    value: "pdf-text-layer",
+    label: `pdf-text-layer (Fast/cheap extraction via PDF text layer)`,
+    hint: "recommended"
+  },
+  {
+    group: "PDF",
+    value: "pdf-llm",
+    label: `pdf-llm (LLM-based PDF extraction; higher cost)`
+  },
+  {
+    group: "PDF",
+    value: "pdf-ocr",
+    label: `pdf-ocr (OCR scanned PDFs; requires native binaries)`,
+    hint: "worker-only"
+  },
+  {
+    group: "Image",
+    value: "image-ocr",
+    label: `image-ocr (Extract text from images via vision LLM)`
+  },
+  {
+    group: "Image",
+    value: "image-caption-llm",
+    label: `image-caption-llm (Generate captions for images via vision LLM)`
+  },
+  {
+    group: "Audio",
+    value: "audio-transcribe",
+    label: `audio-transcribe (Speech-to-text transcription)`
+  },
+  {
+    group: "Video",
+    value: "video-transcribe",
+    label: `video-transcribe (Transcribe video audio track)`
+  },
+  {
+    group: "Video",
+    value: "video-frames",
+    label: `video-frames (Sample frames + analyze via vision LLM; requires ffmpeg)`,
+    hint: "worker-only"
+  },
+  {
+    group: "Files",
+    value: "file-text",
+    label: `file-text (Extract text/markdown/json/html from common text files)`,
+    hint: "recommended"
+  },
+  {
+    group: "Files",
+    value: "file-docx",
+    label: `file-docx (Extract text from .docx files)`
+  },
+  {
+    group: "Files",
+    value: "file-pptx",
+    label: `file-pptx (Extract text from .pptx slides)`
+  },
+  {
+    group: "Files",
+    value: "file-xlsx",
+    label: `file-xlsx (Extract tables from .xlsx spreadsheets)`
+  }
+];
+var AVAILABLE_EXTRACTORS = new Set(EXTRACTOR_OPTIONS.map((o) => o.value));
 async function initCommand(args) {
   const root = await tryFindProjectRoot(process.cwd());
   if (!root) {
@@ -623,17 +756,71 @@ async function initCommand(args) {
     return;
   }
   const aliasBase = String(aliasAnswer).trim();
+  if (parsed.richMedia === false && (parsed.extractors ?? []).length > 0) {
+    throw new Error('Cannot use "--no-rich-media" together with "--extractors".');
+  }
+  const extractorsFromArgs = (parsed.extractors ?? []).filter((x) => AVAILABLE_EXTRACTORS.has(x)).sort();
+  const richMediaAnswer = extractorsFromArgs.length > 0 ? true : typeof parsed.richMedia === "boolean" ? parsed.richMedia : nonInteractive ? false : await confirm2({
+    message: "Enable rich media ingestion (PDF/images/audio/video/files)? This also enables multimodal image embeddings (you can change this later).",
+    initialValue: false
+  });
+  if (isCancel2(richMediaAnswer)) {
+    cancel2("Cancelled.");
+    return;
+  }
+  const richMediaEnabled = Boolean(richMediaAnswer);
+  const selectedExtractorsAnswer = richMediaEnabled || extractorsFromArgs.length > 0 ? nonInteractive ? extractorsFromArgs.length > 0 ? extractorsFromArgs : DEFAULT_RICH_MEDIA_EXTRACTORS : await groupMultiselect({
+    message: "Select extractors to enable (space to toggle, enter to confirm)",
+    options: EXTRACTOR_OPTIONS.reduce((acc, opt) => {
+      acc[opt.group] ??= [];
+      acc[opt.group].push({
+        value: opt.value,
+        label: opt.label,
+        ...opt.hint ? { hint: opt.hint } : {}
+      });
+      return acc;
+    }, {}),
+    initialValues: extractorsFromArgs.length > 0 ? extractorsFromArgs : DEFAULT_RICH_MEDIA_EXTRACTORS,
+    required: false
+  }) : [];
+  if (isCancel2(selectedExtractorsAnswer)) {
+    cancel2("Cancelled.");
+    return;
+  }
+  const selectedExtractors = Array.from(new Set(Array.isArray(selectedExtractorsAnswer) ? selectedExtractorsAnswer : [])).sort();
   const selection = {
     installDir,
     storeAdapter: storeAdapterAnswer,
     projectRoot: root,
     registryRoot,
-    aliasBase
+    aliasBase,
+    richMedia: richMediaEnabled ? {
+      enabled: true,
+      extractors: selectedExtractors
+    } : { enabled: false, extractors: [] }
   };
   await copyRegistryFiles(selection);
+  if (richMediaEnabled && selectedExtractors.length > 0) {
+    for (const extractor of selectedExtractors) {
+      await copyExtractorFiles({
+        projectRoot: root,
+        registryRoot,
+        installDir,
+        extractor,
+        yes: nonInteractive
+      });
+    }
+  }
   const pkg = await readPackageJson(root);
   const { deps, devDeps } = depsForAdapter(storeAdapterAnswer);
-  const merged = mergeDeps(pkg, deps, devDeps);
+  const extractorDeps = {};
+  const extractorDevDeps = {};
+  for (const ex of selectedExtractors) {
+    const r = depsForExtractor(ex);
+    Object.assign(extractorDeps, r.deps);
+    Object.assign(extractorDevDeps, r.devDeps);
+  }
+  const merged = mergeDeps(pkg, { ...deps, ...extractorDeps }, { ...devDeps, ...extractorDevDeps });
   if (merged.changes.length > 0) {
     await writePackageJson(root, merged.pkg);
   }
@@ -643,7 +830,10 @@ async function initCommand(args) {
     aliasBase,
     version: CONFIG_VERSION,
     connectors: existing?.connectors ?? [],
-    extractors: existing?.extractors ?? []
+    extractors: Array.from(new Set([
+      ...existing?.extractors ?? [],
+      ...richMediaEnabled ? selectedExtractors : []
+    ])).sort()
   };
   await writeJsonFile(path5.join(root, CONFIG_FILE), config);
   const pm = await detectPackageManager(root);
@@ -657,6 +847,11 @@ async function initCommand(args) {
     `- Docs: ${path5.join(installDir, "unrag.md")}`,
     `- Config: unrag.config.ts`,
     `- Imports: ${aliasBase}/* and ${aliasBase}/config`,
+    "",
+    `- Rich media: ${richMediaEnabled ? "enabled" : "disabled"}`,
+    richMediaEnabled ? `- Embeddings: multimodal enabled (images can be embedded directly)` : `- Embeddings: text-only (no direct image embedding)`,
+    richMediaEnabled ? `- Extractors: ${selectedExtractors.length > 0 ? selectedExtractors.join(", ") : "none"}` : "",
+    richMediaEnabled ? `  Tip: you can tweak extractors + assetProcessing flags in unrag.config.ts later.` : `  Tip: re-run \`unrag init --rich-media\` (or edit unrag.config.ts) to enable rich media later.`,
     isNext ? tsconfigResult.changed ? `- Next.js: updated ${tsconfigResult.file} (added aliases)` : `- Next.js: no tsconfig changes needed` : `- Next.js: not detected`,
     "",
     merged.changes.length > 0 ? `Added deps: ${merged.changes.map((c) => c.name).join(", ")}` : "Added deps: none",
@@ -685,7 +880,7 @@ function docsUrl(siteRelativePath) {
 var CONFIG_FILE2 = "unrag.json";
 var __filename3 = fileURLToPath2(import.meta.url);
 var __dirname3 = path6.dirname(__filename3);
-var AVAILABLE_EXTRACTORS = [
+var AVAILABLE_EXTRACTORS2 = [
   "pdf-llm",
   "pdf-text-layer",
   "pdf-ocr",
@@ -738,7 +933,7 @@ async function addCommand(args) {
       "  unrag add extractor <name>",
       "",
       "Available connectors: notion",
-      `Available extractors: ${AVAILABLE_EXTRACTORS.join(", ")}`
+      `Available extractors: ${AVAILABLE_EXTRACTORS2.join(", ")}`
     ].join(`
 `));
     return;
@@ -790,10 +985,10 @@ Available connectors: notion`);
     return;
   }
   const extractor = name;
-  if (!extractor || !AVAILABLE_EXTRACTORS.includes(extractor)) {
+  if (!extractor || !AVAILABLE_EXTRACTORS2.includes(extractor)) {
     outro2(`Unknown extractor: ${name}
 
-Available extractors: ${AVAILABLE_EXTRACTORS.join(", ")}`);
+Available extractors: ${AVAILABLE_EXTRACTORS2.join(", ")}`);
     return;
   }
   await copyExtractorFiles({
@@ -844,10 +1039,15 @@ function renderHelp() {
     "  --store <adapter>    drizzle | prisma | raw-sql",
     "  --dir <path>         Install directory (alias: --install-dir)",
     "  --alias <@name>      Import alias base (e.g. @unrag)",
+    "  --rich-media         Enable rich media setup (also enables multimodal embeddings)",
+    "  --no-rich-media      Disable rich media setup",
+    "  --extractors <list>  Comma-separated extractors (implies --rich-media)",
     "",
     "Examples:",
     "  bunx unrag@latest init",
     "  bunx unrag@latest init --yes --store drizzle --dir lib/unrag --alias @unrag",
+    "  bunx unrag@latest init --yes --rich-media",
+    "  bunx unrag@latest init --yes --extractors pdf-text-layer,file-text",
     "  bunx unrag add notion --yes",
     "",
     "Docs:",

package/package.json

@@ -2,10 +2,11 @@
   "name": "unrag",
   "type": "module",
   "repository": "https://github.com/BetterStacks/unrag",
+  "homepage": "https://unrag.dev",
   "bin": {
     "unrag": "./dist/cli/index.js"
   },
-  "version": "0.2.3",
+  "version": "0.2.4",
   "private": false,
   "license": "Apache-2.0",
   "devDependencies": {

package/registry/config/unrag.config.ts

@@ -11,6 +11,8 @@
  * treated like vendored source code.
  */
 
+// @ts-nocheck
+
 // __UNRAG_IMPORTS__
 
 export const unrag = defineUnragConfig({
@@ -26,8 +28,8 @@ export const unrag = defineUnragConfig({
   embedding: {
     provider: "ai",
     config: {
-      type: "text",
-      model: "openai/text-embedding-3-small",
+      type: "text", // __UNRAG_EMBEDDING_TYPE__
+      model: "openai/text-embedding-3-small", // __UNRAG_EMBEDDING_MODEL__
       timeoutMs: 15_000,
     },
   },
@@ -52,13 +54,15 @@ export const unrag = defineUnragConfig({
      * - `import { createPdfLlmExtractor } from "./lib/unrag/extractors/pdf-llm";`
      * - `extractors: [createPdfLlmExtractor()]`
      */
-    extractors: [],
+    extractors: [
+      // __UNRAG_EXTRACTORS__
+    ],
   /**
    * Rich media processing controls.
    *
    * Notes:
-   * - The library defaults are cost-safe (PDF LLM extraction is off).
-   * - This generated config opts you into PDF extraction for convenience.
+   * - This generated config is cost-safe by default (all extraction is off).
+   * - `unrag init` can enable rich media + multimodal embeddings for you.
    * - Tighten fetch allowlists/limits in production if you ingest URL-based assets.
    */
   assetProcessing: {
@@ -74,14 +78,14 @@ export const unrag = defineUnragConfig({
     pdf: {
       // Fast/cheap text-layer extraction (requires installing a PDF text-layer extractor module).
       textLayer: {
-        enabled: false,
+        enabled: false, // __UNRAG_FLAG_pdf_textLayer__
         maxBytes: 15 * 1024 * 1024,
         maxOutputChars: 200_000,
         minChars: 200,
         // maxPages: 200,
       },
       llmExtraction: {
-        enabled: true,
+        enabled: false, // __UNRAG_FLAG_pdf_llmExtraction__
         model: "google/gemini-2.0-flash",
         prompt:
           "Extract all readable text from this PDF as faithfully as possible. Preserve structure with headings and lists when obvious. Output plain text or markdown only. Do not add commentary.",
@@ -91,7 +95,7 @@ export const unrag = defineUnragConfig({
       },
       // Worker-only OCR pipelines typically require native binaries (poppler/tesseract) or external services.
       ocr: {
-        enabled: false,
+        enabled: false, // __UNRAG_FLAG_pdf_ocr__
         maxBytes: 15 * 1024 * 1024,
         maxOutputChars: 200_000,
         minChars: 200,
@@ -104,7 +108,7 @@ export const unrag = defineUnragConfig({
     },
     image: {
       ocr: {
-        enabled: false,
+        enabled: false, // __UNRAG_FLAG_image_ocr__
         model: "google/gemini-2.0-flash",
         prompt:
           "Extract all readable text from this image as faithfully as possible. Output plain text only. Do not add commentary.",
@@ -113,7 +117,7 @@ export const unrag = defineUnragConfig({
         maxOutputChars: 50_000,
       },
       captionLlm: {
-        enabled: false,
+        enabled: false, // __UNRAG_FLAG_image_captionLlm__
         model: "google/gemini-2.0-flash",
         prompt:
           "Write a concise, information-dense caption for this image. Include names, numbers, and labels if visible. Output plain text only.",
@@ -124,7 +128,7 @@ export const unrag = defineUnragConfig({
     },
     audio: {
       transcription: {
-        enabled: false,
+        enabled: false, // __UNRAG_FLAG_audio_transcription__
         model: "openai/whisper-1",
         timeoutMs: 120_000,
         maxBytes: 25 * 1024 * 1024,
@@ -132,13 +136,13 @@ export const unrag = defineUnragConfig({
     },
     video: {
       transcription: {
-        enabled: false,
+        enabled: false, // __UNRAG_FLAG_video_transcription__
         model: "openai/whisper-1",
         timeoutMs: 120_000,
         maxBytes: 50 * 1024 * 1024,
       },
       frames: {
-        enabled: false,
+        enabled: false, // __UNRAG_FLAG_video_frames__
         sampleFps: 0.2,
         maxFrames: 50,
         // ffmpegPath: "/usr/bin/ffmpeg",
@@ -151,10 +155,30 @@ export const unrag = defineUnragConfig({
       },
     },
     file: {
-      text: { enabled: false, maxBytes: 5 * 1024 * 1024, maxOutputChars: 200_000, minChars: 50 },
-      docx: { enabled: false, maxBytes: 15 * 1024 * 1024, maxOutputChars: 200_000, minChars: 50 },
-      pptx: { enabled: false, maxBytes: 30 * 1024 * 1024, maxOutputChars: 200_000, minChars: 50 },
-      xlsx: { enabled: false, maxBytes: 30 * 1024 * 1024, maxOutputChars: 200_000, minChars: 50 },
+      text: {
+        enabled: false, // __UNRAG_FLAG_file_text__
+        maxBytes: 5 * 1024 * 1024,
+        maxOutputChars: 200_000,
+        minChars: 50,
+      },
+      docx: {
+        enabled: false, // __UNRAG_FLAG_file_docx__
+        maxBytes: 15 * 1024 * 1024,
+        maxOutputChars: 200_000,
+        minChars: 50,
+      },
+      pptx: {
+        enabled: false, // __UNRAG_FLAG_file_pptx__
+        maxBytes: 30 * 1024 * 1024,
+        maxOutputChars: 200_000,
+        minChars: 50,
+      },
+      xlsx: {
+        enabled: false, // __UNRAG_FLAG_file_xlsx__
+        maxBytes: 30 * 1024 * 1024,
+        maxOutputChars: 200_000,
+        minChars: 50,
+      },
     },
   },
   },