unrag 0.2.4 → 0.2.6

package/package.json

@@ -6,18 +6,24 @@
   "bin": {
     "unrag": "./dist/cli/index.js"
   },
-  "version": "0.2.4",
+  "version": "0.2.6",
   "private": false,
   "license": "Apache-2.0",
   "devDependencies": {
+    "@ai-sdk/amazon-bedrock": "^3.0.72",
+    "@ai-sdk/cohere": "^3.0.1",
+    "@ai-sdk/google": "^3.0.1",
+    "@ai-sdk/openai": "^3.0.1",
+    "@openrouter/sdk": "^0.3.10",
+    "@prisma/client": "^6.0.0",
     "@types/bun": "latest",
     "@types/pg": "^8.16.0",
-    "@prisma/client": "^6.0.0",
-    "prisma": "^6.0.0",
-    "drizzle-orm": "^0.45.1",
+    "ai": "^6.0.3",
     "drizzle-kit": "^0.31.8",
-    "ai": "^5.0.113",
-    "pg": "^8.16.3"
+    "drizzle-orm": "^0.45.1",
+    "pg": "^8.16.3",
+    "prisma": "^6.0.0",
+    "voyage-ai-provider": "^3.0.0"
   },
   "dependencies": {
     "@clack/prompts": "^0.11.0",

package/registry/config/unrag.config.ts

@@ -18,19 +18,18 @@
 export const unrag = defineUnragConfig({
   defaults: {
   chunking: {
-    chunkSize: 200,
-    chunkOverlap: 40,
+    chunkSize: 200, // __UNRAG_DEFAULT_chunkSize__
+    chunkOverlap: 40, // __UNRAG_DEFAULT_chunkOverlap__
   },
   retrieval: {
-    topK: 8,
+    topK: 8, // __UNRAG_DEFAULT_topK__
   },
   },
   embedding: {
     provider: "ai",
     config: {
-      type: "text", // __UNRAG_EMBEDDING_TYPE__
       model: "openai/text-embedding-3-small", // __UNRAG_EMBEDDING_MODEL__
-      timeoutMs: 15_000,
+      timeoutMs: 15_000, // __UNRAG_EMBEDDING_TIMEOUT__
     },
   },
   engine: {
@@ -41,8 +40,8 @@ export const unrag = defineUnragConfig({
    * - storeDocumentContent: whether the full original document text is stored in `documents.content`.
    */
   storage: {
-    storeChunkContent: true,
-    storeDocumentContent: true,
+    storeChunkContent: true, // __UNRAG_STORAGE_storeChunkContent__
+    storeDocumentContent: true, // __UNRAG_STORAGE_storeDocumentContent__
   },
     /**
      * Optional extractor modules that can process non-text assets into text outputs.
@@ -62,9 +61,10 @@ export const unrag = defineUnragConfig({
    *
    * Notes:
    * - This generated config is cost-safe by default (all extraction is off).
-   * - `unrag init` can enable rich media + multimodal embeddings for you.
+   * - `unrag init --rich-media` can enable rich media ingestion for you (extractors + assetProcessing flags).
    * - Tighten fetch allowlists/limits in production if you ingest URL-based assets.
    */
+  // __UNRAG_ASSET_PROCESSING_BLOCK_START__
   assetProcessing: {
     onUnsupportedAsset: "skip",
     onError: "skip",
@@ -181,6 +181,7 @@ export const unrag = defineUnragConfig({
       },
     },
   },
+  // __UNRAG_ASSET_PROCESSING_BLOCK_END__
   },
 } as const);
 

package/registry/connectors/google-drive/client.ts

@@ -0,0 +1,171 @@
+import type { GoogleDriveAuth } from "./types";
+
+export const DEFAULT_DRIVE_SCOPES = [
+  "https://www.googleapis.com/auth/drive.readonly",
+  "https://www.googleapis.com/auth/drive.metadata.readonly",
+] as const;
+
+type NormalizedAuth =
+  | { kind: "oauth_client"; oauthClient: unknown }
+  | {
+      kind: "oauth_config";
+      clientId: string;
+      clientSecret: string;
+      redirectUri: string;
+      refreshToken: string;
+      accessToken?: string;
+    }
+  | {
+      kind: "service_account";
+      credentials: Record<string, any>;
+      subject?: string;
+    }
+  | { kind: "google_auth"; auth: unknown };
+
+export function normalizeGoogleDriveAuth(auth: GoogleDriveAuth): NormalizedAuth {
+  if (!auth || typeof auth !== "object") {
+    throw new Error("Google Drive auth is required");
+  }
+
+  const kind = (auth as any).kind;
+  if (kind !== "oauth" && kind !== "service_account" && kind !== "google_auth") {
+    throw new Error(`Unknown Google Drive auth kind: ${String(kind)}`);
+  }
+
+  if (kind === "google_auth") {
+    const a = (auth as any).auth;
+    if (!a) throw new Error('Google Drive auth.kind="google_auth" requires auth');
+    return { kind: "google_auth", auth: a };
+  }
+
+  if (kind === "service_account") {
+    const raw = (auth as any).credentialsJson;
+    if (!raw) {
+      throw new Error(
+        'Google Drive auth.kind="service_account" requires credentialsJson'
+      );
+    }
+    const credentials =
+      typeof raw === "string" ? (JSON.parse(raw) as Record<string, any>) : (raw as any);
+    if (!credentials?.client_email || !credentials?.private_key) {
+      throw new Error(
+        'Google Drive service account credentials must include "client_email" and "private_key".'
+      );
+    }
+    return {
+      kind: "service_account",
+      credentials,
+      subject: (auth as any).subject ? String((auth as any).subject) : undefined,
+    };
+  }
+
+  // oauth
+  if ((auth as any).oauthClient) {
+    return { kind: "oauth_client", oauthClient: (auth as any).oauthClient };
+  }
+
+  const { clientId, clientSecret, redirectUri, refreshToken, accessToken } = auth as any;
+  if (!clientId || !clientSecret || !redirectUri || !refreshToken) {
+    throw new Error(
+      'Google Drive auth.kind="oauth" requires either oauthClient or { clientId, clientSecret, redirectUri, refreshToken }'
+    );
+  }
+  return {
+    kind: "oauth_config",
+    clientId: String(clientId),
+    clientSecret: String(clientSecret),
+    redirectUri: String(redirectUri),
+    refreshToken: String(refreshToken),
+    ...(accessToken ? { accessToken: String(accessToken) } : {}),
+  };
+}
+
+const asMessage = (err: unknown) => {
+  if (err instanceof Error) return err.message;
+  try {
+    return typeof err === "string" ? err : JSON.stringify(err);
+  } catch {
+    return String(err);
+  }
+};
+
+/**
+ * Creates a Google Drive API client from a plug-and-play auth input.
+ *
+ * Note: This uses dynamic imports so the core Unrag package does not require
+ * Google dependencies unless the connector is installed into a user project.
+ */
+export async function createGoogleDriveClient(args: {
+  auth: GoogleDriveAuth;
+  scopes?: string[];
+}): Promise<{ drive: any; authClient: any }> {
+  const normalized = normalizeGoogleDriveAuth(args.auth);
+  const scopes = (args.scopes?.length ? args.scopes : DEFAULT_DRIVE_SCOPES) as string[];
+
+  let authClient: any;
+
+  try {
+    if (normalized.kind === "oauth_client") {
+      authClient = normalized.oauthClient;
+    } else if (normalized.kind === "google_auth") {
+      authClient = normalized.auth;
+    } else {
+      // google-auth-library (dynamic)
+      const gal: any = await import("google-auth-library");
+
+      if (normalized.kind === "oauth_config") {
+        const OAuth2Client = gal.OAuth2Client ?? gal.OAuth2;
+        if (!OAuth2Client) {
+          throw new Error("OAuth2Client not found in google-auth-library");
+        }
+        const client = new OAuth2Client(
+          normalized.clientId,
+          normalized.clientSecret,
+          normalized.redirectUri
+        );
+        client.setCredentials({
+          refresh_token: normalized.refreshToken,
+          ...(normalized.accessToken ? { access_token: normalized.accessToken } : {}),
+        });
+        authClient = client;
+      } else {
+        const JWT = gal.JWT;
+        if (!JWT) {
+          throw new Error("JWT not found in google-auth-library");
+        }
+        const c = normalized.credentials;
+        authClient = new JWT({
+          email: c.client_email,
+          key: c.private_key,
+          scopes,
+          ...(normalized.subject ? { subject: normalized.subject } : {}),
+        });
+      }
+    }
+
+    const { google }: any = await import("googleapis");
+    if (!google?.drive) {
+      throw new Error("googleapis.google.drive not found");
+    }
+
+    const drive = google.drive({
+      version: "v3",
+      auth: authClient,
+    });
+
+    return { drive, authClient };
+  } catch (err) {
+    const msg = asMessage(err);
+    if (
+      msg.includes("Cannot find module") &&
+      (msg.includes("googleapis") || msg.includes("google-auth-library"))
+    ) {
+      throw new Error(
+        `Missing Google Drive connector dependencies. Ensure you've installed the connector via \`unrag add google-drive\` (which adds "googleapis" and "google-auth-library"). Original error: ${msg}`
+      );
+    }
+    throw err;
+  }
+}
+
+

package/registry/connectors/google-drive/index.ts

@@ -0,0 +1,10 @@
+export { createGoogleDriveClient } from "./client";
+export {
+  loadGoogleDriveFileDocument,
+  syncGoogleDriveFiles,
+  buildGoogleDriveFileIngestInput,
+} from "./sync";
+export * from "./types";
+export * from "./mime";
+
+

package/registry/connectors/google-drive/mime.ts

@@ -0,0 +1,76 @@
+import type { AssetKind } from "../../core/types";
+
+export const DRIVE_MIME = {
+  folder: "application/vnd.google-apps.folder",
+  shortcut: "application/vnd.google-apps.shortcut",
+  doc: "application/vnd.google-apps.document",
+  sheet: "application/vnd.google-apps.spreadsheet",
+  slides: "application/vnd.google-apps.presentation",
+  drawing: "application/vnd.google-apps.drawing",
+} as const;
+
+export const EXPORT_MIME = {
+  text: "text/plain",
+  csv: "text/csv",
+  pptx:
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  png: "image/png",
+} as const;
+
+export type DriveGoogleNativeKind = "doc" | "sheet" | "slides" | "drawing";
+
+export type DriveMimeClassification =
+  | { kind: "folder" }
+  | { kind: "shortcut" }
+  | { kind: "google_native"; nativeKind: DriveGoogleNativeKind }
+  | { kind: "binary" };
+
+export function classifyDriveMimeType(mimeType: string | undefined): DriveMimeClassification {
+  const mt = String(mimeType ?? "").trim();
+  if (!mt) return { kind: "binary" };
+
+  if (mt === DRIVE_MIME.folder) return { kind: "folder" };
+  if (mt === DRIVE_MIME.shortcut) return { kind: "shortcut" };
+
+  if (mt === DRIVE_MIME.doc) return { kind: "google_native", nativeKind: "doc" };
+  if (mt === DRIVE_MIME.sheet) return { kind: "google_native", nativeKind: "sheet" };
+  if (mt === DRIVE_MIME.slides) return { kind: "google_native", nativeKind: "slides" };
+  if (mt === DRIVE_MIME.drawing) return { kind: "google_native", nativeKind: "drawing" };
+
+  return { kind: "binary" };
+}
+
+export type DriveNativeExportPlan =
+  | { kind: "content"; mimeType: string }
+  | {
+      kind: "asset";
+      assetKind: AssetKind;
+      mimeType: string;
+      filenameExt?: string;
+    }
+  | { kind: "unsupported" };
+
+/**
+ * Default behavior (Notion-like): Google-native files are exported to text-ish content.
+ * Drawings are exported as PNG image assets (no good text representation).
+ */
+export function getNativeExportPlan(nativeKind: DriveGoogleNativeKind): DriveNativeExportPlan {
+  if (nativeKind === "doc") return { kind: "content", mimeType: EXPORT_MIME.text };
+  if (nativeKind === "sheet") return { kind: "content", mimeType: EXPORT_MIME.csv };
+  if (nativeKind === "slides") return { kind: "content", mimeType: EXPORT_MIME.text };
+  if (nativeKind === "drawing") {
+    return { kind: "asset", assetKind: "image", mimeType: EXPORT_MIME.png, filenameExt: "png" };
+  }
+  return { kind: "unsupported" };
+}
+
+export function assetKindFromMediaType(mediaType: string | undefined): AssetKind {
+  const mt = String(mediaType ?? "").trim().toLowerCase();
+  if (mt === "application/pdf") return "pdf";
+  if (mt.startsWith("image/")) return "image";
+  if (mt.startsWith("audio/")) return "audio";
+  if (mt.startsWith("video/")) return "video";
+  return "file";
+}
+
+