unrag 0.2.6 → 0.2.7

package/package.json

@@ -6,7 +6,7 @@
   "bin": {
     "unrag": "./dist/cli/index.js"
   },
-  "version": "0.2.6",
+  "version": "0.2.7",
   "private": false,
   "license": "Apache-2.0",
   "devDependencies": {

package/registry/connectors/google-drive/_api-types.ts

@@ -0,0 +1,60 @@
+/**
+ * Structural types for the Google Drive API.
+ *
+ * These are minimal interfaces that match the googleapis API structure,
+ * allowing the connector to work without depending on googleapis types at compile time.
+ */
+
+export interface DriveFile {
+  id?: string | null;
+  name?: string | null;
+  mimeType?: string | null;
+  size?: string | null;
+  webViewLink?: string | null;
+  modifiedTime?: string | null;
+  parents?: string[] | null;
+  shortcutDetails?: {
+    targetId?: string | null;
+    targetMimeType?: string | null;
+  } | null;
+}
+
+export interface DriveFileList {
+  files?: DriveFile[];
+  nextPageToken?: string | null;
+}
+
+export interface DriveFilesResource {
+  get(params: {
+    fileId: string;
+    fields?: string;
+    alt?: string;
+    supportsAllDrives?: boolean;
+  }): Promise<{ data: DriveFile | ArrayBuffer | string }>;
+
+  list(params: {
+    q?: string;
+    fields?: string;
+    pageToken?: string;
+    pageSize?: number;
+    supportsAllDrives?: boolean;
+    includeItemsFromAllDrives?: boolean;
+  }): Promise<{ data: DriveFileList }>;
+
+  export(params: {
+    fileId: string;
+    mimeType: string;
+  }): Promise<{ data: ArrayBuffer | string }>;
+}
+
+export interface DriveClient {
+  files: DriveFilesResource;
+}
+
+/**
+ * Auth client interface - minimal subset used by the connector.
+ */
+export interface AuthClient {
+  getAccessToken?(): Promise<{ token?: string | null }>;
+}
+

package/registry/connectors/google-drive/client.ts

@@ -1,4 +1,11 @@
-import type { GoogleDriveAuth } from "./types";
+import type { DriveClient, AuthClient } from "./_api-types";
+import type {
+  GoogleDriveAuth,
+  GoogleDriveOAuthAuth,
+  GoogleDriveServiceAccountAuth,
+  GoogleDriveGoogleAuthAuth,
+  ServiceAccountCredentials,
+} from "./types";
 
 export const DEFAULT_DRIVE_SCOPES = [
   "https://www.googleapis.com/auth/drive.readonly",
@@ -17,36 +24,51 @@ type NormalizedAuth =
     }
   | {
       kind: "service_account";
-      credentials: Record<string, any>;
+      credentials: ServiceAccountCredentials;
       subject?: string;
     }
   | { kind: "google_auth"; auth: unknown };
 
+/**
+ * Type guard for service account auth.
+ */
+function isServiceAccountAuth(auth: GoogleDriveAuth): auth is GoogleDriveServiceAccountAuth {
+  return auth.kind === "service_account";
+}
+
+/**
+ * Type guard for google auth.
+ */
+function isGoogleAuth(auth: GoogleDriveAuth): auth is GoogleDriveGoogleAuthAuth {
+  return auth.kind === "google_auth";
+}
+
+/**
+ * Type guard for oauth.
+ */
+function isOAuthAuth(auth: GoogleDriveAuth): auth is GoogleDriveOAuthAuth {
+  return auth.kind === "oauth";
+}
+
 export function normalizeGoogleDriveAuth(auth: GoogleDriveAuth): NormalizedAuth {
   if (!auth || typeof auth !== "object") {
     throw new Error("Google Drive auth is required");
   }
 
-  const kind = (auth as any).kind;
-  if (kind !== "oauth" && kind !== "service_account" && kind !== "google_auth") {
-    throw new Error(`Unknown Google Drive auth kind: ${String(kind)}`);
-  }
-
-  if (kind === "google_auth") {
-    const a = (auth as any).auth;
-    if (!a) throw new Error('Google Drive auth.kind="google_auth" requires auth');
-    return { kind: "google_auth", auth: a };
+  if (isGoogleAuth(auth)) {
+    if (!auth.auth) throw new Error('Google Drive auth.kind="google_auth" requires auth');
+    return { kind: "google_auth", auth: auth.auth };
   }
 
-  if (kind === "service_account") {
-    const raw = (auth as any).credentialsJson;
+  if (isServiceAccountAuth(auth)) {
+    const raw = auth.credentialsJson;
     if (!raw) {
       throw new Error(
         'Google Drive auth.kind="service_account" requires credentialsJson'
       );
     }
-    const credentials =
-      typeof raw === "string" ? (JSON.parse(raw) as Record<string, any>) : (raw as any);
+    const credentials: ServiceAccountCredentials =
+      typeof raw === "string" ? (JSON.parse(raw) as ServiceAccountCredentials) : raw;
     if (!credentials?.client_email || !credentials?.private_key) {
       throw new Error(
         'Google Drive service account credentials must include "client_email" and "private_key".'
@@ -55,29 +77,33 @@ export function normalizeGoogleDriveAuth(auth: GoogleDriveAuth): NormalizedAuth
     return {
       kind: "service_account",
       credentials,
-      subject: (auth as any).subject ? String((auth as any).subject) : undefined,
+      subject: auth.subject ? String(auth.subject) : undefined,
     };
   }
 
-  // oauth
-  if ((auth as any).oauthClient) {
-    return { kind: "oauth_client", oauthClient: (auth as any).oauthClient };
-  }
+  if (isOAuthAuth(auth)) {
+    // oauth
+    if (auth.oauthClient) {
+      return { kind: "oauth_client", oauthClient: auth.oauthClient };
+    }
 
-  const { clientId, clientSecret, redirectUri, refreshToken, accessToken } = auth as any;
-  if (!clientId || !clientSecret || !redirectUri || !refreshToken) {
-    throw new Error(
-      'Google Drive auth.kind="oauth" requires either oauthClient or { clientId, clientSecret, redirectUri, refreshToken }'
-    );
+    const { clientId, clientSecret, redirectUri, refreshToken, accessToken } = auth;
+    if (!clientId || !clientSecret || !redirectUri || !refreshToken) {
+      throw new Error(
+        'Google Drive auth.kind="oauth" requires either oauthClient or { clientId, clientSecret, redirectUri, refreshToken }'
+      );
+    }
+    return {
+      kind: "oauth_config",
+      clientId: String(clientId),
+      clientSecret: String(clientSecret),
+      redirectUri: String(redirectUri),
+      refreshToken: String(refreshToken),
+      ...(accessToken ? { accessToken: String(accessToken) } : {}),
+    };
   }
-  return {
-    kind: "oauth_config",
-    clientId: String(clientId),
-    clientSecret: String(clientSecret),
-    redirectUri: String(redirectUri),
-    refreshToken: String(refreshToken),
-    ...(accessToken ? { accessToken: String(accessToken) } : {}),
-  };
+
+  throw new Error(`Unknown Google Drive auth kind: ${String((auth as Record<string, unknown>).kind)}`);
 }
 
 const asMessage = (err: unknown) => {
@@ -89,6 +115,41 @@ const asMessage = (err: unknown) => {
   }
 };
 
+/**
+ * Google Auth Library module shape for dynamic import.
+ */
+interface GoogleAuthLibraryModule {
+  OAuth2Client?: new (
+    clientId: string,
+    clientSecret: string,
+    redirectUri: string
+  ) => {
+    setCredentials(credentials: Record<string, string>): void;
+  };
+  OAuth2?: new (
+    clientId: string,
+    clientSecret: string,
+    redirectUri: string
+  ) => {
+    setCredentials(credentials: Record<string, string>): void;
+  };
+  JWT?: new (options: {
+    email: string;
+    key: string;
+    scopes: string[];
+    subject?: string;
+  }) => unknown;
+}
+
+/**
+ * Googleapis module shape for dynamic import.
+ */
+interface GoogleApisModule {
+  google: {
+    drive(options: { version: string; auth: unknown }): DriveClient;
+  };
+}
+
 /**
  * Creates a Google Drive API client from a plug-and-play auth input.
  *
@@ -98,11 +159,11 @@ const asMessage = (err: unknown) => {
 export async function createGoogleDriveClient(args: {
   auth: GoogleDriveAuth;
   scopes?: string[];
-}): Promise<{ drive: any; authClient: any }> {
+}): Promise<{ drive: DriveClient; authClient: AuthClient }> {
   const normalized = normalizeGoogleDriveAuth(args.auth);
   const scopes = (args.scopes?.length ? args.scopes : DEFAULT_DRIVE_SCOPES) as string[];
 
-  let authClient: any;
+  let authClient: unknown;
 
   try {
     if (normalized.kind === "oauth_client") {
@@ -111,7 +172,7 @@ export async function createGoogleDriveClient(args: {
       authClient = normalized.auth;
     } else {
       // google-auth-library (dynamic)
-      const gal: any = await import("google-auth-library");
+      const gal = (await import("google-auth-library")) as GoogleAuthLibraryModule;
 
       if (normalized.kind === "oauth_config") {
         const OAuth2Client = gal.OAuth2Client ?? gal.OAuth2;
@@ -143,7 +204,7 @@ export async function createGoogleDriveClient(args: {
       }
     }
 
-    const { google }: any = await import("googleapis");
+    const { google } = (await import("googleapis")) as GoogleApisModule;
     if (!google?.drive) {
       throw new Error("googleapis.google.drive not found");
     }
@@ -153,7 +214,7 @@ export async function createGoogleDriveClient(args: {
       auth: authClient,
     });
 
-    return { drive, authClient };
+    return { drive, authClient: authClient as AuthClient };
   } catch (err) {
     const msg = asMessage(err);
     if (

package/registry/connectors/google-drive/sync.ts

@@ -1,5 +1,6 @@
-import type { IngestResult } from "../../core";
+import type { IngestResult, Metadata } from "../../core";
 import type { AssetInput } from "../../core/types";
+import type { DriveClient, DriveFile } from "./_api-types";
 import { createGoogleDriveClient } from "./client";
 import {
   assetKindFromMediaType,
@@ -17,6 +18,25 @@ import type {
 
 const DEFAULT_MAX_BYTES = 15 * 1024 * 1024; // 15MB
 
+/**
+ * Internal metadata type for Google Drive documents.
+ */
+interface GoogleDriveMetadata extends Metadata {
+  connector: "google-drive";
+  kind: "file" | "folder" | "shortcut";
+  fileId: string;
+  name?: string;
+  mimeType?: string;
+  size?: number;
+  googleNativeKind?: string;
+  unsupportedGoogleMime?: boolean;
+  skippedTooLarge?: boolean;
+  exportedTooLarge?: boolean;
+  shortcutUnresolved?: boolean;
+  exportMimeType?: string;
+  exportFallback?: string;
+}
+
 const joinPrefix = (prefix: string | undefined, rest: string) => {
   const p = (prefix ?? "").trim();
   if (!p) return rest;
@@ -44,10 +64,10 @@ const asMessage = (err: unknown) => {
   }
 };
 
-const toUint8Array = (data: any): Uint8Array => {
+const toUint8Array = (data: unknown): Uint8Array => {
   if (!data) return new Uint8Array();
   if (data instanceof Uint8Array) return data;
-  if (typeof Buffer !== "undefined" && data instanceof Buffer) {
+  if (typeof Buffer !== "undefined" && Buffer.isBuffer(data)) {
     return new Uint8Array(data);
   }
   if (data instanceof ArrayBuffer) return new Uint8Array(data);
@@ -61,51 +81,54 @@ const toUint8Array = (data: any): Uint8Array => {
   return new Uint8Array();
 };
 
-const bytesToText = (bytes: Uint8Array) => {
+const bytesToText = (bytes: Uint8Array): string => {
   return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
 };
 
-const isNotFound = (err: any, treatForbiddenAsNotFound: boolean) => {
+const isNotFound = (err: unknown, treatForbiddenAsNotFound: boolean): boolean => {
+  if (typeof err !== "object" || err === null) return false;
+  const e = err as Record<string, unknown>;
+  const response = e.response as Record<string, unknown> | undefined;
   const status =
-    Number(err?.code ?? err?.status ?? err?.response?.status ?? err?.statusCode) ||
-    Number(err?.response?.status);
+    Number(e.code ?? e.status ?? response?.status ?? e.statusCode ?? 0);
   if (status === 404) return true;
   if (treatForbiddenAsNotFound && status === 403) return true;
   return false;
 };
 
-async function getFileMetadata(drive: any, fileId: string) {
+async function getFileMetadata(drive: DriveClient, fileId: string): Promise<DriveFile> {
   const res = await drive.files.get({
     fileId,
     supportsAllDrives: true,
     fields:
       "id,name,mimeType,size,md5Checksum,modifiedTime,webViewLink,webContentLink,iconLink,shortcutDetails,driveId",
   });
-  return res?.data ?? {};
+  return (res?.data ?? {}) as DriveFile;
 }
 
-async function downloadFileBytes(drive: any, fileId: string): Promise<Uint8Array> {
-  const res = await drive.files.get(
-    { fileId, alt: "media", supportsAllDrives: true },
-    { responseType: "arraybuffer" }
-  );
+async function downloadFileBytes(drive: DriveClient, fileId: string): Promise<Uint8Array> {
+  const res = await drive.files.get({
+    fileId,
+    alt: "media",
+    supportsAllDrives: true,
+  });
   return toUint8Array(res?.data);
 }
 
 async function exportFileBytes(
-  drive: any,
+  drive: DriveClient,
   fileId: string,
   mimeType: string
 ): Promise<Uint8Array> {
-  const res = await drive.files.export(
-    { fileId, mimeType },
-    { responseType: "arraybuffer" }
-  );
+  const res = await drive.files.export({
+    fileId,
+    mimeType,
+  });
   return toUint8Array(res?.data);
 }
 
 export async function loadGoogleDriveFileDocument(args: {
-  drive: any;
+  drive: DriveClient;
   fileId: string;
   sourceIdPrefix?: string;
   options?: {
@@ -128,21 +151,22 @@ export async function loadGoogleDriveFileDocument(args: {
 
   // Handle folders: return a document shape but with no content/assets; callers typically skip.
   if (classification.kind === "folder") {
+    const folderMetadata: GoogleDriveMetadata = {
+      connector: "google-drive",
+      kind: "folder",
+      fileId,
+      name,
+      mimeType: DRIVE_MIME.folder,
+      ...(meta?.webViewLink ? { webViewLink: String(meta.webViewLink) } : {}),
+      ...(meta?.modifiedTime ? { modifiedTime: String(meta.modifiedTime) } : {}),
+    };
     return buildGoogleDriveFileIngestInput({
       fileId,
       sourceIdPrefix: args.sourceIdPrefix,
       content: "",
       assets: [],
-      metadata: {
-        connector: "google-drive",
-        kind: "folder",
-        fileId,
-        name,
-        mimeType: DRIVE_MIME.folder,
-        ...(meta?.webViewLink ? { webViewLink: String(meta.webViewLink) } : {}),
-        ...(meta?.modifiedTime ? { modifiedTime: String(meta.modifiedTime) } : {}),
-      },
-    }) as any;
+      metadata: folderMetadata,
+    });
   }
 
   // Shortcuts: resolve to target if possible (1-level), otherwise let caller decide.
@@ -150,20 +174,21 @@ export async function loadGoogleDriveFileDocument(args: {
     const visited = args._visited ?? new Set<string>();
     if (visited.has(fileId)) {
       // cycle
+      const cycleMetadata: GoogleDriveMetadata = {
+        connector: "google-drive",
+        kind: "shortcut",
+        fileId,
+        name,
+        mimeType: DRIVE_MIME.shortcut,
+        shortcutUnresolved: true,
+      };
       return buildGoogleDriveFileIngestInput({
         fileId,
         sourceIdPrefix: args.sourceIdPrefix,
         content: "",
         assets: [],
-        metadata: {
-          connector: "google-drive",
-          kind: "shortcut",
-          fileId,
-          name,
-          mimeType: DRIVE_MIME.shortcut,
-          shortcutUnresolved: true,
-        },
-      }) as any;
+        metadata: cycleMetadata,
+      });
     }
     visited.add(fileId);
 
@@ -172,20 +197,21 @@ export async function loadGoogleDriveFileDocument(args: {
       : "";
 
     if (!targetId) {
+      const unresolvedMetadata: GoogleDriveMetadata = {
+        connector: "google-drive",
+        kind: "shortcut",
+        fileId,
+        name,
+        mimeType: DRIVE_MIME.shortcut,
+        shortcutUnresolved: true,
+      };
       return buildGoogleDriveFileIngestInput({
         fileId,
         sourceIdPrefix: args.sourceIdPrefix,
         content: "",
         assets: [],
-        metadata: {
-          connector: "google-drive",
-          kind: "shortcut",
-          fileId,
-          name,
-          mimeType: DRIVE_MIME.shortcut,
-          shortcutUnresolved: true,
-        },
-      }) as any;
+        metadata: unresolvedMetadata,
+      });
     }
 
     // Resolve target content/assets but keep sourceId stable to the shortcut file id.
@@ -209,7 +235,7 @@ export async function loadGoogleDriveFileDocument(args: {
     };
   }
 
-  const baseMetadata = {
+  const baseMetadata: Record<string, unknown> = {
     connector: "google-drive",
     kind: "file",
     fileId,
@@ -222,7 +248,7 @@ export async function loadGoogleDriveFileDocument(args: {
     ...(meta?.webContentLink ? { webContentLink: String(meta.webContentLink) } : {}),
     ...(meta?.iconLink ? { iconLink: String(meta.iconLink) } : {}),
     ...(meta?.driveId ? { driveId: String(meta.driveId) } : {}),
-  } as const;
+  };
 
   // Google-native export path
   if (classification.kind === "google_native") {
@@ -238,7 +264,7 @@ export async function loadGoogleDriveFileDocument(args: {
           googleNativeKind: classification.nativeKind,
           unsupportedGoogleMime: true,
         },
-      }) as any;
+      });
     }
 
     // For content export, enforce maxBytesPerFile by bytes length.
@@ -252,7 +278,7 @@ export async function loadGoogleDriveFileDocument(args: {
             content: "",
             assets: [],
             metadata: { ...baseMetadata, exportedTooLarge: true },
-          }) as any;
+          });
         }
         const content = bytesToText(bytes).trim();
         return buildGoogleDriveFileIngestInput({
@@ -261,7 +287,7 @@ export async function loadGoogleDriveFileDocument(args: {
           content,
           assets: [],
           metadata: { ...baseMetadata, googleNativeKind: classification.nativeKind, exportMimeType: plan.mimeType },
-        }) as any;
+        });
       } catch (err) {
         // Slides can fail to export as text; fallback to PPTX unless strict.
         if (classification.nativeKind === "slides" && !strictNativeExport) {
@@ -274,7 +300,7 @@ export async function loadGoogleDriveFileDocument(args: {
                 content: "",
                 assets: [],
                 metadata: { ...baseMetadata, exportedTooLarge: true },
-              }) as any;
+              });
             }
             const asset: AssetInput = {
               assetId: fileId,
@@ -286,7 +312,7 @@ export async function loadGoogleDriveFileDocument(args: {
                 filename: name ? `${name}.pptx` : undefined,
               },
               uri: meta?.webViewLink ? String(meta.webViewLink) : undefined,
-              metadata: { connector: "google-drive", fileId, exportMimeType: EXPORT_MIME.pptx } as any,
+              metadata: { connector: "google-drive", fileId, exportMimeType: EXPORT_MIME.pptx },
             };
             return buildGoogleDriveFileIngestInput({
               fileId,
@@ -294,7 +320,7 @@ export async function loadGoogleDriveFileDocument(args: {
               content: "",
               assets: [asset],
               metadata: { ...baseMetadata, googleNativeKind: "slides", exportFallback: "pptx" },
-            }) as any;
+            });
           } catch {
             // fall through to strict error
           }
@@ -314,7 +340,7 @@ export async function loadGoogleDriveFileDocument(args: {
           content: "",
           assets: [],
           metadata: { ...baseMetadata, exportedTooLarge: true },
-        }) as any;
+        });
       }
 
       const filename = name && plan.filenameExt ? `${name}.${plan.filenameExt}` : name || undefined;
@@ -323,7 +349,7 @@ export async function loadGoogleDriveFileDocument(args: {
         kind: plan.assetKind,
         data: { kind: "bytes", bytes, mediaType: plan.mimeType, ...(filename ? { filename } : {}) },
         uri: meta?.webViewLink ? String(meta.webViewLink) : undefined,
-        metadata: { connector: "google-drive", fileId, exportMimeType: plan.mimeType } as any,
+        metadata: { connector: "google-drive", fileId, exportMimeType: plan.mimeType },
       };
 
       return buildGoogleDriveFileIngestInput({
@@ -332,7 +358,7 @@ export async function loadGoogleDriveFileDocument(args: {
         content: "",
         assets: [asset],
         metadata: { ...baseMetadata, googleNativeKind: classification.nativeKind, exportMimeType: plan.mimeType },
-      }) as any;
+      });
     }
   }
 
@@ -344,7 +370,7 @@ export async function loadGoogleDriveFileDocument(args: {
       content: "",
       assets: [],
       metadata: { ...baseMetadata, skippedTooLarge: true },
-    }) as any;
+    });
   }
 
   const bytes = await downloadFileBytes(args.drive, fileId);
@@ -355,7 +381,7 @@ export async function loadGoogleDriveFileDocument(args: {
       content: "",
       assets: [],
       metadata: { ...baseMetadata, skippedTooLarge: true },
-    }) as any;
+    });
   }
 
   const assetKind = assetKindFromMediaType(mimeType);
@@ -370,7 +396,7 @@ export async function loadGoogleDriveFileDocument(args: {
       ...(filename ? { filename } : {}),
     },
     uri: meta?.webViewLink ? String(meta.webViewLink) : undefined,
-    metadata: { connector: "google-drive", fileId, name, mimeType } as any,
+    metadata: { connector: "google-drive", fileId, name, mimeType },
   };
 
   // For pure binaries, keep content empty; extraction occurs via engine asset processing + extractors.
@@ -379,8 +405,8 @@ export async function loadGoogleDriveFileDocument(args: {
     sourceIdPrefix: args.sourceIdPrefix,
     content: "",
     assets: [asset],
-    metadata: baseMetadata as any,
-  }) as any;
+    metadata: baseMetadata,
+  });
 }
 
 export async function syncGoogleDriveFiles(
@@ -434,8 +460,10 @@ export async function syncGoogleDriveFiles(
         },
       });
 
+      const meta = doc.metadata as Record<string, unknown>;
+
       // Skip folders explicitly (v1).
-      if ((doc.metadata as any)?.kind === "folder") {
+      if (meta.kind === "folder") {
         emit({
           type: "file:skipped",
           fileId,
@@ -446,7 +474,7 @@ export async function syncGoogleDriveFiles(
         continue;
       }
 
-      if ((doc.metadata as any)?.unsupportedGoogleMime) {
+      if (meta.unsupportedGoogleMime) {
         emit({
           type: "file:skipped",
           fileId,
@@ -458,7 +486,7 @@ export async function syncGoogleDriveFiles(
         continue;
       }
 
-      if ((doc.metadata as any)?.skippedTooLarge || (doc.metadata as any)?.exportedTooLarge) {
+      if (meta.skippedTooLarge || meta.exportedTooLarge) {
         emit({
           type: "file:skipped",
           fileId,
@@ -469,7 +497,7 @@ export async function syncGoogleDriveFiles(
         continue;
       }
 
-      if ((doc.metadata as any)?.shortcutUnresolved) {
+      if (meta.shortcutUnresolved) {
         emit({
           type: "file:skipped",
           fileId,
@@ -484,7 +512,7 @@ export async function syncGoogleDriveFiles(
         sourceId: doc.sourceId,
         content: doc.content,
         assets: doc.assets,
-        metadata: doc.metadata as any,
+        metadata: doc.metadata,
       });
 
       succeeded += 1;