unrag 0.2.12 → 0.3.1

package/dist/cli/index.js

@@ -720,6 +720,10 @@ async function copyRegistryFiles(selection) {
       src: path5.join(selection.registryRoot, "core/types.ts"),
       dest: path5.join(installBaseAbs, "core/types.ts")
     },
+    {
+      src: path5.join(selection.registryRoot, "core/connectors.ts"),
+      dest: path5.join(installBaseAbs, "core/connectors.ts")
+    },
     {
       src: path5.join(selection.registryRoot, "core/chunking.ts"),
       dest: path5.join(installBaseAbs, "core/chunking.ts")
@@ -870,7 +874,7 @@ async function copyRegistryFiles(selection) {
         });
         if (isCancel(answer)) {
           cancel("Cancelled.");
-          return;
+          return [];
         }
         if (!answer) {
           continue;
@@ -912,7 +916,7 @@ async function copyConnectorFiles(selection) {
       });
       if (isCancel(answer)) {
         cancel("Cancelled.");
-        return;
+        return [];
       }
       if (!answer) {
         continue;
@@ -3815,6 +3819,7 @@ async function runInstallChecks(state) {
     const coreExists = await exists(path12.join(installDirFull, "core"));
     const storeExists = await exists(path12.join(installDirFull, "store"));
     const embeddingExists = await exists(path12.join(installDirFull, "embedding"));
+    const coreConnectorsExists = coreExists && await exists(path12.join(installDirFull, "core", "connectors.ts"));
     const unragMdExists = await exists(path12.join(installDirFull, "unrag.md"));
     const missingDirs = [];
     if (!coreExists) {
@@ -3826,7 +3831,11 @@ async function runInstallChecks(state) {
     if (!embeddingExists) {
       missingDirs.push("embedding/");
     }
-    if (state.installDirExists && missingDirs.length === 0) {
+    const missingCoreFiles = [];
+    if (!coreConnectorsExists) {
+      missingCoreFiles.push("core/connectors.ts");
+    }
+    if (state.installDirExists && missingDirs.length === 0 && missingCoreFiles.length === 0) {
       results.push({
         id: "install-dir",
         title: "Install directory",
@@ -3834,18 +3843,23 @@ async function runInstallChecks(state) {
         summary: `Install directory found at ${state.installDir}`,
         details: [
           `core/: ${coreExists ? "✓" : "✗"}`,
+          `core/connectors.ts: ${coreConnectorsExists ? "✓" : "✗"}`,
           `store/: ${storeExists ? "✓" : "✗"}`,
           `embedding/: ${embeddingExists ? "✓" : "✗"}`,
           `unrag.md: ${unragMdExists ? "✓" : "✗"}`
         ]
       });
     } else if (state.installDirExists) {
+      const missing = [
+        ...missingDirs,
+        ...missingCoreFiles
+      ].filter(Boolean);
       results.push({
         id: "install-dir",
         title: "Install directory",
         status: "warn",
         summary: "Install directory exists but is incomplete.",
-        details: [`Missing: ${missingDirs.join(", ")}`],
+        details: [`Missing: ${missing.join(", ")}`],
         fixHints: ["Run `unrag init` to reinstall missing files"]
       });
     } else {
@@ -4052,6 +4066,18 @@ async function checkConnectorEnvVars(connector) {
         "GOOGLE_CLIENT_ID",
         "GOOGLE_CLIENT_SECRET"
       ]
+    },
+    onedrive: {
+      required: [],
+      optional: [
+        "AZURE_TENANT_ID",
+        "AZURE_CLIENT_ID",
+        "AZURE_CLIENT_SECRET"
+      ]
+    },
+    dropbox: {
+      required: [],
+      optional: ["DROPBOX_CLIENT_ID", "DROPBOX_CLIENT_SECRET"]
     }
   };
   const envVars = connectorEnvVars[connector];
@@ -6144,7 +6170,7 @@ var __dirname4 = path18.dirname(__filename4);
 var formatDepChanges3 = (changes) => changes.map((c) => `${c.name}${c.action === "update" ? " (update)" : ""}`).join(", ");
 var isEmbeddingProviderName = (v) => v === "ai" || v === "openai" || v === "google" || v === "openrouter" || v === "azure" || v === "vertex" || v === "bedrock" || v === "cohere" || v === "mistral" || v === "together" || v === "ollama" || v === "voyage";
 var isExtractorName = (v) => v === "pdf-llm" || v === "pdf-text-layer" || v === "pdf-ocr" || v === "image-ocr" || v === "image-caption-llm" || v === "audio-transcribe" || v === "video-transcribe" || v === "video-frames" || v === "file-text" || v === "file-docx" || v === "file-pptx" || v === "file-xlsx";
-var isConnectorName = (v) => v === "notion" || v === "google-drive";
+var isConnectorName = (v) => v === "notion" || v === "google-drive" || v === "onedrive" || v === "dropbox";
 var isBatteryName = (v) => v === "reranker" || v === "eval" || v === "debug";
 var toStringList = (xs) => Array.isArray(xs) ? xs.map((x) => String(x).trim()).filter(Boolean) : [];
 var toExtractorNames = (xs) => toStringList(xs).filter(isExtractorName);

package/package.json

@@ -6,7 +6,7 @@
 	"bin": {
 		"unrag": "./dist/cli/index.js"
 	},
-	"version": "0.2.12",
+	"version": "0.3.1",
 	"private": false,
 	"license": "Apache-2.0",
 	"devDependencies": {
@@ -41,6 +41,7 @@
 	"scripts": {
 		"build": "bun build --target=node --format=esm --packages=external --outfile dist/cli/index.js cli/index.ts && bun build --target=node --format=esm --production --outfile dist/debug-tui/index.js registry/debug/tui/App.tsx",
 		"test": "bun test",
+		"types:check": "tsc -p tsconfig.json --noEmit",
 		"lint": "biome check .",
 		"unrag:doctor": "unrag doctor --config .unrag/doctor.json",
 		"unrag:doctor:db": "unrag doctor --config .unrag/doctor.json --db",

package/registry/connectors/dropbox/client.ts

@@ -0,0 +1,110 @@
+import type {
+	DropboxAccessTokenAuth,
+	DropboxAuth,
+	DropboxRefreshTokenAuth
+} from '@registry/connectors/dropbox/types'
+
+const API_BASE = 'https://api.dropboxapi.com/2'
+const CONTENT_BASE = 'https://content.dropboxapi.com/2'
+const TOKEN_URL = 'https://api.dropboxapi.com/oauth2/token'
+
+const isAccessToken = (auth: DropboxAuth): auth is DropboxAccessTokenAuth =>
+	auth.kind === 'access_token'
+
+const isRefreshToken = (auth: DropboxAuth): auth is DropboxRefreshTokenAuth =>
+	auth.kind === 'oauth_refresh_token'
+
+const requestToken = async (params: Record<string, string>) => {
+	const res = await fetch(TOKEN_URL, {
+		method: 'POST',
+		headers: {'Content-Type': 'application/x-www-form-urlencoded'},
+		body: new URLSearchParams(params)
+	})
+	if (!res.ok) {
+		throw new Error(
+			`Dropbox token request failed (${res.status}): ${await res.text()}`
+		)
+	}
+	const data = (await res.json()) as Record<string, unknown>
+	const accessToken = String(data.access_token ?? '')
+	if (!accessToken) {
+		throw new Error('Dropbox token response missing access_token')
+	}
+	return accessToken
+}
+
+export const getDropboxAccessToken = async (
+	auth: DropboxAuth
+): Promise<string> => {
+	if (!auth || typeof auth !== 'object') {
+		throw new Error('Dropbox auth is required')
+	}
+	if (isAccessToken(auth)) {
+		if (!auth.accessToken) {
+			throw new Error('Dropbox access token is required')
+		}
+		return auth.accessToken
+	}
+	if (isRefreshToken(auth)) {
+		if (!auth.clientId || !auth.clientSecret || !auth.refreshToken) {
+			throw new Error(
+				'Dropbox refresh token auth requires clientId, clientSecret, and refreshToken'
+			)
+		}
+		return requestToken({
+			grant_type: 'refresh_token',
+			client_id: auth.clientId,
+			client_secret: auth.clientSecret,
+			refresh_token: auth.refreshToken
+		})
+	}
+	const kind =
+		typeof auth === 'object' && auth !== null
+			? (auth as {kind?: unknown}).kind
+			: undefined
+	throw new Error(`Unknown Dropbox auth kind: ${String(kind)}`)
+}
+
+export const dropboxApiFetch = async <T>(args: {
+	auth: DropboxAuth
+	path: string
+	body?: unknown
+}) => {
+	const token = await getDropboxAccessToken(args.auth)
+	const res = await fetch(`${API_BASE}/${args.path.replace(/^\//, '')}`, {
+		method: 'POST',
+		headers: {
+			Authorization: `Bearer ${token}`,
+			'Content-Type': 'application/json'
+		},
+		body: JSON.stringify(args.body ?? {})
+	})
+	if (!res.ok) {
+		const text = await res.text()
+		throw new Error(`Dropbox API failed (${res.status}): ${text}`)
+	}
+	return (await res.json()) as T
+}
+
+export const dropboxDownload = async (args: {
+	auth: DropboxAuth
+	path: string
+}) => {
+	const token = await getDropboxAccessToken(args.auth)
+	const res = await fetch(`${CONTENT_BASE}/files/download`, {
+		method: 'POST',
+		headers: {
+			Authorization: `Bearer ${token}`,
+			'Dropbox-API-Arg': JSON.stringify({path: args.path})
+		}
+	})
+	if (!res.ok) {
+		throw new Error(
+			`Dropbox download failed (${res.status}): ${await res.text()}`
+		)
+	}
+	return {
+		bytes: new Uint8Array(await res.arrayBuffer()),
+		contentType: res.headers.get('content-type') ?? undefined
+	}
+}

package/registry/connectors/dropbox/index.ts

@@ -0,0 +1,6 @@
+export {
+	dropboxConnector,
+	streamFolder,
+	streamFiles
+} from '@registry/connectors/dropbox/sync'
+export * from '@registry/connectors/dropbox/types'

package/registry/connectors/dropbox/sync.ts

@@ -0,0 +1,429 @@
+import {
+	dropboxApiFetch,
+	dropboxDownload
+} from '@registry/connectors/dropbox/client'
+import type {
+	DropboxAuth,
+	DropboxCheckpoint,
+	StreamDropboxFilesInput,
+	StreamDropboxFolderInput
+} from '@registry/connectors/dropbox/types'
+import type {ConnectorStream} from '@registry/core/connectors'
+import type {AssetInput, Metadata} from '@registry/core/types'
+
+const DEFAULT_MAX_BYTES = 15 * 1024 * 1024 // 15MB
+
+const joinPrefix = (prefix: string | undefined, rest: string) => {
+	const p = (prefix ?? '').trim()
+	if (!p) {
+		return rest
+	}
+	return p.endsWith(':') ? p + rest : `${p}:${rest}`
+}
+
+const buildSourceId = (prefix: string | undefined, pathLower: string) =>
+	joinPrefix(prefix, `dropbox:path:${pathLower}`)
+
+const asMessage = (err: unknown): string => {
+	if (err instanceof Error) {
+		return err.message
+	}
+	try {
+		return typeof err === 'string' ? err : JSON.stringify(err)
+	} catch {
+		return String(err)
+	}
+}
+
+const isTextLike = (mediaType: string | undefined) => {
+	const mt = String(mediaType ?? '')
+		.trim()
+		.toLowerCase()
+	if (!mt) {
+		return false
+	}
+	return (
+		mt.startsWith('text/') ||
+		mt === 'application/json' ||
+		mt === 'application/xml' ||
+		mt === 'text/csv'
+	)
+}
+
+const assetKindFromMediaType = (
+	mediaType: string | undefined
+): AssetInput['kind'] => {
+	const mt = String(mediaType ?? '')
+		.trim()
+		.toLowerCase()
+	if (mt === 'application/pdf') {
+		return 'pdf'
+	}
+	if (mt.startsWith('image/')) {
+		return 'image'
+	}
+	if (mt.startsWith('audio/')) {
+		return 'audio'
+	}
+	if (mt.startsWith('video/')) {
+		return 'video'
+	}
+	return 'file'
+}
+
+type DropboxEntry = {
+	'.tag': 'file' | 'folder' | 'deleted'
+	id?: string
+	name?: string
+	path_lower?: string
+	path_display?: string
+	size?: number
+	client_modified?: string
+	server_modified?: string
+	content_hash?: string
+}
+
+type DropboxListFolderResponse = {
+	entries?: DropboxEntry[]
+	cursor?: string
+	has_more?: boolean
+}
+
+type DropboxMetadataResponse = DropboxEntry
+
+const resolvePath = (entry: DropboxEntry) =>
+	String(entry.path_lower ?? entry.path_display ?? '').trim()
+
+const downloadFile = async (args: {
+	auth: DropboxAuth
+	path: string
+	maxBytesPerFile: number
+}) => {
+	const {bytes, contentType} = await dropboxDownload({
+		auth: args.auth,
+		path: args.path
+	})
+	if (bytes.byteLength > args.maxBytesPerFile) {
+		return {skipped: true as const, reason: 'too_large'}
+	}
+	if (isTextLike(contentType)) {
+		const content = new TextDecoder('utf-8', {fatal: false}).decode(bytes)
+		return {skipped: false as const, content, assets: []}
+	}
+	const asset: AssetInput = {
+		assetId: args.path,
+		kind: assetKindFromMediaType(contentType),
+		data: {
+			kind: 'bytes',
+			bytes,
+			mediaType: contentType || 'application/octet-stream',
+			filename: args.path.split('/').pop()
+		}
+	}
+	return {skipped: false as const, content: '', assets: [asset]}
+}
+
+const buildMetadata = (entry: DropboxEntry): Metadata => ({
+	connector: 'dropbox',
+	kind: 'file',
+	fileId: entry.id ?? '',
+	name: entry.name ?? '',
+	path: entry.path_lower ?? entry.path_display ?? '',
+	size: entry.size,
+	clientModified: entry.client_modified,
+	serverModified: entry.server_modified,
+	contentHash: entry.content_hash
+})
+
+const isNotFound = (err: unknown): boolean => {
+	const message =
+		typeof err === 'object' && err !== null
+			? String((err as {message?: unknown}).message ?? '')
+			: ''
+	return (
+		message.toLowerCase().includes('not_found') || message.includes('404')
+	)
+}
+
+/**
+ * Stream Dropbox folder changes (cursor-based).
+ */
+export async function* streamFolder(
+	input: StreamDropboxFolderInput
+): ConnectorStream<DropboxCheckpoint> {
+	const options = input.options ?? {}
+	const maxBytesPerFile = options.maxBytesPerFile ?? DEFAULT_MAX_BYTES
+	const deleteOnRemoved = options.deleteOnRemoved ?? false
+	const recursive = options.recursive ?? true
+	const folderPath = String(input.folderPath ?? '').trim()
+	if (!folderPath) {
+		throw new Error('Dropbox folderPath is required')
+	}
+
+	let cursor = input.checkpoint?.cursor
+	let processed = 0
+
+	while (true) {
+		const data = cursor
+			? await dropboxApiFetch<DropboxListFolderResponse>({
+					auth: input.auth,
+					path: 'files/list_folder/continue',
+					body: {cursor}
+				})
+			: await dropboxApiFetch<DropboxListFolderResponse>({
+					auth: input.auth,
+					path: 'files/list_folder',
+					body: {
+						path: folderPath,
+						recursive,
+						include_deleted: true
+					}
+				})
+
+		const entries = data.entries ?? []
+		for (const entry of entries) {
+			processed += 1
+			const entryPath = resolvePath(entry)
+			const sourceId = entryPath
+				? buildSourceId(input.sourceIdPrefix, entryPath)
+				: ''
+
+			if (entry['.tag'] === 'deleted') {
+				if (deleteOnRemoved && sourceId) {
+					yield {type: 'delete', input: {sourceId}}
+				}
+				continue
+			}
+			if (entry['.tag'] === 'folder') {
+				continue
+			}
+			if (!entryPath) {
+				continue
+			}
+
+			yield {
+				type: 'progress',
+				message: 'file:start',
+				current: processed,
+				sourceId,
+				entityId: entry.id ?? entryPath
+			}
+
+			if (
+				Number.isFinite(entry.size) &&
+				(entry.size as number) > maxBytesPerFile
+			) {
+				yield {
+					type: 'warning',
+					code: 'file_skipped',
+					message: `Skipping file because it exceeds maxBytesPerFile (${maxBytesPerFile}).`,
+					data: {path: entryPath, sourceId, reason: 'too_large'}
+				}
+				continue
+			}
+
+			try {
+				const payload = await downloadFile({
+					auth: input.auth,
+					path: entryPath,
+					maxBytesPerFile
+				})
+				if (payload.skipped) {
+					yield {
+						type: 'warning',
+						code: 'file_skipped',
+						message: `Skipping file because it exceeds maxBytesPerFile (${maxBytesPerFile}).`,
+						data: {path: entryPath, sourceId, reason: 'too_large'}
+					}
+					continue
+				}
+
+				yield {
+					type: 'upsert',
+					input: {
+						sourceId,
+						content: payload.content,
+						assets: payload.assets,
+						metadata: buildMetadata(entry)
+					}
+				}
+
+				yield {
+					type: 'progress',
+					message: 'file:success',
+					current: processed,
+					sourceId,
+					entityId: entry.id ?? entryPath
+				}
+			} catch (err) {
+				if (isNotFound(err)) {
+					yield {
+						type: 'warning',
+						code: 'file_not_found',
+						message: 'Dropbox file not found or inaccessible.',
+						data: {path: entryPath, sourceId}
+					}
+					if (deleteOnRemoved) {
+						yield {type: 'delete', input: {sourceId}}
+					}
+				} else {
+					yield {
+						type: 'warning',
+						code: 'file_error',
+						message: asMessage(err),
+						data: {path: entryPath, sourceId}
+					}
+				}
+			}
+		}
+
+		if (data.cursor) {
+			yield {
+				type: 'checkpoint',
+				checkpoint: {
+					cursor: data.cursor,
+					path: folderPath
+				}
+			}
+		}
+
+		cursor = data.cursor ?? cursor
+		if (!data.has_more) {
+			break
+		}
+	}
+}
+
+/**
+ * Stream explicit Dropbox file IDs.
+ */
+export async function* streamFiles(
+	input: StreamDropboxFilesInput
+): ConnectorStream<DropboxCheckpoint> {
+	const options = input.options ?? {}
+	const maxBytesPerFile = options.maxBytesPerFile ?? DEFAULT_MAX_BYTES
+	const fileIds = Array.isArray(input.fileIds) ? input.fileIds : []
+	const startIndex = Math.max(0, input.checkpoint?.index ?? 0)
+
+	let processed = 0
+	for (let i = startIndex; i < fileIds.length; i++) {
+		const fileId = String(fileIds[i] ?? '').trim()
+		if (!fileId) {
+			continue
+		}
+		processed += 1
+
+		yield {
+			type: 'progress',
+			message: 'file:start',
+			current: processed,
+			total: fileIds.length,
+			entityId: fileId
+		}
+
+		try {
+			const meta = await dropboxApiFetch<DropboxMetadataResponse>({
+				auth: input.auth,
+				path: 'files/get_metadata',
+				body: {path: fileId}
+			})
+			if (meta['.tag'] !== 'file') {
+				yield {
+					type: 'warning',
+					code: 'file_skipped',
+					message: 'Skipping non-file entry.',
+					data: {fileId}
+				}
+				continue
+			}
+			const entryPath = resolvePath(meta)
+			if (!entryPath) {
+				yield {
+					type: 'warning',
+					code: 'file_skipped',
+					message: 'Missing path for file.',
+					data: {fileId}
+				}
+				continue
+			}
+			const sourceId = buildSourceId(input.sourceIdPrefix, entryPath)
+			if (
+				Number.isFinite(meta.size) &&
+				(meta.size as number) > maxBytesPerFile
+			) {
+				yield {
+					type: 'warning',
+					code: 'file_skipped',
+					message: `Skipping file because it exceeds maxBytesPerFile (${maxBytesPerFile}).`,
+					data: {path: entryPath, sourceId, reason: 'too_large'}
+				}
+				continue
+			}
+			const payload = await downloadFile({
+				auth: input.auth,
+				path: entryPath,
+				maxBytesPerFile
+			})
+			if (payload.skipped) {
+				yield {
+					type: 'warning',
+					code: 'file_skipped',
+					message: `Skipping file because it exceeds maxBytesPerFile (${maxBytesPerFile}).`,
+					data: {path: entryPath, sourceId, reason: 'too_large'}
+				}
+				continue
+			}
+
+			yield {
+				type: 'upsert',
+				input: {
+					sourceId,
+					content: payload.content,
+					assets: payload.assets,
+					metadata: buildMetadata(meta)
+				}
+			}
+
+			yield {
+				type: 'progress',
+				message: 'file:success',
+				current: processed,
+				total: fileIds.length,
+				sourceId,
+				entityId: fileId
+			}
+		} catch (err) {
+			if (isNotFound(err)) {
+				yield {
+					type: 'warning',
+					code: 'file_not_found',
+					message: 'Dropbox file not found or inaccessible.',
+					data: {fileId}
+				}
+			} else {
+				yield {
+					type: 'warning',
+					code: 'file_error',
+					message: asMessage(err),
+					data: {fileId}
+				}
+			}
+		}
+
+		yield {
+			type: 'checkpoint',
+			checkpoint: {
+				index: i + 1,
+				fileId
+			}
+		}
+	}
+}
+
+/**
+ * Exported connector surface for Dropbox.
+ */
+export const dropboxConnector = {
+	streamFolder,
+	streamFiles
+}