unrag 0.2.0 → 0.2.1

package/dist/cli/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 // cli/run.ts
-import { intro, outro as outro2 } from "@clack/prompts";
+import { intro, outro as outro3 } from "@clack/prompts";
 
 // cli/commands/init.ts
 import { cancel as cancel2, isCancel as isCancel2, outro, select, text } from "@clack/prompts";
@@ -47,6 +47,23 @@ async function findUp(startDir, filename) {
     current = parent;
   }
 }
+async function listFilesRecursive(dir) {
+  const entries = await readdir(dir, { withFileTypes: true });
+  const out = [];
+  for (const entry of entries) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      out.push(...await listFilesRecursive(full));
+    } else if (entry.isFile()) {
+      out.push(full);
+    } else {
+      const s = await stat(full).catch(() => null);
+      if (s?.isFile())
+        out.push(full);
+    }
+  }
+  return out;
+}
 
 // cli/lib/registry.ts
 var readText = (filePath) => readFile(filePath, "utf8");
@@ -209,6 +226,42 @@ async function copyRegistryFiles(selection) {
     await writeText(mapping.dest, content);
   }
 }
+async function copyConnectorFiles(selection) {
+  const toAbs = (projectRelative) => path2.join(selection.projectRoot, projectRelative);
+  const installBaseAbs = toAbs(selection.installDir);
+  const connectorRegistryAbs = path2.join(selection.registryRoot, "connectors", selection.connector);
+  if (!await exists(connectorRegistryAbs)) {
+    throw new Error(`Unknown connector registry: ${path2.relative(selection.registryRoot, connectorRegistryAbs)}`);
+  }
+  const files = await listFilesRecursive(connectorRegistryAbs);
+  const destRootAbs = path2.join(installBaseAbs, "connectors", selection.connector);
+  const nonInteractive = Boolean(selection.yes) || !process.stdin.isTTY;
+  for (const src of files) {
+    if (!await exists(src)) {
+      throw new Error(`Registry file missing: ${src}`);
+    }
+    const rel = path2.relative(connectorRegistryAbs, src);
+    const dest = path2.join(destRootAbs, rel);
+    if (await exists(dest)) {
+      if (nonInteractive) {
+        continue;
+      }
+      const answer = await confirm({
+        message: `Overwrite ${path2.relative(selection.projectRoot, dest)}?`,
+        initialValue: false
+      });
+      if (isCancel(answer)) {
+        cancel("Cancelled.");
+        return;
+      }
+      if (!answer) {
+        continue;
+      }
+    }
+    const raw = await readText(src);
+    await writeText(dest, raw);
+  }
+}
 
 // cli/lib/json.ts
 import { readFile as readFile2, writeFile as writeFile2 } from "node:fs/promises";
@@ -286,6 +339,14 @@ function depsForAdapter(adapter) {
   }
   return { deps, devDeps };
 }
+function depsForConnector(connector) {
+  const deps = {};
+  const devDeps = {};
+  if (connector === "notion") {
+    deps["@notionhq/client"] = "^2.2.16";
+  }
+  return { deps, devDeps };
+}
 function installCmd(pm) {
   if (pm === "bun")
     return "bun install";
@@ -488,7 +549,8 @@ async function initCommand(args) {
     installDir,
     storeAdapter: storeAdapterAnswer,
     aliasBase,
-    version: CONFIG_VERSION
+    version: CONFIG_VERSION,
+    connectors: existing?.connectors ?? []
   };
   await writeJsonFile(path5.join(root, CONFIG_FILE), config);
   const pm = await detectPackageManager(root);
@@ -512,19 +574,103 @@ async function initCommand(args) {
 `));
 }
 
+// cli/commands/add.ts
+import { outro as outro2 } from "@clack/prompts";
+import path6 from "node:path";
+import { fileURLToPath as fileURLToPath2 } from "node:url";
+var CONFIG_FILE2 = "unrag.json";
+var __filename3 = fileURLToPath2(import.meta.url);
+var __dirname3 = path6.dirname(__filename3);
+var parseAddArgs = (args) => {
+  const out = {};
+  for (let i = 0;i < args.length; i++) {
+    const a = args[i];
+    if (a === "--yes" || a === "-y") {
+      out.yes = true;
+      continue;
+    }
+    if (!out.connector && !a.startsWith("-")) {
+      out.connector = a;
+      continue;
+    }
+  }
+  return out;
+};
+async function addCommand(args) {
+  const root = await tryFindProjectRoot(process.cwd());
+  if (!root) {
+    throw new Error("Could not find a project root (no package.json found).");
+  }
+  const parsed = parseAddArgs(args);
+  const connector = parsed.connector;
+  if (!connector) {
+    outro2(`Usage: unrag add <connector>
+
+Available connectors: notion`);
+    return;
+  }
+  if (connector !== "notion") {
+    outro2(`Unknown connector: ${connector}
+
+Available connectors: notion`);
+    return;
+  }
+  const configPath = path6.join(root, CONFIG_FILE2);
+  const config = await readJsonFile(configPath);
+  if (!config?.installDir) {
+    throw new Error(`Missing ${CONFIG_FILE2}. Run \`unrag init\` first.`);
+  }
+  const cliPackageRoot = await findUp(__dirname3, "package.json");
+  if (!cliPackageRoot) {
+    throw new Error("Could not locate CLI package root (package.json not found).");
+  }
+  const registryRoot = path6.join(cliPackageRoot, "registry");
+  const nonInteractive = parsed.yes || !process.stdin.isTTY;
+  await copyConnectorFiles({
+    projectRoot: root,
+    registryRoot,
+    installDir: config.installDir,
+    connector,
+    yes: nonInteractive
+  });
+  const pkg = await readPackageJson(root);
+  const { deps, devDeps } = depsForConnector(connector);
+  const merged = mergeDeps(pkg, deps, devDeps);
+  if (merged.changes.length > 0) {
+    await writePackageJson(root, merged.pkg);
+  }
+  const connectors = Array.from(new Set([...config.connectors ?? [], connector])).sort();
+  await writeJsonFile(configPath, { ...config, connectors });
+  outro2([
+    `Installed connector: ${connector}.`,
+    "",
+    `- Code: ${path6.join(config.installDir, "connectors", connector)}`,
+    `- Docs: /docs/connectors/${connector}`,
+    "",
+    merged.changes.length > 0 ? `Added deps: ${merged.changes.map((c) => c.name).join(", ")}` : "Added deps: none",
+    nonInteractive ? "" : "Tip: keep NOTION_TOKEN server-side only (env var)."
+  ].filter(Boolean).join(`
+`));
+}
+
 // cli/run.ts
 async function run(argv) {
   const [, , command, ...rest] = argv;
   intro("unrag");
   if (!command || command === "help" || command === "--help" || command === "-h") {
-    outro2("Usage: unrag init");
+    outro3(["Usage:", "", "- unrag init", "- unrag add <connector>"].join(`
+`));
     return;
   }
   if (command === "init") {
     await initCommand(rest);
     return;
   }
-  outro2(`Unknown command: ${command}`);
+  if (command === "add") {
+    await addCommand(rest);
+    return;
+  }
+  outro3(`Unknown command: ${command}`);
   process.exitCode = 1;
 }
 

package/package.json

@@ -4,7 +4,7 @@
   "bin": {
     "unrag": "./dist/cli/index.js"
   },
-  "version": "0.2.0",
+  "version": "0.2.1",
   "private": false,
   "license": "Apache-2.0",
   "devDependencies": {

package/registry/connectors/notion/client.ts

@@ -0,0 +1,22 @@
+import { Client } from "@notionhq/client";
+
+export type NotionClient = Client;
+
+export type CreateNotionClientInput = {
+  token: string;
+  timeoutMs?: number;
+};
+
+export function createNotionClient(input: CreateNotionClientInput): NotionClient {
+  const token = input.token?.trim();
+  if (!token) throw new Error("NOTION token is required");
+
+  return new Client({
+    auth: token,
+    // @notionhq/client uses undici/fetch under the hood; timeout is supported.
+    // If unsupported in a future version, callers can wrap requests.
+    timeoutMs: input.timeoutMs ?? 30_000,
+  } as any);
+}
+
+

package/registry/connectors/notion/ids.ts

@@ -0,0 +1,39 @@
+const UUID_32_RE = /^[0-9a-f]{32}$/i;
+const UUID_HYPHEN_RE =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+export function normalizeNotionId32(input: string): string {
+  const raw = String(input ?? "").trim();
+  if (!raw) throw new Error("Notion id is required");
+
+  // Try to extract UUID-like tokens from URLs or mixed strings.
+  const token =
+    raw.match(/[0-9a-fA-F]{32}/)?.[0] ??
+    raw.match(/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/)?.[0] ??
+    raw;
+
+  const normalized = token.replaceAll("-", "").toLowerCase();
+  if (!UUID_32_RE.test(normalized)) {
+    throw new Error(`Invalid Notion id: ${input}`);
+  }
+  return normalized;
+}
+
+export function toUuidHyphenated(id32: string): string {
+  const n = normalizeNotionId32(id32);
+  return `${n.slice(0, 8)}-${n.slice(8, 12)}-${n.slice(12, 16)}-${n.slice(
+    16,
+    20
+  )}-${n.slice(20)}`;
+}
+
+export function normalizeNotionPageId32(pageIdOrUrl: string): string {
+  return normalizeNotionId32(pageIdOrUrl);
+}
+
+export function isUuidLike(input: string): boolean {
+  const s = String(input ?? "").trim();
+  return UUID_32_RE.test(s.replaceAll("-", "")) || UUID_HYPHEN_RE.test(s);
+}
+
+

package/registry/connectors/notion/index.ts

@@ -0,0 +1,7 @@
+export { createNotionClient } from "./client";
+export { normalizeNotionId32, normalizeNotionPageId32, toUuidHyphenated } from "./ids";
+export { renderNotionBlocksToText } from "./render";
+export { buildNotionPageIngestInput, loadNotionPageDocument, syncNotionPages } from "./sync";
+export * from "./types";
+
+

package/registry/connectors/notion/render.ts

@@ -0,0 +1,98 @@
+type RichText = { plain_text?: string };
+
+export type NotionBlock = {
+  id: string;
+  type: string;
+  has_children?: boolean;
+  // Notion block payload is keyed by `type`; we keep it loose to stay stable.
+  [key: string]: unknown;
+};
+
+export type NotionBlockNode = {
+  block: NotionBlock;
+  children: NotionBlockNode[];
+};
+
+const rt = (value: unknown): string => {
+  const items = Array.isArray(value) ? (value as RichText[]) : [];
+  return items.map((t) => t?.plain_text ?? "").join("");
+};
+
+const indent = (n: number) => (n > 0 ? "  ".repeat(n) : "");
+
+export function renderNotionBlocksToText(
+  nodes: NotionBlockNode[],
+  opts: { maxDepth?: number } = {}
+): string {
+  const maxDepth = opts.maxDepth ?? 6;
+  const lines: string[] = [];
+
+  const walk = (node: NotionBlockNode, depth: number, listDepth: number) => {
+    if (depth > maxDepth) return;
+    const b = node.block;
+
+    const t = b.type;
+
+    if (t === "paragraph") {
+      const text = rt((b as any).paragraph?.rich_text);
+      if (text.trim()) lines.push(text);
+    } else if (t === "heading_1") {
+      const text = rt((b as any).heading_1?.rich_text);
+      if (text.trim()) lines.push(`# ${text}`);
+    } else if (t === "heading_2") {
+      const text = rt((b as any).heading_2?.rich_text);
+      if (text.trim()) lines.push(`## ${text}`);
+    } else if (t === "heading_3") {
+      const text = rt((b as any).heading_3?.rich_text);
+      if (text.trim()) lines.push(`### ${text}`);
+    } else if (t === "bulleted_list_item") {
+      const text = rt((b as any).bulleted_list_item?.rich_text);
+      if (text.trim()) lines.push(`${indent(listDepth)}- ${text}`);
+    } else if (t === "numbered_list_item") {
+      const text = rt((b as any).numbered_list_item?.rich_text);
+      if (text.trim()) lines.push(`${indent(listDepth)}- ${text}`);
+    } else if (t === "to_do") {
+      const text = rt((b as any).to_do?.rich_text);
+      const checked = Boolean((b as any).to_do?.checked);
+      if (text.trim()) lines.push(`${indent(listDepth)}- [${checked ? "x" : " "}] ${text}`);
+    } else if (t === "quote") {
+      const text = rt((b as any).quote?.rich_text);
+      if (text.trim()) lines.push(`> ${text}`);
+    } else if (t === "callout") {
+      const text = rt((b as any).callout?.rich_text);
+      if (text.trim()) lines.push(text);
+    } else if (t === "code") {
+      const text = rt((b as any).code?.rich_text);
+      const lang = String((b as any).code?.language ?? "").trim();
+      lines.push("```" + lang);
+      if (text.trim()) lines.push(text);
+      lines.push("```");
+    } else if (t === "divider") {
+      lines.push("---");
+    } else {
+      // Unsupported block types are ignored for v1.
+      // This keeps the output focused and avoids surprises.
+    }
+
+    // Render children (nested blocks). For list items, increase listDepth.
+    const nextListDepth =
+      t === "bulleted_list_item" ||
+      t === "numbered_list_item" ||
+      t === "to_do"
+        ? listDepth + 1
+        : listDepth;
+
+    for (const child of node.children) {
+      walk(child, depth + 1, nextListDepth);
+    }
+  };
+
+  for (const node of nodes) {
+    walk(node, 0, 0);
+    lines.push("");
+  }
+
+  return lines.join("\n").trim();
+}
+
+

package/registry/connectors/notion/sync.ts

@@ -0,0 +1,222 @@
+import type { ContextEngine } from "../../core";
+import type { IngestResult } from "../../core/types";
+import { createNotionClient, type NotionClient } from "./client";
+import { normalizeNotionPageId32, toUuidHyphenated } from "./ids";
+import { renderNotionBlocksToText, type NotionBlock, type NotionBlockNode } from "./render";
+import type {
+  BuildNotionPageIngestInputArgs,
+  NotionPageDocument,
+  NotionSyncProgressEvent,
+  SyncNotionPagesInput,
+  SyncNotionPagesResult,
+} from "./types";
+
+const joinPrefix = (prefix: string | undefined, rest: string) => {
+  const p = (prefix ?? "").trim();
+  if (!p) return rest;
+  return p.endsWith(":") ? p + rest : p + ":" + rest;
+};
+
+export function buildNotionPageIngestInput(
+  args: BuildNotionPageIngestInputArgs
+) {
+  const sourceId = joinPrefix(
+    args.sourceIdPrefix,
+    `notion:page:${args.pageId}`
+  );
+
+  return {
+    sourceId,
+    content: args.content,
+    metadata: args.metadata ?? {},
+  };
+}
+
+const richTextToText = (richText: any[] | undefined) =>
+  (Array.isArray(richText) ? richText : [])
+    .map((t) => String(t?.plain_text ?? ""))
+    .join("");
+
+const getNotionPageTitle = (page: any): string => {
+  const props = page?.properties ?? {};
+  for (const key of Object.keys(props)) {
+    const p = props[key];
+    if (p?.type === "title") {
+      return richTextToText(p?.title);
+    }
+  }
+  return "";
+};
+
+async function listAllBlockChildren(
+  notion: NotionClient,
+  blockId: string
+): Promise<NotionBlock[]> {
+  const blocks: NotionBlock[] = [];
+  let cursor: string | undefined = undefined;
+
+  while (true) {
+    const res: any = await notion.blocks.children.list({
+      block_id: blockId,
+      start_cursor: cursor,
+      page_size: 100,
+    });
+
+    blocks.push(...((res?.results ?? []) as NotionBlock[]));
+    if (!res?.has_more) break;
+    cursor = res?.next_cursor ?? undefined;
+    if (!cursor) break;
+  }
+
+  return blocks;
+}
+
+async function buildBlockTree(
+  notion: NotionClient,
+  rootBlockId: string,
+  depth: number,
+  maxDepth: number
+): Promise<NotionBlockNode[]> {
+  const children = await listAllBlockChildren(notion, rootBlockId);
+  const nodes: NotionBlockNode[] = [];
+
+  for (const block of children) {
+    let grandChildren: NotionBlockNode[] = [];
+    if (block.has_children && depth < maxDepth) {
+      grandChildren = await buildBlockTree(notion, block.id, depth + 1, maxDepth);
+    }
+    nodes.push({ block, children: grandChildren });
+  }
+
+  return nodes;
+}
+
+export async function loadNotionPageDocument(args: {
+  notion: NotionClient;
+  pageIdOrUrl: string;
+  sourceIdPrefix?: string;
+  maxDepth?: number;
+}): Promise<NotionPageDocument> {
+  const pageId = normalizeNotionPageId32(args.pageIdOrUrl);
+  const apiId = toUuidHyphenated(pageId);
+
+  const page: any = await args.notion.pages.retrieve({ page_id: apiId });
+  const title = getNotionPageTitle(page);
+  const url = String(page?.url ?? "");
+  const lastEditedTime = String(page?.last_edited_time ?? "");
+
+  const tree = await buildBlockTree(args.notion, apiId, 0, args.maxDepth ?? 4);
+  const body = renderNotionBlocksToText(tree);
+  const content = [title.trim(), body.trim()].filter(Boolean).join("\n\n");
+
+  const metadata = {
+    connector: "notion",
+    kind: "page",
+    pageId,
+    url,
+    title,
+    lastEditedTime,
+  } as const;
+
+  const ingest = buildNotionPageIngestInput({
+    pageId,
+    content,
+    metadata: metadata as any,
+    sourceIdPrefix: args.sourceIdPrefix,
+  });
+
+  return {
+    sourceId: ingest.sourceId,
+    content: ingest.content,
+    metadata: ingest.metadata ?? {},
+  };
+}
+
+const isNotFound = (err: any) => {
+  const status = Number(err?.status ?? err?.statusCode ?? err?.code);
+  if (status === 404) return true;
+  const msg = String(err?.message ?? "");
+  return msg.toLowerCase().includes("could not find");
+};
+
+export async function syncNotionPages(
+  input: SyncNotionPagesInput
+): Promise<SyncNotionPagesResult> {
+  const deleteOnNotFound = input.deleteOnNotFound ?? false;
+
+  const notion = createNotionClient({ token: input.token });
+  const errors: SyncNotionPagesResult["errors"] = [];
+
+  let succeeded = 0;
+  let failed = 0;
+  let deleted = 0;
+
+  for (const rawId of input.pageIds) {
+    const pageId = normalizeNotionPageId32(rawId);
+    const sourceId = joinPrefix(
+      input.sourceIdPrefix,
+      `notion:page:${pageId}`
+    );
+
+    const emit = (event: NotionSyncProgressEvent) => {
+      try {
+        input.onProgress?.(event);
+      } catch {
+        // ignore progress handler errors
+      }
+    };
+
+    emit({ type: "page:start", pageId, sourceId });
+
+    try {
+      const doc = await loadNotionPageDocument({
+        notion,
+        pageIdOrUrl: pageId,
+        sourceIdPrefix: input.sourceIdPrefix,
+      });
+
+      const result: IngestResult = await input.engine.ingest({
+        sourceId: doc.sourceId,
+        content: doc.content,
+        metadata: doc.metadata as any,
+      });
+
+      succeeded += 1;
+      emit({
+        type: "page:success",
+        pageId,
+        sourceId,
+        chunkCount: result.chunkCount,
+      });
+    } catch (err) {
+      if (isNotFound(err)) {
+        emit({ type: "page:not-found", pageId, sourceId });
+        if (deleteOnNotFound) {
+          try {
+            await input.engine.delete({ sourceId });
+            deleted += 1;
+          } catch (deleteErr) {
+            failed += 1;
+            errors.push({ pageId, sourceId, error: deleteErr });
+            emit({ type: "page:error", pageId, sourceId, error: deleteErr });
+          }
+        }
+        continue;
+      }
+
+      failed += 1;
+      errors.push({ pageId, sourceId, error: err });
+      emit({ type: "page:error", pageId, sourceId, error: err });
+    }
+  }
+
+  return {
+    pageCount: input.pageIds.length,
+    succeeded,
+    failed,
+    deleted,
+    errors,
+  };
+}
+
+

package/registry/connectors/notion/types.ts

@@ -0,0 +1,56 @@
+import type { ContextEngine } from "../../core";
+import type { IngestInput } from "../../core/types";
+
+export type NotionSyncProgressEvent =
+  | { type: "page:start"; pageId: string; sourceId: string }
+  | { type: "page:success"; pageId: string; sourceId: string; chunkCount: number }
+  | { type: "page:error"; pageId: string; sourceId: string; error: unknown }
+  | { type: "page:not-found"; pageId: string; sourceId: string };
+
+export type SyncNotionPagesInput = {
+  engine: ContextEngine;
+  /**
+   * Server-side Notion integration token.
+   * Keep this server-only (env var).
+   */
+  token: string;
+  /** Notion page IDs or page URLs. */
+  pageIds: string[];
+  /**
+   * Optional namespace prefix, useful for multi-tenant apps:
+   * `tenant:acme:` -> `tenant:acme:notion:page:<id>`
+   */
+  sourceIdPrefix?: string;
+  /**
+   * When true, if a page is not found/accessible, delete the previously ingested
+   * document for that page (exact sourceId).
+   */
+  deleteOnNotFound?: boolean;
+  /** Optional progress callback. */
+  onProgress?: (event: NotionSyncProgressEvent) => void;
+};
+
+export type SyncNotionPagesResult = {
+  pageCount: number;
+  succeeded: number;
+  failed: number;
+  deleted: number;
+  errors: Array<{ pageId: string; sourceId: string; error: unknown }>;
+};
+
+export type NotionPageDocument = {
+  sourceId: string;
+  content: string;
+  metadata: Record<string, unknown>;
+};
+
+export type BuildNotionPageIngestInputArgs = {
+  pageId: string; // normalized 32-hex (no dashes)
+  content: string;
+  metadata?: Record<string, unknown>;
+  sourceIdPrefix?: string;
+};
+
+export type BuildNotionPageIngestInputResult = IngestInput;
+
+