useathena 0.2.1 → 0.3.0

package/dist/connect/google.js

@@ -0,0 +1,371 @@
+import { createHash, randomBytes } from "node:crypto";
+import { createServer } from "node:http";
+import { spawn } from "node:child_process";
+/**
+ * Google connector (Drive + Gmail, read-only) on the installed-app OAuth flow:
+ * loopback redirect + PKCE, the gcloud model — supermemory-grade connect UX
+ * with zero hosted infrastructure, because a CLI can open a local port where
+ * a web service cannot. The client credentials ship in the package; for
+ * Desktop-type clients Google documents the secret as non-confidential.
+ * During friend testing the consent screen is published-unverified: testers
+ * click through one "Google hasn't verified this app" warning, and refresh
+ * tokens persist (Testing mode would expire them weekly).
+ */
+const CLIENT_ID = process.env.ATHENA_GOOGLE_CLIENT_ID ?? "994548323040-h5k5ikae6e7t33igveqqt4766uk5r4u8.apps.googleusercontent.com";
+const CLIENT_SECRET = process.env.ATHENA_GOOGLE_CLIENT_SECRET ?? "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";
+const AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth";
+const TOKEN_URL = "https://oauth2.googleapis.com/token";
+const SCOPES = "https://www.googleapis.com/auth/drive.readonly https://www.googleapis.com/auth/gmail.readonly";
+const AUTH_TIMEOUT_MS = 5 * 60 * 1000;
+const MAX_FILES = 100;
+const MAX_THREADS = 200;
+const CONTENT_CAP = 100_000;
+const RETRYABLE = new Set([408, 429, 500, 502, 503, 504]);
+// --- OAuth: loopback + PKCE ---
+export function pkcePair() {
+    const verifier = randomBytes(32).toString("base64url");
+    const challenge = createHash("sha256").update(verifier).digest("base64url");
+    return { verifier, challenge };
+}
+export async function acquireGoogleAuth(rl, log) {
+    if (!CLIENT_SECRET) {
+        throw new Error("Google client secret is not configured — set ATHENA_GOOGLE_CLIENT_SECRET (or update this build)");
+    }
+    const { verifier, challenge } = pkcePair();
+    const state = randomBytes(16).toString("base64url");
+    const server = createServer();
+    await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
+    const redirectUri = `http://127.0.0.1:${server.address().port}`;
+    try {
+        const authUrl = `${AUTH_URL}?${new URLSearchParams({
+            client_id: CLIENT_ID,
+            redirect_uri: redirectUri,
+            response_type: "code",
+            scope: SCOPES,
+            code_challenge: challenge,
+            code_challenge_method: "S256",
+            state,
+            access_type: "offline",
+            prompt: "consent",
+        }).toString()}`;
+        log("opening your browser to sign in with Google…");
+        log(`if it does not open (or this is an SSH session), open this URL on any machine:\n  ${authUrl}`);
+        log("after consent the browser lands on a 127.0.0.1 page that fails to load — that is normal");
+        openBrowser(authUrl);
+        // Two ways home, first one wins: the loopback catch (local browser) or a
+        // pasted redirect URL (SSH/headless — the code travels in the URL itself).
+        const aborted = new AbortController();
+        let code;
+        try {
+            code = await Promise.race([waitForCode(server, state), codeFromPaste(rl, state, aborted.signal, log)]);
+        }
+        finally {
+            aborted.abort(); // settle the losing path so the readline is usable afterwards
+        }
+        const token = await postForm(TOKEN_URL, {
+            code,
+            client_id: CLIENT_ID,
+            client_secret: CLIENT_SECRET,
+            code_verifier: verifier,
+            redirect_uri: redirectUri,
+            grant_type: "authorization_code",
+        });
+        const refreshToken = token.refresh_token;
+        const accessToken = token.access_token;
+        if (typeof refreshToken !== "string" || typeof accessToken !== "string") {
+            throw new Error("Google returned no refresh token — remove athena under myaccount.google.com/permissions and retry");
+        }
+        const profile = await apiGet(accessToken, "https://gmail.googleapis.com/gmail/v1/users/me/profile");
+        const email = typeof profile.emailAddress === "string" ? profile.emailAddress : "Google account";
+        return { secret: JSON.stringify({ refreshToken }), label: email };
+    }
+    finally {
+        server.close();
+    }
+}
+function openBrowser(url) {
+    const command = process.platform === "darwin" ? "open" : process.platform === "win32" ? "start" : "xdg-open";
+    try {
+        spawn(command, [url], { stdio: "ignore", detached: true }).unref();
+    }
+    catch {
+        // the URL is printed — opening is best-effort
+    }
+}
+/** Parse a pasted redirect URL (`http://127.0.0.1:…/?code=…&state=…`) into the auth code. */
+export function parseRedirectUrl(input, expectedState) {
+    let url;
+    try {
+        url = new URL(input.trim());
+    }
+    catch {
+        throw new Error("that is not a URL — paste the full address from the failed 127.0.0.1 page");
+    }
+    const error = url.searchParams.get("error");
+    if (error)
+        throw new Error(`Google sign-in failed: ${error}`);
+    const code = url.searchParams.get("code");
+    if (!code)
+        throw new Error("no code in that URL — paste the full address, including everything after the ?");
+    if (url.searchParams.get("state") !== expectedState) {
+        throw new Error("state mismatch — that URL is from a different sign-in attempt; use the most recent one");
+    }
+    return code;
+}
+async function codeFromPaste(rl, expectedState, signal, log) {
+    while (true) {
+        let answer;
+        try {
+            answer = (await rl.question("waiting — or paste the redirect URL here: ", { signal })).trim();
+        }
+        catch {
+            return new Promise(() => { }); // loopback won the race; never settle
+        }
+        if (!answer)
+            continue;
+        try {
+            return parseRedirectUrl(answer, expectedState);
+        }
+        catch (error) {
+            log(String(error instanceof Error ? error.message : error));
+        }
+    }
+}
+function waitForCode(server, expectedState) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error("timed out waiting for the browser sign-in (5 minutes)")), AUTH_TIMEOUT_MS);
+        server.on("request", (request, response) => {
+            const url = new URL(request.url ?? "/", "http://127.0.0.1");
+            const code = url.searchParams.get("code");
+            const state = url.searchParams.get("state");
+            const error = url.searchParams.get("error");
+            if (!code && !error) {
+                response.writeHead(404).end();
+                return;
+            }
+            response
+                .writeHead(200, { "Content-Type": "text/html" })
+                .end("<html><body style='font-family:sans-serif'><h2>athena is connected</h2>You can close this tab and return to the terminal.</body></html>");
+            clearTimeout(timer);
+            if (error)
+                reject(new Error(`Google sign-in failed: ${error}`));
+            else if (state !== expectedState)
+                reject(new Error("OAuth state mismatch — try again"));
+            else
+                resolve(code);
+        });
+    });
+}
+async function postForm(url, form) {
+    const response = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "application/x-www-form-urlencoded" },
+        body: new URLSearchParams(form).toString(),
+        signal: AbortSignal.timeout(30_000),
+    });
+    const body = (await response.json());
+    if (!response.ok) {
+        throw new Error(`google token endpoint: HTTP ${response.status} ${typeof body.error === "string" ? body.error : ""}`);
+    }
+    return body;
+}
+async function apiGet(accessToken, url, attempt = 0) {
+    const response = await fetch(url, {
+        headers: { Authorization: `Bearer ${accessToken}` },
+        signal: AbortSignal.timeout(30_000),
+    });
+    if (!response.ok) {
+        if (attempt < 3 && RETRYABLE.has(response.status)) {
+            await new Promise((resolve) => setTimeout(resolve, 500 * (attempt + 1)));
+            return apiGet(accessToken, url, attempt + 1);
+        }
+        const raw = await response.text();
+        let detail = raw.slice(0, 300);
+        try {
+            const parsed = JSON.parse(raw);
+            if (parsed.error?.message)
+                detail = parsed.error.message;
+        }
+        catch {
+            // not JSON — keep the raw slice
+        }
+        throw new Error(`google api ${new URL(url).pathname}: HTTP ${response.status} — ${detail}`);
+    }
+    const text = await response.text();
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return { __text: text }; // Drive exports and media downloads are not JSON
+    }
+}
+/** Refresh-token → authenticated fetcher for one sync run. */
+export async function googleApi(storedSecret) {
+    const { refreshToken } = JSON.parse(storedSecret);
+    const token = await postForm(TOKEN_URL, {
+        refresh_token: refreshToken,
+        client_id: CLIENT_ID,
+        client_secret: CLIENT_SECRET,
+        grant_type: "refresh_token",
+    });
+    const accessToken = token.access_token;
+    if (typeof accessToken !== "string")
+        throw new Error("google refresh failed — reconnect with: athena connect google");
+    return (url) => apiGet(accessToken, url);
+}
+// --- Drive ---
+const DRIVE_QUERY = [
+    "(mimeType='application/vnd.google-apps.document' or mimeType='text/plain' or mimeType='text/markdown')",
+    "trashed=false",
+].join(" and ");
+async function fetchDrive(api, cursor, log) {
+    const files = [];
+    let pageToken;
+    let truncated = false;
+    outer: do {
+        const params = new URLSearchParams({
+            q: DRIVE_QUERY,
+            orderBy: "modifiedTime desc",
+            pageSize: "50",
+            fields: "nextPageToken,files(id,name,mimeType,modifiedTime,webViewLink)",
+            ...(pageToken ? { pageToken } : {}),
+        });
+        const page = await api(`https://www.googleapis.com/drive/v3/files?${params.toString()}`);
+        for (const raw of Array.isArray(page.files) ? page.files : []) {
+            if (cursor !== undefined && raw.modifiedTime <= cursor)
+                break outer; // sorted desc — the rest is synced
+            if (files.length >= MAX_FILES) {
+                truncated = true;
+                break outer;
+            }
+            files.push(raw);
+        }
+        pageToken = typeof page.nextPageToken === "string" ? page.nextPageToken : undefined;
+    } while (pageToken);
+    if (truncated)
+        log(`drive: capped at the ${MAX_FILES} most recently edited files — older files sync when next edited`);
+    const items = [];
+    for (const file of files) {
+        const url = file.mimeType === "application/vnd.google-apps.document"
+            ? `https://www.googleapis.com/drive/v3/files/${file.id}/export?mimeType=text/plain`
+            : `https://www.googleapis.com/drive/v3/files/${file.id}?alt=media`;
+        let body;
+        try {
+            body = await api(url);
+        }
+        catch (error) {
+            // export-locked shared docs etc. — one bad file must not kill the sync
+            log(`drive: skipping "${file.name}" — ${error instanceof Error ? error.message : String(error)}`);
+            continue;
+        }
+        const content = (typeof body.__text === "string" ? body.__text : JSON.stringify(body)).slice(0, CONTENT_CAP);
+        if (!content.trim())
+            continue;
+        items.push({
+            uri: file.webViewLink ?? `https://drive.google.com/file/d/${file.id}`,
+            title: file.name,
+            content,
+            editedAt: file.modifiedTime,
+            kind: "document",
+        });
+    }
+    const result = { items, truncated };
+    const newest = files[0]?.modifiedTime ?? cursor;
+    if (newest !== undefined)
+        result.cursor = newest;
+    return result;
+}
+// --- Gmail ---
+export function decodeBase64Url(data) {
+    return Buffer.from(data, "base64url").toString("utf8");
+}
+/** Depth-first text/plain extraction across nested multiparts. */
+export function extractText(part) {
+    if (part.mimeType === "text/plain" && part.body?.data)
+        return decodeBase64Url(part.body.data);
+    return (part.parts ?? []).map(extractText).filter(Boolean).join("\n");
+}
+export function renderThread(messages) {
+    const header = (message, name) => message.payload?.headers?.find((h) => h.name?.toLowerCase() === name)?.value ?? "";
+    const subject = messages.map((m) => header(m, "subject")).find((s) => s) ?? "(no subject)";
+    const lines = messages.map((message) => {
+        const date = Number(message.internalDate ?? 0);
+        const day = date > 0 ? new Date(date).toISOString().slice(0, 10) : "????-??-??";
+        const text = message.payload ? extractText(message.payload).trim() : "";
+        return `[${day}] From: ${header(message, "from")}\n${text}`;
+    });
+    const newestMs = messages.reduce((max, m) => Math.max(max, Number(m.internalDate ?? 0)), 0);
+    return { subject, content: `Subject: ${subject}\n\n${lines.join("\n---\n")}`.slice(0, CONTENT_CAP), newestMs };
+}
+async function fetchGmail(api, afterSec, log) {
+    // First sync digs a year deep — sent mail is the knowledge bootstrap; after
+    // that the cursor keeps syncs incremental.
+    const query = afterSec !== undefined ? `in:sent after:${afterSec}` : "in:sent newer_than:365d";
+    const threads = [];
+    let pageToken;
+    do {
+        const params = new URLSearchParams({ q: query, maxResults: "100", ...(pageToken ? { pageToken } : {}) });
+        const list = await api(`https://gmail.googleapis.com/gmail/v1/users/me/threads?${params.toString()}`);
+        threads.push(...(Array.isArray(list.threads) ? list.threads : []));
+        pageToken = typeof list.nextPageToken === "string" ? list.nextPageToken : undefined;
+    } while (pageToken && threads.length < MAX_THREADS);
+    const truncated = threads.length > MAX_THREADS || pageToken !== undefined;
+    if (truncated) {
+        log(`gmail: capped at the ${MAX_THREADS} newest threads — older ones enter only when their thread gets new mail`);
+    }
+    const items = [];
+    let newestMs = 0;
+    for (const thread of threads.slice(0, MAX_THREADS)) {
+        if (!thread.id)
+            continue;
+        let full;
+        try {
+            full = await api(`https://gmail.googleapis.com/gmail/v1/users/me/threads/${thread.id}?format=full`);
+        }
+        catch (error) {
+            log(`gmail: skipping a thread — ${error instanceof Error ? error.message : String(error)}`);
+            continue;
+        }
+        const rendered = renderThread((Array.isArray(full.messages) ? full.messages : []));
+        newestMs = Math.max(newestMs, rendered.newestMs);
+        if (!rendered.content.trim())
+            continue;
+        items.push({
+            uri: `https://mail.google.com/mail/u/0/#all/${thread.id}`,
+            title: rendered.subject,
+            content: rendered.content,
+            editedAt: rendered.newestMs > 0 ? new Date(rendered.newestMs).toISOString() : new Date().toISOString(),
+            kind: "email",
+        });
+    }
+    const result = { items, truncated };
+    const newest = newestMs > 0 ? Math.floor(newestMs / 1000) + 1 : afterSec;
+    if (newest !== undefined)
+        result.cursor = newest;
+    return result;
+}
+export function parseGoogleCursor(cursor) {
+    if (!cursor)
+        return {};
+    try {
+        return JSON.parse(cursor);
+    }
+    catch {
+        return {};
+    }
+}
+export async function fetchGoogleSources(api, cursor, log) {
+    const parsed = parseGoogleCursor(cursor);
+    const drive = await fetchDrive(api, parsed.drive, log);
+    const gmail = await fetchGmail(api, parsed.gmail, log);
+    const next = {};
+    if (drive.cursor !== undefined)
+        next.drive = drive.cursor;
+    if (gmail.cursor !== undefined)
+        next.gmail = gmail.cursor;
+    return {
+        items: [...drive.items, ...gmail.items],
+        cursor: JSON.stringify(next),
+        truncated: drive.truncated || gmail.truncated,
+    };
+}

package/dist/connect/notion.js

@@ -0,0 +1,177 @@
+/**
+ * Notion connector: internal-integration token, pages shared with the
+ * integration are the privacy boundary. Sync is incremental the way the
+ * Notion API affords it: search returns pages newest-edit-first, so we walk
+ * until we pass the last cursor (newest last_edited_time already synced)
+ * and stop. Block rendering is plain text with bounded depth — sources feed
+ * lexical search and fact extraction, not a Notion clone.
+ */
+const NOTION_VERSION = "2022-06-28";
+const API = "https://api.notion.com/v1";
+const MAX_PAGES_PER_SYNC = 200;
+const MAX_BLOCK_DEPTH = 2;
+const RETRYABLE = new Set([408, 429, 500, 502, 503, 504]);
+export function notionFetcher(token) {
+    return async (path, body) => {
+        for (let attempt = 0;; attempt += 1) {
+            const response = await fetch(`${API}${path}`, {
+                method: body === undefined ? "GET" : "POST",
+                headers: {
+                    Authorization: `Bearer ${token}`,
+                    "Notion-Version": NOTION_VERSION,
+                    ...(body === undefined ? {} : { "Content-Type": "application/json" }),
+                },
+                ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+                signal: AbortSignal.timeout(30_000),
+            });
+            if (response.ok)
+                return (await response.json());
+            if (attempt < 3 && RETRYABLE.has(response.status)) {
+                const retryAfter = Number(response.headers.get("retry-after"));
+                await sleep(Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter * 1000 : 500 * (attempt + 1));
+                continue;
+            }
+            const detail = (await response.text()).slice(0, 200);
+            throw new Error(`notion ${path}: HTTP ${response.status} ${detail}`);
+        }
+    };
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/** Validates the token and names the connection: "<workspace> (via <bot>)". */
+export async function validateNotionToken(fetchNotion) {
+    const me = await fetchNotion("/users/me");
+    const bot = (me.bot ?? {});
+    const workspace = typeof bot.workspace_name === "string" ? bot.workspace_name : "Notion workspace";
+    return typeof me.name === "string" && me.name.length > 0 ? `${workspace} (via ${me.name})` : workspace;
+}
+export function parsePage(raw) {
+    if (typeof raw !== "object" || raw === null)
+        return undefined;
+    const page = raw;
+    if (page.object !== "page" || typeof page.id !== "string" || typeof page.last_edited_time !== "string") {
+        return undefined;
+    }
+    return {
+        id: page.id,
+        url: typeof page.url === "string" ? page.url : `https://www.notion.so/${page.id.replaceAll("-", "")}`,
+        title: pageTitle(page.properties),
+        editedAt: page.last_edited_time,
+    };
+}
+export function pageTitle(properties) {
+    if (typeof properties !== "object" || properties === null)
+        return "Untitled";
+    for (const property of Object.values(properties)) {
+        const p = property;
+        if (p?.type === "title" && Array.isArray(p.title)) {
+            const text = richText(p.title);
+            if (text)
+                return text;
+        }
+    }
+    return "Untitled";
+}
+export function richText(spans) {
+    if (!Array.isArray(spans))
+        return "";
+    return spans
+        .map((span) => {
+        const s = span;
+        return typeof s?.plain_text === "string" ? s.plain_text : "";
+    })
+        .join("");
+}
+const BLOCK_PREFIX = {
+    heading_1: (t) => `# ${t}`,
+    heading_2: (t) => `## ${t}`,
+    heading_3: (t) => `### ${t}`,
+    bulleted_list_item: (t) => `- ${t}`,
+    numbered_list_item: (t) => `- ${t}`,
+    to_do: (t) => `- [ ] ${t}`,
+    quote: (t) => `> ${t}`,
+    paragraph: (t) => t,
+    callout: (t) => `> ${t}`,
+    toggle: (t) => t,
+    code: (t) => t,
+};
+export function renderBlock(raw, indent) {
+    if (typeof raw !== "object" || raw === null)
+        return undefined;
+    const block = raw;
+    const type = block.type;
+    if (typeof type !== "string")
+        return undefined;
+    if (type === "child_page") {
+        const child = block.child_page;
+        return `${indent}▸ ${typeof child?.title === "string" ? child.title : "Untitled"}`;
+    }
+    if (type === "divider")
+        return `${indent}---`;
+    const render = BLOCK_PREFIX[type];
+    if (!render)
+        return undefined;
+    const payload = block[type];
+    const text = richText(payload?.rich_text);
+    if (!text.trim())
+        return undefined;
+    return indent + render(text);
+}
+async function renderBlockTree(fetchNotion, blockId, depth) {
+    const lines = [];
+    let cursor;
+    do {
+        const query = cursor ? `?page_size=100&start_cursor=${cursor}` : "?page_size=100";
+        const result = await fetchNotion(`/blocks/${blockId}/children${query}`);
+        for (const raw of Array.isArray(result.results) ? result.results : []) {
+            const line = renderBlock(raw, "  ".repeat(depth));
+            if (line !== undefined)
+                lines.push(line);
+            const block = raw;
+            if (block.has_children === true && block.type !== "child_page" && depth < MAX_BLOCK_DEPTH) {
+                lines.push(...(await renderBlockTree(fetchNotion, block.id, depth + 1)));
+            }
+        }
+        cursor = result.has_more === true && typeof result.next_cursor === "string" ? result.next_cursor : undefined;
+    } while (cursor);
+    return lines;
+}
+/** Pages the cursor has not seen yet, newest first. `cursor` is the newest last_edited_time already synced. */
+export async function fetchNotionPages(fetchNotion, cursor, log) {
+    const pages = [];
+    let searchCursor;
+    let truncated = false;
+    outer: do {
+        const result = await fetchNotion("/search", {
+            filter: { property: "object", value: "page" },
+            sort: { direction: "descending", timestamp: "last_edited_time" },
+            page_size: 100,
+            ...(searchCursor ? { start_cursor: searchCursor } : {}),
+        });
+        for (const raw of Array.isArray(result.results) ? result.results : []) {
+            const page = parsePage(raw);
+            if (!page)
+                continue;
+            if (cursor !== undefined && page.editedAt <= cursor)
+                break outer; // sorted desc — the rest is already synced
+            if (pages.length >= MAX_PAGES_PER_SYNC) {
+                truncated = true;
+                break outer;
+            }
+            pages.push(page);
+        }
+        searchCursor = result.has_more === true && typeof result.next_cursor === "string" ? result.next_cursor : undefined;
+    } while (searchCursor);
+    if (truncated)
+        log(`capped at the ${MAX_PAGES_PER_SYNC} most recently edited pages — older pages sync when they are next edited`);
+    const items = [];
+    for (const page of pages) {
+        const body = (await renderBlockTree(fetchNotion, page.id, 0)).join("\n");
+        items.push({ uri: page.url, title: page.title, content: body, editedAt: page.editedAt });
+        if (items.length % 25 === 0)
+            log(`fetched ${items.length}/${pages.length} pages…`);
+    }
+    const newest = pages[0]?.editedAt ?? cursor;
+    return { items, ...(newest !== undefined ? { cursor: newest } : {}), truncated };
+}

package/dist/connect/secrets.js

@@ -0,0 +1,37 @@
+import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+/**
+ * Connector tokens live in one chmod-0600 JSON file, keyed by connection id —
+ * the same convention as provider API keys in config.json. Deliberately not
+ * the OS keychain: login services and headless machines hit locked-keychain
+ * failures (a documented OpenClaw lesson), and a 0600 file under the user's
+ * home is the same trust boundary the SQLite store already lives in.
+ */
+export function secretsPath() {
+    return process.env.ATHENA_SECRETS ?? join(homedir(), ".athena", "secrets.json");
+}
+function load() {
+    try {
+        return JSON.parse(readFileSync(secretsPath(), "utf8"));
+    }
+    catch {
+        return {};
+    }
+}
+function persist(secrets) {
+    const path = secretsPath();
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, `${JSON.stringify(secrets, null, 2)}\n`, { mode: 0o600 });
+}
+export function saveSecret(key, value) {
+    persist({ ...load(), [key]: value });
+}
+export function loadSecret(key) {
+    return load()[key];
+}
+export function deleteSecret(key) {
+    const secrets = load();
+    delete secrets[key];
+    persist(secrets);
+}