useathena 0.2.1 → 0.3.0

package/dist/connect/slack.js

@@ -0,0 +1,123 @@
+/**
+ * Slack connector: bring-your-own app created from a printed manifest (Slack
+ * allows no localhost OAuth redirects, so distributed OAuth would need hosted
+ * infra — the manifest flow is the validated zero-infra path). Read-only user
+ * token; only public channels the user is a member of are synced, one source
+ * per channel holding the recent transcript. Hash-dedupe in the sync runner
+ * makes the re-walk cheap, so there is no cursor.
+ */
+const API = "https://slack.com/api";
+const MAX_CHANNELS = 100;
+const MESSAGES_PER_CHANNEL = 100;
+const REPLY_THREADS_PER_CHANNEL = 25;
+export const SLACK_MANIFEST = JSON.stringify({
+    display_information: { name: "athena", description: "Read-only knowledge capture for the athena CLI" },
+    oauth_config: { scopes: { user: ["channels:history", "channels:read", "users:read"] } },
+    settings: { org_deploy_enabled: false, socket_mode_enabled: false, token_rotation_enabled: false },
+}, null, 2);
+export function slackFetcher(token) {
+    return async (method, params = {}) => {
+        const query = new URLSearchParams(params).toString();
+        for (let attempt = 0;; attempt += 1) {
+            const response = await fetch(`${API}/${method}${query ? `?${query}` : ""}`, {
+                headers: { Authorization: `Bearer ${token}` },
+                signal: AbortSignal.timeout(30_000),
+            });
+            if (response.status === 429 && attempt < 3) {
+                const retryAfter = Number(response.headers.get("retry-after"));
+                await new Promise((resolve) => setTimeout(resolve, (Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : 1) * 1000));
+                continue;
+            }
+            const body = (await response.json());
+            if (body.ok !== true)
+                throw new Error(`slack ${method}: ${typeof body.error === "string" ? body.error : `HTTP ${response.status}`}`);
+            return body;
+        }
+    };
+}
+export async function validateSlackToken(fetchSlack) {
+    const auth = await fetchSlack("auth.test");
+    const team = typeof auth.team === "string" ? auth.team : "Slack workspace";
+    return typeof auth.user === "string" ? `${team} (as ${auth.user})` : team;
+}
+/** `<@U…>` → @name, `<#C…|name>` → #name, `<url|label>` → label, `<url>` → url. */
+export function renderText(text, users) {
+    return text
+        .replace(/<@([A-Z0-9]+)(?:\|[^>]*)?>/g, (_, id) => `@${users.get(id) ?? id}`)
+        .replace(/<#[A-Z0-9]+\|([^>]+)>/g, "#$1")
+        .replace(/<([^|>]+)\|([^>]+)>/g, "$2")
+        .replace(/<([^>]+)>/g, "$1");
+}
+function renderLine(message, users, indent = "") {
+    const day = new Date(Number(message.ts) * 1000).toISOString().slice(0, 10);
+    const name = message.user !== undefined ? (users.get(message.user) ?? message.user) : "unknown";
+    return `${indent}[${day}] ${name}: ${renderText(message.text, users)}`;
+}
+const hasText = (m) => m.subtype === undefined && typeof m.text === "string" && m.text.trim().length > 0;
+export function renderTranscript(messages, users, replies = new Map()) {
+    return messages
+        .filter(hasText)
+        .sort((a, b) => Number(a.ts) - Number(b.ts))
+        .flatMap((message) => [
+        renderLine(message, users),
+        ...(replies.get(message.ts) ?? [])
+            .filter(hasText)
+            .sort((a, b) => Number(a.ts) - Number(b.ts))
+            .map((reply) => renderLine(reply, users, "    ↳ ")),
+    ])
+        .join("\n");
+}
+async function paginate(fetchSlack, method, params, key) {
+    const all = [];
+    let cursor;
+    do {
+        const body = await fetchSlack(method, cursor ? { ...params, cursor } : params);
+        all.push(...(Array.isArray(body[key]) ? body[key] : []));
+        const metadata = body.response_metadata;
+        cursor = metadata?.next_cursor || undefined;
+    } while (cursor);
+    return all;
+}
+export async function fetchSlackChannels(fetchSlack, _cursor, log) {
+    const users = new Map();
+    for (const raw of await paginate(fetchSlack, "users.list", { limit: "200" }, "members")) {
+        const member = raw;
+        if (member.id === undefined)
+            continue;
+        users.set(member.id, member.profile?.display_name || member.real_name || member.name || member.id);
+    }
+    const channels = (await paginate(fetchSlack, "conversations.list", { types: "public_channel", exclude_archived: "true", limit: "200" }, "channels"))
+        .map((raw) => raw)
+        .filter((c) => c.id !== undefined && c.is_member === true);
+    const truncated = channels.length > MAX_CHANNELS;
+    if (truncated)
+        log(`capped at ${MAX_CHANNELS} of ${channels.length} member channels this sync`);
+    const items = [];
+    for (const channel of channels.slice(0, MAX_CHANNELS)) {
+        const history = await fetchSlack("conversations.history", { channel: channel.id, limit: String(MESSAGES_PER_CHANNEL) });
+        const messages = (Array.isArray(history.messages) ? history.messages : []);
+        // Threads carry the real discussion: pull replies for the most recent
+        // threaded parents (same channels:history scope, one call per thread).
+        const replies = new Map();
+        const parents = messages
+            .filter((m) => (m.reply_count ?? 0) > 0)
+            .sort((a, b) => Number(b.ts) - Number(a.ts))
+            .slice(0, REPLY_THREADS_PER_CHANNEL);
+        for (const parent of parents) {
+            const thread = await fetchSlack("conversations.replies", { channel: channel.id, ts: parent.ts, limit: "100" });
+            const threadMessages = (Array.isArray(thread.messages) ? thread.messages : []);
+            replies.set(parent.ts, threadMessages.filter((m) => m.ts !== parent.ts));
+        }
+        const content = renderTranscript(messages, users, replies);
+        if (!content)
+            continue;
+        const newest = messages.reduce((max, m) => Math.max(max, Number(m.ts) || 0), 0);
+        items.push({
+            uri: `slack://${channel.context_team_id ?? "T"}/${channel.id}`,
+            title: `#${channel.name ?? channel.id}`,
+            content,
+            editedAt: new Date(newest * 1000).toISOString(),
+        });
+    }
+    return { items, truncated };
+}

package/dist/connect/sync.js

@@ -0,0 +1,123 @@
+import { createHash } from "node:crypto";
+import { newId } from "../core/ids.js";
+import { loadSecret } from "./secrets.js";
+import { fetchNotionPages, notionFetcher, validateNotionToken, } from "./notion.js";
+import { fetchSlackChannels, SLACK_MANIFEST, slackFetcher, validateSlackToken } from "./slack.js";
+import { acquireGoogleAuth, fetchGoogleSources, googleApi } from "./google.js";
+export const connectors = {
+    notion: {
+        label: "Notion",
+        sourceKind: "page",
+        connectHint: "token paste, ~2 minutes",
+        tokenHint: "ntn_… or secret_…",
+        instructions: [
+            "Open https://www.notion.so/profile/integrations and click “New integration”",
+            "Type: Internal. Pick the workspace, save, and copy the integration secret",
+            "Share content: on any Notion page → ··· → Connections → add the integration",
+            "(subpages are included automatically — what you share is what athena sees)",
+        ],
+        validate: (token) => validateNotionToken(notionFetcher(token)),
+        fetch: (token, cursor, log) => fetchNotionPages(notionFetcher(token), cursor, log),
+    },
+    slack: {
+        label: "Slack",
+        sourceKind: "thread",
+        connectHint: "app from a manifest, ~3 minutes",
+        tokenHint: "xoxp-…",
+        instructions: [
+            "Open https://api.slack.com/apps?new_app=1 and pick “From a manifest”",
+            "Choose your workspace, paste the manifest below, and create the app",
+            "Click “Install to Workspace” and allow it",
+            "OAuth & Permissions → copy the User OAuth Token (read-only scopes;",
+            "athena syncs only public channels you are a member of)",
+        ],
+        extra: SLACK_MANIFEST,
+        validate: (token) => validateSlackToken(slackFetcher(token)),
+        fetch: (token, cursor, log) => fetchSlackChannels(slackFetcher(token), cursor, log),
+    },
+    google: {
+        label: "Google (Drive + Gmail)",
+        sourceKind: "document",
+        connectHint: "browser sign-in, ~1 minute",
+        instructions: [
+            "A browser window opens — sign in and allow read-only access to Drive and Gmail",
+            "During friend testing Google shows an “unverified app” warning:",
+            "Advanced → “Go to athena (unsafe)” — expected until the app is verified",
+        ],
+        acquire: (rl, log) => acquireGoogleAuth(rl, log),
+        fetch: async (secret, cursor, log) => fetchGoogleSources(await googleApi(secret), cursor, log),
+    },
+};
+const INFLIGHT_FRESH_MS = 15 * 60 * 1000;
+export async function syncConnection(store, connection, log = () => { }) {
+    const token = loadSecret(connection.id);
+    if (!token)
+        throw new Error(`no token stored for ${connection.provider} — reconnect with: athena connect ${connection.provider}`);
+    const connector = connectors[connection.provider];
+    // One sync per connection at a time, across processes (serve timer vs CLI):
+    // a fresh in-flight marker means someone else is on it. Stale markers from
+    // crashed runs expire instead of wedging the connection.
+    const inflightKey = `sync.inflight.${connection.id}`;
+    const inflightSince = store.getMeta(inflightKey);
+    if (inflightSince && Date.now() - Date.parse(inflightSince) < INFLIGHT_FRESH_MS) {
+        throw new Error(`a ${connection.provider} sync is already running (since ${inflightSince}) — it can take minutes on big workspaces`);
+    }
+    store.setMeta(inflightKey, new Date().toISOString());
+    let fetched;
+    try {
+        fetched = await connector.fetch(token, connection.cursor, log);
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        store.saveConnection({ ...connection, lastError: message });
+        store.setMeta(inflightKey, "");
+        throw error;
+    }
+    const result = { added: 0, updated: 0, unchanged: 0 };
+    for (const item of fetched.items) {
+        const contentHash = hashContent(`${item.title}\n${item.content}`);
+        const existing = store.getSourceByOriginUri(item.uri);
+        if (existing?.contentHash === contentHash) {
+            result.unchanged += 1;
+            continue;
+        }
+        store.saveSource({
+            id: existing?.id ?? newId("src"),
+            kind: item.kind ?? connector.sourceKind,
+            title: item.title,
+            content: item.content,
+            contentHash,
+            origin: { connector: connection.provider, uri: item.uri },
+            capturedAt: new Date().toISOString(),
+            accessState: "private",
+        });
+        result[existing ? "updated" : "added"] += 1;
+    }
+    const { lastError: _cleared, ...rest } = connection;
+    store.saveConnection({
+        ...rest,
+        lastSyncedAt: new Date().toISOString(),
+        ...(fetched.cursor !== undefined ? { cursor: fetched.cursor } : {}),
+    });
+    store.setMeta(inflightKey, "");
+    return result;
+}
+const SYNC_DUE_AFTER_MS = 4 * 60 * 60 * 1000;
+/** Sync every connection not synced in the last 4 hours; failures land on the connection, not the caller. */
+export async function syncDueConnections(store, log = () => { }) {
+    for (const connection of store.listConnections()) {
+        const last = connection.lastSyncedAt ? Date.parse(connection.lastSyncedAt) : 0;
+        if (Date.now() - last < SYNC_DUE_AFTER_MS)
+            continue;
+        try {
+            const result = await syncConnection(store, connection, log);
+            log(`${connection.provider}: +${result.added} new, ${result.updated} updated, ${result.unchanged} unchanged`);
+        }
+        catch (error) {
+            log(`${connection.provider} sync failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+}
+function hashContent(content) {
+    return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}

package/dist/store/store.js

@@ -65,6 +65,9 @@ export class AthenaStore {
         recorded_at TEXT NOT NULL, matched_outcome_id TEXT, data TEXT NOT NULL
       );
       CREATE INDEX IF NOT EXISTS idx_drafts_unmatched ON drafts(domain, recorded_at) WHERE matched_outcome_id IS NULL;
+      CREATE TABLE IF NOT EXISTS connections (
+        id TEXT PRIMARY KEY, provider TEXT NOT NULL, created_at TEXT NOT NULL, data TEXT NOT NULL
+      );
       CREATE TABLE IF NOT EXISTS meta (
         key TEXT PRIMARY KEY, value TEXT NOT NULL
       );
@@ -153,6 +156,30 @@ export class AthenaStore {
     getSource(id) {
         return this.getData("sources", id);
     }
+    /** Connector dedupe key: one source per external URI, updated in place on re-sync. */
+    getSourceByOriginUri(uri) {
+        const row = this.db
+            .prepare("SELECT data FROM sources WHERE json_extract(data, '$.origin.uri') = ? LIMIT 1")
+            .get(uri);
+        return row ? rowData(row) : undefined;
+    }
+    // --- connections (external knowledge sources; tokens live in the secret store) ---
+    saveConnection(connection) {
+        this.upsert("connections", "INSERT INTO connections (id, provider, created_at, data) VALUES (?, ?, ?, ?) " +
+            "ON CONFLICT(id) DO UPDATE SET data = excluded.data", [connection.id, connection.provider, connection.createdAt, JSON.stringify(connection)]);
+    }
+    getConnection(id) {
+        return this.getData("connections", id);
+    }
+    listConnections(provider) {
+        const rows = provider
+            ? this.db.prepare("SELECT data FROM connections WHERE provider = ? ORDER BY created_at").all(provider)
+            : this.db.prepare("SELECT data FROM connections ORDER BY created_at").all();
+        return rows.map((rowData));
+    }
+    deleteConnection(id) {
+        this.db.prepare("DELETE FROM connections WHERE id = ?").run(id);
+    }
     saveObject(object) {
         this.upsert("objects", "INSERT INTO objects (id, kind, name, data) VALUES (?, ?, ?, ?) " +
             "ON CONFLICT(id) DO UPDATE SET kind = excluded.kind, name = excluded.name, data = excluded.data", [object.id, object.kind, object.name, JSON.stringify(object)]);

package/docs/schema.md

@@ -269,6 +269,29 @@ type ObjectRelation = {
 };
 ```
 
+### Connection — the explicit layer's front door
+
+A connected external source — Notion, Slack, and Google (Drive + Gmail). Tokens never
+live on the record — they stay in `~/.athena/secrets.json` (chmod 0600, same convention
+as provider keys in config.json). Sync is incremental per provider (newest-edit
+watermark in `cursor`), deduped one source per external URI: re-syncs update the same
+`RawSource` in place when content changed and skip it when not. Synced content enters
+as `accessState: "private"`; what the user shares with the integration is the privacy
+boundary. Connectors own provider work only — they never write the store directly, the
+sync runner does.
+
+```ts
+type Connection = {
+  id: ConnectionId;            // con_…
+  provider: "notion" | "slack" | "google"; // widens as connectors land
+  label: string;               // human name, e.g. the Notion workspace
+  createdAt: string;
+  lastSyncedAt?: string;
+  cursor?: string;             // provider watermark for incremental sync
+  lastError?: string;          // last sync failure, cleared on success
+};
+```
+
 ### Fact — the explicit counterpart of a hypothesis
 
 A declarative, entity-grounded statement extracted from evidence ("Acme's renewal
@@ -406,8 +429,8 @@ No durable-mutation tools. `athena_record` proposals enter the review queue like
 
 SQLite, one file per workspace (`.athena/athena.db`):
 - `instances`, `hypotheses`, `sources`, `objects`, `relations`, `outcomes`, `briefs`,
-  `facts`, `drafts` — JSON column + extracted indexed columns (id, kind, domain, status,
-  observedAt, visibility).
+  `facts`, `drafts`, `connections` — JSON column + extracted indexed columns (id, kind,
+  domain, status, observedAt, visibility).
 - `meta` — key/value operational state (last learn run, auto-learn cooldown).
 - FTS5 over instance summaries/diffs, hypothesis rules, source content, fact statements.
 - `sqlite-vec` over situation/rule/source embeddings (optional lane — everything works without it).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "useathena",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "athena captures tacit knowledge from real work so agents become truly autonomous and reliable.",
   "license": "UNLICENSED",
   "repository": {