stellar-memory 0.5.0

package/dist/scanner/cloud/slack.js

@@ -0,0 +1,282 @@
+/**
+ * scanner/cloud/slack.ts — Slack connector.
+ *
+ * Collects: channel message history, thread conversations, pinned messages.
+ * Intentionally EXCLUDES: DMs and private channels (privacy by default).
+ * DMs can be included by setting credential `include_dms=true` explicitly.
+ *
+ * Authentication: Bot Token (xoxb-…) with scopes:
+ *   channels:history, channels:read, groups:history, groups:read,
+ *   pins:read, users:read (for display names)
+ * Credential key: `bot_token`
+ *
+ * Rate limits: Slack Tier 3 = 50 req/min. We add 60ms between requests.
+ * Exponential backoff on 429 honouring the Retry-After header.
+ *
+ * Incremental sync: channel history accepts `oldest` (Unix timestamp).
+ */
+import { createLogger } from '../../utils/logger.js';
+const log = createLogger('slack');
+const SLACK_API_BASE = 'https://slack.com/api';
+const MAX_RETRIES = 4;
+const REQUEST_GAP_MS = 60; // ≈50 req/min
+const MAX_MESSAGES_PER_CHANNEL = 200;
+// ---------------------------------------------------------------------------
+// SlackConnector
+// ---------------------------------------------------------------------------
+export class SlackConnector {
+    name = 'Slack';
+    type = 'slack';
+    token = null;
+    includeDMs = false;
+    lastRequestAt = 0;
+    userCache = new Map();
+    // -------------------------------------------------------------------------
+    // authenticate
+    // -------------------------------------------------------------------------
+    async authenticate(credentials) {
+        const token = credentials['bot_token'];
+        if (!token || !token.startsWith('xoxb-')) {
+            throw new Error('SlackConnector requires a "bot_token" credential starting with "xoxb-"');
+        }
+        // Verify by calling auth.test
+        const res = await this.call('auth.test', {}, token);
+        if (!res.ok) {
+            throw new Error(`Slack authentication failed: ${res.error ?? 'unknown error'}`);
+        }
+        this.token = token;
+        this.includeDMs = credentials['include_dms'] === 'true';
+        log.info('Slack authentication successful', {
+            team: res.team,
+            botUser: res.user,
+            includeDMs: this.includeDMs,
+        });
+    }
+    isAuthenticated() {
+        return this.token !== null;
+    }
+    // -------------------------------------------------------------------------
+    // fetchDocuments
+    // -------------------------------------------------------------------------
+    async fetchDocuments(since) {
+        this.assertAuthenticated();
+        log.info('Fetching Slack documents', { since: since?.toISOString() });
+        const channels = await this.listChannels();
+        log.info('Found Slack channels', { count: channels.length });
+        const docs = [];
+        for (const channel of channels) {
+            // Fetch pinned messages (highest value; always include regardless of since)
+            try {
+                const pins = await this.fetchPinnedMessages(channel);
+                docs.push(...pins);
+            }
+            catch (err) {
+                log.warn('Failed to fetch pinned messages', { channel: channel.name });
+                log.error('Pins error', err instanceof Error ? err : new Error(String(err)));
+            }
+            // Fetch recent message history
+            try {
+                const history = await this.fetchChannelHistory(channel, since);
+                if (history)
+                    docs.push(history);
+            }
+            catch (err) {
+                log.warn('Failed to fetch channel history', { channel: channel.name });
+                log.error('History error', err instanceof Error ? err : new Error(String(err)));
+            }
+        }
+        return docs;
+    }
+    // -------------------------------------------------------------------------
+    // toMemory
+    // -------------------------------------------------------------------------
+    toMemory(doc) {
+        const isPinned = doc.metadata['docType'] === 'pinned';
+        return {
+            content: doc.content,
+            summary: doc.title.slice(0, 120),
+            type: isPinned ? 'context' : 'observation',
+            tags: ['slack', 'cloud', String(doc.metadata['channelName'] ?? '')].filter(Boolean),
+            source: 'cloud',
+            source_path: doc.url,
+            metadata: {
+                ...doc.metadata,
+                cloudService: 'slack',
+                lastModified: doc.lastModified.toISOString(),
+            },
+        };
+    }
+    // -------------------------------------------------------------------------
+    // Private helpers
+    // -------------------------------------------------------------------------
+    assertAuthenticated() {
+        if (!this.isAuthenticated()) {
+            throw new Error('SlackConnector: not authenticated. Call authenticate() first.');
+        }
+    }
+    async call(method, params, token) {
+        const now = Date.now();
+        const elapsed = now - this.lastRequestAt;
+        if (elapsed < REQUEST_GAP_MS) {
+            await sleep(REQUEST_GAP_MS - elapsed);
+        }
+        this.lastRequestAt = Date.now();
+        const searchParams = new URLSearchParams(Object.entries(params).map(([k, v]) => [k, String(v)]));
+        const res = await fetchWithBackoff(`${SLACK_API_BASE}/${method}?${searchParams}`, {
+            headers: { Authorization: `Bearer ${token ?? this.token}` },
+        });
+        if (!res.ok) {
+            throw new Error(`Slack API HTTP error: ${res.status}`);
+        }
+        return res.json();
+    }
+    async listChannels() {
+        const types = this.includeDMs
+            ? 'public_channel,private_channel,im,mpim'
+            : 'public_channel,private_channel';
+        const data = await this.call('conversations.list', {
+            limit: 200,
+            exclude_archived: true,
+            types,
+        });
+        if (!data.ok) {
+            throw new Error(`conversations.list failed: ${data.error}`);
+        }
+        return data['channels'] ?? [];
+    }
+    async fetchPinnedMessages(channel) {
+        const data = await this.call('pins.list', { channel: channel.id });
+        if (!data.ok)
+            return [];
+        const pins = data['items'] ?? [];
+        const docs = [];
+        for (const pin of pins) {
+            if (!pin.message?.text)
+                continue;
+            const msg = pin.message;
+            const ts = parseSlackTs(msg.ts);
+            const userName = await this.resolveUser(msg.user);
+            const url = buildSlackUrl(channel.id, msg.ts);
+            docs.push({
+                id: `slack-pin-${channel.id}-${msg.ts}`,
+                title: `Pinned in #${channel.name}: ${msg.text.slice(0, 80)}`,
+                content: `**Pinned message in #${channel.name}**\n\n${msg.text}`,
+                url,
+                mimeType: 'text/markdown',
+                lastModified: ts,
+                author: userName,
+                metadata: {
+                    channelId: channel.id,
+                    channelName: channel.name,
+                    docType: 'pinned',
+                    ts: msg.ts,
+                },
+            });
+        }
+        return docs;
+    }
+    async fetchChannelHistory(channel, since) {
+        const params = {
+            channel: channel.id,
+            limit: MAX_MESSAGES_PER_CHANNEL,
+        };
+        if (since) {
+            params['oldest'] = (since.getTime() / 1000).toFixed(6);
+        }
+        const data = await this.call('conversations.history', params);
+        if (!data.ok)
+            return null;
+        const messages = data['messages'] ?? [];
+        if (messages.length === 0)
+            return null;
+        // Collect threads for messages with replies
+        const lines = [`## #${channel.name} — recent messages\n`];
+        for (const msg of messages.slice().reverse()) {
+            const ts = parseSlackTs(msg.ts);
+            const userName = await this.resolveUser(msg.user);
+            const dateStr = ts.toISOString().slice(0, 16).replace('T', ' ');
+            lines.push(`**${userName}** (${dateStr}):\n${msg.text}\n`);
+            // Fetch thread replies if any
+            if (msg.thread_ts && msg.reply_count && msg.reply_count > 0) {
+                const thread = await this.fetchThread(channel.id, msg.thread_ts);
+                for (const reply of thread.slice(1)) { // skip first (= parent)
+                    const rUser = await this.resolveUser(reply.user);
+                    const rTs = parseSlackTs(reply.ts).toISOString().slice(0, 16).replace('T', ' ');
+                    lines.push(`  > **${rUser}** (${rTs}):\n  > ${reply.text}\n`);
+                }
+            }
+        }
+        const newestTs = messages[0]?.ts ?? String(Date.now() / 1000);
+        return {
+            id: `slack-history-${channel.id}`,
+            title: `#${channel.name} history`,
+            content: lines.join('\n'),
+            url: buildSlackUrl(channel.id),
+            mimeType: 'text/markdown',
+            lastModified: parseSlackTs(newestTs),
+            metadata: {
+                channelId: channel.id,
+                channelName: channel.name,
+                messageCount: messages.length,
+                docType: 'channel_history',
+            },
+        };
+    }
+    async fetchThread(channelId, threadTs) {
+        const data = await this.call('conversations.replies', {
+            channel: channelId,
+            ts: threadTs,
+            limit: 50,
+        });
+        if (!data.ok)
+            return [];
+        return data['messages'] ?? [];
+    }
+    async resolveUser(userId) {
+        if (!userId)
+            return 'Unknown';
+        const cached = this.userCache.get(userId);
+        if (cached)
+            return cached;
+        try {
+            const data = await this.call('users.info', { user: userId });
+            if (data.ok) {
+                const user = data['user'];
+                const name = user.real_name ?? user.name ?? userId;
+                this.userCache.set(userId, name);
+                return name;
+            }
+        }
+        catch {
+            // Non-critical; fallback to userId
+        }
+        this.userCache.set(userId, userId);
+        return userId;
+    }
+}
+// ---------------------------------------------------------------------------
+// Utilities
+// ---------------------------------------------------------------------------
+function parseSlackTs(ts) {
+    const seconds = parseFloat(ts);
+    return new Date(seconds * 1000);
+}
+function buildSlackUrl(channelId, ts) {
+    const base = `https://app.slack.com/client/${channelId}`;
+    return ts ? `${base}/${channelId}-${ts.replace('.', '')}` : base;
+}
+async function fetchWithBackoff(url, init, attempt = 0) {
+    const res = await fetch(url, init);
+    if (res.status === 429 && attempt < MAX_RETRIES) {
+        const retryAfter = parseInt(res.headers.get('Retry-After') ?? '1', 10);
+        const delay = Math.max(retryAfter * 1000, Math.min(1000 * 2 ** attempt, 30_000));
+        log.debug('Slack rate limited, retrying', { delay, attempt });
+        await sleep(delay);
+        return fetchWithBackoff(url, init, attempt + 1);
+    }
+    return res;
+}
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+//# sourceMappingURL=slack.js.map

package/dist/scanner/cloud/slack.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"slack.js","sourceRoot":"","sources":["../../../src/scanner/cloud/slack.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAErD,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,cAAc,GAAK,uBAAuB,CAAC;AACjD,MAAM,WAAW,GAAQ,CAAC,CAAC;AAC3B,MAAM,cAAc,GAAK,EAAE,CAAC,CAAC,cAAc;AAC3C,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAqCrC,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,OAAO,cAAc;IAChB,IAAI,GAAG,OAAO,CAAC;IACf,IAAI,GAAG,OAAgB,CAAC;IAEzB,KAAK,GAAkB,IAAI,CAAC;IAC5B,UAAU,GAAG,KAAK,CAAC;IACnB,aAAa,GAAG,CAAC,CAAC;IAClB,SAAS,GAAc,IAAI,GAAG,EAAE,CAAC;IAEzC,4EAA4E;IAC5E,eAAe;IACf,4EAA4E;IAE5E,KAAK,CAAC,YAAY,CAAC,WAAmC;QACpD,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CACb,wEAAwE,CACzE,CAAC;QACJ,CAAC;QAED,8BAA8B;QAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC;QACpD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gCAAgC,GAAG,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;QAClF,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,aAAa,CAAC,KAAK,MAAM,CAAC;QAExD,GAAG,CAAC,IAAI,CAAC,iCAAiC,EAAE;YAC1C,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,OAAO,EAAE,GAAG,CAAC,IAAI;YACjB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC;IAC7B,CAAC;IAED,4EAA4E;IAC5E,iBAAiB;IACjB,4EAA4E;IAE5E,KAAK,CAAC,cAAc,CAAC,KAAY;QAC/B,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAC3B,GAAG,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;QAEtE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC3C,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAE7D,MAAM,IAAI,GAAoB,EAAE,CAAC;QAEjC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,4EAA4E;YAC5E,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;gBACrD,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;YACrB,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvE,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/E,CAAC;YAED,+BAA+B;YAC/B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC/D,IAAI,OAAO;oBAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBACvE,GAAG,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,4EAA4E;IAC5E,WAAW;IACX,4EAA4E;IAE5E,QAAQ,CAAC,GAAkB;QACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC;QACtD,OAAO;YACL,OAAO,EAAM,GAAG,CAAC,OAAO;YACxB,OAAO,EAAM,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACpC,IAAI,EAAS,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa;YACjD,IAAI,EAAS,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;YAC1F,MAAM,EAAO,OAAO;YACpB,WAAW,EAAE,GAAG,CAAC,GAAG;YACpB,QAAQ,EAAE;gBACR,GAAG,GAAG,CAAC,QAAQ;gBACf,YAAY,EAAE,OAAO;gBACrB,YAAY,EAAE,GAAG,CAAC,YAAY,CAAC,WAAW,EAAE;aAC7C;SACF,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,kBAAkB;IAClB,4EAA4E;IAEpE,mBAAmB;QACzB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,IAAI,CAChB,MAAc,EACd,MAAiD,EACjD,KAAc;QAEd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAC,aAAa,CAAC;QACzC,IAAI,OAAO,GAAG,cAAc,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEhC,MAAM,YAAY,GAAG,IAAI,eAAe,CACtC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CACvD,CAAC;QAEF,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,GAAG,cAAc,IAAI,MAAM,IAAI,YAAY,EAAE,EAAE;YAChF,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,KAAK,IAAI,IAAI,CAAC,KAAM,EAAE,EAAE;SAC7D,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,GAAG,CAAC,IAAI,EAAwE,CAAC;IAC1F,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU;YAC3B,CAAC,CAAC,wCAAwC;YAC1C,CAAC,CAAC,gCAAgC,CAAC;QAErC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE;YACjD,KAAK,EAAE,GAAG;YACV,gBAAgB,EAAE,IAAI;YACtB,KAAK;SACN,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,OAAQ,IAAI,CAAC,UAAU,CAAgC,IAAI,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,OAAqB;QACrD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QAExB,MAAM,IAAI,GAAI,IAAI,CAAC,OAAO,CAA4B,IAAI,EAAE,CAAC;QAC7D,MAAM,IAAI,GAAoB,EAAE,CAAC;QAEjC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI;gBAAE,SAAS;YAEjC,MAAM,GAAG,GAAQ,GAAG,CAAC,OAAO,CAAC;YAC7B,MAAM,EAAE,GAAS,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,GAAG,GAAQ,aAAa,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAEnD,IAAI,CAAC,IAAI,CAAC;gBACR,EAAE,EAAY,aAAa,OAAO,CAAC,EAAE,IAAI,GAAG,CAAC,EAAE,EAAE;gBACjD,KAAK,EAAS,cAAc,OAAO,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;gBACpE,OAAO,EAAO,wBAAwB,OAAO,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,EAAE;gBACrE,GAAG;gBACH,QAAQ,EAAM,eAAe;gBAC7B,YAAY,EAAE,EAAE;gBAChB,MAAM,EAAQ,QAAQ;gBACtB,QAAQ,EAAE;oBACR,SAAS,EAAI,OAAO,CAAC,EAAE;oBACvB,WAAW,EAAE,OAAO,CAAC,IAAI;oBACzB,OAAO,EAAM,QAAQ;oBACrB,EAAE,EAAW,GAAG,CAAC,EAAE;iBACpB;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,OAAqB,EACrB,KAAY;QAEZ,MAAM,MAAM,GAA8C;YACxD,OAAO,EAAE,OAAO,CAAC,EAAE;YACnB,KAAK,EAAI,wBAAwB;SAClC,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,MAAM,CAAC,CAAC;QAC9D,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QAE1B,MAAM,QAAQ,GAAI,IAAI,CAAC,UAAU,CAAgC,IAAI,EAAE,CAAC;QACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEvC,4CAA4C;QAC5C,MAAM,KAAK,GAAa,CAAC,OAAO,OAAO,CAAC,IAAI,sBAAsB,CAAC,CAAC;QAEpE,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7C,MAAM,EAAE,GAAS,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClD,MAAM,OAAO,GAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,OAAO,OAAO,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC;YAE3D,8BAA8B;YAC9B,IAAI,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBAC5D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC;gBACjE,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAE,wBAAwB;oBAC9D,MAAM,KAAK,GAAK,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACnD,MAAM,GAAG,GAAO,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;oBACpF,KAAK,CAAC,IAAI,CAAC,SAAS,KAAK,OAAO,GAAG,WAAW,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAE9D,OAAO;YACL,EAAE,EAAY,iBAAiB,OAAO,CAAC,EAAE,EAAE;YAC3C,KAAK,EAAS,IAAI,OAAO,CAAC,IAAI,UAAU;YACxC,OAAO,EAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC9B,GAAG,EAAW,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;YACvC,QAAQ,EAAM,eAAe;YAC7B,YAAY,EAAE,YAAY,CAAC,QAAQ,CAAC;YACpC,QAAQ,EAAE;gBACR,SAAS,EAAK,OAAO,CAAC,EAAE;gBACxB,WAAW,EAAG,OAAO,CAAC,IAAI;gBAC1B,YAAY,EAAE,QAAQ,CAAC,MAAM;gBAC7B,OAAO,EAAO,iBAAiB;aAChC;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,SAAiB,EAAE,QAAgB;QAC3D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE;YACpD,OAAO,EAAE,SAAS;YAClB,EAAE,EAAO,QAAQ;YACjB,KAAK,EAAI,EAAE;SACZ,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,EAAE,CAAC;QACxB,OAAQ,IAAI,CAAC,UAAU,CAAgC,IAAI,EAAE,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,MAAe;QACvC,IAAI,CAAC,MAAM;YAAE,OAAO,SAAS,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAC7D,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAc,CAAC;gBACvC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC;gBACnD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACjC,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,mCAAmC;QACrC,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACnC,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,SAAS,YAAY,CAAC,EAAU;IAC9B,MAAM,OAAO,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IAC/B,OAAO,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,aAAa,CAAC,SAAiB,EAAE,EAAW;IACnD,MAAM,IAAI,GAAG,gCAAgC,SAAS,EAAE,CAAC;IACzD,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,SAAS,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACnE,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,IAAiB,EAAE,OAAO,GAAG,CAAC;IACzE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACnC,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QACvE,MAAM,KAAK,GAAQ,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QACtF,GAAG,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9D,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO,gBAAgB,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AACzD,CAAC"}

package/dist/scanner/cloud/types.d.ts

@@ -0,0 +1,73 @@
+/**
+ * scanner/cloud/types.ts — Common interfaces for cloud connectors.
+ *
+ * Each cloud connector (Google Drive, Notion, GitHub, Slack) implements
+ * CloudConnector. The CloudDocument is the neutral transport format;
+ * toMemory() maps it into the shape required by the storage layer.
+ */
+import type { MemoryType } from '../../engine/types.js';
+export interface CloudDocument {
+    /** Service-specific unique identifier */
+    id: string;
+    /** Human-readable title / subject */
+    title: string;
+    /** Plain-text body content */
+    content: string;
+    /** Canonical URL to the document */
+    url: string;
+    /** MIME type (e.g., "text/plain", "text/markdown") */
+    mimeType: string;
+    /** Last modification timestamp on the cloud service */
+    lastModified: Date;
+    /** Optional display name of the document author */
+    author?: string;
+    /** Arbitrary service-specific metadata */
+    metadata: Record<string, unknown>;
+}
+export interface MemoryCreateInput {
+    content: string;
+    summary: string;
+    type: MemoryType;
+    tags: string[];
+    source: 'cloud';
+    source_path: string;
+    metadata: Record<string, unknown>;
+}
+export interface CloudConnector {
+    /** Human-readable service name */
+    readonly name: string;
+    /** Discriminated union key used for routing / logging */
+    readonly type: 'google-drive' | 'notion' | 'github' | 'slack';
+    /**
+     * Authenticate with the cloud service.
+     * Credentials are passed as a plain-object map so each connector can define
+     * its own required keys without leaking them into the interface.
+     * Credentials must come from environment variables or a local config file —
+     * never from user-supplied untrusted input directly.
+     */
+    authenticate(credentials: Record<string, string>): Promise<void>;
+    /** Returns true if the connector holds valid authentication state */
+    isAuthenticated(): boolean;
+    /**
+     * Fetch documents modified since the given date.
+     * When `since` is omitted the connector performs a full initial scan.
+     * Implementors must honour rate limits and apply exponential backoff on 429s.
+     */
+    fetchDocuments(since?: Date): Promise<CloudDocument[]>;
+    /**
+     * Convert a fetched CloudDocument into the shape needed to create a memory.
+     * Implementors must populate source_path with the document URL so the
+     * deduplication index can find duplicates on re-sync.
+     */
+    toMemory(doc: CloudDocument): MemoryCreateInput;
+}
+export type ConnectorType = CloudConnector['type'];
+/** Represents the persisted authentication state for a connector */
+export interface ConnectorAuthState {
+    type: ConnectorType;
+    authenticated: boolean;
+    authenticatedAt: Date | null;
+    /** Redacted credential summary — never store raw secrets here */
+    summary: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/scanner/cloud/types.js

@@ -0,0 +1,9 @@
+/**
+ * scanner/cloud/types.ts — Common interfaces for cloud connectors.
+ *
+ * Each cloud connector (Google Drive, Notion, GitHub, Slack) implements
+ * CloudConnector. The CloudDocument is the neutral transport format;
+ * toMemory() maps it into the shape required by the storage layer.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/scanner/cloud/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/scanner/cloud/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG"}

package/dist/scanner/index.d.ts

@@ -0,0 +1,35 @@
+import type { ScanConfig, ScanResult } from './types.js';
+export type ScanProgressEvent = {
+    phase: 'collecting' | 'collected' | 'processing' | 'path_complete';
+    path: string;
+    currentFile?: string;
+    totalFiles?: number;
+    scannedFiles?: number;
+    createdMemories?: number;
+    skippedFiles?: number;
+    errorFiles?: number;
+};
+export declare const FULL_SCAN_EXTRA_EXCLUDES: string[];
+export declare class StellarScanner {
+    private readonly config;
+    private stopWatchers;
+    constructor(config?: Partial<ScanConfig>);
+    scan(): Promise<ScanResult>;
+    scanPath(dirPath: string, opts?: {
+        recursive?: boolean;
+        includeGit?: boolean;
+    }): Promise<ScanResult>;
+    scanWithProgress(opts: {
+        paths?: string[];
+        includeGit?: boolean;
+        onProgress: (event: ScanProgressEvent) => void;
+        abortSignal?: AbortSignal;
+    }): Promise<ScanResult>;
+    watch(): void;
+    stop(): void;
+    private _processFile;
+    private _insertGitMemories;
+    private _ensureDataSource;
+}
+export declare function listDataSources(): import("./types.js").DataSource[];
+//# sourceMappingURL=index.d.ts.map