research-powerpack-mcp 3.0.0

package/dist/clients/reddit.js

@@ -0,0 +1,179 @@
+/**
+ * Reddit OAuth API Client
+ * Fetches posts and comments sorted by score (most upvoted first)
+ */
+import { REDDIT } from '../config/index.js';
+export function calculateCommentAllocation(postCount) {
+    const totalBudget = REDDIT.MAX_COMMENT_BUDGET;
+    const perPostBase = Math.floor(totalBudget / postCount);
+    const perPostCapped = Math.min(perPostBase, REDDIT.MAX_COMMENTS_PER_POST);
+    return { totalBudget, perPostBase, perPostCapped, redistributed: false };
+}
+export class RedditClient {
+    clientId;
+    clientSecret;
+    token = null;
+    tokenExpiry = 0;
+    userAgent = 'script:research-powerpack-mcp:v3.0 (by /u/research-powerpack)';
+    constructor(clientId, clientSecret) {
+        this.clientId = clientId;
+        this.clientSecret = clientSecret;
+    }
+    async auth() {
+        if (this.token && Date.now() < this.tokenExpiry - 60000)
+            return this.token;
+        const credentials = Buffer.from(`${this.clientId}:${this.clientSecret}`).toString('base64');
+        const res = await fetch('https://www.reddit.com/api/v1/access_token', {
+            method: 'POST',
+            headers: {
+                'Authorization': `Basic ${credentials}`,
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'User-Agent': this.userAgent,
+            },
+            body: 'grant_type=client_credentials',
+        });
+        if (!res.ok) {
+            const text = await res.text().catch(() => '');
+            throw new Error(`Reddit auth failed: ${res.status} ${text}`);
+        }
+        const data = await res.json();
+        this.token = data.access_token;
+        this.tokenExpiry = Date.now() + data.expires_in * 1000;
+        return this.token;
+    }
+    parseUrl(url) {
+        const m = url.match(/reddit\.com\/r\/([^\/]+)\/comments\/([a-z0-9]+)/i);
+        return m ? { sub: m[1], id: m[2] } : null;
+    }
+    async getPost(url, maxComments = 100) {
+        const parsed = this.parseUrl(url);
+        if (!parsed)
+            throw new Error(`Invalid Reddit URL: ${url}`);
+        const token = await this.auth();
+        const limit = Math.min(maxComments, 500);
+        let lastError = null;
+        for (let attempt = 0; attempt < REDDIT.RETRY_COUNT; attempt++) {
+            try {
+                const apiUrl = `https://oauth.reddit.com/r/${parsed.sub}/comments/${parsed.id}?sort=top&limit=${limit}&depth=10&raw_json=1`;
+                const res = await fetch(apiUrl, {
+                    headers: {
+                        'Authorization': `Bearer ${token}`,
+                        'User-Agent': this.userAgent,
+                    },
+                });
+                if (res.status === 429) {
+                    const delay = REDDIT.RETRY_DELAYS[attempt] || 32000;
+                    console.error(`[Reddit] Rate limited. Retry ${attempt + 1}/${REDDIT.RETRY_COUNT} after ${delay}ms`);
+                    await this.delay(delay);
+                    continue;
+                }
+                if (!res.ok)
+                    throw new Error(`Reddit API error: ${res.status}`);
+                const [postListing, commentListing] = await res.json();
+                const p = postListing?.data?.children?.[0]?.data;
+                if (!p)
+                    throw new Error(`Post not found: ${url}`);
+                const post = {
+                    title: p.title,
+                    author: p.author || '[deleted]',
+                    subreddit: p.subreddit,
+                    body: this.formatBody(p),
+                    score: p.score,
+                    commentCount: p.num_comments,
+                    url: `https://reddit.com${p.permalink}`,
+                    created: new Date(p.created_utc * 1000),
+                    flair: p.link_flair_text || undefined,
+                    isNsfw: p.over_18 || false,
+                    isPinned: p.stickied || false,
+                };
+                const comments = this.extractComments(commentListing?.data?.children || [], maxComments, post.author);
+                return { post, comments, allocatedComments: maxComments, actualComments: post.commentCount };
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                if (attempt < REDDIT.RETRY_COUNT - 1) {
+                    const delay = REDDIT.RETRY_DELAYS[attempt] || 2000;
+                    console.error(`[Reddit] Error: ${lastError.message}. Retry ${attempt + 1}/${REDDIT.RETRY_COUNT}`);
+                    await this.delay(delay);
+                }
+            }
+        }
+        throw lastError || new Error('Failed to fetch Reddit post');
+    }
+    formatBody(p) {
+        if (p.selftext?.trim())
+            return p.selftext;
+        if (p.is_self)
+            return '';
+        if (p.url)
+            return `**Link:** ${p.url}`;
+        return '';
+    }
+    extractComments(children, maxComments, opAuthor) {
+        const result = [];
+        const extract = (items, depth = 0) => {
+            const sorted = [...items].sort((a, b) => (b.data?.score || 0) - (a.data?.score || 0));
+            for (const c of sorted) {
+                if (result.length >= maxComments)
+                    return;
+                if (c.kind !== 't1' || !c.data?.author || c.data.author === '[deleted]')
+                    continue;
+                result.push({
+                    author: c.data.author,
+                    body: c.data.body || '',
+                    score: c.data.score || 0,
+                    depth,
+                    isOP: c.data.author === opAuthor,
+                });
+                if (c.data.replies?.data?.children && result.length < maxComments) {
+                    extract(c.data.replies.data.children, depth + 1);
+                }
+            }
+        };
+        extract(children);
+        return result;
+    }
+    async getPosts(urls, maxComments = 100) {
+        if (urls.length <= REDDIT.BATCH_SIZE) {
+            const results = await Promise.all(urls.map(u => this.getPost(u, maxComments).catch(e => e)));
+            return new Map(urls.map((u, i) => [u, results[i]]));
+        }
+        return (await this.batchGetPosts(urls, maxComments)).results;
+    }
+    async batchGetPosts(urls, maxCommentsOverride, fetchComments = true, onBatchComplete) {
+        const totalBatches = Math.ceil(urls.length / REDDIT.BATCH_SIZE);
+        const allResults = new Map();
+        let rateLimitHits = 0;
+        const allocation = calculateCommentAllocation(urls.length);
+        const commentsPerPost = fetchComments ? (maxCommentsOverride || allocation.perPostCapped) : 0;
+        console.error(`[Reddit] Fetching ${urls.length} posts in ${totalBatches} batch(es), ${commentsPerPost} comments/post`);
+        for (let batchNum = 0; batchNum < totalBatches; batchNum++) {
+            const startIdx = batchNum * REDDIT.BATCH_SIZE;
+            const batchUrls = urls.slice(startIdx, startIdx + REDDIT.BATCH_SIZE);
+            console.error(`[Reddit] Batch ${batchNum + 1}/${totalBatches} (${batchUrls.length} posts)`);
+            const batchResults = await Promise.allSettled(batchUrls.map(url => this.getPost(url, commentsPerPost)));
+            for (let i = 0; i < batchResults.length; i++) {
+                const result = batchResults[i];
+                const url = batchUrls[i] || '';
+                if (result.status === 'fulfilled') {
+                    allResults.set(url, result.value);
+                }
+                else {
+                    const errorMsg = result.reason?.message || String(result.reason);
+                    if (errorMsg.includes('429') || errorMsg.includes('rate'))
+                        rateLimitHits++;
+                    allResults.set(url, new Error(errorMsg));
+                }
+            }
+            onBatchComplete?.(batchNum + 1, totalBatches, allResults.size);
+            console.error(`[Reddit] Batch ${batchNum + 1} complete (${allResults.size}/${urls.length})`);
+            if (batchNum < totalBatches - 1)
+                await this.delay(500);
+        }
+        return { results: allResults, batchesProcessed: totalBatches, totalPosts: urls.length, rateLimitHits, commentAllocation: allocation };
+    }
+    delay(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+}
+//# sourceMappingURL=reddit.js.map

package/dist/clients/reddit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reddit.js","sourceRoot":"","sources":["../../src/clients/reddit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AA8C5C,MAAM,UAAU,0BAA0B,CAAC,SAAiB;IAC1D,MAAM,WAAW,GAAG,MAAM,CAAC,kBAAkB,CAAC;IAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC,CAAC;IACxD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAC1E,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,YAAY;IAKH;IAA0B;IAJtC,KAAK,GAAkB,IAAI,CAAC;IAC5B,WAAW,GAAG,CAAC,CAAC;IAChB,SAAS,GAAG,+DAA+D,CAAC;IAEpF,YAAoB,QAAgB,EAAU,YAAoB;QAA9C,aAAQ,GAAR,QAAQ,CAAQ;QAAU,iBAAY,GAAZ,YAAY,CAAQ;IAAG,CAAC;IAE9D,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QAE3E,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAE5F,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE;YACpE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,eAAe,EAAE,SAAS,WAAW,EAAE;gBACvC,cAAc,EAAE,mCAAmC;gBACnD,YAAY,EAAE,IAAI,CAAC,SAAS;aAC7B;YACD,IAAI,EAAE,+BAA+B;SACtC,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAkD,CAAC;QAC9E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvD,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAEO,QAAQ,CAAC,GAAW;QAC1B,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACxE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,WAAW,GAAG,GAAG;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;QAE3D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QAEzC,IAAI,SAAS,GAAiB,IAAI,CAAC;QACnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YAC9D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,8BAA8B,MAAM,CAAC,GAAG,aAAa,MAAM,CAAC,EAAE,mBAAmB,KAAK,sBAAsB,CAAC;gBAE5H,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;oBAC9B,OAAO,EAAE;wBACP,eAAe,EAAE,UAAU,KAAK,EAAE;wBAClC,YAAY,EAAE,IAAI,CAAC,SAAS;qBAC7B;iBACF,CAAC,CAAC;gBAEH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvB,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC;oBACpD,OAAO,CAAC,KAAK,CAAC,gCAAgC,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,WAAW,UAAU,KAAK,IAAI,CAAC,CAAC;oBACpG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBACxB,SAAS;gBACX,CAAC;gBAED,IAAI,CAAC,GAAG,CAAC,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;gBAEhE,MAAM,CAAC,WAAW,EAAE,cAAc,CAAC,GAAG,MAAM,GAAG,CAAC,IAAI,EAAgB,CAAC;gBACrE,MAAM,CAAC,GAAG,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;gBACjD,IAAI,CAAC,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;gBAElD,MAAM,IAAI,GAAS;oBACjB,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,WAAW;oBAC/B,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;oBACxB,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,YAAY,EAAE,CAAC,CAAC,YAAY;oBAC5B,GAAG,EAAE,qBAAqB,CAAC,CAAC,SAAS,EAAE;oBACvC,OAAO,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC;oBACvC,KAAK,EAAE,CAAC,CAAC,eAAe,IAAI,SAAS;oBACrC,MAAM,EAAE,CAAC,CAAC,OAAO,IAAI,KAAK;oBAC1B,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,KAAK;iBAC9B,CAAC;gBAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,cAAc,EAAE,IAAI,EAAE,QAAQ,IAAI,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEtG,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,WAAW,EAAE,cAAc,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;YAC/F,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACtE,IAAI,OAAO,GAAG,MAAM,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;oBACrC,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;oBACnD,OAAO,CAAC,KAAK,CAAC,mBAAmB,SAAS,CAAC,OAAO,WAAW,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;oBAClG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,SAAS,IAAI,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC9D,CAAC;IAEO,UAAU,CAAC,CAAM;QACvB,IAAI,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAC1C,IAAI,CAAC,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,GAAG;YAAE,OAAO,aAAa,CAAC,CAAC,GAAG,EAAE,CAAC;QACvC,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,eAAe,CAAC,QAAe,EAAE,WAAmB,EAAE,QAAgB;QAC5E,MAAM,MAAM,GAAc,EAAE,CAAC;QAE7B,MAAM,OAAO,GAAG,CAAC,KAAY,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE;YAC1C,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;YAEtF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACvB,IAAI,MAAM,CAAC,MAAM,IAAI,WAAW;oBAAE,OAAO;gBACzC,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW;oBAAE,SAAS;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;oBACrB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE;oBACvB,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;oBACxB,KAAK;oBACL,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ;iBACjC,CAAC,CAAC;gBAEH,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;oBAClE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,OAAO,CAAC,QAAQ,CAAC,CAAC;QAClB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAc,EAAE,WAAW,GAAG,GAAG;QAC9C,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAU,CAAC,CAAC,CACnE,CAAC;YACF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,IAAc,EACd,mBAA4B,EAC5B,aAAa,GAAG,IAAI,EACpB,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;QACzD,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,MAAM,UAAU,GAAG,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3D,MAAM,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,mBAAmB,IAAI,UAAU,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9F,OAAO,CAAC,KAAK,CAAC,qBAAqB,IAAI,CAAC,MAAM,aAAa,YAAY,eAAe,eAAe,gBAAgB,CAAC,CAAC;QAEvH,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC;YAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;YAErE,OAAO,CAAC,KAAK,CAAC,kBAAkB,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,SAAS,CAAC,CAAC;YAE5F,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,CACzD,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;gBACpC,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAAE,aAAa,EAAE,CAAC;oBAC3E,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;YAED,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;YAC/D,OAAO,CAAC,KAAK,CAAC,kBAAkB,QAAQ,GAAG,CAAC,cAAc,UAAU,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YAE7F,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC;gBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,iBAAiB,EAAE,UAAU,EAAE,CAAC;IACxI,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;CACF"}

package/dist/clients/research.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Deep Research Client
+ * Handles research API requests with web search capabilities
+ */
+export interface ResearchParams {
+    question: string;
+    systemPrompt?: string;
+    reasoningEffort?: 'low' | 'medium' | 'high';
+    maxSearchResults?: number;
+    maxTokens?: number;
+    temperature?: number;
+    responseFormat?: {
+        type: 'json_object' | 'text';
+    };
+}
+export interface ResearchResponse {
+    id: string;
+    model: string;
+    created: number;
+    content: string;
+    finishReason?: string;
+    usage?: {
+        promptTokens: number;
+        completionTokens: number;
+        totalTokens: number;
+        sourcesUsed?: number;
+    };
+    annotations?: Array<{
+        type: 'url_citation';
+        url: string;
+        title: string;
+        startIndex: number;
+        endIndex: number;
+    }>;
+}
+export declare class ResearchClient {
+    private client;
+    constructor();
+    research(params: ResearchParams): Promise<ResearchResponse>;
+}
+//# sourceMappingURL=research.d.ts.map

package/dist/clients/research.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"research.d.ts","sourceRoot":"","sources":["../../src/clients/research.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;IAC5C,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,aAAa,GAAG,MAAM,CAAA;KAAE,CAAC;CACnD;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,gBAAgB,EAAE,MAAM,CAAC;QACzB,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,WAAW,CAAC,EAAE,KAAK,CAAC;QAClB,IAAI,EAAE,cAAc,CAAC;QACrB,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;CACJ;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAS;;IAcjB,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAqElE"}

package/dist/clients/research.js

@@ -0,0 +1,77 @@
+/**
+ * Deep Research Client
+ * Handles research API requests with web search capabilities
+ */
+import OpenAI from 'openai';
+import { RESEARCH } from '../config/index.js';
+export class ResearchClient {
+    client;
+    constructor() {
+        if (!RESEARCH.API_KEY) {
+            throw new Error('OPENROUTER_API_KEY is required for research');
+        }
+        this.client = new OpenAI({
+            baseURL: RESEARCH.BASE_URL,
+            apiKey: RESEARCH.API_KEY,
+            timeout: RESEARCH.TIMEOUT_MS,
+        });
+    }
+    async research(params) {
+        const { question, systemPrompt, reasoningEffort = RESEARCH.REASONING_EFFORT, maxSearchResults = RESEARCH.MAX_URLS, maxTokens = 32000, temperature = 0.3, responseFormat, } = params;
+        const messages = [];
+        if (systemPrompt) {
+            messages.push({ role: 'system', content: systemPrompt });
+        }
+        messages.push({ role: 'user', content: question });
+        const requestPayload = {
+            model: RESEARCH.MODEL,
+            messages,
+            temperature,
+            reasoning_effort: reasoningEffort,
+            max_completion_tokens: maxTokens,
+            search_parameters: {
+                mode: 'on',
+                max_search_results: Math.min(maxSearchResults, 30),
+                return_citations: true,
+                sources: [{ type: 'web' }],
+            },
+        };
+        if (responseFormat) {
+            requestPayload.response_format = responseFormat;
+        }
+        try {
+            const response = await this.client.chat.completions.create(requestPayload);
+            const choice = response.choices?.[0];
+            const message = choice?.message;
+            return {
+                id: response.id,
+                model: response.model,
+                created: response.created,
+                content: message?.content || '',
+                finishReason: choice?.finish_reason,
+                usage: response.usage ? {
+                    promptTokens: response.usage.prompt_tokens,
+                    completionTokens: response.usage.completion_tokens,
+                    totalTokens: response.usage.total_tokens,
+                    sourcesUsed: response.usage.num_sources_used,
+                } : undefined,
+                annotations: message?.annotations?.map((a) => ({
+                    type: 'url_citation',
+                    url: a.url_citation?.url,
+                    title: a.url_citation?.title,
+                    startIndex: a.url_citation?.start_index,
+                    endIndex: a.url_citation?.end_index,
+                })),
+            };
+        }
+        catch (error) {
+            const err = error;
+            console.error('[ERROR] Research request failed:', err.message);
+            if (err.response) {
+                console.error('[ERROR] Response:', JSON.stringify(err.response.data, null, 2));
+            }
+            throw error;
+        }
+    }
+}
+//# sourceMappingURL=research.js.map

package/dist/clients/research.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"research.js","sourceRoot":"","sources":["../../src/clients/research.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAiC9C,MAAM,OAAO,cAAc;IACjB,MAAM,CAAS;IAEvB;QACE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,OAAO,EAAE,QAAQ,CAAC,QAAQ;YAC1B,MAAM,EAAE,QAAQ,CAAC,OAAO;YACxB,OAAO,EAAE,QAAQ,CAAC,UAAU;SAC7B,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAsB;QACnC,MAAM,EACJ,QAAQ,EACR,YAAY,EACZ,eAAe,GAAG,QAAQ,CAAC,gBAAgB,EAC3C,gBAAgB,GAAG,QAAQ,CAAC,QAAQ,EACpC,SAAS,GAAG,KAAK,EACjB,WAAW,GAAG,GAAG,EACjB,cAAc,GACf,GAAG,MAAM,CAAC;QAEX,MAAM,QAAQ,GAAwD,EAAE,CAAC;QACzE,IAAI,YAAY,EAAE,CAAC;YACjB,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEnD,MAAM,cAAc,GAA4B;YAC9C,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ;YACR,WAAW;YACX,gBAAgB,EAAE,eAAe;YACjC,qBAAqB,EAAE,SAAS;YAChC,iBAAiB,EAAE;gBACjB,IAAI,EAAE,IAAI;gBACV,kBAAkB,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAClD,gBAAgB,EAAE,IAAI;gBACtB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;aAC3B;SACF,CAAC;QAEF,IAAI,cAAc,EAAE,CAAC;YACnB,cAAc,CAAC,eAAe,GAAG,cAAc,CAAC;QAClD,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,cAAqB,CAAC,CAAC;YAClF,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,MAAM,EAAE,OAAc,CAAC;YAEvC,OAAO;gBACL,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,KAAK,EAAE,QAAQ,CAAC,KAAK;gBACrB,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE;gBAC/B,YAAY,EAAE,MAAM,EAAE,aAAa;gBACnC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;oBACtB,YAAY,EAAE,QAAQ,CAAC,KAAK,CAAC,aAAa;oBAC1C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,CAAC,iBAAiB;oBAClD,WAAW,EAAE,QAAQ,CAAC,KAAK,CAAC,YAAY;oBACxC,WAAW,EAAG,QAAQ,CAAC,KAAa,CAAC,gBAAgB;iBACtD,CAAC,CAAC,CAAC,SAAS;gBACb,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;oBAClD,IAAI,EAAE,cAAuB;oBAC7B,GAAG,EAAE,CAAC,CAAC,YAAY,EAAE,GAAG;oBACxB,KAAK,EAAE,CAAC,CAAC,YAAY,EAAE,KAAK;oBAC5B,UAAU,EAAE,CAAC,CAAC,YAAY,EAAE,WAAW;oBACvC,QAAQ,EAAE,CAAC,CAAC,YAAY,EAAE,SAAS;iBACpC,CAAC,CAAC;aACJ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG,KAA6E,CAAC;YAC1F,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/D,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACjB,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YACjF,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;CACF"}

package/dist/clients/scraper.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Web Scraper Client
+ * Generic interface for URL scraping with automatic fallback modes
+ */
+export interface ScrapeRequest {
+    url: string;
+    mode?: 'basic' | 'javascript';
+    timeout?: number;
+    country?: string;
+}
+export interface ScrapeResponse {
+    content: string;
+    statusCode: number;
+    credits: number;
+    headers?: Record<string, string>;
+}
+export interface BatchScrapeResult {
+    results: Array<ScrapeResponse & {
+        url: string;
+    }>;
+    batchesProcessed: number;
+    totalAttempted: number;
+    rateLimitHits: number;
+}
+export declare class ScraperClient {
+    private apiKey;
+    private baseURL;
+    constructor(apiKey?: string);
+    scrape(request: ScrapeRequest, maxRetries?: 3): Promise<ScrapeResponse>;
+    scrapeWithFallback(url: string, options?: {
+        timeout?: number;
+    }): Promise<ScrapeResponse>;
+    scrapeMultiple(urls: string[], options?: {
+        timeout?: number;
+    }): Promise<Array<ScrapeResponse & {
+        url: string;
+    }>>;
+    batchScrape(urls: string[], options?: {
+        timeout?: number;
+    }, onBatchComplete?: (batchNum: number, totalBatches: number, processed: number) => void): Promise<BatchScrapeResult>;
+    private processBatch;
+    private delay;
+}
+//# sourceMappingURL=scraper.d.ts.map

package/dist/clients/scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,OAAO,GAAG,YAAY,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA2B;gBAE9B,MAAM,CAAC,EAAE,MAAM;IASrB,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,UAAU,IAAsB,GAAG,OAAO,CAAC,cAAc,CAAC;IAgFzF,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,cAAc,CAAC;IAyC5F,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IASpH,WAAW,CACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,EAClC,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,GACpF,OAAO,CAAC,iBAAiB,CAAC;YA8Cf,YAAY;IAW1B,OAAO,CAAC,KAAK;CAGd"}

package/dist/clients/scraper.js

@@ -0,0 +1,171 @@
+/**
+ * Web Scraper Client
+ * Generic interface for URL scraping with automatic fallback modes
+ */
+import { parseEnv, SCRAPER } from '../config/index.js';
+export class ScraperClient {
+    apiKey;
+    baseURL = 'https://api.scrape.do';
+    constructor(apiKey) {
+        const env = parseEnv();
+        this.apiKey = apiKey || env.SCRAPER_API_KEY;
+        if (!this.apiKey) {
+            throw new Error('SCRAPEDO_API_KEY is required');
+        }
+    }
+    async scrape(request, maxRetries = SCRAPER.RETRY_COUNT) {
+        const { url, mode = 'basic', timeout = 30, country } = request;
+        const params = new URLSearchParams({
+            url: url,
+            token: this.apiKey,
+            timeout: String(timeout * 1000),
+        });
+        if (mode === 'javascript') {
+            params.append('render', 'true');
+        }
+        if (country) {
+            params.append('geoCode', country.toUpperCase());
+        }
+        const apiUrl = `${this.baseURL}?${params.toString()}`;
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+            try {
+                const response = await fetch(apiUrl, {
+                    method: 'GET',
+                    headers: { Accept: 'text/html,application/json' },
+                });
+                if (!response.ok) {
+                    const errorText = await response.text().catch(() => '');
+                    if (response.status === 429) {
+                        if (attempt < maxRetries - 1) {
+                            const delay = SCRAPER.RETRY_DELAYS[attempt] || 8000;
+                            console.error(`[Scraper] Rate limited (429). Retry ${attempt + 1}/${maxRetries} after ${delay}ms`);
+                            await this.delay(delay);
+                            continue;
+                        }
+                        throw new Error('Scraper rate limited. Try fewer URLs or retry later.');
+                    }
+                    if (response.status >= 400 && response.status < 500) {
+                        throw new Error(`Scraper API error: ${response.status} ${errorText}`);
+                    }
+                    if (attempt < maxRetries - 1) {
+                        const delay = SCRAPER.RETRY_DELAYS[attempt] || 8000;
+                        console.error(`[Scraper] Retry ${attempt + 1}/${maxRetries} after ${delay}ms (status: ${response.status})`);
+                        await this.delay(delay);
+                        continue;
+                    }
+                    throw new Error(`Scraper API error: ${response.status} ${errorText}`);
+                }
+                const content = await response.text();
+                const credits = mode === 'javascript' ? 5 : 1;
+                return {
+                    content,
+                    statusCode: response.status,
+                    credits,
+                    headers: Object.fromEntries(response.headers.entries()),
+                };
+            }
+            catch (error) {
+                if (attempt === maxRetries - 1) {
+                    throw new Error(`Failed to scrape URL after ${maxRetries} attempts: ${error instanceof Error ? error.message : String(error)}`);
+                }
+                if (error instanceof Error && !error.message.includes('API error') && !error.message.includes('rate limited')) {
+                    const delay = SCRAPER.RETRY_DELAYS[attempt] || 8000;
+                    console.error(`[Scraper] Network error, retry ${attempt + 1}/${maxRetries} after ${delay}ms: ${error.message}`);
+                    await this.delay(delay);
+                }
+                else {
+                    throw error;
+                }
+            }
+        }
+        throw new Error('Unexpected retry loop exit');
+    }
+    async scrapeWithFallback(url, options = {}) {
+        const attempts = [
+            { mode: 'basic', description: 'basic mode' },
+            { mode: 'javascript', description: 'javascript rendering' },
+            { mode: 'javascript', country: 'us', description: 'javascript + US geo-targeting' },
+        ];
+        const attemptResults = [];
+        for (const attempt of attempts) {
+            try {
+                const result = await this.scrape({
+                    url,
+                    mode: attempt.mode,
+                    timeout: options.timeout,
+                    country: attempt.country,
+                });
+                if (result.statusCode >= 200 && result.statusCode < 300) {
+                    if (attemptResults.length > 0) {
+                        console.error(`[Scraper] Success with ${attempt.description} after ${attemptResults.length} failed attempt(s)`);
+                    }
+                    return result;
+                }
+                if (result.statusCode === 404) {
+                    return { content: '404 - Page not found', statusCode: 404, credits: result.credits };
+                }
+                attemptResults.push(`${attempt.description}: ${result.statusCode}`);
+                console.error(`[Scraper] Failed with ${attempt.description} (status: ${result.statusCode}), trying next fallback...`);
+            }
+            catch (error) {
+                const msg = error instanceof Error ? error.message : String(error);
+                attemptResults.push(`${attempt.description}: ${msg}`);
+                console.error(`[Scraper] Error with ${attempt.description}: ${msg}`);
+            }
+        }
+        throw new Error(`Failed to scrape ${url} after trying all fallback modes:\n${attemptResults.join('\n')}`);
+    }
+    async scrapeMultiple(urls, options = {}) {
+        if (urls.length <= SCRAPER.BATCH_SIZE) {
+            return this.processBatch(urls, options);
+        }
+        const result = await this.batchScrape(urls, options);
+        return result.results;
+    }
+    async batchScrape(urls, options = {}, onBatchComplete) {
+        const totalBatches = Math.ceil(urls.length / SCRAPER.BATCH_SIZE);
+        const allResults = [];
+        let rateLimitHits = 0;
+        console.error(`[Scraper] Starting batch processing: ${urls.length} URLs in ${totalBatches} batch(es)`);
+        for (let batchNum = 0; batchNum < totalBatches; batchNum++) {
+            const startIdx = batchNum * SCRAPER.BATCH_SIZE;
+            const endIdx = Math.min(startIdx + SCRAPER.BATCH_SIZE, urls.length);
+            const batchUrls = urls.slice(startIdx, endIdx);
+            console.error(`[Scraper] Processing batch ${batchNum + 1}/${totalBatches} (${batchUrls.length} URLs)`);
+            const batchResults = await Promise.allSettled(batchUrls.map(url => this.scrapeWithFallback(url, options)));
+            for (let i = 0; i < batchResults.length; i++) {
+                const result = batchResults[i];
+                const url = batchUrls[i] || '';
+                if (result.status === 'fulfilled') {
+                    allResults.push({ ...result.value, url });
+                }
+                else {
+                    const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
+                    if (errorMsg.includes('rate limited') || errorMsg.includes('429')) {
+                        rateLimitHits++;
+                    }
+                    allResults.push({ url, content: `Error: ${errorMsg}`, statusCode: 500, credits: 0 });
+                }
+            }
+            onBatchComplete?.(batchNum + 1, totalBatches, allResults.length);
+            console.error(`[Scraper] Completed batch ${batchNum + 1}/${totalBatches} (${allResults.length}/${urls.length} total)`);
+            if (batchNum < totalBatches - 1) {
+                await this.delay(500);
+            }
+        }
+        return { results: allResults, batchesProcessed: totalBatches, totalAttempted: urls.length, rateLimitHits };
+    }
+    async processBatch(urls, options) {
+        const results = await Promise.allSettled(urls.map(url => this.scrapeWithFallback(url, options)));
+        return results.map((result, index) => {
+            if (result.status === 'fulfilled') {
+                return { ...result.value, url: urls[index] || '' };
+            }
+            return { url: urls[index] || '', content: `Error: ${result.reason}`, statusCode: 500, credits: 0 };
+        });
+    }
+    delay(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+}
+//# sourceMappingURL=scraper.js.map

package/dist/clients/scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAuBvD,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,OAAO,GAAG,uBAAuB,CAAC;IAE1C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,eAAe,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW;QACnE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAE/D,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QAEtD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;oBACnC,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,EAAE,MAAM,EAAE,4BAA4B,EAAE;iBAClD,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;oBAExD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;wBAC5B,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;4BAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;4BACpD,OAAO,CAAC,KAAK,CAAC,uCAAuC,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,KAAK,IAAI,CAAC,CAAC;4BACnG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;4BACxB,SAAS;wBACX,CAAC;wBACD,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;oBAC1E,CAAC;oBAED,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;wBACpD,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;oBACxE,CAAC;oBAED,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;wBACpD,OAAO,CAAC,KAAK,CAAC,mBAAmB,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,KAAK,eAAe,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;wBAC5G,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBACxB,SAAS;oBACX,CAAC;oBAED,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC;gBACxE,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACtC,MAAM,OAAO,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE9C,OAAO;oBACL,OAAO;oBACP,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,OAAO;oBACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;iBACxD,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO,KAAK,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,cAAc,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAClI,CAAC;gBAED,IAAI,KAAK,YAAY,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;oBAC9G,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;oBACpD,OAAO,CAAC,KAAK,CAAC,kCAAkC,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,KAAK,OAAO,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;oBAChH,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,UAAgC,EAAE;QACtE,MAAM,QAAQ,GAAmF;YAC/F,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;YAC5C,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,sBAAsB,EAAE;YAC3D,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;SACpF,CAAC;QAEF,MAAM,cAAc,GAAa,EAAE,CAAC;QAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;oBAC/B,GAAG;oBACH,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;iBACzB,CAAC,CAAC;gBAEH,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;oBACxD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC9B,OAAO,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,WAAW,UAAU,cAAc,CAAC,MAAM,oBAAoB,CAAC,CAAC;oBAClH,CAAC;oBACD,OAAO,MAAM,CAAC;gBAChB,CAAC;gBAED,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC9B,OAAO,EAAE,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvF,CAAC;gBAED,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;gBACpE,OAAO,CAAC,KAAK,CAAC,yBAAyB,OAAO,CAAC,WAAW,aAAa,MAAM,CAAC,UAAU,4BAA4B,CAAC,CAAC;YACxH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnE,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,GAAG,EAAE,CAAC,CAAC;gBACtD,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,WAAW,KAAK,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,sCAAsC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAc,EAAE,UAAgC,EAAE;QACrE,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,WAAW,CACf,IAAc,EACd,UAAgC,EAAE,EAClC,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,UAAU,GAA4C,EAAE,CAAC;QAC/D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,OAAO,CAAC,KAAK,CAAC,wCAAwC,IAAI,CAAC,MAAM,YAAY,YAAY,YAAY,CAAC,CAAC;QAEvG,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE/C,OAAO,CAAC,KAAK,CAAC,8BAA8B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,QAAQ,CAAC,CAAC;YAEvG,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAC5D,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC5C,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAEhG,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;wBAClE,aAAa,EAAE,CAAC;oBAClB,CAAC;oBAED,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,UAAU,QAAQ,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;gBACvF,CAAC;YACH,CAAC;YAED,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACjE,OAAO,CAAC,KAAK,CAAC,6BAA6B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAEvH,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IAC7G,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,IAAc,EAAE,OAA6B;QACtE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QAEjG,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACrD,CAAC;YACD,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACrG,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;CACF"}

package/dist/clients/search.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Web Search Client
+ * Generic interface for web search via Google (Serper implementation)
+ */
+export interface SearchResult {
+    title: string;
+    link: string;
+    snippet: string;
+    date?: string;
+    position: number;
+}
+export interface KeywordSearchResult {
+    keyword: string;
+    results: SearchResult[];
+    totalResults: number;
+    related: string[];
+}
+export interface MultipleSearchResponse {
+    searches: KeywordSearchResult[];
+    totalKeywords: number;
+    executionTime: number;
+}
+export interface RedditSearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+    date?: string;
+}
+export declare class SearchClient {
+    private apiKey;
+    private baseURL;
+    constructor(apiKey?: string);
+    /**
+     * Search multiple keywords in parallel
+     */
+    searchMultiple(keywords: string[]): Promise<MultipleSearchResponse>;
+    /**
+     * Search Reddit via Google (adds site:reddit.com automatically)
+     */
+    searchReddit(query: string, dateAfter?: string): Promise<RedditSearchResult[]>;
+    /**
+     * Search Reddit with multiple queries in parallel
+     */
+    searchRedditMultiple(queries: string[], dateAfter?: string): Promise<Map<string, RedditSearchResult[]>>;
+}
+//# sourceMappingURL=search.d.ts.map

package/dist/clients/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,CAAC,EAAE,MAAM;IAS3B;;OAEG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAkDzE;;OAEG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAwBpF;;OAEG;IACG,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;CAM9G"}