scorchcrawl-mcp 1.0.2 → 1.1.0

package/dist/copilot-agent.js

@@ -0,0 +1,619 @@
+/**
+ * Copilot SDK Agent Engine
+ *
+ * Replaces the proprietary cloud-only agent with a Copilot SDK-powered
+ * agent that uses scorchcrawl tools (scrape, search, map, extract) as custom tools.
+ *
+ * Model selection is controlled via COPILOT_AGENT_MODELS environment variable.
+ */
+import { CopilotClient } from '@github/copilot-sdk';
+import { v4 as uuidv4 } from 'uuid';
+import { RateLimitGuard, buildRateLimitConfig, buildErrorHook, findStaleJobs, } from './rate-limiter.js';
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Parse COPILOT_AGENT_MODELS env var into an array of model names */
+export function parseAllowedModels() {
+    const envModels = process.env.COPILOT_AGENT_MODELS;
+    if (!envModels) {
+        // Defaults from user requirements
+        return ['gpt-4.1', 'gpt-4o', 'gpt-5-mini', 'grok-code-fast-1'];
+    }
+    return envModels
+        .split(',')
+        .map((m) => m.trim())
+        .filter(Boolean);
+}
+/** Get default model (first in list or from env) */
+export function getDefaultModel() {
+    const defaultFromEnv = process.env.COPILOT_AGENT_DEFAULT_MODEL;
+    if (defaultFromEnv)
+        return defaultFromEnv.trim();
+    const allowed = parseAllowedModels();
+    return allowed[0] || 'gpt-4.1';
+}
+/** Build the agent configuration from environment */
+export function buildAgentConfig() {
+    const config = {
+        allowedModels: parseAllowedModels(),
+        defaultModel: getDefaultModel(),
+    };
+    // Optional BYOK provider
+    if (process.env.COPILOT_AGENT_PROVIDER_TYPE && process.env.COPILOT_AGENT_PROVIDER_BASE_URL) {
+        config.provider = {
+            type: process.env.COPILOT_AGENT_PROVIDER_TYPE,
+            baseUrl: process.env.COPILOT_AGENT_PROVIDER_BASE_URL,
+            apiKey: process.env.COPILOT_AGENT_PROVIDER_API_KEY,
+        };
+    }
+    return config;
+}
+// ---------------------------------------------------------------------------
+// ScorchCrawl Tools for Copilot SDK Agent
+// ---------------------------------------------------------------------------
+/**
+ * Build the custom tools that the Copilot SDK agent will have access to.
+ * These wrap the scraping engine client methods so the agent can scrape, search,
+ * map, crawl, and extract data from the web.
+ */
+function buildScrapingTools(client, origin) {
+    return [
+        {
+            name: 'web_scrape',
+            description: 'Scrape content from a single URL. Returns markdown content by default. ' +
+                'Use formats parameter to request JSON extraction with a schema, or other formats like html, screenshot, links. ' +
+                'If the result is empty, blocked, or shows anti-bot content, use web_screenshot to visually inspect the page.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'The URL to scrape' },
+                    formats: {
+                        type: 'array',
+                        description: 'Output formats: "markdown", "html", "links", or JSON extraction object',
+                        items: { type: 'string' },
+                    },
+                    onlyMainContent: {
+                        type: 'boolean',
+                        description: 'Extract only the main content, excluding nav/footer/etc',
+                    },
+                    waitFor: {
+                        type: 'number',
+                        description: 'Wait time in ms for JS-rendered pages (default 0)',
+                    },
+                },
+                required: ['url'],
+            },
+            handler: async (args) => {
+                try {
+                    const a = args;
+                    const { url, ...opts } = a;
+                    const res = await client.scrape(String(url), {
+                        ...opts,
+                        origin,
+                    });
+                    return {
+                        textResultForLlm: JSON.stringify(res, null, 2),
+                        resultType: 'success',
+                    };
+                }
+                catch (err) {
+                    return {
+                        textResultForLlm: `Scrape failed: ${err.message || err}`,
+                        resultType: 'failure',
+                        error: String(err.message || err),
+                    };
+                }
+            },
+        },
+        {
+            name: 'web_screenshot',
+            description: 'Take a screenshot of a web page for visual inspection. ' +
+                'Use this when web_scrape returns empty, blocked, or anti-bot content. ' +
+                'The screenshot is passed directly to your vision capability so you can analyze what is on the page — ' +
+                'cookie popups, CAPTCHAs, anti-bot walls, or the actual rendered content.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'The URL to screenshot' },
+                    fullPage: {
+                        type: 'boolean',
+                        description: 'Capture the full scrollable page (default: false, viewport only)',
+                    },
+                    waitFor: {
+                        type: 'number',
+                        description: 'Wait time in ms for JS rendering before screenshot (default 3000)',
+                    },
+                },
+                required: ['url'],
+            },
+            handler: async (args) => {
+                try {
+                    const a = args;
+                    const url = String(a.url);
+                    const fullPage = a.fullPage === true;
+                    const waitFor = typeof a.waitFor === 'number' ? a.waitFor : 3000;
+                    const screenshotFormat = {
+                        type: 'screenshot',
+                        fullPage,
+                    };
+                    const res = await client.scrape(url, {
+                        formats: [screenshotFormat],
+                        waitFor,
+                        origin,
+                    });
+                    const resAny = res;
+                    const screenshot = resAny?.screenshot || resAny?.data?.screenshot;
+                    const metadata = resAny?.metadata || resAny?.data?.metadata || {};
+                    if (screenshot) {
+                        const imgBytes = Buffer.from(screenshot, 'base64').length;
+                        const description = [
+                            `Screenshot of: ${url}`,
+                            `Title: ${metadata.title || 'Unknown'}`,
+                            `Status: ${metadata.statusCode || 'Unknown'}`,
+                            `Size: ${Math.round(imgBytes / 1024)}KB`,
+                            '',
+                            'Analyze the attached image to determine:',
+                            '- Is there a cookie consent popup blocking content?',
+                            '- Is there a CAPTCHA or anti-bot challenge?',
+                            '- Is the page content actually rendered?',
+                            '- What text/data is visible?',
+                        ].join('\n');
+                        return {
+                            textResultForLlm: description,
+                            binaryResultsForLlm: [{
+                                    data: screenshot,
+                                    mimeType: 'image/png',
+                                    type: 'image',
+                                    description: `Screenshot of ${url}`,
+                                }],
+                            resultType: 'success',
+                        };
+                    }
+                    return {
+                        textResultForLlm: `Screenshot request completed but no image was returned. Page metadata: ${JSON.stringify(metadata, null, 2)}`,
+                        resultType: 'success',
+                    };
+                }
+                catch (err) {
+                    return {
+                        textResultForLlm: `Screenshot failed: ${err.message || err}. The page may be unreachable or the screenshot service unavailable.`,
+                        resultType: 'failure',
+                        error: String(err.message || err),
+                    };
+                }
+            },
+        },
+        {
+            name: 'web_search',
+            description: 'Search the web for information. Returns search results with titles, URLs, and snippets. ' +
+                'Supports search operators like site:, inurl:, intitle:, and quoted phrases.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    query: { type: 'string', description: 'The search query' },
+                    limit: {
+                        type: 'number',
+                        description: 'Maximum number of results (default 5)',
+                    },
+                },
+                required: ['query'],
+            },
+            handler: async (args) => {
+                try {
+                    const a = args;
+                    const { query, ...opts } = a;
+                    const res = await client.search(String(query), {
+                        ...opts,
+                        origin,
+                    });
+                    return {
+                        textResultForLlm: JSON.stringify(res, null, 2),
+                        resultType: 'success',
+                    };
+                }
+                catch (err) {
+                    return {
+                        textResultForLlm: `Search failed: ${err.message || err}`,
+                        resultType: 'failure',
+                        error: String(err.message || err),
+                    };
+                }
+            },
+        },
+        {
+            name: 'web_map',
+            description: 'Map a website to discover all indexed URLs. Use the search parameter to find specific pages. ' +
+                'Useful when you need to find the right page before scraping.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    url: { type: 'string', description: 'The website URL to map' },
+                    search: {
+                        type: 'string',
+                        description: 'Optional search query to filter URLs',
+                    },
+                    limit: {
+                        type: 'number',
+                        description: 'Maximum number of URLs to return',
+                    },
+                },
+                required: ['url'],
+            },
+            handler: async (args) => {
+                try {
+                    const a = args;
+                    const { url, ...opts } = a;
+                    const res = await client.map(String(url), {
+                        ...opts,
+                        origin,
+                    });
+                    return {
+                        textResultForLlm: JSON.stringify(res, null, 2),
+                        resultType: 'success',
+                    };
+                }
+                catch (err) {
+                    return {
+                        textResultForLlm: `Map failed: ${err.message || err}`,
+                        resultType: 'failure',
+                        error: String(err.message || err),
+                    };
+                }
+            },
+        },
+        {
+            name: 'web_extract',
+            description: 'Extract structured information from web pages using LLM capabilities. ' +
+                'Provide URLs, a prompt describing what to extract, and an optional JSON schema for the output format.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    urls: {
+                        type: 'array',
+                        items: { type: 'string' },
+                        description: 'URLs to extract information from',
+                    },
+                    prompt: {
+                        type: 'string',
+                        description: 'Description of what information to extract',
+                    },
+                    schema: {
+                        type: 'object',
+                        description: 'JSON schema for structured output',
+                    },
+                },
+                required: ['urls'],
+            },
+            handler: async (args) => {
+                try {
+                    const a = args;
+                    const res = await client.extract({
+                        urls: a.urls,
+                        prompt: a.prompt,
+                        schema: a.schema,
+                        origin,
+                    });
+                    return {
+                        textResultForLlm: JSON.stringify(res, null, 2),
+                        resultType: 'success',
+                    };
+                }
+                catch (err) {
+                    return {
+                        textResultForLlm: `Extract failed: ${err.message || err}`,
+                        resultType: 'failure',
+                        error: String(err.message || err),
+                    };
+                }
+            },
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Agent Engine
+// ---------------------------------------------------------------------------
+/** In-memory store for agent jobs */
+const agentJobs = new Map();
+// ---------------------------------------------------------------------------
+// Rate Limiting & Concurrency Guard (singleton)
+// ---------------------------------------------------------------------------
+const rateLimitGuard = new RateLimitGuard(buildRateLimitConfig());
+/** Periodic stale-job reaper */
+setInterval(() => {
+    const staleIds = findStaleJobs(agentJobs.values(), rateLimitGuard.config.staleJobTimeoutMs);
+    for (const id of staleIds) {
+        const job = agentJobs.get(id);
+        if (job) {
+            job.status = 'failed';
+            job.error = `Job timed out after ${rateLimitGuard.config.staleJobTimeoutMs / 1000}s without completing.`;
+            job.completedAt = Date.now();
+            // Release concurrency slot (use empty-string key for unknown user)
+            rateLimitGuard.release(job._userKey || '');
+            console.warn(`[RateLimit] Reaped stale job ${id}`);
+        }
+    }
+}, rateLimitGuard.config.gcIntervalMs);
+/**
+ * Cache of CopilotClient instances keyed by GitHub token.
+ * The empty-string key holds the server-wide (env-based) client.
+ * Entries are evicted after MAX_CLIENT_AGE_MS of inactivity.
+ */
+const clientCache = new Map();
+const MAX_CLIENT_AGE_MS = 30 * 60 * 1000; // 30 min
+/** Periodically purge stale clients */
+setInterval(() => {
+    const now = Date.now();
+    for (const [key, entry] of clientCache) {
+        if (now - entry.lastUsed > MAX_CLIENT_AGE_MS) {
+            try {
+                entry.client.stop();
+            }
+            catch { /* ignore */ }
+            clientCache.delete(key);
+        }
+    }
+}, 5 * 60 * 1000); // every 5 min
+/**
+ * Get or create a CopilotClient for the given token.
+ * If `userToken` is provided it takes priority; otherwise
+ * the server-wide GITHUB_TOKEN env var is used.
+ *
+ * Exported so the summarization layer in response-utils.ts can
+ * reuse the same cached Copilot client instances.
+ */
+export async function getCopilotClient(userToken) {
+    const cacheKey = userToken || '';
+    const existing = clientCache.get(cacheKey);
+    if (existing) {
+        existing.lastUsed = Date.now();
+        return existing.client;
+    }
+    const clientOpts = {};
+    // Allow configuring Copilot CLI path
+    if (process.env.COPILOT_CLI_PATH) {
+        clientOpts.cliPath = process.env.COPILOT_CLI_PATH;
+    }
+    // Allow connecting to external CLI server
+    if (process.env.COPILOT_CLI_URL) {
+        clientOpts.cliUrl = process.env.COPILOT_CLI_URL;
+    }
+    // Per-user token takes priority, then env var
+    const token = userToken || process.env.GITHUB_TOKEN;
+    if (token) {
+        clientOpts.githubToken = token;
+    }
+    const client = new CopilotClient(clientOpts);
+    clientCache.set(cacheKey, { client, lastUsed: Date.now() });
+    return client;
+}
+/**
+ * Start a new agent job and wait for it to complete.
+ * Returns the final result (or error) directly — no polling needed.
+ *
+ * Rate limiting is enforced before the job is accepted:
+ *   1. Per-user and global concurrency limits
+ *   2. Sliding-window request rate
+ *   3. Proactive Copilot quota check
+ */
+export async function startAgent(request, scrapingClient, origin, config, copilotToken) {
+    const jobId = uuidv4();
+    const userKey = copilotToken || '__server__';
+    // --- Rate limit gate ---
+    const gate = rateLimitGuard.check(userKey);
+    if (!gate.allowed) {
+        return {
+            id: jobId,
+            status: 'rate_limited',
+            rateLimited: true,
+            retryAfterSeconds: gate.retryAfterSeconds,
+            error: gate.reason,
+        };
+    }
+    // Validate model
+    const requestedModel = request.model || config.defaultModel;
+    if (!config.allowedModels.includes(requestedModel)) {
+        return {
+            id: jobId,
+            status: 'failed',
+            error: `Model "${requestedModel}" is not in the allowed list: ${config.allowedModels.join(', ')}`,
+        };
+    }
+    // --- Acquire concurrency slot & record request ---
+    rateLimitGuard.acquire(userKey);
+    // Create the job entry
+    const job = {
+        id: jobId,
+        status: 'processing',
+        prompt: request.prompt,
+        createdAt: Date.now(),
+        _userKey: userKey,
+    };
+    agentJobs.set(jobId, job);
+    // Run the agent synchronously — wait for completion
+    try {
+        await runAgentJob(job, request, scrapingClient, origin, config, requestedModel, copilotToken);
+    }
+    catch (err) {
+        job.status = 'failed';
+        job.error = String(err.message || err);
+        job.completedAt = Date.now();
+    }
+    finally {
+        // Always release the concurrency slot when the job finishes
+        rateLimitGuard.release(userKey);
+    }
+    return {
+        id: jobId,
+        status: job.status,
+        data: job.result || undefined,
+        error: job.error || undefined,
+        duration: job.completedAt
+            ? `${((job.completedAt - job.createdAt) / 1000).toFixed(1)}s`
+            : undefined,
+    };
+}
+/**
+ * Run an agent job using the Copilot SDK.
+ */
+async function runAgentJob(job, request, scrapingClient, origin, config, model, copilotToken) {
+    try {
+        job.progress = 'Initializing Copilot SDK agent...';
+        const client = await getCopilotClient(copilotToken);
+        const tools = buildScrapingTools(scrapingClient, origin);
+        // Build session config
+        const sessionConfig = {
+            model,
+            tools,
+            // Disable all built-in tools - we only want our scorchcrawl tools
+            availableTools: tools.map((t) => t.name),
+            systemMessage: {
+                mode: 'replace',
+                content: buildSystemPrompt(request),
+            },
+        };
+        // Add BYOK provider if configured
+        if (config.provider) {
+            sessionConfig.provider = {
+                type: config.provider.type,
+                baseUrl: config.provider.baseUrl,
+                ...(config.provider.apiKey && { apiKey: config.provider.apiKey }),
+            };
+        }
+        job.progress = `Creating agent session with model: ${model}`;
+        const session = await client.createSession(sessionConfig);
+        // Register error hook for intelligent retry/abort decisions
+        try {
+            const errorHook = buildErrorHook(job.id);
+            if (typeof session.registerHooks === 'function') {
+                session.registerHooks({ onErrorOccurred: errorHook });
+            }
+        }
+        catch {
+            // Hooks may not be supported in all SDK versions — non-fatal
+        }
+        // Listen for quota snapshots from assistant.usage events
+        try {
+            const userKey = copilotToken || '__server__';
+            session.on?.('assistant.usage', (event) => {
+                const snapshots = event?.data?.quotaSnapshots || event?.quotaSnapshots;
+                if (snapshots) {
+                    // Take the first available quota category (usually 'chat')
+                    const snap = snapshots.chat || Object.values(snapshots)[0];
+                    if (snap) {
+                        rateLimitGuard.quota.update(userKey, {
+                            remainingPercent: snap.remainingPercentage ?? snap.percent_remaining ?? 100,
+                            usedRequests: snap.usedRequests ?? snap.remaining ?? 0,
+                            entitlementRequests: snap.entitlementRequests ?? snap.entitlement ?? -1,
+                            isUnlimited: snap.isUnlimitedEntitlement ?? snap.unlimited ?? false,
+                            resetDate: snap.resetDate ?? undefined,
+                        });
+                    }
+                }
+            });
+        }
+        catch {
+            // Event subscription may not be supported — non-fatal
+        }
+        try {
+            job.progress = 'Agent is researching...';
+            // Build the user prompt
+            let userPrompt = request.prompt;
+            if (request.urls && request.urls.length > 0) {
+                userPrompt += `\n\nFocus on these URLs:\n${request.urls.map((u) => `- ${u}`).join('\n')}`;
+            }
+            if (request.schema) {
+                userPrompt += `\n\nReturn the results in this JSON schema format:\n${JSON.stringify(request.schema, null, 2)}`;
+            }
+            // Determine timeout: request-level > env var > SDK default (60s)
+            const timeoutMs = request.timeout
+                || (process.env.AGENT_TIMEOUT_MS ? parseInt(process.env.AGENT_TIMEOUT_MS, 10) : undefined)
+                || undefined; // undefined lets SDK use its own 60s default
+            // Send the prompt and wait for completion
+            const response = await session.sendAndWait({ prompt: userPrompt }, timeoutMs);
+            job.status = 'completed';
+            job.completedAt = Date.now();
+            job.result = {
+                success: true,
+                data: response?.data?.content || 'No response generated',
+                model,
+            };
+        }
+        finally {
+            // Clean up the session
+            try {
+                await session.destroy();
+            }
+            catch {
+                // Ignore cleanup errors
+            }
+        }
+    }
+    catch (err) {
+        job.status = 'failed';
+        job.error = `Agent error: ${err.message || err}`;
+        job.completedAt = Date.now();
+    }
+}
+/**
+ * Build the system prompt for the agent session.
+ */
+function buildSystemPrompt(request) {
+    return `You are an autonomous web research agent. Your job is to browse the internet, search for information, navigate through pages, and extract structured data based on the user's query.
+
+You have access to the following web tools:
+- **web_search**: Search the web for information. Start with this for broad queries.
+- **web_scrape**: Scrape content from a specific URL. Use this to get page content.
+- **web_screenshot**: Take a screenshot of a page. The image is sent directly to you for vision analysis. Use this when scrape returns empty/blocked content.
+- **web_map**: Map a website to discover all its URLs. Use this to find specific pages on a site.
+- **web_extract**: Extract structured data from URLs using LLM. Use this for structured extraction.
+
+## Research Strategy
+
+1. **Start with search** to find relevant URLs if no specific URLs are provided.
+2. **Map websites** to discover relevant pages when you need to explore a site's structure.
+3. **Scrape specific pages** to get their content.
+4. **If scrape fails or returns empty/blocked content**, use **web_screenshot** to visually inspect the page and determine what's blocking (cookie popup, CAPTCHA, anti-bot, JS rendering issue).
+5. **Extract structured data** when you need specific fields from pages.
+
+## Visual Inspection Fallback
+
+When web_scrape returns any of these, ALWAYS follow up with web_screenshot:
+- Empty or very short content (< 100 characters)
+- Anti-bot messages ("Checking your browser", "Access denied", "Just a moment")
+- Cookie consent popup HTML instead of actual content
+- CAPTCHA challenges
+- Generic error pages
+
+The screenshot helps you understand what the user would see in a browser, and report back what's actually on the page even when text extraction fails.
+
+## Guidelines
+
+- Be thorough: check multiple sources when possible.
+- Be efficient: don't scrape pages that aren't relevant.
+- If a page fails to load or returns empty content, try an alternative approach (screenshot, different URL, waitFor parameter).
+- Always provide your final answer with all the information you gathered.
+- If the user provided a JSON schema, format your final response to match that schema.
+- Cite your sources with URLs when providing information.
+- When reporting screenshot results, describe what you observed on the page.
+
+## Important
+
+- You MUST use the provided tools to gather information. Do not make up or hallucinate data.
+- If you cannot find the requested information after reasonable effort, say so clearly.
+- Provide complete, well-structured responses.`;
+}
+/**
+ * Gracefully shut down the Copilot client.
+ */
+export async function shutdownAgent() {
+    rateLimitGuard.shutdown();
+    for (const [, entry] of clientCache) {
+        try {
+            await entry.client.stop();
+        }
+        catch {
+            // Ignore shutdown errors
+        }
+    }
+    clientCache.clear();
+}