scorchcrawl-mcp 1.0.2 → 1.2.0

package/dist/lib/fastmcp/index.js

@@ -0,0 +1,6 @@
+/**
+ * Re-export of FastMCP (MIT-licensed, originally "firecrawl-fastmcp" by Frank Fiegel).
+ * We re-export from this local path so our codebase doesn't reference
+ * the upstream package name in import statements.
+ */
+export { FastMCP } from 'firecrawl-fastmcp';

package/dist/lib/scorch-client/index.js

@@ -0,0 +1,9 @@
+/**
+ * Re-export of the Firecrawl JS SDK (MIT-licensed, @mendable/firecrawl-js).
+ * This is the HTTP client that talks to our local engine API.
+ * We re-export here so our source files import from a local path
+ * without referencing the upstream package name.
+ */
+import FirecrawlApp from '@mendable/firecrawl-js';
+export default FirecrawlApp;
+export { FirecrawlApp as ScorchClient };

package/dist/local-scraper.js

@@ -0,0 +1,319 @@
+/**
+ * Local Scraper — fetches pages through the LOCAL machine's IP and converts to markdown.
+ * Used when SCORCHCRAWL_LOCAL_PROXY=true is set, so scraping traffic exits through
+ * the user's residential IP instead of the server's datacenter IP.
+ *
+ * Falls back to the remote ScorchCrawl API for features that need server-side
+ * processing (search, crawl, extract, agent, JSON schema extraction).
+ *
+ * SPA Detection:
+ *   When the fetched HTML looks like a Single Page Application shell
+ *   (minimal text, loading indicators, heavy JS bundles), the scraper returns
+ *   a `SPA_SKELETON_DETECTED` error so the caller can retry via the engine's
+ *   Playwright-backed scraper which executes JavaScript.
+ */
+import TurndownService from 'turndown';
+import * as cheerio from 'cheerio';
+// Lazy-init singleton
+let _turndown = null;
+function getTurndown() {
+    if (!_turndown) {
+        _turndown = new TurndownService({
+            headingStyle: 'atx',
+            codeBlockStyle: 'fenced',
+            bulletListMarker: '-',
+        });
+        // Strip script/style/nav/footer tags
+        _turndown.remove(['script', 'style', 'noscript', 'iframe']);
+    }
+    return _turndown;
+}
+// ---------------------------------------------------------------------------
+// SPA / JS-rendered page detection
+// ---------------------------------------------------------------------------
+/**
+ * Common phrases found in SPA shell HTML before JS hydrates the page.
+ * Matched case-insensitively against the visible body text.
+ */
+const SPA_LOADING_PATTERNS = [
+    'loading...',
+    'loading…',
+    'please wait',
+    'just a moment',
+    'checking your browser',
+    'one moment please',
+    'redirecting',
+    'enable javascript',
+    'javascript is required',
+    'javascript must be enabled',
+    'this app requires javascript',
+    'you need to enable javascript',
+    'noscript',
+];
+/**
+ * CSS selectors whose sole presence (with no other meaningful content)
+ * strongly indicates a JS-only SPA shell.
+ */
+const SPA_ROOT_SELECTORS = [
+    '#root', // React (CRA, Vite)
+    '#app', // Vue
+    '#__next', // Next.js
+    '#__nuxt', // Nuxt
+    '#svelte', // SvelteKit
+    'app-root', // Angular
+    '#___gatsby', // Gatsby
+    '#main-app', // misc
+];
+/** Minimum characters of visible text for a page to be considered "real" content. */
+const MIN_MEANINGFUL_TEXT_LENGTH = 200;
+/** Ratio: if (script bytes / total HTML bytes) exceeds this, it's likely a SPA shell. */
+const SCRIPT_HEAVY_RATIO = 0.65;
+/**
+ * Inspect raw HTML + extracted text to decide if the page is a SPA shell
+ * that hasn't been hydrated (no JS execution happened).
+ *
+ * Returns a short reason string if SPA-like, or `null` if the page looks real.
+ */
+export function detectSPASkeleton(rawHtml, _bodyText, $) {
+    // Get visible text only (strip script, style, noscript content)
+    const $clone = cheerio.load($.html());
+    $clone('script, style, noscript').remove();
+    const visibleText = $clone('body').text() || '';
+    const trimmedText = visibleText.replace(/\s+/g, ' ').trim();
+    const lowerText = trimmedText.toLowerCase();
+    // 1. Very little visible text — likely a shell that JS would populate
+    if (trimmedText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
+        // Check for SPA root containers
+        for (const sel of SPA_ROOT_SELECTORS) {
+            const el = $(sel);
+            if (el.length > 0) {
+                const innerText = el.text().replace(/\s+/g, ' ').trim();
+                if (innerText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
+                    return `SPA root container "${sel}" with minimal content (${innerText.length} chars)`;
+                }
+            }
+        }
+        // Check for loading phrases in the sparse text
+        for (const pattern of SPA_LOADING_PATTERNS) {
+            if (lowerText.includes(pattern)) {
+                return `Loading indicator detected: "${pattern}"`;
+            }
+        }
+        // Even without a known root, < 50 chars of body text is almost certainly a shell
+        if (trimmedText.length < 50) {
+            return `Near-empty body text (${trimmedText.length} chars)`;
+        }
+    }
+    // 2. Loading phrases in an otherwise short page (< 500 chars)
+    if (trimmedText.length < 500) {
+        for (const pattern of SPA_LOADING_PATTERNS) {
+            if (lowerText.includes(pattern)) {
+                return `Short page with loading indicator: "${pattern}"`;
+            }
+        }
+    }
+    // 3. Script-heavy pages: mostly <script> tags, very little content
+    const scriptContent = $('script')
+        .toArray()
+        .reduce((sum, el) => sum + ($(el).html()?.length || 0), 0);
+    const htmlLength = rawHtml.length;
+    if (htmlLength > 1000 &&
+        scriptContent / htmlLength > SCRIPT_HEAVY_RATIO &&
+        trimmedText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
+        return `Script-heavy page (${Math.round((scriptContent / htmlLength) * 100)}% scripts, ${trimmedText.length} chars text)`;
+    }
+    return null;
+}
+/**
+ * Fetches a URL locally (through the user's IP) and converts to markdown.
+ */
+export async function localScrape(url, options = {}) {
+    const timeout = options.timeout || 30000;
+    // Determine requested formats
+    const formats = (options.formats || ['markdown']).map((f) => typeof f === 'string' ? f : f.type);
+    const wantMarkdown = formats.includes('markdown');
+    const wantHtml = formats.includes('html');
+    const wantRawHtml = formats.includes('rawHtml');
+    const wantLinks = formats.includes('links');
+    // Needs JSON/screenshot/branding? Can't do locally — return null to fall back
+    const needsServerSide = formats.some((f) => f === 'json' || f === 'screenshot' || f === 'branding' || f === 'summary');
+    if (needsServerSide) {
+        return { success: false, error: 'FORMAT_NEEDS_SERVER' };
+    }
+    try {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeout);
+        const fetchOptions = {
+            signal: controller.signal,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+                'Accept-Language': 'en-US,en;q=0.9',
+                'Accept-Encoding': 'gzip, deflate, br',
+                'Cache-Control': 'no-cache',
+                ...(options.headers || {}),
+            },
+            redirect: 'follow',
+        };
+        // Node 18+ native TLS rejection control
+        if (options.skipTlsVerification) {
+            fetchOptions.dispatcher = undefined; // handled below
+            process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
+        }
+        const response = await fetch(url, fetchOptions);
+        clearTimeout(timer);
+        if (options.skipTlsVerification) {
+            delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
+        }
+        const rawHtml = await response.text();
+        const statusCode = response.status;
+        const contentType = response.headers.get('content-type') || 'text/html';
+        // Parse with Cheerio
+        const $ = cheerio.load(rawHtml);
+        // Extract metadata
+        const title = $('title').first().text().trim() ||
+            $('meta[property="og:title"]').attr('content') ||
+            '';
+        const description = $('meta[name="description"]').attr('content') ||
+            $('meta[property="og:description"]').attr('content') ||
+            '';
+        const language = $('html').attr('lang') || '';
+        // Remove unwanted elements if onlyMainContent
+        if (options.onlyMainContent) {
+            $('nav, header, footer, aside, .sidebar, .menu, .navigation, .breadcrumb, .cookie-banner, .ad, .advertisement, [role="navigation"], [role="banner"], [role="complementary"]').remove();
+        }
+        // Apply excludeTags
+        if (options.excludeTags?.length) {
+            $(options.excludeTags.join(', ')).remove();
+        }
+        // Get the target HTML content
+        let targetHtml;
+        if (options.includeTags?.length) {
+            targetHtml = options.includeTags
+                .map((sel) => $(sel).html() || '')
+                .filter(Boolean)
+                .join('\n');
+        }
+        else if (options.onlyMainContent) {
+            // Try to find main content area
+            const mainSelectors = [
+                'main',
+                'article',
+                '[role="main"]',
+                '.main-content',
+                '.content',
+                '#content',
+                '#main',
+            ];
+            let mainHtml = '';
+            for (const sel of mainSelectors) {
+                const el = $(sel).first();
+                if (el.length && (el.html()?.length || 0) > 100) {
+                    mainHtml = el.html() || '';
+                    break;
+                }
+            }
+            targetHtml = mainHtml || $('body').html() || rawHtml;
+        }
+        else {
+            targetHtml = $('body').html() || rawHtml;
+        }
+        // Build response
+        const data = {
+            metadata: {
+                title,
+                description: description || undefined,
+                language: language || undefined,
+                sourceURL: url,
+                url: response.url || url,
+                statusCode,
+                contentType,
+                proxyUsed: 'local',
+            },
+        };
+        if (wantMarkdown) {
+            data.markdown = getTurndown().turndown(targetHtml);
+        }
+        if (wantHtml) {
+            data.html = targetHtml;
+        }
+        if (wantRawHtml) {
+            data.rawHtml = rawHtml;
+        }
+        if (wantLinks) {
+            const links = [];
+            $('a[href]').each((_, el) => {
+                const href = $(el).attr('href');
+                if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
+                    try {
+                        links.push(new URL(href, url).href);
+                    }
+                    catch {
+                        links.push(href);
+                    }
+                }
+            });
+            data.links = [...new Set(links)];
+        }
+        // --- SPA detection: check if the fetched content is a JS-only shell ---
+        const bodyText = $('body').text() || '';
+        const spaReason = detectSPASkeleton(rawHtml, bodyText, $);
+        if (spaReason) {
+            return { success: false, error: 'SPA_SKELETON_DETECTED', data };
+        }
+        return { success: true, data };
+    }
+    catch (err) {
+        if (err.name === 'AbortError') {
+            return { success: false, error: `Timeout after ${timeout}ms` };
+        }
+        return { success: false, error: err.message || String(err) };
+    }
+}
+/**
+ * Check if local proxy mode is enabled.
+ * Controlled by:
+ *   - SCORCHCRAWL_LOCAL_PROXY=true env var
+ *   - ?localProxy=true query param in SCORCHCRAWL_API_URL
+ */
+export function isLocalProxyEnabled() {
+    // Check env var
+    if (process.env.SCORCHCRAWL_LOCAL_PROXY?.toLowerCase() === 'true' ||
+        process.env.SCORCHCRAWL_LOCAL_PROXY === '1') {
+        return true;
+    }
+    // Check URL query param
+    const apiUrl = process.env.SCORCHCRAWL_API_URL;
+    if (apiUrl) {
+        try {
+            const parsed = new URL(apiUrl);
+            if (parsed.searchParams.get('localProxy') === 'true' ||
+                parsed.searchParams.get('localProxy') === '1') {
+                return true;
+            }
+        }
+        catch {
+            // ignore invalid URL
+        }
+    }
+    return false;
+}
+/**
+ * Returns SCORCHCRAWL_API_URL without the localProxy query param
+ * (so the scraping SDK doesn't pass it to the API).
+ */
+export function getCleanApiUrl() {
+    const apiUrl = process.env.SCORCHCRAWL_API_URL;
+    if (!apiUrl)
+        return undefined;
+    try {
+        const parsed = new URL(apiUrl);
+        parsed.searchParams.delete('localProxy');
+        const cleaned = parsed.toString();
+        // Remove trailing ? if no other params
+        return cleaned.replace(/\?$/, '');
+    }
+    catch {
+        return apiUrl;
+    }
+}

package/dist/rate-limiter.js

@@ -0,0 +1,361 @@
+/**
+ * Rate Limiting, Concurrency Control & Quota Monitoring
+ *
+ * The Copilot SDK / CLI engine handles HTTP-level rate limiting automatically:
+ *   - 429 retries (up to 5) with exponential backoff + jitter
+ *   - `retry-after` header respect
+ *   - HTTP/2 GOAWAY recovery
+ *   - 402 quota-exceeded errors surfaced as session.error
+ *
+ * This module adds the APPLICATION-LEVEL safeguards that the SDK doesn't:
+ *   1. Per-user and global concurrency limits for agent jobs
+ *   2. Quota snapshot tracking with proactive rejection
+ *   3. Sliding-window request rate tracking to avoid hitting 429 at all
+ *   4. Stale job garbage collection
+ *   5. Error classification and hook configuration
+ */
+export function buildRateLimitConfig() {
+    return {
+        maxGlobalConcurrency: envInt('RATE_LIMIT_MAX_GLOBAL_CONCURRENCY', 10),
+        maxPerUserConcurrency: envInt('RATE_LIMIT_MAX_PER_USER_CONCURRENCY', 3),
+        rateLimitWindowMs: envInt('RATE_LIMIT_WINDOW_MS', 60_000),
+        maxRequestsPerWindow: envInt('RATE_LIMIT_MAX_REQUESTS_PER_WINDOW', 20),
+        maxGlobalRequestsPerWindow: envInt('RATE_LIMIT_MAX_GLOBAL_REQUESTS_PER_WINDOW', 60),
+        quotaRejectThresholdPercent: envInt('RATE_LIMIT_QUOTA_REJECT_PERCENT', 5),
+        staleJobTimeoutMs: envInt('RATE_LIMIT_STALE_JOB_TIMEOUT_MS', 10 * 60 * 1000),
+        gcIntervalMs: envInt('RATE_LIMIT_GC_INTERVAL_MS', 60_000),
+    };
+}
+function envInt(key, fallback) {
+    const v = process.env[key];
+    if (v == null)
+        return fallback;
+    const n = parseInt(v, 10);
+    return Number.isFinite(n) ? n : fallback;
+}
+// ---------------------------------------------------------------------------
+// ConcurrencyTracker
+// ---------------------------------------------------------------------------
+/**
+ * Tracks in-flight agent jobs both globally and per-user.
+ * Call `acquire()` before starting a job, `release()` when done.
+ */
+export class ConcurrencyTracker {
+    config;
+    globalActive = 0;
+    perUser = new Map();
+    constructor(config) {
+        this.config = config;
+    }
+    /** Check if a new job is allowed for the given user key */
+    canAcquire(userKey) {
+        if (this.globalActive >= this.config.maxGlobalConcurrency) {
+            return {
+                allowed: false,
+                reason: `Server is at maximum capacity (${this.config.maxGlobalConcurrency} concurrent jobs). Please retry shortly.`,
+                retryAfterSeconds: 10,
+            };
+        }
+        const userCount = this.perUser.get(userKey) || 0;
+        if (userCount >= this.config.maxPerUserConcurrency) {
+            return {
+                allowed: false,
+                reason: `You already have ${userCount} concurrent agent jobs (max ${this.config.maxPerUserConcurrency}). Wait for a job to complete before starting another.`,
+                retryAfterSeconds: 15,
+            };
+        }
+        return { allowed: true };
+    }
+    acquire(userKey) {
+        this.globalActive++;
+        this.perUser.set(userKey, (this.perUser.get(userKey) || 0) + 1);
+    }
+    release(userKey) {
+        this.globalActive = Math.max(0, this.globalActive - 1);
+        const cur = this.perUser.get(userKey) || 0;
+        if (cur <= 1) {
+            this.perUser.delete(userKey);
+        }
+        else {
+            this.perUser.set(userKey, cur - 1);
+        }
+    }
+    /** Current stats (for observability / health endpoint) */
+    stats() {
+        return {
+            global: this.globalActive,
+            perUser: Object.fromEntries(this.perUser),
+        };
+    }
+}
+// ---------------------------------------------------------------------------
+// SlidingWindowRateLimiter
+// ---------------------------------------------------------------------------
+/**
+ * Sliding-window rate limiter that tracks request timestamps.
+ * This prevents us from *sending* too many requests in the first place,
+ * so we rarely hit the API-level 429.
+ */
+export class SlidingWindowRateLimiter {
+    config;
+    /** Map<userKey, timestamp[]> */
+    perUser = new Map();
+    globalTimestamps = [];
+    constructor(config) {
+        this.config = config;
+    }
+    check(userKey) {
+        const now = Date.now();
+        const windowStart = now - this.config.rateLimitWindowMs;
+        // --- Global check ---
+        this.globalTimestamps = this.globalTimestamps.filter((t) => t > windowStart);
+        if (this.globalTimestamps.length >= this.config.maxGlobalRequestsPerWindow) {
+            const oldest = this.globalTimestamps[0];
+            const retryAfter = Math.ceil((oldest + this.config.rateLimitWindowMs - now) / 1000);
+            return {
+                allowed: false,
+                reason: `Global rate limit reached (${this.config.maxGlobalRequestsPerWindow} requests per ${this.config.rateLimitWindowMs / 1000}s window). Please wait.`,
+                retryAfterSeconds: Math.max(1, retryAfter),
+            };
+        }
+        // --- Per-user check ---
+        let userTs = this.perUser.get(userKey) || [];
+        userTs = userTs.filter((t) => t > windowStart);
+        this.perUser.set(userKey, userTs);
+        if (userTs.length >= this.config.maxRequestsPerWindow) {
+            const oldest = userTs[0];
+            const retryAfter = Math.ceil((oldest + this.config.rateLimitWindowMs - now) / 1000);
+            return {
+                allowed: false,
+                reason: `Rate limit reached (${this.config.maxRequestsPerWindow} requests per ${this.config.rateLimitWindowMs / 1000}s window). Please wait.`,
+                retryAfterSeconds: Math.max(1, retryAfter),
+            };
+        }
+        return { allowed: true };
+    }
+    /** Record that a request was made */
+    record(userKey) {
+        const now = Date.now();
+        this.globalTimestamps.push(now);
+        const userTs = this.perUser.get(userKey) || [];
+        userTs.push(now);
+        this.perUser.set(userKey, userTs);
+    }
+    /** Purge old timestamps (called by GC) */
+    gc() {
+        const cutoff = Date.now() - this.config.rateLimitWindowMs;
+        this.globalTimestamps = this.globalTimestamps.filter((t) => t > cutoff);
+        for (const [key, ts] of this.perUser) {
+            const filtered = ts.filter((t) => t > cutoff);
+            if (filtered.length === 0) {
+                this.perUser.delete(key);
+            }
+            else {
+                this.perUser.set(key, filtered);
+            }
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// QuotaMonitor
+// ---------------------------------------------------------------------------
+/**
+ * Tracks Copilot quota snapshots per user to proactively reject requests
+ * when quota is about to be exhausted.
+ *
+ * Quota data is fed from Copilot session `assistant.usage` events.
+ */
+export class QuotaMonitor {
+    config;
+    perUser = new Map();
+    constructor(config) {
+        this.config = config;
+    }
+    /** Update quota snapshot for a user (call from session event handler) */
+    update(userKey, snapshot) {
+        const existing = this.perUser.get(userKey);
+        this.perUser.set(userKey, {
+            remainingPercent: snapshot.remainingPercent ?? existing?.remainingPercent ?? 100,
+            usedRequests: snapshot.usedRequests ?? existing?.usedRequests ?? 0,
+            entitlementRequests: snapshot.entitlementRequests ?? existing?.entitlementRequests ?? -1,
+            isUnlimited: snapshot.isUnlimited ?? existing?.isUnlimited ?? false,
+            resetDate: snapshot.resetDate ?? existing?.resetDate,
+            lastUpdated: Date.now(),
+        });
+    }
+    /** Check if the user has sufficient quota to start a new agent job */
+    check(userKey) {
+        const info = this.perUser.get(userKey);
+        // No quota info yet → allow (first request, or info not yet retrieved)
+        if (!info)
+            return { allowed: true };
+        // Unlimited entitlement → always allow
+        if (info.isUnlimited)
+            return { allowed: true };
+        // Stale quota info (> 5 min old) → allow but log warning
+        if (Date.now() - info.lastUpdated > 5 * 60 * 1000) {
+            return { allowed: true };
+        }
+        if (info.remainingPercent <= this.config.quotaRejectThresholdPercent) {
+            const resetMsg = info.resetDate
+                ? ` Quota resets on ${info.resetDate}.`
+                : '';
+            return {
+                allowed: false,
+                reason: `Copilot quota nearly exhausted (${info.remainingPercent.toFixed(1)}% remaining, ${info.usedRequests}/${info.entitlementRequests} used).${resetMsg} Upgrade your plan or wait for reset.`,
+            };
+        }
+        return { allowed: true };
+    }
+    /** Get quota info for a user (for diagnostics) */
+    get(userKey) {
+        return this.perUser.get(userKey);
+    }
+    /** Purge old entries (called by GC) */
+    gc() {
+        const cutoff = Date.now() - 30 * 60 * 1000; // 30 min
+        for (const [key, info] of this.perUser) {
+            if (info.lastUpdated < cutoff) {
+                this.perUser.delete(key);
+            }
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Unified RateLimitGuard
+// ---------------------------------------------------------------------------
+/**
+ * Facade that combines concurrency tracking, sliding-window rate limiting,
+ * and quota monitoring into a single `guard.check()` call.
+ */
+export class RateLimitGuard {
+    config;
+    concurrency;
+    rateLimit;
+    quota;
+    gcTimer = null;
+    constructor(config) {
+        this.config = config || buildRateLimitConfig();
+        this.concurrency = new ConcurrencyTracker(this.config);
+        this.rateLimit = new SlidingWindowRateLimiter(this.config);
+        this.quota = new QuotaMonitor(this.config);
+        // Start periodic GC
+        this.gcTimer = setInterval(() => {
+            this.rateLimit.gc();
+            this.quota.gc();
+        }, this.config.gcIntervalMs);
+        // Don't block process exit
+        if (this.gcTimer.unref)
+            this.gcTimer.unref();
+    }
+    /**
+     * Check ALL rate-limit gates in priority order.
+     * Returns the first rejection, or { allowed: true } if all pass.
+     */
+    check(userKey) {
+        // 1. Concurrency (cheapest check)
+        const concResult = this.concurrency.canAcquire(userKey);
+        if (!concResult.allowed)
+            return concResult;
+        // 2. Sliding-window rate limit
+        const rateResult = this.rateLimit.check(userKey);
+        if (!rateResult.allowed)
+            return rateResult;
+        // 3. Quota (proactive rejection)
+        const quotaResult = this.quota.check(userKey);
+        if (!quotaResult.allowed)
+            return quotaResult;
+        return { allowed: true };
+    }
+    /** Record that a request was admitted and is now in-flight */
+    acquire(userKey) {
+        this.concurrency.acquire(userKey);
+        this.rateLimit.record(userKey);
+    }
+    /** Release concurrency slot when job finishes */
+    release(userKey) {
+        this.concurrency.release(userKey);
+    }
+    /** Stats for observability */
+    stats() {
+        return {
+            concurrency: this.concurrency.stats(),
+            config: {
+                maxGlobalConcurrency: this.config.maxGlobalConcurrency,
+                maxPerUserConcurrency: this.config.maxPerUserConcurrency,
+                maxRequestsPerWindow: this.config.maxRequestsPerWindow,
+                rateLimitWindowMs: this.config.rateLimitWindowMs,
+                quotaRejectThresholdPercent: this.config.quotaRejectThresholdPercent,
+            },
+        };
+    }
+    /** Clean shutdown */
+    shutdown() {
+        if (this.gcTimer) {
+            clearInterval(this.gcTimer);
+            this.gcTimer = null;
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Copilot SDK session error hook factory
+// ---------------------------------------------------------------------------
+/**
+ * Build an `onErrorOccurred` hook for Copilot SDK sessions.
+ *
+ * This integrates with the SDK's ErrorOccurredHookOutput to make
+ * intelligent retry/skip/abort decisions:
+ *   - model_call + recoverable → retry (up to 2 extra)
+ *   - tool_execution → skip the failed tool, continue agent
+ *   - quota / auth errors → abort immediately
+ */
+export function buildErrorHook(jobId, logger) {
+    return (input) => {
+        const log = logger || console;
+        log.warn(`[RateLimit] Agent ${jobId} error`, {
+            context: input.errorContext,
+            recoverable: input.recoverable,
+            error: input.error.substring(0, 200),
+        });
+        // Quota / auth errors → abort immediately, don't waste retries
+        const lowerErr = input.error.toLowerCase();
+        if (lowerErr.includes('quota') ||
+            lowerErr.includes('402') ||
+            lowerErr.includes('not licensed') ||
+            lowerErr.includes('authentication')) {
+            return { errorHandling: 'abort', suppressOutput: false };
+        }
+        // Rate limit errors after SDK exhausted its 5 retries → abort
+        if (lowerErr.includes('rate limit') || lowerErr.includes('429')) {
+            return {
+                errorHandling: 'abort',
+                suppressOutput: false,
+                userNotification: 'Rate limit reached. Please retry later.',
+            };
+        }
+        // Model call errors that are recoverable → retry up to 2 additional times
+        if (input.errorContext === 'model_call' && input.recoverable) {
+            return { errorHandling: 'retry', retryCount: 2 };
+        }
+        // Tool execution errors → skip the failed tool, let agent continue
+        if (input.errorContext === 'tool_execution') {
+            return { errorHandling: 'skip' };
+        }
+        // Everything else → abort
+        return { errorHandling: 'abort' };
+    };
+}
+/**
+ * Identify agent jobs that have been "processing" longer than the timeout.
+ * The caller is responsible for actually marking/removing them.
+ */
+export function findStaleJobs(jobs, timeoutMs) {
+    const now = Date.now();
+    const stale = [];
+    for (const job of jobs) {
+        if (job.status === 'processing' && now - job.createdAt > timeoutMs) {
+            stale.push(job.id);
+        }
+    }
+    return stale;
+}