scorchcrawl-mcp 1.2.0 → 1.2.1

package/dist/bundle.js

@@ -1,2777 +0,0 @@
-#!/usr/bin/env node
-
-// src/index.ts
-import dotenv from "dotenv";
-
-// src/lib/fastmcp/index.ts
-import { FastMCP } from "firecrawl-fastmcp";
-
-// src/index.ts
-import { z } from "zod";
-
-// src/lib/scorch-client/index.ts
-import FirecrawlApp from "@mendable/firecrawl-js";
-var scorch_client_default = FirecrawlApp;
-
-// src/local-scraper.ts
-import TurndownService from "turndown";
-import * as cheerio from "cheerio";
-var _turndown = null;
-function getTurndown() {
-  if (!_turndown) {
-    _turndown = new TurndownService({
-      headingStyle: "atx",
-      codeBlockStyle: "fenced",
-      bulletListMarker: "-"
-    });
-    _turndown.remove(["script", "style", "noscript", "iframe"]);
-  }
-  return _turndown;
-}
-var SPA_LOADING_PATTERNS = [
-  "loading...",
-  "loading\u2026",
-  "please wait",
-  "just a moment",
-  "checking your browser",
-  "one moment please",
-  "redirecting",
-  "enable javascript",
-  "javascript is required",
-  "javascript must be enabled",
-  "this app requires javascript",
-  "you need to enable javascript",
-  "noscript"
-];
-var SPA_ROOT_SELECTORS = [
-  "#root",
-  // React (CRA, Vite)
-  "#app",
-  // Vue
-  "#__next",
-  // Next.js
-  "#__nuxt",
-  // Nuxt
-  "#svelte",
-  // SvelteKit
-  "app-root",
-  // Angular
-  "#___gatsby",
-  // Gatsby
-  "#main-app"
-  // misc
-];
-var MIN_MEANINGFUL_TEXT_LENGTH = 200;
-var SCRIPT_HEAVY_RATIO = 0.65;
-function detectSPASkeleton(rawHtml, _bodyText, $) {
-  const $clone = cheerio.load($.html());
-  $clone("script, style, noscript").remove();
-  const visibleText = $clone("body").text() || "";
-  const trimmedText = visibleText.replace(/\s+/g, " ").trim();
-  const lowerText = trimmedText.toLowerCase();
-  if (trimmedText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
-    for (const sel of SPA_ROOT_SELECTORS) {
-      const el = $(sel);
-      if (el.length > 0) {
-        const innerText = el.text().replace(/\s+/g, " ").trim();
-        if (innerText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
-          return `SPA root container "${sel}" with minimal content (${innerText.length} chars)`;
-        }
-      }
-    }
-    for (const pattern of SPA_LOADING_PATTERNS) {
-      if (lowerText.includes(pattern)) {
-        return `Loading indicator detected: "${pattern}"`;
-      }
-    }
-    if (trimmedText.length < 50) {
-      return `Near-empty body text (${trimmedText.length} chars)`;
-    }
-  }
-  if (trimmedText.length < 500) {
-    for (const pattern of SPA_LOADING_PATTERNS) {
-      if (lowerText.includes(pattern)) {
-        return `Short page with loading indicator: "${pattern}"`;
-      }
-    }
-  }
-  const scriptContent = $("script").toArray().reduce((sum, el) => sum + ($(el).html()?.length || 0), 0);
-  const htmlLength = rawHtml.length;
-  if (htmlLength > 1e3 && scriptContent / htmlLength > SCRIPT_HEAVY_RATIO && trimmedText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
-    return `Script-heavy page (${Math.round(scriptContent / htmlLength * 100)}% scripts, ${trimmedText.length} chars text)`;
-  }
-  return null;
-}
-async function localScrape(url, options = {}) {
-  const timeout = options.timeout || 3e4;
-  const formats = (options.formats || ["markdown"]).map(
-    (f) => typeof f === "string" ? f : f.type
-  );
-  const wantMarkdown = formats.includes("markdown");
-  const wantHtml = formats.includes("html");
-  const wantRawHtml = formats.includes("rawHtml");
-  const wantLinks = formats.includes("links");
-  const needsServerSide = formats.some(
-    (f) => f === "json" || f === "screenshot" || f === "branding" || f === "summary"
-  );
-  if (needsServerSide) {
-    return { success: false, error: "FORMAT_NEEDS_SERVER" };
-  }
-  try {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), timeout);
-    const fetchOptions = {
-      signal: controller.signal,
-      headers: {
-        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-        "Accept-Language": "en-US,en;q=0.9",
-        "Accept-Encoding": "gzip, deflate, br",
-        "Cache-Control": "no-cache",
-        ...options.headers || {}
-      },
-      redirect: "follow"
-    };
-    if (options.skipTlsVerification) {
-      fetchOptions.dispatcher = void 0;
-      process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
-    }
-    const response = await fetch(url, fetchOptions);
-    clearTimeout(timer);
-    if (options.skipTlsVerification) {
-      delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
-    }
-    const rawHtml = await response.text();
-    const statusCode = response.status;
-    const contentType = response.headers.get("content-type") || "text/html";
-    const $ = cheerio.load(rawHtml);
-    const title = $("title").first().text().trim() || $('meta[property="og:title"]').attr("content") || "";
-    const description = $('meta[name="description"]').attr("content") || $('meta[property="og:description"]').attr("content") || "";
-    const language = $("html").attr("lang") || "";
-    if (options.onlyMainContent) {
-      $(
-        'nav, header, footer, aside, .sidebar, .menu, .navigation, .breadcrumb, .cookie-banner, .ad, .advertisement, [role="navigation"], [role="banner"], [role="complementary"]'
-      ).remove();
-    }
-    if (options.excludeTags?.length) {
-      $(options.excludeTags.join(", ")).remove();
-    }
-    let targetHtml;
-    if (options.includeTags?.length) {
-      targetHtml = options.includeTags.map((sel) => $(sel).html() || "").filter(Boolean).join("\n");
-    } else if (options.onlyMainContent) {
-      const mainSelectors = [
-        "main",
-        "article",
-        '[role="main"]',
-        ".main-content",
-        ".content",
-        "#content",
-        "#main"
-      ];
-      let mainHtml = "";
-      for (const sel of mainSelectors) {
-        const el = $(sel).first();
-        if (el.length && (el.html()?.length || 0) > 100) {
-          mainHtml = el.html() || "";
-          break;
-        }
-      }
-      targetHtml = mainHtml || $("body").html() || rawHtml;
-    } else {
-      targetHtml = $("body").html() || rawHtml;
-    }
-    const data = {
-      metadata: {
-        title,
-        description: description || void 0,
-        language: language || void 0,
-        sourceURL: url,
-        url: response.url || url,
-        statusCode,
-        contentType,
-        proxyUsed: "local"
-      }
-    };
-    if (wantMarkdown) {
-      data.markdown = getTurndown().turndown(targetHtml);
-    }
-    if (wantHtml) {
-      data.html = targetHtml;
-    }
-    if (wantRawHtml) {
-      data.rawHtml = rawHtml;
-    }
-    if (wantLinks) {
-      const links = [];
-      $("a[href]").each((_, el) => {
-        const href = $(el).attr("href");
-        if (href && !href.startsWith("#") && !href.startsWith("javascript:")) {
-          try {
-            links.push(new URL(href, url).href);
-          } catch {
-            links.push(href);
-          }
-        }
-      });
-      data.links = [...new Set(links)];
-    }
-    const bodyText = $("body").text() || "";
-    const spaReason = detectSPASkeleton(rawHtml, bodyText, $);
-    if (spaReason) {
-      return { success: false, error: "SPA_SKELETON_DETECTED", data };
-    }
-    return { success: true, data };
-  } catch (err) {
-    if (err.name === "AbortError") {
-      return { success: false, error: `Timeout after ${timeout}ms` };
-    }
-    return { success: false, error: err.message || String(err) };
-  }
-}
-function isLocalProxyEnabled() {
-  if (process.env.SCORCHCRAWL_LOCAL_PROXY?.toLowerCase() === "true" || process.env.SCORCHCRAWL_LOCAL_PROXY === "1") {
-    return true;
-  }
-  const apiUrl = process.env.SCORCHCRAWL_API_URL;
-  if (apiUrl) {
-    try {
-      const parsed = new URL(apiUrl);
-      if (parsed.searchParams.get("localProxy") === "true" || parsed.searchParams.get("localProxy") === "1") {
-        return true;
-      }
-    } catch {
-    }
-  }
-  return false;
-}
-function getCleanApiUrl() {
-  const apiUrl = process.env.SCORCHCRAWL_API_URL;
-  if (!apiUrl) return void 0;
-  try {
-    const parsed = new URL(apiUrl);
-    parsed.searchParams.delete("localProxy");
-    const cleaned = parsed.toString();
-    return cleaned.replace(/\?$/, "");
-  } catch {
-    return apiUrl;
-  }
-}
-
-// src/copilot-agent.ts
-import { CopilotClient } from "@github/copilot-sdk";
-import { v4 as uuidv4 } from "uuid";
-
-// src/rate-limiter.ts
-function buildRateLimitConfig() {
-  return {
-    maxGlobalConcurrency: envInt("RATE_LIMIT_MAX_GLOBAL_CONCURRENCY", 10),
-    maxPerUserConcurrency: envInt("RATE_LIMIT_MAX_PER_USER_CONCURRENCY", 3),
-    rateLimitWindowMs: envInt("RATE_LIMIT_WINDOW_MS", 6e4),
-    maxRequestsPerWindow: envInt("RATE_LIMIT_MAX_REQUESTS_PER_WINDOW", 20),
-    maxGlobalRequestsPerWindow: envInt("RATE_LIMIT_MAX_GLOBAL_REQUESTS_PER_WINDOW", 60),
-    quotaRejectThresholdPercent: envInt("RATE_LIMIT_QUOTA_REJECT_PERCENT", 5),
-    staleJobTimeoutMs: envInt("RATE_LIMIT_STALE_JOB_TIMEOUT_MS", 10 * 60 * 1e3),
-    gcIntervalMs: envInt("RATE_LIMIT_GC_INTERVAL_MS", 6e4)
-  };
-}
-function envInt(key, fallback) {
-  const v = process.env[key];
-  if (v == null) return fallback;
-  const n = parseInt(v, 10);
-  return Number.isFinite(n) ? n : fallback;
-}
-var ConcurrencyTracker = class {
-  constructor(config) {
-    this.config = config;
-  }
-  globalActive = 0;
-  perUser = /* @__PURE__ */ new Map();
-  /** Check if a new job is allowed for the given user key */
-  canAcquire(userKey) {
-    if (this.globalActive >= this.config.maxGlobalConcurrency) {
-      return {
-        allowed: false,
-        reason: `Server is at maximum capacity (${this.config.maxGlobalConcurrency} concurrent jobs). Please retry shortly.`,
-        retryAfterSeconds: 10
-      };
-    }
-    const userCount = this.perUser.get(userKey) || 0;
-    if (userCount >= this.config.maxPerUserConcurrency) {
-      return {
-        allowed: false,
-        reason: `You already have ${userCount} concurrent agent jobs (max ${this.config.maxPerUserConcurrency}). Wait for a job to complete before starting another.`,
-        retryAfterSeconds: 15
-      };
-    }
-    return { allowed: true };
-  }
-  acquire(userKey) {
-    this.globalActive++;
-    this.perUser.set(userKey, (this.perUser.get(userKey) || 0) + 1);
-  }
-  release(userKey) {
-    this.globalActive = Math.max(0, this.globalActive - 1);
-    const cur = this.perUser.get(userKey) || 0;
-    if (cur <= 1) {
-      this.perUser.delete(userKey);
-    } else {
-      this.perUser.set(userKey, cur - 1);
-    }
-  }
-  /** Current stats (for observability / health endpoint) */
-  stats() {
-    return {
-      global: this.globalActive,
-      perUser: Object.fromEntries(this.perUser)
-    };
-  }
-};
-var SlidingWindowRateLimiter = class {
-  constructor(config) {
-    this.config = config;
-  }
-  /** Map<userKey, timestamp[]> */
-  perUser = /* @__PURE__ */ new Map();
-  globalTimestamps = [];
-  check(userKey) {
-    const now = Date.now();
-    const windowStart = now - this.config.rateLimitWindowMs;
-    this.globalTimestamps = this.globalTimestamps.filter((t) => t > windowStart);
-    if (this.globalTimestamps.length >= this.config.maxGlobalRequestsPerWindow) {
-      const oldest = this.globalTimestamps[0];
-      const retryAfter = Math.ceil((oldest + this.config.rateLimitWindowMs - now) / 1e3);
-      return {
-        allowed: false,
-        reason: `Global rate limit reached (${this.config.maxGlobalRequestsPerWindow} requests per ${this.config.rateLimitWindowMs / 1e3}s window). Please wait.`,
-        retryAfterSeconds: Math.max(1, retryAfter)
-      };
-    }
-    let userTs = this.perUser.get(userKey) || [];
-    userTs = userTs.filter((t) => t > windowStart);
-    this.perUser.set(userKey, userTs);
-    if (userTs.length >= this.config.maxRequestsPerWindow) {
-      const oldest = userTs[0];
-      const retryAfter = Math.ceil((oldest + this.config.rateLimitWindowMs - now) / 1e3);
-      return {
-        allowed: false,
-        reason: `Rate limit reached (${this.config.maxRequestsPerWindow} requests per ${this.config.rateLimitWindowMs / 1e3}s window). Please wait.`,
-        retryAfterSeconds: Math.max(1, retryAfter)
-      };
-    }
-    return { allowed: true };
-  }
-  /** Record that a request was made */
-  record(userKey) {
-    const now = Date.now();
-    this.globalTimestamps.push(now);
-    const userTs = this.perUser.get(userKey) || [];
-    userTs.push(now);
-    this.perUser.set(userKey, userTs);
-  }
-  /** Purge old timestamps (called by GC) */
-  gc() {
-    const cutoff = Date.now() - this.config.rateLimitWindowMs;
-    this.globalTimestamps = this.globalTimestamps.filter((t) => t > cutoff);
-    for (const [key, ts] of this.perUser) {
-      const filtered = ts.filter((t) => t > cutoff);
-      if (filtered.length === 0) {
-        this.perUser.delete(key);
-      } else {
-        this.perUser.set(key, filtered);
-      }
-    }
-  }
-};
-var QuotaMonitor = class {
-  constructor(config) {
-    this.config = config;
-  }
-  perUser = /* @__PURE__ */ new Map();
-  /** Update quota snapshot for a user (call from session event handler) */
-  update(userKey, snapshot) {
-    const existing = this.perUser.get(userKey);
-    this.perUser.set(userKey, {
-      remainingPercent: snapshot.remainingPercent ?? existing?.remainingPercent ?? 100,
-      usedRequests: snapshot.usedRequests ?? existing?.usedRequests ?? 0,
-      entitlementRequests: snapshot.entitlementRequests ?? existing?.entitlementRequests ?? -1,
-      isUnlimited: snapshot.isUnlimited ?? existing?.isUnlimited ?? false,
-      resetDate: snapshot.resetDate ?? existing?.resetDate,
-      lastUpdated: Date.now()
-    });
-  }
-  /** Check if the user has sufficient quota to start a new agent job */
-  check(userKey) {
-    const info = this.perUser.get(userKey);
-    if (!info) return { allowed: true };
-    if (info.isUnlimited) return { allowed: true };
-    if (Date.now() - info.lastUpdated > 5 * 60 * 1e3) {
-      return { allowed: true };
-    }
-    if (info.remainingPercent <= this.config.quotaRejectThresholdPercent) {
-      const resetMsg = info.resetDate ? ` Quota resets on ${info.resetDate}.` : "";
-      return {
-        allowed: false,
-        reason: `Copilot quota nearly exhausted (${info.remainingPercent.toFixed(1)}% remaining, ${info.usedRequests}/${info.entitlementRequests} used).${resetMsg} Upgrade your plan or wait for reset.`
-      };
-    }
-    return { allowed: true };
-  }
-  /** Get quota info for a user (for diagnostics) */
-  get(userKey) {
-    return this.perUser.get(userKey);
-  }
-  /** Purge old entries (called by GC) */
-  gc() {
-    const cutoff = Date.now() - 30 * 60 * 1e3;
-    for (const [key, info] of this.perUser) {
-      if (info.lastUpdated < cutoff) {
-        this.perUser.delete(key);
-      }
-    }
-  }
-};
-var RateLimitGuard = class {
-  config;
-  concurrency;
-  rateLimit;
-  quota;
-  gcTimer = null;
-  constructor(config) {
-    this.config = config || buildRateLimitConfig();
-    this.concurrency = new ConcurrencyTracker(this.config);
-    this.rateLimit = new SlidingWindowRateLimiter(this.config);
-    this.quota = new QuotaMonitor(this.config);
-    this.gcTimer = setInterval(() => {
-      this.rateLimit.gc();
-      this.quota.gc();
-    }, this.config.gcIntervalMs);
-    if (this.gcTimer.unref) this.gcTimer.unref();
-  }
-  /**
-   * Check ALL rate-limit gates in priority order.
-   * Returns the first rejection, or { allowed: true } if all pass.
-   */
-  check(userKey) {
-    const concResult = this.concurrency.canAcquire(userKey);
-    if (!concResult.allowed) return concResult;
-    const rateResult = this.rateLimit.check(userKey);
-    if (!rateResult.allowed) return rateResult;
-    const quotaResult = this.quota.check(userKey);
-    if (!quotaResult.allowed) return quotaResult;
-    return { allowed: true };
-  }
-  /** Record that a request was admitted and is now in-flight */
-  acquire(userKey) {
-    this.concurrency.acquire(userKey);
-    this.rateLimit.record(userKey);
-  }
-  /** Release concurrency slot when job finishes */
-  release(userKey) {
-    this.concurrency.release(userKey);
-  }
-  /** Stats for observability */
-  stats() {
-    return {
-      concurrency: this.concurrency.stats(),
-      config: {
-        maxGlobalConcurrency: this.config.maxGlobalConcurrency,
-        maxPerUserConcurrency: this.config.maxPerUserConcurrency,
-        maxRequestsPerWindow: this.config.maxRequestsPerWindow,
-        rateLimitWindowMs: this.config.rateLimitWindowMs,
-        quotaRejectThresholdPercent: this.config.quotaRejectThresholdPercent
-      }
-    };
-  }
-  /** Clean shutdown */
-  shutdown() {
-    if (this.gcTimer) {
-      clearInterval(this.gcTimer);
-      this.gcTimer = null;
-    }
-  }
-};
-function buildErrorHook(jobId, logger) {
-  return (input) => {
-    const log = logger || console;
-    log.warn(`[RateLimit] Agent ${jobId} error`, {
-      context: input.errorContext,
-      recoverable: input.recoverable,
-      error: input.error.substring(0, 200)
-    });
-    const lowerErr = input.error.toLowerCase();
-    if (lowerErr.includes("quota") || lowerErr.includes("402") || lowerErr.includes("not licensed") || lowerErr.includes("authentication")) {
-      return { errorHandling: "abort", suppressOutput: false };
-    }
-    if (lowerErr.includes("rate limit") || lowerErr.includes("429")) {
-      return {
-        errorHandling: "abort",
-        suppressOutput: false,
-        userNotification: "Rate limit reached. Please retry later."
-      };
-    }
-    if (input.errorContext === "model_call" && input.recoverable) {
-      return { errorHandling: "retry", retryCount: 2 };
-    }
-    if (input.errorContext === "tool_execution") {
-      return { errorHandling: "skip" };
-    }
-    return { errorHandling: "abort" };
-  };
-}
-function findStaleJobs(jobs, timeoutMs) {
-  const now = Date.now();
-  const stale = [];
-  for (const job of jobs) {
-    if (job.status === "processing" && now - job.createdAt > timeoutMs) {
-      stale.push(job.id);
-    }
-  }
-  return stale;
-}
-
-// src/copilot-agent.ts
-function parseAllowedModels() {
-  const envModels = process.env.COPILOT_AGENT_MODELS;
-  if (!envModels) {
-    return ["gpt-4.1", "gpt-4o", "gpt-5-mini", "grok-code-fast-1"];
-  }
-  return envModels.split(",").map((m) => m.trim()).filter(Boolean);
-}
-function getDefaultModel() {
-  const defaultFromEnv = process.env.COPILOT_AGENT_DEFAULT_MODEL;
-  if (defaultFromEnv) return defaultFromEnv.trim();
-  const allowed = parseAllowedModels();
-  return allowed[0] || "gpt-4.1";
-}
-function buildAgentConfig() {
-  const config = {
-    allowedModels: parseAllowedModels(),
-    defaultModel: getDefaultModel()
-  };
-  if (process.env.COPILOT_AGENT_PROVIDER_TYPE && process.env.COPILOT_AGENT_PROVIDER_BASE_URL) {
-    config.provider = {
-      type: process.env.COPILOT_AGENT_PROVIDER_TYPE,
-      baseUrl: process.env.COPILOT_AGENT_PROVIDER_BASE_URL,
-      apiKey: process.env.COPILOT_AGENT_PROVIDER_API_KEY
-    };
-  }
-  return config;
-}
-function buildScrapingTools(client, origin) {
-  return [
-    {
-      name: "web_scrape",
-      description: "Scrape content from a single URL. Returns markdown content by default. Use formats parameter to request JSON extraction with a schema, or other formats like html, screenshot, links. If the result is empty, blocked, or shows anti-bot content, use web_screenshot to visually inspect the page.",
-      parameters: {
-        type: "object",
-        properties: {
-          url: { type: "string", description: "The URL to scrape" },
-          formats: {
-            type: "array",
-            description: 'Output formats: "markdown", "html", "links", or JSON extraction object',
-            items: { type: "string" }
-          },
-          onlyMainContent: {
-            type: "boolean",
-            description: "Extract only the main content, excluding nav/footer/etc"
-          },
-          waitFor: {
-            type: "number",
-            description: "Wait time in ms for JS-rendered pages (default 0)"
-          }
-        },
-        required: ["url"]
-      },
-      handler: async (args2) => {
-        try {
-          const a = args2;
-          const { url, ...opts } = a;
-          const res = await client.scrape(String(url), {
-            ...opts,
-            origin
-          });
-          return {
-            textResultForLlm: JSON.stringify(res, null, 2),
-            resultType: "success"
-          };
-        } catch (err) {
-          return {
-            textResultForLlm: `Scrape failed: ${err.message || err}`,
-            resultType: "failure",
-            error: String(err.message || err)
-          };
-        }
-      }
-    },
-    {
-      name: "web_screenshot",
-      description: "Take a screenshot of a web page for visual inspection. Use this when web_scrape returns empty, blocked, or anti-bot content. The screenshot is passed directly to your vision capability so you can analyze what is on the page \u2014 cookie popups, CAPTCHAs, anti-bot walls, or the actual rendered content.",
-      parameters: {
-        type: "object",
-        properties: {
-          url: { type: "string", description: "The URL to screenshot" },
-          fullPage: {
-            type: "boolean",
-            description: "Capture the full scrollable page (default: false, viewport only)"
-          },
-          waitFor: {
-            type: "number",
-            description: "Wait time in ms for JS rendering before screenshot (default 3000)"
-          }
-        },
-        required: ["url"]
-      },
-      handler: async (args2) => {
-        try {
-          const a = args2;
-          const url = String(a.url);
-          const fullPage = a.fullPage === true;
-          const waitFor = typeof a.waitFor === "number" ? a.waitFor : 3e3;
-          const screenshotFormat = {
-            type: "screenshot",
-            fullPage
-          };
-          const res = await client.scrape(url, {
-            formats: [screenshotFormat],
-            waitFor,
-            origin
-          });
-          const resAny = res;
-          const screenshot = resAny?.screenshot || resAny?.data?.screenshot;
-          const metadata = resAny?.metadata || resAny?.data?.metadata || {};
-          if (screenshot) {
-            const imgBytes = Buffer.from(screenshot, "base64").length;
-            const description = [
-              `Screenshot of: ${url}`,
-              `Title: ${metadata.title || "Unknown"}`,
-              `Status: ${metadata.statusCode || "Unknown"}`,
-              `Size: ${Math.round(imgBytes / 1024)}KB`,
-              "",
-              "Analyze the attached image to determine:",
-              "- Is there a cookie consent popup blocking content?",
-              "- Is there a CAPTCHA or anti-bot challenge?",
-              "- Is the page content actually rendered?",
-              "- What text/data is visible?"
-            ].join("\n");
-            return {
-              textResultForLlm: description,
-              binaryResultsForLlm: [{
-                data: screenshot,
-                mimeType: "image/png",
-                type: "image",
-                description: `Screenshot of ${url}`
-              }],
-              resultType: "success"
-            };
-          }
-          return {
-            textResultForLlm: `Screenshot request completed but no image was returned. Page metadata: ${JSON.stringify(metadata, null, 2)}`,
-            resultType: "success"
-          };
-        } catch (err) {
-          return {
-            textResultForLlm: `Screenshot failed: ${err.message || err}. The page may be unreachable or the screenshot service unavailable.`,
-            resultType: "failure",
-            error: String(err.message || err)
-          };
-        }
-      }
-    },
-    {
-      name: "web_search",
-      description: "Search the web for information. Returns search results with titles, URLs, and snippets. Supports search operators like site:, inurl:, intitle:, and quoted phrases.",
-      parameters: {
-        type: "object",
-        properties: {
-          query: { type: "string", description: "The search query" },
-          limit: {
-            type: "number",
-            description: "Maximum number of results (default 5)"
-          }
-        },
-        required: ["query"]
-      },
-      handler: async (args2) => {
-        try {
-          const a = args2;
-          const { query, ...opts } = a;
-          const res = await client.search(String(query), {
-            ...opts,
-            origin
-          });
-          return {
-            textResultForLlm: JSON.stringify(res, null, 2),
-            resultType: "success"
-          };
-        } catch (err) {
-          return {
-            textResultForLlm: `Search failed: ${err.message || err}`,
-            resultType: "failure",
-            error: String(err.message || err)
-          };
-        }
-      }
-    },
-    {
-      name: "web_map",
-      description: "Map a website to discover all indexed URLs. Use the search parameter to find specific pages. Useful when you need to find the right page before scraping.",
-      parameters: {
-        type: "object",
-        properties: {
-          url: { type: "string", description: "The website URL to map" },
-          search: {
-            type: "string",
-            description: "Optional search query to filter URLs"
-          },
-          limit: {
-            type: "number",
-            description: "Maximum number of URLs to return"
-          }
-        },
-        required: ["url"]
-      },
-      handler: async (args2) => {
-        try {
-          const a = args2;
-          const { url, ...opts } = a;
-          const res = await client.map(String(url), {
-            ...opts,
-            origin
-          });
-          return {
-            textResultForLlm: JSON.stringify(res, null, 2),
-            resultType: "success"
-          };
-        } catch (err) {
-          return {
-            textResultForLlm: `Map failed: ${err.message || err}`,
-            resultType: "failure",
-            error: String(err.message || err)
-          };
-        }
-      }
-    },
-    {
-      name: "web_extract",
-      description: "Extract structured information from web pages using LLM capabilities. Provide URLs, a prompt describing what to extract, and an optional JSON schema for the output format.",
-      parameters: {
-        type: "object",
-        properties: {
-          urls: {
-            type: "array",
-            items: { type: "string" },
-            description: "URLs to extract information from"
-          },
-          prompt: {
-            type: "string",
-            description: "Description of what information to extract"
-          },
-          schema: {
-            type: "object",
-            description: "JSON schema for structured output"
-          }
-        },
-        required: ["urls"]
-      },
-      handler: async (args2) => {
-        try {
-          const a = args2;
-          const res = await client.extract({
-            urls: a.urls,
-            prompt: a.prompt,
-            schema: a.schema,
-            origin
-          });
-          return {
-            textResultForLlm: JSON.stringify(res, null, 2),
-            resultType: "success"
-          };
-        } catch (err) {
-          return {
-            textResultForLlm: `Extract failed: ${err.message || err}`,
-            resultType: "failure",
-            error: String(err.message || err)
-          };
-        }
-      }
-    }
-  ];
-}
-var agentJobs = /* @__PURE__ */ new Map();
-var rateLimitGuard = new RateLimitGuard(buildRateLimitConfig());
-setInterval(() => {
-  const staleIds = findStaleJobs(agentJobs.values(), rateLimitGuard.config.staleJobTimeoutMs);
-  for (const id of staleIds) {
-    const job = agentJobs.get(id);
-    if (job) {
-      job.status = "failed";
-      job.error = `Job timed out after ${rateLimitGuard.config.staleJobTimeoutMs / 1e3}s without completing.`;
-      job.completedAt = Date.now();
-      rateLimitGuard.release(job._userKey || "");
-      console.warn(`[RateLimit] Reaped stale job ${id}`);
-    }
-  }
-}, rateLimitGuard.config.gcIntervalMs);
-var clientCache = /* @__PURE__ */ new Map();
-var MAX_CLIENT_AGE_MS = 30 * 60 * 1e3;
-setInterval(() => {
-  const now = Date.now();
-  for (const [key, entry] of clientCache) {
-    if (now - entry.lastUsed > MAX_CLIENT_AGE_MS) {
-      try {
-        entry.client.stop();
-      } catch {
-      }
-      clientCache.delete(key);
-    }
-  }
-}, 5 * 60 * 1e3);
-async function getCopilotClient(userToken) {
-  const cacheKey = userToken || "";
-  const existing = clientCache.get(cacheKey);
-  if (existing) {
-    existing.lastUsed = Date.now();
-    return existing.client;
-  }
-  const clientOpts = {};
-  if (process.env.COPILOT_CLI_PATH) {
-    clientOpts.cliPath = process.env.COPILOT_CLI_PATH;
-  }
-  if (process.env.COPILOT_CLI_URL) {
-    clientOpts.cliUrl = process.env.COPILOT_CLI_URL;
-  }
-  const token = userToken || process.env.GITHUB_TOKEN;
-  if (token) {
-    clientOpts.githubToken = token;
-  }
-  const client = new CopilotClient(clientOpts);
-  clientCache.set(cacheKey, { client, lastUsed: Date.now() });
-  return client;
-}
-async function startAgent(request, scrapingClient, origin, config, copilotToken) {
-  const jobId = uuidv4();
-  const userKey = copilotToken || "__server__";
-  const gate = rateLimitGuard.check(userKey);
-  if (!gate.allowed) {
-    return {
-      id: jobId,
-      status: "rate_limited",
-      rateLimited: true,
-      retryAfterSeconds: gate.retryAfterSeconds,
-      error: gate.reason
-    };
-  }
-  const requestedModel = request.model || config.defaultModel;
-  if (!config.allowedModels.includes(requestedModel)) {
-    return {
-      id: jobId,
-      status: "failed",
-      error: `Model "${requestedModel}" is not in the allowed list: ${config.allowedModels.join(", ")}`
-    };
-  }
-  rateLimitGuard.acquire(userKey);
-  const job = {
-    id: jobId,
-    status: "processing",
-    prompt: request.prompt,
-    createdAt: Date.now(),
-    _userKey: userKey
-  };
-  agentJobs.set(jobId, job);
-  try {
-    await runAgentJob(job, request, scrapingClient, origin, config, requestedModel, copilotToken);
-  } catch (err) {
-    job.status = "failed";
-    job.error = String(err.message || err);
-    job.completedAt = Date.now();
-  } finally {
-    rateLimitGuard.release(userKey);
-  }
-  return {
-    id: jobId,
-    status: job.status,
-    data: job.result || void 0,
-    error: job.error || void 0,
-    duration: job.completedAt ? `${((job.completedAt - job.createdAt) / 1e3).toFixed(1)}s` : void 0
-  };
-}
-async function runAgentJob(job, request, scrapingClient, origin, config, model, copilotToken) {
-  try {
-    job.progress = "Initializing Copilot SDK agent...";
-    const client = await getCopilotClient(copilotToken);
-    const tools = buildScrapingTools(scrapingClient, origin);
-    const sessionConfig = {
-      model,
-      tools,
-      // Disable all built-in tools - we only want our scorchcrawl tools
-      availableTools: tools.map((t) => t.name),
-      systemMessage: {
-        mode: "replace",
-        content: buildSystemPrompt(request)
-      }
-    };
-    if (config.provider) {
-      sessionConfig.provider = {
-        type: config.provider.type,
-        baseUrl: config.provider.baseUrl,
-        ...config.provider.apiKey && { apiKey: config.provider.apiKey }
-      };
-    }
-    job.progress = `Creating agent session with model: ${model}`;
-    const session = await client.createSession(sessionConfig);
-    try {
-      const errorHook = buildErrorHook(job.id);
-      if (typeof session.registerHooks === "function") {
-        session.registerHooks({ onErrorOccurred: errorHook });
-      }
-    } catch {
-    }
-    try {
-      const userKey = copilotToken || "__server__";
-      session.on?.("assistant.usage", (event) => {
-        const snapshots = event?.data?.quotaSnapshots || event?.quotaSnapshots;
-        if (snapshots) {
-          const snap = snapshots.chat || Object.values(snapshots)[0];
-          if (snap) {
-            rateLimitGuard.quota.update(userKey, {
-              remainingPercent: snap.remainingPercentage ?? snap.percent_remaining ?? 100,
-              usedRequests: snap.usedRequests ?? snap.remaining ?? 0,
-              entitlementRequests: snap.entitlementRequests ?? snap.entitlement ?? -1,
-              isUnlimited: snap.isUnlimitedEntitlement ?? snap.unlimited ?? false,
-              resetDate: snap.resetDate ?? void 0
-            });
-          }
-        }
-      });
-    } catch {
-    }
-    try {
-      job.progress = "Agent is researching...";
-      let userPrompt = request.prompt;
-      if (request.urls && request.urls.length > 0) {
-        userPrompt += `
-
-Focus on these URLs:
-${request.urls.map((u) => `- ${u}`).join("\n")}`;
-      }
-      if (request.schema) {
-        userPrompt += `
-
-Return the results in this JSON schema format:
-${JSON.stringify(request.schema, null, 2)}`;
-      }
-      const timeoutMs = request.timeout || (process.env.AGENT_TIMEOUT_MS ? parseInt(process.env.AGENT_TIMEOUT_MS, 10) : void 0) || void 0;
-      const response = await session.sendAndWait({ prompt: userPrompt }, timeoutMs);
-      job.status = "completed";
-      job.completedAt = Date.now();
-      job.result = {
-        success: true,
-        data: response?.data?.content || "No response generated",
-        model
-      };
-    } finally {
-      try {
-        await session.destroy();
-      } catch {
-      }
-    }
-  } catch (err) {
-    job.status = "failed";
-    job.error = `Agent error: ${err.message || err}`;
-    job.completedAt = Date.now();
-  }
-}
-function buildSystemPrompt(request) {
-  return `You are an autonomous web research agent. Your job is to browse the internet, search for information, navigate through pages, and extract structured data based on the user's query.
-
-You have access to the following web tools:
-- **web_search**: Search the web for information. Start with this for broad queries.
-- **web_scrape**: Scrape content from a specific URL. Use this to get page content.
-- **web_screenshot**: Take a screenshot of a page. The image is sent directly to you for vision analysis. Use this when scrape returns empty/blocked content.
-- **web_map**: Map a website to discover all its URLs. Use this to find specific pages on a site.
-- **web_extract**: Extract structured data from URLs using LLM. Use this for structured extraction.
-
-## Research Strategy
-
-1. **Start with search** to find relevant URLs if no specific URLs are provided.
-2. **Map websites** to discover relevant pages when you need to explore a site's structure.
-3. **Scrape specific pages** to get their content.
-4. **If scrape fails or returns empty/blocked content**, use **web_screenshot** to visually inspect the page and determine what's blocking (cookie popup, CAPTCHA, anti-bot, JS rendering issue).
-5. **Extract structured data** when you need specific fields from pages.
-
-## Visual Inspection Fallback
-
-When web_scrape returns any of these, ALWAYS follow up with web_screenshot:
-- Empty or very short content (< 100 characters)
-- Anti-bot messages ("Checking your browser", "Access denied", "Just a moment")
-- Cookie consent popup HTML instead of actual content
-- CAPTCHA challenges
-- Generic error pages
-
-The screenshot helps you understand what the user would see in a browser, and report back what's actually on the page even when text extraction fails.
-
-## Guidelines
-
-- Be thorough: check multiple sources when possible.
-- Be efficient: don't scrape pages that aren't relevant.
-- If a page fails to load or returns empty content, try an alternative approach (screenshot, different URL, waitFor parameter).
-- Always provide your final answer with all the information you gathered.
-- If the user provided a JSON schema, format your final response to match that schema.
-- Cite your sources with URLs when providing information.
-- When reporting screenshot results, describe what you observed on the page.
-
-## Important
-
-- You MUST use the provided tools to gather information. Do not make up or hallucinate data.
-- If you cannot find the requested information after reasonable effort, say so clearly.
-- Provide complete, well-structured responses.`;
-}
-async function shutdownAgent() {
-  rateLimitGuard.shutdown();
-  for (const [, entry] of clientCache) {
-    try {
-      await entry.client.stop();
-    } catch {
-    }
-  }
-  clientCache.clear();
-}
-
-// src/response-utils.ts
-import { createHash } from "node:crypto";
-var MAX_CONTENT_CHARS = parseInt(
-  process.env.SCORCHCRAWL_MAX_CONTENT_CHARS || "25000",
-  10
-);
-var SUMMARIZE_AFTER_WORDS = parseInt(
-  process.env.SCORCHCRAWL_SUMMARIZE_AFTER_WORDS || "5000",
-  10
-);
-var SUMMARIZE_MODEL = process.env.SCORCHCRAWL_SUMMARIZE_MODEL || "gpt-4o";
-var SUMMARIZE_CACHE_SIZE = parseInt(
-  process.env.SCORCHCRAWL_SUMMARIZE_CACHE_SIZE || "100",
-  10
-);
-var SUMMARIZE_MAX_PER_MINUTE = parseInt(
-  process.env.SCORCHCRAWL_SUMMARIZE_MAX_PER_MINUTE || "10",
-  10
-);
-var SUMMARIZE_TIMEOUT_MS = parseInt(
-  process.env.SCORCHCRAWL_SUMMARIZE_TIMEOUT_MS || "300000",
-  10
-);
-function mapError(err) {
-  const raw = normalizeErrorString(err);
-  const lower = raw.toLowerCase();
-  if (matches(lower, ["captcha", "recaptcha", "hcaptcha", "turnstile"])) {
-    return {
-      code: "CAPTCHA_REQUIRED",
-      message: "Page requires CAPTCHA verification.",
-      suggestions: [
-        'Try proxy: "stealth" to avoid CAPTCHA triggers',
-        "Use scorch_agent which may navigate around CAPTCHAs",
-        "Try scorch_search to find the content from a cached source"
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["403", "forbidden", "cloudflare", "cf-ray", "challenge", "access denied"])) {
-    return {
-      code: "ACCESS_DENIED",
-      message: "This site blocks automated access.",
-      suggestions: [
-        "Try scorch_search to find cached/indexed content instead",
-        "Add waitFor: 5000 to let challenge pages resolve",
-        'Try with proxy: "stealth" for enhanced bot evasion'
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["404", "not found", "page not found", "does not exist"])) {
-    return {
-      code: "NOT_FOUND",
-      message: "Page does not exist at this URL.",
-      suggestions: [
-        "Use scorch_map to discover correct URLs on the site",
-        "Check if the URL has a typo or outdated path",
-        "Try scorch_search to find the current location of this content"
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["429", "rate limit", "too many requests", "throttl"])) {
-    return {
-      code: "RATE_LIMITED",
-      message: "Rate limited by the target site.",
-      suggestions: [
-        "Wait 30 seconds and retry",
-        "Try a different URL on the same site",
-        "Use scorch_search to find the content from a different source"
-      ],
-      originalError: raw
-    };
-  }
-  if (matchesStatusRange(lower, 500, 599)) {
-    return {
-      code: "SERVER_ERROR",
-      message: "The target site returned a server error.",
-      suggestions: [
-        "Retry once \u2014 the site may be experiencing temporary issues",
-        "Try again in a few minutes if it persists"
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["timeout", "aborterror", "aborted", "timed out", "etimedout"])) {
-    return {
-      code: "TIMEOUT",
-      message: "Page took too long to load.",
-      suggestions: [
-        "Add onlyMainContent: true to reduce processing time",
-        "Increase waitFor to give JS more time to render",
-        "Try with a simpler format like markdown instead of screenshot"
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["econnrefused", "enotfound", "dns", "getaddrinfo", "connect failed"])) {
-    if (matches(lower, ["localhost", "127.0.0.1", "0.0.0.0", "3002", "engine", "api"])) {
-      return {
-        code: "ENGINE_UNAVAILABLE",
-        message: "Scraping engine is not running.",
-        suggestions: [
-          "Check that Docker services are up: docker compose up -d",
-          "Verify SCORCHCRAWL_API_URL is correct in your config"
-        ],
-        originalError: raw
-      };
-    }
-    return {
-      code: "CONNECTION_FAILED",
-      message: "Cannot reach this URL.",
-      suggestions: [
-        "Check if the URL is correct and accessible",
-        "The site may be down \u2014 try again later",
-        "Try scorch_search to find alternative sources"
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["ssl", "tls", "certificate", "cert", "self-signed", "unable to verify"])) {
-    return {
-      code: "TLS_ERROR",
-      message: "SSL certificate issue with this site.",
-      suggestions: [
-        "Try with skipTlsVerification: true"
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["empty", "no content", "no markdown", "no extractable"])) {
-    return {
-      code: "EMPTY_CONTENT",
-      message: "Page returned no extractable content.",
-      suggestions: [
-        "Use scorch_map with a search param to find the right page",
-        "Add waitFor: 5000 for JavaScript-rendered pages",
-        'Try with formats: ["html"] to see the raw page structure'
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["spa_skeleton_detected", "spa detected", "javascript is required"])) {
-    return {
-      code: "SPA_DETECTED",
-      message: "Page requires JavaScript rendering (SPA detected).",
-      suggestions: [
-        "Add waitFor: 5000 to allow JavaScript to render",
-        "The engine will automatically retry with Playwright"
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["geo", "geoblocked", "not available in your", "region", "country restriction"])) {
-    return {
-      code: "GEO_BLOCKED",
-      message: "Content is geo-restricted.",
-      suggestions: [
-        'Use location parameter with a different country: location: { country: "US" }',
-        'Try proxy: "stealth" which may route through a different region'
-      ],
-      originalError: raw
-    };
-  }
-  if (matches(lower, ["paywall", "subscribe", "premium content", "members only"])) {
-    return {
-      code: "PAYWALL",
-      message: "Content is behind a paywall.",
-      suggestions: [
-        "Try scorch_search to find freely available alternatives",
-        "Use scorch_map to find non-paywalled pages on the same site"
-      ],
-      originalError: raw
-    };
-  }
-  return {
-    code: "UNKNOWN_ERROR",
-    message: `Scraping failed: ${truncateString(raw, 200)}`,
-    suggestions: [
-      "Try a different URL or format",
-      "Use scorch_search as an alternative data source"
-    ],
-    originalError: raw
-  };
-}
-function normalizeErrorString(err) {
-  if (err == null) return "Unknown error";
-  if (err instanceof Error) {
-    const axiosStatus = err?.response?.status;
-    const status = err?.status || err?.statusCode || axiosStatus;
-    const prefix = status ? `HTTP ${status}: ` : "";
-    return `${prefix}${err.message}`;
-  }
-  if (typeof err === "string") return err;
-  try {
-    const s = JSON.stringify(err);
-    return s || "Unknown error";
-  } catch {
-    return String(err);
-  }
-}
-function matches(lower, patterns) {
-  return patterns.some((p) => lower.includes(p));
-}
-function matchesStatusRange(lower, from, to) {
-  const statusMatch = lower.match(/(?:status\s*(?:code\s*)?|http\s*)(\d{3})/);
-  if (statusMatch) {
-    const code = parseInt(statusMatch[1], 10);
-    return code >= from && code <= to;
-  }
-  for (let s = from; s <= to; s++) {
-    if (lower.includes(String(s))) return true;
-  }
-  return false;
-}
-function truncateString(s, maxLen) {
-  if (s.length <= maxLen) return s;
-  return s.slice(0, maxLen - 3) + "...";
-}
-var CONTENT_FIELDS = ["markdown", "html", "rawHtml"];
-var SINGLE_PAGE_MAX_FRACTION = 0.3;
-function truncateAtBoundary(content, maxChars) {
-  if (content.length <= maxChars) {
-    return { text: content, wasTruncated: false };
-  }
-  const searchWindow = content.slice(0, maxChars);
-  const headingMatch = searchWindow.lastIndexOf("\n#");
-  const paraMatch = searchWindow.lastIndexOf("\n\n");
-  const lineMatch = searchWindow.lastIndexOf("\n");
-  const sentenceMatch = Math.max(
-    searchWindow.lastIndexOf(". "),
-    searchWindow.lastIndexOf(".\n")
-  );
-  let breakPoint = -1;
-  const minBreak = Math.floor(maxChars * 0.7);
-  if (headingMatch > minBreak) breakPoint = headingMatch;
-  else if (paraMatch > minBreak) breakPoint = paraMatch;
-  else if (lineMatch > minBreak) breakPoint = lineMatch;
-  else if (sentenceMatch > minBreak) breakPoint = sentenceMatch + 1;
-  else breakPoint = maxChars;
-  const truncated = content.slice(0, breakPoint).trimEnd();
-  return { text: truncated, wasTruncated: true };
-}
-function truncateContent(data) {
-  if (MAX_CONTENT_CHARS <= 0) {
-    const serialized2 = JSON.stringify(data, null, 2);
-    return { result: data, wasTruncated: false, originalLength: serialized2.length };
-  }
-  const serialized = JSON.stringify(data, null, 2);
-  if (serialized.length <= MAX_CONTENT_CHARS) {
-    return { result: data, wasTruncated: false, originalLength: serialized.length };
-  }
-  const cloned = JSON.parse(serialized);
-  let wasTruncated = false;
-  if (cloned && typeof cloned === "object") {
-    const target = cloned.data || cloned;
-    for (const field of CONTENT_FIELDS) {
-      if (typeof target[field] === "string" && target[field].length > 0) {
-        const otherFieldsSize = serialized.length - target[field].length;
-        const fieldBudget = Math.max(
-          MAX_CONTENT_CHARS - otherFieldsSize,
-          Math.floor(MAX_CONTENT_CHARS * 0.5)
-          // at least 50% of limit
-        );
-        if (target[field].length > fieldBudget) {
-          const { text } = truncateAtBoundary(target[field], fieldBudget);
-          const originalLen = target[field].length;
-          target[field] = text + buildTruncationNotice(text.length, originalLen);
-          wasTruncated = true;
-        }
-      }
-    }
-    if (Array.isArray(target)) {
-      const result = truncateCrawlArray(target);
-      return { result: cloned, wasTruncated: result.wasTruncated, originalLength: serialized.length };
-    }
-    if (wasTruncated) {
-      const meta = target.metadata || target;
-      meta._truncated = true;
-      meta._originalLength = serialized.length;
-    }
-  }
-  return { result: cloned, wasTruncated, originalLength: serialized.length };
-}
-function truncateCrawlArray(pages) {
-  if (pages.length === 0) return { wasTruncated: false };
-  const perPageLimit = Math.floor(MAX_CONTENT_CHARS * SINGLE_PAGE_MAX_FRACTION);
-  let wasTruncated = false;
-  let totalSize = 0;
-  for (let i = 0; i < pages.length; i++) {
-    const page = pages[i];
-    if (!page || typeof page !== "object") continue;
-    const target = page.data || page;
-    for (const field of CONTENT_FIELDS) {
-      if (typeof target[field] === "string" && target[field].length > perPageLimit) {
-        const { text } = truncateAtBoundary(target[field], perPageLimit);
-        target[field] = text + buildTruncationNotice(text.length, target[field].length);
-        wasTruncated = true;
-      }
-    }
-    totalSize += JSON.stringify(page).length;
-    if (totalSize > MAX_CONTENT_CHARS && i < pages.length - 1) {
-      const originalCount = pages.length;
-      pages.length = i + 1;
-      pages.push({
-        _notice: `Showing ${i + 1} of ${originalCount} crawled pages. Use scorch_check_crawl_status with pagination to see more.`
-      });
-      wasTruncated = true;
-      break;
-    }
-  }
-  return { wasTruncated };
-}
-function buildTruncationNotice(shownChars, originalChars) {
-  return `
-
----
-[Content truncated. Showing ~${Math.round(shownChars / 1e3)}k of ~${Math.round(originalChars / 1e3)}k characters.
-To get specific information, try:
-- Use JSON format with a schema to extract only the data you need
-- Add onlyMainContent: true to exclude navigation/footers
-- Use scorch_map to find a more specific page URL]`;
-}
-var summaryCache = /* @__PURE__ */ new Map();
-var CACHE_TTL_MS = 30 * 60 * 1e3;
-var summarizationTimestamps = [];
-function wordCount(text) {
-  return text.split(/\s+/).filter(Boolean).length;
-}
-function summaryKey(url, markdown) {
-  const fingerprint = url + markdown.slice(0, 1e3);
-  return createHash("sha256").update(fingerprint).digest("hex");
-}
-function canSummarize() {
-  if (SUMMARIZE_MAX_PER_MINUTE <= 0) return false;
-  const now = Date.now();
-  const windowStart = now - 6e4;
-  while (summarizationTimestamps.length > 0 && summarizationTimestamps[0] < windowStart) {
-    summarizationTimestamps.shift();
-  }
-  return summarizationTimestamps.length < SUMMARIZE_MAX_PER_MINUTE;
-}
-function recordSummarization() {
-  summarizationTimestamps.push(Date.now());
-}
-function evictCache() {
-  if (summaryCache.size <= SUMMARIZE_CACHE_SIZE) return;
-  const entries = [...summaryCache.entries()].sort(
-    (a, b) => a[1].timestamp - b[1].timestamp
-  );
-  const toDelete = entries.slice(0, entries.length - SUMMARIZE_CACHE_SIZE);
-  for (const [key] of toDelete) {
-    summaryCache.delete(key);
-  }
-}
-async function summarizeIfNeeded(markdown, url, getCopilotClientFn, copilotToken) {
-  if (SUMMARIZE_AFTER_WORDS <= 0) {
-    return { markdown, summarized: false };
-  }
-  const words = wordCount(markdown);
-  if (words <= SUMMARIZE_AFTER_WORDS) {
-    return { markdown, summarized: false };
-  }
-  const key = summaryKey(url, markdown);
-  const cached = summaryCache.get(key);
-  if (cached && Date.now() - cached.timestamp < CACHE_TTL_MS) {
-    return {
-      markdown: cached.summary,
-      summarized: true,
-      meta: {
-        _summarized: true,
-        _originalWordCount: words,
-        _summaryWordCount: cached.wordCount,
-        _summarizedBy: SUMMARIZE_MODEL,
-        _cachedResult: true
-      }
-    };
-  }
-  if (!getCopilotClientFn) {
-    return { markdown, summarized: false };
-  }
-  if (!canSummarize()) {
-    return { markdown, summarized: false };
-  }
-  try {
-    const client = await getCopilotClientFn(copilotToken);
-    if (!client) {
-      return { markdown, summarized: false };
-    }
-    const targetWords = Math.min(SUMMARIZE_AFTER_WORDS, Math.floor(words * 0.3));
-    const sessionConfig = {
-      model: SUMMARIZE_MODEL,
-      systemMessage: {
-        mode: "replace",
-        content: buildSummarizationPrompt(words, targetWords)
-      }
-    };
-    const session = await client.createSession(sessionConfig);
-    recordSummarization();
-    try {
-      const response = await session.sendAndWait({ prompt: markdown }, SUMMARIZE_TIMEOUT_MS);
-      let summary = "";
-      if (response && typeof response === "object") {
-        const r = response;
-        if (r.data && typeof r.data === "object") {
-          const d = r.data;
-          if (typeof d.content === "string") summary = d.content;
-          else if (typeof d.text === "string") summary = d.text;
-        }
-        if (!summary && typeof r.content === "string") summary = r.content;
-        if (!summary && typeof r.text === "string") summary = r.text;
-        if (!summary && r.message && typeof r.message === "object") {
-          const m = r.message;
-          if (typeof m.content === "string") summary = m.content;
-        }
-      } else if (typeof response === "string") {
-        summary = response;
-      }
-      if (!summary || summary.length < 50) {
-        console.error(`[Summarization] Response too short or empty (${summary.length} chars), response keys: ${response ? Object.keys(response).join(", ") : "null"}`);
-        return { markdown, summarized: false };
-      }
-      const summaryWords = wordCount(summary);
-      summaryCache.set(key, {
-        summary,
-        timestamp: Date.now(),
-        wordCount: summaryWords
-      });
-      evictCache();
-      return {
-        markdown: summary,
-        summarized: true,
-        meta: {
-          _summarized: true,
-          _originalWordCount: words,
-          _summaryWordCount: summaryWords,
-          _summarizedBy: SUMMARIZE_MODEL
-        }
-      };
-    } finally {
-      try {
-        await session.destroy();
-      } catch {
-      }
-    }
-  } catch (err) {
-    const errMsg = err instanceof Error ? err.message : String(err);
-    const errStack = err instanceof Error ? err.stack : "";
-    console.error(`[Summarization] Failed for ${url}: ${errMsg}`);
-    if (errStack) console.error(`[Summarization] Stack: ${errStack}`);
-    return {
-      markdown,
-      summarized: false,
-      meta: { _summarizationFailed: true, _summarizationError: errMsg }
-    };
-  }
-}
-function buildSummarizationPrompt(originalWords, targetWords) {
-  return `You are a content summarizer for a web scraping tool. 
-Summarize the following web page content, preserving:
-- ALL factual data (numbers, dates, names, prices, specifications)
-- Key arguments, conclusions, and findings
-- Structural organization (use headings, lists)
-- Any code snippets or technical details
-
-Do NOT:
-- Add opinions or analysis
-- Omit specific data points (numbers, names, URLs)
-- Change the meaning or emphasis of the content
-- Add any preamble like "Here is a summary" \u2014 just output the summary directly
-
-Original content word count: ${originalWords}
-Target summary: ~${targetWords} words`;
-}
-async function safeExecute(fn, context) {
-  try {
-    return await fn();
-  } catch (err) {
-    const mapped = mapError(err);
-    console.error(`[${context.tool}] ${mapped.code}`, {
-      url: context.url,
-      original: mapped.originalError
-    });
-    return JSON.stringify(
-      {
-        success: false,
-        error: mapped.message,
-        code: mapped.code,
-        suggestions: mapped.suggestions
-      },
-      null,
-      2
-    );
-  }
-}
-async function processResponse(data, options) {
-  if (!options?.skipSummarization && options?.getCopilotClientFn) {
-    const md = extractMarkdown(data);
-    const wc = md ? wordCount(md) : 0;
-    console.error(`[Summarization] Check: markdown=${md ? md.length : 0} chars, ${wc} words, threshold=${SUMMARIZE_AFTER_WORDS}`);
-    if (md && wc > SUMMARIZE_AFTER_WORDS && SUMMARIZE_AFTER_WORDS > 0) {
-      console.error(`[Summarization] Attempting summarization for ${options.url || "unknown url"} (${wc} words)`);
-      const result2 = await summarizeIfNeeded(
-        md,
-        options.url || "",
-        options.getCopilotClientFn,
-        options.copilotToken
-      );
-      if (result2.summarized) {
-        const updated = injectMarkdown(data, result2.markdown, result2.meta || {});
-        return JSON.stringify(updated, null, 2);
-      }
-      if (result2.meta) {
-        console.error(`[Summarization] Failed, falling back to truncation. Meta:`, JSON.stringify(result2.meta));
-      }
-    }
-  } else {
-    console.error(`[Summarization] Skipped: skipSummarization=${options?.skipSummarization}, hasCopilotFn=${!!options?.getCopilotClientFn}`);
-  }
-  const { result, wasTruncated } = truncateContent(data);
-  return JSON.stringify(result, null, 2);
-}
-function processResponseSync(data) {
-  const { result } = truncateContent(data);
-  return JSON.stringify(result, null, 2);
-}
-function extractMarkdown(data) {
-  if (!data || typeof data !== "object") return null;
-  const obj = data;
-  if (typeof obj.markdown === "string") return obj.markdown;
-  if (obj.data && typeof obj.data === "object") {
-    const inner = obj.data;
-    if (typeof inner.markdown === "string") return inner.markdown;
-  }
-  return null;
-}
-function injectMarkdown(data, newMarkdown, meta) {
-  const serialized = JSON.stringify(data);
-  const cloned = JSON.parse(serialized);
-  if (typeof cloned.markdown === "string") {
-    cloned.markdown = newMarkdown;
-    Object.assign(cloned.metadata || cloned, meta);
-    return cloned;
-  }
-  if (cloned.data && typeof cloned.data === "object") {
-    if (typeof cloned.data.markdown === "string") {
-      cloned.data.markdown = newMarkdown;
-      Object.assign(cloned.data.metadata || cloned.data, meta);
-      return cloned;
-    }
-  }
-  return cloned;
-}
-var ANTI_BOT_SIGNATURES = [
-  // Cloudflare
-  "checking your browser",
-  "cf-browser-verification",
-  "just a moment",
-  "cf-challenge-running",
-  "cf_chl_opt",
-  "ray id:",
-  // DataDome
-  "datadome",
-  "dd.js",
-  // PerimeterX
-  "perimeterx",
-  "px-captcha",
-  "human challenge",
-  // Incapsula / Imperva
-  "incapsula",
-  "imperva",
-  "_incapsula_resource",
-  // Generic
-  "access denied",
-  "bot detected",
-  "automated access",
-  "please verify you are a human",
-  "captcha",
-  // Akamai
-  "akamai",
-  "ak_bmsc",
-  // Shape Security
-  "shape security"
-];
-function scoreResponseQuality(data) {
-  const issues = [];
-  let score = 100;
-  let antiBotDetected = false;
-  let antiBotSystem;
-  if (!data || typeof data !== "object") {
-    return { score: 0, antiBotDetected: false, issues: ["No response data"] };
-  }
-  const obj = data;
-  const inner = obj.data && typeof obj.data === "object" ? obj.data : obj;
-  if (obj.success === false) {
-    score -= 50;
-    issues.push("Response indicates failure");
-  }
-  const md = typeof inner.markdown === "string" ? inner.markdown : "";
-  const html = typeof inner.html === "string" ? inner.html : typeof inner.rawHtml === "string" ? inner.rawHtml : "";
-  const content = md || html;
-  const lower = content.toLowerCase();
-  for (const sig of ANTI_BOT_SIGNATURES) {
-    if (lower.includes(sig)) {
-      antiBotDetected = true;
-      if (sig.includes("cf-") || sig.includes("cloudflare") || sig === "just a moment" || sig === "checking your browser" || sig === "ray id:") {
-        antiBotSystem = "Cloudflare";
-      } else if (sig.includes("datadome")) {
-        antiBotSystem = "DataDome";
-      } else if (sig.includes("perimeterx") || sig.includes("px-captcha")) {
-        antiBotSystem = "PerimeterX";
-      } else if (sig.includes("incapsula") || sig.includes("imperva")) {
-        antiBotSystem = "Imperva/Incapsula";
-      } else if (sig.includes("akamai") || sig.includes("ak_bmsc")) {
-        antiBotSystem = "Akamai";
-      } else if (sig.includes("captcha")) {
-        antiBotSystem = "CAPTCHA";
-      } else {
-        antiBotSystem = "Unknown";
-      }
-      score -= 40;
-      issues.push(`Anti-bot protection detected: ${antiBotSystem}`);
-      break;
-    }
-  }
-  if (content.length === 0) {
-    score -= 30;
-    issues.push("No content extracted");
-  } else if (content.length < 100) {
-    score -= 20;
-    issues.push("Very minimal content (< 100 chars)");
-  } else if (content.length < 500) {
-    score -= 10;
-    issues.push("Thin content (< 500 chars)");
-  }
-  const navPatterns = ["sign in", "log in", "menu", "navigation", "footer", "header"];
-  const navCount = navPatterns.filter((p) => lower.includes(p)).length;
-  const contentWords = content.split(/\s+/).length;
-  if (contentWords < 50 && navCount >= 3) {
-    score -= 20;
-    issues.push("Content appears to be navigation-only (possible SPA shell)");
-  }
-  const errorPagePatterns = ["page not found", "404", "error occurred", "something went wrong", "under maintenance"];
-  if (errorPagePatterns.some((p) => lower.includes(p)) && contentWords < 200) {
-    score -= 15;
-    issues.push("Content appears to be an error page");
-  }
-  return {
-    score: Math.max(0, Math.min(100, score)),
-    antiBotDetected,
-    ...antiBotSystem && { antiBotSystem },
-    issues
-  };
-}
-
-// src/scrape-cache.ts
-import { createHash as createHash2 } from "node:crypto";
-var SCRAPE_CACHE_TTL_MS = parseInt(
-  process.env.SCORCHCRAWL_SCRAPE_CACHE_TTL_MS || "120000",
-  10
-);
-var SCRAPE_CACHE_MAX = parseInt(
-  process.env.SCORCHCRAWL_SCRAPE_CACHE_MAX || "50",
-  10
-);
-var scrapeCache = /* @__PURE__ */ new Map();
-function scrapeCacheKey(url, options) {
-  const fingerprint = JSON.stringify({ url, ...options });
-  return createHash2("sha256").update(fingerprint).digest("hex");
-}
-function getCachedScrape(key) {
-  const entry = scrapeCache.get(key);
-  if (!entry) return null;
-  if (Date.now() - entry.timestamp > SCRAPE_CACHE_TTL_MS) {
-    scrapeCache.delete(key);
-    return null;
-  }
-  return entry.response;
-}
-function setCachedScrape(key, response) {
-  scrapeCache.set(key, { response, timestamp: Date.now() });
-  if (scrapeCache.size > SCRAPE_CACHE_MAX) {
-    const oldest = scrapeCache.keys().next().value;
-    if (oldest) scrapeCache.delete(oldest);
-  }
-}
-
-// src/index.ts
-dotenv.config({ debug: false, quiet: true });
-function extractApiKey(headers) {
-  const headerAuth = headers["authorization"];
-  const headerApiKey = headers["x-scorchcrawl-api-key"] || headers["x-api-key"];
-  if (headerApiKey) {
-    return Array.isArray(headerApiKey) ? headerApiKey[0] : headerApiKey;
-  }
-  if (typeof headerAuth === "string" && headerAuth.toLowerCase().startsWith("bearer ")) {
-    return headerAuth.slice(7).trim();
-  }
-  return void 0;
-}
-function extractCopilotToken(headers) {
-  const copilotHeader = headers["x-copilot-token"] || headers["x-github-token"];
-  if (copilotHeader) {
-    return Array.isArray(copilotHeader) ? copilotHeader[0] : copilotHeader;
-  }
-  return process.env.GITHUB_TOKEN || void 0;
-}
-function removeEmptyTopLevel(obj) {
-  const out = {};
-  for (const [k, v] of Object.entries(obj)) {
-    if (v == null) continue;
-    if (typeof v === "string" && v.trim() === "") continue;
-    if (Array.isArray(v) && v.length === 0) continue;
-    if (typeof v === "object" && !Array.isArray(v) && Object.keys(v).length === 0)
-      continue;
-    out[k] = v;
-  }
-  return out;
-}
-var ConsoleLogger = class {
-  shouldLog = process.env.CLOUD_SERVICE === "true" || process.env.SSE_LOCAL === "true" || process.env.HTTP_STREAMABLE_SERVER === "true";
-  debug(...args2) {
-    if (this.shouldLog) {
-      console.debug("[DEBUG]", (/* @__PURE__ */ new Date()).toISOString(), ...args2);
-    }
-  }
-  error(...args2) {
-    if (this.shouldLog) {
-      console.error("[ERROR]", (/* @__PURE__ */ new Date()).toISOString(), ...args2);
-    }
-  }
-  info(...args2) {
-    if (this.shouldLog) {
-      console.log("[INFO]", (/* @__PURE__ */ new Date()).toISOString(), ...args2);
-    }
-  }
-  log(...args2) {
-    if (this.shouldLog) {
-      console.log("[LOG]", (/* @__PURE__ */ new Date()).toISOString(), ...args2);
-    }
-  }
-  warn(...args2) {
-    if (this.shouldLog) {
-      console.warn("[WARN]", (/* @__PURE__ */ new Date()).toISOString(), ...args2);
-    }
-  }
-};
-var server = new FastMCP({
-  name: "scorchcrawl",
-  version: "1.0.0",
-  logger: new ConsoleLogger(),
-  roots: { enabled: false },
-  authenticate: async (request) => {
-    const copilotToken = extractCopilotToken(request.headers);
-    if (process.env.CLOUD_SERVICE === "true") {
-      const apiKey = extractApiKey(request.headers);
-      if (!apiKey) {
-        throw new Error("API key is required");
-      }
-      return { scraperApiKey: apiKey, copilotToken };
-    } else {
-      if (!process.env.SCORCHCRAWL_API_KEY && !process.env.SCORCHCRAWL_API_URL) {
-        console.error(
-          "Either SCORCHCRAWL_API_KEY or SCORCHCRAWL_API_URL must be provided"
-        );
-        process.exit(1);
-      }
-      return { scraperApiKey: process.env.SCORCHCRAWL_API_KEY, copilotToken };
-    }
-  },
-  // Lightweight health endpoint for LB checks
-  health: {
-    enabled: true,
-    message: "ok",
-    path: "/health",
-    status: 200
-  }
-});
-function createClient(apiKey) {
-  const cleanUrl = getCleanApiUrl();
-  const config = {
-    ...cleanUrl && {
-      apiUrl: cleanUrl
-    }
-  };
-  if (apiKey) {
-    config.apiKey = apiKey;
-  }
-  return new scorch_client_default(config);
-}
-var ORIGIN = "scorchcrawl-mcp";
-var SAFE_MODE = process.env.CLOUD_SERVICE === "true";
-function getClient(session) {
-  if (process.env.CLOUD_SERVICE === "true") {
-    if (!session || !session.scraperApiKey) {
-      throw new Error("Unauthorized");
-    }
-    return createClient(session.scraperApiKey);
-  }
-  if (!process.env.SCORCHCRAWL_API_URL && (!session || !session.scraperApiKey)) {
-    throw new Error(
-      "Unauthorized: API key is required when not using a self-hosted instance"
-    );
-  }
-  return createClient(session?.scraperApiKey);
-}
-function asText(data) {
-  return processResponseSync(data);
-}
-var safeActionTypes = ["wait", "screenshot", "scroll", "scrape"];
-var otherActions = [
-  "click",
-  "write",
-  "press",
-  "executeJavascript",
-  "generatePDF"
-];
-var allActionTypes = [...safeActionTypes, ...otherActions];
-var allowedActionTypes = SAFE_MODE ? safeActionTypes : allActionTypes;
-var scrapeParamsSchema = z.object({
-  url: z.string().url(),
-  formats: z.array(
-    z.union([
-      z.enum([
-        "markdown",
-        "html",
-        "rawHtml",
-        "screenshot",
-        "links",
-        "summary",
-        "changeTracking",
-        "branding"
-      ]),
-      z.object({
-        type: z.literal("json"),
-        prompt: z.string().optional(),
-        schema: z.record(z.string(), z.any()).optional()
-      }),
-      z.object({
-        type: z.literal("screenshot"),
-        fullPage: z.boolean().optional(),
-        quality: z.number().optional(),
-        viewport: z.object({ width: z.number(), height: z.number() }).optional()
-      })
-    ])
-  ).optional(),
-  parsers: z.array(
-    z.union([
-      z.enum(["pdf"]),
-      z.object({
-        type: z.enum(["pdf"]),
-        maxPages: z.number().int().min(1).max(1e4).optional()
-      })
-    ])
-  ).optional(),
-  onlyMainContent: z.boolean().optional(),
-  includeTags: z.array(z.string()).optional(),
-  excludeTags: z.array(z.string()).optional(),
-  waitFor: z.number().optional(),
-  ...SAFE_MODE ? {} : {
-    actions: z.array(
-      z.object({
-        type: z.enum(allowedActionTypes),
-        selector: z.string().optional(),
-        milliseconds: z.number().optional(),
-        text: z.string().optional(),
-        key: z.string().optional(),
-        direction: z.enum(["up", "down"]).optional(),
-        script: z.string().optional(),
-        fullPage: z.boolean().optional()
-      })
-    ).optional()
-  },
-  mobile: z.boolean().optional(),
-  skipTlsVerification: z.boolean().optional(),
-  removeBase64Images: z.boolean().optional(),
-  location: z.object({
-    country: z.string().optional(),
-    languages: z.array(z.string()).optional()
-  }).optional(),
-  storeInCache: z.boolean().optional(),
-  zeroDataRetention: z.boolean().optional(),
-  maxAge: z.number().optional(),
-  proxy: z.enum(["basic", "stealth", "enhanced", "auto"]).optional()
-});
-server.addTool({
-  name: "scorch_scrape",
-  annotations: {
-    title: "Scrape Web Page",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: true,
-    openWorldHint: true
-  },
-  description: `
-Scrape content from a single URL with advanced options.
-This is the most powerful, fastest and most reliable scraper tool, if available you should always default to using this tool for any web scraping needs.
-
-**Best for:** Single page content extraction, when you know exactly which page contains the information.
-**Not recommended for:** Multiple pages (call scrape multiple times or use crawl), unknown page location (use search).
-**Common mistakes:** Using markdown format when extracting specific data points (use JSON instead).
-**Other Features:** Use 'branding' format to extract brand identity (colors, fonts, typography, spacing, UI components) for design analysis or style replication.
-
-**CRITICAL - Format Selection (you MUST follow this):**
-When the user asks for SPECIFIC data points, you MUST use JSON format with a schema. Only use markdown when the user needs the ENTIRE page content.
-
-**Use JSON format when user asks for:**
-- Parameters, fields, or specifications (e.g., "get the header parameters", "what are the required fields")
-- Prices, numbers, or structured data (e.g., "extract the pricing", "get the product details")
-- API details, endpoints, or technical specs (e.g., "find the authentication endpoint")
-- Lists of items or properties (e.g., "list the features", "get all the options")
-- Any specific piece of information from a page
-
-**Use markdown format ONLY when:**
-- User wants to read/summarize an entire article or blog post
-- User needs to see all content on a page without specific extraction
-- User explicitly asks for the full page content
-
-**Handling JavaScript-rendered pages (SPAs):**
-If JSON extraction returns empty, minimal, or just navigation content, the page is likely JavaScript-rendered or the content is on a different URL. Try these steps IN ORDER:
-1. **Add waitFor parameter:** Set \`waitFor: 5000\` to \`waitFor: 10000\` to allow JavaScript to render before extraction
-2. **Try a different URL:** If the URL has a hash fragment (#section), try the base URL or look for a direct page URL
-3. **Use scorch_map to find the correct page:** Large documentation sites or SPAs often spread content across multiple URLs. Use \`scorch_map\` with a \`search\` parameter to discover the specific page containing your target content, then scrape that URL directly.
-   Example: If scraping "https://docs.example.com/reference" fails to find webhook parameters, use \`scorch_map\` with \`{"url": "https://docs.example.com/reference", "search": "webhook"}\` to find URLs like "/reference/webhook-events", then scrape that specific page.
-4. **Use scorch_agent:** As a last resort for heavily dynamic pages where map+scrape still fails, use the agent which can autonomously navigate and research
-
-**Usage Example (JSON format - REQUIRED for specific data extraction):**
-\`\`\`json
-{
-  "name": "scorch_scrape",
-  "arguments": {
-    "url": "https://example.com/api-docs",
-    "formats": [{
-      "type": "json",
-      "prompt": "Extract the header parameters for the authentication endpoint",
-      "schema": {
-        "type": "object",
-        "properties": {
-          "parameters": {
-            "type": "array",
-            "items": {
-              "type": "object",
-              "properties": {
-                "name": { "type": "string" },
-                "type": { "type": "string" },
-                "required": { "type": "boolean" },
-                "description": { "type": "string" }
-              }
-            }
-          }
-        }
-      }
-    }]
-  }
-}
-\`\`\`
-**Usage Example (markdown format - ONLY when full content genuinely needed):**
-\`\`\`json
-{
-  "name": "scorch_scrape",
-  "arguments": {
-    "url": "https://example.com/article",
-    "formats": ["markdown"],
-    "onlyMainContent": true
-  }
-}
-\`\`\`
-**Usage Example (branding format - extract brand identity):**
-\`\`\`json
-{
-  "name": "scorch_scrape",
-  "arguments": {
-    "url": "https://example.com",
-    "formats": ["branding"]
-  }
-}
-\`\`\`
-**Branding format:** Extracts comprehensive brand identity (colors, fonts, typography, spacing, logo, UI components) for design analysis or style replication.
-**Performance:** Add maxAge parameter for 500% faster scrapes using cached data.
-**Returns:** JSON structured data, markdown, branding profile, or other formats as specified.
-${SAFE_MODE ? "**Safe Mode:** Read-only content extraction. Interactive actions (click, write, executeJavascript) are disabled for security." : ""}
-`,
-  parameters: scrapeParamsSchema,
-  execute: async (args2, { session, log }) => {
-    const { url, ...options } = args2;
-    const cleaned = removeEmptyTopLevel(options);
-    return safeExecute(async () => {
-      const cacheKey = scrapeCacheKey(String(url), cleaned);
-      const cached = getCachedScrape(cacheKey);
-      if (cached) {
-        log.info("Returning cached scrape result", { url: String(url) });
-        return cached;
-      }
-      if (isLocalProxyEnabled()) {
-        log.info("Scraping URL via LOCAL PROXY (user IP)", { url: String(url) });
-        const localResult = await localScrape(String(url), cleaned);
-        if (!localResult.success && localResult.error === "FORMAT_NEEDS_SERVER") {
-          log.info("Format requires server-side processing, falling back to API", { url: String(url) });
-          const client2 = getClient(session);
-          const res2 = await client2.scrape(String(url), {
-            ...cleaned,
-            origin: ORIGIN
-          });
-          return await processResponse(res2, {
-            url: String(url),
-            getCopilotClientFn: getCopilotClient,
-            copilotToken: session?.copilotToken
-          });
-        }
-        if (!localResult.success && localResult.error === "SPA_SKELETON_DETECTED") {
-          const waitMs = cleaned.waitFor || 5e3;
-          log.info(
-            `SPA skeleton detected, retrying via engine with waitFor=${waitMs}ms`,
-            { url: String(url) }
-          );
-          const client2 = getClient(session);
-          const res2 = await client2.scrape(String(url), {
-            ...cleaned,
-            waitFor: waitMs,
-            origin: ORIGIN
-          });
-          return await processResponse(res2, {
-            url: String(url),
-            getCopilotClientFn: getCopilotClient,
-            copilotToken: session?.copilotToken
-          });
-        }
-        if (!localResult.success && localResult.error) {
-          const mapped = mapError(localResult.error);
-          return JSON.stringify({
-            success: false,
-            error: mapped.message,
-            code: mapped.code,
-            suggestions: mapped.suggestions
-          }, null, 2);
-        }
-        return await processResponse(localResult, {
-          url: String(url),
-          getCopilotClientFn: getCopilotClient,
-          copilotToken: session?.copilotToken
-        });
-      }
-      const client = getClient(session);
-      log.info("Scraping URL", { url: String(url) });
-      const res = await client.scrape(String(url), {
-        ...cleaned,
-        origin: ORIGIN
-      });
-      const quality = scoreResponseQuality(res);
-      if (quality.score < 30 && !cleaned.waitFor) {
-        log.info("Low quality response detected, retrying with waitFor=5000", {
-          url: String(url),
-          score: quality.score,
-          issues: quality.issues
-        });
-        const retryRes = await client.scrape(String(url), {
-          ...cleaned,
-          waitFor: 5e3,
-          origin: ORIGIN
-        });
-        const retryQuality = scoreResponseQuality(retryRes);
-        const bestRes = retryQuality.score >= quality.score ? retryRes : res;
-        const bestQuality = retryQuality.score >= quality.score ? retryQuality : quality;
-        const result2 = await processResponse(bestRes, {
-          url: String(url),
-          getCopilotClientFn: getCopilotClient,
-          copilotToken: session?.copilotToken
-        });
-        const parsed2 = JSON.parse(result2);
-        parsed2._quality = bestQuality;
-        const enriched2 = JSON.stringify(parsed2, null, 2);
-        setCachedScrape(cacheKey, enriched2);
-        return enriched2;
-      }
-      const result = await processResponse(res, {
-        url: String(url),
-        getCopilotClientFn: getCopilotClient,
-        copilotToken: session?.copilotToken
-      });
-      const parsed = JSON.parse(result);
-      parsed._quality = quality;
-      const enriched = JSON.stringify(parsed, null, 2);
-      setCachedScrape(cacheKey, enriched);
-      return enriched;
-    }, { tool: "scorch_scrape", url: String(url) });
-  }
-});
-server.addTool({
-  name: "scorch_batch_scrape",
-  annotations: {
-    title: "Batch Scrape URLs",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: true,
-    openWorldHint: true
-  },
-  description: `
-Scrape multiple URLs in a single request. Much faster than calling scorch_scrape multiple times.
-
-**Best for:** Extracting content from 2-50 known URLs in one batch.
-**Not recommended for:** Single URL (use scorch_scrape); unknown URLs (use scorch_search or scorch_map first).
-
-**Usage Example:**
-\`\`\`json
-{
-  "name": "scorch_batch_scrape",
-  "arguments": {
-    "urls": ["https://example.com/page1", "https://example.com/page2"],
-    "formats": ["markdown"],
-    "onlyMainContent": true
-  }
-}
-\`\`\`
-**Returns:** Results for each URL with individual success/failure status.
-`,
-  parameters: z.object({
-    urls: z.array(z.string().url()).min(1).max(50),
-    formats: scrapeParamsSchema.shape.formats.optional(),
-    onlyMainContent: z.boolean().optional(),
-    includeTags: z.array(z.string()).optional(),
-    excludeTags: z.array(z.string()).optional(),
-    waitFor: z.number().optional(),
-    mobile: z.boolean().optional(),
-    skipTlsVerification: z.boolean().optional()
-  }),
-  execute: async (args2, { session, log }) => {
-    const { urls, ...options } = args2;
-    const cleaned = removeEmptyTopLevel(options);
-    return safeExecute(async () => {
-      const client = getClient(session);
-      log.info("Batch scraping URLs", { count: urls.length });
-      const res = await client.batchScrape(urls, {
-        ...cleaned,
-        origin: ORIGIN
-      });
-      return await processResponse(res, {
-        skipSummarization: true
-      });
-    }, { tool: "scorch_batch_scrape" });
-  }
-});
-server.addTool({
-  name: "scorch_map",
-  annotations: {
-    title: "Map Website URLs",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: true,
-    openWorldHint: true
-  },
-  description: `
-Map a website to discover all indexed URLs on the site.
-
-**Best for:** Discovering URLs on a website before deciding what to scrape; finding specific sections or pages within a large site; locating the correct page when scrape returns empty or incomplete results.
-**Not recommended for:** When you already know which specific URL you need (use scrape); when you need the content of the pages (use scrape after mapping).
-**Common mistakes:** Using crawl to discover URLs instead of map; jumping straight to scorch_agent when scrape fails instead of using map first to find the right page.
-
-**IMPORTANT - Use map before agent:** If \`scorch_scrape\` returns empty, minimal, or irrelevant content, use \`scorch_map\` with the \`search\` parameter to find the specific page URL containing your target content. This is faster and cheaper than using \`scorch_agent\`. Only use the agent as a last resort after map+scrape fails.
-
-**Prompt Example:** "Find the webhook documentation page on this API docs site."
-**Usage Example (discover all URLs):**
-\`\`\`json
-{
-  "name": "scorch_map",
-  "arguments": {
-    "url": "https://example.com"
-  }
-}
-\`\`\`
-**Usage Example (search for specific content - RECOMMENDED when scrape fails):**
-\`\`\`json
-{
-  "name": "scorch_map",
-  "arguments": {
-    "url": "https://docs.example.com/api",
-    "search": "webhook events"
-  }
-}
-\`\`\`
-**Returns:** Array of URLs found on the site, filtered by search query if provided.
-`,
-  parameters: z.object({
-    url: z.string().url(),
-    search: z.string().optional(),
-    sitemap: z.enum(["include", "skip", "only"]).optional(),
-    includeSubdomains: z.boolean().optional(),
-    limit: z.number().optional(),
-    ignoreQueryParameters: z.boolean().optional()
-  }),
-  execute: async (args2, { session, log }) => {
-    const { url, ...options } = args2;
-    return safeExecute(async () => {
-      const client = getClient(session);
-      const cleaned = removeEmptyTopLevel(options);
-      log.info("Mapping URL", { url: String(url) });
-      const res = await client.map(String(url), {
-        ...cleaned,
-        origin: ORIGIN
-      });
-      return asText(res);
-    }, { tool: "scorch_map", url: String(url) });
-  }
-});
-server.addTool({
-  name: "scorch_search",
-  annotations: {
-    title: "Web Search",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: false,
-    openWorldHint: true
-  },
-  description: `
-Search the web and optionally extract content from search results. This is the most powerful web search tool available, and if available you should always default to using this tool for any web search needs.
-
-The query also supports search operators, that you can use if needed to refine the search:
-| Operator | Functionality | Examples |
----|-|-|
-| \`""\` | Non-fuzzy matches a string of text | \`"ScorchCrawl"\`
-| \`-\` | Excludes certain keywords or negates other operators | \`-bad\`, \`-site:example.com\`
-| \`site:\` | Only returns results from a specified website | \`site:example.com\`
-| \`inurl:\` | Only returns results that include a word in the URL | \`inurl:example\`
-| \`allinurl:\` | Only returns results that include multiple words in the URL | \`allinurl:git example\`
-| \`intitle:\` | Only returns results that include a word in the title of the page | \`intitle:ScorchCrawl\`
-| \`allintitle:\` | Only returns results that include multiple words in the title of the page | \`allintitle:example playground\`
-| \`related:\` | Only returns results that are related to a specific domain | \`related:example.com\`
-| \`imagesize:\` | Only returns images with exact dimensions | \`imagesize:1920x1080\`
-| \`larger:\` | Only returns images larger than specified dimensions | \`larger:1920x1080\`
-
-**Best for:** Finding specific information across multiple websites, when you don't know which website has the information; when you need the most relevant content for a query.
-**Not recommended for:** When you need to search the filesystem. When you already know which website to scrape (use scrape); when you need comprehensive coverage of a single website (use map or crawl.
-**Common mistakes:** Using crawl or map for open-ended questions (use search instead).
-**Prompt Example:** "Find the latest research papers on AI published in 2023."
-**Sources:** web, images, news, default to web unless needed images or news.
-**Scrape Options:** Only use scrapeOptions when you think it is absolutely necessary. When you do so default to a lower limit to avoid timeouts, 5 or lower.
-**Optimal Workflow:** Search first using scorch_search without formats, then after fetching the results, use the scrape tool to get the content of the relevantpage(s) that you want to scrape
-
-**Usage Example without formats (Preferred):**
-\`\`\`json
-{
-  "name": "scorch_search",
-  "arguments": {
-    "query": "top AI companies",
-    "limit": 5,
-    "sources": [
-      { "type": "web" }
-    ]
-  }
-}
-\`\`\`
-**Usage Example with formats:**
-\`\`\`json
-{
-  "name": "scorch_search",
-  "arguments": {
-    "query": "latest AI research papers 2023",
-    "limit": 5,
-    "lang": "en",
-    "country": "us",
-    "sources": [
-      { "type": "web" },
-      { "type": "images" },
-      { "type": "news" }
-    ],
-    "scrapeOptions": {
-      "formats": ["markdown"],
-      "onlyMainContent": true
-    }
-  }
-}
-\`\`\`
-**Returns:** Array of search results (with optional scraped content).
-`,
-  parameters: z.object({
-    query: z.string().min(1),
-    limit: z.number().optional(),
-    tbs: z.string().optional(),
-    filter: z.string().optional(),
-    location: z.string().optional(),
-    sources: z.array(z.object({ type: z.enum(["web", "images", "news"]) })).optional(),
-    scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional(),
-    enterprise: z.array(z.enum(["default", "anon", "zdr"])).optional()
-  }),
-  execute: async (args2, { session, log }) => {
-    const { query, ...opts } = args2;
-    return safeExecute(async () => {
-      const client = getClient(session);
-      const cleaned = removeEmptyTopLevel(opts);
-      log.info("Searching", { query: String(query) });
-      const res = await client.search(query, {
-        ...cleaned,
-        origin: ORIGIN
-      });
-      return await processResponse(res, {
-        skipSummarization: true
-      });
-    }, { tool: "scorch_search" });
-  }
-});
-server.addTool({
-  name: "scorch_crawl",
-  annotations: {
-    title: "Crawl Website",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: false,
-    openWorldHint: true
-  },
-  description: `
- Starts a crawl job on a website and extracts content from all pages.
- 
- **Best for:** Extracting content from multiple related pages, when you need comprehensive coverage.
- **Not recommended for:** Extracting content from a single page (use scrape); when token limits are a concern (use map + batch_scrape); when you need fast results (crawling can be slow).
- **Warning:** Crawl responses can be very large and may exceed token limits. Limit the crawl depth and number of pages, or use map + batch_scrape for better control.
- **Common mistakes:** Setting limit or maxDiscoveryDepth too high (causes token overflow) or too low (causes missing pages); using crawl for a single page (use scrape instead). Using a /* wildcard is not recommended.
- **Prompt Example:** "Get all blog posts from the first two levels of example.com/blog."
- **Usage Example:**
- \`\`\`json
- {
-   "name": "scorch_crawl",
-   "arguments": {
-     "url": "https://example.com/blog/*",
-     "maxDiscoveryDepth": 5,
-     "limit": 20,
-     "allowExternalLinks": false,
-     "deduplicateSimilarURLs": true,
-     "sitemap": "include"
-   }
- }
- \`\`\`
- **Returns:** Operation ID for status checking; use scorch_check_crawl_status to check progress.
- ${SAFE_MODE ? "**Safe Mode:** Read-only crawling. Webhooks and interactive actions are disabled for security." : ""}
- `,
-  parameters: z.object({
-    url: z.string(),
-    prompt: z.string().optional(),
-    excludePaths: z.array(z.string()).optional(),
-    includePaths: z.array(z.string()).optional(),
-    maxDiscoveryDepth: z.number().optional(),
-    sitemap: z.enum(["skip", "include", "only"]).optional(),
-    limit: z.number().optional(),
-    allowExternalLinks: z.boolean().optional(),
-    allowSubdomains: z.boolean().optional(),
-    crawlEntireDomain: z.boolean().optional(),
-    delay: z.number().optional(),
-    maxConcurrency: z.number().optional(),
-    ...SAFE_MODE ? {} : {
-      webhook: z.union([
-        z.string(),
-        z.object({
-          url: z.string(),
-          headers: z.record(z.string(), z.string()).optional()
-        })
-      ]).optional()
-    },
-    deduplicateSimilarURLs: z.boolean().optional(),
-    ignoreQueryParameters: z.boolean().optional(),
-    scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional()
-  }),
-  execute: async (args2, { session, log }) => {
-    const { url, ...options } = args2;
-    return safeExecute(async () => {
-      const client = getClient(session);
-      const cleaned = removeEmptyTopLevel(options);
-      log.info("Starting crawl", { url: String(url) });
-      const res = await client.crawl(String(url), {
-        ...cleaned,
-        origin: ORIGIN
-      });
-      return await processResponse(res, {
-        url: String(url),
-        skipSummarization: true
-      });
-    }, { tool: "scorch_crawl", url: String(url) });
-  }
-});
-server.addTool({
-  name: "scorch_check_crawl_status",
-  annotations: {
-    title: "Check Crawl Status",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: true,
-    openWorldHint: false
-  },
-  description: `
-Check the status of a crawl job.
-
-**Usage Example:**
-\`\`\`json
-{
-  "name": "scorch_check_crawl_status",
-  "arguments": {
-    "id": "550e8400-e29b-41d4-a716-446655440000"
-  }
-}
-\`\`\`
-**Returns:** Status and progress of the crawl job, including results if available.
-`,
-  parameters: z.object({ id: z.string() }),
-  execute: async (args2, { session }) => {
-    return safeExecute(async () => {
-      const client = getClient(session);
-      const res = await client.getCrawlStatus(args2.id);
-      return asText(res);
-    }, { tool: "scorch_check_crawl_status" });
-  }
-});
-server.addTool({
-  name: "scorch_extract",
-  annotations: {
-    title: "Extract Structured Data",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: true,
-    openWorldHint: true
-  },
-  description: `
-Extract structured information from web pages using LLM capabilities. Supports both cloud AI and self-hosted LLM extraction.
-
-**Best for:** Extracting specific structured data like prices, names, details from web pages.
-**Not recommended for:** When you need the full content of a page (use scrape); when you're not looking for specific structured data.
-**Arguments:**
-- urls: Array of URLs to extract information from
-- prompt: Custom prompt for the LLM extraction
-- schema: JSON schema for structured data extraction
-- allowExternalLinks: Allow extraction from external links
-- enableWebSearch: Enable web search for additional context
-- includeSubdomains: Include subdomains in extraction
-**Prompt Example:** "Extract the product name, price, and description from these product pages."
-**Usage Example:**
-\`\`\`json
-{
-  "name": "scorch_extract",
-  "arguments": {
-    "urls": ["https://example.com/page1", "https://example.com/page2"],
-    "prompt": "Extract product information including name, price, and description",
-    "schema": {
-      "type": "object",
-      "properties": {
-        "name": { "type": "string" },
-        "price": { "type": "number" },
-        "description": { "type": "string" }
-      },
-      "required": ["name", "price"]
-    },
-    "allowExternalLinks": false,
-    "enableWebSearch": false,
-    "includeSubdomains": false
-  }
-}
-\`\`\`
-**Returns:** Extracted structured data as defined by your schema.
-`,
-  parameters: z.object({
-    urls: z.array(z.string()),
-    prompt: z.string().optional(),
-    schema: z.record(z.string(), z.any()).optional(),
-    allowExternalLinks: z.boolean().optional(),
-    enableWebSearch: z.boolean().optional(),
-    includeSubdomains: z.boolean().optional()
-  }),
-  execute: async (args2, { session, log }) => {
-    const a = args2;
-    return safeExecute(async () => {
-      const client = getClient(session);
-      log.info("Extracting from URLs", {
-        count: Array.isArray(a.urls) ? a.urls.length : 0
-      });
-      const extractBody = removeEmptyTopLevel({
-        urls: a.urls,
-        prompt: a.prompt,
-        schema: a.schema || void 0,
-        allowExternalLinks: a.allowExternalLinks,
-        enableWebSearch: a.enableWebSearch,
-        includeSubdomains: a.includeSubdomains,
-        origin: ORIGIN
-      });
-      const res = await client.extract(extractBody);
-      return await processResponse(res, { skipSummarization: true });
-    }, { tool: "scorch_extract" });
-  }
-});
-var agentConfig = buildAgentConfig();
-var allowedModelsList = agentConfig.allowedModels.join(", ");
-server.addTool({
-  name: "scorch_agent",
-  annotations: {
-    title: "AI Research Agent",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: false,
-    openWorldHint: true
-  },
-  description: `
-Autonomous web research agent powered by GitHub Copilot SDK. This is a separate AI agent layer that independently browses the internet, searches for information, navigates through pages, and extracts structured data based on your query. You describe what you need, and the agent figures out where to find it.
-
-**How it works:** The agent uses GitHub Copilot SDK to orchestrate web research using scorchcrawl tools (scrape, search, map, extract). It runs synchronously \u2014 the tool call blocks until the agent finishes and returns the complete result.
-
-**Available models:** ${allowedModelsList}
-
-**Expected wait times:**
-- Simple queries with provided URLs: 30 seconds - 1 minute
-- Complex research across multiple sites: 2-5 minutes
-- Deep research tasks: 5+ minutes
-
-**Best for:** Complex research tasks where you don't know the exact URLs; multi-source data gathering; finding information scattered across the web; extracting data from JavaScript-heavy SPAs that fail with regular scrape.
-**Not recommended for:** Simple single-page scraping where you know the URL (use scrape with JSON format instead - faster and cheaper).
-
-**Arguments:**
-- prompt: Natural language description of the data you want (required, max 10,000 characters)
-- urls: Optional array of URLs to focus the agent on specific pages
-- schema: Optional JSON schema for structured output
-- model: Optional model override (must be one of: ${allowedModelsList})
-- timeout: Optional timeout in milliseconds (default: AGENT_TIMEOUT_MS env var or 60000). Increase for complex research tasks.
-
-**Prompt Example:** "Find the founders of ScorchCrawl and their backgrounds"
-**Usage Example:**
-\`\`\`json
-{
-  "name": "scorch_agent",
-  "arguments": {
-    "prompt": "Find the top 5 AI startups founded in 2024 and their funding amounts",
-    "schema": {
-      "type": "object",
-      "properties": {
-        "startups": {
-          "type": "array",
-          "items": {
-            "type": "object",
-            "properties": {
-              "name": { "type": "string" },
-              "funding": { "type": "string" },
-              "founded": { "type": "string" }
-            }
-          }
-        }
-      }
-    }
-  }
-}
-\`\`\`
-**Usage Example (with URLs - agent focuses on specific pages):**
-\`\`\`json
-{
-  "name": "scorch_agent",
-  "arguments": {
-    "urls": ["https://docs.example.com", "https://example.com/pricing"],
-    "prompt": "Compare the features and pricing information from these pages"
-  }
-}
-\`\`\`
-**Returns:** The complete research result directly, including extracted data, model used, and duration.
-`,
-  parameters: z.object({
-    prompt: z.string().min(1).max(1e4),
-    urls: z.array(z.string().url()).optional(),
-    schema: z.record(z.string(), z.any()).optional(),
-    model: z.string().optional(),
-    timeout: z.number().int().min(5e3).max(6e5).optional().describe("Timeout in milliseconds (5000-600000). Default: AGENT_TIMEOUT_MS env var or 60000.")
-  }),
-  execute: async (args2, { session, log }) => {
-    const client = getClient(session);
-    const a = args2;
-    const requestedModel = a.model;
-    if (requestedModel && !agentConfig.allowedModels.includes(requestedModel)) {
-      return asText({
-        success: false,
-        error: `Model "${requestedModel}" is not allowed. Available models: ${allowedModelsList}`
-      });
-    }
-    log.info("Starting Copilot SDK agent", {
-      prompt: a.prompt.substring(0, 100),
-      urlCount: Array.isArray(a.urls) ? a.urls.length : 0,
-      model: requestedModel || agentConfig.defaultModel
-    });
-    try {
-      const result = await startAgent(
-        {
-          prompt: a.prompt,
-          urls: a.urls,
-          schema: a.schema || void 0,
-          model: requestedModel,
-          timeout: a.timeout
-        },
-        client,
-        ORIGIN,
-        agentConfig,
-        session?.copilotToken
-      );
-      if (result.rateLimited) {
-        return asText({
-          success: false,
-          rateLimited: true,
-          error: result.error,
-          retryAfterSeconds: result.retryAfterSeconds,
-          hint: "Wait for the specified duration and retry."
-        });
-      }
-      return asText(result);
-    } catch (err) {
-      log.error("Failed to start agent", { error: err.message });
-      return asText({
-        success: false,
-        error: `Failed to start agent: ${err.message || err}`
-      });
-    }
-  }
-});
-server.addTool({
-  name: "scorch_agent_models",
-  annotations: {
-    title: "List Agent Models",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: true,
-    openWorldHint: false
-  },
-  description: `
-List the available models for the Copilot SDK agent. These models are configured via the COPILOT_AGENT_MODELS environment variable.
-
-**Returns:** List of allowed models and the current default model.
-`,
-  parameters: z.object({}),
-  execute: async () => {
-    return asText({
-      allowedModels: parseAllowedModels(),
-      defaultModel: getDefaultModel()
-    });
-  }
-});
-server.addTool({
-  name: "scorch_screenshot",
-  annotations: {
-    title: "Screenshot Web Page",
-    readOnlyHint: true,
-    destructiveHint: false,
-    idempotentHint: true,
-    openWorldHint: true
-  },
-  description: `
-Take a screenshot of a web page for visual inspection. Returns a base64-encoded PNG image.
-
-**Best for:** Debugging why scrape returns empty/blocked content; seeing cookie popups, CAPTCHAs, anti-bot challenges; inspecting JS-rendered pages; visual verification of page state.
-**Not recommended for:** Extracting text content (use scorch_scrape instead).
-
-**When to use:**
-- After scorch_scrape returns empty, minimal, or anti-bot content
-- To verify that a page is rendering correctly
-- To see cookie consent popups or CAPTCHA challenges
-- To inspect visual-only content (charts, infographics)
-
-**Usage Example:**
-\`\`\`json
-{
-  "name": "scorch_screenshot",
-  "arguments": {
-    "url": "https://example.com",
-    "fullPage": false,
-    "waitFor": 3000
-  }
-}
-\`\`\`
-**Returns:** Base64-encoded PNG screenshot with page metadata (title, status code).
-`,
-  parameters: z.object({
-    url: z.string().url(),
-    fullPage: z.boolean().optional(),
-    waitFor: z.number().optional(),
-    quality: z.number().min(1).max(100).optional(),
-    viewport: z.object({
-      width: z.number(),
-      height: z.number()
-    }).optional()
-  }),
-  execute: async (args2, { session, log }) => {
-    const a = args2;
-    const url = String(a.url);
-    const fullPage = a.fullPage === true;
-    const waitFor = typeof a.waitFor === "number" ? a.waitFor : 3e3;
-    return safeExecute(async () => {
-      const client = getClient(session);
-      log.info("Taking screenshot", { url });
-      const screenshotFormat = {
-        type: "screenshot",
-        fullPage
-      };
-      if (a.quality) screenshotFormat.quality = a.quality;
-      if (a.viewport) screenshotFormat.viewport = a.viewport;
-      const res = await client.scrape(url, {
-        formats: [screenshotFormat],
-        waitFor,
-        origin: ORIGIN
-      });
-      const resAny = res;
-      const screenshot = resAny?.screenshot || resAny?.data?.screenshot;
-      const metadata = resAny?.metadata || resAny?.data?.metadata || {};
-      return JSON.stringify({
-        success: !!screenshot,
-        url,
-        metadata: {
-          title: metadata.title || null,
-          statusCode: metadata.statusCode || null,
-          description: metadata.description || null
-        },
-        screenshot: screenshot || null,
-        screenshotSizeKB: screenshot ? Math.round(screenshot.length / 1024) : 0
-      }, null, 2);
-    }, { tool: "scorch_screenshot", url });
-  }
-});
-process.on("SIGINT", async () => {
-  await shutdownAgent();
-  process.exit(0);
-});
-process.on("SIGTERM", async () => {
-  await shutdownAgent();
-  process.exit(0);
-});
-var PORT = Number(process.env.PORT || 3e3);
-var HOST = process.env.CLOUD_SERVICE === "true" ? "0.0.0.0" : process.env.HOST || "localhost";
-var args;
-if (process.env.CLOUD_SERVICE === "true" || process.env.SSE_LOCAL === "true" || process.env.HTTP_STREAMABLE_SERVER === "true") {
-  args = {
-    transportType: "httpStream",
-    httpStream: {
-      port: PORT,
-      host: HOST,
-      stateless: true
-    }
-  };
-} else {
-  args = {
-    transportType: "stdio"
-  };
-}
-await server.start(args);