scorchcrawl-mcp 2.1.0 → 2.1.1

package/dist/index.js

@@ -1,717 +0,0 @@
-#!/usr/bin/env node
-import dotenv from 'dotenv';
-import { FastMCP } from './lib/fastmcp/index.js';
-import { z } from 'zod';
-import ScorchClient from './lib/scorch-client/index.js';
-import { localScrape, isLocalProxyEnabled, getCleanApiUrl } from './local-scraper.js';
-import { mapError, safeExecute, processResponse, processResponseSync, } from './response-utils.js';
-dotenv.config({ debug: false, quiet: true });
-function extractApiKey(headers) {
-    const headerAuth = headers['authorization'];
-    const headerApiKey = (headers['x-scorchcrawl-api-key'] ||
-        headers['x-api-key']);
-    if (headerApiKey) {
-        return Array.isArray(headerApiKey) ? headerApiKey[0] : headerApiKey;
-    }
-    if (typeof headerAuth === 'string' &&
-        headerAuth.toLowerCase().startsWith('bearer ')) {
-        return headerAuth.slice(7).trim();
-    }
-    return undefined;
-}
-function removeEmptyTopLevel(obj) {
-    const out = {};
-    for (const [k, v] of Object.entries(obj)) {
-        if (v == null)
-            continue;
-        if (typeof v === 'string' && v.trim() === '')
-            continue;
-        if (Array.isArray(v) && v.length === 0)
-            continue;
-        if (typeof v === 'object' &&
-            !Array.isArray(v) &&
-            Object.keys(v).length === 0)
-            continue;
-        // @ts-expect-error dynamic assignment
-        out[k] = v;
-    }
-    return out;
-}
-class ConsoleLogger {
-    shouldLog = process.env.CLOUD_SERVICE === 'true' ||
-        process.env.SSE_LOCAL === 'true' ||
-        process.env.HTTP_STREAMABLE_SERVER === 'true';
-    debug(...args) {
-        if (this.shouldLog) {
-            console.debug('[DEBUG]', new Date().toISOString(), ...args);
-        }
-    }
-    error(...args) {
-        if (this.shouldLog) {
-            console.error('[ERROR]', new Date().toISOString(), ...args);
-        }
-    }
-    info(...args) {
-        if (this.shouldLog) {
-            console.log('[INFO]', new Date().toISOString(), ...args);
-        }
-    }
-    log(...args) {
-        if (this.shouldLog) {
-            console.log('[LOG]', new Date().toISOString(), ...args);
-        }
-    }
-    warn(...args) {
-        if (this.shouldLog) {
-            console.warn('[WARN]', new Date().toISOString(), ...args);
-        }
-    }
-}
-const server = new FastMCP({
-    name: 'scorchcrawl',
-    version: '1.0.0',
-    logger: new ConsoleLogger(),
-    roots: { enabled: false },
-    authenticate: async (request) => {
-        if (process.env.CLOUD_SERVICE === 'true') {
-            const apiKey = extractApiKey(request.headers);
-            if (!apiKey) {
-                throw new Error('API key is required');
-            }
-            return { scraperApiKey: apiKey };
-        }
-        else {
-            // For self-hosted instances, API key is optional if SCORCHCRAWL_API_URL is provided
-            if (!process.env.SCORCHCRAWL_API_KEY && !process.env.SCORCHCRAWL_API_URL) {
-                console.error('Either SCORCHCRAWL_API_KEY or SCORCHCRAWL_API_URL must be provided');
-                process.exit(1);
-            }
-            return { scraperApiKey: process.env.SCORCHCRAWL_API_KEY };
-        }
-    },
-    // Lightweight health endpoint for LB checks
-    health: {
-        enabled: true,
-        message: 'ok',
-        path: '/health',
-        status: 200,
-    },
-});
-function createClient(apiKey) {
-    // Use cleaned URL (strips ?localProxy= param so the server doesn't see it)
-    const cleanUrl = getCleanApiUrl();
-    const config = {
-        ...(cleanUrl && {
-            apiUrl: cleanUrl,
-        }),
-    };
-    // Only add apiKey if it's provided (required for cloud, optional for self-hosted)
-    if (apiKey) {
-        config.apiKey = apiKey;
-    }
-    return new ScorchClient(config);
-}
-const ORIGIN = 'scorchcrawl-mcp';
-// Safe mode is enabled by default for cloud service to comply with ChatGPT safety requirements
-const SAFE_MODE = process.env.CLOUD_SERVICE === 'true';
-function getClient(session) {
-    // For cloud service, API key is required
-    if (process.env.CLOUD_SERVICE === 'true') {
-        if (!session || !session.scraperApiKey) {
-            throw new Error('Unauthorized');
-        }
-        return createClient(session.scraperApiKey);
-    }
-    // For self-hosted instances, API key is optional if SCORCHCRAWL_API_URL is provided
-    if (!process.env.SCORCHCRAWL_API_URL &&
-        (!session || !session.scraperApiKey)) {
-        throw new Error('Unauthorized: API key is required when not using a self-hosted instance');
-    }
-    return createClient(session?.scraperApiKey);
-}
-function asText(data) {
-    return processResponseSync(data);
-}
-// scrape tool (v2 semantics, minimal args)
-// Centralized scrape params (used by scrape, and referenced in search/crawl scrapeOptions)
-// Define safe action types
-const safeActionTypes = ['wait', 'screenshot', 'scroll', 'scrape'];
-const otherActions = [
-    'click',
-    'write',
-    'press',
-    'executeJavascript',
-    'generatePDF',
-];
-const allActionTypes = [...safeActionTypes, ...otherActions];
-// Use appropriate action types based on safe mode
-const allowedActionTypes = SAFE_MODE ? safeActionTypes : allActionTypes;
-const scrapeParamsSchema = z.object({
-    url: z.string().url(),
-    formats: z
-        .array(z.union([
-        z.enum([
-            'markdown',
-            'html',
-            'rawHtml',
-            'screenshot',
-            'links',
-            'summary',
-            'changeTracking',
-            'branding',
-        ]),
-        z.object({
-            type: z.literal('json'),
-            prompt: z.string().optional(),
-            schema: z.record(z.string(), z.any()).optional(),
-        }),
-        z.object({
-            type: z.literal('screenshot'),
-            fullPage: z.boolean().optional(),
-            quality: z.number().optional(),
-            viewport: z
-                .object({ width: z.number(), height: z.number() })
-                .optional(),
-        }),
-    ]))
-        .optional(),
-    parsers: z
-        .array(z.union([
-        z.enum(['pdf']),
-        z.object({
-            type: z.enum(['pdf']),
-            maxPages: z.number().int().min(1).max(10000).optional(),
-        }),
-    ]))
-        .optional(),
-    onlyMainContent: z.boolean().optional(),
-    includeTags: z.array(z.string()).optional(),
-    excludeTags: z.array(z.string()).optional(),
-    waitFor: z.number().optional(),
-    ...(SAFE_MODE
-        ? {}
-        : {
-            actions: z
-                .array(z.object({
-                type: z.enum(allowedActionTypes),
-                selector: z.string().optional(),
-                milliseconds: z.number().optional(),
-                text: z.string().optional(),
-                key: z.string().optional(),
-                direction: z.enum(['up', 'down']).optional(),
-                script: z.string().optional(),
-                fullPage: z.boolean().optional(),
-            }))
-                .optional(),
-        }),
-    mobile: z.boolean().optional(),
-    skipTlsVerification: z.boolean().optional(),
-    removeBase64Images: z.boolean().optional(),
-    location: z
-        .object({
-        country: z.string().optional(),
-        languages: z.array(z.string()).optional(),
-    })
-        .optional(),
-    storeInCache: z.boolean().optional(),
-    zeroDataRetention: z.boolean().optional(),
-    maxAge: z.number().optional(),
-    proxy: z.enum(['basic', 'stealth', 'enhanced', 'auto']).optional(),
-});
-server.addTool({
-    name: 'scorch_scrape',
-    description: `
-Scrape content from a single URL with advanced options.
-This is the most powerful, fastest and most reliable scraper tool, if available you should always default to using this tool for any web scraping needs.
-
-**Best for:** Single page content extraction, when you know exactly which page contains the information.
-**Not recommended for:** Multiple pages (call scrape multiple times or use crawl), unknown page location (use search).
-**Common mistakes:** Using markdown format when extracting specific data points (use JSON instead).
-**Other Features:** Use 'branding' format to extract brand identity (colors, fonts, typography, spacing, UI components) for design analysis or style replication.
-
-**CRITICAL - Format Selection (you MUST follow this):**
-When the user asks for SPECIFIC data points, you MUST use JSON format with a schema. Only use markdown when the user needs the ENTIRE page content.
-
-**Use JSON format when user asks for:**
-- Parameters, fields, or specifications (e.g., "get the header parameters", "what are the required fields")
-- Prices, numbers, or structured data (e.g., "extract the pricing", "get the product details")
-- API details, endpoints, or technical specs (e.g., "find the authentication endpoint")
-- Lists of items or properties (e.g., "list the features", "get all the options")
-- Any specific piece of information from a page
-
-**Use markdown format ONLY when:**
-- User wants to read/summarize an entire article or blog post
-- User needs to see all content on a page without specific extraction
-- User explicitly asks for the full page content
-
-**Handling JavaScript-rendered pages (SPAs):**
-If JSON extraction returns empty, minimal, or just navigation content, the page is likely JavaScript-rendered or the content is on a different URL. Try these steps IN ORDER:
-1. **Add waitFor parameter:** Set \`waitFor: 5000\` to \`waitFor: 10000\` to allow JavaScript to render before extraction
-2. **Try a different URL:** If the URL has a hash fragment (#section), try the base URL or look for a direct page URL
-3. **Use scorch_map to find the correct page:** Large documentation sites or SPAs often spread content across multiple URLs. Use \`scorch_map\` with a \`search\` parameter to discover the specific page containing your target content, then scrape that URL directly.
-   Example: If scraping "https://docs.example.com/reference" fails to find webhook parameters, use \`scorch_map\` with \`{"url": "https://docs.example.com/reference", "search": "webhook"}\` to find URLs like "/reference/webhook-events", then scrape that specific page.
-4. **Escalate manually:** If map+scrape still fails, inspect the rendered page yourself and decide whether a custom context-cutter or page-specific flow is warranted
-
-**Usage Example (JSON format - REQUIRED for specific data extraction):**
-\`\`\`json
-{
-  "name": "scorch_scrape",
-  "arguments": {
-    "url": "https://example.com/api-docs",
-    "formats": [{
-      "type": "json",
-      "prompt": "Extract the header parameters for the authentication endpoint",
-      "schema": {
-        "type": "object",
-        "properties": {
-          "parameters": {
-            "type": "array",
-            "items": {
-              "type": "object",
-              "properties": {
-                "name": { "type": "string" },
-                "type": { "type": "string" },
-                "required": { "type": "boolean" },
-                "description": { "type": "string" }
-              }
-            }
-          }
-        }
-      }
-    }]
-  }
-}
-\`\`\`
-**Usage Example (markdown format - ONLY when full content genuinely needed):**
-\`\`\`json
-{
-  "name": "scorch_scrape",
-  "arguments": {
-    "url": "https://example.com/article",
-    "formats": ["markdown"],
-    "onlyMainContent": true
-  }
-}
-\`\`\`
-**Usage Example (branding format - extract brand identity):**
-\`\`\`json
-{
-  "name": "scorch_scrape",
-  "arguments": {
-    "url": "https://example.com",
-    "formats": ["branding"]
-  }
-}
-\`\`\`
-**Branding format:** Extracts comprehensive brand identity (colors, fonts, typography, spacing, logo, UI components) for design analysis or style replication.
-**Performance:** Add maxAge parameter for 500% faster scrapes using cached data.
-**Returns:** JSON structured data, markdown, branding profile, or other formats as specified.
-${SAFE_MODE
-        ? '**Safe Mode:** Read-only content extraction. Interactive actions (click, write, executeJavascript) are disabled for security.'
-        : ''}
-`,
-    parameters: scrapeParamsSchema,
-    execute: async (args, { session, log }) => {
-        const { url, ...options } = args;
-        const cleaned = removeEmptyTopLevel(options);
-        return safeExecute(async () => {
-            // --- Local proxy mode: fetch through user's IP ---
-            if (isLocalProxyEnabled()) {
-                log.info('Scraping URL via LOCAL PROXY (user IP)', { url: String(url) });
-                const localResult = await localScrape(String(url), cleaned);
-                // If the requested format needs server-side processing, fall back
-                if (!localResult.success && localResult.error === 'FORMAT_NEEDS_SERVER') {
-                    log.info('Format requires server-side processing, falling back to API', { url: String(url) });
-                    const client = getClient(session);
-                    const res = await client.scrape(String(url), {
-                        ...cleaned,
-                        origin: ORIGIN,
-                    });
-                    return await processResponse(res, {
-                        url: String(url),
-                    });
-                }
-                // SPA detected: the local fetch returned a JS-only shell.
-                // Retry via the engine's Playwright scraper with waitFor for JS rendering.
-                if (!localResult.success && localResult.error === 'SPA_SKELETON_DETECTED') {
-                    const waitMs = cleaned.waitFor || 5000;
-                    log.info(`SPA skeleton detected, retrying via engine with waitFor=${waitMs}ms`, { url: String(url) });
-                    const client = getClient(session);
-                    const res = await client.scrape(String(url), {
-                        ...cleaned,
-                        waitFor: waitMs,
-                        origin: ORIGIN,
-                    });
-                    return await processResponse(res, {
-                        url: String(url),
-                    });
-                }
-                // Errors from local scraper
-                if (!localResult.success && localResult.error) {
-                    const mapped = mapError(localResult.error);
-                    return JSON.stringify({
-                        success: false,
-                        error: mapped.message,
-                        code: mapped.code,
-                        suggestions: mapped.suggestions,
-                    }, null, 2);
-                }
-                return await processResponse(localResult, {
-                    url: String(url),
-                });
-            }
-            // --- Normal mode: use remote scraping API ---
-            const client = getClient(session);
-            log.info('Scraping URL', { url: String(url) });
-            const res = await client.scrape(String(url), {
-                ...cleaned,
-                origin: ORIGIN,
-            });
-            return await processResponse(res, {
-                url: String(url),
-            });
-        }, { tool: 'scorch_scrape', url: String(url) });
-    },
-});
-server.addTool({
-    name: 'scorch_map',
-    description: `
-Map a website to discover all indexed URLs on the site.
-
-**Best for:** Discovering URLs on a website before deciding what to scrape; finding specific sections or pages within a large site; locating the correct page when scrape returns empty or incomplete results.
-**Not recommended for:** When you already know which specific URL you need (use scrape); when you need the content of the pages (use scrape after mapping).
-**Common mistakes:** Using crawl to discover URLs instead of map; giving up on map too early when scrape fails to find the right page.
-
-**IMPORTANT - Use map before custom handling:** If \`scorch_scrape\` returns empty, minimal, or irrelevant content, use \`scorch_map\` with the \`search\` parameter to find the specific page URL containing your target content before building any page-specific handling.
-
-**Prompt Example:** "Find the webhook documentation page on this API docs site."
-**Usage Example (discover all URLs):**
-\`\`\`json
-{
-  "name": "scorch_map",
-  "arguments": {
-    "url": "https://example.com"
-  }
-}
-\`\`\`
-**Usage Example (search for specific content - RECOMMENDED when scrape fails):**
-\`\`\`json
-{
-  "name": "scorch_map",
-  "arguments": {
-    "url": "https://docs.example.com/api",
-    "search": "webhook events"
-  }
-}
-\`\`\`
-**Returns:** Array of URLs found on the site, filtered by search query if provided.
-`,
-    parameters: z.object({
-        url: z.string().url(),
-        search: z.string().optional(),
-        sitemap: z.enum(['include', 'skip', 'only']).optional(),
-        includeSubdomains: z.boolean().optional(),
-        limit: z.number().optional(),
-        ignoreQueryParameters: z.boolean().optional(),
-    }),
-    execute: async (args, { session, log }) => {
-        const { url, ...options } = args;
-        return safeExecute(async () => {
-            const client = getClient(session);
-            const cleaned = removeEmptyTopLevel(options);
-            log.info('Mapping URL', { url: String(url) });
-            const res = await client.map(String(url), {
-                ...cleaned,
-                origin: ORIGIN,
-            });
-            return asText(res);
-        }, { tool: 'scorch_map', url: String(url) });
-    },
-});
-server.addTool({
-    name: 'scorch_search',
-    description: `
-Search the web and optionally extract content from search results. This is the most powerful web search tool available, and if available you should always default to using this tool for any web search needs.
-
-The query also supports search operators, that you can use if needed to refine the search:
-| Operator | Functionality | Examples |
----|-|-|
-| \`"\"\` | Non-fuzzy matches a string of text | \`"ScorchCrawl"\`
-| \`-\` | Excludes certain keywords or negates other operators | \`-bad\`, \`-site:example.com\`
-| \`site:\` | Only returns results from a specified website | \`site:example.com\`
-| \`inurl:\` | Only returns results that include a word in the URL | \`inurl:example\`
-| \`allinurl:\` | Only returns results that include multiple words in the URL | \`allinurl:git example\`
-| \`intitle:\` | Only returns results that include a word in the title of the page | \`intitle:ScorchCrawl\`
-| \`allintitle:\` | Only returns results that include multiple words in the title of the page | \`allintitle:example playground\`
-| \`related:\` | Only returns results that are related to a specific domain | \`related:example.com\`
-| \`imagesize:\` | Only returns images with exact dimensions | \`imagesize:1920x1080\`
-| \`larger:\` | Only returns images larger than specified dimensions | \`larger:1920x1080\`
-
-**Best for:** Finding specific information across multiple websites, when you don't know which website has the information; when you need the most relevant content for a query.
-**Not recommended for:** When you need to search the filesystem. When you already know which website to scrape (use scrape); when you need comprehensive coverage of a single website (use map or crawl.
-**Common mistakes:** Using crawl or map for open-ended questions (use search instead).
-**Prompt Example:** "Find the latest research papers on AI published in 2023."
-**Sources:** web, images, news, default to web unless needed images or news.
-**Scrape Options:** Only use scrapeOptions when you think it is absolutely necessary. When you do so default to a lower limit to avoid timeouts, 5 or lower.
-**Optimal Workflow:** Search first using scorch_search without formats, then after fetching the results, use the scrape tool to get the content of the relevantpage(s) that you want to scrape
-
-**Usage Example without formats (Preferred):**
-\`\`\`json
-{
-  "name": "scorch_search",
-  "arguments": {
-    "query": "top AI companies",
-    "limit": 5,
-    "sources": [
-      { "type": "web" }
-    ]
-  }
-}
-\`\`\`
-**Usage Example with formats:**
-\`\`\`json
-{
-  "name": "scorch_search",
-  "arguments": {
-    "query": "latest AI research papers 2023",
-    "limit": 5,
-    "lang": "en",
-    "country": "us",
-    "sources": [
-      { "type": "web" },
-      { "type": "images" },
-      { "type": "news" }
-    ],
-    "scrapeOptions": {
-      "formats": ["markdown"],
-      "onlyMainContent": true
-    }
-  }
-}
-\`\`\`
-**Returns:** Array of search results (with optional scraped content).
-`,
-    parameters: z.object({
-        query: z.string().min(1),
-        limit: z.number().optional(),
-        tbs: z.string().optional(),
-        filter: z.string().optional(),
-        location: z.string().optional(),
-        sources: z
-            .array(z.object({ type: z.enum(['web', 'images', 'news']) }))
-            .optional(),
-        scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional(),
-        enterprise: z.array(z.enum(['default', 'anon', 'zdr'])).optional(),
-    }),
-    execute: async (args, { session, log }) => {
-        const { query, ...opts } = args;
-        return safeExecute(async () => {
-            const client = getClient(session);
-            const cleaned = removeEmptyTopLevel(opts);
-            log.info('Searching', { query: String(query) });
-            const res = await client.search(query, {
-                ...cleaned,
-                origin: ORIGIN,
-            });
-            // Search results are already concise snippets — skip summarization
-            return await processResponse(res, {
-                skipSummarization: true,
-            });
-        }, { tool: 'scorch_search' });
-    },
-});
-server.addTool({
-    name: 'scorch_crawl',
-    description: `
- Starts a crawl job on a website and extracts content from all pages.
- 
- **Best for:** Extracting content from multiple related pages, when you need comprehensive coverage.
- **Not recommended for:** Extracting content from a single page (use scrape); when token limits are a concern (use map + batch_scrape); when you need fast results (crawling can be slow).
- **Warning:** Crawl responses can be very large and may exceed token limits. Limit the crawl depth and number of pages, or use map + batch_scrape for better control.
- **Common mistakes:** Setting limit or maxDiscoveryDepth too high (causes token overflow) or too low (causes missing pages); using crawl for a single page (use scrape instead). Using a /* wildcard is not recommended.
- **Prompt Example:** "Get all blog posts from the first two levels of example.com/blog."
- **Usage Example:**
- \`\`\`json
- {
-   "name": "scorch_crawl",
-   "arguments": {
-     "url": "https://example.com/blog/*",
-     "maxDiscoveryDepth": 5,
-     "limit": 20,
-     "allowExternalLinks": false,
-     "deduplicateSimilarURLs": true,
-     "sitemap": "include"
-   }
- }
- \`\`\`
- **Returns:** Operation ID for status checking; use scorch_check_crawl_status to check progress.
- ${SAFE_MODE
-        ? '**Safe Mode:** Read-only crawling. Webhooks and interactive actions are disabled for security.'
-        : ''}
- `,
-    parameters: z.object({
-        url: z.string(),
-        prompt: z.string().optional(),
-        excludePaths: z.array(z.string()).optional(),
-        includePaths: z.array(z.string()).optional(),
-        maxDiscoveryDepth: z.number().optional(),
-        sitemap: z.enum(['skip', 'include', 'only']).optional(),
-        limit: z.number().optional(),
-        allowExternalLinks: z.boolean().optional(),
-        allowSubdomains: z.boolean().optional(),
-        crawlEntireDomain: z.boolean().optional(),
-        delay: z.number().optional(),
-        maxConcurrency: z.number().optional(),
-        ...(SAFE_MODE
-            ? {}
-            : {
-                webhook: z
-                    .union([
-                    z.string(),
-                    z.object({
-                        url: z.string(),
-                        headers: z.record(z.string(), z.string()).optional(),
-                    }),
-                ])
-                    .optional(),
-            }),
-        deduplicateSimilarURLs: z.boolean().optional(),
-        ignoreQueryParameters: z.boolean().optional(),
-        scrapeOptions: scrapeParamsSchema.omit({ url: true }).partial().optional(),
-    }),
-    execute: async (args, { session, log }) => {
-        const { url, ...options } = args;
-        return safeExecute(async () => {
-            const client = getClient(session);
-            const cleaned = removeEmptyTopLevel(options);
-            log.info('Starting crawl', { url: String(url) });
-            const res = await client.crawl(String(url), {
-                ...cleaned,
-                origin: ORIGIN,
-            });
-            // Crawl results use truncation only (no summarization — multi-page)
-            return await processResponse(res, {
-                url: String(url),
-                skipSummarization: true,
-            });
-        }, { tool: 'scorch_crawl', url: String(url) });
-    },
-});
-server.addTool({
-    name: 'scorch_check_crawl_status',
-    description: `
-Check the status of a crawl job.
-
-**Usage Example:**
-\`\`\`json
-{
-  "name": "scorch_check_crawl_status",
-  "arguments": {
-    "id": "550e8400-e29b-41d4-a716-446655440000"
-  }
-}
-\`\`\`
-**Returns:** Status and progress of the crawl job, including results if available.
-`,
-    parameters: z.object({ id: z.string() }),
-    execute: async (args, { session }) => {
-        return safeExecute(async () => {
-            const client = getClient(session);
-            const res = await client.getCrawlStatus(args.id);
-            return asText(res);
-        }, { tool: 'scorch_check_crawl_status' });
-    },
-});
-server.addTool({
-    name: 'scorch_extract',
-    description: `
-Extract structured information from web pages using LLM capabilities. Supports both cloud AI and self-hosted LLM extraction.
-
-**Best for:** Extracting specific structured data like prices, names, details from web pages.
-**Not recommended for:** When you need the full content of a page (use scrape); when you're not looking for specific structured data.
-**Arguments:**
-- urls: Array of URLs to extract information from
-- prompt: Custom prompt for the LLM extraction
-- schema: JSON schema for structured data extraction
-- allowExternalLinks: Allow extraction from external links
-- enableWebSearch: Enable web search for additional context
-- includeSubdomains: Include subdomains in extraction
-**Prompt Example:** "Extract the product name, price, and description from these product pages."
-**Usage Example:**
-\`\`\`json
-{
-  "name": "scorch_extract",
-  "arguments": {
-    "urls": ["https://example.com/page1", "https://example.com/page2"],
-    "prompt": "Extract product information including name, price, and description",
-    "schema": {
-      "type": "object",
-      "properties": {
-        "name": { "type": "string" },
-        "price": { "type": "number" },
-        "description": { "type": "string" }
-      },
-      "required": ["name", "price"]
-    },
-    "allowExternalLinks": false,
-    "enableWebSearch": false,
-    "includeSubdomains": false
-  }
-}
-\`\`\`
-**Returns:** Extracted structured data as defined by your schema.
-`,
-    parameters: z.object({
-        urls: z.array(z.string()),
-        prompt: z.string().optional(),
-        schema: z.record(z.string(), z.any()).optional(),
-        allowExternalLinks: z.boolean().optional(),
-        enableWebSearch: z.boolean().optional(),
-        includeSubdomains: z.boolean().optional(),
-    }),
-    execute: async (args, { session, log }) => {
-        const a = args;
-        return safeExecute(async () => {
-            const client = getClient(session);
-            log.info('Extracting from URLs', {
-                count: Array.isArray(a.urls) ? a.urls.length : 0,
-            });
-            const extractBody = removeEmptyTopLevel({
-                urls: a.urls,
-                prompt: a.prompt,
-                schema: a.schema || undefined,
-                allowExternalLinks: a.allowExternalLinks,
-                enableWebSearch: a.enableWebSearch,
-                includeSubdomains: a.includeSubdomains,
-                origin: ORIGIN,
-            });
-            const res = await client.extract(extractBody);
-            // Extract returns structured data — skip summarization
-            return await processResponse(res, { skipSummarization: true });
-        }, { tool: 'scorch_extract' });
-    },
-});
-const PORT = Number(process.env.PORT || 3000);
-const HOST = process.env.CLOUD_SERVICE === 'true'
-    ? '0.0.0.0'
-    : process.env.HOST || 'localhost';
-let args;
-if (process.env.CLOUD_SERVICE === 'true' ||
-    process.env.SSE_LOCAL === 'true' ||
-    process.env.HTTP_STREAMABLE_SERVER === 'true') {
-    args = {
-        transportType: 'httpStream',
-        httpStream: {
-            port: PORT,
-            host: HOST,
-            stateless: true,
-        },
-    };
-}
-else {
-    // default: stdio
-    args = {
-        transportType: 'stdio',
-    };
-}
-await server.start(args);