scorchcrawl-mcp 2.1.1 → 2.2.0

package/dist/response-utils.js

@@ -0,0 +1,375 @@
+/**
+ * Response Processing Utilities
+ *
+ * Two layers of intelligent response handling for MCP tool results:
+ *   1. Error Mapping — classifies errors and returns actionable guidance
+ *   2. Content Truncation — smart paragraph-boundary truncation with notices
+ *
+ * No competitor MCP scraper implements any of these. This is a ScorchCrawl
+ * differentiator.
+ */
+// ---------------------------------------------------------------------------
+// Configuration (from environment)
+// ---------------------------------------------------------------------------
+const MAX_CONTENT_CHARS = parseInt(process.env.SCORCHCRAWL_MAX_CONTENT_CHARS || '25000', 10);
+/**
+ * Classify a raw error (from fetch, SDK, or engine) into a structured
+ * MappedError with actionable guidance for the LLM.
+ */
+export function mapError(err) {
+    const raw = normalizeErrorString(err);
+    const lower = raw.toLowerCase();
+    // --- HTTP status-based patterns ---
+    if (matches(lower, ['403', 'forbidden', 'cloudflare', 'cf-ray', 'challenge', 'access denied'])) {
+        return {
+            code: 'ACCESS_DENIED',
+            message: 'This site blocks automated access.',
+            suggestions: [
+                'Try scorch_search to find cached/indexed content instead',
+                'Add waitFor: 5000 to let challenge pages resolve',
+                'Try with proxy: "stealth" for enhanced bot evasion',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['404', 'not found', 'page not found', 'does not exist'])) {
+        return {
+            code: 'NOT_FOUND',
+            message: 'Page does not exist at this URL.',
+            suggestions: [
+                'Use scorch_map to discover correct URLs on the site',
+                'Check if the URL has a typo or outdated path',
+                'Try scorch_search to find the current location of this content',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['429', 'rate limit', 'too many requests', 'throttl'])) {
+        return {
+            code: 'RATE_LIMITED',
+            message: 'Rate limited by the target site.',
+            suggestions: [
+                'Wait 30 seconds and retry',
+                'Try a different URL on the same site',
+                'Use scorch_search to find the content from a different source',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matchesStatusRange(lower, 500, 599)) {
+        return {
+            code: 'SERVER_ERROR',
+            message: 'The target site returned a server error.',
+            suggestions: [
+                'Retry once — the site may be experiencing temporary issues',
+                'Try again in a few minutes if it persists',
+            ],
+            originalError: raw,
+        };
+    }
+    // --- Network/connection errors ---
+    if (matches(lower, ['timeout', 'aborterror', 'aborted', 'timed out', 'etimedout'])) {
+        return {
+            code: 'TIMEOUT',
+            message: 'Page took too long to load.',
+            suggestions: [
+                'Add onlyMainContent: true to reduce processing time',
+                'Increase waitFor to give JS more time to render',
+                'Try with a simpler format like markdown instead of screenshot',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['econnrefused', 'enotfound', 'dns', 'getaddrinfo', 'connect failed'])) {
+        // Distinguish between engine-down and remote-site-down
+        if (matches(lower, ['localhost', '127.0.0.1', '0.0.0.0', '3002', 'engine', 'api'])) {
+            return {
+                code: 'ENGINE_UNAVAILABLE',
+                message: 'Scraping engine is not running.',
+                suggestions: [
+                    'Check that Docker services are up: docker compose up -d',
+                    'Verify SCORCHCRAWL_API_URL is correct in your config',
+                ],
+                originalError: raw,
+            };
+        }
+        return {
+            code: 'CONNECTION_FAILED',
+            message: 'Cannot reach this URL.',
+            suggestions: [
+                'Check if the URL is correct and accessible',
+                'The site may be down — try again later',
+                'Try scorch_search to find alternative sources',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['ssl', 'tls', 'certificate', 'cert', 'self-signed', 'unable to verify'])) {
+        return {
+            code: 'TLS_ERROR',
+            message: 'SSL certificate issue with this site.',
+            suggestions: [
+                'Try with skipTlsVerification: true',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['empty', 'no content', 'no markdown', 'no extractable'])) {
+        return {
+            code: 'EMPTY_CONTENT',
+            message: 'Page returned no extractable content.',
+            suggestions: [
+                'Use scorch_map with a search param to find the right page',
+                'Add waitFor: 5000 for JavaScript-rendered pages',
+                'Try with formats: ["html"] to see the raw page structure',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['spa_skeleton_detected', 'spa detected', 'javascript is required'])) {
+        return {
+            code: 'SPA_DETECTED',
+            message: 'Page requires JavaScript rendering (SPA detected).',
+            suggestions: [
+                'Add waitFor: 5000 to allow JavaScript to render',
+                'The engine will automatically retry with Playwright',
+            ],
+            originalError: raw,
+        };
+    }
+    // --- Catch-all ---
+    return {
+        code: 'UNKNOWN_ERROR',
+        message: `Scraping failed: ${truncateString(raw, 200)}`,
+        suggestions: [
+            'Try a different URL or format',
+            'Use scorch_search as an alternative data source',
+        ],
+        originalError: raw,
+    };
+}
+/** Normalize any error-like value to a string */
+function normalizeErrorString(err) {
+    if (err == null)
+        return 'Unknown error';
+    if (err instanceof Error) {
+        // Include status code from Axios/fetch errors
+        const axiosStatus = err?.response?.status;
+        const status = err?.status || err?.statusCode || axiosStatus;
+        const prefix = status ? `HTTP ${status}: ` : '';
+        return `${prefix}${err.message}`;
+    }
+    if (typeof err === 'string')
+        return err;
+    try {
+        const s = JSON.stringify(err);
+        return s || 'Unknown error';
+    }
+    catch {
+        return String(err);
+    }
+}
+/** Check if a lowered string contains any of the patterns */
+function matches(lower, patterns) {
+    return patterns.some((p) => lower.includes(p));
+}
+/** Check if an error string refers to an HTTP status in a range */
+function matchesStatusRange(lower, from, to) {
+    const statusMatch = lower.match(/(?:status\s*(?:code\s*)?|http\s*)(\d{3})/);
+    if (statusMatch) {
+        const code = parseInt(statusMatch[1], 10);
+        return code >= from && code <= to;
+    }
+    // Also check for bare 5xx patterns
+    for (let s = from; s <= to; s++) {
+        if (lower.includes(String(s)))
+            return true;
+    }
+    return false;
+}
+/** Safely truncate a string for display */
+function truncateString(s, maxLen) {
+    if (s.length <= maxLen)
+        return s;
+    return s.slice(0, maxLen - 3) + '...';
+}
+// ---------------------------------------------------------------------------
+// Feature 2: Content Truncation
+// ---------------------------------------------------------------------------
+/**
+ * Content fields that may need truncation.
+ * Metadata, links, and structured data are NEVER truncated.
+ */
+const CONTENT_FIELDS = ['markdown', 'html', 'rawHtml'];
+/**
+ * For crawl results: max fraction of total limit a single page can use.
+ */
+const SINGLE_PAGE_MAX_FRACTION = 0.3;
+/**
+ * Truncate a markdown/html string at the nearest paragraph or heading
+ * boundary before the character limit.
+ */
+export function truncateAtBoundary(content, maxChars) {
+    if (content.length <= maxChars) {
+        return { text: content, wasTruncated: false };
+    }
+    // Search backward from maxChars for a good break point
+    const searchWindow = content.slice(0, maxChars);
+    // Priority 1: Last heading boundary (# at start of line)
+    const headingMatch = searchWindow.lastIndexOf('\n#');
+    // Priority 2: Last double newline (paragraph boundary)
+    const paraMatch = searchWindow.lastIndexOf('\n\n');
+    // Priority 3: Last single newline
+    const lineMatch = searchWindow.lastIndexOf('\n');
+    // Priority 4: Last sentence end
+    const sentenceMatch = Math.max(searchWindow.lastIndexOf('. '), searchWindow.lastIndexOf('.\n'));
+    // Pick the best break point (prefer heading > paragraph > line > sentence)
+    let breakPoint = -1;
+    // Only use if it's in the last 30% of the window (don't cut too aggressively)
+    const minBreak = Math.floor(maxChars * 0.7);
+    if (headingMatch > minBreak)
+        breakPoint = headingMatch;
+    else if (paraMatch > minBreak)
+        breakPoint = paraMatch;
+    else if (lineMatch > minBreak)
+        breakPoint = lineMatch;
+    else if (sentenceMatch > minBreak)
+        breakPoint = sentenceMatch + 1; // include the period
+    else
+        breakPoint = maxChars; // hard cut as last resort
+    const truncated = content.slice(0, breakPoint).trimEnd();
+    return { text: truncated, wasTruncated: true };
+}
+/**
+ * Process a single scrape/crawl data object, truncating content fields
+ * while preserving metadata and structured data.
+ *
+ * Returns the processed data and truncation metadata.
+ */
+export function truncateContent(data) {
+    if (MAX_CONTENT_CHARS <= 0) {
+        const serialized = JSON.stringify(data, null, 2);
+        return { result: data, wasTruncated: false, originalLength: serialized.length };
+    }
+    const serialized = JSON.stringify(data, null, 2);
+    if (serialized.length <= MAX_CONTENT_CHARS) {
+        return { result: data, wasTruncated: false, originalLength: serialized.length };
+    }
+    // Deep clone to avoid mutating the original
+    const cloned = JSON.parse(serialized);
+    let wasTruncated = false;
+    // Handle single scrape result (has data.markdown, data.html, etc.)
+    if (cloned && typeof cloned === 'object') {
+        const target = cloned.data || cloned;
+        for (const field of CONTENT_FIELDS) {
+            if (typeof target[field] === 'string' && target[field].length > 0) {
+                // Budget for this field: total limit minus overhead from other fields
+                const otherFieldsSize = serialized.length - target[field].length;
+                const fieldBudget = Math.max(MAX_CONTENT_CHARS - otherFieldsSize, Math.floor(MAX_CONTENT_CHARS * 0.5));
+                if (target[field].length > fieldBudget) {
+                    const { text } = truncateAtBoundary(target[field], fieldBudget);
+                    const originalLen = target[field].length;
+                    target[field] = text + buildTruncationNotice(text.length, originalLen);
+                    wasTruncated = true;
+                }
+            }
+        }
+        // Handle crawl results (array of pages)
+        if (Array.isArray(target)) {
+            const result = truncateCrawlArray(target);
+            return { result: cloned, wasTruncated: result.wasTruncated, originalLength: serialized.length };
+        }
+        // Add truncation metadata
+        if (wasTruncated) {
+            const meta = target.metadata || target;
+            meta._truncated = true;
+            meta._originalLength = serialized.length;
+        }
+    }
+    return { result: cloned, wasTruncated, originalLength: serialized.length };
+}
+/**
+ * Truncate a crawl result array: limit per-page content and total page count.
+ */
+function truncateCrawlArray(pages) {
+    if (pages.length === 0)
+        return { wasTruncated: false };
+    const perPageLimit = Math.floor(MAX_CONTENT_CHARS * SINGLE_PAGE_MAX_FRACTION);
+    let wasTruncated = false;
+    let totalSize = 0;
+    for (let i = 0; i < pages.length; i++) {
+        const page = pages[i];
+        if (!page || typeof page !== 'object')
+            continue;
+        const target = page.data || page;
+        for (const field of CONTENT_FIELDS) {
+            if (typeof target[field] === 'string' && target[field].length > perPageLimit) {
+                const { text } = truncateAtBoundary(target[field], perPageLimit);
+                target[field] = text + buildTruncationNotice(text.length, target[field].length);
+                wasTruncated = true;
+            }
+        }
+        totalSize += JSON.stringify(page).length;
+        // If total exceeds limit, truncate the array
+        if (totalSize > MAX_CONTENT_CHARS && i < pages.length - 1) {
+            const originalCount = pages.length;
+            pages.length = i + 1;
+            pages.push({
+                _notice: `Showing ${i + 1} of ${originalCount} crawled pages. Use scorch_check_crawl_status with pagination to see more.`,
+            });
+            wasTruncated = true;
+            break;
+        }
+    }
+    return { wasTruncated };
+}
+/** Build a truncation notice for appending to content */
+function buildTruncationNotice(shownChars, originalChars) {
+    return `\n\n---\n[Content truncated. Showing ~${Math.round(shownChars / 1000)}k of ~${Math.round(originalChars / 1000)}k characters.\nTo get specific information, try:\n- Use JSON format with a schema to extract only the data you need\n- Add onlyMainContent: true to exclude navigation/footers\n- Use scorch_map to find a more specific page URL]`;
+}
+// ---------------------------------------------------------------------------
+// Combined Processing Pipeline
+// ---------------------------------------------------------------------------
+/**
+ * Safe wrapper for tool execution. Catches errors and returns
+ * mapped, LLM-friendly error responses.
+ */
+export async function safeExecute(fn, context) {
+    try {
+        return await fn();
+    }
+    catch (err) {
+        const mapped = mapError(err);
+        console.error(`[${context.tool}] ${mapped.code}`, {
+            url: context.url,
+            original: mapped.originalError,
+        });
+        return JSON.stringify({
+            success: false,
+            error: mapped.message,
+            code: mapped.code,
+            suggestions: mapped.suggestions,
+        }, null, 2);
+    }
+}
+/**
+ * Process a successful response: truncate content if needed.
+ *
+ * Replaces the old `asText()` function.
+ */
+export async function processResponse(data, options) {
+    const { result, wasTruncated } = truncateContent(data);
+    return JSON.stringify(result, null, 2);
+}
+/**
+ * Simple `asText` replacement for backward compatibility.
+ * Applies truncation only.
+ */
+export function processResponseSync(data) {
+    const { result } = truncateContent(data);
+    return JSON.stringify(result, null, 2);
+}
+// ---------------------------------------------------------------------------
+// Exports for testing
+// ---------------------------------------------------------------------------
+export { MAX_CONTENT_CHARS };

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "scorchcrawl-mcp",
-  "version": "2.1.1",
-  "description": "MCP client for ScorchCrawl — connect Copilot to a self-hosted stealth web scraping server",
+  "version": "2.2.0",
+  "description": "Open-source MCP server for web scraping with stealth bot-detection bypass.",
   "type": "module",
   "bin": {
-    "scorchcrawl-mcp": "dist/cli.js"
+    "scorchcrawl-mcp": "dist/index.js"
   },
   "files": [
     "dist"
@@ -13,18 +13,29 @@
     "access": "public"
   },
   "scripts": {
-    "build": "tsc",
-    "start": "node dist/cli.js",
-    "test": "npm run test:unit && npm run test:integration && npm run test:e2e",
-    "test:unit": "npm run build && node --test test/unit/*.test.mjs",
-    "test:integration": "npm run build && node --test test/integration/*.test.mjs",
-    "test:e2e": "npm run build && node --test test/e2e/*.test.mjs",
-    "smoke": "node test/smoke/smoke.js",
+    "build": "tsc && node -e \"require('fs').chmodSync('dist/index.js', '755')\"",
+    "start": "node dist/index.js",
+    "start:server": "HTTP_STREAMABLE_SERVER=true node dist/index.js",
+    "dev": "tsc --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:integration": "vitest run --config vitest.integration.config.ts",
+    "test:coverage": "vitest run --coverage",
+    "lint": "eslint src/**/*.ts",
+    "lint:fix": "eslint src/**/*.ts --fix",
+    "format": "prettier --write .",
     "prepare": "npm run build"
   },
   "license": "AGPL-3.0",
   "dependencies": {
-    "dotenv": "^17.2.2"
+    "@mendable/firecrawl-js": "^4.9.3",
+    "cheerio": "^1.2.0",
+    "dotenv": "^17.2.2",
+    "firecrawl-fastmcp": "^1.0.4",
+    "turndown": "^7.2.2",
+    "typescript": "^5.9.2",
+    "uuid": "^11.1.0",
+    "zod": "^4.1.5"
   },
   "engines": {
     "node": ">=18.0.0"
@@ -32,12 +43,22 @@
   "keywords": [
     "mcp",
     "scorchcrawl",
-    "copilot",
     "web-scraping",
-    "stealth"
+    "stealth",
+    "bot-detection-bypass",
+    "crawler",
+    "content-extraction"
   ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/user/scorchcrawl.git"
+  },
+  "author": "ScorchCrawl Contributors",
+  "homepage": "https://github.com/user/scorchcrawl#readme",
   "devDependencies": {
-    "@types/node": "^25.3.5",
-    "typescript": "^5.9.3"
+    "@types/node": "^22.0.0",
+    "@types/turndown": "^5.0.5",
+    "@types/uuid": "^10.0.0",
+    "vitest": "^3.2.1"
   }
 }

package/dist/cli.js

@@ -1,207 +0,0 @@
-#!/usr/bin/env node
-/**
- * scorchcrawl-mcp CLI
- *
- * A thin wrapper that connects to a ScorchCrawl server and exposes its MCP
- * tools over stdio transport. This lets MCP clients (VS Code, Copilot CLI,
- * etc.) use a remote ScorchCrawl server as if it were a local MCP server.
- *
- * Usage:
- *   SCORCHCRAWL_URL=http://localhost:24787 scorchcrawl-mcp
- *
- * Or with a remote server + API key:
- *   SCORCHCRAWL_URL=https://your-server.com/mcp-api/scorchcrawl/YOUR_KEY scorchcrawl-mcp
- *
- * Environment variables:
- *   SCORCHCRAWL_URL          - Base URL of the ScorchCrawl MCP server (required)
- *   GITHUB_TOKEN             - GitHub PAT for Copilot SDK agent (optional, sent as x-copilot-token)
- *   SCORCHCRAWL_LOCAL_PROXY  - Set to "true" to route scraping through your local IP
- */
-import { pathToFileURL } from 'url';
-async function loadDotenv() {
-    try {
-        const dotenv = await import('dotenv');
-        dotenv.config({ quiet: true });
-    }
-    catch {
-        // Allow direct source execution in constrained environments.
-    }
-}
-const DEFAULT_MCP_URL = 'http://localhost:24787';
-function trimTrailingSlash(value) {
-    return value.replace(/\/+$/, '');
-}
-function normalizeServerBaseUrl(value) {
-    const trimmed = trimTrailingSlash(value.trim());
-    if (!trimmed) {
-        return DEFAULT_MCP_URL;
-    }
-    if (trimmed.endsWith('/mcp')) {
-        return trimmed.slice(0, -4);
-    }
-    return trimmed;
-}
-function deriveLocalMcpUrl(apiUrl) {
-    const localHosts = new Set(['localhost', '127.0.0.1', '::1']);
-    if (!localHosts.has(apiUrl.hostname)) {
-        return null;
-    }
-    if (apiUrl.port === '24786') {
-        const derived = new URL(apiUrl.toString());
-        derived.port = '24787';
-        derived.pathname = '';
-        derived.search = '';
-        derived.hash = '';
-        return normalizeServerBaseUrl(derived.toString());
-    }
-    return null;
-}
-export function resolveServerConfig(env = process.env) {
-    const warnings = [];
-    const directUrl = env.SCORCHCRAWL_URL?.trim();
-    if (directUrl) {
-        return {
-            serverBaseUrl: normalizeServerBaseUrl(directUrl),
-            source: 'SCORCHCRAWL_URL',
-            warnings,
-        };
-    }
-    const apiUrlValue = env.SCORCHCRAWL_API_URL?.trim();
-    if (!apiUrlValue) {
-        return {
-            serverBaseUrl: DEFAULT_MCP_URL,
-            source: 'default',
-            warnings,
-        };
-    }
-    let parsed;
-    try {
-        parsed = new URL(apiUrlValue);
-    }
-    catch {
-        throw new Error(`Invalid SCORCHCRAWL_API_URL: ${apiUrlValue}. Set SCORCHCRAWL_URL to your MCP server URL instead.`);
-    }
-    if (parsed.port === '24787' || parsed.pathname.endsWith('/mcp')) {
-        warnings.push('SCORCHCRAWL_API_URL is being used as an MCP endpoint alias. Prefer SCORCHCRAWL_URL for the npm client.');
-        return {
-            serverBaseUrl: normalizeServerBaseUrl(apiUrlValue),
-            source: 'SCORCHCRAWL_API_URL',
-            warnings,
-        };
-    }
-    const derivedLocalUrl = deriveLocalMcpUrl(parsed);
-    if (derivedLocalUrl) {
-        warnings.push(`SCORCHCRAWL_API_URL points to the scraping engine (${apiUrlValue}). The npm client needs the MCP server, so it will use ${derivedLocalUrl} instead.`);
-        return {
-            serverBaseUrl: derivedLocalUrl,
-            source: 'SCORCHCRAWL_API_URL',
-            warnings,
-        };
-    }
-    throw new Error(`SCORCHCRAWL_API_URL points to the scraping engine (${apiUrlValue}), not the MCP server. Set SCORCHCRAWL_URL to the MCP endpoint, for example http://localhost:24787.`);
-}
-export function isLocalProxyEnabled(env = process.env) {
-    return env.SCORCHCRAWL_LOCAL_PROXY === 'true';
-}
-/**
- * Forward a JSON-RPC request to the remote ScorchCrawl server.
- */
-export async function forwardToServer(request, serverBaseUrl, githubToken) {
-    const url = `${serverBaseUrl}/mcp`;
-    const headers = {
-        'Content-Type': 'application/json',
-        'Accept': 'application/json, text/event-stream',
-    };
-    if (githubToken) {
-        headers['x-copilot-token'] = githubToken;
-    }
-    try {
-        const response = await fetch(url, {
-            method: 'POST',
-            headers,
-            body: JSON.stringify(request),
-        });
-        const text = await response.text();
-        // Handle SSE responses (event: message\ndata: {...})
-        if (text.startsWith('event:') || text.startsWith('data:')) {
-            const dataLine = text.split('\n').find(l => l.startsWith('data:'));
-            if (dataLine) {
-                return JSON.parse(dataLine.slice(5).trim());
-            }
-        }
-        // Handle direct JSON responses
-        if (text.trim().startsWith('{')) {
-            return JSON.parse(text);
-        }
-        return {
-            jsonrpc: '2.0',
-            error: { code: -32603, message: `Unexpected response: ${text.substring(0, 200)}` },
-            id: request.id,
-        };
-    }
-    catch (err) {
-        return {
-            jsonrpc: '2.0',
-            error: { code: -32603, message: `Connection failed: ${err.message}` },
-            id: request.id,
-        };
-    }
-}
-/**
- * Read JSON-RPC messages from stdin and forward to the server.
- */
-async function main() {
-    await loadDotenv();
-    const { serverBaseUrl, warnings } = resolveServerConfig(process.env);
-    const githubToken = process.env.GITHUB_TOKEN;
-    const localProxy = isLocalProxyEnabled(process.env);
-    if (!serverBaseUrl || serverBaseUrl === DEFAULT_MCP_URL) {
-        process.stderr.write(`[scorchcrawl-mcp] Connecting to ${serverBaseUrl}\n` +
-            `[scorchcrawl-mcp] Set SCORCHCRAWL_URL to change the server address\n`);
-    }
-    else {
-        process.stderr.write(`[scorchcrawl-mcp] Connecting to ${serverBaseUrl}\n`);
-    }
-    for (const warning of warnings) {
-        process.stderr.write(`[scorchcrawl-mcp] ${warning}\n`);
-    }
-    if (localProxy) {
-        process.stderr.write('[scorchcrawl-mcp] Local proxy mode: ON (scraping through your IP)\n');
-        process.stderr.write('[scorchcrawl-mcp] Note: this only works when the MCP server itself is running on this machine.\n');
-    }
-    // Read from stdin line by line
-    let buffer = '';
-    process.stdin.setEncoding('utf8');
-    process.stdin.on('data', async (chunk) => {
-        buffer += chunk;
-        // Process complete lines
-        const lines = buffer.split('\n');
-        buffer = lines.pop() || '';
-        for (const line of lines) {
-            const trimmed = line.trim();
-            if (!trimmed)
-                continue;
-            try {
-                const request = JSON.parse(trimmed);
-                const response = await forwardToServer(request, serverBaseUrl, githubToken);
-                process.stdout.write(JSON.stringify(response) + '\n');
-            }
-            catch (err) {
-                const errorResponse = {
-                    jsonrpc: '2.0',
-                    error: { code: -32700, message: `Parse error: ${err.message}` },
-                };
-                process.stdout.write(JSON.stringify(errorResponse) + '\n');
-            }
-        }
-    });
-    process.stdin.on('end', () => {
-        process.exit(0);
-    });
-}
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
-    main().catch((err) => {
-        process.stderr.write(`[scorchcrawl-mcp] Fatal error: ${err.message}\n`);
-        process.exit(1);
-    });
-}