scorchcrawl-mcp 2.1.1 → 2.3.0

package/dist/local-scraper.js

@@ -0,0 +1,319 @@
+/**
+ * Local Scraper — fetches pages through the LOCAL machine's IP and converts to markdown.
+ * Used when SCORCHCRAWL_LOCAL_PROXY=true is set, so scraping traffic exits through
+ * the user's residential IP instead of the server's datacenter IP.
+ *
+ * Falls back to the remote ScorchCrawl API for features that need server-side
+ * processing (search, crawl, extract, agent, JSON schema extraction).
+ *
+ * SPA Detection:
+ *   When the fetched HTML looks like a Single Page Application shell
+ *   (minimal text, loading indicators, heavy JS bundles), the scraper returns
+ *   a `SPA_SKELETON_DETECTED` error so the caller can retry via the engine's
+ *   Playwright-backed scraper which executes JavaScript.
+ */
+import TurndownService from 'turndown';
+import * as cheerio from 'cheerio';
+// Lazy-init singleton
+let _turndown = null;
+function getTurndown() {
+    if (!_turndown) {
+        _turndown = new TurndownService({
+            headingStyle: 'atx',
+            codeBlockStyle: 'fenced',
+            bulletListMarker: '-',
+        });
+        // Strip script/style/nav/footer tags
+        _turndown.remove(['script', 'style', 'noscript', 'iframe']);
+    }
+    return _turndown;
+}
+// ---------------------------------------------------------------------------
+// SPA / JS-rendered page detection
+// ---------------------------------------------------------------------------
+/**
+ * Common phrases found in SPA shell HTML before JS hydrates the page.
+ * Matched case-insensitively against the visible body text.
+ */
+const SPA_LOADING_PATTERNS = [
+    'loading...',
+    'loading…',
+    'please wait',
+    'just a moment',
+    'checking your browser',
+    'one moment please',
+    'redirecting',
+    'enable javascript',
+    'javascript is required',
+    'javascript must be enabled',
+    'this app requires javascript',
+    'you need to enable javascript',
+    'noscript',
+];
+/**
+ * CSS selectors whose sole presence (with no other meaningful content)
+ * strongly indicates a JS-only SPA shell.
+ */
+const SPA_ROOT_SELECTORS = [
+    '#root', // React (CRA, Vite)
+    '#app', // Vue
+    '#__next', // Next.js
+    '#__nuxt', // Nuxt
+    '#svelte', // SvelteKit
+    'app-root', // Angular
+    '#___gatsby', // Gatsby
+    '#main-app', // misc
+];
+/** Minimum characters of visible text for a page to be considered "real" content. */
+const MIN_MEANINGFUL_TEXT_LENGTH = 200;
+/** Ratio: if (script bytes / total HTML bytes) exceeds this, it's likely a SPA shell. */
+const SCRIPT_HEAVY_RATIO = 0.65;
+/**
+ * Inspect raw HTML + extracted text to decide if the page is a SPA shell
+ * that hasn't been hydrated (no JS execution happened).
+ *
+ * Returns a short reason string if SPA-like, or `null` if the page looks real.
+ */
+export function detectSPASkeleton(rawHtml, _bodyText, $) {
+    // Get visible text only (strip script, style, noscript content)
+    const $clone = cheerio.load($.html());
+    $clone('script, style, noscript').remove();
+    const visibleText = $clone('body').text() || '';
+    const trimmedText = visibleText.replace(/\s+/g, ' ').trim();
+    const lowerText = trimmedText.toLowerCase();
+    // 1. Very little visible text — likely a shell that JS would populate
+    if (trimmedText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
+        // Check for SPA root containers
+        for (const sel of SPA_ROOT_SELECTORS) {
+            const el = $(sel);
+            if (el.length > 0) {
+                const innerText = el.text().replace(/\s+/g, ' ').trim();
+                if (innerText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
+                    return `SPA root container "${sel}" with minimal content (${innerText.length} chars)`;
+                }
+            }
+        }
+        // Check for loading phrases in the sparse text
+        for (const pattern of SPA_LOADING_PATTERNS) {
+            if (lowerText.includes(pattern)) {
+                return `Loading indicator detected: "${pattern}"`;
+            }
+        }
+        // Even without a known root, < 50 chars of body text is almost certainly a shell
+        if (trimmedText.length < 50) {
+            return `Near-empty body text (${trimmedText.length} chars)`;
+        }
+    }
+    // 2. Loading phrases in an otherwise short page (< 500 chars)
+    if (trimmedText.length < 500) {
+        for (const pattern of SPA_LOADING_PATTERNS) {
+            if (lowerText.includes(pattern)) {
+                return `Short page with loading indicator: "${pattern}"`;
+            }
+        }
+    }
+    // 3. Script-heavy pages: mostly <script> tags, very little content
+    const scriptContent = $('script')
+        .toArray()
+        .reduce((sum, el) => sum + ($(el).html()?.length || 0), 0);
+    const htmlLength = rawHtml.length;
+    if (htmlLength > 1000 &&
+        scriptContent / htmlLength > SCRIPT_HEAVY_RATIO &&
+        trimmedText.length < MIN_MEANINGFUL_TEXT_LENGTH) {
+        return `Script-heavy page (${Math.round((scriptContent / htmlLength) * 100)}% scripts, ${trimmedText.length} chars text)`;
+    }
+    return null;
+}
+/**
+ * Fetches a URL locally (through the user's IP) and converts to markdown.
+ */
+export async function localScrape(url, options = {}) {
+    const timeout = options.timeout || 30000;
+    // Determine requested formats
+    const formats = (options.formats || ['markdown']).map((f) => typeof f === 'string' ? f : f.type);
+    const wantMarkdown = formats.includes('markdown');
+    const wantHtml = formats.includes('html');
+    const wantRawHtml = formats.includes('rawHtml');
+    const wantLinks = formats.includes('links');
+    // Needs JSON/screenshot/branding? Can't do locally — return null to fall back
+    const needsServerSide = formats.some((f) => f === 'json' || f === 'screenshot' || f === 'branding' || f === 'summary');
+    if (needsServerSide) {
+        return { success: false, error: 'FORMAT_NEEDS_SERVER' };
+    }
+    try {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeout);
+        const fetchOptions = {
+            signal: controller.signal,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+                'Accept-Language': 'en-US,en;q=0.9',
+                'Accept-Encoding': 'gzip, deflate, br',
+                'Cache-Control': 'no-cache',
+                ...(options.headers || {}),
+            },
+            redirect: 'follow',
+        };
+        // Node 18+ native TLS rejection control
+        if (options.skipTlsVerification) {
+            fetchOptions.dispatcher = undefined; // handled below
+            process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0';
+        }
+        const response = await fetch(url, fetchOptions);
+        clearTimeout(timer);
+        if (options.skipTlsVerification) {
+            delete process.env.NODE_TLS_REJECT_UNAUTHORIZED;
+        }
+        const rawHtml = await response.text();
+        const statusCode = response.status;
+        const contentType = response.headers.get('content-type') || 'text/html';
+        // Parse with Cheerio
+        const $ = cheerio.load(rawHtml);
+        // Extract metadata
+        const title = $('title').first().text().trim() ||
+            $('meta[property="og:title"]').attr('content') ||
+            '';
+        const description = $('meta[name="description"]').attr('content') ||
+            $('meta[property="og:description"]').attr('content') ||
+            '';
+        const language = $('html').attr('lang') || '';
+        // Remove unwanted elements if onlyMainContent
+        if (options.onlyMainContent) {
+            $('nav, header, footer, aside, .sidebar, .menu, .navigation, .breadcrumb, .cookie-banner, .ad, .advertisement, [role="navigation"], [role="banner"], [role="complementary"]').remove();
+        }
+        // Apply excludeTags
+        if (options.excludeTags?.length) {
+            $(options.excludeTags.join(', ')).remove();
+        }
+        // Get the target HTML content
+        let targetHtml;
+        if (options.includeTags?.length) {
+            targetHtml = options.includeTags
+                .map((sel) => $(sel).html() || '')
+                .filter(Boolean)
+                .join('\n');
+        }
+        else if (options.onlyMainContent) {
+            // Try to find main content area
+            const mainSelectors = [
+                'main',
+                'article',
+                '[role="main"]',
+                '.main-content',
+                '.content',
+                '#content',
+                '#main',
+            ];
+            let mainHtml = '';
+            for (const sel of mainSelectors) {
+                const el = $(sel).first();
+                if (el.length && (el.html()?.length || 0) > 100) {
+                    mainHtml = el.html() || '';
+                    break;
+                }
+            }
+            targetHtml = mainHtml || $('body').html() || rawHtml;
+        }
+        else {
+            targetHtml = $('body').html() || rawHtml;
+        }
+        // Build response
+        const data = {
+            metadata: {
+                title,
+                description: description || undefined,
+                language: language || undefined,
+                sourceURL: url,
+                url: response.url || url,
+                statusCode,
+                contentType,
+                proxyUsed: 'local',
+            },
+        };
+        if (wantMarkdown) {
+            data.markdown = getTurndown().turndown(targetHtml);
+        }
+        if (wantHtml) {
+            data.html = targetHtml;
+        }
+        if (wantRawHtml) {
+            data.rawHtml = rawHtml;
+        }
+        if (wantLinks) {
+            const links = [];
+            $('a[href]').each((_, el) => {
+                const href = $(el).attr('href');
+                if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {
+                    try {
+                        links.push(new URL(href, url).href);
+                    }
+                    catch {
+                        links.push(href);
+                    }
+                }
+            });
+            data.links = [...new Set(links)];
+        }
+        // --- SPA detection: check if the fetched content is a JS-only shell ---
+        const bodyText = $('body').text() || '';
+        const spaReason = detectSPASkeleton(rawHtml, bodyText, $);
+        if (spaReason) {
+            return { success: false, error: 'SPA_SKELETON_DETECTED', data };
+        }
+        return { success: true, data };
+    }
+    catch (err) {
+        if (err.name === 'AbortError') {
+            return { success: false, error: `Timeout after ${timeout}ms` };
+        }
+        return { success: false, error: err.message || String(err) };
+    }
+}
+/**
+ * Check if local proxy mode is enabled.
+ * Controlled by:
+ *   - SCORCHCRAWL_LOCAL_PROXY=true env var
+ *   - ?localProxy=true query param in SCORCHCRAWL_API_URL
+ */
+export function isLocalProxyEnabled() {
+    // Check env var
+    if (process.env.SCORCHCRAWL_LOCAL_PROXY?.toLowerCase() === 'true' ||
+        process.env.SCORCHCRAWL_LOCAL_PROXY === '1') {
+        return true;
+    }
+    // Check URL query param
+    const apiUrl = process.env.SCORCHCRAWL_API_URL;
+    if (apiUrl) {
+        try {
+            const parsed = new URL(apiUrl);
+            if (parsed.searchParams.get('localProxy') === 'true' ||
+                parsed.searchParams.get('localProxy') === '1') {
+                return true;
+            }
+        }
+        catch {
+            // ignore invalid URL
+        }
+    }
+    return false;
+}
+/**
+ * Returns SCORCHCRAWL_API_URL without the localProxy query param
+ * (so the scraping SDK doesn't pass it to the API).
+ */
+export function getCleanApiUrl() {
+    const apiUrl = process.env.SCORCHCRAWL_API_URL;
+    if (!apiUrl)
+        return undefined;
+    try {
+        const parsed = new URL(apiUrl);
+        parsed.searchParams.delete('localProxy');
+        const cleaned = parsed.toString();
+        // Remove trailing ? if no other params
+        return cleaned.replace(/\?$/, '');
+    }
+    catch {
+        return apiUrl;
+    }
+}

package/dist/response-utils.js

@@ -0,0 +1,375 @@
+/**
+ * Response Processing Utilities
+ *
+ * Two layers of intelligent response handling for MCP tool results:
+ *   1. Error Mapping — classifies errors and returns actionable guidance
+ *   2. Content Truncation — smart paragraph-boundary truncation with notices
+ *
+ * No competitor MCP scraper implements any of these. This is a ScorchCrawl
+ * differentiator.
+ */
+// ---------------------------------------------------------------------------
+// Configuration (from environment)
+// ---------------------------------------------------------------------------
+const MAX_CONTENT_CHARS = parseInt(process.env.SCORCHCRAWL_MAX_CONTENT_CHARS || '25000', 10);
+/**
+ * Classify a raw error (from fetch, SDK, or engine) into a structured
+ * MappedError with actionable guidance for the LLM.
+ */
+export function mapError(err) {
+    const raw = normalizeErrorString(err);
+    const lower = raw.toLowerCase();
+    // --- HTTP status-based patterns ---
+    if (matches(lower, ['403', 'forbidden', 'cloudflare', 'cf-ray', 'challenge', 'access denied'])) {
+        return {
+            code: 'ACCESS_DENIED',
+            message: 'This site blocks automated access.',
+            suggestions: [
+                'Try scorch_search to find cached/indexed content instead',
+                'Add waitFor: 5000 to let challenge pages resolve',
+                'Try with proxy: "stealth" for enhanced bot evasion',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['404', 'not found', 'page not found', 'does not exist'])) {
+        return {
+            code: 'NOT_FOUND',
+            message: 'Page does not exist at this URL.',
+            suggestions: [
+                'Use scorch_map to discover correct URLs on the site',
+                'Check if the URL has a typo or outdated path',
+                'Try scorch_search to find the current location of this content',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['429', 'rate limit', 'too many requests', 'throttl'])) {
+        return {
+            code: 'RATE_LIMITED',
+            message: 'Rate limited by the target site.',
+            suggestions: [
+                'Wait 30 seconds and retry',
+                'Try a different URL on the same site',
+                'Use scorch_search to find the content from a different source',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matchesStatusRange(lower, 500, 599)) {
+        return {
+            code: 'SERVER_ERROR',
+            message: 'The target site returned a server error.',
+            suggestions: [
+                'Retry once — the site may be experiencing temporary issues',
+                'Try again in a few minutes if it persists',
+            ],
+            originalError: raw,
+        };
+    }
+    // --- Network/connection errors ---
+    if (matches(lower, ['timeout', 'aborterror', 'aborted', 'timed out', 'etimedout'])) {
+        return {
+            code: 'TIMEOUT',
+            message: 'Page took too long to load.',
+            suggestions: [
+                'Add onlyMainContent: true to reduce processing time',
+                'Increase waitFor to give JS more time to render',
+                'Try with a simpler format like markdown instead of screenshot',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['econnrefused', 'enotfound', 'dns', 'getaddrinfo', 'connect failed'])) {
+        // Distinguish between engine-down and remote-site-down
+        if (matches(lower, ['localhost', '127.0.0.1', '0.0.0.0', '3002', 'engine', 'api'])) {
+            return {
+                code: 'ENGINE_UNAVAILABLE',
+                message: 'Scraping engine is not running.',
+                suggestions: [
+                    'Check that Docker services are up: docker compose up -d',
+                    'Verify SCORCHCRAWL_API_URL is correct in your config',
+                ],
+                originalError: raw,
+            };
+        }
+        return {
+            code: 'CONNECTION_FAILED',
+            message: 'Cannot reach this URL.',
+            suggestions: [
+                'Check if the URL is correct and accessible',
+                'The site may be down — try again later',
+                'Try scorch_search to find alternative sources',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['ssl', 'tls', 'certificate', 'cert', 'self-signed', 'unable to verify'])) {
+        return {
+            code: 'TLS_ERROR',
+            message: 'SSL certificate issue with this site.',
+            suggestions: [
+                'Try with skipTlsVerification: true',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['empty', 'no content', 'no markdown', 'no extractable'])) {
+        return {
+            code: 'EMPTY_CONTENT',
+            message: 'Page returned no extractable content.',
+            suggestions: [
+                'Use scorch_map with a search param to find the right page',
+                'Add waitFor: 5000 for JavaScript-rendered pages',
+                'Try with formats: ["html"] to see the raw page structure',
+            ],
+            originalError: raw,
+        };
+    }
+    if (matches(lower, ['spa_skeleton_detected', 'spa detected', 'javascript is required'])) {
+        return {
+            code: 'SPA_DETECTED',
+            message: 'Page requires JavaScript rendering (SPA detected).',
+            suggestions: [
+                'Add waitFor: 5000 to allow JavaScript to render',
+                'The engine will automatically retry with Playwright',
+            ],
+            originalError: raw,
+        };
+    }
+    // --- Catch-all ---
+    return {
+        code: 'UNKNOWN_ERROR',
+        message: `Scraping failed: ${truncateString(raw, 200)}`,
+        suggestions: [
+            'Try a different URL or format',
+            'Use scorch_search as an alternative data source',
+        ],
+        originalError: raw,
+    };
+}
+/** Normalize any error-like value to a string */
+function normalizeErrorString(err) {
+    if (err == null)
+        return 'Unknown error';
+    if (err instanceof Error) {
+        // Include status code from Axios/fetch errors
+        const axiosStatus = err?.response?.status;
+        const status = err?.status || err?.statusCode || axiosStatus;
+        const prefix = status ? `HTTP ${status}: ` : '';
+        return `${prefix}${err.message}`;
+    }
+    if (typeof err === 'string')
+        return err;
+    try {
+        const s = JSON.stringify(err);
+        return s || 'Unknown error';
+    }
+    catch {
+        return String(err);
+    }
+}
+/** Check if a lowered string contains any of the patterns */
+function matches(lower, patterns) {
+    return patterns.some((p) => lower.includes(p));
+}
+/** Check if an error string refers to an HTTP status in a range */
+function matchesStatusRange(lower, from, to) {
+    const statusMatch = lower.match(/(?:status\s*(?:code\s*)?|http\s*)(\d{3})/);
+    if (statusMatch) {
+        const code = parseInt(statusMatch[1], 10);
+        return code >= from && code <= to;
+    }
+    // Also check for bare 5xx patterns
+    for (let s = from; s <= to; s++) {
+        if (lower.includes(String(s)))
+            return true;
+    }
+    return false;
+}
+/** Safely truncate a string for display */
+function truncateString(s, maxLen) {
+    if (s.length <= maxLen)
+        return s;
+    return s.slice(0, maxLen - 3) + '...';
+}
+// ---------------------------------------------------------------------------
+// Feature 2: Content Truncation
+// ---------------------------------------------------------------------------
+/**
+ * Content fields that may need truncation.
+ * Metadata, links, and structured data are NEVER truncated.
+ */
+const CONTENT_FIELDS = ['markdown', 'html', 'rawHtml'];
+/**
+ * For crawl results: max fraction of total limit a single page can use.
+ */
+const SINGLE_PAGE_MAX_FRACTION = 0.3;
+/**
+ * Truncate a markdown/html string at the nearest paragraph or heading
+ * boundary before the character limit.
+ */
+export function truncateAtBoundary(content, maxChars) {
+    if (content.length <= maxChars) {
+        return { text: content, wasTruncated: false };
+    }
+    // Search backward from maxChars for a good break point
+    const searchWindow = content.slice(0, maxChars);
+    // Priority 1: Last heading boundary (# at start of line)
+    const headingMatch = searchWindow.lastIndexOf('\n#');
+    // Priority 2: Last double newline (paragraph boundary)
+    const paraMatch = searchWindow.lastIndexOf('\n\n');
+    // Priority 3: Last single newline
+    const lineMatch = searchWindow.lastIndexOf('\n');
+    // Priority 4: Last sentence end
+    const sentenceMatch = Math.max(searchWindow.lastIndexOf('. '), searchWindow.lastIndexOf('.\n'));
+    // Pick the best break point (prefer heading > paragraph > line > sentence)
+    let breakPoint = -1;
+    // Only use if it's in the last 30% of the window (don't cut too aggressively)
+    const minBreak = Math.floor(maxChars * 0.7);
+    if (headingMatch > minBreak)
+        breakPoint = headingMatch;
+    else if (paraMatch > minBreak)
+        breakPoint = paraMatch;
+    else if (lineMatch > minBreak)
+        breakPoint = lineMatch;
+    else if (sentenceMatch > minBreak)
+        breakPoint = sentenceMatch + 1; // include the period
+    else
+        breakPoint = maxChars; // hard cut as last resort
+    const truncated = content.slice(0, breakPoint).trimEnd();
+    return { text: truncated, wasTruncated: true };
+}
+/**
+ * Process a single scrape/crawl data object, truncating content fields
+ * while preserving metadata and structured data.
+ *
+ * Returns the processed data and truncation metadata.
+ */
+export function truncateContent(data) {
+    if (MAX_CONTENT_CHARS <= 0) {
+        const serialized = JSON.stringify(data, null, 2);
+        return { result: data, wasTruncated: false, originalLength: serialized.length };
+    }
+    const serialized = JSON.stringify(data, null, 2);
+    if (serialized.length <= MAX_CONTENT_CHARS) {
+        return { result: data, wasTruncated: false, originalLength: serialized.length };
+    }
+    // Deep clone to avoid mutating the original
+    const cloned = JSON.parse(serialized);
+    let wasTruncated = false;
+    // Handle single scrape result (has data.markdown, data.html, etc.)
+    if (cloned && typeof cloned === 'object') {
+        const target = cloned.data || cloned;
+        for (const field of CONTENT_FIELDS) {
+            if (typeof target[field] === 'string' && target[field].length > 0) {
+                // Budget for this field: total limit minus overhead from other fields
+                const otherFieldsSize = serialized.length - target[field].length;
+                const fieldBudget = Math.max(MAX_CONTENT_CHARS - otherFieldsSize, Math.floor(MAX_CONTENT_CHARS * 0.5));
+                if (target[field].length > fieldBudget) {
+                    const { text } = truncateAtBoundary(target[field], fieldBudget);
+                    const originalLen = target[field].length;
+                    target[field] = text + buildTruncationNotice(text.length, originalLen);
+                    wasTruncated = true;
+                }
+            }
+        }
+        // Handle crawl results (array of pages)
+        if (Array.isArray(target)) {
+            const result = truncateCrawlArray(target);
+            return { result: cloned, wasTruncated: result.wasTruncated, originalLength: serialized.length };
+        }
+        // Add truncation metadata
+        if (wasTruncated) {
+            const meta = target.metadata || target;
+            meta._truncated = true;
+            meta._originalLength = serialized.length;
+        }
+    }
+    return { result: cloned, wasTruncated, originalLength: serialized.length };
+}
+/**
+ * Truncate a crawl result array: limit per-page content and total page count.
+ */
+function truncateCrawlArray(pages) {
+    if (pages.length === 0)
+        return { wasTruncated: false };
+    const perPageLimit = Math.floor(MAX_CONTENT_CHARS * SINGLE_PAGE_MAX_FRACTION);
+    let wasTruncated = false;
+    let totalSize = 0;
+    for (let i = 0; i < pages.length; i++) {
+        const page = pages[i];
+        if (!page || typeof page !== 'object')
+            continue;
+        const target = page.data || page;
+        for (const field of CONTENT_FIELDS) {
+            if (typeof target[field] === 'string' && target[field].length > perPageLimit) {
+                const { text } = truncateAtBoundary(target[field], perPageLimit);
+                target[field] = text + buildTruncationNotice(text.length, target[field].length);
+                wasTruncated = true;
+            }
+        }
+        totalSize += JSON.stringify(page).length;
+        // If total exceeds limit, truncate the array
+        if (totalSize > MAX_CONTENT_CHARS && i < pages.length - 1) {
+            const originalCount = pages.length;
+            pages.length = i + 1;
+            pages.push({
+                _notice: `Showing ${i + 1} of ${originalCount} crawled pages. Use scorch_check_crawl_status with pagination to see more.`,
+            });
+            wasTruncated = true;
+            break;
+        }
+    }
+    return { wasTruncated };
+}
+/** Build a truncation notice for appending to content */
+function buildTruncationNotice(shownChars, originalChars) {
+    return `\n\n---\n[Content truncated. Showing ~${Math.round(shownChars / 1000)}k of ~${Math.round(originalChars / 1000)}k characters.\nTo get specific information, try:\n- Use JSON format with a schema to extract only the data you need\n- Add onlyMainContent: true to exclude navigation/footers\n- Use scorch_map to find a more specific page URL]`;
+}
+// ---------------------------------------------------------------------------
+// Combined Processing Pipeline
+// ---------------------------------------------------------------------------
+/**
+ * Safe wrapper for tool execution. Catches errors and returns
+ * mapped, LLM-friendly error responses.
+ */
+export async function safeExecute(fn, context) {
+    try {
+        return await fn();
+    }
+    catch (err) {
+        const mapped = mapError(err);
+        console.error(`[${context.tool}] ${mapped.code}`, {
+            url: context.url,
+            original: mapped.originalError,
+        });
+        return JSON.stringify({
+            success: false,
+            error: mapped.message,
+            code: mapped.code,
+            suggestions: mapped.suggestions,
+        }, null, 2);
+    }
+}
+/**
+ * Process a successful response: truncate content if needed.
+ *
+ * Replaces the old `asText()` function.
+ */
+export async function processResponse(data, options) {
+    const { result, wasTruncated } = truncateContent(data);
+    return JSON.stringify(result, null, 2);
+}
+/**
+ * Simple `asText` replacement for backward compatibility.
+ * Applies truncation only.
+ */
+export function processResponseSync(data) {
+    const { result } = truncateContent(data);
+    return JSON.stringify(result, null, 2);
+}
+// ---------------------------------------------------------------------------
+// Exports for testing
+// ---------------------------------------------------------------------------
+export { MAX_CONTENT_CHARS };