scorchcrawl-mcp 1.2.0 → 2.0.0

package/dist/response-utils.js

@@ -26,19 +26,6 @@ const SUMMARIZE_TIMEOUT_MS = parseInt(process.env.SCORCHCRAWL_SUMMARIZE_TIMEOUT_
 export function mapError(err) {
     const raw = normalizeErrorString(err);
     const lower = raw.toLowerCase();
-    // --- CAPTCHA (checked before ACCESS_DENIED since 'challenge' overlaps) ---
-    if (matches(lower, ['captcha', 'recaptcha', 'hcaptcha', 'turnstile'])) {
-        return {
-            code: 'CAPTCHA_REQUIRED',
-            message: 'Page requires CAPTCHA verification.',
-            suggestions: [
-                'Try proxy: "stealth" to avoid CAPTCHA triggers',
-                'Use scorch_agent which may navigate around CAPTCHAs',
-                'Try scorch_search to find the content from a cached source',
-            ],
-            originalError: raw,
-        };
-    }
     // --- HTTP status-based patterns ---
     if (matches(lower, ['403', 'forbidden', 'cloudflare', 'cf-ray', 'challenge', 'access denied'])) {
         return {
@@ -157,28 +144,6 @@ export function mapError(err) {
             originalError: raw,
         };
     }
-    if (matches(lower, ['geo', 'geoblocked', 'not available in your', 'region', 'country restriction'])) {
-        return {
-            code: 'GEO_BLOCKED',
-            message: 'Content is geo-restricted.',
-            suggestions: [
-                'Use location parameter with a different country: location: { country: "US" }',
-                'Try proxy: "stealth" which may route through a different region',
-            ],
-            originalError: raw,
-        };
-    }
-    if (matches(lower, ['paywall', 'subscribe', 'premium content', 'members only'])) {
-        return {
-            code: 'PAYWALL',
-            message: 'Content is behind a paywall.',
-            suggestions: [
-                'Try scorch_search to find freely available alternatives',
-                'Use scorch_map to find non-paywalled pages on the same site',
-            ],
-            originalError: raw,
-        };
-    }
     // --- Catch-all ---
     return {
         code: 'UNKNOWN_ERROR',
@@ -676,114 +641,3 @@ function injectMarkdown(data, newMarkdown, meta) {
 // ---------------------------------------------------------------------------
 export { MAX_CONTENT_CHARS, SUMMARIZE_AFTER_WORDS, SUMMARIZE_MODEL };
 export { summaryCache, summarizationTimestamps };
-// ---------------------------------------------------------------------------
-// Feature 4: Response Quality Scoring
-// ---------------------------------------------------------------------------
-/** Known anti-bot / blocked-page signatures */
-const ANTI_BOT_SIGNATURES = [
-    // Cloudflare
-    'checking your browser', 'cf-browser-verification', 'just a moment',
-    'cf-challenge-running', 'cf_chl_opt', 'ray id:',
-    // DataDome
-    'datadome', 'dd.js',
-    // PerimeterX
-    'perimeterx', 'px-captcha', 'human challenge',
-    // Incapsula / Imperva
-    'incapsula', 'imperva', '_incapsula_resource',
-    // Generic
-    'access denied', 'bot detected', 'automated access',
-    'please verify you are a human', 'captcha',
-    // Akamai
-    'akamai', 'ak_bmsc',
-    // Shape Security
-    'shape security',
-];
-/**
- * Score the quality of a scrape response (0-100).
- * Detects anti-bot pages, empty/minimal content, and common failure patterns.
- */
-export function scoreResponseQuality(data) {
-    const issues = [];
-    let score = 100;
-    let antiBotDetected = false;
-    let antiBotSystem;
-    if (!data || typeof data !== 'object') {
-        return { score: 0, antiBotDetected: false, issues: ['No response data'] };
-    }
-    const obj = data;
-    const inner = (obj.data && typeof obj.data === 'object' ? obj.data : obj);
-    // Check for explicit failure
-    if (obj.success === false) {
-        score -= 50;
-        issues.push('Response indicates failure');
-    }
-    // Extract text content for analysis
-    const md = (typeof inner.markdown === 'string' ? inner.markdown : '');
-    const html = (typeof inner.html === 'string' ? inner.html : typeof inner.rawHtml === 'string' ? inner.rawHtml : '');
-    const content = md || html;
-    const lower = content.toLowerCase();
-    // --- Anti-bot detection ---
-    for (const sig of ANTI_BOT_SIGNATURES) {
-        if (lower.includes(sig)) {
-            antiBotDetected = true;
-            // Identify the specific system
-            if (sig.includes('cf-') || sig.includes('cloudflare') || sig === 'just a moment' || sig === 'checking your browser' || sig === 'ray id:') {
-                antiBotSystem = 'Cloudflare';
-            }
-            else if (sig.includes('datadome')) {
-                antiBotSystem = 'DataDome';
-            }
-            else if (sig.includes('perimeterx') || sig.includes('px-captcha')) {
-                antiBotSystem = 'PerimeterX';
-            }
-            else if (sig.includes('incapsula') || sig.includes('imperva')) {
-                antiBotSystem = 'Imperva/Incapsula';
-            }
-            else if (sig.includes('akamai') || sig.includes('ak_bmsc')) {
-                antiBotSystem = 'Akamai';
-            }
-            else if (sig.includes('captcha')) {
-                antiBotSystem = 'CAPTCHA';
-            }
-            else {
-                antiBotSystem = 'Unknown';
-            }
-            score -= 40;
-            issues.push(`Anti-bot protection detected: ${antiBotSystem}`);
-            break;
-        }
-    }
-    // --- Content length analysis ---
-    if (content.length === 0) {
-        score -= 30;
-        issues.push('No content extracted');
-    }
-    else if (content.length < 100) {
-        score -= 20;
-        issues.push('Very minimal content (< 100 chars)');
-    }
-    else if (content.length < 500) {
-        score -= 10;
-        issues.push('Thin content (< 500 chars)');
-    }
-    // --- Navigation-only content detection (SPA shells) ---
-    const navPatterns = ['sign in', 'log in', 'menu', 'navigation', 'footer', 'header'];
-    const navCount = navPatterns.filter(p => lower.includes(p)).length;
-    const contentWords = content.split(/\s+/).length;
-    if (contentWords < 50 && navCount >= 3) {
-        score -= 20;
-        issues.push('Content appears to be navigation-only (possible SPA shell)');
-    }
-    // --- Error page detection ---
-    const errorPagePatterns = ['page not found', '404', 'error occurred', 'something went wrong', 'under maintenance'];
-    if (errorPagePatterns.some(p => lower.includes(p)) && contentWords < 200) {
-        score -= 15;
-        issues.push('Content appears to be an error page');
-    }
-    return {
-        score: Math.max(0, Math.min(100, score)),
-        antiBotDetected,
-        ...(antiBotSystem && { antiBotSystem }),
-        issues,
-    };
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scorchcrawl-mcp",
-  "version": "1.2.0",
+  "version": "2.0.0",
   "description": "Open-source Copilot SDK-compliant MCP server for web scraping with stealth bot-detection bypass.",
   "type": "module",
   "bin": {
@@ -13,8 +13,7 @@
     "access": "public"
   },
   "scripts": {
-    "build": "tsc --noEmit && esbuild src/index.ts --bundle --platform=node --target=node18 --format=esm --outfile=dist/index.js --external:effect --external:@valibot/to-json-schema --external:sury --minify --keep-names --banner:js='#!/usr/bin/env node\nimport{createRequire as __cr}from\"module\";const require=__cr(import.meta.url);' && chmod 755 dist/index.js",
-    "build:tsc": "tsc",
+    "build": "tsc && node -e \"require('fs').chmodSync('dist/index.js', '755')\"",
     "start": "node dist/index.js",
     "start:server": "HTTP_STREAMABLE_SERVER=true node dist/index.js",
     "dev": "tsc --watch",
@@ -63,7 +62,6 @@
     "@types/node": "^22.0.0",
     "@types/turndown": "^5.0.5",
     "@types/uuid": "^10.0.0",
-    "esbuild": "^0.25.0",
     "vitest": "^3.2.1"
   }
 }