spectrawl 0.4.2 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "The unified web layer for AI agents. Search (8 engines), stealth browse, auth, act on 24 platforms. Self-hosted.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/browse/index.js

@@ -127,12 +127,21 @@ class BrowseEngine {
 
       await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 })
 
-      // Human-like delays
-      await page.waitForTimeout(800 + Math.random() * 1500)
-      await page.evaluate(() => {
-        window.scrollBy({ top: Math.floor(Math.random() * 400) + 100, behavior: 'smooth' })
-      })
-      await page.waitForTimeout(300 + Math.random() * 700)
+      if (opts.fastMode) {
+        // Crawl mode: minimal delays, just enough for lazy-load triggers
+        await page.waitForTimeout(400)
+        await page.evaluate(() => {
+          window.scrollBy({ top: 500, behavior: 'instant' })
+        })
+        await page.waitForTimeout(200)
+      } else {
+        // Normal browse: full human-like delays
+        await page.waitForTimeout(800 + Math.random() * 1500)
+        await page.evaluate(() => {
+          window.scrollBy({ top: Math.floor(Math.random() * 400) + 100, behavior: 'smooth' })
+        })
+        await page.waitForTimeout(300 + Math.random() * 700)
+      }
 
       const result = {}
 
@@ -155,9 +164,18 @@ class BrowseEngine {
 
       result.url = page.url()
       result.title = await page.title()
+      result.statusCode = null // playwright doesn't expose easily, but we detect blocks below
       result.cached = false
       result.engine = this._engine
 
+      // Detect block pages (Cloudflare, Akamai, etc.)
+      const blockInfo = detectBlockPage(result.content, result.title, result.html, result.url)
+      if (blockInfo) {
+        result.blocked = true
+        result.blockType = blockInfo.type
+        result.blockDetail = blockInfo.detail
+      }
+
       if (!opts.screenshot) {
         this.cache?.set('scrape', url, { content: result.content, url: result.url, title: result.title })
       }
@@ -279,4 +297,81 @@ class BrowseEngine {
   }
 }
 
+/**
+ * Detect block/challenge pages from CDNs and bot protection services.
+ * Returns { type, detail } if blocked, null if clean.
+ */
+function detectBlockPage(content, title, html, url) {
+  const text = (content || '').toLowerCase()
+  const titleLower = (title || '').toLowerCase()
+  const htmlLower = (html || '').toLowerCase()
+
+  // Cloudflare
+  if (htmlLower.includes('cf-error-details') || htmlLower.includes('cf_chl_opt') ||
+      text.includes('attention required') && text.includes('cloudflare') ||
+      text.includes('checking if the site connection is secure') ||
+      titleLower.includes('just a moment') && htmlLower.includes('cloudflare') ||
+      text.includes('ray id:') && text.includes('cloudflare')) {
+    return { type: 'cloudflare', detail: 'Cloudflare bot challenge or block page detected' }
+  }
+
+  // Cloudflare RFC 9457 structured error (new format)
+  if (htmlLower.includes('application/problem+json') || 
+      text.includes('error 1') && text.includes('cloudflare') ||
+      htmlLower.includes('"type":') && htmlLower.includes('cloudflare.com/errors/')) {
+    return { type: 'cloudflare-rfc9457', detail: 'Cloudflare structured error response (RFC 9457)' }
+  }
+
+  // Akamai
+  if (text.includes('access denied') && htmlLower.includes('akamai') ||
+      htmlLower.includes('akamaighost') ||
+      text.includes('reference #') && text.includes('access denied')) {
+    return { type: 'akamai', detail: 'Akamai bot detection triggered' }
+  }
+
+  // AWS WAF
+  if (text.includes('request blocked') && htmlLower.includes('aws') ||
+      htmlLower.includes('awswaf')) {
+    return { type: 'aws-waf', detail: 'AWS WAF blocked the request' }
+  }
+
+  // Imperva / Incapsula
+  if (htmlLower.includes('incapsula') || htmlLower.includes('imperva') ||
+      text.includes('request unsuccessful') && text.includes('incapsula')) {
+    return { type: 'imperva', detail: 'Imperva/Incapsula bot detection triggered' }
+  }
+
+  // DataDome
+  if (htmlLower.includes('datadome') || htmlLower.includes('dd.js')) {
+    return { type: 'datadome', detail: 'DataDome bot detection triggered' }
+  }
+
+  // PerimeterX / HUMAN
+  if (htmlLower.includes('perimeterx') || htmlLower.includes('px-captcha') ||
+      htmlLower.includes('human security')) {
+    return { type: 'perimeterx', detail: 'PerimeterX/HUMAN bot detection triggered' }
+  }
+
+  // hCaptcha challenge
+  if (htmlLower.includes('hcaptcha.com') && htmlLower.includes('h-captcha')) {
+    return { type: 'hcaptcha', detail: 'hCaptcha challenge page' }
+  }
+
+  // reCAPTCHA challenge (standalone, not embedded)
+  if (htmlLower.includes('recaptcha') && text.length < 500 &&
+      (titleLower === '' || titleLower.includes('blocked') || titleLower.includes('verify'))) {
+    return { type: 'recaptcha', detail: 'reCAPTCHA challenge page' }
+  }
+
+  // Generic bot detection signals
+  if (text.length < 200 && (
+      text.includes('access denied') || text.includes('403 forbidden') ||
+      text.includes('bot detected') || text.includes('automated access') ||
+      text.includes('please verify you are human') || text.includes('are you a robot'))) {
+    return { type: 'generic', detail: 'Generic bot detection or access denied page' }
+  }
+
+  return null
+}
+
 module.exports = { BrowseEngine }

package/src/crawl.js

@@ -260,8 +260,10 @@ class CrawlEngine {
     const totalMB = Math.floor(os.totalmem() / 1024 / 1024)
     const freeMB = Math.floor(os.freemem() / 1024 / 1024)
     const concurrency = detectConcurrency()
-    // Estimate: each page takes ~4s with stealth delays
-    const pagesPerMinute = concurrency * 15  // ~4s per page
+    // Realistic: ~0.8s per page with fast mode, limited by shared browser pipeline
+    // Concurrency helps but not linearly — shared browser bottleneck
+    const effectiveConcurrency = Math.min(concurrency, 5) // diminishing returns past 5
+    const pagesPerMinute = Math.floor(effectiveConcurrency * 30)  // ~2s effective per page with overhead
     return {
       totalRamMB: totalMB,
       freeRamMB: freeMB,
@@ -274,14 +276,32 @@ class CrawlEngine {
 
   async _fetchPage(url, config, cookies) {
     try {
-      const result = await this.browseEngine.browse(url, {
+      const browseOpts = {
         stealth: config.stealth,
         _cookies: cookies,
         timeout: config.timeout,
         html: true,
-        noCache: true
-      })
+        noCache: true,
+        fastMode: true
+      }
+      let result = await this.browseEngine.browse(url, browseOpts)
+
+      // Auto-retry with full stealth if blocked
+      if (result?.blocked && browseOpts.fastMode) {
+        console.log(`[crawl] Block detected on ${url} (${result.blockType}) — retrying with full stealth`)
+        result = await this.browseEngine.browse(url, {
+          ...browseOpts,
+          fastMode: false,
+          stealth: true,
+          camoufox: true
+        })
+      }
+
       if (result?.content) {
+        // Skip if still blocked after retry
+        if (result.blocked) {
+          throw new Error(`Blocked by ${result.blockType}: ${result.blockDetail}`)
+        }
         const linkSource = result.html || result.content
         return {
           title: result.title || '',

package/src/server.js

@@ -159,7 +159,15 @@ const server = http.createServer(async (req, res) => {
     return error(res, 404, 'Not found')
   } catch (err) {
     console.error('Server error:', err)
-    return error(res, 500, err.message)
+    const status = err.statusCode || 500
+    const extra = {}
+    if (err.retryable) extra.retryable = true
+    if (err.suggestion) extra.suggestion = err.suggestion
+    if (err.blocked) {
+      extra.retryable = true
+      extra.suggestion = 'Retry with stealth:true or use Camoufox engine'
+    }
+    return error(res, status, err.message, extra)
   }
 })
 
@@ -168,8 +176,29 @@ function json(res, data, status = 200) {
   res.end(JSON.stringify(data))
 }
 
-function error(res, status, message) {
-  json(res, { error: message }, status)
+/**
+ * RFC 9457-style structured error responses.
+ * Machine-readable for AI agents consuming our API.
+ */
+function error(res, status, message, extra = {}) {
+  const errorTypes = {
+    400: 'bad-request',
+    401: 'unauthorized',
+    403: 'forbidden',
+    404: 'not-found',
+    429: 'rate-limited',
+    500: 'internal-error',
+    502: 'upstream-error',
+    503: 'service-unavailable'
+  }
+  const body = {
+    type: `https://spectrawl.dev/errors/${errorTypes[status] || 'unknown'}`,
+    status,
+    title: errorTypes[status] ? errorTypes[status].replace(/-/g, ' ') : 'error',
+    detail: message,
+    ...extra
+  }
+  json(res, body, status)
 }
 
 function readBody(req) {