spectrawl 0.3.14 → 0.3.16

package/index.d.ts

@@ -17,6 +17,7 @@ declare module 'spectrawl' {
       defaultEngine?: string
       proxy?: { type: string; host: string; port: number; username?: string; password?: string }
       humanlike?: { minDelay?: number; maxDelay?: number; scrollBehavior?: boolean }
+      captcha?: { apiKey?: string; model?: string }
     }
     auth?: {
       refreshInterval?: string

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.3.14",
+  "version": "0.3.16",
   "description": "The unified web layer for AI agents. Search (6 engines), stealth browse (Camoufox + Playwright), auth (cookies, multi-account), act (24 adapters, 30+ platforms), proxy rotation. Self-hosted, free.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/browse/captcha-solver.js

@@ -0,0 +1,162 @@
+const https = require('https')
+const fs = require('fs')
+
+/**
+ * CAPTCHA solver using Gemini Vision.
+ * Free tier: 1,500 req/day (gemini-2.0-flash).
+ * 
+ * Handles: image CAPTCHAs, text CAPTCHAs, simple challenges.
+ * Does NOT handle: reCAPTCHA v2/v3, hCaptcha, Cloudflare Turnstile
+ * (those require token solving services like 2captcha).
+ * 
+ * Strategy: Playwright stealth bypasses most CAPTCHAs.
+ * This is the fallback when a visual CAPTCHA appears.
+ */
+
+class CaptchaSolver {
+  constructor(config = {}) {
+    this.apiKey = config.apiKey || process.env.GEMINI_API_KEY
+    this.model = config.model || 'gemini-2.0-flash'
+  }
+
+  /**
+   * Detect if a page has a CAPTCHA challenge.
+   * Returns { hasCaptcha, type, selector } or null.
+   */
+  async detect(page) {
+    return page.evaluate(() => {
+      // Check for common CAPTCHA indicators
+      const indicators = [
+        // reCAPTCHA
+        { selector: '.g-recaptcha, #recaptcha, [data-sitekey]', type: 'recaptcha' },
+        // hCaptcha
+        { selector: '.h-captcha, [data-hcaptcha-sitekey]', type: 'hcaptcha' },
+        // Cloudflare Turnstile
+        { selector: '.cf-turnstile, [data-turnstile-sitekey]', type: 'turnstile' },
+        // Image CAPTCHA (solvable with vision)
+        { selector: 'img[src*="captcha"], img[alt*="captcha"], .captcha-image', type: 'image' },
+        // Text/math CAPTCHA
+        { selector: '[class*="captcha"] input, #captcha-input', type: 'text' },
+      ]
+
+      for (const { selector, type } of indicators) {
+        const el = document.querySelector(selector)
+        if (el) return { hasCaptcha: true, type, selector }
+      }
+
+      // Check page text for CAPTCHA mentions
+      const bodyText = document.body?.innerText?.toLowerCase() || ''
+      if (bodyText.includes('verify you are human') || bodyText.includes('complete the captcha')) {
+        return { hasCaptcha: true, type: 'unknown', selector: null }
+      }
+
+      return { hasCaptcha: false, type: null, selector: null }
+    })
+  }
+
+  /**
+   * Attempt to solve a visual CAPTCHA using Gemini Vision.
+   * Takes a screenshot of the CAPTCHA element, sends to Gemini, returns answer.
+   */
+  async solveImage(page, captchaSelector) {
+    if (!this.apiKey) {
+      throw new Error('GEMINI_API_KEY required for CAPTCHA solving')
+    }
+
+    // Screenshot the CAPTCHA element
+    const element = await page.$(captchaSelector)
+    if (!element) throw new Error(`CAPTCHA element not found: ${captchaSelector}`)
+
+    const screenshot = await element.screenshot({ type: 'png' })
+    const base64 = screenshot.toString('base64')
+
+    // Ask Gemini to solve it
+    const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`
+    const body = JSON.stringify({
+      contents: [{
+        parts: [
+          { text: 'This is a CAPTCHA image. What text, numbers, or answer does it show? Reply with ONLY the answer, nothing else.' },
+          { inline_data: { mime_type: 'image/png', data: base64 } }
+        ]
+      }],
+      generationConfig: { temperature: 0, maxOutputTokens: 100 }
+    })
+
+    const data = await this._post(url, body)
+    const answer = data.candidates?.[0]?.content?.parts?.[0]?.text?.trim()
+    return answer || null
+  }
+
+  /**
+   * Full solve flow: detect → solve → fill → submit.
+   * Returns true if solved, false if unsolvable type.
+   */
+  async trySolve(page) {
+    const detection = await this.detect(page)
+    if (!detection?.hasCaptcha) return { solved: false, reason: 'no captcha detected' }
+
+    // Token-based CAPTCHAs — can't solve with vision
+    if (['recaptcha', 'hcaptcha', 'turnstile'].includes(detection.type)) {
+      return { solved: false, reason: `${detection.type} requires token solving (2captcha/CapSolver)` }
+    }
+
+    // Image CAPTCHA — solve with Gemini Vision
+    if (detection.type === 'image') {
+      try {
+        const answer = await this.solveImage(page, detection.selector)
+        if (!answer) return { solved: false, reason: 'gemini could not read captcha' }
+
+        // Find the input field near the CAPTCHA
+        const inputSelector = await page.evaluate((captchaSelector) => {
+          const captcha = document.querySelector(captchaSelector)
+          // Look for nearby input
+          const parent = captcha?.closest('form') || captcha?.parentElement
+          const input = parent?.querySelector('input[type="text"], input:not([type])')
+          if (input) {
+            input.id = input.id || '__spectrawl_captcha_input'
+            return '#' + input.id
+          }
+          return null
+        }, detection.selector)
+
+        if (inputSelector) {
+          await page.fill(inputSelector, answer)
+          return { solved: true, answer, type: 'image' }
+        }
+        return { solved: false, reason: 'could not find captcha input field', answer }
+      } catch (e) {
+        return { solved: false, reason: e.message }
+      }
+    }
+
+    return { solved: false, reason: `unsupported captcha type: ${detection.type}` }
+  }
+
+  _post(url, body) {
+    return new Promise((resolve, reject) => {
+      const urlObj = new URL(url)
+      const req = https.request({
+        hostname: urlObj.hostname,
+        path: urlObj.pathname + urlObj.search,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(body)
+        }
+      }, res => {
+        let data = ''
+        res.on('data', c => data += c)
+        res.on('end', () => {
+          try { resolve(JSON.parse(data)) }
+          catch (e) { reject(new Error('Invalid Gemini response')) }
+        })
+      })
+      req.on('error', reject)
+      req.setTimeout(15000, () => { req.destroy(); reject(new Error('Gemini vision timeout')) })
+      req.write(body)
+      req.end()
+    })
+  }
+}
+
+module.exports = { CaptchaSolver }

package/src/browse/index.js

@@ -12,6 +12,7 @@ const os = require('os')
 const path = require('path')
 const { CamoufoxClient } = require('./camoufox')
 const { getCamoufoxPath, isInstalled } = require('./install-stealth')
+const { CaptchaSolver } = require('./captcha-solver')
 
 class BrowseEngine {
   constructor(config = {}, cache) {
@@ -23,6 +24,9 @@ class BrowseEngine {
     this.remoteCamoufox = config.camoufox?.url ? new CamoufoxClient(config.camoufox) : null
     this._remoteCamoufoxAvailable = null
 
+    // CAPTCHA solver (Gemini Vision fallback)
+    this.captchaSolver = new CaptchaSolver(config.captcha || {})
+
     // Which engine we're using
     this._engine = null
   }

package/src/browse/install-stealth.js

@@ -114,15 +114,60 @@ async function install() {
   // Download
   await download(url, zipPath)
 
-  // Extract
+  // Extract — try multiple methods (large zip64 files break some tools)
   console.log('  Extracting...')
   fs.mkdirSync(extractDir, { recursive: true })
 
-  try {
-    execSync(`unzip -o "${zipPath}" -d "${extractDir}"`, { stdio: 'pipe' })
-  } catch (e) {
-    // Try with built-in tools on systems without unzip
-    execSync(`python3 -c "import zipfile; zipfile.ZipFile('${zipPath}').extractall('${extractDir}')"`, { stdio: 'pipe' })
+  const extractMethods = [
+    // 1. unzip (most common on Linux/Mac)
+    () => execSync(`unzip -o "${zipPath}" -d "${extractDir}"`, { stdio: 'pipe' }),
+    // 2. 7z (handles zip64 reliably)
+    () => execSync(`7z x -o"${extractDir}" -y "${zipPath}"`, { stdio: 'pipe' }),
+    // 3. bsdtar (available on many systems, handles zip64)
+    () => execSync(`bsdtar -xf "${zipPath}" -C "${extractDir}"`, { stdio: 'pipe' }),
+    // 4. Node.js built-in (no external deps, handles zip64)
+    () => {
+      const { execSync: es } = require('child_process')
+      es(`node -e "
+        const fs = require('fs');
+        const zlib = require('zlib');
+        const { execFileSync } = require('child_process');
+        // Use jar if available (JDK)
+        execFileSync('jar', ['xf', '${zipPath}'], { cwd: '${extractDir}', stdio: 'pipe' });
+      "`, { stdio: 'pipe' })
+    },
+    // 5. Python with explicit zip64 support
+    () => execSync(`python3 -c "
+import zipfile, sys
+try:
+    z = zipfile.ZipFile('${zipPath}', allowZip64=True)
+    z.extractall('${extractDir}')
+    z.close()
+except Exception as e:
+    print(f'Python extract failed: {e}', file=sys.stderr)
+    sys.exit(1)
+"`, { stdio: 'pipe' }),
+  ]
+
+  let extracted = false
+  for (const method of extractMethods) {
+    try {
+      method()
+      extracted = true
+      break
+    } catch (e) {
+      continue
+    }
+  }
+
+  if (!extracted) {
+    fs.unlinkSync(zipPath)
+    throw new Error(
+      'Could not extract Camoufox archive. Install one of: unzip, 7z, or bsdtar.\n' +
+      '  Ubuntu/Debian: sudo apt-get install unzip\n' +
+      '  macOS: brew install p7zip\n' +
+      '  Alpine: apk add unzip'
+    )
   }
 
   // Clean up zip

package/src/search/scraper.js

@@ -12,7 +12,7 @@ const { jinaExtract } = require('./engines/jina')
  */
 async function scrapeUrls(urls, opts = {}) {
   const results = {}
-  const timeout = opts.timeout || 3000  // 3s hard cutoff per URL (was 10s)
+  const timeout = opts.timeout || 5000  // 5s per URL — balances speed vs quality
   const concurrent = opts.concurrent || 5
   const engine = opts.engine || 'auto' // 'jina', 'readability', 'auto'
 
@@ -35,7 +35,7 @@ async function scrapeUrls(urls, opts = {}) {
 }
 
 async function scrapeUrl(url, opts = {}) {
-  const { timeout = 3000, engine = 'auto', browse } = opts
+  const { timeout = 5000, engine = 'auto', browse } = opts
 
   // Try Jina first if available (better markdown output)
   if (engine === 'jina' || engine === 'auto') {