spectrawl 0.4.3 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.4.3",
+  "version": "0.6.0",
   "description": "The unified web layer for AI agents. Search (8 engines), stealth browse, auth, act on 24 platforms. Self-hosted.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/agent.js

@@ -0,0 +1,295 @@
+/**
+ * Spectrawl Agent Engine
+ * Natural language browser actions — "click the sign in button", "fill the search box with query".
+ * Uses LLM to interpret page DOM and generate Playwright actions.
+ */
+
+const https = require('https')
+
+class AgentEngine {
+  constructor(browseEngine, config = {}) {
+    this.browseEngine = browseEngine
+    this.apiKey = config.apiKey || process.env.GEMINI_API_KEY
+    this.openaiKey = config.openaiKey || process.env.OPENAI_API_KEY
+    this.model = config.model || 'gemini-2.0-flash'
+  }
+
+  /**
+   * Execute a natural language action on a page.
+   * @param {string} url - URL to navigate to
+   * @param {string} instruction - what to do (e.g. "click the login button")
+   * @param {object} opts - options
+   * @param {number} opts.maxSteps - max number of actions to take (default 5)
+   * @param {boolean} opts.screenshot - take screenshot after action
+   * @param {number} opts.timeout - timeout per action in ms
+   */
+  async act(url, instruction, opts = {}) {
+    const maxSteps = opts.maxSteps || 5
+    const timeout = opts.timeout || 30000
+    const startTime = Date.now()
+    const steps = []
+
+    // Get a browser page
+    const { page, context } = await this.browseEngine.getPage({ url, timeout })
+
+    try {
+      // Wait for page to be ready
+      await page.waitForLoadState('domcontentloaded', { timeout: 10000 }).catch(() => {})
+      await page.waitForTimeout(1000)
+
+      for (let i = 0; i < maxSteps; i++) {
+        // Get simplified DOM
+        const dom = await this._getSimplifiedDOM(page)
+
+        // Ask LLM what to do
+        const action = await this._planAction(dom, instruction, steps, page.url())
+
+        if (action.done) {
+          steps.push({ step: i + 1, action: 'done', reason: action.reason })
+          break
+        }
+
+        // Execute the action
+        try {
+          const result = await this._executeAction(page, action)
+          steps.push({ step: i + 1, ...action, result: result || 'ok' })
+
+          // Wait for potential navigation/load
+          await page.waitForTimeout(500 + Math.random() * 1000)
+          await page.waitForLoadState('domcontentloaded', { timeout: 5000 }).catch(() => {})
+        } catch (err) {
+          steps.push({ step: i + 1, ...action, error: err.message })
+          // Continue trying if there are more steps
+        }
+      }
+
+      // Get final page state
+      const finalContent = await page.evaluate(() => document.body?.innerText?.slice(0, 10000) || '')
+      const finalUrl = page.url()
+      const finalTitle = await page.title()
+
+      let screenshot = null
+      if (opts.screenshot) {
+        screenshot = await page.screenshot({ type: 'png', fullPage: false })
+      }
+
+      return {
+        success: steps.some(s => s.action === 'done' || !s.error),
+        url: finalUrl,
+        title: finalTitle,
+        steps,
+        content: finalContent,
+        screenshot,
+        duration: Date.now() - startTime
+      }
+    } finally {
+      await context.close().catch(() => {})
+    }
+  }
+
+  /**
+   * Get a simplified DOM representation for the LLM.
+   * Strips noise, keeps interactive elements with indices.
+   */
+  async _getSimplifiedDOM(page) {
+    return page.evaluate(() => {
+      const elements = []
+      const interactiveSelectors = [
+        'a[href]', 'button', 'input', 'textarea', 'select',
+        '[role="button"]', '[role="link"]', '[role="tab"]',
+        '[onclick]', '[type="submit"]', 'label'
+      ]
+
+      const allElements = document.querySelectorAll(interactiveSelectors.join(','))
+
+      allElements.forEach((el, idx) => {
+        if (!el.offsetParent && el.tagName !== 'INPUT') return // skip hidden
+        const rect = el.getBoundingClientRect()
+        if (rect.width === 0 && rect.height === 0) return
+
+        const tag = el.tagName.toLowerCase()
+        const type = el.type || ''
+        const text = (el.textContent || '').trim().slice(0, 100)
+        const placeholder = el.placeholder || ''
+        const ariaLabel = el.getAttribute('aria-label') || ''
+        const href = el.href || ''
+        const value = el.value || ''
+        const name = el.name || ''
+        const id = el.id || ''
+
+        // Create a unique selector for this element
+        let selector = tag
+        if (id) selector = `#${id}`
+        else if (name) selector = `${tag}[name="${name}"]`
+        else if (ariaLabel) selector = `${tag}[aria-label="${ariaLabel}"]`
+
+        elements.push({
+          idx,
+          tag,
+          type,
+          text: text.slice(0, 80),
+          placeholder,
+          ariaLabel,
+          href: href.slice(0, 100),
+          value,
+          selector,
+          id,
+          name
+        })
+      })
+
+      return {
+        title: document.title,
+        url: location.href,
+        elements: elements.slice(0, 100) // cap at 100 elements
+      }
+    })
+  }
+
+  /**
+   * Ask LLM to plan the next action.
+   */
+  async _planAction(dom, instruction, previousSteps, currentUrl) {
+    const prompt = `You are a browser automation agent. Given the current page state and instruction, determine the next action.
+
+Current URL: ${currentUrl}
+Page title: ${dom.title}
+
+Interactive elements on page:
+${dom.elements.map(e => `[${e.idx}] <${e.tag}${e.type ? ` type="${e.type}"` : ''}${e.id ? ` id="${e.id}"` : ''}${e.name ? ` name="${e.name}"` : ''}> ${e.text || e.placeholder || e.ariaLabel || e.href || '(empty)'}`).join('\n')}
+
+Instruction: ${instruction}
+
+Previous steps: ${previousSteps.length > 0 ? JSON.stringify(previousSteps) : 'none'}
+
+Respond with a JSON object:
+- If the instruction is complete: {"done": true, "reason": "why it's done"}
+- To click: {"action": "click", "elementIdx": 5, "reason": "clicking the login button"}
+- To type: {"action": "type", "elementIdx": 3, "text": "hello", "reason": "filling search box"}
+- To select: {"action": "select", "elementIdx": 7, "value": "option1", "reason": "selecting dropdown"}
+- To press a key: {"action": "press", "key": "Enter", "reason": "submitting form"}
+- To scroll: {"action": "scroll", "direction": "down", "reason": "loading more content"}
+
+Only return valid JSON. No explanation.`
+
+    const result = await this._llmCall(prompt)
+    return result
+  }
+
+  /**
+   * Execute a planned action on the page.
+   */
+  async _executeAction(page, action) {
+    switch (action.action) {
+      case 'click': {
+        const elements = await page.$$('a[href], button, input, textarea, select, [role="button"], [role="link"], [role="tab"], [onclick], [type="submit"], label')
+        const visibleElements = []
+        for (const el of elements) {
+          const visible = await el.isVisible().catch(() => false)
+          if (visible) visibleElements.push(el)
+        }
+        const target = visibleElements[action.elementIdx]
+        if (!target) throw new Error(`Element [${action.elementIdx}] not found`)
+        await target.click({ timeout: 5000 })
+        return 'clicked'
+      }
+
+      case 'type': {
+        const elements = await page.$$('a[href], button, input, textarea, select, [role="button"], [role="link"], [role="tab"], [onclick], [type="submit"], label')
+        const visibleElements = []
+        for (const el of elements) {
+          const visible = await el.isVisible().catch(() => false)
+          if (visible) visibleElements.push(el)
+        }
+        const target = visibleElements[action.elementIdx]
+        if (!target) throw new Error(`Element [${action.elementIdx}] not found`)
+        await target.fill('')
+        await target.type(action.text, { delay: 50 + Math.random() * 100 })
+        return 'typed'
+      }
+
+      case 'select': {
+        const elements = await page.$$('a[href], button, input, textarea, select, [role="button"], [role="link"], [role="tab"], [onclick], [type="submit"], label')
+        const visibleElements = []
+        for (const el of elements) {
+          const visible = await el.isVisible().catch(() => false)
+          if (visible) visibleElements.push(el)
+        }
+        const target = visibleElements[action.elementIdx]
+        if (!target) throw new Error(`Element [${action.elementIdx}] not found`)
+        await target.selectOption(action.value)
+        return 'selected'
+      }
+
+      case 'press':
+        await page.keyboard.press(action.key)
+        return 'pressed'
+
+      case 'scroll':
+        await page.evaluate((dir) => {
+          window.scrollBy(0, dir === 'up' ? -500 : 500)
+        }, action.direction)
+        return 'scrolled'
+
+      default:
+        throw new Error(`Unknown action: ${action.action}`)
+    }
+  }
+
+  async _llmCall(prompt) {
+    if (this.apiKey) {
+      const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`
+      const body = {
+        contents: [{ parts: [{ text: prompt }] }],
+        generationConfig: { responseMimeType: 'application/json', temperature: 0.1 }
+      }
+
+      const response = await this._post(url, body)
+      const text = response?.candidates?.[0]?.content?.parts?.[0]?.text
+      if (!text) throw new Error('Empty LLM response')
+      return JSON.parse(text)
+    } else if (this.openaiKey) {
+      const url = 'https://api.openai.com/v1/chat/completions'
+      const body = {
+        model: 'gpt-4o-mini',
+        messages: [{ role: 'user', content: prompt }],
+        response_format: { type: 'json_object' },
+        temperature: 0.1
+      }
+      const response = await this._post(url, body, { 'Authorization': `Bearer ${this.openaiKey}` })
+      return JSON.parse(response?.choices?.[0]?.message?.content)
+    }
+    throw new Error('No LLM API key configured')
+  }
+
+  _post(url, body, extraHeaders = {}) {
+    return new Promise((resolve, reject) => {
+      const urlObj = new URL(url)
+      const data = JSON.stringify(body)
+      const opts = {
+        hostname: urlObj.hostname,
+        path: urlObj.pathname + urlObj.search,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(data),
+          ...extraHeaders
+        }
+      }
+      const req = https.request(opts, res => {
+        let responseData = ''
+        res.on('data', chunk => responseData += chunk)
+        res.on('end', () => {
+          try { resolve(JSON.parse(responseData)) }
+          catch (e) { reject(new Error(`Invalid JSON: ${responseData.slice(0, 200)}`)) }
+        })
+      })
+      req.on('error', reject)
+      req.setTimeout(30000, () => { req.destroy(); reject(new Error('LLM timeout')) })
+      req.write(data)
+      req.end()
+    })
+  }
+}
+
+module.exports = { AgentEngine }

package/src/browse/index.js

@@ -120,6 +120,40 @@ class BrowseEngine {
     const context = await this._createContext(browser, opts)
     const page = await context.newPage()
 
+    // Network request capturing
+    const networkRequests = []
+    if (opts.captureNetwork) {
+      page.on('request', req => {
+        const resourceType = req.resourceType()
+        if (['xhr', 'fetch'].includes(resourceType)) {
+          networkRequests.push({
+            url: req.url(),
+            method: req.method(),
+            resourceType,
+            headers: opts.captureNetworkHeaders ? req.headers() : undefined,
+            postData: req.postData() || undefined
+          })
+        }
+      })
+      page.on('response', async res => {
+        const req = res.request()
+        const resourceType = req.resourceType()
+        if (['xhr', 'fetch'].includes(resourceType)) {
+          const existing = networkRequests.find(r => r.url === req.url() && r.method === req.method())
+          if (existing) {
+            existing.status = res.status()
+            existing.contentType = res.headers()['content-type'] || null
+            if (opts.captureNetworkBody) {
+              try {
+                const body = await res.text().catch(() => null)
+                if (body && body.length < 50000) existing.body = body
+              } catch (e) { /* ignore */ }
+            }
+          }
+        }
+      })
+    }
+
     try {
       if (opts._cookies) {
         await context.addCookies(opts._cookies)
@@ -164,9 +198,23 @@ class BrowseEngine {
 
       result.url = page.url()
       result.title = await page.title()
+      result.statusCode = null // playwright doesn't expose easily, but we detect blocks below
       result.cached = false
       result.engine = this._engine
 
+      // Attach captured network requests
+      if (opts.captureNetwork && networkRequests.length > 0) {
+        result.networkRequests = networkRequests
+      }
+
+      // Detect block pages (Cloudflare, Akamai, etc.)
+      const blockInfo = detectBlockPage(result.content, result.title, result.html, result.url)
+      if (blockInfo) {
+        result.blocked = true
+        result.blockType = blockInfo.type
+        result.blockDetail = blockInfo.detail
+      }
+
       if (!opts.screenshot) {
         this.cache?.set('scrape', url, { content: result.content, url: result.url, title: result.title })
       }
@@ -288,4 +336,81 @@ class BrowseEngine {
   }
 }
 
+/**
+ * Detect block/challenge pages from CDNs and bot protection services.
+ * Returns { type, detail } if blocked, null if clean.
+ */
+function detectBlockPage(content, title, html, url) {
+  const text = (content || '').toLowerCase()
+  const titleLower = (title || '').toLowerCase()
+  const htmlLower = (html || '').toLowerCase()
+
+  // Cloudflare
+  if (htmlLower.includes('cf-error-details') || htmlLower.includes('cf_chl_opt') ||
+      text.includes('attention required') && text.includes('cloudflare') ||
+      text.includes('checking if the site connection is secure') ||
+      titleLower.includes('just a moment') && htmlLower.includes('cloudflare') ||
+      text.includes('ray id:') && text.includes('cloudflare')) {
+    return { type: 'cloudflare', detail: 'Cloudflare bot challenge or block page detected' }
+  }
+
+  // Cloudflare RFC 9457 structured error (new format)
+  if (htmlLower.includes('application/problem+json') || 
+      text.includes('error 1') && text.includes('cloudflare') ||
+      htmlLower.includes('"type":') && htmlLower.includes('cloudflare.com/errors/')) {
+    return { type: 'cloudflare-rfc9457', detail: 'Cloudflare structured error response (RFC 9457)' }
+  }
+
+  // Akamai
+  if (text.includes('access denied') && htmlLower.includes('akamai') ||
+      htmlLower.includes('akamaighost') ||
+      text.includes('reference #') && text.includes('access denied')) {
+    return { type: 'akamai', detail: 'Akamai bot detection triggered' }
+  }
+
+  // AWS WAF
+  if (text.includes('request blocked') && htmlLower.includes('aws') ||
+      htmlLower.includes('awswaf')) {
+    return { type: 'aws-waf', detail: 'AWS WAF blocked the request' }
+  }
+
+  // Imperva / Incapsula
+  if (htmlLower.includes('incapsula') || htmlLower.includes('imperva') ||
+      text.includes('request unsuccessful') && text.includes('incapsula')) {
+    return { type: 'imperva', detail: 'Imperva/Incapsula bot detection triggered' }
+  }
+
+  // DataDome
+  if (htmlLower.includes('datadome') || htmlLower.includes('dd.js')) {
+    return { type: 'datadome', detail: 'DataDome bot detection triggered' }
+  }
+
+  // PerimeterX / HUMAN
+  if (htmlLower.includes('perimeterx') || htmlLower.includes('px-captcha') ||
+      htmlLower.includes('human security')) {
+    return { type: 'perimeterx', detail: 'PerimeterX/HUMAN bot detection triggered' }
+  }
+
+  // hCaptcha challenge
+  if (htmlLower.includes('hcaptcha.com') && htmlLower.includes('h-captcha')) {
+    return { type: 'hcaptcha', detail: 'hCaptcha challenge page' }
+  }
+
+  // reCAPTCHA challenge (standalone, not embedded)
+  if (htmlLower.includes('recaptcha') && text.length < 500 &&
+      (titleLower === '' || titleLower.includes('blocked') || titleLower.includes('verify'))) {
+    return { type: 'recaptcha', detail: 'reCAPTCHA challenge page' }
+  }
+
+  // Generic bot detection signals
+  if (text.length < 200 && (
+      text.includes('access denied') || text.includes('403 forbidden') ||
+      text.includes('bot detected') || text.includes('automated access') ||
+      text.includes('please verify you are human') || text.includes('are you a robot'))) {
+    return { type: 'generic', detail: 'Generic bot detection or access denied page' }
+  }
+
+  return null
+}
+
 module.exports = { BrowseEngine }

package/src/crawl.js

@@ -90,6 +90,29 @@ class CrawlEngine {
     const failed = []
     let activeCount = 0
 
+    // Sitemap-based crawling: pre-seed queue with sitemap URLs
+    if (config.useSitemap !== false) {
+      try {
+        const sitemapUrls = await fetchSitemap(startUrl)
+        if (sitemapUrls.length > 0) {
+          for (const sUrl of sitemapUrls.slice(0, config.maxPages)) {
+            const norm = normalizeUrl(sUrl)
+            if (!visited.has(norm) && this._inScope(sUrl, baseDomain, basePrefix, config.scope)) {
+              if (!config.skipPatterns.some(p => p.test(sUrl))) {
+                if (this._matchesFilters(sUrl, config.includePatterns, config.excludePatterns)) {
+                  visited.add(norm)
+                  queue.push({ url: sUrl, depth: 0 })
+                }
+              }
+            }
+          }
+          console.log(`[crawl] Pre-seeded ${Math.min(sitemapUrls.length, config.maxPages)} URLs from sitemap`)
+        }
+      } catch (e) {
+        console.log(`[crawl] Sitemap fetch failed, continuing with link discovery`)
+      }
+    }
+
     // Process queue with concurrency control
     const processUrl = async (item) => {
       const { url, depth } = item
@@ -177,6 +200,14 @@ class CrawlEngine {
       }).join('\n\n---\n\n')
     }
 
+    // Webhook notification
+    if (config.webhook) {
+      sendWebhook(config.webhook, {
+        event: 'crawl_complete',
+        ...result
+      }).catch(err => console.error('[crawl] Webhook failed:', err.message))
+    }
+
     return result
   }
 
@@ -276,15 +307,32 @@ class CrawlEngine {
 
   async _fetchPage(url, config, cookies) {
     try {
-      const result = await this.browseEngine.browse(url, {
+      const browseOpts = {
         stealth: config.stealth,
         _cookies: cookies,
         timeout: config.timeout,
         html: true,
         noCache: true,
-        fastMode: true  // crawl mode: reduced delays for speed
-      })
+        fastMode: true
+      }
+      let result = await this.browseEngine.browse(url, browseOpts)
+
+      // Auto-retry with full stealth if blocked
+      if (result?.blocked && browseOpts.fastMode) {
+        console.log(`[crawl] Block detected on ${url} (${result.blockType}) — retrying with full stealth`)
+        result = await this.browseEngine.browse(url, {
+          ...browseOpts,
+          fastMode: false,
+          stealth: true,
+          camoufox: true
+        })
+      }
+
       if (result?.content) {
+        // Skip if still blocked after retry
+        if (result.blocked) {
+          throw new Error(`Blocked by ${result.blockType}: ${result.blockDetail}`)
+        }
         const linkSource = result.html || result.content
         return {
           title: result.title || '',
@@ -354,6 +402,93 @@ function resolveUrl(url, base) {
   }
 }
 
+/**
+ * Fetch and parse sitemap.xml for a domain.
+ * Returns array of URLs found in the sitemap.
+ */
+async function fetchSitemap(startUrl) {
+  const { URL } = require('url')
+  const base = new URL(startUrl)
+  const sitemapUrls = [
+    `${base.origin}/sitemap.xml`,
+    `${base.origin}/sitemap_index.xml`,
+    `${base.origin}/sitemap/sitemap.xml`
+  ]
+
+  for (const sitemapUrl of sitemapUrls) {
+    try {
+      const xml = await fetchText(sitemapUrl, 5000)
+      if (!xml || !xml.includes('<url') && !xml.includes('<sitemap')) continue
+
+      const urls = []
+      // Extract <loc> URLs from sitemap
+      const locMatches = xml.matchAll(/<loc>\s*(.*?)\s*<\/loc>/gi)
+      for (const match of locMatches) {
+        const loc = match[1].trim()
+        // If it's a sitemap index, recursively fetch
+        if (loc.endsWith('.xml') || loc.includes('sitemap')) {
+          try {
+            const subXml = await fetchText(loc, 5000)
+            if (subXml) {
+              const subMatches = subXml.matchAll(/<loc>\s*(.*?)\s*<\/loc>/gi)
+              for (const subMatch of subMatches) {
+                const subLoc = subMatch[1].trim()
+                if (!subLoc.endsWith('.xml')) urls.push(subLoc)
+              }
+            }
+          } catch (e) { /* skip failed sub-sitemaps */ }
+        } else {
+          urls.push(loc)
+        }
+      }
+      if (urls.length > 0) {
+        console.log(`[crawl] Found sitemap at ${sitemapUrl} with ${urls.length} URLs`)
+        return urls
+      }
+    } catch (e) { /* try next */ }
+  }
+  return []
+}
+
+function fetchText(url, timeout = 5000) {
+  const h = url.startsWith('https') ? require('https') : require('http')
+  return new Promise((resolve, reject) => {
+    const req = h.get(url, { timeout, headers: { 'User-Agent': 'Spectrawl/1.0 (sitemap crawler)' } }, res => {
+      if (res.statusCode !== 200) return resolve(null)
+      let data = ''
+      res.on('data', chunk => data += chunk)
+      res.on('end', () => resolve(data))
+    })
+    req.on('error', () => resolve(null))
+    req.setTimeout(timeout, () => { req.destroy(); resolve(null) })
+  })
+}
+
+/**
+ * Send a webhook notification.
+ */
+async function sendWebhook(webhookUrl, data) {
+  const h = webhookUrl.startsWith('https') ? require('https') : require('http')
+  const body = JSON.stringify(data)
+  const urlObj = new URL(webhookUrl)
+  return new Promise((resolve) => {
+    const req = h.request({
+      hostname: urlObj.hostname,
+      port: urlObj.port,
+      path: urlObj.pathname + urlObj.search,
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) }
+    }, res => {
+      res.on('data', () => {})
+      res.on('end', () => resolve(true))
+    })
+    req.on('error', () => resolve(false))
+    req.setTimeout(10000, () => { req.destroy(); resolve(false) })
+    req.write(body)
+    req.end()
+  })
+}
+
 function normalizeUrl(url) {
   try {
     const u = new URL(url)