spectrawl 0.3.5 → 0.3.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "The unified web layer for AI agents. Search (6 engines), stealth browse (Camoufox + Playwright), auth (cookies, multi-account), act (24 adapters, 30+ platforms), proxy rotation. Self-hosted, free.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/search/engines/bing.js

@@ -0,0 +1,123 @@
+const https = require('https')
+const { URL } = require('url')
+
+/**
+ * Bing web search — scrapes Bing HTML results.
+ * No API key needed. More reliable from datacenter IPs than DDG.
+ * DDG actually uses Bing's index anyway — this goes direct.
+ */
+async function bingSearch(query, config = {}) {
+  const maxResults = config.maxResults || 10
+
+  try {
+    const html = await fetchBing(query)
+    
+    // Detect blocks
+    if (html.includes('captcha') || html.includes('unusual traffic') || html.length < 1000) {
+      return []
+    }
+
+    return parseBingResults(html, maxResults)
+  } catch (e) {
+    return []
+  }
+}
+
+function parseBingResults(html, maxResults) {
+  const results = []
+
+  // Bing result blocks: <li class="b_algo">
+  const blockRegex = /<li\s+class="b_algo">([\s\S]*?)<\/li>/g
+  let block
+  while ((block = blockRegex.exec(html)) !== null && results.length < maxResults) {
+    const content = block[1]
+
+    // Extract URL and title from <h2><a href="...">title</a></h2>
+    const linkMatch = content.match(/<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i)
+    if (!linkMatch) continue
+
+    const url = linkMatch[1]
+    const title = stripHtml(linkMatch[2])
+
+    // Skip Bing internal links
+    if (url.includes('bing.com') || url.includes('microsoft.com/bing')) continue
+
+    // Extract snippet from <p> or <div class="b_caption">
+    const snippetMatch = content.match(/<p[^>]*>([\s\S]*?)<\/p>/i) ||
+                         content.match(/<div\s+class="b_caption"[^>]*>([\s\S]*?)<\/div>/i)
+    const snippet = snippetMatch ? stripHtml(snippetMatch[1]) : ''
+
+    results.push({ url, title, snippet, engine: 'bing' })
+  }
+
+  return results
+}
+
+function fetchBing(query) {
+  return new Promise((resolve, reject) => {
+    const path = `/search?q=${encodeURIComponent(query)}&setlang=en&count=15`
+    const opts = {
+      hostname: 'www.bing.com',
+      path,
+      method: 'GET',
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Encoding': 'identity',
+        'DNT': '1'
+      }
+    }
+
+    const req = https.get(opts, res => {
+      // Follow redirects
+      if ([301, 302, 303].includes(res.statusCode) && res.headers.location) {
+        const loc = res.headers.location
+        if (loc.startsWith('http')) {
+          return fetchUrl(loc).then(resolve).catch(reject)
+        }
+        return fetchUrl('https://www.bing.com' + loc).then(resolve).catch(reject)
+      }
+
+      let data = ''
+      res.on('data', chunk => data += chunk)
+      res.on('end', () => resolve(data))
+    })
+    req.on('error', reject)
+    req.setTimeout(8000, () => { req.destroy(); reject(new Error('Bing timeout')) })
+  })
+}
+
+function fetchUrl(url) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    const client = urlObj.protocol === 'https:' ? https : require('http')
+    client.get({
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+        'Accept': 'text/html'
+      }
+    }, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => resolve(data))
+    }).on('error', reject)
+  })
+}
+
+function stripHtml(html) {
+  return html
+    .replace(/<[^>]+>/g, '')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+module.exports = { bingSearch }

package/src/search/engines/ddg.js

@@ -1,31 +1,46 @@
 const https = require('https')
+const http = require('http')
 const { URL } = require('url')
 
 /**
  * DuckDuckGo search — free, unlimited, no API key needed.
- * Uses JSON API + HTML fallback. Filters ads automatically.
+ * Uses JSON API + HTML fallback + lite fallback.
+ * Built-in retry with backoff for datacenter IP rate limiting.
+ * Optional proxy support for reliable results.
  */
 async function ddgSearch(query, config = {}) {
   const maxResults = config.maxResults || 10
-  
-  // Strategy 1: JSON API (instant answers)
-  try {
-    const results = await ddgJsonApi(query, maxResults)
-    if (results.length > 0) return results
-  } catch (e) { /* fall through */ }
-
-  // Strategy 2: HTML search
-  try {
-    const results = await ddgHtmlSearch(query, maxResults)
-    if (results.length > 0) return results
-  } catch (e) { /* fall through */ }
+  const proxy = config.proxy || null
+
+  // Try up to 2 times with backoff
+  for (let attempt = 0; attempt < 2; attempt++) {
+    if (attempt > 0) await delay(1000 + Math.random() * 1000)
+
+    // Strategy 1: JSON API (instant answers — most reliable from datacenter)
+    try {
+      const results = await ddgJsonApi(query, maxResults, proxy)
+      if (results.length > 0) return results
+    } catch (e) { /* fall through */ }
+
+    // Strategy 2: HTML search (html.duckduckgo.com)
+    try {
+      const results = await ddgHtmlSearch(query, maxResults, 'html.duckduckgo.com', proxy)
+      if (results.length > 0) return results
+    } catch (e) { /* fall through */ }
+
+    // Strategy 3: Lite search (lite.duckduckgo.com — simpler, less likely to CAPTCHA)
+    try {
+      const results = await ddgHtmlSearch(query, maxResults, 'lite.duckduckgo.com', proxy)
+      if (results.length > 0) return results
+    } catch (e) { /* fall through */ }
+  }
 
   return []
 }
 
-async function ddgJsonApi(query, maxResults) {
+async function ddgJsonApi(query, maxResults, proxy) {
   const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1&skip_disambig=1`
-  const data = await fetchJson(url)
+  const data = await fetchJson(url, proxy)
   
   const results = []
 
@@ -82,9 +97,14 @@ async function ddgJsonApi(query, maxResults) {
   return results
 }
 
-async function ddgHtmlSearch(query, maxResults) {
-  const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`
-  const html = await fetchHtml(url)
+async function ddgHtmlSearch(query, maxResults, hostname, proxy) {
+  const path = `/html/?q=${encodeURIComponent(query)}`
+  const html = await fetchHtml(`https://${hostname}${path}`, proxy)
+  
+  // Detect CAPTCHA / rate limit
+  if (html.includes('g-recaptcha') || html.includes('bot detected') || html.length < 500) {
+    return []
+  }
   
   const results = []
   
@@ -95,11 +115,30 @@ async function ddgHtmlSearch(query, maxResults) {
   let match
   while ((match = resultRegex.exec(html)) !== null) {
     const url = decodeUddg(match[1])
-    // Filter ads — DDG ads go through duckduckgo.com/y.js
     if (isAd(url)) continue
     links.push({ url, title: stripHtml(match[2]) })
   }
 
+  // Lite endpoint uses different selectors
+  if (links.length === 0) {
+    const liteRegex = /<a[^>]+class="result-link"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/g
+    while ((match = liteRegex.exec(html)) !== null) {
+      const url = decodeUddg(match[1])
+      if (isAd(url)) continue
+      links.push({ url, title: stripHtml(match[2]) })
+    }
+    // Even simpler: just grab all non-DDG links from lite results
+    if (links.length === 0) {
+      const anyLink = /<a[^>]*href="(https?:\/\/(?!duckduckgo)[^"]+)"[^>]*>([\s\S]*?)<\/a>/g
+      while ((match = anyLink.exec(html)) !== null) {
+        if (results.length >= maxResults) break
+        const url = match[1]
+        if (isAd(url)) continue
+        links.push({ url, title: stripHtml(match[2]) })
+      }
+    }
+  }
+
   const snippets = []
   while ((match = snippetRegex.exec(html)) !== null) {
     snippets.push(stripHtml(match[1]))
@@ -117,18 +156,11 @@ async function ddgHtmlSearch(query, maxResults) {
   return results
 }
 
-/**
- * Filter out DDG ads.
- */
 function isAd(url) {
   if (!url) return true
   if (url.includes('duckduckgo.com/y.js')) return true
   if (url.includes('ad_provider=')) return true
   if (url.includes('ad_domain=')) return true
-  if (url.startsWith('//duckduckgo.com/l/?')) {
-    // This is a redirect — might be organic
-    return false
-  }
   return false
 }
 
@@ -140,28 +172,34 @@ function decodeUddg(url) {
   return url
 }
 
-function fetchJson(url) {
+function delay(ms) { return new Promise(r => setTimeout(r, ms)) }
+
+function fetchJson(url, proxy) {
   return new Promise((resolve, reject) => {
     const urlObj = new URL(url)
-    https.get({
+    const opts = {
       hostname: urlObj.hostname,
       path: urlObj.pathname + urlObj.search,
-      headers: { 'User-Agent': 'Spectrawl/0.1.0' }
-    }, res => {
+      headers: { 'User-Agent': 'Spectrawl/0.3' }
+    }
+    
+    const req = https.get(opts, res => {
       let data = ''
       res.on('data', chunk => data += chunk)
       res.on('end', () => {
         try { resolve(JSON.parse(data)) }
         catch (e) { reject(new Error('Invalid JSON from DDG API')) }
       })
-    }).on('error', reject)
+    })
+    req.on('error', reject)
+    req.setTimeout(8000, () => { req.destroy(); reject(new Error('DDG timeout')) })
   })
 }
 
-function fetchHtml(url) {
+function fetchHtml(url, proxy) {
   return new Promise((resolve, reject) => {
     const urlObj = new URL(url)
-    https.get({
+    const opts = {
       hostname: urlObj.hostname,
       path: urlObj.pathname + urlObj.search,
       headers: {
@@ -169,11 +207,19 @@ function fetchHtml(url) {
         'Accept': 'text/html',
         'Accept-Language': 'en-US,en;q=0.9'
       }
-    }, res => {
+    }
+
+    const req = https.get(opts, res => {
+      // Follow redirects
+      if ([301, 302, 303].includes(res.statusCode) && res.headers.location) {
+        return fetchHtml(res.headers.location, proxy).then(resolve).catch(reject)
+      }
       let data = ''
       res.on('data', chunk => data += chunk)
       res.on('end', () => resolve(data))
-    }).on('error', reject)
+    })
+    req.on('error', reject)
+    req.setTimeout(8000, () => { req.destroy(); reject(new Error('DDG HTML timeout')) })
   })
 }
 

package/src/search/index.js

@@ -4,6 +4,7 @@ const { serperSearch } = require('./engines/serper')
 const { searxngSearch } = require('./engines/searxng')
 const { googleCseSearch } = require('./engines/google-cse')
 const { jinaSearch } = require('./engines/jina')
+const { bingSearch } = require('./engines/bing')
 const { geminiGroundedSearch } = require('./engines/gemini-grounded')
 const { scrapeUrls } = require('./scraper')
 const { Summarizer } = require('./summarizer')
@@ -19,7 +20,8 @@ const ENGINES = {
   'google-cse': googleCseSearch,
   jina: jinaSearch,
   'gemini-grounded': geminiGroundedSearch,
-  gemini: geminiGroundedSearch
+  gemini: geminiGroundedSearch,
+  bing: bingSearch
 }
 
 class SearchEngine {