spectrawl 0.3.0 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "The unified web layer for AI agents. Search (6 engines), stealth browse (Camoufox + Playwright), auth (cookies, multi-account), act (24 adapters, 30+ platforms), proxy rotation. Self-hosted, free.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/search/engines/gemini-grounded.js

@@ -0,0 +1,130 @@
+const https = require('https')
+
+/**
+ * Gemini Grounded Search — uses Google's Gemini API with built-in Google Search.
+ * Free tier: 1,500 req/day for Flash.
+ * Returns both an AI answer AND the search results it found.
+ * 
+ * This is basically free Google search + AI summarization in one call.
+ */
+async function geminiGroundedSearch(query, config = {}) {
+  const apiKey = config.apiKey || process.env.GEMINI_API_KEY
+  if (!apiKey) throw new Error('GEMINI_API_KEY required for grounded search')
+
+  const model = config.model || 'gemini-2.0-flash'
+  const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`
+
+  const body = JSON.stringify({
+    contents: [{
+      parts: [{ text: `Search the web and provide relevant results for: ${query}` }]
+    }],
+    tools: [{ google_search: {} }],
+    generationConfig: {
+      temperature: 0.1,
+      maxOutputTokens: 1000
+    }
+  })
+
+  const data = await post(url, body)
+
+  if (data.error) {
+    throw new Error(`Gemini grounded search: ${data.error.message}`)
+  }
+
+  // Extract grounding metadata (search results)
+  const candidate = data.candidates?.[0]
+  const grounding = candidate?.groundingMetadata
+  const chunks = grounding?.groundingChunks || []
+  const answer = candidate?.content?.parts?.map(p => p.text).filter(Boolean).join('\n') || ''
+
+  // Resolve redirect URLs to actual URLs (parallel, with timeout)
+  const rawResults = chunks.map((chunk, i) => ({
+    title: chunk.web?.title || `Result ${i + 1}`,
+    redirectUrl: chunk.web?.uri || '',
+    snippet: '',
+    source: 'gemini-grounded'
+  })).filter(r => r.redirectUrl)
+
+  // Follow redirects to get real URLs
+  const resolved = await Promise.all(
+    rawResults.map(r => resolveRedirect(r.redirectUrl).catch(() => r.redirectUrl))
+  )
+
+  const results = rawResults.map((r, i) => ({
+    ...r,
+    url: resolved[i] || r.redirectUrl
+  }))
+
+  // Add confidence scores from grounding supports
+  const supports = grounding?.groundingSupports || []
+  for (const support of supports) {
+    const indices = support.groundingChunkIndices || []
+    const scores = support.confidenceScores || []
+    indices.forEach((idx, j) => {
+      if (results[idx] && scores[j]) {
+        results[idx].confidence = Math.max(results[idx].confidence || 0, scores[j])
+      }
+    })
+  }
+
+  // Attach the AI answer as metadata
+  if (results.length > 0) {
+    results._groundedAnswer = answer
+  }
+
+  return results
+}
+
+/**
+ * Follow a redirect URL to get the actual destination.
+ */
+function resolveRedirect(url) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    const client = urlObj.protocol === 'https:' ? https : require('http')
+    const req = client.request({
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      method: 'HEAD',
+      headers: { 'User-Agent': 'Spectrawl/0.3' }
+    }, res => {
+      if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location) {
+        resolve(res.headers.location)
+      } else {
+        resolve(url)
+      }
+    })
+    req.on('error', () => resolve(url))
+    req.setTimeout(3000, () => { req.destroy(); resolve(url) })
+    req.end()
+  })
+}
+
+function post(url, body) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    const opts = {
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(body)
+      }
+    }
+    const req = https.request(opts, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { reject(new Error(`Invalid Gemini response: ${data.slice(0, 200)}`)) }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(15000, () => { req.destroy(); reject(new Error('Gemini grounded search timeout')) })
+    req.write(body)
+    req.end()
+  })
+}
+
+module.exports = { geminiGroundedSearch }

package/src/search/index.js

@@ -4,6 +4,7 @@ const { serperSearch } = require('./engines/serper')
 const { searxngSearch } = require('./engines/searxng')
 const { googleCseSearch } = require('./engines/google-cse')
 const { jinaSearch } = require('./engines/jina')
+const { geminiGroundedSearch } = require('./engines/gemini-grounded')
 const { scrapeUrls } = require('./scraper')
 const { Summarizer } = require('./summarizer')
 const { Reranker } = require('./reranker')
@@ -15,7 +16,9 @@ const ENGINES = {
   brave: braveSearch,
   serper: serperSearch,
   'google-cse': googleCseSearch,
-  jina: jinaSearch
+  jina: jinaSearch,
+  'gemini-grounded': geminiGroundedSearch,
+  gemini: geminiGroundedSearch
 }
 
 class SearchEngine {
@@ -105,41 +108,49 @@ class SearchEngine {
     }
 
     // Check cache
-    const cacheKey = `deep:${query}:${JSON.stringify(opts)}`
+    const cacheKey = `deep:${opts.mode || 'full'}:${query}`
     const cached = this.cache?.get('search', cacheKey)
     if (cached) return { ...cached, cached: true }
 
-    // Step 1: Query expansion
+    // Step 1: Query expansion (skip if using Gemini grounded — it searches Google natively)
     let queries = [query]
-    if (this.expander && opts.expand !== false) {
+    const usesGrounded = this.cascade.includes('gemini-grounded') || this.cascade.includes('gemini')
+    if (this.expander && opts.expand !== false && !usesGrounded) {
       queries = await this.expander.expand(query)
     }
 
-    // Step 2: Search across all query variants (with stagger to avoid rate limits)
+    // Step 2: Search across all query variants
+    // When using Gemini Grounded, also run DDG in parallel for volume
     const resultSets = []
-    for (const q of queries) {
-      try {
-        const r = await this._rawSearch(q, opts)
-        resultSets.push(r)
-      } catch (e) {
-        resultSets.push([])
+    if (usesGrounded) {
+      // Parallel: Gemini for quality + DDG for volume
+      const [groundedResults, ddgResults] = await Promise.all([
+        this._rawSearch(query, { ...opts, engines: ['gemini-grounded', 'gemini'] }).catch(() => []),
+        this._rawSearch(query, { ...opts, engines: ['ddg'] }).catch(() => [])
+      ])
+      resultSets.push(groundedResults, ddgResults)
+    } else {
+      for (const q of queries) {
+        try {
+          const r = await this._rawSearch(q, opts)
+          resultSets.push(r)
+        } catch (e) {
+          resultSets.push([])
+        }
+        if (queries.length > 1) await new Promise(r => setTimeout(r, 300))
       }
-      // Small delay between queries to avoid rate limiting
-      if (queries.length > 1) await new Promise(r => setTimeout(r, 300))
     }
 
     // Step 3: Merge and deduplicate
-    let results = this.expander
-      ? this.expander.mergeResults(resultSets)
-      : dedupeResults(resultSets.flat())
+    let results = dedupeResults(resultSets.flat())
 
     // Step 4: Rerank by relevance
     if (this.reranker && opts.rerank !== false) {
       results = await this.reranker.rerank(query, results)
     }
 
-    // Step 5: Parallel scrape top N for full content
-    const scrapeCount = opts.scrapeTop ?? this.scrapeTop ?? 5
+    // Step 5: Parallel scrape top N for full content (skip in fast mode)
+    const scrapeCount = opts.mode === 'fast' ? 0 : (opts.scrapeTop ?? this.scrapeTop ?? 5)
     if (scrapeCount > 0 && results.length > 0) {
       const urls = results.slice(0, scrapeCount).map(r => r.url)
       const scraped = await scrapeUrls(urls)
@@ -188,8 +199,9 @@ class SearchEngine {
   async _rawSearch(query, opts = {}) {
     let results = []
     const minResults = opts.minResults || 5
+    const cascade = opts.engines || this.cascade
 
-    for (const engineName of this.cascade) {
+    for (const engineName of cascade) {
       const engine = ENGINES[engineName]
       if (!engine) continue
 

package/src/search/scraper.js

@@ -16,17 +16,20 @@ async function scrapeUrls(urls, opts = {}) {
   const concurrent = opts.concurrent || 3
   const engine = opts.engine || 'auto' // 'jina', 'readability', 'auto'
 
-  for (let i = 0; i < urls.length; i += concurrent) {
-    const batch = urls.slice(i, i + concurrent)
-    const promises = batch.map(url => scrapeUrl(url, { timeout, engine }).catch(() => null))
-    const batchResults = await Promise.all(promises)
-    
-    batch.forEach((url, idx) => {
-      if (batchResults[idx]) {
-        results[url] = batchResults[idx]
-      }
-    })
-  }
+  // All URLs in parallel (with per-URL timeout)
+  const promises = urls.map(url => {
+    const p = scrapeUrl(url, { timeout, engine }).catch(() => null)
+    // Hard timeout per URL
+    const timer = new Promise(resolve => setTimeout(() => resolve(null), timeout + 1000))
+    return Promise.race([p, timer])
+  })
+  const allResults = await Promise.all(promises)
+  
+  urls.forEach((url, idx) => {
+    if (allResults[idx]) {
+      results[url] = allResults[idx]
+    }
+  })
 
   return results
 }