spectrawl 0.3.2 → 0.3.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "The unified web layer for AI agents. Search (6 engines), stealth browse (Camoufox + Playwright), auth (cookies, multi-account), act (24 adapters, 30+ platforms), proxy rotation. Self-hosted, free.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/search/index.js

@@ -144,8 +144,8 @@ class SearchEngine {
     // Step 3: Merge and deduplicate
     let results = dedupeResults(resultSets.flat())
 
-    // Step 4: Rerank by relevance
-    if (this.reranker && opts.rerank !== false) {
+    // Step 4: Rerank by relevance (skip for Gemini Grounded — it already returns scored results)
+    if (this.reranker && opts.rerank !== false && !usesGrounded) {
       results = await this.reranker.rerank(query, results)
     }
 
@@ -167,7 +167,7 @@ class SearchEngine {
     let answer = null
     const summarizer = this.summarizer || (this.reranker ? new Summarizer({
       provider: 'gemini',
-      model: 'gemini-2.0-flash',
+      model: 'gemini-2.5-flash',
       apiKey: process.env.GEMINI_API_KEY
     }) : null)
 
@@ -177,12 +177,12 @@ class SearchEngine {
 
     const response = {
       answer,
-      sources: results.map(r => ({
+      sources: results.map((r, i) => ({
         title: r.title,
         url: r.url,
         snippet: r.snippet,
         content: r.fullContent?.slice(0, 2000) || r.snippet || '',
-        score: r.score || null
+        score: r.score || r.confidence || Math.max(0.5, 1 - (i * 0.05))
       })),
       queries, // show which queries were used
       cached: false

package/src/search/query-expander.js

@@ -8,7 +8,7 @@ const https = require('https')
 class QueryExpander {
   constructor(config = {}) {
     this.provider = config.provider || 'gemini'
-    this.model = config.model || 'gemini-2.0-flash'
+    this.model = config.model || 'gemini-2.5-flash'
     this.apiKey = config.apiKey || process.env.GEMINI_API_KEY
     this.variants = config.variants || 3
   }
@@ -69,7 +69,7 @@ Example: ["alternative query 1", "alternative query 2", "alternative query 3"]`
 
   async _call(prompt) {
     if (this.provider === 'gemini') {
-      const model = this.model || 'gemini-2.0-flash'
+      const model = this.model || 'gemini-2.5-flash'
       const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${this.apiKey}`
       const body = JSON.stringify({
         contents: [{ parts: [{ text: prompt }] }],

package/src/search/reranker.js

@@ -8,7 +8,7 @@ const https = require('https')
 class Reranker {
   constructor(config = {}) {
     this.provider = config.provider || 'gemini'
-    this.model = config.model || 'gemini-2.0-flash'
+    this.model = config.model || 'gemini-2.5-flash'
     this.apiKey = config.apiKey || process.env.GEMINI_API_KEY
   }
 
@@ -55,7 +55,7 @@ No explanation, just the array.`
 
   async _call(prompt) {
     if (this.provider === 'gemini') {
-      const model = this.model || 'gemini-2.0-flash'
+      const model = this.model || 'gemini-2.5-flash'
       const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${this.apiKey}`
       const body = JSON.stringify({
         contents: [{ parts: [{ text: prompt }] }],

package/src/search/scraper.js

@@ -35,13 +35,13 @@ async function scrapeUrls(urls, opts = {}) {
 }
 
 async function scrapeUrl(url, opts = {}) {
-  const { timeout = 10000, engine = 'auto' } = opts
+  const { timeout = 10000, engine = 'auto', browse } = opts
 
   // Try Jina first if available (better markdown output)
   if (engine === 'jina' || engine === 'auto') {
     try {
       const result = await jinaExtract(url)
-      if (result.content && result.content.length > 100) {
+      if (result.content && result.content.length > 200) {
         return result.content
       }
     } catch (e) {
@@ -49,9 +49,44 @@ async function scrapeUrl(url, opts = {}) {
     }
   }
 
-  // Readability fallback
-  const html = await fetchPage(url, timeout)
-  return extractMarkdown(html)
+  // Readability fallback (HTTP fetch + HTML→markdown)
+  try {
+    const html = await fetchPage(url, timeout)
+    const content = extractMarkdown(html)
+    if (content && content.length > 200) {
+      return content
+    }
+  } catch (e) {
+    // Fall through to browser
+  }
+
+  // Browser fallback for JS-rendered pages or when extraction is too short
+  // This is where we beat Tavily — they can't render JS pages
+  if (browse !== false) {
+    try {
+      const { BrowseEngine } = require('../browse')
+      const browser = new BrowseEngine()
+      const result = await browser.browse(url, { 
+        timeout, 
+        extractText: true,
+        screenshot: false 
+      })
+      await browser.close()
+      if (result.text && result.text.length > 200) {
+        return result.text
+      }
+    } catch (e) {
+      // All methods exhausted
+    }
+  }
+
+  // Return whatever we got, even if short
+  try {
+    const html = await fetchPage(url, timeout)
+    return extractMarkdown(html)
+  } catch (e) {
+    return ''
+  }
 }
 
 function fetchPage(url, timeout = 10000, redirects = 3) {

package/src/search/summarizer.js

@@ -110,7 +110,7 @@ Answer:`
   }
 
   async _gemini(prompt) {
-    const model = this.model || 'gemini-2.0-flash'
+    const model = this.model || 'gemini-2.5-flash'
     const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${this.apiKey}`
     const body = JSON.stringify({
       contents: [{ parts: [{ text: prompt }] }],