spectrawl 0.3.11 → 0.3.13

package/README.md

@@ -2,7 +2,7 @@
 
 The unified web layer for AI agents. Search, browse, authenticate, and act on platforms — one tool, self-hosted, free.
 
-**Free Tavily alternative** with Google-quality results via Gemini Grounded Search.
+**5,000 free searches/month** with Google-quality results via Gemini Grounded Search. Better answers than Tavily. Self-hosted.
 
 ## What It Does
 
@@ -57,18 +57,23 @@ export GEMINI_API_KEY=your-free-key  # Get one at aistudio.google.com
 const { Spectrawl } = require('spectrawl')
 const web = new Spectrawl()
 
-// Deep search — like Tavily but free, with better answers
+// Deep search — returns sources for your agent/LLM to process
 const result = await web.deepSearch('how to build an MCP server in Node.js')
-console.log(result.answer)    // AI-generated answer with [1] [2] citations
 console.log(result.sources)   // [{ title, url, content, score }]
 
+// With AI summary (opt-in — uses extra Gemini call)
+const withAnswer = await web.deepSearch('query', { summarize: true })
+console.log(withAnswer.answer)  // AI-generated answer with [1] [2] citations
+
 // Fast mode — snippets only, skip scraping
 const fast = await web.deepSearch('query', { mode: 'fast' })
 
-// Basic search — raw results, no AI
+// Basic search — raw results
 const basic = await web.search('query')
 ```
 
+> **Why no summary by default?** Your agent already has an LLM. If we summarize AND your agent summarizes, you're paying two LLMs for one answer. We return rich sources — your agent does the rest.
+
 ## vs Tavily
 
 | | Tavily | Spectrawl |
@@ -77,7 +82,7 @@ const basic = await web.search('query')
 | Answer quality | Generic (3 items) | **Detailed** (12+ items) ✅ |
 | Inline citations | ❌ | **[1] [2] [3]** ✅ |
 | Results per query | 10 | **12-19** ✅ |
-| Cost | $0.01/query | **Free** ✅ |
+| Cost | $0.01/query | **Free** (5K/mo) ✅ |
 | Self-hosted | No | **Yes** ✅ |
 | Source ranking | No | **Domain trust scoring** ✅ |
 | Stealth scraping | No | **Yes** ✅ |
@@ -110,7 +115,7 @@ Query → Gemini Grounded + DDG (parallel)
   → Merge & deduplicate (12-19 results)
   → Source quality ranking (boost GitHub/SO/Reddit, penalize SEO spam)
   → Parallel scraping (Jina → readability → Playwright fallback)
-  → AI summarization with [1] [2] citations (gemini-2.5-flash)
+  → Returns sources to your agent (AI summary opt-in with summarize: true)
 ```
 
 ### What you get without any keys

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.3.11",
+  "version": "0.3.13",
   "description": "The unified web layer for AI agents. Search (6 engines), stealth browse (Camoufox + Playwright), auth (cookies, multi-account), act (24 adapters, 30+ platforms), proxy rotation. Self-hosted, free.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/search/index.js

@@ -37,6 +37,13 @@ class SearchEngine {
     this.reranker = geminiKey ? new Reranker({ apiKey: geminiKey, ...config.reranker }) : null
     this.expander = geminiKey ? new QueryExpander({ apiKey: geminiKey, ...config.expander }) : null
     this.sourceRanker = new SourceRanker(config.sourceRanker || {})
+
+    // One-time warning if no Gemini key
+    if (!geminiKey && !SearchEngine._keyWarned) {
+      SearchEngine._keyWarned = true
+      console.warn('\n⚠️  No GEMINI_API_KEY set. Using DDG fallback (limited quality, unreliable from servers).')
+      console.warn('   Get a free key (no credit card): https://aistudio.google.com/apikey\n')
+    }
   }
 
   /**
@@ -188,16 +195,18 @@ class SearchEngine {
       }
     }
 
-    // Step 6: Summarize with citations
+    // Step 6: Summarize with citations (opt-in — most agents have their own LLM)
     let answer = null
-    const summarizer = this.summarizer || (this.reranker ? new Summarizer({
-      provider: 'gemini',
-      model: 'gemini-2.5-flash',
-      apiKey: process.env.GEMINI_API_KEY
-    }) : null)
+    if (opts.summarize === true) {
+      const summarizer = this.summarizer || (process.env.GEMINI_API_KEY ? new Summarizer({
+        provider: 'gemini',
+        model: 'gemini-2.5-flash',
+        apiKey: process.env.GEMINI_API_KEY
+      }) : null)
 
-    if (summarizer) {
-      answer = await summarizer.summarize(query, results)
+      if (summarizer) {
+        answer = await summarizer.summarize(query, results)
+      }
     }
 
     const response = {