spectrawl 0.3.18 → 0.3.22

package/README.md

@@ -2,7 +2,7 @@
 
 The unified web layer for AI agents. Search, browse, authenticate, and act on platforms — one package, self-hosted.
 
-**5,000 free searches/month** via Gemini Grounded Search. Full page scraping, stealth browsing, 24 platform adapters.
+**5,000 free searches/month** via Gemini Grounded Search. Full page scraping, stealth browsing, 19 platform adapters.
 
 ## What It Does
 
@@ -55,7 +55,7 @@ Different tools for different needs.
 | Returns | Snippets + AI answer | Full page content + snippets |
 | Self-hosted | No | Yes |
 | Stealth browsing | No | Yes (Camoufox + Playwright) |
-| Platform posting | No | 24 adapters |
+| Platform posting | No | 19 adapters |
 | Auth management | No | Cookie store + auto-refresh |
 | Cached repeats | No | <1ms |
 
@@ -63,15 +63,16 @@ Different tools for different needs.
 
 ## Search
 
-Default cascade: **Gemini Grounded → Brave → DDG**
+Default cascade: **Gemini Grounded → Tavily → Brave**
 
 Gemini Grounded Search gives you Google-quality results through the Gemini API. Free tier: 5,000 grounded queries/month.
 
 | Engine | Free Tier | Key Required | Default |
 |--------|-----------|-------------|---------|
 | **Gemini Grounded** | 5,000/month | `GEMINI_API_KEY` | ✅ Primary |
-| Brave | 2,000/month | `BRAVE_API_KEY` | ✅ Fallback |
-| DuckDuckGo | Unlimited | None | ✅ Last resort |
+| Tavily | 1,000/month | `TAVILY_API_KEY` | ✅ 1st fallback |
+| Brave | 2,000/month | `BRAVE_API_KEY` | ✅ 2nd fallback |
+| DuckDuckGo | Unlimited | None | Available |
 | Bing | Unlimited | None | Available |
 | Serper | 2,500 trial | `SERPER_API_KEY` | Available |
 | Google CSE | 100/day | `GOOGLE_CSE_KEY` | Available |
@@ -123,9 +124,9 @@ const accounts = await web.auth.getStatus()
 
 Cookie refresh cron fires `cookie_expiring` and `cookie_expired` events before accounts go stale.
 
-## Act — 24 Platform Adapters
+## Act — 19 Platform Adapters
 
-Post to 24+ platforms with one API:
+Post to 19 platforms with one API:
 
 ```js
 await web.act('github', 'create-issue', { repo: 'user/repo', title: 'Bug report', body: '...' })
@@ -134,7 +135,7 @@ await web.act('devto', 'post', { title: '...', body: '...', tags: ['ai'] })
 await web.act('huggingface', 'create-repo', { name: 'my-model', type: 'model' })
 ```
 
-**Live tested:** GitHub ✅, Reddit ✅, Dev.to ✅, HuggingFace ✅, X (reads) ✅
+**Live tested:** GitHub ✅, Reddit ✅, Dev.to ✅, HuggingFace ✅, X (reads) ✅, Hashnode ✅, Discord ✅, Product Hunt ✅
 
 | Platform | Auth Method | Actions |
 |----------|-------------|---------|
@@ -153,7 +154,10 @@ await web.act('huggingface', 'create-repo', { name: 'my-model', type: 'model' })
 | Quora | Browser automation | answer |
 | HuggingFace | Hub API | repo, model card, upload |
 | BetaList | REST API | submit |
-| **14 Directories** | Generic adapter | submit |
+| AlternativeTo | Cookie session | submit, claim |
+| DevHunt | Supabase auth | submit, upvote |
+| SaaSHub | Generic adapter | submit |
+| **Generic Directory** | Configurable | submit |
 
 Built-in rate limiting, content dedup (MD5, 24h window), and dead letter queue for retries.
 
@@ -216,7 +220,7 @@ npx spectrawl install-stealth   # download Camoufox browser
 ```json
 {
   "search": {
-    "cascade": ["gemini-grounded", "brave", "ddg"],
+    "cascade": ["gemini-grounded", "tavily", "brave"],
     "scrapeTop": 5
   },
   "cache": {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "spectrawl",
-  "version": "0.3.18",
-  "description": "The unified web layer for AI agents. Search (6 engines), stealth browse (Camoufox + Playwright), auth (cookies, multi-account), act (24 adapters, 30+ platforms), proxy rotation. Self-hosted, free.",
+  "version": "0.3.22",
+  "description": "The unified web layer for AI agents. Search (8 engines), stealth browse, auth, act on 24 platforms. Self-hosted.",
   "main": "src/index.js",
   "types": "index.d.ts",
   "bin": {

package/src/server.js

@@ -61,6 +61,65 @@ const server = http.createServer(async (req, res) => {
       return json(res, result)
     }
 
+    // Threads OAuth callback
+    if (req.method === 'GET' && path === '/auth/callback/threads') {
+      const code = url.searchParams.get('code')
+      const errParam = url.searchParams.get('error')
+      if (errParam) {
+        res.writeHead(200, { 'Content-Type': 'text/html' })
+        return res.end(`<h2>❌ Auth error: ${errParam}</h2>`)
+      }
+      if (!code) {
+        res.writeHead(400, { 'Content-Type': 'text/html' })
+        return res.end('<h2>❌ No code received</h2>')
+      }
+      try {
+        // Exchange code for token
+        const fetch = require('node:https')
+        const params = new URLSearchParams({
+          client_id: '1574846783732558',
+          client_secret: 'f8589ca3523b0ea5bab3fac2c2ae4c15',
+          code,
+          grant_type: 'authorization_code',
+          redirect_uri: 'https://gateway.xanos.org/auth/callback/threads'
+        })
+        const tokenRes = await new Promise((resolve, reject) => {
+          const postData = params.toString()
+          const options = {
+            hostname: 'graph.threads.net',
+            path: '/oauth/access_token',
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/x-www-form-urlencoded',
+              'Content-Length': Buffer.byteLength(postData)
+            }
+          }
+          const req2 = fetch.request(options, (r) => {
+            let data = ''
+            r.on('data', chunk => data += chunk)
+            r.on('end', () => resolve(JSON.parse(data)))
+          })
+          req2.on('error', reject)
+          req2.write(postData)
+          req2.end()
+        })
+        // Save to credentials
+        const fs = require('fs')
+        const credsPath = '/root/.openclaw/workspace-dijiclaw/.openclaw/credentials/threads-api.json'
+        const creds = JSON.parse(fs.readFileSync(credsPath, 'utf8'))
+        creds.user_token = tokenRes.access_token
+        creds.user_id = tokenRes.user_id
+        creds.token_type = tokenRes.token_type
+        creds.note = 'User token saved via OAuth callback'
+        fs.writeFileSync(credsPath, JSON.stringify(creds, null, 2))
+        res.writeHead(200, { 'Content-Type': 'text/html' })
+        return res.end('<h2>✅ Threads connected! You can close this tab.</h2>')
+      } catch (e) {
+        res.writeHead(500, { 'Content-Type': 'text/html' })
+        return res.end(`<h2>❌ Token exchange failed: ${e.message}</h2>`)
+      }
+    }
+
     return error(res, 404, 'Not found')
   } catch (err) {
     console.error('Server error:', err)