spectrawl 0.1.0

package/src/mcp.js

@@ -0,0 +1,195 @@
+/**
+ * MCP (Model Context Protocol) server for Spectrawl.
+ * Exposes search, browse, act, auth, and status as MCP tools.
+ * Communicates over stdio (standard MCP transport).
+ */
+
+const { Spectrawl } = require('./index')
+
+const TOOLS = [
+  {
+    name: 'web_search',
+    description: 'Search the web using free API cascade (DuckDuckGo, Brave, Serper). Returns results with optional LLM summary and full page content.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: { type: 'string', description: 'Search query' },
+        summarize: { type: 'boolean', description: 'Generate LLM summary with citations', default: false },
+        scrapeTop: { type: 'number', description: 'Number of top results to scrape for full content', default: 3 },
+        minResults: { type: 'number', description: 'Minimum results before trying next engine', default: 5 }
+      },
+      required: ['query']
+    }
+  },
+  {
+    name: 'web_browse',
+    description: 'Browse a URL with stealth anti-detection. Extracts text content, optionally takes screenshots. Supports authenticated sessions.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        url: { type: 'string', description: 'URL to browse' },
+        auth: { type: 'string', description: 'Platform name to use stored auth (e.g. "reddit", "x")' },
+        screenshot: { type: 'boolean', description: 'Take a screenshot', default: false },
+        html: { type: 'boolean', description: 'Return raw HTML', default: false },
+        stealth: { type: 'boolean', description: 'Force stealth browser mode', default: false }
+      },
+      required: ['url']
+    }
+  },
+  {
+    name: 'web_act',
+    description: 'Perform an authenticated action on a platform (post, comment, like, etc). Supports X/Twitter, Reddit, Dev.to.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        platform: { type: 'string', description: 'Platform name', enum: ['x', 'reddit', 'devto', 'hashnode', 'linkedin', 'ih'] },
+        action: { type: 'string', description: 'Action to perform (post, comment, like, delete)' },
+        account: { type: 'string', description: 'Account handle (e.g. @myhandle)' },
+        text: { type: 'string', description: 'Text content for post/comment' },
+        title: { type: 'string', description: 'Title (for Reddit/Dev.to posts)' },
+        subreddit: { type: 'string', description: 'Subreddit name (Reddit only)' },
+        tags: { type: 'array', items: { type: 'string' }, description: 'Tags (Dev.to only)' }
+      },
+      required: ['platform', 'action']
+    }
+  },
+  {
+    name: 'web_auth',
+    description: 'Manage platform authentication. Add, remove, or list accounts.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        action: { type: 'string', description: 'Auth action', enum: ['list', 'add', 'remove'] },
+        platform: { type: 'string', description: 'Platform name' },
+        account: { type: 'string', description: 'Account handle' }
+      },
+      required: ['action']
+    }
+  },
+  {
+    name: 'web_status',
+    description: 'Check health status of all authenticated accounts. Shows cookie expiry, OAuth status, and issues.',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+      required: []
+    }
+  }
+]
+
+class MCPServer {
+  constructor(configPath) {
+    this.spectrawl = new Spectrawl(configPath)
+    this._buffer = ''
+  }
+
+  async start() {
+    process.stdin.setEncoding('utf8')
+    process.stdin.on('data', (chunk) => {
+      this._buffer += chunk
+      this._processBuffer()
+    })
+    process.stdin.on('end', () => {
+      this.spectrawl.close()
+    })
+  }
+
+  _processBuffer() {
+    const lines = this._buffer.split('\n')
+    this._buffer = lines.pop() || ''
+    
+    for (const line of lines) {
+      if (!line.trim()) continue
+      try {
+        const msg = JSON.parse(line)
+        this._handleMessage(msg)
+      } catch (e) {
+        // Not JSON, ignore
+      }
+    }
+  }
+
+  async _handleMessage(msg) {
+    if (msg.method === 'initialize') {
+      this._send({
+        jsonrpc: '2.0',
+        id: msg.id,
+        result: {
+          protocolVersion: '2024-11-05',
+          capabilities: { tools: {} },
+          serverInfo: { name: 'spectrawl', version: '0.1.0' }
+        }
+      })
+    } else if (msg.method === 'tools/list') {
+      this._send({
+        jsonrpc: '2.0',
+        id: msg.id,
+        result: { tools: TOOLS }
+      })
+    } else if (msg.method === 'tools/call') {
+      const result = await this._handleToolCall(msg.params.name, msg.params.arguments || {})
+      this._send({
+        jsonrpc: '2.0',
+        id: msg.id,
+        result: {
+          content: [{ type: 'text', text: JSON.stringify(result, null, 2) }]
+        }
+      })
+    } else if (msg.method === 'notifications/initialized') {
+      // Client acknowledged init, nothing to do
+    }
+  }
+
+  async _handleToolCall(name, args) {
+    try {
+      switch (name) {
+        case 'web_search':
+          return await this.spectrawl.search(args.query, {
+            summarize: args.summarize,
+            scrapeTop: args.scrapeTop,
+            minResults: args.minResults
+          })
+        case 'web_browse':
+          return await this.spectrawl.browse(args.url, {
+            auth: args.auth,
+            screenshot: args.screenshot,
+            html: args.html,
+            stealth: args.stealth
+          })
+        case 'web_act':
+          return await this.spectrawl.act(args.platform, args.action, {
+            account: args.account,
+            text: args.text,
+            title: args.title,
+            subreddit: args.subreddit,
+            tags: args.tags
+          })
+        case 'web_auth':
+          if (args.action === 'list') return await this.spectrawl.status()
+          if (args.action === 'remove') {
+            await this.spectrawl.auth.remove(args.platform, args.account)
+            return { removed: `${args.platform}/${args.account}` }
+          }
+          return { error: 'Use CLI for adding accounts: spectrawl login <platform>' }
+        case 'web_status':
+          return await this.spectrawl.status()
+        default:
+          return { error: `Unknown tool: ${name}` }
+      }
+    } catch (err) {
+      return { error: err.message }
+    }
+  }
+
+  _send(msg) {
+    process.stdout.write(JSON.stringify(msg) + '\n')
+  }
+}
+
+// Run if called directly
+if (require.main === module) {
+  const server = new MCPServer()
+  server.start()
+}
+
+module.exports = { MCPServer, TOOLS }

package/src/search/engines/brave.js

@@ -0,0 +1,62 @@
+const https = require('https')
+
+/**
+ * Brave Search API — 2000 queries/month free tier.
+ * Requires BRAVE_API_KEY in config or env.
+ */
+async function braveSearch(query, config = {}) {
+  const apiKey = config.apiKey || process.env.BRAVE_API_KEY
+  if (!apiKey) throw new Error('Brave API key not configured')
+
+  const maxResults = config.maxResults || 10
+  const params = new URLSearchParams({
+    q: query,
+    count: String(maxResults)
+  })
+
+  const data = await fetchJson(`https://api.search.brave.com/res/v1/web/search?${params}`, {
+    'X-Subscription-Token': apiKey,
+    'Accept': 'application/json'
+  })
+
+  if (!data.web?.results) return []
+
+  return data.web.results.map(r => ({
+    url: r.url,
+    title: r.title,
+    snippet: r.description || '',
+    engine: 'brave'
+  }))
+}
+
+function fetchJson(url, headers = {}) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    const opts = {
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      method: 'GET',
+      headers: {
+        ...headers,
+        'User-Agent': 'Spectrawl/0.1.0'
+      }
+    }
+
+    const req = https.request(opts, (res) => {
+      let data = ''
+      res.on('data', chunk => data += chunk)
+      res.on('end', () => {
+        try {
+          resolve(JSON.parse(data))
+        } catch (e) {
+          reject(new Error(`Brave API returned invalid JSON: ${data.slice(0, 200)}`))
+        }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(10000, () => { req.destroy(); reject(new Error('Brave API timeout')) })
+    req.end()
+  })
+}
+
+module.exports = { braveSearch }

package/src/search/engines/ddg.js

@@ -0,0 +1,192 @@
+const https = require('https')
+const { URL } = require('url')
+
+/**
+ * DuckDuckGo search — free, unlimited, no API key needed.
+ * Uses JSON API + HTML fallback. Filters ads automatically.
+ */
+async function ddgSearch(query, config = {}) {
+  const maxResults = config.maxResults || 10
+  
+  // Strategy 1: JSON API (instant answers)
+  try {
+    const results = await ddgJsonApi(query, maxResults)
+    if (results.length > 0) return results
+  } catch (e) { /* fall through */ }
+
+  // Strategy 2: HTML search
+  try {
+    const results = await ddgHtmlSearch(query, maxResults)
+    if (results.length > 0) return results
+  } catch (e) { /* fall through */ }
+
+  return []
+}
+
+async function ddgJsonApi(query, maxResults) {
+  const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1&skip_disambig=1`
+  const data = await fetchJson(url)
+  
+  const results = []
+
+  if (data.AbstractURL && data.Abstract) {
+    results.push({
+      url: data.AbstractURL,
+      title: data.Heading || query,
+      snippet: data.Abstract,
+      engine: 'ddg'
+    })
+  }
+
+  if (data.RelatedTopics) {
+    for (const topic of data.RelatedTopics) {
+      if (results.length >= maxResults) break
+      if (topic.FirstURL && topic.Text) {
+        results.push({
+          url: topic.FirstURL,
+          title: topic.Text.slice(0, 100),
+          snippet: topic.Text,
+          engine: 'ddg'
+        })
+      }
+      if (topic.Topics) {
+        for (const sub of topic.Topics) {
+          if (results.length >= maxResults) break
+          if (sub.FirstURL && sub.Text) {
+            results.push({
+              url: sub.FirstURL,
+              title: sub.Text.slice(0, 100),
+              snippet: sub.Text,
+              engine: 'ddg'
+            })
+          }
+        }
+      }
+    }
+  }
+
+  if (data.Results) {
+    for (const r of data.Results) {
+      if (results.length >= maxResults) break
+      if (r.FirstURL && r.Text) {
+        results.push({
+          url: r.FirstURL,
+          title: r.Text.slice(0, 100),
+          snippet: r.Text,
+          engine: 'ddg'
+        })
+      }
+    }
+  }
+
+  return results
+}
+
+async function ddgHtmlSearch(query, maxResults) {
+  const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`
+  const html = await fetchHtml(url)
+  
+  const results = []
+  
+  const resultRegex = /<a[^>]+class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/g
+  const snippetRegex = /<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g
+
+  const links = []
+  let match
+  while ((match = resultRegex.exec(html)) !== null) {
+    const url = decodeUddg(match[1])
+    // Filter ads — DDG ads go through duckduckgo.com/y.js
+    if (isAd(url)) continue
+    links.push({ url, title: stripHtml(match[2]) })
+  }
+
+  const snippets = []
+  while ((match = snippetRegex.exec(html)) !== null) {
+    snippets.push(stripHtml(match[1]))
+  }
+
+  for (let i = 0; i < Math.min(links.length, maxResults); i++) {
+    results.push({
+      url: links[i].url,
+      title: links[i].title,
+      snippet: snippets[i] || '',
+      engine: 'ddg'
+    })
+  }
+
+  return results
+}
+
+/**
+ * Filter out DDG ads.
+ */
+function isAd(url) {
+  if (!url) return true
+  if (url.includes('duckduckgo.com/y.js')) return true
+  if (url.includes('ad_provider=')) return true
+  if (url.includes('ad_domain=')) return true
+  if (url.startsWith('//duckduckgo.com/l/?')) {
+    // This is a redirect — might be organic
+    return false
+  }
+  return false
+}
+
+function decodeUddg(url) {
+  if (url.includes('uddg=')) {
+    const match = url.match(/uddg=([^&]+)/)
+    if (match) return decodeURIComponent(match[1])
+  }
+  return url
+}
+
+function fetchJson(url) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    https.get({
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      headers: { 'User-Agent': 'Spectrawl/0.1.0' }
+    }, res => {
+      let data = ''
+      res.on('data', chunk => data += chunk)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { reject(new Error('Invalid JSON from DDG API')) }
+      })
+    }).on('error', reject)
+  })
+}
+
+function fetchHtml(url) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    https.get({
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+        'Accept': 'text/html',
+        'Accept-Language': 'en-US,en;q=0.9'
+      }
+    }, res => {
+      let data = ''
+      res.on('data', chunk => data += chunk)
+      res.on('end', () => resolve(data))
+    }).on('error', reject)
+  })
+}
+
+function stripHtml(html) {
+  return html
+    .replace(/<[^>]+>/g, '')
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/\s+/g, ' ')
+    .trim()
+}
+
+module.exports = { ddgSearch }

package/src/search/engines/google-cse.js

@@ -0,0 +1,50 @@
+const https = require('https')
+
+/**
+ * Google Custom Search Engine — 100 queries/day free.
+ * Requires GOOGLE_CSE_KEY and GOOGLE_CSE_ID in config or env.
+ */
+async function googleCseSearch(query, config = {}) {
+  const apiKey = config.apiKey || process.env.GOOGLE_CSE_KEY
+  const cseId = config.cseId || process.env.GOOGLE_CSE_ID
+  if (!apiKey || !cseId) throw new Error('Google CSE key/ID not configured')
+
+  const maxResults = Math.min(config.maxResults || 10, 10) // Google caps at 10
+  const params = new URLSearchParams({
+    key: apiKey,
+    cx: cseId,
+    q: query,
+    num: String(maxResults)
+  })
+
+  const data = await fetchJson(`https://www.googleapis.com/customsearch/v1?${params}`)
+
+  if (!data.items) return []
+
+  return data.items.map(r => ({
+    url: r.link,
+    title: r.title,
+    snippet: r.snippet || '',
+    engine: 'google-cse'
+  }))
+}
+
+function fetchJson(url) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    https.get({
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      headers: { 'User-Agent': 'Spectrawl/0.1.0' }
+    }, res => {
+      let data = ''
+      res.on('data', chunk => data += chunk)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { reject(new Error('Invalid JSON from Google CSE')) }
+      })
+    }).on('error', reject)
+  })
+}
+
+module.exports = { googleCseSearch }

package/src/search/engines/jina.js

@@ -0,0 +1,76 @@
+const https = require('https')
+
+/**
+ * Jina Reader — AI-optimized content extraction.
+ * Prepend r.jina.ai/ to any URL for clean markdown output.
+ * Free tier available, no API key required for basic use.
+ */
+async function jinaExtract(url, config = {}) {
+  const apiKey = config.apiKey || process.env.JINA_API_KEY
+  const readerUrl = `https://r.jina.ai/${url}`
+
+  const headers = {
+    'Accept': 'application/json',
+    'User-Agent': 'Spectrawl/0.1.0'
+  }
+  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`
+
+  const data = await fetchJson(readerUrl, headers)
+
+  return {
+    content: data.data?.content || data.content || '',
+    title: data.data?.title || data.title || '',
+    url: data.data?.url || url,
+    description: data.data?.description || ''
+  }
+}
+
+/**
+ * Jina Search — search + extract in one call.
+ * Prepend s.jina.ai/ to a query for search results.
+ */
+async function jinaSearch(query, config = {}) {
+  const apiKey = config.apiKey || process.env.JINA_API_KEY
+  const searchUrl = `https://s.jina.ai/${encodeURIComponent(query)}`
+  const maxResults = config.maxResults || 5
+
+  const headers = {
+    'Accept': 'application/json',
+    'User-Agent': 'Spectrawl/0.1.0'
+  }
+  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`
+
+  const data = await fetchJson(searchUrl, headers)
+
+  const results = (data.data || []).slice(0, maxResults)
+  return results.map(r => ({
+    url: r.url,
+    title: r.title || '',
+    snippet: r.description || '',
+    fullContent: r.content || '',
+    engine: 'jina'
+  }))
+}
+
+function fetchJson(url, headers = {}) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    https.get({
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      headers
+    }, res => {
+      let data = ''
+      res.on('data', chunk => data += chunk)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { 
+          // Jina sometimes returns plain text/markdown
+          resolve({ content: data, title: '', url }) 
+        }
+      })
+    }).on('error', reject)
+  })
+}
+
+module.exports = { jinaExtract, jinaSearch }

package/src/search/engines/searxng.js

@@ -0,0 +1,69 @@
+const http = require('http')
+const https = require('https')
+
+/**
+ * SearXNG — self-hosted metasearch engine.
+ * Aggregates 70+ search engines (Google, Bing, DDG, etc.)
+ * Free, unlimited, no API key needed.
+ * 
+ * Requires a SearXNG instance running (self-hosted or public).
+ * Default: http://localhost:8888 (local Docker instance)
+ * 
+ * Docker quick start:
+ *   docker run -d -p 8888:8080 searxng/searxng
+ */
+async function searxngSearch(query, config = {}) {
+  const baseUrl = config.url || process.env.SEARXNG_URL || 'http://localhost:8888'
+  const maxResults = config.maxResults || 10
+  const engines = config.engines || '' // empty = all engines
+  const categories = config.categories || 'general'
+
+  const params = new URLSearchParams({
+    q: query,
+    format: 'json',
+    categories,
+    pageno: '1'
+  })
+  
+  if (engines) params.set('engines', engines)
+
+  const url = `${baseUrl}/search?${params}`
+  const data = await fetchJson(url)
+
+  if (!data.results) return []
+
+  return data.results.slice(0, maxResults).map(r => ({
+    url: r.url,
+    title: r.title || '',
+    snippet: r.content || '',
+    engine: r.engine || 'searxng',
+    engines: r.engines || [],
+    score: r.score || 0
+  }))
+}
+
+function fetchJson(url) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    const client = urlObj.protocol === 'https:' ? https : http
+
+    client.get({
+      hostname: urlObj.hostname,
+      port: urlObj.port,
+      path: urlObj.pathname + urlObj.search,
+      headers: {
+        'Accept': 'application/json',
+        'User-Agent': 'Spectrawl/0.1.0'
+      }
+    }, res => {
+      let data = ''
+      res.on('data', chunk => data += chunk)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { reject(new Error(`SearXNG returned invalid JSON: ${data.slice(0, 200)}`)) }
+      })
+    }).on('error', reject)
+  })
+}
+
+module.exports = { searxngSearch }