spectrawl 0.1.0

package/src/browse/install-stealth.js

@@ -0,0 +1,188 @@
+#!/usr/bin/env node
+
+/**
+ * Camoufox binary installer for Spectrawl.
+ * Downloads prebuilt anti-detect Firefox from Camoufox releases.
+ * 
+ * Usage: npx spectrawl install-stealth
+ * 
+ * Same model as `npx playwright install chromium` — downloads the
+ * right binary for your OS/arch and stores it locally.
+ */
+
+const https = require('https')
+const http = require('http')
+const fs = require('fs')
+const path = require('path')
+const { execSync } = require('child_process')
+const os = require('os')
+
+const CAMOUFOX_VERSION = 'v135.0.1-beta.24'
+const INSTALL_DIR = path.join(os.homedir(), '.spectrawl', 'browsers', 'camoufox')
+
+function getPlatformAsset() {
+  const platform = os.platform()
+  const arch = os.arch()
+
+  if (platform === 'linux') {
+    if (arch === 'x64') return `camoufox-135.0.1-beta.24-lin.x86_64.zip`
+    if (arch === 'arm64') return `camoufox-135.0.1-beta.24-lin.arm64.zip`
+    if (arch === 'ia32') return `camoufox-135.0.1-beta.24-lin.i686.zip`
+  }
+  if (platform === 'darwin') {
+    return `camoufox-135.0.1-beta.24-mac.universal.zip`
+  }
+  // Windows not yet supported in Camoufox latest
+  throw new Error(`Unsupported platform: ${platform}-${arch}. Camoufox supports Linux (x64/arm64) and macOS.`)
+}
+
+function downloadUrl(assetName) {
+  return `https://github.com/daijro/camoufox/releases/download/${CAMOUFOX_VERSION}/${assetName}`
+}
+
+function followRedirects(url) {
+  return new Promise((resolve, reject) => {
+    const proto = url.startsWith('https') ? https : http
+    proto.get(url, { headers: { 'User-Agent': 'spectrawl' } }, (res) => {
+      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+        followRedirects(res.headers.location).then(resolve).catch(reject)
+        return
+      }
+      if (res.statusCode !== 200) {
+        reject(new Error(`Download failed: HTTP ${res.statusCode}`))
+        return
+      }
+      resolve(res)
+    }).on('error', reject)
+  })
+}
+
+async function download(url, dest) {
+  const dir = path.dirname(dest)
+  fs.mkdirSync(dir, { recursive: true })
+
+  console.log(`Downloading Camoufox ${CAMOUFOX_VERSION}...`)
+  console.log(`  From: ${url}`)
+  console.log(`  To: ${dest}`)
+
+  const res = await followRedirects(url)
+  const total = parseInt(res.headers['content-length'], 10) || 0
+  let downloaded = 0
+  let lastPercent = 0
+
+  const file = fs.createWriteStream(dest)
+
+  return new Promise((resolve, reject) => {
+    res.on('data', (chunk) => {
+      file.write(chunk)
+      downloaded += chunk.length
+      if (total) {
+        const percent = Math.floor((downloaded / total) * 100)
+        if (percent >= lastPercent + 10) {
+          process.stdout.write(`  ${percent}%`)
+          if (percent < 100) process.stdout.write(' ')
+          lastPercent = percent
+        }
+      }
+    })
+    res.on('end', () => {
+      file.end()
+      console.log('\n  Download complete.')
+      resolve()
+    })
+    res.on('error', reject)
+    file.on('error', reject)
+  })
+}
+
+async function install() {
+  const asset = getPlatformAsset()
+  const url = downloadUrl(asset)
+  const zipPath = path.join(INSTALL_DIR, asset)
+  const extractDir = INSTALL_DIR
+
+  // Check if already installed
+  const markerFile = path.join(INSTALL_DIR, '.version')
+  if (fs.existsSync(markerFile)) {
+    const installed = fs.readFileSync(markerFile, 'utf8').trim()
+    if (installed === CAMOUFOX_VERSION) {
+      console.log(`Camoufox ${CAMOUFOX_VERSION} already installed at ${INSTALL_DIR}`)
+      return { path: INSTALL_DIR, version: CAMOUFOX_VERSION }
+    }
+  }
+
+  // Download
+  await download(url, zipPath)
+
+  // Extract
+  console.log('  Extracting...')
+  fs.mkdirSync(extractDir, { recursive: true })
+
+  try {
+    execSync(`unzip -o "${zipPath}" -d "${extractDir}"`, { stdio: 'pipe' })
+  } catch (e) {
+    // Try with built-in tools on systems without unzip
+    execSync(`python3 -c "import zipfile; zipfile.ZipFile('${zipPath}').extractall('${extractDir}')"`, { stdio: 'pipe' })
+  }
+
+  // Clean up zip
+  fs.unlinkSync(zipPath)
+
+  // Find the binary
+  const binaryName = os.platform() === 'darwin' ? 'camoufox' : 'camoufox-bin'
+  const possiblePaths = [
+    path.join(extractDir, 'camoufox', binaryName),
+    path.join(extractDir, binaryName),
+  ]
+
+  let binaryPath = null
+  for (const p of possiblePaths) {
+    if (fs.existsSync(p)) {
+      binaryPath = p
+      fs.chmodSync(p, 0o755)
+      break
+    }
+  }
+
+  // Write version marker
+  fs.writeFileSync(markerFile, CAMOUFOX_VERSION)
+
+  console.log(`\n✅ Camoufox ${CAMOUFOX_VERSION} installed.`)
+  console.log(`   Binary: ${binaryPath || 'in ' + extractDir}`)
+  console.log(`   Spectrawl will use it automatically for stealth browsing.`)
+
+  return { path: extractDir, binary: binaryPath, version: CAMOUFOX_VERSION }
+}
+
+/**
+ * Get the Camoufox binary path if installed.
+ */
+function getCamoufoxPath() {
+  const binaryName = os.platform() === 'darwin' ? 'camoufox' : 'camoufox-bin'
+  const possiblePaths = [
+    path.join(INSTALL_DIR, 'camoufox', binaryName),
+    path.join(INSTALL_DIR, binaryName),
+  ]
+
+  for (const p of possiblePaths) {
+    if (fs.existsSync(p)) return p
+  }
+  return null
+}
+
+/**
+ * Check if Camoufox is installed.
+ */
+function isInstalled() {
+  return getCamoufoxPath() !== null
+}
+
+// Run as CLI
+if (require.main === module) {
+  install().catch(err => {
+    console.error('❌ Installation failed:', err.message)
+    process.exit(1)
+  })
+}
+
+module.exports = { install, getCamoufoxPath, isInstalled, INSTALL_DIR }

package/src/cache.js

@@ -0,0 +1,82 @@
+const Database = require('better-sqlite3')
+const path = require('path')
+const fs = require('fs')
+const crypto = require('crypto')
+
+class Cache {
+  constructor(config = {}) {
+    const dbPath = config.path || './data/cache.db'
+    fs.mkdirSync(path.dirname(dbPath), { recursive: true })
+    
+    this.db = new Database(dbPath)
+    this.db.pragma('journal_mode = WAL')
+    this.ttls = {
+      search: config.searchTtl || 3600,
+      scrape: config.scrapeTtl || 86400,
+      screenshot: config.screenshotTtl || 3600
+    }
+    
+    this._init()
+  }
+
+  _init() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS cache (
+        key TEXT PRIMARY KEY,
+        type TEXT NOT NULL,
+        value TEXT NOT NULL,
+        created_at INTEGER NOT NULL,
+        ttl INTEGER NOT NULL
+      )
+    `)
+    
+    // Clean expired entries on startup
+    this.db.prepare('DELETE FROM cache WHERE created_at + ttl < ?').run(now())
+  }
+
+  get(type, key) {
+    const hash = this._hash(type, key)
+    const row = this.db.prepare(
+      'SELECT value FROM cache WHERE key = ? AND created_at + ttl > ?'
+    ).get(hash, now())
+    
+    return row ? JSON.parse(row.value) : null
+  }
+
+  set(type, key, value) {
+    const hash = this._hash(type, key)
+    const ttl = this.ttls[type] || 3600
+    
+    this.db.prepare(`
+      INSERT OR REPLACE INTO cache (key, type, value, created_at, ttl)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(hash, type, JSON.stringify(value), now(), ttl)
+  }
+
+  invalidate(type, key) {
+    const hash = this._hash(type, key)
+    this.db.prepare('DELETE FROM cache WHERE key = ?').run(hash)
+  }
+
+  clear(type) {
+    if (type) {
+      this.db.prepare('DELETE FROM cache WHERE type = ?').run(type)
+    } else {
+      this.db.prepare('DELETE FROM cache').run()
+    }
+  }
+
+  close() {
+    this.db.close()
+  }
+
+  _hash(type, key) {
+    return crypto.createHash('sha256').update(`${type}:${key}`).digest('hex')
+  }
+}
+
+function now() {
+  return Math.floor(Date.now() / 1000)
+}
+
+module.exports = { Cache }

package/src/cli.js

@@ -0,0 +1,160 @@
+#!/usr/bin/env node
+
+const { Spectrawl } = require('./index')
+const { DEFAULTS } = require('./config')
+const fs = require('fs')
+const path = require('path')
+
+const args = process.argv.slice(2)
+const command = args[0]
+
+async function main() {
+  switch (command) {
+    case 'init':
+      return init()
+    case 'search':
+      return search(args.slice(1).join(' '))
+    case 'status':
+      return status()
+    case 'serve':
+      return serve()
+    case 'mcp':
+      return mcp()
+    case 'install-stealth':
+      return installStealth()
+    case 'version':
+      console.log('spectrawl v0.1.0')
+      return
+    default:
+      return help()
+  }
+}
+
+function init() {
+  const configPath = path.join(process.cwd(), 'spectrawl.json')
+  if (fs.existsSync(configPath)) {
+    console.log('spectrawl.json already exists')
+    return
+  }
+
+  const config = {
+    port: DEFAULTS.port,
+    search: {
+      cascade: DEFAULTS.search.cascade,
+      scrapeTop: DEFAULTS.search.scrapeTop
+    },
+    cache: {
+      path: DEFAULTS.cache.path,
+      searchTtl: DEFAULTS.cache.searchTtl,
+      scrapeTtl: DEFAULTS.cache.scrapeTtl
+    },
+    concurrency: DEFAULTS.concurrency
+  }
+
+  fs.writeFileSync(configPath, JSON.stringify(config, null, 2))
+  console.log('Created spectrawl.json')
+}
+
+async function search(query) {
+  if (!query) {
+    console.error('Usage: spectrawl search "your query"')
+    process.exit(1)
+  }
+
+  const web = new Spectrawl()
+  try {
+    console.log(`Searching: "${query}"...\n`)
+    const results = await web.search(query, { summarize: false })
+    
+    if (results.answer) {
+      console.log('Answer:', results.answer, '\n')
+    }
+
+    for (const source of results.sources) {
+      console.log(`  ${source.title}`)
+      console.log(`  ${source.url}`)
+      console.log(`  ${source.snippet?.slice(0, 150)}`)
+      console.log()
+    }
+
+    console.log(`${results.sources.length} results${results.cached ? ' (cached)' : ''}`)
+  } finally {
+    await web.close()
+  }
+}
+
+async function status() {
+  const web = new Spectrawl()
+  try {
+    const accounts = await web.status()
+    
+    if (accounts.length === 0) {
+      console.log('No accounts configured. Run: spectrawl login <platform> --account @handle')
+      return
+    }
+
+    for (const acc of accounts) {
+      const icon = acc.status === 'valid' ? '✅' :
+                   acc.status === 'expiring' ? '⚠️' : '❌'
+      const extra = acc.expiresAt ? ` (expires ${acc.expiresAt})` : ''
+      console.log(`${icon} ${acc.platform}/${acc.account} — ${acc.status}${extra}`)
+    }
+  } finally {
+    await web.close()
+  }
+}
+
+async function serve() {
+  // Start the HTTP server
+  require('./server')
+}
+
+async function mcp() {
+  // Start as MCP server (stdio transport)
+  const { MCPServer } = require('./mcp')
+  const server = new MCPServer()
+  server.start()
+}
+
+async function installStealth() {
+  const { install, isInstalled } = require('./browse/install-stealth')
+  if (isInstalled()) {
+    const { getCamoufoxPath } = require('./browse/install-stealth')
+    console.log(`Camoufox already installed at ${getCamoufoxPath()}`)
+    console.log('Spectrawl will use it automatically.')
+    return
+  }
+  await install()
+}
+
+function help() {
+  console.log(`
+🌐 Spectrawl — The unified web layer for AI agents.
+
+Commands:
+  init                     Create spectrawl.json config
+  search "query"           Search the web
+  status                   Check auth health for all accounts
+  serve [--port N]         Start HTTP server
+  mcp                      Start MCP server (stdio)
+  install-stealth          Download Camoufox anti-detect browser
+  version                  Show version
+
+Examples:
+  spectrawl init
+  spectrawl search "best dental clinics in seoul"
+  spectrawl status
+  spectrawl serve --port 3900
+  spectrawl mcp
+`)
+}
+
+function getFlag(flag) {
+  const idx = args.indexOf(flag)
+  return idx !== -1 ? args[idx + 1] : null
+}
+
+main().catch(err => {
+  console.error('Error:', err.message)
+  process.exit(1)
+})

package/src/config.js

@@ -0,0 +1,65 @@
+const fs = require('fs')
+const path = require('path')
+
+const DEFAULTS = {
+  port: 3900,
+  search: {
+    cascade: ['searxng', 'ddg', 'brave', 'serper'],
+    scrapeTop: 3,
+    searxng: { url: 'http://localhost:8888' },
+    llm: null // { provider, model, apiKey }
+  },
+  browse: {
+    defaultEngine: 'playwright',
+    proxy: null, // { type, host, port, username, password }
+    humanlike: {
+      minDelay: 500,
+      maxDelay: 2000,
+      scrollBehavior: true
+    }
+  },
+  auth: {
+    refreshInterval: '4h',
+    cookieStore: './data/cookies.db'
+  },
+  cache: {
+    path: './data/cache.db',
+    searchTtl: 3600,      // 1 hour
+    scrapeTtl: 86400,     // 24 hours
+    screenshotTtl: 3600   // 1 hour
+  },
+  rateLimit: {
+    x: { postsPerHour: 5, minDelayMs: 30000 },
+    reddit: { postsPerHour: 3, minDelayMs: 600000 }
+  },
+  concurrency: 3
+}
+
+function loadConfig(configPath) {
+  const filePath = configPath || path.join(process.cwd(), 'spectrawl.json')
+  
+  let userConfig = {}
+  if (fs.existsSync(filePath)) {
+    try {
+      userConfig = JSON.parse(fs.readFileSync(filePath, 'utf8'))
+    } catch (e) {
+      console.warn(`Warning: Could not parse ${filePath}:`, e.message)
+    }
+  }
+
+  return deepMerge(DEFAULTS, userConfig)
+}
+
+function deepMerge(target, source) {
+  const result = { ...target }
+  for (const key of Object.keys(source)) {
+    if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
+      result[key] = deepMerge(target[key] || {}, source[key])
+    } else {
+      result[key] = source[key]
+    }
+  }
+  return result
+}
+
+module.exports = { loadConfig, DEFAULTS }

package/src/events.js

@@ -0,0 +1,57 @@
+/**
+ * Event system for Spectrawl.
+ * Proactive notifications to agents about auth state changes.
+ */
+
+class EventEmitter {
+  constructor() {
+    this._handlers = {}
+  }
+
+  on(event, handler) {
+    if (!this._handlers[event]) this._handlers[event] = []
+    this._handlers[event].push(handler)
+    return this
+  }
+
+  off(event, handler) {
+    if (!this._handlers[event]) return
+    this._handlers[event] = this._handlers[event].filter(h => h !== handler)
+    return this
+  }
+
+  emit(event, data) {
+    const handlers = this._handlers[event] || []
+    for (const handler of handlers) {
+      try {
+        handler(data)
+      } catch (err) {
+        console.warn(`Event handler error for ${event}:`, err.message)
+      }
+    }
+    
+    // Also emit to wildcard handlers
+    const wildcards = this._handlers['*'] || []
+    for (const handler of wildcards) {
+      try {
+        handler({ event, ...data })
+      } catch (err) {
+        console.warn(`Wildcard handler error:`, err.message)
+      }
+    }
+  }
+}
+
+// Standard events
+const EVENTS = {
+  COOKIE_EXPIRING: 'cookie_expiring',
+  COOKIE_EXPIRED: 'cookie_expired',
+  AUTH_FAILED: 'auth_failed',
+  AUTH_REFRESHED: 'auth_refreshed',
+  RATE_LIMITED: 'rate_limited',
+  ACTION_FAILED: 'action_failed',
+  ACTION_SUCCESS: 'action_success',
+  HEALTH_CHECK: 'health_check'
+}
+
+module.exports = { EventEmitter, EVENTS }

package/src/index.js

@@ -0,0 +1,108 @@
+/**
+ * Spectrawl — The unified web layer for AI agents.
+ * Search, browse, authenticate, act.
+ */
+
+const { SearchEngine } = require('./search')
+const { BrowseEngine } = require('./browse')
+const { AuthManager } = require('./auth')
+const { ActEngine } = require('./act')
+const { Cache } = require('./cache')
+const { EventEmitter, EVENTS } = require('./events')
+const { CookieRefresher } = require('./auth/refresh')
+const { loadConfig } = require('./config')
+
+class Spectrawl {
+  constructor(configPath) {
+    this.config = loadConfig(configPath)
+    this.events = new EventEmitter()
+    this.cache = new Cache(this.config.cache)
+    this.searchEngine = new SearchEngine(this.config.search, this.cache)
+    this.browseEngine = new BrowseEngine(this.config.browse, this.cache)
+    this.auth = new AuthManager(this.config.auth)
+    this.actEngine = new ActEngine(this.config, this.auth, this.browseEngine)
+    this.refresher = new CookieRefresher(this.auth, this.events, this.config.auth)
+  }
+
+  /**
+   * Search the web using free API cascade.
+   * @param {string} query - Search query
+   * @param {object} opts - { summarize, scrapeTop, engines }
+   * @returns {Promise<{answer?, sources[], cached}>}
+   */
+  async search(query, opts = {}) {
+    return this.searchEngine.search(query, opts)
+  }
+
+  /**
+   * Browse a URL with stealth and optional auth.
+   * @param {string} url - URL to browse
+   * @param {object} opts - { auth, screenshot, extract, stealth }
+   * @returns {Promise<{content, html, screenshot?, cookies?}>}
+   */
+  async browse(url, opts = {}) {
+    if (opts.auth) {
+      const cookies = await this.auth.getCookies(opts.auth)
+      opts._cookies = cookies
+    }
+    return this.browseEngine.browse(url, opts)
+  }
+
+  /**
+   * Perform an authenticated action on a platform.
+   * @param {string} platform - Platform name (x, reddit, devto, etc.)
+   * @param {string} action - Action name (post, comment, like, etc.)
+   * @param {object} params - Action parameters
+   * @returns {Promise<{success, data?, error?}>}
+   */
+  async act(platform, action, params = {}) {
+    const result = await this.actEngine.execute(platform, action, params)
+    
+    if (result.success) {
+      this.events.emit(EVENTS.ACTION_SUCCESS, { platform, action, ...result })
+    } else {
+      this.events.emit(EVENTS.ACTION_FAILED, { platform, action, ...result })
+    }
+    
+    return result
+  }
+
+  /**
+   * Register event handler.
+   * Events: cookie_expiring, cookie_expired, auth_failed, auth_refreshed,
+   *         rate_limited, action_failed, action_success, health_check
+   * @param {string} event - Event name
+   * @param {function} handler - Event handler
+   */
+  on(event, handler) {
+    this.events.on(event, handler)
+    return this
+  }
+
+  /**
+   * Start the cookie refresh cron.
+   * Call this when running as a server to auto-monitor auth health.
+   */
+  startRefreshCron() {
+    this.refresher.start()
+  }
+
+  /**
+   * Get health status of all authenticated sessions.
+   * @returns {Promise<Array<{platform, account, status, expires?}>>}
+   */
+  async status() {
+    return this.auth.getStatus()
+  }
+
+  /**
+   * Shut down gracefully.
+   */
+  async close() {
+    this.refresher.stop()
+    await this.browseEngine.close()
+    await this.cache.close()
+  }
+}
+
+module.exports = { Spectrawl, EVENTS }