spectrawl 0.1.0

package/src/auth/index.js

@@ -0,0 +1,132 @@
+const Database = require('better-sqlite3')
+const path = require('path')
+const fs = require('fs')
+
+class AuthManager {
+  constructor(config = {}) {
+    const dbPath = config.cookieStore || './data/cookies.db'
+    fs.mkdirSync(path.dirname(dbPath), { recursive: true })
+    
+    this.db = new Database(dbPath)
+    this.db.pragma('journal_mode = WAL')
+    this.refreshInterval = parseInterval(config.refreshInterval || '4h')
+    
+    this._init()
+  }
+
+  _init() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS accounts (
+        id TEXT PRIMARY KEY,
+        platform TEXT NOT NULL,
+        account TEXT NOT NULL,
+        method TEXT NOT NULL,
+        cookies TEXT,
+        oauth_token TEXT,
+        oauth_refresh TEXT,
+        credentials TEXT,
+        status TEXT DEFAULT 'unknown',
+        last_check INTEGER,
+        expires_at INTEGER,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL,
+        UNIQUE(platform, account)
+      )
+    `)
+  }
+
+  /**
+   * Add a new account.
+   */
+  async add(platform, opts = {}) {
+    const id = `${platform}:${opts.account}`
+    const now = Math.floor(Date.now() / 1000)
+    
+    this.db.prepare(`
+      INSERT OR REPLACE INTO accounts 
+      (id, platform, account, method, cookies, oauth_token, oauth_refresh, credentials, status, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      id, platform, opts.account, opts.method || 'cookie',
+      opts.cookies ? JSON.stringify(opts.cookies) : null,
+      opts.oauthToken || null,
+      opts.oauthRefresh || null,
+      opts.credentials ? JSON.stringify(opts.credentials) : null,
+      'valid', now, now
+    )
+
+    return { id, platform, account: opts.account, status: 'valid' }
+  }
+
+  /**
+   * Get cookies for a platform/account.
+   */
+  async getCookies(platformOrId, account) {
+    let row
+    if (account) {
+      row = this.db.prepare('SELECT * FROM accounts WHERE platform = ? AND account = ?').get(platformOrId, account)
+    } else {
+      // Try as id first, then as platform (return first account)
+      row = this.db.prepare('SELECT * FROM accounts WHERE id = ?').get(platformOrId) ||
+            this.db.prepare('SELECT * FROM accounts WHERE platform = ? ORDER BY updated_at DESC LIMIT 1').get(platformOrId)
+    }
+
+    if (!row) return null
+    if (row.cookies) return JSON.parse(row.cookies)
+    return null
+  }
+
+  /**
+   * Update cookies for an account.
+   */
+  async updateCookies(platform, account, cookies, expiresAt) {
+    const now = Math.floor(Date.now() / 1000)
+    this.db.prepare(`
+      UPDATE accounts SET cookies = ?, status = 'valid', expires_at = ?, updated_at = ?, last_check = ?
+      WHERE platform = ? AND account = ?
+    `).run(JSON.stringify(cookies), expiresAt || null, now, now, platform, account)
+  }
+
+  /**
+   * Get health status of all accounts.
+   */
+  async getStatus() {
+    const rows = this.db.prepare('SELECT * FROM accounts ORDER BY platform, account').all()
+    const now = Math.floor(Date.now() / 1000)
+
+    return rows.map(row => {
+      let status = row.status
+      if (row.expires_at) {
+        const remaining = row.expires_at - now
+        if (remaining <= 0) status = 'expired'
+        else if (remaining < 7200) status = 'expiring'
+      }
+
+      return {
+        platform: row.platform,
+        account: row.account,
+        method: row.method,
+        status,
+        expiresAt: row.expires_at ? new Date(row.expires_at * 1000).toISOString() : null,
+        lastCheck: row.last_check ? new Date(row.last_check * 1000).toISOString() : null
+      }
+    })
+  }
+
+  /**
+   * Remove an account.
+   */
+  async remove(platform, account) {
+    this.db.prepare('DELETE FROM accounts WHERE platform = ? AND account = ?').run(platform, account)
+  }
+}
+
+function parseInterval(str) {
+  const match = str.match(/^(\d+)(h|m|s)$/)
+  if (!match) return 14400 // default 4h
+  const [, num, unit] = match
+  const multipliers = { h: 3600, m: 60, s: 1 }
+  return parseInt(num) * (multipliers[unit] || 3600)
+}
+
+module.exports = { AuthManager }

package/src/auth/refresh.js

@@ -0,0 +1,111 @@
+/**
+ * Cookie refresh cron.
+ * Periodically checks cookie validity and refreshes when needed.
+ */
+
+class CookieRefresher {
+  constructor(authManager, events, config = {}) {
+    this.auth = authManager
+    this.events = events
+    this.interval = config.refreshInterval || 14400 // 4h default
+    this.warningThreshold = config.warningThreshold || 7200 // 2h before expiry
+    this._timer = null
+  }
+
+  /**
+   * Start the refresh cron.
+   */
+  start() {
+    if (this._timer) return
+    
+    // Run immediately, then on interval
+    this._check()
+    this._timer = setInterval(() => this._check(), this.interval * 1000)
+    
+    console.log(`Cookie refresh cron started (every ${this.interval}s)`)
+  }
+
+  /**
+   * Stop the refresh cron.
+   */
+  stop() {
+    if (this._timer) {
+      clearInterval(this._timer)
+      this._timer = null
+    }
+  }
+
+  async _check() {
+    try {
+      const accounts = await this.auth.getStatus()
+      const now = Math.floor(Date.now() / 1000)
+
+      for (const account of accounts) {
+        if (account.status === 'expired') {
+          this.events.emit('cookie_expired', {
+            platform: account.platform,
+            account: account.account,
+            expiredAt: account.expiresAt
+          })
+
+          // Attempt auto-refresh
+          await this._tryRefresh(account)
+        } else if (account.status === 'expiring') {
+          const remaining = account.expiresAt 
+            ? Math.floor((new Date(account.expiresAt).getTime() / 1000) - now)
+            : 0
+
+          this.events.emit('cookie_expiring', {
+            platform: account.platform,
+            account: account.account,
+            expiresIn: remaining,
+            expiresAt: account.expiresAt
+          })
+        }
+      }
+    } catch (err) {
+      console.warn('Cookie refresh check failed:', err.message)
+    }
+  }
+
+  async _tryRefresh(account) {
+    // Platform-specific refresh strategies
+    try {
+      switch (account.method) {
+        case 'oauth':
+          await this._refreshOAuth(account)
+          break
+        case 'cookie':
+          // Can't auto-refresh cookies without browser
+          // Emit event so user/agent knows
+          this.events.emit('auth_failed', {
+            platform: account.platform,
+            account: account.account,
+            reason: 'Cookie expired. Manual re-login required.',
+            suggestion: `Run: spectrawl login ${account.platform} --account ${account.account}`
+          })
+          break
+      }
+    } catch (err) {
+      this.events.emit('auth_failed', {
+        platform: account.platform,
+        account: account.account,
+        reason: err.message,
+        suggestion: `Run: spectrawl login ${account.platform} --account ${account.account}`
+      })
+    }
+  }
+
+  async _refreshOAuth(account) {
+    // TODO: implement OAuth token refresh per platform
+    // For now, emit that refresh is needed
+    this.events.emit('auth_failed', {
+      platform: account.platform,
+      account: account.account,
+      reason: 'OAuth refresh not yet implemented',
+      suggestion: `Run: spectrawl login ${account.platform} --oauth`
+    })
+  }
+}
+
+module.exports = { CookieRefresher }

package/src/browse/camoufox.js

@@ -0,0 +1,164 @@
+const http = require('http')
+
+/**
+ * Camoufox client — connects to existing Camoufox HTTP service.
+ * Camoufox is a modified Firefox with anti-fingerprint patches.
+ * Runs as a persistent service, we just send commands via REST API.
+ * 
+ * Default: http://localhost:9869 (existing service on Hetzner)
+ */
+class CamoufoxClient {
+  constructor(config = {}) {
+    this.baseUrl = config.url || process.env.CAMOUFOX_URL || 'http://localhost:9869'
+    this.timeout = config.timeout || 30000
+  }
+
+  /**
+   * Check if Camoufox service is running.
+   */
+  async health() {
+    try {
+      const data = await this._get('/health')
+      return { available: true, url: data.url }
+    } catch (e) {
+      return { available: false, error: e.message }
+    }
+  }
+
+  /**
+   * Navigate to a URL.
+   */
+  async navigate(url, opts = {}) {
+    return this._post('/navigate', {
+      url,
+      timeout: opts.timeout || this.timeout,
+      wait: opts.wait || 3000
+    })
+  }
+
+  /**
+   * Get page text content.
+   */
+  async getText() {
+    return this._get('/text')
+  }
+
+  /**
+   * Take a screenshot.
+   */
+  async screenshot() {
+    return this._get('/screenshot')
+  }
+
+  /**
+   * Click an element.
+   */
+  async click(selector, opts = {}) {
+    return this._post('/click', {
+      selector,
+      timeout: opts.timeout || 10000,
+      wait: opts.wait || 1000
+    })
+  }
+
+  /**
+   * Type text.
+   */
+  async type(text, opts = {}) {
+    return this._post('/type', {
+      text,
+      delay: opts.delay || 20
+    })
+  }
+
+  /**
+   * Press a key.
+   */
+  async press(key) {
+    return this._post('/press', { key })
+  }
+
+  /**
+   * Update cookies.
+   */
+  async setCookies(cookies) {
+    return this._post('/cookies', { cookies })
+  }
+
+  /**
+   * Post to Reddit (uses built-in Reddit automation).
+   */
+  async redditPost(subreddit, title, body) {
+    return this._post('/post', { subreddit, title, body })
+  }
+
+  /**
+   * Reply to Reddit post/comment.
+   */
+  async redditReply(url, text) {
+    return this._post('/reply', { url, text })
+  }
+
+  /**
+   * Delete a Reddit post.
+   */
+  async redditDelete(url) {
+    return this._post('/delete', { url })
+  }
+
+  async _get(path) {
+    return new Promise((resolve, reject) => {
+      const urlObj = new URL(this.baseUrl + path)
+      http.get({
+        hostname: urlObj.hostname,
+        port: urlObj.port,
+        path: urlObj.pathname,
+        timeout: this.timeout
+      }, res => {
+        let data = ''
+        res.on('data', c => data += c)
+        res.on('end', () => {
+          try {
+            const parsed = JSON.parse(data)
+            if (res.statusCode >= 400) reject(new Error(parsed.error || `HTTP ${res.statusCode}`))
+            else resolve(parsed)
+          } catch (e) { reject(new Error(`Invalid response from Camoufox: ${data.slice(0, 200)}`)) }
+        })
+      }).on('error', reject)
+    })
+  }
+
+  async _post(path, body) {
+    return new Promise((resolve, reject) => {
+      const urlObj = new URL(this.baseUrl + path)
+      const bodyStr = JSON.stringify(body)
+      const opts = {
+        hostname: urlObj.hostname,
+        port: urlObj.port,
+        path: urlObj.pathname,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(bodyStr)
+        },
+        timeout: this.timeout
+      }
+      const req = http.request(opts, res => {
+        let data = ''
+        res.on('data', c => data += c)
+        res.on('end', () => {
+          try {
+            const parsed = JSON.parse(data)
+            if (res.statusCode >= 400) reject(new Error(parsed.error || `HTTP ${res.statusCode}`))
+            else resolve(parsed)
+          } catch (e) { reject(new Error(`Invalid response from Camoufox: ${data.slice(0, 200)}`)) }
+        })
+      })
+      req.on('error', reject)
+      req.write(bodyStr)
+      req.end()
+    })
+  }
+}
+
+module.exports = { CamoufoxClient }

package/src/browse/index.js

@@ -0,0 +1,278 @@
+/**
+ * Browse engine — three tiers of stealth.
+ * 
+ * Tier 1: playwright-extra + stealth plugin (default, npm install)
+ * Tier 2: Camoufox binary (npx spectrawl install-stealth, engine-level anti-detect)
+ * Tier 3: Remote Camoufox service (set camoufox.url, for existing deployments)
+ * 
+ * Auto-detects best available. No config needed for most users.
+ */
+
+const os = require('os')
+const path = require('path')
+const { CamoufoxClient } = require('./camoufox')
+const { getCamoufoxPath, isInstalled } = require('./install-stealth')
+
+class BrowseEngine {
+  constructor(config = {}, cache) {
+    this.config = config
+    this.cache = cache
+    this.browser = null
+
+    // Remote Camoufox service (existing deployment)
+    this.remoteCamoufox = config.camoufox?.url ? new CamoufoxClient(config.camoufox) : null
+    this._remoteCamoufoxAvailable = null
+
+    // Which engine we're using
+    this._engine = null
+  }
+
+  /**
+   * Browse a URL and extract content.
+   */
+  async browse(url, opts = {}) {
+    if (!opts.noCache && !opts.screenshot) {
+      const cached = this.cache?.get('scrape', url)
+      if (cached) return { ...cached, cached: true }
+    }
+
+    // Force remote Camoufox if explicitly requested
+    if (opts.camoufox && this.remoteCamoufox) {
+      return this._browseRemoteCamoufox(url, opts)
+    }
+
+    try {
+      return await this._browsePlaywright(url, opts)
+    } catch (err) {
+      // If blocked and remote Camoufox available, try that
+      if (this._isBlocked(err) && this.remoteCamoufox) {
+        console.log(`Blocked on ${url}, escalating to remote Camoufox`)
+        return this._browseRemoteCamoufox(url, opts)
+      }
+
+      if (this._isBlocked(err)) {
+        const hint = isInstalled()
+          ? 'Site has strong anti-bot. Try configuring a residential proxy.'
+          : 'Run `npx spectrawl install-stealth` for engine-level anti-detect.'
+        err.message = `Blocked on ${url}: ${err.message}. ${hint}`
+      }
+      throw err
+    }
+  }
+
+  /**
+   * Launch Playwright with the best available browser.
+   * Priority: Camoufox binary > stealth Chromium > vanilla Chromium
+   */
+  async _getBrowser() {
+    if (this.browser) return this.browser
+
+    // Tier 2: Local Camoufox binary (engine-level anti-detect)
+    const camoufoxBinary = getCamoufoxPath()
+    if (camoufoxBinary) {
+      try {
+        const { firefox } = require('playwright')
+        this.browser = await firefox.launch({
+          executablePath: camoufoxBinary,
+          headless: true,
+          args: ['--no-remote']
+        })
+        this._engine = 'camoufox'
+        console.log('Browse engine: Camoufox (engine-level anti-detect)')
+        return this.browser
+      } catch (e) {
+        console.log(`Camoufox binary failed: ${e.message}, falling back`)
+      }
+    }
+
+    // Tier 1: playwright-extra + stealth plugin
+    try {
+      const { chromium } = require('playwright-extra')
+      const stealth = require('puppeteer-extra-plugin-stealth')
+      chromium.use(stealth())
+
+      this.browser = await chromium.launch({
+        headless: true,
+        args: ['--no-sandbox', '--disable-setuid-sandbox']
+      })
+      this._engine = 'stealth-playwright'
+      console.log('Browse engine: stealth Playwright (JS-level anti-detect)')
+      return this.browser
+    } catch (e) {
+      // Tier 0: vanilla playwright
+      const { chromium } = require('playwright')
+      this.browser = await chromium.launch({
+        headless: true,
+        args: ['--no-sandbox', '--disable-setuid-sandbox']
+      })
+      this._engine = 'playwright'
+      console.log('Browse engine: vanilla Playwright (no anti-detect — install playwright-extra)')
+      return this.browser
+    }
+  }
+
+  async _browsePlaywright(url, opts) {
+    const browser = await this._getBrowser()
+    const context = await this._createContext(browser, opts)
+    const page = await context.newPage()
+
+    try {
+      if (opts._cookies) {
+        await context.addCookies(opts._cookies)
+      }
+
+      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 })
+
+      // Human-like delays
+      await page.waitForTimeout(800 + Math.random() * 1500)
+      await page.evaluate(() => {
+        window.scrollBy({ top: Math.floor(Math.random() * 400) + 100, behavior: 'smooth' })
+      })
+      await page.waitForTimeout(300 + Math.random() * 700)
+
+      const result = {}
+
+      if (opts.extract !== false) {
+        result.content = await page.evaluate(() => {
+          const main = document.querySelector('main, article, [role="main"]') || document.body
+          return main.innerText
+        })
+      }
+
+      if (opts.html) result.html = await page.content()
+
+      if (opts.screenshot) {
+        result.screenshot = await page.screenshot({
+          type: 'png', fullPage: opts.fullPage || false
+        })
+      }
+
+      if (opts.saveCookies) result.cookies = await context.cookies()
+
+      result.url = page.url()
+      result.title = await page.title()
+      result.cached = false
+      result.engine = this._engine
+
+      if (!opts.screenshot) {
+        this.cache?.set('scrape', url, { content: result.content, url: result.url, title: result.title })
+      }
+
+      return result
+    } finally {
+      await page.close()
+      await context.close()
+    }
+  }
+
+  async _browseRemoteCamoufox(url, opts) {
+    if (this._remoteCamoufoxAvailable === null) {
+      const health = await this.remoteCamoufox.health()
+      this._remoteCamoufoxAvailable = health.available
+    }
+
+    if (!this._remoteCamoufoxAvailable) {
+      throw new Error('Remote Camoufox configured but not running. Check camoufox.url.')
+    }
+
+    if (opts._cookies) await this.remoteCamoufox.setCookies(opts._cookies)
+    await this.remoteCamoufox.navigate(url, { wait: 3000 })
+
+    const result = { engine: 'remote-camoufox', cached: false }
+
+    if (opts.extract !== false) {
+      const textData = await this.remoteCamoufox.getText()
+      result.content = textData.text
+      result.title = textData.title
+      result.url = textData.url
+    }
+
+    if (opts.screenshot) {
+      const ssData = await this.remoteCamoufox.screenshot()
+      result.screenshotPath = ssData.path
+    }
+
+    if (!opts.screenshot) {
+      this.cache?.set('scrape', url, { content: result.content, url: result.url, title: result.title })
+    }
+
+    return result
+  }
+
+  _isBlocked(err) {
+    const msg = (err.message || '').toLowerCase()
+    return msg.includes('captcha') || msg.includes('blocked') || msg.includes('403') ||
+           msg.includes('access denied') || msg.includes('challenge') ||
+           msg.includes('cloudflare') || msg.includes('bot detection')
+  }
+
+  async _createContext(browser, opts) {
+    const resolutions = [
+      { width: 1920, height: 1080 }, { width: 1536, height: 864 },
+      { width: 1440, height: 900 }, { width: 1366, height: 768 },
+      { width: 2560, height: 1440 }
+    ]
+    const viewport = resolutions[Math.floor(Math.random() * resolutions.length)]
+
+    const userAgents = [
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0',
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15'
+    ]
+
+    const contextOpts = {
+      userAgent: userAgents[Math.floor(Math.random() * userAgents.length)],
+      viewport,
+      locale: 'en-US',
+      timezoneId: 'America/New_York',
+      colorScheme: 'light',
+      deviceScaleFactor: Math.random() > 0.5 ? 1 : 2
+    }
+
+    if (this.config.proxy) {
+      contextOpts.proxy = {
+        server: `${this.config.proxy.host}:${this.config.proxy.port}`,
+        username: this.config.proxy.username,
+        password: this.config.proxy.password
+      }
+    }
+
+    return browser.newContext(contextOpts)
+  }
+
+  /**
+   * Get a raw Playwright page for direct interaction.
+   * Used by platform adapters that need browser automation (e.g., IH).
+   * Caller is responsible for closing the page and context.
+   * 
+   * @param {object} opts - { _cookies, url }
+   * @returns {{ page, context, engine }}
+   */
+  async getPage(opts = {}) {
+    const browser = await this._getBrowser()
+    const context = await this._createContext(browser, opts)
+
+    if (opts._cookies) {
+      await context.addCookies(opts._cookies)
+    }
+
+    const page = await context.newPage()
+
+    if (opts.url) {
+      await page.goto(opts.url, { waitUntil: 'domcontentloaded', timeout: 30000 })
+      await page.waitForTimeout(800 + Math.random() * 1500)
+    }
+
+    return { page, context, engine: this._engine }
+  }
+
+  async close() {
+    if (this.browser) {
+      await this.browser.close()
+      this.browser = null
+    }
+  }
+}
+
+module.exports = { BrowseEngine }