spectrawl 0.1.0

package/src/act/adapters/x.js

@@ -0,0 +1,202 @@
+const https = require('https')
+const crypto = require('crypto')
+
+/**
+ * X (Twitter) platform adapter.
+ * Methods: Cookie API (GraphQL) with OAuth 1.0a fallback.
+ */
+class XAdapter {
+  /**
+   * Execute an action on X.
+   * @param {string} action - post, like, retweet, delete
+   * @param {object} params - { account, text, mediaIds, tweetId, _cookies }
+   * @param {object} ctx - { auth, browse }
+   */
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+        return this._post(params, ctx)
+      case 'like':
+        return this._like(params, ctx)
+      case 'retweet':
+        return this._retweet(params, ctx)
+      case 'delete':
+        return this._delete(params, ctx)
+      default:
+        throw new Error(`Unsupported X action: ${action}`)
+    }
+  }
+
+  async _post(params, ctx) {
+    const { text, account, _cookies } = params
+
+    // Try Cookie API (GraphQL) first
+    if (_cookies) {
+      return this._graphqlPost(text, _cookies)
+    }
+
+    // Try OAuth 1.0a if configured
+    const oauthCreds = await ctx.auth.getCookies('x', account)
+    if (oauthCreds?.oauth) {
+      return this._oauthPost(text, oauthCreds.oauth)
+    }
+
+    throw new Error(`No auth available for X account ${account}. Run: spectrawl login x --account ${account}`)
+  }
+
+  async _graphqlPost(text, cookies) {
+    // X GraphQL CreateTweet mutation
+    const csrfToken = cookies.find(c => c.name === 'ct0')?.value
+    if (!csrfToken) throw new Error('Missing ct0 CSRF token in X cookies')
+
+    const cookieStr = cookies.map(c => `${c.name}=${c.value}`).join('; ')
+    
+    const body = JSON.stringify({
+      variables: {
+        tweet_text: text,
+        dark_request: false,
+        media: { media_entities: [], possibly_sensitive: false },
+        semantic_annotation_ids: []
+      },
+      features: {
+        communities_web_enable_tweet_community_results_fetch: true,
+        c9s_tweet_anatomy_moderator_badge_enabled: true,
+        tweetypie_unmention_optimization_enabled: true,
+        responsive_web_edit_tweet_api_enabled: true,
+        graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+        view_counts_everywhere_api_enabled: true,
+        longform_notetweets_consumption_enabled: true,
+        responsive_web_twitter_article_tweet_consumption_enabled: true,
+        tweet_awards_web_tipping_enabled: false,
+        creator_subscriptions_quote_tweet_preview_enabled: false,
+        longform_notetweets_rich_text_read_enabled: true,
+        longform_notetweets_inline_media_enabled: true,
+        articles_preview_enabled: true,
+        rweb_video_timestamps_enabled: true,
+        rweb_tipjar_consumption_enabled: true,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        freedom_of_speech_not_reach_fetch_enabled: true,
+        standardized_nudges_misinfo: true,
+        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+        responsive_web_enhance_cards_enabled: false
+      },
+      queryId: 'bDE2rBtZb3uyrczSZ_pI9g'
+    })
+
+    const data = await postJson(
+      'https://x.com/i/api/graphql/bDE2rBtZb3uyrczSZ_pI9g/CreateTweet',
+      body,
+      {
+        'Authorization': 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+        'Content-Type': 'application/json',
+        'Cookie': cookieStr,
+        'X-Csrf-Token': csrfToken,
+        'X-Twitter-Auth-Type': 'OAuth2Session',
+        'X-Twitter-Active-User': 'yes'
+      }
+    )
+
+    if (data.errors) {
+      throw new Error(`X API error: ${data.errors[0]?.message || JSON.stringify(data.errors)}`)
+    }
+
+    const tweetId = data.data?.create_tweet?.tweet_results?.result?.rest_id
+    return { tweetId, url: tweetId ? `https://x.com/i/status/${tweetId}` : null }
+  }
+
+  async _oauthPost(text, oauth) {
+    // OAuth 1.0a — for accounts with API keys
+    const { consumerKey, consumerSecret, accessToken, accessTokenSecret } = oauth
+    
+    const url = 'https://api.x.com/2/tweets'
+    const body = JSON.stringify({ text })
+    
+    const authHeader = generateOAuthHeader('POST', url, {}, {
+      consumerKey, consumerSecret, accessToken, accessTokenSecret
+    })
+
+    const data = await postJson(url, body, {
+      'Authorization': authHeader,
+      'Content-Type': 'application/json'
+    })
+
+    return { tweetId: data.data?.id, url: `https://x.com/i/status/${data.data?.id}` }
+  }
+
+  async _like(params, ctx) {
+    // TODO: implement like via GraphQL
+    throw new Error('X like not yet implemented')
+  }
+
+  async _retweet(params, ctx) {
+    // TODO: implement retweet via GraphQL
+    throw new Error('X retweet not yet implemented')
+  }
+
+  async _delete(params, ctx) {
+    // TODO: implement delete via GraphQL
+    throw new Error('X delete not yet implemented')
+  }
+}
+
+function generateOAuthHeader(method, url, params, creds) {
+  const oauthParams = {
+    oauth_consumer_key: creds.consumerKey,
+    oauth_nonce: crypto.randomBytes(16).toString('hex'),
+    oauth_signature_method: 'HMAC-SHA1',
+    oauth_timestamp: Math.floor(Date.now() / 1000).toString(),
+    oauth_token: creds.accessToken,
+    oauth_version: '1.0'
+  }
+
+  const allParams = { ...params, ...oauthParams }
+  const sortedKeys = Object.keys(allParams).sort()
+  const paramStr = sortedKeys.map(k => `${encodeRFC3986(k)}=${encodeRFC3986(allParams[k])}`).join('&')
+  const baseStr = `${method}&${encodeRFC3986(url)}&${encodeRFC3986(paramStr)}`
+  const signingKey = `${encodeRFC3986(creds.consumerSecret)}&${encodeRFC3986(creds.accessTokenSecret)}`
+  
+  oauthParams.oauth_signature = crypto
+    .createHmac('sha1', signingKey)
+    .update(baseStr)
+    .digest('base64')
+
+  const header = Object.keys(oauthParams)
+    .sort()
+    .map(k => `${encodeRFC3986(k)}="${encodeRFC3986(oauthParams[k])}"`)
+    .join(', ')
+
+  return `OAuth ${header}`
+}
+
+function encodeRFC3986(str) {
+  return encodeURIComponent(str).replace(/[!'()*]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase())
+}
+
+function postJson(url, body, headers) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    const opts = {
+      hostname: urlObj.hostname,
+      path: urlObj.pathname + urlObj.search,
+      method: 'POST',
+      headers: { ...headers, 'Content-Length': Buffer.byteLength(body) }
+    }
+    const req = https.request(opts, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { reject(new Error(`Invalid response: ${data.slice(0, 200)}`)) }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(15000, () => { req.destroy(); reject(new Error('X API timeout')) })
+    req.write(body)
+    req.end()
+  })
+}
+
+module.exports = { XAdapter }

package/src/act/form-filler.js

@@ -0,0 +1,94 @@
+/**
+ * Form filler — handles platform-specific input quirks.
+ * Solves: contentEditable divs, shadow DOMs, React controlled inputs.
+ */
+
+/**
+ * Fill a contentEditable div (X compose box, Notion, etc.)
+ * Regular Playwright .fill() doesn't work on these.
+ */
+async function fillContentEditable(page, selector, text) {
+  await page.click(selector)
+  await page.waitForTimeout(200)
+  
+  // execCommand("insertText") is the only reliable method for contentEditable
+  await page.evaluate(({ selector, text }) => {
+    const el = document.querySelector(selector)
+    if (el) {
+      el.focus()
+      document.execCommand('selectAll', false, null)
+      document.execCommand('insertText', false, text)
+    }
+  }, { selector, text })
+}
+
+/**
+ * Fill a React controlled input.
+ * React ignores .value changes — need to trigger native input events.
+ */
+async function fillReactInput(page, selector, text) {
+  await page.click(selector)
+  await page.waitForTimeout(100)
+  
+  // Clear existing value
+  await page.evaluate((selector) => {
+    const el = document.querySelector(selector)
+    if (el) {
+      const nativeInputValueSetter = Object.getOwnPropertyDescriptor(
+        window.HTMLInputElement.prototype, 'value'
+      ).set
+      nativeInputValueSetter.call(el, '')
+      el.dispatchEvent(new Event('input', { bubbles: true }))
+    }
+  }, selector)
+  
+  // Type character by character (most reliable for React)
+  for (const char of text) {
+    await page.keyboard.press(char === ' ' ? 'Space' : char)
+    await page.waitForTimeout(10 + Math.random() * 30)
+  }
+}
+
+/**
+ * Fill a shadow DOM input.
+ * Playwright can pierce shadow DOM with >> syntax.
+ */
+async function fillShadowInput(page, hostSelector, inputSelector, text) {
+  const selector = `${hostSelector} >> ${inputSelector}`
+  await page.fill(selector, text)
+}
+
+/**
+ * Smart fill — detects input type and uses appropriate method.
+ */
+async function smartFill(page, selector, text, opts = {}) {
+  const inputType = await page.evaluate((selector) => {
+    const el = document.querySelector(selector)
+    if (!el) return 'not_found'
+    if (el.contentEditable === 'true' || el.getAttribute('contenteditable')) return 'contentEditable'
+    if (el.shadowRoot) return 'shadow'
+    
+    // Check if React-controlled (has __reactFiber or __reactInternalInstance)
+    const keys = Object.keys(el)
+    if (keys.some(k => k.startsWith('__react'))) return 'react'
+    
+    return 'standard'
+  }, selector)
+
+  switch (inputType) {
+    case 'contentEditable':
+      return fillContentEditable(page, selector, text)
+    case 'react':
+      return fillReactInput(page, selector, text)
+    case 'shadow':
+      return fillShadowInput(page, opts.hostSelector || selector, opts.inputSelector || 'input', text)
+    case 'standard':
+      return page.fill(selector, text)
+    default:
+      // Last resort: click and type
+      await page.click(selector)
+      await page.keyboard.type(text, { delay: opts.delay || 20 })
+  }
+}
+
+module.exports = { fillContentEditable, fillReactInput, fillShadowInput, smartFill }

package/src/act/index.js

@@ -0,0 +1,159 @@
+/**
+ * Act engine — authenticated actions on platforms.
+ * Delegates to platform-specific adapters.
+ * Includes rate limiting, deduplication, and dead letter queue.
+ */
+
+const crypto = require('crypto')
+const { XAdapter } = require('./adapters/x')
+const { RedditAdapter } = require('./adapters/reddit')
+const { DevtoAdapter } = require('./adapters/devto')
+const { HashnodeAdapter } = require('./adapters/hashnode')
+const { LinkedInAdapter } = require('./adapters/linkedin')
+const { IHAdapter } = require('./adapters/ih')
+const { RateLimiter } = require('./rate-limiter')
+
+const adapters = {
+  x: new XAdapter(),
+  twitter: new XAdapter(),
+  reddit: new RedditAdapter(),
+  devto: new DevtoAdapter(),
+  'dev.to': new DevtoAdapter(),
+  hashnode: new HashnodeAdapter(),
+  linkedin: new LinkedInAdapter(),
+  ih: new IHAdapter(),
+  indiehackers: new IHAdapter()
+}
+
+class ActEngine {
+  constructor(config, auth, browse) {
+    this.config = config
+    this.auth = auth
+    this.browse = browse
+    this.rateLimiter = new RateLimiter({
+      dbPath: config.cache?.path?.replace('cache.db', 'ratelimit.db') || './data/ratelimit.db',
+      limits: config.rateLimit || {}
+    })
+  }
+
+  /**
+   * Execute an action on a platform.
+   * @param {string} platform - Platform name
+   * @param {string} action - Action name (post, comment, like, etc.)
+   * @param {object} params - Action parameters
+   */
+  async execute(platform, action, params = {}) {
+    const adapter = this._getAdapter(platform)
+    if (!adapter) {
+      return {
+        success: false,
+        error: 'unsupported_platform',
+        detail: `No adapter for platform "${platform}". Supported: ${Object.keys(adapters).join(', ') || 'none yet'}`,
+        suggestion: 'Platform adapters are being added. Check back soon.'
+      }
+    }
+
+    // Get auth for this platform/account
+    const account = params.account
+    if (account) {
+      const cookies = await this.auth.getCookies(platform, account)
+      if (!cookies) {
+        return {
+          success: false,
+          error: 'auth_missing',
+          detail: `No auth found for ${platform}/${account}.`,
+          suggestion: `Run: spectrawl login ${platform} --account ${account}`
+        }
+      }
+      params._cookies = cookies
+    }
+
+    // Check rate limits
+    const rateCheck = this.rateLimiter.check(platform, action, params)
+    if (!rateCheck.allowed) {
+      return {
+        success: false,
+        error: 'rate_limited',
+        detail: rateCheck.reason,
+        retryAfter: rateCheck.retryAfter,
+        suggestion: `Wait ${rateCheck.retryAfter}s or adjust limits in spectrawl.json`
+      }
+    }
+
+    // Check deduplication (same content posted in last 24h)
+    const contentHash = params.text || params.title || params.body
+      ? crypto.createHash('md5').update(`${platform}:${action}:${params.text || ''}${params.title || ''}`).digest('hex')
+      : null
+
+    if (contentHash && this.rateLimiter.isDuplicate(platform, contentHash)) {
+      return {
+        success: false,
+        error: 'duplicate',
+        detail: `Same content already posted to ${platform} in the last 24h`,
+        suggestion: 'Change the content or wait 24h'
+      }
+    }
+
+    try {
+      const result = await adapter.execute(action, params, {
+        auth: this.auth,
+        browse: this.browse
+      })
+
+      // Log success
+      this.rateLimiter.log(platform, action, {
+        account, contentHash, status: 'success'
+      })
+
+      return { success: true, ...result }
+    } catch (err) {
+      // Log failure
+      this.rateLimiter.log(platform, action, {
+        account, contentHash, status: 'failed',
+        error: err.message, retryCount: params._retryCount || 0
+      })
+
+      return {
+        success: false,
+        error: categorizeError(err),
+        detail: err.message,
+        suggestion: getSuggestion(err, platform, account)
+      }
+    }
+  }
+
+  _getAdapter(platform) {
+    return adapters[platform] || null
+  }
+
+  /**
+   * Register a platform adapter.
+   */
+  static registerAdapter(platform, adapter) {
+    adapters[platform] = adapter
+  }
+}
+
+function categorizeError(err) {
+  const msg = err.message.toLowerCase()
+  if (msg.includes('cookie') || msg.includes('auth') || msg.includes('login')) return 'auth_expired'
+  if (msg.includes('captcha')) return 'captcha_required'
+  if (msg.includes('rate') || msg.includes('429')) return 'rate_limited'
+  if (msg.includes('fingerprint') || msg.includes('blocked')) return 'fingerprint_blocked'
+  if (msg.includes('timeout')) return 'timeout'
+  return 'unknown'
+}
+
+function getSuggestion(err, platform, account) {
+  const category = categorizeError(err)
+  const suggestions = {
+    auth_expired: `Run: spectrawl login ${platform}${account ? ` --account ${account}` : ''}`,
+    captcha_required: `Manual intervention needed. Run: spectrawl login ${platform} --manual`,
+    rate_limited: `Wait and retry. Check rate limits in spectrawl.json`,
+    fingerprint_blocked: `Try with stealth mode: spectrawl browse --stealth`,
+    timeout: `Network issue. Check proxy settings.`
+  }
+  return suggestions[category] || 'Check logs for details.'
+}
+
+module.exports = { ActEngine }

package/src/act/rate-limiter.js

@@ -0,0 +1,143 @@
+const Database = require('better-sqlite3')
+const path = require('path')
+const fs = require('fs')
+
+/**
+ * Rate limiter for platform actions.
+ * Tracks action history and enforces per-platform limits.
+ * Also handles action deduplication and dead letter queue.
+ */
+class RateLimiter {
+  constructor(config = {}) {
+    const dbPath = config.dbPath || './data/ratelimit.db'
+    fs.mkdirSync(path.dirname(dbPath), { recursive: true })
+    
+    this.db = new Database(dbPath)
+    this.db.pragma('journal_mode = WAL')
+    this.limits = config.limits || {}
+    
+    this._init()
+  }
+
+  _init() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS action_log (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        platform TEXT NOT NULL,
+        account TEXT,
+        action TEXT NOT NULL,
+        content_hash TEXT,
+        status TEXT DEFAULT 'success',
+        error TEXT,
+        retry_count INTEGER DEFAULT 0,
+        created_at INTEGER NOT NULL
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_action_log_platform 
+        ON action_log(platform, created_at);
+      
+      CREATE INDEX IF NOT EXISTS idx_action_log_hash 
+        ON action_log(content_hash, created_at);
+    `)
+  }
+
+  /**
+   * Check if an action is allowed under rate limits.
+   * @returns {{ allowed: boolean, reason?: string, retryAfter?: number }}
+   */
+  check(platform, action, params = {}) {
+    const limit = this.limits[platform]
+    if (!limit) return { allowed: true }
+
+    const now = Math.floor(Date.now() / 1000)
+    const hourAgo = now - 3600
+
+    // Check posts per hour
+    if (limit.postsPerHour) {
+      const count = this.db.prepare(
+        'SELECT COUNT(*) as cnt FROM action_log WHERE platform = ? AND action = ? AND created_at > ? AND status = ?'
+      ).get(platform, action, hourAgo, 'success')
+
+      if (count.cnt >= limit.postsPerHour) {
+        // Find when the oldest action in this window will expire
+        const oldest = this.db.prepare(
+          'SELECT created_at FROM action_log WHERE platform = ? AND action = ? AND created_at > ? AND status = ? ORDER BY created_at ASC LIMIT 1'
+        ).get(platform, action, hourAgo, 'success')
+
+        const retryAfter = oldest ? (oldest.created_at + 3600 - now) : 3600
+        return {
+          allowed: false,
+          reason: `Rate limit: max ${limit.postsPerHour} ${action}s per hour on ${platform}`,
+          retryAfter
+        }
+      }
+    }
+
+    // Check minimum delay between actions
+    if (limit.minDelayMs) {
+      const last = this.db.prepare(
+        'SELECT created_at FROM action_log WHERE platform = ? AND status = ? ORDER BY created_at DESC LIMIT 1'
+      ).get(platform, 'success')
+
+      if (last) {
+        const elapsed = (now - last.created_at) * 1000
+        if (elapsed < limit.minDelayMs) {
+          return {
+            allowed: false,
+            reason: `Min delay: wait ${Math.ceil((limit.minDelayMs - elapsed) / 1000)}s between actions on ${platform}`,
+            retryAfter: Math.ceil((limit.minDelayMs - elapsed) / 1000)
+          }
+        }
+      }
+    }
+
+    return { allowed: true }
+  }
+
+  /**
+   * Check if this action is a duplicate (same content recently posted).
+   */
+  isDuplicate(platform, contentHash, windowSeconds = 86400) {
+    const cutoff = Math.floor(Date.now() / 1000) - windowSeconds
+    const existing = this.db.prepare(
+      'SELECT id FROM action_log WHERE platform = ? AND content_hash = ? AND created_at > ? AND status = ?'
+    ).get(platform, contentHash, cutoff, 'success')
+
+    return !!existing
+  }
+
+  /**
+   * Log an action (success or failure).
+   */
+  log(platform, action, params = {}) {
+    const now = Math.floor(Date.now() / 1000)
+    this.db.prepare(`
+      INSERT INTO action_log (platform, account, action, content_hash, status, error, retry_count, created_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      platform,
+      params.account || null,
+      action,
+      params.contentHash || null,
+      params.status || 'success',
+      params.error || null,
+      params.retryCount || 0,
+      now
+    )
+  }
+
+  /**
+   * Get failed actions for retry (dead letter queue).
+   */
+  getFailedActions(maxRetries = 3) {
+    return this.db.prepare(
+      'SELECT * FROM action_log WHERE status = ? AND retry_count < ? ORDER BY created_at ASC'
+    ).all('failed', maxRetries)
+  }
+
+  close() {
+    this.db.close()
+  }
+}
+
+module.exports = { RateLimiter }