spectrawl 0.1.2 → 0.2.1

package/README.md

@@ -19,7 +19,14 @@ npm install spectrawl
 
 **Auth** — Persistent cookie storage (SQLite), multi-account management, automatic cookie refresh, expiry alerts.
 
-**Act** — Post to X, Reddit, Dev.to, Hashnode, LinkedIn, IndieHackers. Rate limiting, content dedup, dead letter queue for retries.
+**Act** — 24 platform adapters covering 30+ sites:
+- **Content platforms:** X, Reddit, LinkedIn, Dev.to, Hashnode, IndieHackers, Medium, Hacker News, Quora
+- **Developer:** GitHub (repos, issues, releases), HuggingFace (models, datasets), Discord (bot + webhooks)
+- **Launch/SEO:** Product Hunt, BetaList, AlternativeTo, SaaSHub, DevHunt, AppSumo
+- **Directories:** Generic adapter for MicroLaunch, Uneed, Peerlist, Fazier, BetaPage, LaunchingNext, StartupStash, SideProjectors, TAIFT, Futurepedia, Crunchbase, G2, StackShare, YouTube
+- Rate limiting, content dedup, dead letter queue for retries.
+
+**Proxy** — Rotating proxy server. One endpoint (`localhost:8080`) for all your tools. Round-robin, random, or least-used strategies. Health checking with auto-failover.
 
 ## Quick Start
 
@@ -123,6 +130,19 @@ Configure the cascade in `spectrawl.json`:
 | Hashnode | GraphQL API | post |
 | LinkedIn | Cookie API (Voyager) | post |
 | IndieHackers | Browser automation | post, comment, upvote |
+| Medium | REST API | post (markdown) |
+| GitHub | REST v3 | repo, file, issue, release |
+| Discord | Bot API + webhooks | send, thread |
+| Product Hunt | GraphQL v2 | launch, comment, upvote |
+| Hacker News | Cookie/form POST | submit, comment, upvote |
+| YouTube | Data API v3 | comment, playlist, update |
+| Quora | Browser automation | answer, question |
+| HuggingFace | Hub API | repo, model card, upload |
+| BetaList | REST API | submit |
+| AlternativeTo | Browser automation | submit |
+| SaaSHub | Browser automation | submit |
+| DevHunt | Browser automation | submit |
+| **30+ Directories** | Generic adapter | submit (MicroLaunch, Uneed, TAIFT, Futurepedia, Crunchbase, G2, etc.) |
 
 ## Configuration
 
@@ -141,10 +161,11 @@ Configure the cascade in `spectrawl.json`:
     "scrapeTtl": 24
   },
   "proxy": {
-    "host": "proxy.example.com",
-    "port": "8080",
-    "username": "user",
-    "password": "pass"
+    "localPort": 8080,
+    "strategy": "round-robin",
+    "upstreams": [
+      { "url": "http://user:pass@proxy1.example.com:8080" }
+    ]
   },
   "camoufox": {
     "url": "http://localhost:9869"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "spectrawl",
-  "version": "0.1.2",
-  "description": "The unified web layer for AI agents. Search, browse, authenticate, act — one tool, self-hosted, free.",
+  "version": "0.2.1",
+  "description": "The unified web layer for AI agents. Search (6 engines), stealth browse (Camoufox + Playwright), auth (cookies, multi-account), act (24 adapters, 30+ platforms), proxy rotation. Self-hosted, free.",
   "main": "src/index.js",
   "types": "index.d.ts",
   "bin": {

package/src/act/adapters/alternativeto.js

@@ -0,0 +1,64 @@
+/**
+ * AlternativeTo platform adapter.
+ * No public API — browser automation for submitting alternatives.
+ * High SEO value: people search "[tool] alternative" constantly.
+ */
+class AlternativeToAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      default:
+        throw new Error(`Unsupported AlternativeTo action: ${action}. Requires browser automation.`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { name, url, description, category, tags, _cookies, _browse } = params
+    if (!_browse) throw new Error('AlternativeTo requires browse engine')
+
+    const page = await _browse('https://alternativeto.net/add/', {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const pw = page._page
+    if (!pw) throw new Error('AlternativeTo requires getPage access')
+
+    // Fill the submission form
+    await pw.waitForSelector('input[name="Name"], #Name', { timeout: 10000 })
+
+    // Name
+    const nameInput = await pw.$('input[name="Name"], #Name')
+    if (nameInput) { await nameInput.fill(name) }
+
+    // URL
+    const urlInput = await pw.$('input[name="Url"], input[name="url"], #Url')
+    if (urlInput) { await urlInput.fill(url) }
+
+    // Description
+    const descInput = await pw.$('textarea[name="Description"], #Description')
+    if (descInput) { await descInput.fill(description || '') }
+
+    // Tags
+    if (tags?.length) {
+      const tagInput = await pw.$('input[name="tags"], input[placeholder*="tag"]')
+      if (tagInput) {
+        for (const tag of tags) {
+          await tagInput.fill(tag)
+          await pw.keyboard.press('Enter')
+          await pw.waitForTimeout(300)
+        }
+      }
+    }
+
+    // Submit
+    await pw.click('button[type="submit"], input[type="submit"]')
+    await pw.waitForTimeout(3000)
+
+    return { submitted: true, name, url }
+  }
+}
+
+module.exports = { AlternativeToAdapter }

package/src/act/adapters/betalist.js

@@ -0,0 +1,66 @@
+const https = require('https')
+
+/**
+ * BetaList platform adapter.
+ * Uses their submission API (POST to betalist.com/api/v1/startups).
+ * Requires API key from betalist.com/developers
+ */
+class BetaListAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      default:
+        throw new Error(`Unsupported BetaList action: ${action}`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { name, url, tagline, description, email, account } = params
+    const token = this._getToken(account, ctx)
+
+    const body = JSON.stringify({
+      startup: {
+        name,
+        url,
+        one_liner: tagline,
+        description: description || tagline,
+        email: email || ''
+      }
+    })
+
+    const data = await new Promise((resolve, reject) => {
+      const opts = {
+        hostname: 'betalist.com',
+        path: '/api/v1/startups',
+        method: 'POST',
+        headers: {
+          'Authorization': `Token ${token}`,
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(body)
+        }
+      }
+      const req = https.request(opts, res => {
+        let data = ''
+        res.on('data', c => data += c)
+        res.on('end', () => {
+          try { resolve(JSON.parse(data)) }
+          catch (e) { resolve({ raw: data, status: res.statusCode }) }
+        })
+      })
+      req.on('error', reject)
+      req.write(body)
+      req.end()
+    })
+
+    return { submitted: true, ...data }
+  }
+
+  _getToken(account, ctx) {
+    if (account?.apiKey) return account.apiKey
+    return ctx?.config?.accounts?.betalist?.apiKey || process.env.BETALIST_API_KEY || ''
+  }
+}
+
+module.exports = { BetaListAdapter }

package/src/act/adapters/devhunt.js

@@ -0,0 +1,56 @@
+const https = require('https')
+
+/**
+ * DevHunt platform adapter.
+ * Dev-focused Product Hunt alternative.
+ * Uses GitHub OAuth — you log in with GitHub and submit tools.
+ * Browser automation for submission.
+ */
+class DevHuntAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      default:
+        throw new Error(`Unsupported DevHunt action: ${action}`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { name, url, description, tagline, githubUrl, _cookies, _browse } = params
+    if (!_browse) throw new Error('DevHunt requires browse engine')
+
+    const page = await _browse('https://devhunt.org/submit', {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const pw = page._page
+    if (!pw) throw new Error('DevHunt requires getPage access')
+
+    await pw.waitForSelector('input, form', { timeout: 10000 })
+
+    // Fill form fields
+    const fields = [
+      ['input[name="name"], input[placeholder*="name"]', name],
+      ['input[name="url"], input[placeholder*="url"], input[placeholder*="URL"]', url],
+      ['input[name="tagline"], input[placeholder*="tagline"]', tagline || description?.slice(0, 100)],
+      ['textarea[name="description"], textarea[placeholder*="description"]', description],
+      ['input[name="github_url"], input[placeholder*="github"]', githubUrl]
+    ]
+
+    for (const [selector, value] of fields) {
+      if (!value) continue
+      const el = await pw.$(selector)
+      if (el) await el.fill(value)
+    }
+
+    await pw.click('button[type="submit"], input[type="submit"]')
+    await pw.waitForTimeout(3000)
+
+    return { submitted: true, name, url }
+  }
+}
+
+module.exports = { DevHuntAdapter }

package/src/act/adapters/directory.js

@@ -0,0 +1,219 @@
+/**
+ * Generic directory submission adapter.
+ * Handles Tier 2 launch/SaaS directories that all have similar forms:
+ * name, url, tagline, description, email, category.
+ * 
+ * Supported directories (with custom selectors):
+ * - MicroLaunch, Uneed, Peerlist, Fazier, BetaPage
+ * - LaunchingNext, StartupStash, SideProjectors
+ * - TAIFT, Futurepedia, Toolify, OpenTools
+ * - Crunchbase, G2, StackShare (profile claim)
+ * 
+ * Falls back to generic form detection for unknown directories.
+ */
+
+const DIRECTORY_CONFIGS = {
+  microlaunch: {
+    submitUrl: 'https://microlaunch.net/submit',
+    fields: {
+      name: 'input[name="name"], input[placeholder*="product name"]',
+      url: 'input[name="url"], input[type="url"]',
+      tagline: 'input[name="tagline"], input[placeholder*="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  uneed: {
+    submitUrl: 'https://www.uneed.best/submit',
+    fields: {
+      name: 'input[name="name"]',
+      url: 'input[name="url"], input[type="url"]',
+      tagline: 'input[name="tagline"], input[name="slogan"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  peerlist: {
+    submitUrl: 'https://peerlist.io/projects/new',
+    fields: {
+      name: 'input[name="name"], input[placeholder*="project name"]',
+      url: 'input[name="url"], input[type="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  fazier: {
+    submitUrl: 'https://fazier.com/submit',
+    fields: {
+      name: 'input[name="name"]',
+      url: 'input[name="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  betapage: {
+    submitUrl: 'https://betapage.co/submit-startup',
+    fields: {
+      name: 'input[name="name"], #startup_name',
+      url: 'input[name="url"], input[name="website"]',
+      tagline: 'input[name="tagline"], input[name="oneliner"]',
+      description: 'textarea[name="description"]',
+      email: 'input[name="email"], input[type="email"]',
+    }
+  },
+  launchingnext: {
+    submitUrl: 'https://www.launchingnext.com/submit/',
+    fields: {
+      name: 'input[name="name"], input[id="name"]',
+      url: 'input[name="url"], input[id="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+      email: 'input[name="email"]',
+    }
+  },
+  startupstash: {
+    submitUrl: 'https://startupstash.com/add-listing/',
+    fields: {
+      name: 'input[name="title"], input[name="name"]',
+      url: 'input[name="url"], input[name="website"]',
+      description: 'textarea[name="description"], textarea[name="content"]',
+    }
+  },
+  sideprojectors: {
+    submitUrl: 'https://www.sideprojectors.com/project/new',
+    fields: {
+      name: 'input[name="title"], input[name="name"]',
+      url: 'input[name="url"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  taift: {
+    submitUrl: 'https://theresanaiforthat.com/submit/',
+    fields: {
+      name: 'input[name="name"], input[placeholder*="name"]',
+      url: 'input[name="url"], input[type="url"]',
+      tagline: 'input[name="tagline"], input[name="short_description"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  futurepedia: {
+    submitUrl: 'https://www.futurepedia.io/submit-tool',
+    fields: {
+      name: 'input[name="name"]',
+      url: 'input[name="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  crunchbase: {
+    submitUrl: 'https://www.crunchbase.com/add-new',
+    fields: {
+      name: 'input[name="name"], input[placeholder*="organization"]',
+      url: 'input[name="url"], input[name="website"]',
+      description: 'textarea[name="description"], textarea[name="short_description"]',
+    }
+  },
+  g2: {
+    submitUrl: 'https://www.g2.com/products/new',
+    fields: {
+      name: 'input[name="name"], input[name="product_name"]',
+      url: 'input[name="url"], input[name="website"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  stackshare: {
+    submitUrl: 'https://stackshare.io/tools/new',
+    fields: {
+      name: 'input[name="name"]',
+      url: 'input[name="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  appsumo: {
+    submitUrl: 'https://sell.appsumo.com/',
+    fields: {
+      name: 'input[name="name"], input[name="product_name"]',
+      url: 'input[name="url"]',
+      description: 'textarea[name="description"]',
+    }
+  }
+}
+
+class DirectoryAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      case 'list-directories':
+        return { directories: Object.keys(DIRECTORY_CONFIGS) }
+      default:
+        throw new Error(`Unsupported Directory action: ${action}`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { directory, name, url, tagline, description, email, category, _cookies, _browse } = params
+    if (!_browse) throw new Error('Directory submission requires browse engine')
+
+    const config = DIRECTORY_CONFIGS[directory?.toLowerCase()]
+    const submitUrl = config?.submitUrl || params.submitUrl
+    if (!submitUrl) {
+      throw new Error(`Unknown directory "${directory}". Known: ${Object.keys(DIRECTORY_CONFIGS).join(', ')}. Or pass submitUrl directly.`)
+    }
+
+    const page = await _browse(submitUrl, {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const pw = page._page
+    if (!pw) throw new Error('Directory adapter requires getPage access')
+
+    await pw.waitForSelector('input, form', { timeout: 15000 })
+
+    // Fill known fields
+    const fields = config?.fields || {}
+    const data = { name, url, tagline, description, email, category }
+
+    for (const [field, value] of Object.entries(data)) {
+      if (!value) continue
+      
+      // Try config selector first, then generic selectors
+      const selectors = [
+        fields[field],
+        `input[name="${field}"]`,
+        `textarea[name="${field}"]`,
+        `input[placeholder*="${field}"]`,
+        `#${field}`
+      ].filter(Boolean)
+
+      for (const sel of selectors) {
+        try {
+          const el = await pw.$(sel)
+          if (el) {
+            const tag = await el.evaluate(e => e.tagName.toLowerCase())
+            if (tag === 'textarea') {
+              await el.fill(value)
+            } else {
+              await el.fill(value)
+            }
+            break
+          }
+        } catch (e) { /* selector didn't match, try next */ }
+      }
+    }
+
+    // Try to submit
+    try {
+      await pw.click('button[type="submit"], input[type="submit"], button:has-text("Submit"), button:has-text("Launch"), button:has-text("Add")')
+      await pw.waitForTimeout(3000)
+    } catch (e) {
+      // No submit button found — might need manual submission
+      return { filled: true, submitted: false, note: 'Form filled but submit button not found', directory }
+    }
+
+    return { submitted: true, directory: directory || submitUrl, name, url }
+  }
+}
+
+module.exports = { DirectoryAdapter, DIRECTORY_CONFIGS }

package/src/act/adapters/discord.js

@@ -0,0 +1,131 @@
+const https = require('https')
+
+/**
+ * Discord platform adapter.
+ * Uses Discord Bot API or webhooks.
+ * Bot token for full access, webhooks for simple posting.
+ */
+class DiscordAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'send':
+        return this._sendMessage(params, ctx)
+      case 'webhook':
+        return this._webhook(params, ctx)
+      case 'create-thread':
+        return this._createThread(params, ctx)
+      default:
+        throw new Error(`Unsupported Discord action: ${action}`)
+    }
+  }
+
+  async _sendMessage(params, ctx) {
+    const { channelId, content, embeds, account } = params
+    const token = this._getToken(account, ctx)
+
+    const body = { content: content || '' }
+    if (embeds) body.embeds = embeds
+
+    const data = await discordApi('POST', `/channels/${channelId}/messages`, body, token)
+    return { messageId: data.id, channelId: data.channel_id }
+  }
+
+  async _webhook(params, ctx) {
+    const { webhookUrl, content, username, avatarUrl, embeds } = params
+    if (!webhookUrl) throw new Error('Discord webhook URL required')
+
+    const body = { content: content || '' }
+    if (username) body.username = username
+    if (avatarUrl) body.avatar_url = avatarUrl
+    if (embeds) body.embeds = embeds
+
+    const urlObj = new URL(webhookUrl)
+    const data = await new Promise((resolve, reject) => {
+      const json = JSON.stringify(body)
+      const opts = {
+        hostname: urlObj.hostname,
+        path: urlObj.pathname + urlObj.search,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(json)
+        }
+      }
+      const req = https.request(opts, res => {
+        let data = ''
+        res.on('data', c => data += c)
+        res.on('end', () => {
+          if (res.statusCode === 204) return resolve({ sent: true })
+          try { resolve(JSON.parse(data)) }
+          catch (e) { resolve({ sent: res.statusCode < 300 }) }
+        })
+      })
+      req.on('error', reject)
+      req.write(json)
+      req.end()
+    })
+
+    return { sent: true, ...data }
+  }
+
+  async _createThread(params, ctx) {
+    const { channelId, name, content, account } = params
+    const token = this._getToken(account, ctx)
+
+    const thread = await discordApi('POST', `/channels/${channelId}/threads`, {
+      name,
+      type: 11, // PUBLIC_THREAD
+      auto_archive_duration: 1440
+    }, token)
+
+    if (content) {
+      await discordApi('POST', `/channels/${thread.id}/messages`, { content }, token)
+    }
+
+    return { threadId: thread.id, name: thread.name }
+  }
+
+  _getToken(account, ctx) {
+    if (account?.botToken) return `Bot ${account.botToken}`
+    const token = ctx?.config?.accounts?.discord?.botToken ||
+                  process.env.DISCORD_BOT_TOKEN
+    if (!token) throw new Error('Discord bot token required')
+    return `Bot ${token}`
+  }
+}
+
+function discordApi(method, path, body, token) {
+  return new Promise((resolve, reject) => {
+    const json = JSON.stringify(body)
+    const opts = {
+      hostname: 'discord.com',
+      path: `/api/v10${path}`,
+      method,
+      headers: {
+        'Authorization': token,
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(json),
+        'User-Agent': 'Spectrawl/0.1.0'
+      }
+    }
+    const req = https.request(opts, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try {
+          const parsed = data ? JSON.parse(data) : {}
+          if (res.statusCode >= 400) {
+            reject(new Error(`Discord API ${res.statusCode}: ${parsed.message || data.slice(0, 200)}`))
+          } else { resolve(parsed) }
+        } catch (e) { reject(new Error(`Invalid Discord response: ${data.slice(0, 200)}`)) }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(15000, () => { req.destroy(); reject(new Error('Discord API timeout')) })
+    req.write(json)
+    req.end()
+  })
+}
+
+module.exports = { DiscordAdapter }