spectrawl 0.1.1 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spectrawl",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "The unified web layer for AI agents. Search, browse, authenticate, act — one tool, self-hosted, free.",
   "main": "src/index.js",
   "types": "index.d.ts",

package/src/act/adapters/alternativeto.js

@@ -0,0 +1,64 @@
+/**
+ * AlternativeTo platform adapter.
+ * No public API — browser automation for submitting alternatives.
+ * High SEO value: people search "[tool] alternative" constantly.
+ */
+class AlternativeToAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      default:
+        throw new Error(`Unsupported AlternativeTo action: ${action}. Requires browser automation.`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { name, url, description, category, tags, _cookies, _browse } = params
+    if (!_browse) throw new Error('AlternativeTo requires browse engine')
+
+    const page = await _browse('https://alternativeto.net/add/', {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const pw = page._page
+    if (!pw) throw new Error('AlternativeTo requires getPage access')
+
+    // Fill the submission form
+    await pw.waitForSelector('input[name="Name"], #Name', { timeout: 10000 })
+
+    // Name
+    const nameInput = await pw.$('input[name="Name"], #Name')
+    if (nameInput) { await nameInput.fill(name) }
+
+    // URL
+    const urlInput = await pw.$('input[name="Url"], input[name="url"], #Url')
+    if (urlInput) { await urlInput.fill(url) }
+
+    // Description
+    const descInput = await pw.$('textarea[name="Description"], #Description')
+    if (descInput) { await descInput.fill(description || '') }
+
+    // Tags
+    if (tags?.length) {
+      const tagInput = await pw.$('input[name="tags"], input[placeholder*="tag"]')
+      if (tagInput) {
+        for (const tag of tags) {
+          await tagInput.fill(tag)
+          await pw.keyboard.press('Enter')
+          await pw.waitForTimeout(300)
+        }
+      }
+    }
+
+    // Submit
+    await pw.click('button[type="submit"], input[type="submit"]')
+    await pw.waitForTimeout(3000)
+
+    return { submitted: true, name, url }
+  }
+}
+
+module.exports = { AlternativeToAdapter }

package/src/act/adapters/betalist.js

@@ -0,0 +1,66 @@
+const https = require('https')
+
+/**
+ * BetaList platform adapter.
+ * Uses their submission API (POST to betalist.com/api/v1/startups).
+ * Requires API key from betalist.com/developers
+ */
+class BetaListAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      default:
+        throw new Error(`Unsupported BetaList action: ${action}`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { name, url, tagline, description, email, account } = params
+    const token = this._getToken(account, ctx)
+
+    const body = JSON.stringify({
+      startup: {
+        name,
+        url,
+        one_liner: tagline,
+        description: description || tagline,
+        email: email || ''
+      }
+    })
+
+    const data = await new Promise((resolve, reject) => {
+      const opts = {
+        hostname: 'betalist.com',
+        path: '/api/v1/startups',
+        method: 'POST',
+        headers: {
+          'Authorization': `Token ${token}`,
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(body)
+        }
+      }
+      const req = https.request(opts, res => {
+        let data = ''
+        res.on('data', c => data += c)
+        res.on('end', () => {
+          try { resolve(JSON.parse(data)) }
+          catch (e) { resolve({ raw: data, status: res.statusCode }) }
+        })
+      })
+      req.on('error', reject)
+      req.write(body)
+      req.end()
+    })
+
+    return { submitted: true, ...data }
+  }
+
+  _getToken(account, ctx) {
+    if (account?.apiKey) return account.apiKey
+    return ctx?.config?.accounts?.betalist?.apiKey || process.env.BETALIST_API_KEY || ''
+  }
+}
+
+module.exports = { BetaListAdapter }

package/src/act/adapters/devhunt.js

@@ -0,0 +1,56 @@
+const https = require('https')
+
+/**
+ * DevHunt platform adapter.
+ * Dev-focused Product Hunt alternative.
+ * Uses GitHub OAuth — you log in with GitHub and submit tools.
+ * Browser automation for submission.
+ */
+class DevHuntAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      default:
+        throw new Error(`Unsupported DevHunt action: ${action}`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { name, url, description, tagline, githubUrl, _cookies, _browse } = params
+    if (!_browse) throw new Error('DevHunt requires browse engine')
+
+    const page = await _browse('https://devhunt.org/submit', {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const pw = page._page
+    if (!pw) throw new Error('DevHunt requires getPage access')
+
+    await pw.waitForSelector('input, form', { timeout: 10000 })
+
+    // Fill form fields
+    const fields = [
+      ['input[name="name"], input[placeholder*="name"]', name],
+      ['input[name="url"], input[placeholder*="url"], input[placeholder*="URL"]', url],
+      ['input[name="tagline"], input[placeholder*="tagline"]', tagline || description?.slice(0, 100)],
+      ['textarea[name="description"], textarea[placeholder*="description"]', description],
+      ['input[name="github_url"], input[placeholder*="github"]', githubUrl]
+    ]
+
+    for (const [selector, value] of fields) {
+      if (!value) continue
+      const el = await pw.$(selector)
+      if (el) await el.fill(value)
+    }
+
+    await pw.click('button[type="submit"], input[type="submit"]')
+    await pw.waitForTimeout(3000)
+
+    return { submitted: true, name, url }
+  }
+}
+
+module.exports = { DevHuntAdapter }

package/src/act/adapters/directory.js

@@ -0,0 +1,219 @@
+/**
+ * Generic directory submission adapter.
+ * Handles Tier 2 launch/SaaS directories that all have similar forms:
+ * name, url, tagline, description, email, category.
+ * 
+ * Supported directories (with custom selectors):
+ * - MicroLaunch, Uneed, Peerlist, Fazier, BetaPage
+ * - LaunchingNext, StartupStash, SideProjectors
+ * - TAIFT, Futurepedia, Toolify, OpenTools
+ * - Crunchbase, G2, StackShare (profile claim)
+ * 
+ * Falls back to generic form detection for unknown directories.
+ */
+
+const DIRECTORY_CONFIGS = {
+  microlaunch: {
+    submitUrl: 'https://microlaunch.net/submit',
+    fields: {
+      name: 'input[name="name"], input[placeholder*="product name"]',
+      url: 'input[name="url"], input[type="url"]',
+      tagline: 'input[name="tagline"], input[placeholder*="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  uneed: {
+    submitUrl: 'https://www.uneed.best/submit',
+    fields: {
+      name: 'input[name="name"]',
+      url: 'input[name="url"], input[type="url"]',
+      tagline: 'input[name="tagline"], input[name="slogan"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  peerlist: {
+    submitUrl: 'https://peerlist.io/projects/new',
+    fields: {
+      name: 'input[name="name"], input[placeholder*="project name"]',
+      url: 'input[name="url"], input[type="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  fazier: {
+    submitUrl: 'https://fazier.com/submit',
+    fields: {
+      name: 'input[name="name"]',
+      url: 'input[name="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  betapage: {
+    submitUrl: 'https://betapage.co/submit-startup',
+    fields: {
+      name: 'input[name="name"], #startup_name',
+      url: 'input[name="url"], input[name="website"]',
+      tagline: 'input[name="tagline"], input[name="oneliner"]',
+      description: 'textarea[name="description"]',
+      email: 'input[name="email"], input[type="email"]',
+    }
+  },
+  launchingnext: {
+    submitUrl: 'https://www.launchingnext.com/submit/',
+    fields: {
+      name: 'input[name="name"], input[id="name"]',
+      url: 'input[name="url"], input[id="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+      email: 'input[name="email"]',
+    }
+  },
+  startupstash: {
+    submitUrl: 'https://startupstash.com/add-listing/',
+    fields: {
+      name: 'input[name="title"], input[name="name"]',
+      url: 'input[name="url"], input[name="website"]',
+      description: 'textarea[name="description"], textarea[name="content"]',
+    }
+  },
+  sideprojectors: {
+    submitUrl: 'https://www.sideprojectors.com/project/new',
+    fields: {
+      name: 'input[name="title"], input[name="name"]',
+      url: 'input[name="url"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  taift: {
+    submitUrl: 'https://theresanaiforthat.com/submit/',
+    fields: {
+      name: 'input[name="name"], input[placeholder*="name"]',
+      url: 'input[name="url"], input[type="url"]',
+      tagline: 'input[name="tagline"], input[name="short_description"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  futurepedia: {
+    submitUrl: 'https://www.futurepedia.io/submit-tool',
+    fields: {
+      name: 'input[name="name"]',
+      url: 'input[name="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  crunchbase: {
+    submitUrl: 'https://www.crunchbase.com/add-new',
+    fields: {
+      name: 'input[name="name"], input[placeholder*="organization"]',
+      url: 'input[name="url"], input[name="website"]',
+      description: 'textarea[name="description"], textarea[name="short_description"]',
+    }
+  },
+  g2: {
+    submitUrl: 'https://www.g2.com/products/new',
+    fields: {
+      name: 'input[name="name"], input[name="product_name"]',
+      url: 'input[name="url"], input[name="website"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  stackshare: {
+    submitUrl: 'https://stackshare.io/tools/new',
+    fields: {
+      name: 'input[name="name"]',
+      url: 'input[name="url"]',
+      tagline: 'input[name="tagline"]',
+      description: 'textarea[name="description"]',
+    }
+  },
+  appsumo: {
+    submitUrl: 'https://sell.appsumo.com/',
+    fields: {
+      name: 'input[name="name"], input[name="product_name"]',
+      url: 'input[name="url"]',
+      description: 'textarea[name="description"]',
+    }
+  }
+}
+
+class DirectoryAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      case 'list-directories':
+        return { directories: Object.keys(DIRECTORY_CONFIGS) }
+      default:
+        throw new Error(`Unsupported Directory action: ${action}`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { directory, name, url, tagline, description, email, category, _cookies, _browse } = params
+    if (!_browse) throw new Error('Directory submission requires browse engine')
+
+    const config = DIRECTORY_CONFIGS[directory?.toLowerCase()]
+    const submitUrl = config?.submitUrl || params.submitUrl
+    if (!submitUrl) {
+      throw new Error(`Unknown directory "${directory}". Known: ${Object.keys(DIRECTORY_CONFIGS).join(', ')}. Or pass submitUrl directly.`)
+    }
+
+    const page = await _browse(submitUrl, {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const pw = page._page
+    if (!pw) throw new Error('Directory adapter requires getPage access')
+
+    await pw.waitForSelector('input, form', { timeout: 15000 })
+
+    // Fill known fields
+    const fields = config?.fields || {}
+    const data = { name, url, tagline, description, email, category }
+
+    for (const [field, value] of Object.entries(data)) {
+      if (!value) continue
+      
+      // Try config selector first, then generic selectors
+      const selectors = [
+        fields[field],
+        `input[name="${field}"]`,
+        `textarea[name="${field}"]`,
+        `input[placeholder*="${field}"]`,
+        `#${field}`
+      ].filter(Boolean)
+
+      for (const sel of selectors) {
+        try {
+          const el = await pw.$(sel)
+          if (el) {
+            const tag = await el.evaluate(e => e.tagName.toLowerCase())
+            if (tag === 'textarea') {
+              await el.fill(value)
+            } else {
+              await el.fill(value)
+            }
+            break
+          }
+        } catch (e) { /* selector didn't match, try next */ }
+      }
+    }
+
+    // Try to submit
+    try {
+      await pw.click('button[type="submit"], input[type="submit"], button:has-text("Submit"), button:has-text("Launch"), button:has-text("Add")')
+      await pw.waitForTimeout(3000)
+    } catch (e) {
+      // No submit button found — might need manual submission
+      return { filled: true, submitted: false, note: 'Form filled but submit button not found', directory }
+    }
+
+    return { submitted: true, directory: directory || submitUrl, name, url }
+  }
+}
+
+module.exports = { DirectoryAdapter, DIRECTORY_CONFIGS }

package/src/act/adapters/discord.js

@@ -0,0 +1,131 @@
+const https = require('https')
+
+/**
+ * Discord platform adapter.
+ * Uses Discord Bot API or webhooks.
+ * Bot token for full access, webhooks for simple posting.
+ */
+class DiscordAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'send':
+        return this._sendMessage(params, ctx)
+      case 'webhook':
+        return this._webhook(params, ctx)
+      case 'create-thread':
+        return this._createThread(params, ctx)
+      default:
+        throw new Error(`Unsupported Discord action: ${action}`)
+    }
+  }
+
+  async _sendMessage(params, ctx) {
+    const { channelId, content, embeds, account } = params
+    const token = this._getToken(account, ctx)
+
+    const body = { content: content || '' }
+    if (embeds) body.embeds = embeds
+
+    const data = await discordApi('POST', `/channels/${channelId}/messages`, body, token)
+    return { messageId: data.id, channelId: data.channel_id }
+  }
+
+  async _webhook(params, ctx) {
+    const { webhookUrl, content, username, avatarUrl, embeds } = params
+    if (!webhookUrl) throw new Error('Discord webhook URL required')
+
+    const body = { content: content || '' }
+    if (username) body.username = username
+    if (avatarUrl) body.avatar_url = avatarUrl
+    if (embeds) body.embeds = embeds
+
+    const urlObj = new URL(webhookUrl)
+    const data = await new Promise((resolve, reject) => {
+      const json = JSON.stringify(body)
+      const opts = {
+        hostname: urlObj.hostname,
+        path: urlObj.pathname + urlObj.search,
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Content-Length': Buffer.byteLength(json)
+        }
+      }
+      const req = https.request(opts, res => {
+        let data = ''
+        res.on('data', c => data += c)
+        res.on('end', () => {
+          if (res.statusCode === 204) return resolve({ sent: true })
+          try { resolve(JSON.parse(data)) }
+          catch (e) { resolve({ sent: res.statusCode < 300 }) }
+        })
+      })
+      req.on('error', reject)
+      req.write(json)
+      req.end()
+    })
+
+    return { sent: true, ...data }
+  }
+
+  async _createThread(params, ctx) {
+    const { channelId, name, content, account } = params
+    const token = this._getToken(account, ctx)
+
+    const thread = await discordApi('POST', `/channels/${channelId}/threads`, {
+      name,
+      type: 11, // PUBLIC_THREAD
+      auto_archive_duration: 1440
+    }, token)
+
+    if (content) {
+      await discordApi('POST', `/channels/${thread.id}/messages`, { content }, token)
+    }
+
+    return { threadId: thread.id, name: thread.name }
+  }
+
+  _getToken(account, ctx) {
+    if (account?.botToken) return `Bot ${account.botToken}`
+    const token = ctx?.config?.accounts?.discord?.botToken ||
+                  process.env.DISCORD_BOT_TOKEN
+    if (!token) throw new Error('Discord bot token required')
+    return `Bot ${token}`
+  }
+}
+
+function discordApi(method, path, body, token) {
+  return new Promise((resolve, reject) => {
+    const json = JSON.stringify(body)
+    const opts = {
+      hostname: 'discord.com',
+      path: `/api/v10${path}`,
+      method,
+      headers: {
+        'Authorization': token,
+        'Content-Type': 'application/json',
+        'Content-Length': Buffer.byteLength(json),
+        'User-Agent': 'Spectrawl/0.1.0'
+      }
+    }
+    const req = https.request(opts, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try {
+          const parsed = data ? JSON.parse(data) : {}
+          if (res.statusCode >= 400) {
+            reject(new Error(`Discord API ${res.statusCode}: ${parsed.message || data.slice(0, 200)}`))
+          } else { resolve(parsed) }
+        } catch (e) { reject(new Error(`Invalid Discord response: ${data.slice(0, 200)}`)) }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(15000, () => { req.destroy(); reject(new Error('Discord API timeout')) })
+    req.write(json)
+    req.end()
+  })
+}
+
+module.exports = { DiscordAdapter }

package/src/act/adapters/github.js

@@ -0,0 +1,152 @@
+const https = require('https')
+
+/**
+ * GitHub platform adapter.
+ * Uses GitHub REST API v3 with personal access tokens.
+ * Actions: create repo, create/update file, create discussion, create issue
+ */
+class GitHubAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'create-repo':
+        return this._createRepo(params, ctx)
+      case 'create-file':
+      case 'push':
+        return this._createFile(params, ctx)
+      case 'create-issue':
+        return this._createIssue(params, ctx)
+      case 'create-release':
+        return this._createRelease(params, ctx)
+      case 'update-readme':
+        return this._updateReadme(params, ctx)
+      default:
+        throw new Error(`Unsupported GitHub action: ${action}`)
+    }
+  }
+
+  async _createRepo(params, ctx) {
+    const { name, description, homepage, topics, isPrivate, account } = params
+    const token = this._getToken(account, ctx)
+
+    const data = await ghApi('POST', '/user/repos', {
+      name,
+      description: description || '',
+      homepage: homepage || '',
+      private: isPrivate || false,
+      auto_init: true
+    }, token)
+
+    if (topics?.length) {
+      await ghApi('PUT', `/repos/${data.full_name}/topics`, { names: topics }, token)
+    }
+
+    return { repoId: data.id, url: data.html_url, fullName: data.full_name }
+  }
+
+  async _createFile(params, ctx) {
+    const { repo, path, content, message, branch, account } = params
+    const token = this._getToken(account, ctx)
+
+    // Check if file exists (for updates)
+    let sha
+    try {
+      const existing = await ghApi('GET', `/repos/${repo}/contents/${path}${branch ? `?ref=${branch}` : ''}`, null, token)
+      sha = existing.sha
+    } catch (e) { /* file doesn't exist, that's fine */ }
+
+    const body = {
+      message: message || `Update ${path}`,
+      content: Buffer.from(content).toString('base64'),
+      branch: branch || 'main'
+    }
+    if (sha) body.sha = sha
+
+    const data = await ghApi('PUT', `/repos/${repo}/contents/${path}`, body, token)
+    return { url: data.content?.html_url, sha: data.content?.sha }
+  }
+
+  async _createIssue(params, ctx) {
+    const { repo, title, body, labels, account } = params
+    const token = this._getToken(account, ctx)
+
+    const data = await ghApi('POST', `/repos/${repo}/issues`, {
+      title,
+      body: body || '',
+      labels: labels || []
+    }, token)
+
+    return { issueId: data.number, url: data.html_url }
+  }
+
+  async _createRelease(params, ctx) {
+    const { repo, tag, name, body, draft, prerelease, account } = params
+    const token = this._getToken(account, ctx)
+
+    const data = await ghApi('POST', `/repos/${repo}/releases`, {
+      tag_name: tag,
+      name: name || tag,
+      body: body || '',
+      draft: draft || false,
+      prerelease: prerelease || false
+    }, token)
+
+    return { releaseId: data.id, url: data.html_url }
+  }
+
+  async _updateReadme(params, ctx) {
+    const { repo, content, account } = params
+    return this._createFile({ repo, path: 'README.md', content, message: 'Update README', account }, ctx)
+  }
+
+  _getToken(account, ctx) {
+    if (account?.token) return account.token
+    const token = ctx?.config?.accounts?.github?.token ||
+                  process.env.GITHUB_TOKEN
+    if (!token) throw new Error('GitHub token required')
+    return token
+  }
+}
+
+function ghApi(method, path, body, token) {
+  return new Promise((resolve, reject) => {
+    const opts = {
+      hostname: 'api.github.com',
+      path,
+      method,
+      headers: {
+        'Authorization': `Bearer ${token}`,
+        'Accept': 'application/vnd.github+json',
+        'User-Agent': 'Spectrawl/0.1.0',
+        'X-GitHub-Api-Version': '2022-11-28'
+      }
+    }
+
+    if (body && method !== 'GET') {
+      const json = JSON.stringify(body)
+      opts.headers['Content-Type'] = 'application/json'
+      opts.headers['Content-Length'] = Buffer.byteLength(json)
+    }
+
+    const req = https.request(opts, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try {
+          const parsed = data ? JSON.parse(data) : {}
+          if (res.statusCode >= 400) {
+            reject(new Error(`GitHub API ${res.statusCode}: ${parsed.message || data.slice(0, 200)}`))
+          } else {
+            resolve(parsed)
+          }
+        } catch (e) { reject(new Error(`Invalid GitHub response: ${data.slice(0, 200)}`)) }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(15000, () => { req.destroy(); reject(new Error('GitHub API timeout')) })
+    if (body && method !== 'GET') req.write(JSON.stringify(body))
+    req.end()
+  })
+}
+
+module.exports = { GitHubAdapter }