spectrawl 0.1.1 → 0.2.0

package/src/act/adapters/hackernews.js

@@ -0,0 +1,161 @@
+const https = require('https')
+const http = require('http')
+
+/**
+ * Hacker News platform adapter.
+ * HN has no official write API — uses Firebase API for reading
+ * and browser automation / cookie-based form submission for posting.
+ * 
+ * Read: Firebase API (no auth needed)
+ * Write: Cookie-based form POST to news.ycombinator.com
+ */
+class HackerNewsAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      case 'comment':
+        return this._comment(params, ctx)
+      case 'upvote':
+        return this._upvote(params, ctx)
+      default:
+        throw new Error(`Unsupported HN action: ${action}`)
+    }
+  }
+
+  /**
+   * Submit a story to HN.
+   * Requires login cookie (user session).
+   */
+  async _submit(params, ctx) {
+    const { title, url, text, _cookies } = params
+    const cookie = this._getCookie(_cookies)
+
+    // Get FNID (form nonce) from submit page
+    const submitPage = await fetchHN('/submit', cookie)
+    const fnidMatch = submitPage.match(/name="fnid" value="([^"]+)"/)
+    if (!fnidMatch) throw new Error('Could not get HN submit form token. Check cookie validity.')
+    const fnid = fnidMatch[1]
+
+    // Submit the form
+    const form = new URLSearchParams()
+    form.append('fnid', fnid)
+    form.append('fnop', 'submit-page')
+    form.append('title', title)
+    if (url) form.append('url', url)
+    if (text && !url) form.append('text', text)
+
+    const result = await postHN('/r', form.toString(), cookie)
+
+    // HN redirects to /newest on success
+    if (result.includes('newest') || result.includes('item?id=')) {
+      const idMatch = result.match(/item\?id=(\d+)/)
+      return {
+        url: idMatch ? `https://news.ycombinator.com/item?id=${idMatch[1]}` : 'https://news.ycombinator.com/newest',
+        submitted: true
+      }
+    }
+
+    // Check for rate limit or error
+    if (result.includes('submitting too fast')) {
+      throw new Error('HN rate limit: submitting too fast')
+    }
+
+    return { submitted: true, url: 'https://news.ycombinator.com/newest' }
+  }
+
+  async _comment(params, ctx) {
+    const { parentId, text, _cookies } = params
+    const cookie = this._getCookie(_cookies)
+
+    // Get the item page to find comment form HMAC
+    const itemPage = await fetchHN(`/item?id=${parentId}`, cookie)
+    const hmacMatch = itemPage.match(/name="hmac" value="([^"]+)"/)
+    if (!hmacMatch) throw new Error('Could not get HN comment form token')
+
+    const form = new URLSearchParams()
+    form.append('parent', parentId)
+    form.append('goto', `item?id=${parentId}`)
+    form.append('hmac', hmacMatch[1])
+    form.append('text', text)
+
+    await postHN('/comment', form.toString(), cookie)
+    return { commented: true, parentId }
+  }
+
+  async _upvote(params, ctx) {
+    const { itemId, _cookies } = params
+    const cookie = this._getCookie(_cookies)
+
+    // Get vote link from item page
+    const itemPage = await fetchHN(`/item?id=${itemId}`, cookie)
+    const voteMatch = itemPage.match(/id="up_(\d+)"[^>]*href="([^"]+)"/)
+    if (!voteMatch) throw new Error('Could not find upvote link (already voted or not logged in)')
+
+    await fetchHN(voteMatch[2], cookie)
+    return { upvoted: true, itemId }
+  }
+
+  _getCookie(cookies) {
+    if (!cookies) throw new Error('HN cookies required for posting')
+    if (typeof cookies === 'string') return cookies
+    
+    const userCookie = cookies.find(c => c.name === 'user')
+    if (!userCookie) throw new Error('HN user cookie not found')
+    return `user=${userCookie.value}`
+  }
+}
+
+function fetchHN(path, cookie) {
+  return new Promise((resolve, reject) => {
+    https.get({
+      hostname: 'news.ycombinator.com',
+      path,
+      headers: {
+        'Cookie': cookie,
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+      }
+    }, res => {
+      // Follow redirects
+      if (res.statusCode === 302 || res.statusCode === 301) {
+        const loc = res.headers.location
+        if (loc) return fetchHN(loc.startsWith('http') ? new URL(loc).pathname + new URL(loc).search : loc, cookie).then(resolve).catch(reject)
+      }
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => resolve(data))
+    }).on('error', reject)
+  })
+}
+
+function postHN(path, body, cookie) {
+  return new Promise((resolve, reject) => {
+    const opts = {
+      hostname: 'news.ycombinator.com',
+      path,
+      method: 'POST',
+      headers: {
+        'Cookie': cookie,
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Content-Length': Buffer.byteLength(body),
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+      }
+    }
+    const req = https.request(opts, res => {
+      let data = ''
+      // Follow redirect and capture location
+      if (res.statusCode === 302 || res.statusCode === 301) {
+        data = res.headers.location || ''
+      }
+      res.on('data', c => data += c)
+      res.on('end', () => resolve(data))
+    })
+    req.on('error', reject)
+    req.setTimeout(15000, () => { req.destroy(); reject(new Error('HN timeout')) })
+    req.write(body)
+    req.end()
+  })
+}
+
+module.exports = { HackerNewsAdapter }

package/src/act/adapters/huggingface.js

@@ -0,0 +1,103 @@
+const https = require('https')
+
+/**
+ * HuggingFace platform adapter.
+ * Uses HF Hub API for model/dataset/space management.
+ * Docs: https://huggingface.co/docs/hub/api
+ */
+class HuggingFaceAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'create-repo':
+        return this._createRepo(params, ctx)
+      case 'create-model-card':
+        return this._createModelCard(params, ctx)
+      case 'upload-file':
+        return this._uploadFile(params, ctx)
+      default:
+        throw new Error(`Unsupported HuggingFace action: ${action}`)
+    }
+  }
+
+  async _createRepo(params, ctx) {
+    const { name, type, private: isPrivate, account } = params
+    const token = this._getToken(account, ctx)
+
+    const data = await hfApi('POST', '/api/repos/create', {
+      name,
+      type: type || 'model', // model, dataset, space
+      private: isPrivate || false
+    }, token)
+
+    return {
+      url: data.url || `https://huggingface.co/${data.repoId || name}`,
+      repoId: data.repoId
+    }
+  }
+
+  async _createModelCard(params, ctx) {
+    const { repo, content, account } = params
+    const token = this._getToken(account, ctx)
+
+    // Upload README.md to the repo
+    const data = await hfApi('PUT', `/api/${repo}/upload/main/README.md`, content, token, 'text/plain')
+    return { updated: true, repo }
+  }
+
+  async _uploadFile(params, ctx) {
+    const { repo, path, content, branch, account } = params
+    const token = this._getToken(account, ctx)
+
+    const data = await hfApi('PUT', `/api/${repo}/upload/${branch || 'main'}/${path}`, content, token,
+      typeof content === 'string' ? 'text/plain' : 'application/octet-stream')
+    return { uploaded: true, path }
+  }
+
+  _getToken(account, ctx) {
+    if (account?.token) return account.token
+    const token = ctx?.config?.accounts?.huggingface?.token ||
+                  process.env.HF_TOKEN ||
+                  process.env.HUGGINGFACE_TOKEN
+    if (!token) throw new Error('HuggingFace token required. Get one from huggingface.co/settings/tokens')
+    return token
+  }
+}
+
+function hfApi(method, path, body, token, contentType) {
+  return new Promise((resolve, reject) => {
+    const isJson = !contentType || contentType === 'application/json'
+    const payload = isJson && typeof body === 'object' ? JSON.stringify(body) : (body || '')
+    const opts = {
+      hostname: 'huggingface.co',
+      path,
+      method,
+      headers: {
+        'Authorization': `Bearer ${token}`,
+        'Content-Type': contentType || 'application/json',
+        'Content-Length': Buffer.byteLength(payload)
+      }
+    }
+    const req = https.request(opts, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try {
+          const parsed = data ? JSON.parse(data) : {}
+          if (res.statusCode >= 400) {
+            reject(new Error(`HF API ${res.statusCode}: ${parsed.error || data.slice(0, 200)}`))
+          } else { resolve(parsed) }
+        } catch (e) {
+          if (res.statusCode < 300) resolve({ raw: data })
+          else reject(new Error(`HF API error: ${data.slice(0, 200)}`))
+        }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(30000, () => { req.destroy(); reject(new Error('HF API timeout')) })
+    req.write(payload)
+    req.end()
+  })
+}
+
+module.exports = { HuggingFaceAdapter }

package/src/act/adapters/medium.js

@@ -0,0 +1,107 @@
+const https = require('https')
+
+/**
+ * Medium platform adapter.
+ * Uses Medium's REST API with integration tokens.
+ * Docs: https://github.com/Medium/medium-api-docs
+ */
+class MediumAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+        return this._post(params, ctx)
+      default:
+        throw new Error(`Unsupported Medium action: ${action}`)
+    }
+  }
+
+  async _post(params, ctx) {
+    const { title, body, tags, publishStatus, account } = params
+    const token = this._getToken(account, ctx)
+
+    // Get user ID first
+    const me = await apiGet('https://api.medium.com/v1/me', token)
+    const userId = me.data?.id
+    if (!userId) throw new Error('Could not get Medium user ID')
+
+    const article = {
+      title,
+      contentFormat: 'markdown',
+      content: body,
+      tags: tags || [],
+      publishStatus: publishStatus || 'public' // public, draft, unlisted
+    }
+
+    const data = await apiPost(
+      `https://api.medium.com/v1/users/${userId}/posts`,
+      JSON.stringify(article),
+      token
+    )
+
+    return {
+      postId: data.data?.id,
+      url: data.data?.url,
+      title: data.data?.title
+    }
+  }
+
+  _getToken(account, ctx) {
+    if (account?.apiKey) return account.apiKey
+    const token = ctx?.config?.accounts?.medium?.apiKey ||
+                  process.env.MEDIUM_API_KEY
+    if (!token) throw new Error('Medium API key required. Get one from medium.com/me/settings')
+    return token
+  }
+}
+
+function apiGet(url, token) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    https.get({
+      hostname: urlObj.hostname,
+      path: urlObj.pathname,
+      headers: {
+        'Authorization': `Bearer ${token}`,
+        'Accept': 'application/json'
+      }
+    }, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { reject(new Error(`Invalid Medium response: ${data.slice(0, 200)}`)) }
+      })
+    }).on('error', reject)
+  })
+}
+
+function apiPost(url, body, token) {
+  return new Promise((resolve, reject) => {
+    const urlObj = new URL(url)
+    const opts = {
+      hostname: urlObj.hostname,
+      path: urlObj.pathname,
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${token}`,
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+        'Content-Length': Buffer.byteLength(body)
+      }
+    }
+    const req = https.request(opts, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { reject(new Error(`Invalid Medium response: ${data.slice(0, 200)}`)) }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(15000, () => { req.destroy(); reject(new Error('Medium API timeout')) })
+    req.write(body)
+    req.end()
+  })
+}
+
+module.exports = { MediumAdapter }

package/src/act/adapters/producthunt.js

@@ -0,0 +1,143 @@
+const https = require('https')
+
+/**
+ * Product Hunt platform adapter.
+ * Uses GraphQL API v2.
+ * Docs: https://api.producthunt.com/v2/docs
+ */
+class ProductHuntAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'launch':
+        return this._launch(params, ctx)
+      case 'comment':
+        return this._comment(params, ctx)
+      case 'upvote':
+        return this._upvote(params, ctx)
+      default:
+        throw new Error(`Unsupported Product Hunt action: ${action}`)
+    }
+  }
+
+  async _launch(params, ctx) {
+    const { name, tagline, url, description, topics, thumbnailUrl, account } = params
+    const token = this._getToken(account, ctx)
+
+    // Note: Product Hunt API v2 doesn't directly support creating posts
+    // via API anymore — launches go through producthunt.com/posts/new
+    // This uses the maker tools endpoint where available
+    const query = `
+      mutation CreatePost($input: PostCreateInput!) {
+        postCreate(input: $input) {
+          post {
+            id
+            name
+            tagline
+            url
+            votesCount
+          }
+        }
+      }
+    `
+
+    const data = await graphql(query, {
+      input: {
+        name,
+        tagline,
+        url,
+        description: description || '',
+        topicIds: topics || [],
+        thumbnailUrl: thumbnailUrl || ''
+      }
+    }, token)
+
+    const post = data.data?.postCreate?.post
+    if (!post) {
+      throw new Error(`Product Hunt launch failed: ${JSON.stringify(data.errors || data)}`)
+    }
+
+    return { postId: post.id, url: post.url, name: post.name }
+  }
+
+  async _comment(params, ctx) {
+    const { postId, body, account } = params
+    const token = this._getToken(account, ctx)
+
+    const query = `
+      mutation CreateComment($input: CommentCreateInput!) {
+        commentCreate(input: $input) {
+          comment {
+            id
+            body
+          }
+        }
+      }
+    `
+
+    const data = await graphql(query, {
+      input: { postId, body }
+    }, token)
+
+    const comment = data.data?.commentCreate?.comment
+    return { commentId: comment?.id }
+  }
+
+  async _upvote(params, ctx) {
+    const { postId, account } = params
+    const token = this._getToken(account, ctx)
+
+    const query = `
+      mutation VotePost($input: PostVoteInput!) {
+        postVote(input: $input) {
+          node {
+            id
+            votesCount
+          }
+        }
+      }
+    `
+
+    const data = await graphql(query, { input: { postId } }, token)
+    return { votes: data.data?.postVote?.node?.votesCount }
+  }
+
+  _getToken(account, ctx) {
+    if (account?.token) return account.token
+    const token = ctx?.config?.accounts?.producthunt?.token ||
+                  process.env.PRODUCTHUNT_TOKEN
+    if (!token) throw new Error('Product Hunt API token required. Get one from producthunt.com/v2/oauth/applications')
+    return token
+  }
+}
+
+function graphql(query, variables, token) {
+  return new Promise((resolve, reject) => {
+    const body = JSON.stringify({ query, variables })
+    const opts = {
+      hostname: 'api.producthunt.com',
+      path: '/v2/api/graphql',
+      method: 'POST',
+      headers: {
+        'Authorization': `Bearer ${token}`,
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+        'Content-Length': Buffer.byteLength(body)
+      }
+    }
+    const req = https.request(opts, res => {
+      let data = ''
+      res.on('data', c => data += c)
+      res.on('end', () => {
+        try { resolve(JSON.parse(data)) }
+        catch (e) { reject(new Error(`Invalid PH response: ${data.slice(0, 200)}`)) }
+      })
+    })
+    req.on('error', reject)
+    req.setTimeout(15000, () => { req.destroy(); reject(new Error('PH API timeout')) })
+    req.write(body)
+    req.end()
+  })
+}
+
+module.exports = { ProductHuntAdapter }

package/src/act/adapters/quora.js

@@ -0,0 +1,89 @@
+/**
+ * Quora platform adapter.
+ * No public API — requires browser automation.
+ * Uses Spectrawl's browse engine for stealth interaction.
+ */
+class QuoraAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'answer':
+        return this._answer(params, ctx)
+      case 'question':
+        return this._askQuestion(params, ctx)
+      default:
+        throw new Error(`Unsupported Quora action: ${action}. Requires browser automation.`)
+    }
+  }
+
+  async _answer(params, ctx) {
+    const { questionUrl, text, _cookies, _browse } = params
+    if (!_browse) throw new Error('Quora requires browse engine. Pass _browse: spectrawl.browse')
+
+    const page = await _browse(questionUrl, {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const playwright = page._page
+    if (!playwright) throw new Error('Quora adapter requires getPage access')
+
+    // Click answer button
+    await playwright.click('[class*="AnswerButton"], button:has-text("Answer")')
+    await playwright.waitForTimeout(1000)
+
+    // Find the contenteditable answer box
+    const editor = await playwright.$('[class*="editor"], [contenteditable="true"], .q-box [role="textbox"]')
+    if (!editor) throw new Error('Could not find Quora answer editor')
+
+    // Type the answer
+    await editor.click()
+    await playwright.keyboard.type(text, { delay: 30 })
+    await playwright.waitForTimeout(500)
+
+    // Submit
+    await playwright.click('button:has-text("Submit"), button:has-text("Post")')
+    await playwright.waitForTimeout(2000)
+
+    return { answered: true, questionUrl }
+  }
+
+  async _askQuestion(params, ctx) {
+    const { question, details, topics, _cookies, _browse } = params
+    if (!_browse) throw new Error('Quora requires browse engine')
+
+    const page = await _browse('https://www.quora.com/', {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const playwright = page._page
+    if (!playwright) throw new Error('Quora adapter requires getPage access')
+
+    // Click "Add question" button
+    await playwright.click('button:has-text("Add question"), [class*="AddQuestion"]')
+    await playwright.waitForTimeout(1000)
+
+    // Type question in the modal
+    const input = await playwright.$('[placeholder*="question"], [class*="QuestionInput"] input, textarea')
+    if (!input) throw new Error('Could not find question input')
+    await input.type(question, { delay: 30 })
+
+    // Add details if provided
+    if (details) {
+      const detailInput = await playwright.$('[placeholder*="details"], [class*="details"] [contenteditable]')
+      if (detailInput) {
+        await detailInput.click()
+        await playwright.keyboard.type(details, { delay: 20 })
+      }
+    }
+
+    // Submit
+    await playwright.click('button:has-text("Add question"), button:has-text("Submit")')
+    await playwright.waitForTimeout(2000)
+
+    return { asked: true, question }
+  }
+}
+
+module.exports = { QuoraAdapter }

package/src/act/adapters/saashub.js

@@ -0,0 +1,47 @@
+/**
+ * SaaSHub platform adapter.
+ * Browser automation — no public API.
+ * Good for alternative comparisons and SEO.
+ */
+class SaaSHubAdapter {
+  async execute(action, params, ctx) {
+    switch (action) {
+      case 'post':
+      case 'submit':
+        return this._submit(params, ctx)
+      default:
+        throw new Error(`Unsupported SaaSHub action: ${action}`)
+    }
+  }
+
+  async _submit(params, ctx) {
+    const { name, url, description, category, alternatives, _cookies, _browse } = params
+    if (!_browse) throw new Error('SaaSHub requires browse engine')
+
+    const page = await _browse('https://www.saashub.com/submit', {
+      cookies: _cookies,
+      getPage: true
+    })
+
+    const pw = page._page
+    if (!pw) throw new Error('SaaSHub requires getPage access')
+
+    await pw.waitForSelector('input[name="name"], #product_name', { timeout: 10000 })
+
+    const nameInput = await pw.$('input[name="name"], #product_name')
+    if (nameInput) await nameInput.fill(name)
+
+    const urlInput = await pw.$('input[name="url"], input[name="website"]')
+    if (urlInput) await urlInput.fill(url)
+
+    const descInput = await pw.$('textarea[name="description"]')
+    if (descInput) await descInput.fill(description || '')
+
+    await pw.click('button[type="submit"], input[type="submit"]')
+    await pw.waitForTimeout(3000)
+
+    return { submitted: true, name, url }
+  }
+}
+
+module.exports = { SaaSHubAdapter }