thumbgate 1.4.2 → 1.4.4

package/scripts/social-analytics/pollers/plausible.js

@@ -1,245 +0,0 @@
-'use strict';
-
-/**
- * Plausible Analytics poller — reads web traffic metrics via the Plausible API.
- *
- * Required environment variables:
- *   PLAUSIBLE_API_KEY  — Plausible API token
- *   PLAUSIBLE_SITE_ID  — Site domain registered in Plausible (e.g. example.com)
- */
-
-const { upsertMetric, initDb } = require('../store');
-
-const PLAUSIBLE_BASE = 'https://plausible.io/api/v1';
-
-/**
- * Builds a Plausible API request URL and executes it.
- * Adds site_id and all extra params to the query string.
- * Throws on non-2xx responses or API-level error bodies.
- *
- * @param {string} endpoint - Path under /api/v1 (e.g. '/stats/aggregate')
- * @param {Record<string, string|number>} params - Additional query parameters.
- * @returns {Promise<object>} Parsed JSON response body.
- */
-async function plausibleQuery(endpoint, params = {}) {
-  const apiKey = process.env.PLAUSIBLE_API_KEY;
-  const siteId = process.env.PLAUSIBLE_SITE_ID || 'thumbgate-production.up.railway.app';
-
-  if (!apiKey) throw new Error('PLAUSIBLE_API_KEY is not set');
-
-  const qs = new URLSearchParams({ site_id: siteId, ...params });
-  const url = `${PLAUSIBLE_BASE}${endpoint}?${qs.toString()}`;
-
-  const res = await fetch(url, {
-    headers: {
-      Authorization: `Bearer ${apiKey}`,
-    },
-  });
-
-  if (!res.ok) {
-    const body = await res.text().catch(() => '');
-    throw new Error(`Plausible API ${res.status} for ${endpoint}: ${body}`);
-  }
-
-  const json = await res.json();
-  if (json.error) {
-    throw new Error(`Plausible API error for ${endpoint}: ${json.error}`);
-  }
-
-  return json;
-}
-
-/**
- * Fetches aggregate site metrics: visitors, pageviews, bounce_rate, visit_duration.
- *
- * @param {string} [period='7d'] - Plausible period string (e.g. '7d', '30d', 'month').
- * @returns {Promise<object>} Aggregate results object from Plausible.
- */
-async function getVisitors(period = '7d') {
-  return plausibleQuery('/stats/aggregate', {
-    metrics: 'visitors,pageviews,bounce_rate,visit_duration',
-    period,
-  });
-}
-
-/**
- * Fetches traffic source breakdown.
- *
- * @param {string} [period='7d'] - Plausible period string.
- * @returns {Promise<object>} Breakdown results by visit:source.
- */
-async function getSourceAttribution(period = '7d') {
-  return plausibleQuery('/stats/breakdown', {
-    property: 'visit:source',
-    metrics: 'visitors,events',
-    period,
-    limit: 20,
-  });
-}
-
-/**
- * Fetches breakdown by a specific UTM parameter.
- *
- * @param {string} utmParam - UTM dimension (e.g. 'utm_source', 'utm_medium', 'utm_campaign').
- * @param {string} [period='7d'] - Plausible period string.
- * @returns {Promise<object>} Breakdown results for the given UTM property.
- */
-async function getUTMBreakdown(utmParam, period = '7d') {
-  return plausibleQuery('/stats/breakdown', {
-    property: `visit:${utmParam}`,
-    metrics: 'visitors,events',
-    period,
-  });
-}
-
-/**
- * Fetches aggregate event count for a named custom event.
- *
- * @param {string} eventName - Plausible custom event name (e.g. 'CTA Click').
- * @param {string} [period='7d'] - Plausible period string.
- * @returns {Promise<object>} Aggregate results with events count.
- */
-async function getCustomEventCount(eventName, period = '7d') {
-  return plausibleQuery('/stats/aggregate', {
-    metrics: 'events',
-    period,
-    filters: `event:name==${eventName}`,
-  });
-}
-
-/**
- * Builds a funnel object from visitors, CTA Click, Checkout Start, and Purchase events.
- * Computes conversion rates between each stage.
- *
- * @param {string} [period='7d'] - Plausible period string.
- * @returns {Promise<object>} Funnel metrics with conversion rates.
- */
-async function getFunnelMetrics(period = '7d') {
-  const [visitorsRes, ctaRes, checkoutRes, purchaseRes] = await Promise.all([
-    getVisitors(period),
-    getCustomEventCount('CTA Click', period),
-    getCustomEventCount('Checkout Start', period),
-    getCustomEventCount('Purchase', period),
-  ]);
-
-  const visitors = visitorsRes.results?.visitors?.value ?? 0;
-  const ctaClicks = ctaRes.results?.events?.value ?? 0;
-  const checkoutStarts = checkoutRes.results?.events?.value ?? 0;
-  const purchases = purchaseRes.results?.events?.value ?? 0;
-
-  const rate = (numerator, denominator) =>
-    denominator > 0 ? Math.round((numerator / denominator) * 10000) / 100 : 0;
-
-  return {
-    period,
-    visitors,
-    cta_clicks: ctaClicks,
-    checkout_starts: checkoutStarts,
-    purchases,
-    visitor_to_cta_pct: rate(ctaClicks, visitors),
-    cta_to_checkout_pct: rate(checkoutStarts, ctaClicks),
-    checkout_to_purchase_pct: rate(purchases, checkoutStarts),
-    visitor_to_purchase_pct: rate(purchases, visitors),
-  };
-}
-
-/**
- * Main polling entry point. Fetches visitors, source attribution, and funnel
- * metrics for the last 7 days and upserts results into the engagement_metrics
- * table as platform='web', content_type='page'.
- *
- * @param {import('better-sqlite3').Database} db - Initialised db instance.
- * @returns {Promise<object>} Summary of stored results.
- */
-async function pollPlausible(db) {
-  const siteId = process.env.PLAUSIBLE_SITE_ID || 'thumbgate-production.up.railway.app';
-  if (!process.env.PLAUSIBLE_API_KEY) throw new Error('PLAUSIBLE_API_KEY is not set');
-
-  const period = '7d';
-  const fetchedAt = new Date().toISOString();
-  const metricDate = fetchedAt.slice(0, 10);
-
-  console.log(`[plausible] Fetching web metrics for site ${siteId} (${period})…`);
-
-  const [visitorsRes, sourceRes, funnelRes] = await Promise.all([
-    getVisitors(period),
-    getSourceAttribution(period),
-    getFunnelMetrics(period),
-  ]);
-
-  const aggregateResults = visitorsRes.results ?? {};
-  const visitors = aggregateResults.visitors?.value ?? 0;
-  const pageviews = aggregateResults.pageviews?.value ?? 0;
-  const bounceRate = aggregateResults.bounce_rate?.value ?? 0;
-  const visitDuration = aggregateResults.visit_duration?.value ?? 0;
-
-  console.log(
-    `[plausible] visitors=${visitors} pageviews=${pageviews} ` +
-      `bounce_rate=${bounceRate}% avg_duration=${visitDuration}s`
-  );
-
-  // Store aggregate web traffic as a single canonical record for the period.
-  const record = {
-    platform: 'web',
-    content_type: 'page',
-    post_id: `${siteId}:${period}:${metricDate}`,
-    post_url: `https://${siteId}`,
-    published_at: null,
-    metric_date: metricDate,
-    impressions: pageviews,
-    reach: visitors,
-    likes: 0,
-    comments: 0,
-    shares: 0,
-    saves: 0,
-    clicks: funnelRes.cta_clicks,
-    video_views: 0,
-    followers_delta: 0,
-    extra_json: JSON.stringify({
-      period,
-      bounce_rate: bounceRate,
-      visit_duration_seconds: visitDuration,
-      sources: sourceRes.results ?? [],
-      funnel: funnelRes,
-    }),
-    fetched_at: fetchedAt,
-  };
-
-  upsertMetric(db, record);
-  console.log(`[plausible] Upserted web metric record for ${metricDate}`);
-  console.log(
-    `[plausible] Funnel: ${funnelRes.visitors} visitors → ` +
-      `${funnelRes.cta_clicks} CTA clicks → ` +
-      `${funnelRes.checkout_starts} checkouts → ` +
-      `${funnelRes.purchases} purchases (${funnelRes.visitor_to_purchase_pct}%)`
-  );
-
-  console.log('[plausible] Poll complete.');
-  return { record, funnel: funnelRes };
-}
-
-module.exports = {
-  plausibleQuery,
-  getVisitors,
-  getSourceAttribution,
-  getUTMBreakdown,
-  getCustomEventCount,
-  getFunnelMetrics,
-  pollPlausible,
-};
-
-// Allow running directly: node scripts/social-analytics/pollers/plausible.js
-if (require.main === module) {
-  (async () => {
-    const db = initDb();
-    try {
-      const results = await pollPlausible(db);
-      console.log('[plausible] Results:', JSON.stringify(results, null, 2));
-    } catch (err) {
-      console.error('[plausible] Fatal error:', err.message);
-      process.exit(1);
-    } finally {
-      db.close();
-    }
-  })();
-}

package/scripts/social-analytics/pollers/reddit.js

@@ -1,306 +0,0 @@
-'use strict';
-
-/**
- * reddit.js
- * Polls the Reddit API (OAuth2 password flow) for user post engagement data.
- *
- * Required env vars:
- *   REDDIT_CLIENT_ID      — OAuth2 application client ID (required)
- *   REDDIT_CLIENT_SECRET  — OAuth2 application client secret (required)
- *   REDDIT_USERNAME       — Reddit account username (required)
- *   REDDIT_PASSWORD       — Reddit account password (required)
- *   REDDIT_USER_AGENT     — Custom User-Agent string (default: thumbgate/1.0 by <username>)
- *
- * Reddit API reference:
- *   POST https://www.reddit.com/api/v1/access_token  — token exchange
- *   GET  https://oauth.reddit.com/user/{username}/submitted
- *   GET  https://oauth.reddit.com/api/info
- *   GET  https://oauth.reddit.com/api/v1/me
- */
-
-const REDDIT_API_BASE = 'https://oauth.reddit.com';
-const REDDIT_TOKEN_URL = 'https://www.reddit.com/api/v1/access_token';
-
-/**
- * Exchange Reddit credentials for an OAuth2 access token using the password grant flow.
- *
- * @param {string} clientId
- * @param {string} clientSecret
- * @param {string} username
- * @param {string} password
- * @returns {Promise<string>} access_token
- */
-async function getRedditToken(clientId, clientSecret, username, password) {
-  if (!clientId) throw new Error('clientId is required');
-  if (!clientSecret) throw new Error('clientSecret is required');
-  if (!username) throw new Error('username is required');
-  if (!password) throw new Error('password is required');
-
-  const credentials = Buffer.from(`${clientId}:${clientSecret}`).toString('base64');
-  const body = new URLSearchParams({
-    grant_type: 'password',
-    username,
-    password,
-  });
-
-  console.log('[reddit] Fetching access token');
-
-  const res = await fetch(REDDIT_TOKEN_URL, {
-    method: 'POST',
-    headers: {
-      Authorization: `Basic ${credentials}`,
-      'Content-Type': 'application/x-www-form-urlencoded',
-      'User-Agent': `thumbgate/1.0 by ${username}`,
-    },
-    body: body.toString(),
-  });
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`Reddit token endpoint ${res.status}: ${text}`);
-  }
-
-  const json = await res.json();
-
-  if (json.error) {
-    throw new Error(`Reddit token error: ${json.error} — ${json.message || ''}`);
-  }
-
-  const token = json.access_token;
-  if (!token) {
-    throw new Error(`Reddit token response missing access_token: ${JSON.stringify(json)}`);
-  }
-
-  console.log('[reddit] Access token obtained');
-  return token;
-}
-
-/**
- * Fetch the user's most recent submitted posts (up to 25).
- *
- * @param {string} token - Reddit OAuth2 access token
- * @param {string} username - Reddit username
- * @param {string} userAgent - User-Agent header value
- * @returns {Promise<object[]>} Array of post data objects
- */
-async function fetchUserPosts(token, username, userAgent) {
-  if (!token) throw new Error('token is required');
-  if (!username) throw new Error('username is required');
-
-  const url = `${REDDIT_API_BASE}/user/${encodeURIComponent(username)}/submitted?sort=new&limit=25`;
-
-  console.log(`[reddit] Fetching submitted posts for u/${username}`);
-
-  const res = await fetch(url, {
-    headers: {
-      Authorization: `Bearer ${token}`,
-      'User-Agent': userAgent,
-    },
-  });
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`Reddit API ${res.status} for ${url}: ${text}`);
-  }
-
-  const json = await res.json();
-  const posts = json.data?.children?.map((child) => child.data) ?? [];
-
-  console.log(`[reddit] Retrieved ${posts.length} posts`);
-  return posts;
-}
-
-/**
- * Fetch detailed metrics for a single post by its base-36 post ID.
- *
- * @param {string} token - Reddit OAuth2 access token
- * @param {string} postId - Reddit post ID (base-36, e.g. "abc123")
- * @param {string} userAgent - User-Agent header value
- * @returns {Promise<object|null>} Post data object, or null if not found
- */
-async function fetchPostDetails(token, postId, userAgent) {
-  if (!token) throw new Error('token is required');
-  if (!postId) throw new Error('postId is required');
-
-  const url = `${REDDIT_API_BASE}/api/info?id=t3_${postId}`;
-
-  const res = await fetch(url, {
-    headers: {
-      Authorization: `Bearer ${token}`,
-      'User-Agent': userAgent,
-    },
-  });
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`Reddit API ${res.status} for ${url}: ${text}`);
-  }
-
-  const json = await res.json();
-  const items = json.data?.children ?? [];
-  return items.length > 0 ? items[0].data : null;
-}
-
-/**
- * Fetch the authenticated user's karma breakdown from /api/v1/me.
- *
- * @param {string} token - Reddit OAuth2 access token
- * @param {string} userAgent - User-Agent header value
- * @returns {Promise<{ link_karma: number, comment_karma: number, total_karma: number }>}
- */
-async function fetchUserKarma(token, userAgent) {
-  if (!token) throw new Error('token is required');
-
-  const url = `${REDDIT_API_BASE}/api/v1/me`;
-
-  console.log('[reddit] Fetching user karma from /api/v1/me');
-
-  const res = await fetch(url, {
-    headers: {
-      Authorization: `Bearer ${token}`,
-      'User-Agent': userAgent,
-    },
-  });
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`Reddit API ${res.status} for ${url}: ${text}`);
-  }
-
-  const json = await res.json();
-
-  const linkKarma = json.link_karma ?? 0;
-  const commentKarma = json.comment_karma ?? 0;
-  const totalKarma = json.total_karma ?? linkKarma + commentKarma;
-
-  console.log(
-    `[reddit] Karma: link_karma=${linkKarma} comment_karma=${commentKarma} total=${totalKarma}`
-  );
-
-  return { link_karma: linkKarma, comment_karma: commentKarma, total_karma: totalKarma };
-}
-
-/**
- * Main polling entry point.
- *
- * Obtains an OAuth2 token, fetches the user's recent posts and karma, normalizes
- * each post record, and upserts into the analytics database. Also records a
- * follower snapshot using total karma as the follower-equivalent metric.
- *
- * @param {import('better-sqlite3').Database} db
- * @returns {Promise<void>}
- */
-async function pollReddit(db) {
-  const clientId = process.env.REDDIT_CLIENT_ID;
-  const clientSecret = process.env.REDDIT_CLIENT_SECRET;
-  const username = process.env.REDDIT_USERNAME;
-  const password = process.env.REDDIT_PASSWORD;
-  const userAgent =
-    process.env.REDDIT_USER_AGENT || `thumbgate/1.0 by ${username}`;
-
-  if (!clientId) throw new Error('REDDIT_CLIENT_ID environment variable is required');
-  if (!clientSecret) throw new Error('REDDIT_CLIENT_SECRET environment variable is required');
-  if (!username) throw new Error('REDDIT_USERNAME environment variable is required');
-  if (!password) throw new Error('REDDIT_PASSWORD environment variable is required');
-
-  const token = await getRedditToken(clientId, clientSecret, username, password);
-
-  const [posts, karma] = await Promise.all([
-    fetchUserPosts(token, username, userAgent),
-    fetchUserKarma(token, userAgent),
-  ]);
-
-  // Lazy-require sibling modules so they can be built/tested independently.
-  const { normalizeRedditMetric: normalizeMetric } = require('../normalizer');
-  const { upsertMetric, upsertFollowerSnapshot } = require('../store');
-
-  const fetchedAt = new Date().toISOString();
-  const today = fetchedAt.slice(0, 10);
-
-  for (const post of posts) {
-    const postId = String(post.id || '');
-    if (!postId) continue;
-
-    // created_utc is a Unix timestamp (seconds).
-    const publishedAt = post.created_utc
-      ? new Date(post.created_utc * 1000).toISOString()
-      : null;
-    const metricDate = publishedAt ? publishedAt.slice(0, 10) : today;
-
-    const extraPayload = {
-      upvote_ratio: post.upvote_ratio ?? null,
-      subreddit: post.subreddit ?? null,
-    };
-    if (post.title) extraPayload.title = post.title;
-    if (post.url) extraPayload.url = post.url;
-
-    const raw = {
-      id: postId,
-      platform: 'reddit',
-      content_type: post.is_self === false ? 'link' : 'post',
-      post_id: postId,
-      post_url: post.permalink ? `https://www.reddit.com${post.permalink}` : null,
-      published_at: publishedAt,
-      metric_date: metricDate,
-      // score = upvotes - downvotes (Reddit's net engagement signal)
-      likes: post.score ?? 0,
-      comments: post.num_comments ?? 0,
-      // Reddit does not expose view counts via the API for regular users
-      impressions: 0,
-      reach: 0,
-      shares: 0,
-      saves: 0,
-      clicks: 0,
-      video_views: 0,
-      followers_delta: 0,
-      extra_json: JSON.stringify(extraPayload),
-      fetched_at: fetchedAt,
-    };
-
-    const normalized = normalizeMetric(raw);
-    upsertMetric(db, normalized);
-
-    console.log(
-      `[reddit] Upserted post id=${postId} subreddit=${post.subreddit} ` +
-        `score=${post.score} comments=${post.num_comments} ` +
-        `upvote_ratio=${post.upvote_ratio} date=${metricDate}`
-    );
-  }
-
-  console.log(`[reddit] Upserted ${posts.length} post metric records`);
-
-  // Record follower snapshot — total karma serves as Reddit's engagement proxy.
-  const totalKarma = karma.total_karma;
-  upsertFollowerSnapshot(db, {
-    platform: 'reddit',
-    follower_count: totalKarma,
-    snapshot_date: today,
-  });
-
-  console.log(
-    `[reddit] Follower snapshot upserted: platform=reddit ` +
-      `follower_count(karma)=${totalKarma} date=${today}`
-  );
-}
-
-module.exports = { getRedditToken, fetchUserPosts, fetchPostDetails, fetchUserKarma, pollReddit };
-
-// ---------------------------------------------------------------------------
-// Stand-alone execution
-// ---------------------------------------------------------------------------
-if (require.main === module) {
-  const { initDb } = require('../store');
-
-  const db = initDb();
-
-  pollReddit(db)
-    .then(() => {
-      console.log('[reddit] Poll complete.');
-      db.close();
-    })
-    .catch((err) => {
-      console.error('[reddit] Poll failed:', err.message);
-      db.close();
-      process.exit(1);
-    });
-}