thumbgate 0.9.9

package/scripts/social-analytics/pollers/reddit.js

@@ -0,0 +1,306 @@
+'use strict';
+
+/**
+ * reddit.js
+ * Polls the Reddit API (OAuth2 password flow) for user post engagement data.
+ *
+ * Required env vars:
+ *   REDDIT_CLIENT_ID      — OAuth2 application client ID (required)
+ *   REDDIT_CLIENT_SECRET  — OAuth2 application client secret (required)
+ *   REDDIT_USERNAME       — Reddit account username (required)
+ *   REDDIT_PASSWORD       — Reddit account password (required)
+ *   REDDIT_USER_AGENT     — Custom User-Agent string (default: thumbgate/1.0 by <username>)
+ *
+ * Reddit API reference:
+ *   POST https://www.reddit.com/api/v1/access_token  — token exchange
+ *   GET  https://oauth.reddit.com/user/{username}/submitted
+ *   GET  https://oauth.reddit.com/api/info
+ *   GET  https://oauth.reddit.com/api/v1/me
+ */
+
+const REDDIT_API_BASE = 'https://oauth.reddit.com';
+const REDDIT_TOKEN_URL = 'https://www.reddit.com/api/v1/access_token';
+
+/**
+ * Exchange Reddit credentials for an OAuth2 access token using the password grant flow.
+ *
+ * @param {string} clientId
+ * @param {string} clientSecret
+ * @param {string} username
+ * @param {string} password
+ * @returns {Promise<string>} access_token
+ */
+async function getRedditToken(clientId, clientSecret, username, password) {
+  if (!clientId) throw new Error('clientId is required');
+  if (!clientSecret) throw new Error('clientSecret is required');
+  if (!username) throw new Error('username is required');
+  if (!password) throw new Error('password is required');
+
+  const credentials = Buffer.from(`${clientId}:${clientSecret}`).toString('base64');
+  const body = new URLSearchParams({
+    grant_type: 'password',
+    username,
+    password,
+  });
+
+  console.log('[reddit] Fetching access token');
+
+  const res = await fetch(REDDIT_TOKEN_URL, {
+    method: 'POST',
+    headers: {
+      Authorization: `Basic ${credentials}`,
+      'Content-Type': 'application/x-www-form-urlencoded',
+      'User-Agent': `thumbgate/1.0 by ${username}`,
+    },
+    body: body.toString(),
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`Reddit token endpoint ${res.status}: ${text}`);
+  }
+
+  const json = await res.json();
+
+  if (json.error) {
+    throw new Error(`Reddit token error: ${json.error} — ${json.message || ''}`);
+  }
+
+  const token = json.access_token;
+  if (!token) {
+    throw new Error(`Reddit token response missing access_token: ${JSON.stringify(json)}`);
+  }
+
+  console.log('[reddit] Access token obtained');
+  return token;
+}
+
+/**
+ * Fetch the user's most recent submitted posts (up to 25).
+ *
+ * @param {string} token - Reddit OAuth2 access token
+ * @param {string} username - Reddit username
+ * @param {string} userAgent - User-Agent header value
+ * @returns {Promise<object[]>} Array of post data objects
+ */
+async function fetchUserPosts(token, username, userAgent) {
+  if (!token) throw new Error('token is required');
+  if (!username) throw new Error('username is required');
+
+  const url = `${REDDIT_API_BASE}/user/${encodeURIComponent(username)}/submitted?sort=new&limit=25`;
+
+  console.log(`[reddit] Fetching submitted posts for u/${username}`);
+
+  const res = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'User-Agent': userAgent,
+    },
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`Reddit API ${res.status} for ${url}: ${text}`);
+  }
+
+  const json = await res.json();
+  const posts = json.data?.children?.map((child) => child.data) ?? [];
+
+  console.log(`[reddit] Retrieved ${posts.length} posts`);
+  return posts;
+}
+
+/**
+ * Fetch detailed metrics for a single post by its base-36 post ID.
+ *
+ * @param {string} token - Reddit OAuth2 access token
+ * @param {string} postId - Reddit post ID (base-36, e.g. "abc123")
+ * @param {string} userAgent - User-Agent header value
+ * @returns {Promise<object|null>} Post data object, or null if not found
+ */
+async function fetchPostDetails(token, postId, userAgent) {
+  if (!token) throw new Error('token is required');
+  if (!postId) throw new Error('postId is required');
+
+  const url = `${REDDIT_API_BASE}/api/info?id=t3_${postId}`;
+
+  const res = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'User-Agent': userAgent,
+    },
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`Reddit API ${res.status} for ${url}: ${text}`);
+  }
+
+  const json = await res.json();
+  const items = json.data?.children ?? [];
+  return items.length > 0 ? items[0].data : null;
+}
+
+/**
+ * Fetch the authenticated user's karma breakdown from /api/v1/me.
+ *
+ * @param {string} token - Reddit OAuth2 access token
+ * @param {string} userAgent - User-Agent header value
+ * @returns {Promise<{ link_karma: number, comment_karma: number, total_karma: number }>}
+ */
+async function fetchUserKarma(token, userAgent) {
+  if (!token) throw new Error('token is required');
+
+  const url = `${REDDIT_API_BASE}/api/v1/me`;
+
+  console.log('[reddit] Fetching user karma from /api/v1/me');
+
+  const res = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'User-Agent': userAgent,
+    },
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`Reddit API ${res.status} for ${url}: ${text}`);
+  }
+
+  const json = await res.json();
+
+  const linkKarma = json.link_karma ?? 0;
+  const commentKarma = json.comment_karma ?? 0;
+  const totalKarma = json.total_karma ?? linkKarma + commentKarma;
+
+  console.log(
+    `[reddit] Karma: link_karma=${linkKarma} comment_karma=${commentKarma} total=${totalKarma}`
+  );
+
+  return { link_karma: linkKarma, comment_karma: commentKarma, total_karma: totalKarma };
+}
+
+/**
+ * Main polling entry point.
+ *
+ * Obtains an OAuth2 token, fetches the user's recent posts and karma, normalizes
+ * each post record, and upserts into the analytics database. Also records a
+ * follower snapshot using total karma as the follower-equivalent metric.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @returns {Promise<void>}
+ */
+async function pollReddit(db) {
+  const clientId = process.env.REDDIT_CLIENT_ID;
+  const clientSecret = process.env.REDDIT_CLIENT_SECRET;
+  const username = process.env.REDDIT_USERNAME;
+  const password = process.env.REDDIT_PASSWORD;
+  const userAgent =
+    process.env.REDDIT_USER_AGENT || `thumbgate/1.0 by ${username}`;
+
+  if (!clientId) throw new Error('REDDIT_CLIENT_ID environment variable is required');
+  if (!clientSecret) throw new Error('REDDIT_CLIENT_SECRET environment variable is required');
+  if (!username) throw new Error('REDDIT_USERNAME environment variable is required');
+  if (!password) throw new Error('REDDIT_PASSWORD environment variable is required');
+
+  const token = await getRedditToken(clientId, clientSecret, username, password);
+
+  const [posts, karma] = await Promise.all([
+    fetchUserPosts(token, username, userAgent),
+    fetchUserKarma(token, userAgent),
+  ]);
+
+  // Lazy-require sibling modules so they can be built/tested independently.
+  const { normalizeRedditMetric: normalizeMetric } = require('../normalizer');
+  const { upsertMetric, upsertFollowerSnapshot } = require('../store');
+
+  const fetchedAt = new Date().toISOString();
+  const today = fetchedAt.slice(0, 10);
+
+  for (const post of posts) {
+    const postId = String(post.id || '');
+    if (!postId) continue;
+
+    // created_utc is a Unix timestamp (seconds).
+    const publishedAt = post.created_utc
+      ? new Date(post.created_utc * 1000).toISOString()
+      : null;
+    const metricDate = publishedAt ? publishedAt.slice(0, 10) : today;
+
+    const extraPayload = {
+      upvote_ratio: post.upvote_ratio ?? null,
+      subreddit: post.subreddit ?? null,
+    };
+    if (post.title) extraPayload.title = post.title;
+    if (post.url) extraPayload.url = post.url;
+
+    const raw = {
+      id: postId,
+      platform: 'reddit',
+      content_type: post.is_self === false ? 'link' : 'post',
+      post_id: postId,
+      post_url: post.permalink ? `https://www.reddit.com${post.permalink}` : null,
+      published_at: publishedAt,
+      metric_date: metricDate,
+      // score = upvotes - downvotes (Reddit's net engagement signal)
+      likes: post.score ?? 0,
+      comments: post.num_comments ?? 0,
+      // Reddit does not expose view counts via the API for regular users
+      impressions: 0,
+      reach: 0,
+      shares: 0,
+      saves: 0,
+      clicks: 0,
+      video_views: 0,
+      followers_delta: 0,
+      extra_json: JSON.stringify(extraPayload),
+      fetched_at: fetchedAt,
+    };
+
+    const normalized = normalizeMetric(raw);
+    upsertMetric(db, normalized);
+
+    console.log(
+      `[reddit] Upserted post id=${postId} subreddit=${post.subreddit} ` +
+        `score=${post.score} comments=${post.num_comments} ` +
+        `upvote_ratio=${post.upvote_ratio} date=${metricDate}`
+    );
+  }
+
+  console.log(`[reddit] Upserted ${posts.length} post metric records`);
+
+  // Record follower snapshot — total karma serves as Reddit's engagement proxy.
+  const totalKarma = karma.total_karma;
+  upsertFollowerSnapshot(db, {
+    platform: 'reddit',
+    follower_count: totalKarma,
+    snapshot_date: today,
+  });
+
+  console.log(
+    `[reddit] Follower snapshot upserted: platform=reddit ` +
+      `follower_count(karma)=${totalKarma} date=${today}`
+  );
+}
+
+module.exports = { getRedditToken, fetchUserPosts, fetchPostDetails, fetchUserKarma, pollReddit };
+
+// ---------------------------------------------------------------------------
+// Stand-alone execution
+// ---------------------------------------------------------------------------
+if (require.main === module) {
+  const { initDb } = require('../store');
+
+  const db = initDb();
+
+  pollReddit(db)
+    .then(() => {
+      console.log('[reddit] Poll complete.');
+      db.close();
+    })
+    .catch((err) => {
+      console.error('[reddit] Poll failed:', err.message);
+      db.close();
+      process.exit(1);
+    });
+}

package/scripts/social-analytics/pollers/threads.js

@@ -0,0 +1,233 @@
+'use strict';
+
+/**
+ * Threads engagement poller using the Threads API (Meta, launched June 2024).
+ *
+ * Threads API base: https://graph.threads.net/v1.0
+ * Authentication: Meta OAuth (Threads Login). Long-lived tokens last 60 days.
+ *
+ * Required environment variables:
+ *   THREADS_ACCESS_TOKEN — long-lived Threads user access token
+ *   THREADS_USER_ID      — Threads user ID (numeric string)
+ *
+ * Handle: @igor.ganapolsky
+ */
+
+const { upsertMetric, upsertFollowerSnapshot, initDb } = require('../store');
+
+const THREADS_BASE = 'https://graph.threads.net/v1.0';
+
+/**
+ * Fetches the most recent 25 threads for a user.
+ *
+ * @param {string} token  - Threads access token.
+ * @param {string} userId - Threads user ID.
+ * @returns {Promise<object[]>} Array of thread objects.
+ */
+async function fetchThreadsPosts(token, userId) {
+  const fields = [
+    'id',
+    'text',
+    'timestamp',
+    'media_type',
+    'permalink',
+    'is_reply',
+    'likes',
+    'replies',
+    'reposts',
+    'quotes',
+  ].join(',');
+
+  const url =
+    `${THREADS_BASE}/${userId}/threads` +
+    `?fields=${fields}&limit=25&access_token=${token}`;
+
+  const res = await fetch(url);
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(`fetchThreadsPosts HTTP ${res.status}: ${body}`);
+  }
+
+  const json = await res.json();
+  if (json.error) {
+    throw new Error(`fetchThreadsPosts API error: ${JSON.stringify(json.error)}`);
+  }
+
+  return json.data ?? [];
+}
+
+/**
+ * Fetches insights for a single thread (post-level metrics).
+ * Errors are caught and logged so polling continues for other threads.
+ *
+ * @param {string} token   - Threads access token.
+ * @param {string} mediaId - Threads post ID.
+ * @returns {Promise<object>} Map of metric name -> value (defaults to 0 on error).
+ */
+async function fetchThreadsInsights(token, mediaId) {
+  const url =
+    `${THREADS_BASE}/${mediaId}/insights` +
+    `?metric=views,likes,replies,reposts,quotes&access_token=${token}`;
+
+  try {
+    const res = await fetch(url);
+    if (!res.ok) {
+      const body = await res.text();
+      console.warn(`fetchThreadsInsights HTTP ${res.status} for ${mediaId}: ${body}`);
+      return {};
+    }
+
+    const json = await res.json();
+    if (json.error) {
+      console.warn(
+        `fetchThreadsInsights API warning for ${mediaId}: ${JSON.stringify(json.error)}`
+      );
+      return {};
+    }
+
+    // Reshape array of { name, values } -> flat { name: value }
+    const result = {};
+    for (const item of json.data ?? []) {
+      const val = Array.isArray(item.values) ? item.values[0]?.value : item.value;
+      result[item.name] = typeof val === 'number' ? val : 0;
+    }
+    return result;
+  } catch (err) {
+    console.warn(`fetchThreadsInsights error for ${mediaId}: ${err.message}`);
+    return {};
+  }
+}
+
+/**
+ * Fetches the Threads user profile including follower count.
+ *
+ * @param {string} token  - Threads access token.
+ * @param {string} userId - Threads user ID.
+ * @returns {Promise<object>} Profile object with id, username, followers_count, etc.
+ */
+async function fetchThreadsProfile(token, userId) {
+  const url =
+    `${THREADS_BASE}/${userId}` +
+    `?fields=id,username,threads_profile_picture_url,threads_biography,followers_count` +
+    `&access_token=${token}`;
+
+  const res = await fetch(url);
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(`fetchThreadsProfile HTTP ${res.status}: ${body}`);
+  }
+
+  const json = await res.json();
+  if (json.error) {
+    throw new Error(`fetchThreadsProfile API error: ${JSON.stringify(json.error)}`);
+  }
+
+  return json;
+}
+
+/**
+ * Main entry point. Polls Threads for recent posts and profile data,
+ * normalises the data, and upserts it into the local SQLite database.
+ *
+ * @param {import('better-sqlite3').Database} db - Initialised db instance.
+ */
+async function pollThreads(db) {
+  const token = process.env.THREADS_ACCESS_TOKEN;
+  const userId = process.env.THREADS_USER_ID;
+
+  if (!token) throw new Error('THREADS_ACCESS_TOKEN is not set');
+  if (!userId) throw new Error('THREADS_USER_ID is not set');
+
+  const fetchedAt = new Date().toISOString();
+  const today = fetchedAt.slice(0, 10);
+
+  console.log(`[threads] Fetching posts for user ${userId}…`);
+  const posts = await fetchThreadsPosts(token, userId);
+  console.log(`[threads] Got ${posts.length} thread posts`);
+
+  for (const post of posts) {
+    const postId = post.id;
+
+    // Fetch per-post insights (best-effort; not all post types support all metrics).
+    const insights = await fetchThreadsInsights(token, postId);
+
+    // Threads API returns likes/replies/reposts/quotes both on the post object
+    // and via insights. Prefer insights values when available; fall back to
+    // the top-level fields returned by the posts endpoint.
+    const likes = insights.likes ?? post.likes ?? 0;
+    const replies = insights.replies ?? post.replies ?? 0;
+    const reposts = insights.reposts ?? post.reposts ?? 0;
+    const quotes = insights.quotes ?? post.quotes ?? 0;
+    const views = insights.views ?? 0;
+
+    const raw = {
+      platform: 'threads',
+      content_type: 'thread',
+      post_id: postId,
+      post_url: post.permalink ?? null,
+      published_at: post.timestamp ?? null,
+      metric_date: today,
+      impressions: views,
+      reach: 0,
+      likes: likes,
+      comments: replies,
+      shares: reposts + quotes,
+      saves: 0,
+      clicks: 0,
+      video_views: 0,
+      followers_delta: 0,
+      fetched_at: fetchedAt,
+      extra_json: JSON.stringify({
+        media_type: post.media_type ?? null,
+        is_reply: post.is_reply ?? false,
+        text_snippet: (post.text ?? '').slice(0, 120),
+        reposts,
+        quotes,
+      }),
+    };
+
+    upsertMetric(db, raw);
+    console.log(`[threads] Upserted metric for post ${postId}`);
+  }
+
+  // Fetch profile for follower snapshot.
+  try {
+    console.log('[threads] Fetching profile for follower snapshot…');
+    const profile = await fetchThreadsProfile(token, userId);
+    const followerCount = profile.followers_count ?? 0;
+
+    upsertFollowerSnapshot(db, {
+      platform: 'threads',
+      follower_count: followerCount,
+      snapshot_date: today,
+    });
+    console.log(`[threads] Upserted follower snapshot: ${followerCount} followers`);
+  } catch (err) {
+    // Profile failure should not abort per-post metric storage.
+    console.warn(`[threads] Profile fetch error (non-fatal): ${err.message}`);
+  }
+
+  console.log('[threads] Poll complete.');
+}
+
+module.exports = {
+  fetchThreadsPosts,
+  fetchThreadsInsights,
+  fetchThreadsProfile,
+  pollThreads,
+};
+
+// Allow running directly: node scripts/social-analytics/pollers/threads.js
+if (require.main === module) {
+  (async () => {
+    const db = initDb();
+    try {
+      await pollThreads(db);
+    } catch (err) {
+      console.error('[threads] Fatal error:', err.message);
+      process.exit(1);
+    } finally {
+      db.close();
+    }
+  })();
+}