thumbgate 0.9.10

package/scripts/social-analytics/pollers/threads.js

@@ -0,0 +1,233 @@
+'use strict';
+
+/**
+ * Threads engagement poller using the Threads API (Meta, launched June 2024).
+ *
+ * Threads API base: https://graph.threads.net/v1.0
+ * Authentication: Meta OAuth (Threads Login). Long-lived tokens last 60 days.
+ *
+ * Required environment variables:
+ *   THREADS_ACCESS_TOKEN — long-lived Threads user access token
+ *   THREADS_USER_ID      — Threads user ID (numeric string)
+ *
+ * Handle: @igor.ganapolsky
+ */
+
+const { upsertMetric, upsertFollowerSnapshot, initDb } = require('../store');
+
+const THREADS_BASE = 'https://graph.threads.net/v1.0';
+
+/**
+ * Fetches the most recent 25 threads for a user.
+ *
+ * @param {string} token  - Threads access token.
+ * @param {string} userId - Threads user ID.
+ * @returns {Promise<object[]>} Array of thread objects.
+ */
+async function fetchThreadsPosts(token, userId) {
+  const fields = [
+    'id',
+    'text',
+    'timestamp',
+    'media_type',
+    'permalink',
+    'is_reply',
+    'likes',
+    'replies',
+    'reposts',
+    'quotes',
+  ].join(',');
+
+  const url =
+    `${THREADS_BASE}/${userId}/threads` +
+    `?fields=${fields}&limit=25&access_token=${token}`;
+
+  const res = await fetch(url);
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(`fetchThreadsPosts HTTP ${res.status}: ${body}`);
+  }
+
+  const json = await res.json();
+  if (json.error) {
+    throw new Error(`fetchThreadsPosts API error: ${JSON.stringify(json.error)}`);
+  }
+
+  return json.data ?? [];
+}
+
+/**
+ * Fetches insights for a single thread (post-level metrics).
+ * Errors are caught and logged so polling continues for other threads.
+ *
+ * @param {string} token   - Threads access token.
+ * @param {string} mediaId - Threads post ID.
+ * @returns {Promise<object>} Map of metric name -> value (defaults to 0 on error).
+ */
+async function fetchThreadsInsights(token, mediaId) {
+  const url =
+    `${THREADS_BASE}/${mediaId}/insights` +
+    `?metric=views,likes,replies,reposts,quotes&access_token=${token}`;
+
+  try {
+    const res = await fetch(url);
+    if (!res.ok) {
+      const body = await res.text();
+      console.warn(`fetchThreadsInsights HTTP ${res.status} for ${mediaId}: ${body}`);
+      return {};
+    }
+
+    const json = await res.json();
+    if (json.error) {
+      console.warn(
+        `fetchThreadsInsights API warning for ${mediaId}: ${JSON.stringify(json.error)}`
+      );
+      return {};
+    }
+
+    // Reshape array of { name, values } -> flat { name: value }
+    const result = {};
+    for (const item of json.data ?? []) {
+      const val = Array.isArray(item.values) ? item.values[0]?.value : item.value;
+      result[item.name] = typeof val === 'number' ? val : 0;
+    }
+    return result;
+  } catch (err) {
+    console.warn(`fetchThreadsInsights error for ${mediaId}: ${err.message}`);
+    return {};
+  }
+}
+
+/**
+ * Fetches the Threads user profile including follower count.
+ *
+ * @param {string} token  - Threads access token.
+ * @param {string} userId - Threads user ID.
+ * @returns {Promise<object>} Profile object with id, username, followers_count, etc.
+ */
+async function fetchThreadsProfile(token, userId) {
+  const url =
+    `${THREADS_BASE}/${userId}` +
+    `?fields=id,username,threads_profile_picture_url,threads_biography,followers_count` +
+    `&access_token=${token}`;
+
+  const res = await fetch(url);
+  if (!res.ok) {
+    const body = await res.text();
+    throw new Error(`fetchThreadsProfile HTTP ${res.status}: ${body}`);
+  }
+
+  const json = await res.json();
+  if (json.error) {
+    throw new Error(`fetchThreadsProfile API error: ${JSON.stringify(json.error)}`);
+  }
+
+  return json;
+}
+
+/**
+ * Main entry point. Polls Threads for recent posts and profile data,
+ * normalises the data, and upserts it into the local SQLite database.
+ *
+ * @param {import('better-sqlite3').Database} db - Initialised db instance.
+ */
+async function pollThreads(db) {
+  const token = process.env.THREADS_ACCESS_TOKEN;
+  const userId = process.env.THREADS_USER_ID;
+
+  if (!token) throw new Error('THREADS_ACCESS_TOKEN is not set');
+  if (!userId) throw new Error('THREADS_USER_ID is not set');
+
+  const fetchedAt = new Date().toISOString();
+  const today = fetchedAt.slice(0, 10);
+
+  console.log(`[threads] Fetching posts for user ${userId}…`);
+  const posts = await fetchThreadsPosts(token, userId);
+  console.log(`[threads] Got ${posts.length} thread posts`);
+
+  for (const post of posts) {
+    const postId = post.id;
+
+    // Fetch per-post insights (best-effort; not all post types support all metrics).
+    const insights = await fetchThreadsInsights(token, postId);
+
+    // Threads API returns likes/replies/reposts/quotes both on the post object
+    // and via insights. Prefer insights values when available; fall back to
+    // the top-level fields returned by the posts endpoint.
+    const likes = insights.likes ?? post.likes ?? 0;
+    const replies = insights.replies ?? post.replies ?? 0;
+    const reposts = insights.reposts ?? post.reposts ?? 0;
+    const quotes = insights.quotes ?? post.quotes ?? 0;
+    const views = insights.views ?? 0;
+
+    const raw = {
+      platform: 'threads',
+      content_type: 'thread',
+      post_id: postId,
+      post_url: post.permalink ?? null,
+      published_at: post.timestamp ?? null,
+      metric_date: today,
+      impressions: views,
+      reach: 0,
+      likes: likes,
+      comments: replies,
+      shares: reposts + quotes,
+      saves: 0,
+      clicks: 0,
+      video_views: 0,
+      followers_delta: 0,
+      fetched_at: fetchedAt,
+      extra_json: JSON.stringify({
+        media_type: post.media_type ?? null,
+        is_reply: post.is_reply ?? false,
+        text_snippet: (post.text ?? '').slice(0, 120),
+        reposts,
+        quotes,
+      }),
+    };
+
+    upsertMetric(db, raw);
+    console.log(`[threads] Upserted metric for post ${postId}`);
+  }
+
+  // Fetch profile for follower snapshot.
+  try {
+    console.log('[threads] Fetching profile for follower snapshot…');
+    const profile = await fetchThreadsProfile(token, userId);
+    const followerCount = profile.followers_count ?? 0;
+
+    upsertFollowerSnapshot(db, {
+      platform: 'threads',
+      follower_count: followerCount,
+      snapshot_date: today,
+    });
+    console.log(`[threads] Upserted follower snapshot: ${followerCount} followers`);
+  } catch (err) {
+    // Profile failure should not abort per-post metric storage.
+    console.warn(`[threads] Profile fetch error (non-fatal): ${err.message}`);
+  }
+
+  console.log('[threads] Poll complete.');
+}
+
+module.exports = {
+  fetchThreadsPosts,
+  fetchThreadsInsights,
+  fetchThreadsProfile,
+  pollThreads,
+};
+
+// Allow running directly: node scripts/social-analytics/pollers/threads.js
+if (require.main === module) {
+  (async () => {
+    const db = initDb();
+    try {
+      await pollThreads(db);
+    } catch (err) {
+      console.error('[threads] Fatal error:', err.message);
+      process.exit(1);
+    } finally {
+      db.close();
+    }
+  })();
+}

package/scripts/social-analytics/pollers/tiktok.js

@@ -0,0 +1,203 @@
+'use strict';
+
+/**
+ * tiktok.js
+ * Polls TikTok Content Posting API v2 for video engagement data.
+ *
+ * Required env vars:
+ *   TIKTOK_ACCESS_TOKEN  — OAuth 2.0 access token with required scopes (required)
+ *
+ * Scopes needed: video.list, user.info.basic, user.info.stats
+ */
+
+const path = require('path');
+
+const TIKTOK_API_BASE = 'https://open.tiktokapis.com';
+
+/**
+ * Fetch the authenticated user's recent videos.
+ *
+ * @param {string} token - TikTok OAuth access token
+ * @returns {Promise<object[]>} Array of video objects
+ */
+async function fetchTikTokVideos(token) {
+  if (!token) {
+    throw new Error('TIKTOK_ACCESS_TOKEN is required');
+  }
+
+  const url = `${TIKTOK_API_BASE}/v2/video/list/`;
+  const body = {
+    max_count: 20,
+    fields: ['id', 'title', 'view_count', 'like_count', 'comment_count', 'share_count', 'create_time', 'duration'],
+  };
+
+  console.log('[tiktok] Fetching video list');
+
+  const res = await fetch(url, {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(body),
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`TikTok API ${res.status} for ${url}: ${text}`);
+  }
+
+  const json = await res.json();
+
+  if (json.error && json.error.code !== 'ok') {
+    throw new Error(`TikTok API error: ${json.error.code} — ${json.error.message}`);
+  }
+
+  const videos = json.data?.videos ?? [];
+  console.log(`[tiktok] Retrieved ${videos.length} videos`);
+  return videos;
+}
+
+/**
+ * Fetch the authenticated user's profile stats.
+ *
+ * @param {string} token - TikTok OAuth access token
+ * @returns {Promise<object>} User info object with follower_count, likes_count, video_count
+ */
+async function fetchTikTokUserInfo(token) {
+  if (!token) {
+    throw new Error('TIKTOK_ACCESS_TOKEN is required');
+  }
+
+  const url = `${TIKTOK_API_BASE}/v2/user/info/?fields=follower_count,likes_count,video_count`;
+
+  console.log('[tiktok] Fetching user info');
+
+  const res = await fetch(url, {
+    method: 'GET',
+    headers: {
+      Authorization: `Bearer ${token}`,
+    },
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`TikTok API ${res.status} for ${url}: ${text}`);
+  }
+
+  const json = await res.json();
+
+  if (json.error && json.error.code !== 'ok') {
+    throw new Error(`TikTok API error: ${json.error.code} — ${json.error.message}`);
+  }
+
+  const userInfo = json.data?.user ?? {};
+  console.log(
+    `[tiktok] User info: follower_count=${userInfo.follower_count} ` +
+      `likes_count=${userInfo.likes_count} video_count=${userInfo.video_count}`
+  );
+  return userInfo;
+}
+
+/**
+ * Main polling entry point.
+ *
+ * Fetches TikTok videos and user info, normalizes each video record,
+ * and upserts into the analytics database. Also records a follower snapshot.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @returns {Promise<void>}
+ */
+async function pollTikTok(db) {
+  const token = process.env.TIKTOK_ACCESS_TOKEN;
+  if (!token) {
+    throw new Error('TIKTOK_ACCESS_TOKEN environment variable is required');
+  }
+
+  const [videos, userInfo] = await Promise.all([
+    fetchTikTokVideos(token),
+    fetchTikTokUserInfo(token),
+  ]);
+
+  // Lazy-require sibling modules so they can be built/tested independently.
+  const { normalizeTikTokMetric: normalizeMetric } = require('../normalizer');
+  const { upsertMetric, upsertFollowerSnapshot } = require('../store');
+
+  const fetchedAt = new Date().toISOString();
+  const today = fetchedAt.slice(0, 10);
+
+  for (const video of videos) {
+    // create_time is a Unix timestamp (seconds); convert to ISO date string.
+    const publishedDate = video.create_time
+      ? new Date(video.create_time * 1000).toISOString()
+      : null;
+    const metricDate = publishedDate ? publishedDate.slice(0, 10) : today;
+
+    const raw = {
+      platform: 'tiktok',
+      content_type: 'video',
+      post_id: video.id,
+      post_url: null,
+      published_at: publishedDate,
+      metric_date: metricDate,
+      video_views: video.view_count ?? 0,
+      likes: video.like_count ?? 0,
+      comments: video.comment_count ?? 0,
+      shares: video.share_count ?? 0,
+      impressions: 0,
+      reach: 0,
+      saves: 0,
+      clicks: 0,
+      followers_delta: 0,
+      extra_json: JSON.stringify({ title: video.title ?? '', duration: video.duration ?? 0 }),
+      fetched_at: fetchedAt,
+    };
+
+    const normalized = normalizeMetric(raw);
+    upsertMetric(db, normalized);
+
+    console.log(
+      `[tiktok] Upserted video id=${video.id} ` +
+        `views=${video.view_count} likes=${video.like_count} ` +
+        `comments=${video.comment_count} shares=${video.share_count} ` +
+        `date=${metricDate}`
+    );
+  }
+
+  console.log(`[tiktok] Upserted ${videos.length} video metric records`);
+
+  // Record follower snapshot.
+  const followerCount = userInfo.follower_count ?? 0;
+  upsertFollowerSnapshot(db, {
+    platform: 'tiktok',
+    follower_count: followerCount,
+    snapshot_date: today,
+  });
+
+  console.log(
+    `[tiktok] Follower snapshot upserted: platform=tiktok ` +
+      `follower_count=${followerCount} date=${today}`
+  );
+}
+
+module.exports = { fetchTikTokVideos, fetchTikTokUserInfo, pollTikTok };
+
+// ---------------------------------------------------------------------------
+// Stand-alone execution
+// ---------------------------------------------------------------------------
+if (require.main === module) {
+  const { initDb } = require('../store');
+
+  const db = initDb();
+
+  pollTikTok(db)
+    .then(() => {
+      console.log('[tiktok] Poll complete.');
+      db.close();
+    })
+    .catch((err) => {
+      console.error('[tiktok] Poll failed:', err.message);
+      db.close();
+      process.exit(1);
+    });
+}

package/scripts/social-analytics/pollers/x.js

@@ -0,0 +1,227 @@
+'use strict';
+
+/**
+ * x.js
+ * Polls X (Twitter) API v2 for tweet engagement data and profile metrics.
+ *
+ * Required env vars:
+ *   X_BEARER_TOKEN  — App-only Bearer Token (required)
+ *   X_USER_ID       — Numeric X user ID to poll (required)
+ *
+ * Rate limits (Basic tier):
+ *   User tweet timeline: 10 requests / 15 minutes per app.
+ *   User lookup: 300 requests / 15 minutes per app.
+ *   Exceeding these limits returns HTTP 429; callers should implement
+ *   exponential backoff or schedule polls no more frequently than every
+ *   2 minutes to stay within the Basic tier ceiling.
+ *
+ * X API v2 reference:
+ *   GET /2/users/:id/tweets
+ *   GET /2/users/:id
+ */
+
+const X_API_BASE = 'https://api.twitter.com/2';
+
+/**
+ * Fetches the most recent 20 tweets for a user.
+ *
+ * @param {string} token  - X Bearer Token
+ * @param {string} userId - Numeric X user ID
+ * @returns {Promise<{ data: object[], includes?: object }>}
+ */
+async function fetchUserTweets(token, userId) {
+  if (!token) throw new Error('X_BEARER_TOKEN is required');
+  if (!userId) throw new Error('X_USER_ID is required');
+
+  const params = new URLSearchParams({
+    max_results: '20',
+    'tweet.fields': 'public_metrics,created_at,text',
+    expansions: 'attachments.media_keys',
+    'media.fields': 'preview_image_url,type,url',
+  });
+
+  const url = `${X_API_BASE}/users/${userId}/tweets?${params}`;
+
+  console.log(`[x:poller] Fetching tweets for user_id=${userId}`);
+
+  const res = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+    },
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`X API ${res.status} for ${url}: ${text}`);
+  }
+
+  const json = await res.json();
+
+  if (json.errors && !json.data) {
+    throw new Error(`X API errors: ${JSON.stringify(json.errors)}`);
+  }
+
+  const tweets = json.data ?? [];
+  console.log(`[x:poller] Retrieved ${tweets.length} tweets`);
+  return { data: tweets, includes: json.includes ?? {} };
+}
+
+/**
+ * Fetches the public profile metrics for a user.
+ *
+ * Returns an object with:
+ *   followers_count, following_count, tweet_count, listed_count
+ *
+ * @param {string} token  - X Bearer Token
+ * @param {string} userId - Numeric X user ID
+ * @returns {Promise<object>} User object including public_metrics field
+ */
+async function fetchUserProfile(token, userId) {
+  if (!token) throw new Error('X_BEARER_TOKEN is required');
+  if (!userId) throw new Error('X_USER_ID is required');
+
+  const params = new URLSearchParams({
+    'user.fields': 'public_metrics,description,profile_image_url',
+  });
+
+  const url = `${X_API_BASE}/users/${userId}?${params}`;
+
+  console.log(`[x:poller] Fetching profile for user_id=${userId}`);
+
+  const res = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+    },
+  });
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`X API ${res.status} for ${url}: ${text}`);
+  }
+
+  const json = await res.json();
+
+  if (json.errors && !json.data) {
+    throw new Error(`X API errors: ${JSON.stringify(json.errors)}`);
+  }
+
+  const user = json.data ?? {};
+  const metrics = user.public_metrics ?? {};
+  console.log(
+    `[x:poller] Profile: followers=${metrics.followers_count} ` +
+      `following=${metrics.following_count} tweets=${metrics.tweet_count} ` +
+      `listed=${metrics.listed_count}`
+  );
+  return user;
+}
+
+/**
+ * Main polling entry point.
+ *
+ * Fetches recent tweets and the user profile, normalizes each tweet record,
+ * and upserts into the analytics database. Also records a follower snapshot.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @returns {Promise<void>}
+ */
+async function pollX(db) {
+  const token = process.env.X_BEARER_TOKEN;
+  if (!token) throw new Error('X_BEARER_TOKEN environment variable is required');
+
+  const userId = process.env.X_USER_ID;
+  if (!userId) throw new Error('X_USER_ID environment variable is required');
+
+  const [tweetsResult, profile] = await Promise.all([
+    fetchUserTweets(token, userId),
+    fetchUserProfile(token, userId),
+  ]);
+
+  // Lazy-require sibling modules so they can be built/tested independently.
+  const { normalizeXMetric: normalizeMetric } = require('../normalizer');
+  const { upsertMetric, upsertFollowerSnapshot } = require('../store');
+
+  const fetchedAt = new Date().toISOString();
+  const today = fetchedAt.slice(0, 10);
+  const tweets = tweetsResult.data;
+
+  for (const tweet of tweets) {
+    const metrics = tweet.public_metrics ?? {};
+
+    // Metric mapping per spec:
+    //   impressions  = impression_count
+    //   likes        = like_count
+    //   comments     = reply_count
+    //   shares       = retweet_count + quote_count
+    //   saves        = bookmark_count
+    const raw = {
+      id: tweet.id,
+      platform: 'x',
+      content_type: 'tweet',
+      post_id: tweet.id,
+      post_url: `https://x.com/i/web/status/${tweet.id}`,
+      published_at: tweet.created_at ?? null,
+      metric_date: tweet.created_at ? tweet.created_at.slice(0, 10) : today,
+      impressions: metrics.impression_count ?? 0,
+      reach: 0,
+      likes: metrics.like_count ?? 0,
+      comments: metrics.reply_count ?? 0,
+      shares: (metrics.retweet_count ?? 0) + (metrics.quote_count ?? 0),
+      saves: metrics.bookmark_count ?? 0,
+      clicks: metrics.url_link_clicks ?? 0,
+      video_views: metrics.video_view_count ?? 0,
+      followers_delta: 0,
+      extra_json: JSON.stringify({ text: (tweet.text ?? '').slice(0, 280) }),
+      fetched_at: fetchedAt,
+    };
+
+    const normalized = normalizeMetric(raw);
+    upsertMetric(db, normalized);
+
+    console.log(
+      `[x:poller] Upserted tweet id=${tweet.id} ` +
+        `impressions=${metrics.impression_count ?? 0} ` +
+        `likes=${metrics.like_count ?? 0} ` +
+        `replies=${metrics.reply_count ?? 0} ` +
+        `retweets=${metrics.retweet_count ?? 0} ` +
+        `bookmarks=${metrics.bookmark_count ?? 0} ` +
+        `date=${raw.metric_date}`
+    );
+  }
+
+  console.log(`[x:poller] Upserted ${tweets.length} tweet metric records`);
+
+  // Record follower snapshot.
+  const followersCount = profile.public_metrics?.followers_count ?? 0;
+  upsertFollowerSnapshot(db, {
+    platform: 'x',
+    follower_count: followersCount,
+    snapshot_date: today,
+  });
+
+  console.log(
+    `[x:poller] Follower snapshot upserted: platform=x ` +
+      `follower_count=${followersCount} date=${today}`
+  );
+}
+
+module.exports = { fetchUserTweets, fetchUserProfile, pollX };
+
+// ---------------------------------------------------------------------------
+// Stand-alone execution
+// ---------------------------------------------------------------------------
+if (require.main === module) {
+  const { initDb } = require('../store');
+
+  const db = initDb();
+
+  pollX(db)
+    .then(() => {
+      console.log('[x:poller] Poll complete.');
+      db.close();
+    })
+    .catch((err) => {
+      console.error('[x:poller] Poll failed:', err.message);
+      db.close();
+      process.exit(1);
+    });
+}