thumbgate 1.4.3 → 1.4.5

package/scripts/social-analytics/pollers/threads.js

@@ -1,233 +0,0 @@
-'use strict';
-
-/**
- * Threads engagement poller using the Threads API (Meta, launched June 2024).
- *
- * Threads API base: https://graph.threads.net/v1.0
- * Authentication: Meta OAuth (Threads Login). Long-lived tokens last 60 days.
- *
- * Required environment variables:
- *   THREADS_ACCESS_TOKEN — long-lived Threads user access token
- *   THREADS_USER_ID      — Threads user ID (numeric string)
- *
- * Handle: @igor.ganapolsky
- */
-
-const { upsertMetric, upsertFollowerSnapshot, initDb } = require('../store');
-
-const THREADS_BASE = 'https://graph.threads.net/v1.0';
-
-/**
- * Fetches the most recent 25 threads for a user.
- *
- * @param {string} token  - Threads access token.
- * @param {string} userId - Threads user ID.
- * @returns {Promise<object[]>} Array of thread objects.
- */
-async function fetchThreadsPosts(token, userId) {
-  const fields = [
-    'id',
-    'text',
-    'timestamp',
-    'media_type',
-    'permalink',
-    'is_reply',
-    'likes',
-    'replies',
-    'reposts',
-    'quotes',
-  ].join(',');
-
-  const url =
-    `${THREADS_BASE}/${userId}/threads` +
-    `?fields=${fields}&limit=25&access_token=${token}`;
-
-  const res = await fetch(url);
-  if (!res.ok) {
-    const body = await res.text();
-    throw new Error(`fetchThreadsPosts HTTP ${res.status}: ${body}`);
-  }
-
-  const json = await res.json();
-  if (json.error) {
-    throw new Error(`fetchThreadsPosts API error: ${JSON.stringify(json.error)}`);
-  }
-
-  return json.data ?? [];
-}
-
-/**
- * Fetches insights for a single thread (post-level metrics).
- * Errors are caught and logged so polling continues for other threads.
- *
- * @param {string} token   - Threads access token.
- * @param {string} mediaId - Threads post ID.
- * @returns {Promise<object>} Map of metric name -> value (defaults to 0 on error).
- */
-async function fetchThreadsInsights(token, mediaId) {
-  const url =
-    `${THREADS_BASE}/${mediaId}/insights` +
-    `?metric=views,likes,replies,reposts,quotes&access_token=${token}`;
-
-  try {
-    const res = await fetch(url);
-    if (!res.ok) {
-      const body = await res.text();
-      console.warn(`fetchThreadsInsights HTTP ${res.status} for ${mediaId}: ${body}`);
-      return {};
-    }
-
-    const json = await res.json();
-    if (json.error) {
-      console.warn(
-        `fetchThreadsInsights API warning for ${mediaId}: ${JSON.stringify(json.error)}`
-      );
-      return {};
-    }
-
-    // Reshape array of { name, values } -> flat { name: value }
-    const result = {};
-    for (const item of json.data ?? []) {
-      const val = Array.isArray(item.values) ? item.values[0]?.value : item.value;
-      result[item.name] = typeof val === 'number' ? val : 0;
-    }
-    return result;
-  } catch (err) {
-    console.warn(`fetchThreadsInsights error for ${mediaId}: ${err.message}`);
-    return {};
-  }
-}
-
-/**
- * Fetches the Threads user profile including follower count.
- *
- * @param {string} token  - Threads access token.
- * @param {string} userId - Threads user ID.
- * @returns {Promise<object>} Profile object with id, username, followers_count, etc.
- */
-async function fetchThreadsProfile(token, userId) {
-  const url =
-    `${THREADS_BASE}/${userId}` +
-    `?fields=id,username,threads_profile_picture_url,threads_biography,followers_count` +
-    `&access_token=${token}`;
-
-  const res = await fetch(url);
-  if (!res.ok) {
-    const body = await res.text();
-    throw new Error(`fetchThreadsProfile HTTP ${res.status}: ${body}`);
-  }
-
-  const json = await res.json();
-  if (json.error) {
-    throw new Error(`fetchThreadsProfile API error: ${JSON.stringify(json.error)}`);
-  }
-
-  return json;
-}
-
-/**
- * Main entry point. Polls Threads for recent posts and profile data,
- * normalises the data, and upserts it into the local SQLite database.
- *
- * @param {import('better-sqlite3').Database} db - Initialised db instance.
- */
-async function pollThreads(db) {
-  const token = process.env.THREADS_ACCESS_TOKEN;
-  const userId = process.env.THREADS_USER_ID;
-
-  if (!token) throw new Error('THREADS_ACCESS_TOKEN is not set');
-  if (!userId) throw new Error('THREADS_USER_ID is not set');
-
-  const fetchedAt = new Date().toISOString();
-  const today = fetchedAt.slice(0, 10);
-
-  console.log(`[threads] Fetching posts for user ${userId}…`);
-  const posts = await fetchThreadsPosts(token, userId);
-  console.log(`[threads] Got ${posts.length} thread posts`);
-
-  for (const post of posts) {
-    const postId = post.id;
-
-    // Fetch per-post insights (best-effort; not all post types support all metrics).
-    const insights = await fetchThreadsInsights(token, postId);
-
-    // Threads API returns likes/replies/reposts/quotes both on the post object
-    // and via insights. Prefer insights values when available; fall back to
-    // the top-level fields returned by the posts endpoint.
-    const likes = insights.likes ?? post.likes ?? 0;
-    const replies = insights.replies ?? post.replies ?? 0;
-    const reposts = insights.reposts ?? post.reposts ?? 0;
-    const quotes = insights.quotes ?? post.quotes ?? 0;
-    const views = insights.views ?? 0;
-
-    const raw = {
-      platform: 'threads',
-      content_type: 'thread',
-      post_id: postId,
-      post_url: post.permalink ?? null,
-      published_at: post.timestamp ?? null,
-      metric_date: today,
-      impressions: views,
-      reach: 0,
-      likes: likes,
-      comments: replies,
-      shares: reposts + quotes,
-      saves: 0,
-      clicks: 0,
-      video_views: 0,
-      followers_delta: 0,
-      fetched_at: fetchedAt,
-      extra_json: JSON.stringify({
-        media_type: post.media_type ?? null,
-        is_reply: post.is_reply ?? false,
-        text_snippet: (post.text ?? '').slice(0, 120),
-        reposts,
-        quotes,
-      }),
-    };
-
-    upsertMetric(db, raw);
-    console.log(`[threads] Upserted metric for post ${postId}`);
-  }
-
-  // Fetch profile for follower snapshot.
-  try {
-    console.log('[threads] Fetching profile for follower snapshot…');
-    const profile = await fetchThreadsProfile(token, userId);
-    const followerCount = profile.followers_count ?? 0;
-
-    upsertFollowerSnapshot(db, {
-      platform: 'threads',
-      follower_count: followerCount,
-      snapshot_date: today,
-    });
-    console.log(`[threads] Upserted follower snapshot: ${followerCount} followers`);
-  } catch (err) {
-    // Profile failure should not abort per-post metric storage.
-    console.warn(`[threads] Profile fetch error (non-fatal): ${err.message}`);
-  }
-
-  console.log('[threads] Poll complete.');
-}
-
-module.exports = {
-  fetchThreadsPosts,
-  fetchThreadsInsights,
-  fetchThreadsProfile,
-  pollThreads,
-};
-
-// Allow running directly: node scripts/social-analytics/pollers/threads.js
-if (require.main === module) {
-  (async () => {
-    const db = initDb();
-    try {
-      await pollThreads(db);
-    } catch (err) {
-      console.error('[threads] Fatal error:', err.message);
-      process.exit(1);
-    } finally {
-      db.close();
-    }
-  })();
-}

package/scripts/social-analytics/pollers/tiktok.js

@@ -1,203 +0,0 @@
-'use strict';
-
-/**
- * tiktok.js
- * Polls TikTok Content Posting API v2 for video engagement data.
- *
- * Required env vars:
- *   TIKTOK_ACCESS_TOKEN  — OAuth 2.0 access token with required scopes (required)
- *
- * Scopes needed: video.list, user.info.basic, user.info.stats
- */
-
-const path = require('path');
-
-const TIKTOK_API_BASE = 'https://open.tiktokapis.com';
-
-/**
- * Fetch the authenticated user's recent videos.
- *
- * @param {string} token - TikTok OAuth access token
- * @returns {Promise<object[]>} Array of video objects
- */
-async function fetchTikTokVideos(token) {
-  if (!token) {
-    throw new Error('TIKTOK_ACCESS_TOKEN is required');
-  }
-
-  const url = `${TIKTOK_API_BASE}/v2/video/list/`;
-  const body = {
-    max_count: 20,
-    fields: ['id', 'title', 'view_count', 'like_count', 'comment_count', 'share_count', 'create_time', 'duration'],
-  };
-
-  console.log('[tiktok] Fetching video list');
-
-  const res = await fetch(url, {
-    method: 'POST',
-    headers: {
-      Authorization: `Bearer ${token}`,
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify(body),
-  });
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`TikTok API ${res.status} for ${url}: ${text}`);
-  }
-
-  const json = await res.json();
-
-  if (json.error && json.error.code !== 'ok') {
-    throw new Error(`TikTok API error: ${json.error.code} — ${json.error.message}`);
-  }
-
-  const videos = json.data?.videos ?? [];
-  console.log(`[tiktok] Retrieved ${videos.length} videos`);
-  return videos;
-}
-
-/**
- * Fetch the authenticated user's profile stats.
- *
- * @param {string} token - TikTok OAuth access token
- * @returns {Promise<object>} User info object with follower_count, likes_count, video_count
- */
-async function fetchTikTokUserInfo(token) {
-  if (!token) {
-    throw new Error('TIKTOK_ACCESS_TOKEN is required');
-  }
-
-  const url = `${TIKTOK_API_BASE}/v2/user/info/?fields=follower_count,likes_count,video_count`;
-
-  console.log('[tiktok] Fetching user info');
-
-  const res = await fetch(url, {
-    method: 'GET',
-    headers: {
-      Authorization: `Bearer ${token}`,
-    },
-  });
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`TikTok API ${res.status} for ${url}: ${text}`);
-  }
-
-  const json = await res.json();
-
-  if (json.error && json.error.code !== 'ok') {
-    throw new Error(`TikTok API error: ${json.error.code} — ${json.error.message}`);
-  }
-
-  const userInfo = json.data?.user ?? {};
-  console.log(
-    `[tiktok] User info: follower_count=${userInfo.follower_count} ` +
-      `likes_count=${userInfo.likes_count} video_count=${userInfo.video_count}`
-  );
-  return userInfo;
-}
-
-/**
- * Main polling entry point.
- *
- * Fetches TikTok videos and user info, normalizes each video record,
- * and upserts into the analytics database. Also records a follower snapshot.
- *
- * @param {import('better-sqlite3').Database} db
- * @returns {Promise<void>}
- */
-async function pollTikTok(db) {
-  const token = process.env.TIKTOK_ACCESS_TOKEN;
-  if (!token) {
-    throw new Error('TIKTOK_ACCESS_TOKEN environment variable is required');
-  }
-
-  const [videos, userInfo] = await Promise.all([
-    fetchTikTokVideos(token),
-    fetchTikTokUserInfo(token),
-  ]);
-
-  // Lazy-require sibling modules so they can be built/tested independently.
-  const { normalizeTikTokMetric: normalizeMetric } = require('../normalizer');
-  const { upsertMetric, upsertFollowerSnapshot } = require('../store');
-
-  const fetchedAt = new Date().toISOString();
-  const today = fetchedAt.slice(0, 10);
-
-  for (const video of videos) {
-    // create_time is a Unix timestamp (seconds); convert to ISO date string.
-    const publishedDate = video.create_time
-      ? new Date(video.create_time * 1000).toISOString()
-      : null;
-    const metricDate = publishedDate ? publishedDate.slice(0, 10) : today;
-
-    const raw = {
-      platform: 'tiktok',
-      content_type: 'video',
-      post_id: video.id,
-      post_url: null,
-      published_at: publishedDate,
-      metric_date: metricDate,
-      video_views: video.view_count ?? 0,
-      likes: video.like_count ?? 0,
-      comments: video.comment_count ?? 0,
-      shares: video.share_count ?? 0,
-      impressions: 0,
-      reach: 0,
-      saves: 0,
-      clicks: 0,
-      followers_delta: 0,
-      extra_json: JSON.stringify({ title: video.title ?? '', duration: video.duration ?? 0 }),
-      fetched_at: fetchedAt,
-    };
-
-    const normalized = normalizeMetric(raw);
-    upsertMetric(db, normalized);
-
-    console.log(
-      `[tiktok] Upserted video id=${video.id} ` +
-        `views=${video.view_count} likes=${video.like_count} ` +
-        `comments=${video.comment_count} shares=${video.share_count} ` +
-        `date=${metricDate}`
-    );
-  }
-
-  console.log(`[tiktok] Upserted ${videos.length} video metric records`);
-
-  // Record follower snapshot.
-  const followerCount = userInfo.follower_count ?? 0;
-  upsertFollowerSnapshot(db, {
-    platform: 'tiktok',
-    follower_count: followerCount,
-    snapshot_date: today,
-  });
-
-  console.log(
-    `[tiktok] Follower snapshot upserted: platform=tiktok ` +
-      `follower_count=${followerCount} date=${today}`
-  );
-}
-
-module.exports = { fetchTikTokVideos, fetchTikTokUserInfo, pollTikTok };
-
-// ---------------------------------------------------------------------------
-// Stand-alone execution
-// ---------------------------------------------------------------------------
-if (require.main === module) {
-  const { initDb } = require('../store');
-
-  const db = initDb();
-
-  pollTikTok(db)
-    .then(() => {
-      console.log('[tiktok] Poll complete.');
-      db.close();
-    })
-    .catch((err) => {
-      console.error('[tiktok] Poll failed:', err.message);
-      db.close();
-      process.exit(1);
-    });
-}

package/scripts/social-analytics/pollers/x.js

@@ -1,227 +0,0 @@
-'use strict';
-
-/**
- * x.js
- * Polls X (Twitter) API v2 for tweet engagement data and profile metrics.
- *
- * Required env vars:
- *   X_BEARER_TOKEN  — App-only Bearer Token (required)
- *   X_USER_ID       — Numeric X user ID to poll (required)
- *
- * Rate limits (Basic tier):
- *   User tweet timeline: 10 requests / 15 minutes per app.
- *   User lookup: 300 requests / 15 minutes per app.
- *   Exceeding these limits returns HTTP 429; callers should implement
- *   exponential backoff or schedule polls no more frequently than every
- *   2 minutes to stay within the Basic tier ceiling.
- *
- * X API v2 reference:
- *   GET /2/users/:id/tweets
- *   GET /2/users/:id
- */
-
-const X_API_BASE = 'https://api.twitter.com/2';
-
-/**
- * Fetches the most recent 20 tweets for a user.
- *
- * @param {string} token  - X Bearer Token
- * @param {string} userId - Numeric X user ID
- * @returns {Promise<{ data: object[], includes?: object }>}
- */
-async function fetchUserTweets(token, userId) {
-  if (!token) throw new Error('X_BEARER_TOKEN is required');
-  if (!userId) throw new Error('X_USER_ID is required');
-
-  const params = new URLSearchParams({
-    max_results: '20',
-    'tweet.fields': 'public_metrics,created_at,text',
-    expansions: 'attachments.media_keys',
-    'media.fields': 'preview_image_url,type,url',
-  });
-
-  const url = `${X_API_BASE}/users/${userId}/tweets?${params}`;
-
-  console.log(`[x:poller] Fetching tweets for user_id=${userId}`);
-
-  const res = await fetch(url, {
-    headers: {
-      Authorization: `Bearer ${token}`,
-    },
-  });
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`X API ${res.status} for ${url}: ${text}`);
-  }
-
-  const json = await res.json();
-
-  if (json.errors && !json.data) {
-    throw new Error(`X API errors: ${JSON.stringify(json.errors)}`);
-  }
-
-  const tweets = json.data ?? [];
-  console.log(`[x:poller] Retrieved ${tweets.length} tweets`);
-  return { data: tweets, includes: json.includes ?? {} };
-}
-
-/**
- * Fetches the public profile metrics for a user.
- *
- * Returns an object with:
- *   followers_count, following_count, tweet_count, listed_count
- *
- * @param {string} token  - X Bearer Token
- * @param {string} userId - Numeric X user ID
- * @returns {Promise<object>} User object including public_metrics field
- */
-async function fetchUserProfile(token, userId) {
-  if (!token) throw new Error('X_BEARER_TOKEN is required');
-  if (!userId) throw new Error('X_USER_ID is required');
-
-  const params = new URLSearchParams({
-    'user.fields': 'public_metrics,description,profile_image_url',
-  });
-
-  const url = `${X_API_BASE}/users/${userId}?${params}`;
-
-  console.log(`[x:poller] Fetching profile for user_id=${userId}`);
-
-  const res = await fetch(url, {
-    headers: {
-      Authorization: `Bearer ${token}`,
-    },
-  });
-
-  if (!res.ok) {
-    const text = await res.text().catch(() => '');
-    throw new Error(`X API ${res.status} for ${url}: ${text}`);
-  }
-
-  const json = await res.json();
-
-  if (json.errors && !json.data) {
-    throw new Error(`X API errors: ${JSON.stringify(json.errors)}`);
-  }
-
-  const user = json.data ?? {};
-  const metrics = user.public_metrics ?? {};
-  console.log(
-    `[x:poller] Profile: followers=${metrics.followers_count} ` +
-      `following=${metrics.following_count} tweets=${metrics.tweet_count} ` +
-      `listed=${metrics.listed_count}`
-  );
-  return user;
-}
-
-/**
- * Main polling entry point.
- *
- * Fetches recent tweets and the user profile, normalizes each tweet record,
- * and upserts into the analytics database. Also records a follower snapshot.
- *
- * @param {import('better-sqlite3').Database} db
- * @returns {Promise<void>}
- */
-async function pollX(db) {
-  const token = process.env.X_BEARER_TOKEN;
-  if (!token) throw new Error('X_BEARER_TOKEN environment variable is required');
-
-  const userId = process.env.X_USER_ID;
-  if (!userId) throw new Error('X_USER_ID environment variable is required');
-
-  const [tweetsResult, profile] = await Promise.all([
-    fetchUserTweets(token, userId),
-    fetchUserProfile(token, userId),
-  ]);
-
-  // Lazy-require sibling modules so they can be built/tested independently.
-  const { normalizeXMetric: normalizeMetric } = require('../normalizer');
-  const { upsertMetric, upsertFollowerSnapshot } = require('../store');
-
-  const fetchedAt = new Date().toISOString();
-  const today = fetchedAt.slice(0, 10);
-  const tweets = tweetsResult.data;
-
-  for (const tweet of tweets) {
-    const metrics = tweet.public_metrics ?? {};
-
-    // Metric mapping per spec:
-    //   impressions  = impression_count
-    //   likes        = like_count
-    //   comments     = reply_count
-    //   shares       = retweet_count + quote_count
-    //   saves        = bookmark_count
-    const raw = {
-      id: tweet.id,
-      platform: 'x',
-      content_type: 'tweet',
-      post_id: tweet.id,
-      post_url: `https://x.com/i/web/status/${tweet.id}`,
-      published_at: tweet.created_at ?? null,
-      metric_date: tweet.created_at ? tweet.created_at.slice(0, 10) : today,
-      impressions: metrics.impression_count ?? 0,
-      reach: 0,
-      likes: metrics.like_count ?? 0,
-      comments: metrics.reply_count ?? 0,
-      shares: (metrics.retweet_count ?? 0) + (metrics.quote_count ?? 0),
-      saves: metrics.bookmark_count ?? 0,
-      clicks: metrics.url_link_clicks ?? 0,
-      video_views: metrics.video_view_count ?? 0,
-      followers_delta: 0,
-      extra_json: JSON.stringify({ text: (tweet.text ?? '').slice(0, 280) }),
-      fetched_at: fetchedAt,
-    };
-
-    const normalized = normalizeMetric(raw);
-    upsertMetric(db, normalized);
-
-    console.log(
-      `[x:poller] Upserted tweet id=${tweet.id} ` +
-        `impressions=${metrics.impression_count ?? 0} ` +
-        `likes=${metrics.like_count ?? 0} ` +
-        `replies=${metrics.reply_count ?? 0} ` +
-        `retweets=${metrics.retweet_count ?? 0} ` +
-        `bookmarks=${metrics.bookmark_count ?? 0} ` +
-        `date=${raw.metric_date}`
-    );
-  }
-
-  console.log(`[x:poller] Upserted ${tweets.length} tweet metric records`);
-
-  // Record follower snapshot.
-  const followersCount = profile.public_metrics?.followers_count ?? 0;
-  upsertFollowerSnapshot(db, {
-    platform: 'x',
-    follower_count: followersCount,
-    snapshot_date: today,
-  });
-
-  console.log(
-    `[x:poller] Follower snapshot upserted: platform=x ` +
-      `follower_count=${followersCount} date=${today}`
-  );
-}
-
-module.exports = { fetchUserTweets, fetchUserProfile, pollX };
-
-// ---------------------------------------------------------------------------
-// Stand-alone execution
-// ---------------------------------------------------------------------------
-if (require.main === module) {
-  const { initDb } = require('../store');
-
-  const db = initDb();
-
-  pollX(db)
-    .then(() => {
-      console.log('[x:poller] Poll complete.');
-      db.close();
-    })
-    .catch((err) => {
-      console.error('[x:poller] Poll failed:', err.message);
-      db.close();
-      process.exit(1);
-    });
-}