thumbgate 1.4.3 → 1.4.5

package/scripts/social-analytics/pollers/instagram.js

@@ -1,253 +0,0 @@
-'use strict';
-
-/**
- * Instagram engagement poller using the Instagram Graph API v21.0.
- *
- * Required environment variables:
- *   INSTAGRAM_ACCESS_TOKEN  — long-lived page/user access token
- *   INSTAGRAM_USER_ID       — Instagram business/creator account user ID
- */
-
-const { normalize } = require('../normalizer');
-const { upsertMetric, upsertFollowerSnapshot, initDb } = require('../store');
-
-const GRAPH_BASE = 'https://graph.facebook.com/v21.0';
-
-// Map Instagram media_type to our normalised content_type.
-const MEDIA_TYPE_MAP = {
-  CAROUSEL_ALBUM: 'carousel',
-  IMAGE: 'image',
-  VIDEO: 'reel',
-};
-
-/**
- * Fetches the most recent 25 media items for a user.
- *
- * @param {string} token  - Instagram access token.
- * @param {string} userId - Instagram user ID.
- * @returns {Promise<object[]>} Array of media objects.
- */
-async function fetchInstagramMedia(token, userId) {
-  const fields = [
-    'id',
-    'caption',
-    'media_type',
-    'permalink',
-    'timestamp',
-    'like_count',
-    'comments_count',
-    'children{media_type,media_url}',
-  ].join(',');
-
-  const url =
-    `${GRAPH_BASE}/${userId}/media` +
-    `?fields=${fields}&limit=25&access_token=${token}`;
-
-  const res = await fetch(url);
-  if (!res.ok) {
-    const body = await res.text();
-    throw new Error(`fetchInstagramMedia HTTP ${res.status}: ${body}`);
-  }
-
-  const json = await res.json();
-  if (json.error) {
-    throw new Error(`fetchInstagramMedia API error: ${JSON.stringify(json.error)}`);
-  }
-
-  return json.data ?? [];
-}
-
-/**
- * Fetches insights for a single media item.
- * Some media types (e.g. stories) do not support all metrics — errors are
- * caught and logged so polling continues for other posts.
- *
- * @param {string} token   - Instagram access token.
- * @param {string} mediaId - Instagram media ID.
- * @returns {Promise<object>} Map of metric name -> value (defaults to 0 on error).
- */
-async function fetchMediaInsights(token, mediaId) {
-  const metrics = 'impressions,reach,saved,shares';
-  const url =
-    `${GRAPH_BASE}/${mediaId}/insights` +
-    `?metric=${metrics}&access_token=${token}`;
-
-  try {
-    const res = await fetch(url);
-    if (!res.ok) {
-      const body = await res.text();
-      console.warn(`fetchMediaInsights HTTP ${res.status} for ${mediaId}: ${body}`);
-      return {};
-    }
-
-    const json = await res.json();
-    if (json.error) {
-      // Some media types (Reels, Stories older than 24 h, etc.) return errors
-      // for certain metrics. Log and return an empty result rather than throw.
-      console.warn(
-        `fetchMediaInsights API warning for ${mediaId}: ${JSON.stringify(json.error)}`
-      );
-      return {};
-    }
-
-    // Reshape array of { name, values } -> flat { name: value }
-    const result = {};
-    for (const item of json.data ?? []) {
-      // Instagram returns period-scoped values; take the first value entry.
-      const val = Array.isArray(item.values) ? item.values[0]?.value : item.value;
-      result[item.name] = typeof val === 'number' ? val : 0;
-    }
-    return result;
-  } catch (err) {
-    console.warn(`fetchMediaInsights error for ${mediaId}: ${err.message}`);
-    return {};
-  }
-}
-
-/**
- * Fetches account-level insights: reach, impressions, follower_count.
- *
- * @param {string} token  - Instagram access token.
- * @param {string} userId - Instagram user ID.
- * @param {string} since  - Unix timestamp string (start of window).
- * @param {string} until  - Unix timestamp string (end of window).
- * @returns {Promise<object[]>} Raw data array from the API response.
- */
-async function fetchAccountInsights(token, userId, since, until) {
-  const metrics = 'reach,impressions,follower_count';
-  const url =
-    `${GRAPH_BASE}/${userId}/insights` +
-    `?metric=${metrics}&period=day&since=${since}&until=${until}&access_token=${token}`;
-
-  const res = await fetch(url);
-  if (!res.ok) {
-    const body = await res.text();
-    throw new Error(`fetchAccountInsights HTTP ${res.status}: ${body}`);
-  }
-
-  const json = await res.json();
-  if (json.error) {
-    throw new Error(
-      `fetchAccountInsights API error: ${JSON.stringify(json.error)}`
-    );
-  }
-
-  return json.data ?? [];
-}
-
-/**
- * Main entry point. Polls Instagram for recent media and account insights,
- * normalises the data, and upserts it into the local SQLite database.
- *
- * @param {import('better-sqlite3').Database} db - Initialised db instance.
- */
-async function pollInstagram(db) {
-  const token = process.env.INSTAGRAM_ACCESS_TOKEN;
-  const userId = process.env.INSTAGRAM_USER_ID;
-
-  if (!token) throw new Error('INSTAGRAM_ACCESS_TOKEN is not set');
-  if (!userId) throw new Error('INSTAGRAM_USER_ID is not set');
-
-  const fetchedAt = new Date().toISOString();
-  const today = fetchedAt.slice(0, 10);
-
-  console.log(`[instagram] Fetching media for user ${userId}…`);
-  const mediaItems = await fetchInstagramMedia(token, userId);
-  console.log(`[instagram] Got ${mediaItems.length} media items`);
-
-  for (const item of mediaItems) {
-    const mediaId = item.id;
-
-    // Fetch per-post insights (best-effort; some types may not support all).
-    const insights = await fetchMediaInsights(token, mediaId);
-
-    const rawContentType = MEDIA_TYPE_MAP[item.media_type] ?? 'image';
-
-    // Build a raw record compatible with the normalizer contract.
-    const raw = {
-      platform: 'instagram',
-      content_type: rawContentType,
-      post_id: mediaId,
-      post_url: item.permalink ?? null,
-      published_at: item.timestamp ?? null,
-      metric_date: today,
-      impressions: insights.impressions ?? 0,
-      reach: insights.reach ?? 0,
-      likes: item.like_count ?? 0,
-      comments: item.comments_count ?? 0,
-      shares: insights.shares ?? 0,
-      saves: insights.saved ?? 0,
-      clicks: 0,
-      video_views: 0,
-      followers_delta: 0,
-      fetched_at: fetchedAt,
-      extra_json: {
-        media_type: item.media_type,
-        caption_snippet: (item.caption ?? '').slice(0, 120),
-        children_count: item.children?.data?.length ?? null,
-      },
-    };
-
-    const normalised = normalize(raw);
-    upsertMetric(db, normalised);
-    console.log(`[instagram] Upserted metric for post ${mediaId} (${rawContentType})`);
-  }
-
-  // Fetch account-level follower count for today's snapshot.
-  const nowTs = Math.floor(Date.now() / 1000);
-  const dayAgoTs = nowTs - 86400;
-
-  try {
-    console.log('[instagram] Fetching account insights for follower snapshot…');
-    const accountData = await fetchAccountInsights(
-      token,
-      userId,
-      String(dayAgoTs),
-      String(nowTs)
-    );
-
-    // Find the follower_count series and take the latest value.
-    const followerSeries = accountData.find((d) => d.name === 'follower_count');
-    if (followerSeries) {
-      const latestEntry = (followerSeries.values ?? []).slice(-1)[0];
-      const followerCount = latestEntry?.value ?? 0;
-
-      upsertFollowerSnapshot(db, {
-        platform: 'instagram',
-        follower_count: followerCount,
-        snapshot_date: today,
-      });
-      console.log(`[instagram] Upserted follower snapshot: ${followerCount} followers`);
-    } else {
-      console.warn('[instagram] follower_count not found in account insights response');
-    }
-  } catch (err) {
-    // Account insights failure should not abort per-post metric storage.
-    console.warn(`[instagram] Account insights error (non-fatal): ${err.message}`);
-  }
-
-  console.log('[instagram] Poll complete.');
-}
-
-module.exports = {
-  fetchInstagramMedia,
-  fetchMediaInsights,
-  fetchAccountInsights,
-  pollInstagram,
-};
-
-// Allow running directly: node scripts/social-analytics/pollers/instagram.js
-if (require.main === module) {
-  (async () => {
-    const { initDb } = require('../store');
-    const db = initDb();
-    try {
-      await pollInstagram(db);
-    } catch (err) {
-      console.error('[instagram] Fatal error:', err.message);
-      process.exit(1);
-    } finally {
-      db.close();
-    }
-  })();
-}

package/scripts/social-analytics/pollers/linkedin.js

@@ -1,340 +0,0 @@
-'use strict';
-
-/**
- * LinkedIn engagement poller using the LinkedIn Posts API and related REST endpoints.
- *
- * Required environment variables:
- *   LINKEDIN_ACCESS_TOKEN  — OAuth 2.0 access token with r_liteprofile, r_organization_social,
- *                            and r_1st_connections_size scopes (required)
- *   LINKEDIN_PERSON_URN    — Authenticated member URN, e.g. urn:li:person:XXXXX (required)
- *
- * LinkedIn API references:
- *   Posts API:       https://api.linkedin.com/rest/posts
- *   Social Metadata: https://api.linkedin.com/v2/socialMetadata/{encoded-urn}
- *   Profile:         https://api.linkedin.com/v2/me
- *   Network sizes:   https://api.linkedin.com/v2/networkSizes/{encoded-urn}
- */
-
-const LI_REST_BASE = 'https://api.linkedin.com/rest';
-const LI_V2_BASE = 'https://api.linkedin.com/v2';
-
-// Standard headers required by LinkedIn's versioned REST API.
-function buildRestHeaders(token) {
-  return {
-    Authorization: `Bearer ${token}`,
-    'LinkedIn-Version': '202601',
-    'X-Restli-Protocol-Version': '2.0.0',
-  };
-}
-
-function buildV2Headers(token) {
-  return {
-    Authorization: `Bearer ${token}`,
-    'X-Restli-Protocol-Version': '2.0.0',
-  };
-}
-
-/**
- * Safely coerces a value to an integer, returning 0 for null/undefined/NaN.
- * @param {*} v
- * @returns {number}
- */
-function toInt(v) {
-  const n = parseInt(v, 10);
-  return Number.isFinite(n) ? n : 0;
-}
-
-/**
- * Fetches the most recent posts authored by the authenticated member.
- *
- * Uses the Posts API (LinkedIn-Version: 202401).
- * Returns up to 20 posts sorted by recency.
- *
- * @param {string} token     - LinkedIn OAuth access token.
- * @param {string} personUrn - Member URN, e.g. "urn:li:person:XXXXX".
- * @returns {Promise<object[]>} Array of post objects.
- */
-async function fetchLinkedInPosts(token, personUrn) {
-  if (!token) throw new Error('fetchLinkedInPosts: token is required');
-  if (!personUrn) throw new Error('fetchLinkedInPosts: personUrn is required');
-
-  const url =
-    `${LI_REST_BASE}/posts` +
-    `?author=${encodeURIComponent(personUrn)}&q=author&count=20`;
-
-  console.log(`[linkedin] Fetching posts for ${personUrn}`);
-
-  const res = await fetch(url, { headers: buildRestHeaders(token) });
-  if (!res.ok) {
-    const body = await res.text().catch(() => '');
-    throw new Error(`fetchLinkedInPosts HTTP ${res.status}: ${body}`);
-  }
-
-  const json = await res.json();
-  if (json.serviceErrorCode || json.code) {
-    throw new Error(`fetchLinkedInPosts API error: ${JSON.stringify(json)}`);
-  }
-
-  return json.elements ?? [];
-}
-
-/**
- * Fetches share statistics (impressions, likes, comments, shares) for a post.
- *
- * LinkedIn's analytics APIs have significant restrictions for personal accounts vs company pages.
- * Strategy:
- *   1. Try the organizational entity share statistics endpoint (works for company pages).
- *   2. Fall back to /v2/socialMetadata (works for personal member posts).
- *
- * @param {string} token   - LinkedIn OAuth access token.
- * @param {string} postUrn - Post URN, e.g. "urn:li:share:XXXXX" or "urn:li:ugcPost:XXXXX".
- * @returns {Promise<object>} Normalized stats: { impressions, likes, comments, shares }.
- */
-async function fetchPostAnalytics(token, postUrn) {
-  if (!token) throw new Error('fetchPostAnalytics: token is required');
-  if (!postUrn) throw new Error('fetchPostAnalytics: postUrn is required');
-
-  // Attempt 1: Organizational entity share statistics (company pages / pages with analytics).
-  const orgStatsUrl =
-    `${LI_REST_BASE}/organizationalEntityShareStatistics` +
-    `?q=organizationalEntity&shares=List(${encodeURIComponent(postUrn)})`;
-
-  try {
-    const res = await fetch(orgStatsUrl, { headers: buildRestHeaders(token) });
-    if (res.ok) {
-      const json = await res.json();
-      const el = (json.elements ?? [])[0];
-      if (el) {
-        const stats = el.totalShareStatistics ?? {};
-        console.log(`[linkedin] Got org share stats for ${postUrn}`);
-        return {
-          impressions: toInt(stats.impressionCount),
-          likes: toInt(stats.likeCount),
-          comments: toInt(stats.commentCount),
-          shares: toInt(stats.shareCount),
-          clicks: toInt(stats.clickCount),
-        };
-      }
-    }
-  } catch (err) {
-    console.warn(`[linkedin] Org share stats error for ${postUrn} (trying fallback): ${err.message}`);
-  }
-
-  // Attempt 2: socialMetadata — available for personal member posts.
-  const socialMetaUrl = `${LI_V2_BASE}/socialMetadata/${encodeURIComponent(postUrn)}`;
-
-  try {
-    const res = await fetch(socialMetaUrl, { headers: buildV2Headers(token) });
-    if (res.ok) {
-      const json = await res.json();
-      console.log(`[linkedin] Got socialMetadata for ${postUrn}`);
-      return {
-        impressions: 0, // socialMetadata does not expose impression counts.
-        likes: toInt(json.numLikes ?? json.likes?.paging?.total ?? 0),
-        comments: toInt(json.numComments ?? json.comments?.paging?.total ?? 0),
-        shares: toInt(json.numShares ?? json.shares?.paging?.total ?? 0),
-        clicks: 0,
-      };
-    }
-    const body = await res.text().catch(() => '');
-    console.warn(`[linkedin] socialMetadata HTTP ${res.status} for ${postUrn}: ${body}`);
-  } catch (err) {
-    console.warn(`[linkedin] socialMetadata error for ${postUrn}: ${err.message}`);
-  }
-
-  // Both endpoints failed — return zeroed stats so polling continues.
-  return { impressions: 0, likes: 0, comments: 0, shares: 0, clicks: 0 };
-}
-
-/**
- * Fetches the authenticated member's basic profile information.
- *
- * @param {string} token - LinkedIn OAuth access token.
- * @returns {Promise<object>} Profile object with id, localizedFirstName, localizedLastName, etc.
- */
-async function fetchLinkedInProfile(token) {
-  if (!token) throw new Error('fetchLinkedInProfile: token is required');
-
-  // Try /v2/userinfo first (works with openid+profile scopes), fall back to /v2/me.
-  for (const url of [`${LI_V2_BASE}/userinfo`, `${LI_V2_BASE}/me`]) {
-    const res = await fetch(url, { headers: buildV2Headers(token) });
-    if (!res.ok) {
-      const body = await res.text().catch(() => '');
-      console.warn(`[linkedin] ${url} HTTP ${res.status}: ${body.slice(0, 120)}`);
-      continue;
-    }
-    const json = await res.json();
-    if (json.serviceErrorCode) {
-      console.warn(`[linkedin] ${url} API error: ${JSON.stringify(json).slice(0, 120)}`);
-      continue;
-    }
-    return json;
-  }
-  throw new Error('fetchLinkedInProfile: all endpoints failed');
-}
-
-/**
- * Fetches the network/connection size for a member or company.
- *
- * For company pages, uses CompanyFollowedByMember edgeType.
- * For personal accounts the networkSizes endpoint returns connection count
- * using MEMBER_TO_MEMBER_CONNECTION edgeType.
- *
- * LinkedIn limits this to the authenticated member's own network size
- * due to privacy restrictions.
- *
- * @param {string} token     - LinkedIn OAuth access token.
- * @param {string} personUrn - Member or organization URN.
- * @returns {Promise<number>} Follower / connection count.
- */
-async function fetchFollowerCount(token, personUrn) {
-  if (!token) throw new Error('fetchFollowerCount: token is required');
-  if (!personUrn) throw new Error('fetchFollowerCount: personUrn is required');
-
-  // Determine edge type based on URN type.
-  const isOrg = personUrn.includes('urn:li:organization:');
-  const edgeType = isOrg
-    ? 'CompanyFollowedByMember'
-    : 'MEMBER_TO_MEMBER_CONNECTION';
-
-  const url =
-    `${LI_V2_BASE}/networkSizes/${encodeURIComponent(personUrn)}` +
-    `?edgeType=${edgeType}`;
-
-  try {
-    const res = await fetch(url, { headers: buildV2Headers(token) });
-    if (!res.ok) {
-      const body = await res.text().catch(() => '');
-      console.warn(`[linkedin] fetchFollowerCount HTTP ${res.status}: ${body}`);
-      return 0;
-    }
-    const json = await res.json();
-    // Response shape: { firstDegreeSize: N } or { followerCount: N }
-    return toInt(json.firstDegreeSize ?? json.followerCount ?? 0);
-  } catch (err) {
-    console.warn(`[linkedin] fetchFollowerCount error: ${err.message}`);
-    return 0;
-  }
-}
-
-/**
- * Main entry point. Polls LinkedIn for recent posts and per-post analytics,
- * normalises the data, and upserts it into the local SQLite database.
- *
- * Also records a follower snapshot for today.
- *
- * @param {import('better-sqlite3').Database} db - Initialised db instance.
- */
-async function pollLinkedIn(db) {
-  const token = process.env.LINKEDIN_ACCESS_TOKEN;
-  const personUrn = process.env.LINKEDIN_PERSON_URN;
-
-  if (!token) throw new Error('LINKEDIN_ACCESS_TOKEN is not set');
-  if (!personUrn) throw new Error('LINKEDIN_PERSON_URN is not set');
-
-  const { upsertMetric, upsertFollowerSnapshot } = require('../store');
-
-  const fetchedAt = new Date().toISOString();
-  const today = fetchedAt.slice(0, 10);
-
-  console.log(`[linkedin] Starting poll for ${personUrn}`);
-
-  let posts = [];
-  try {
-    posts = await fetchLinkedInPosts(token, personUrn);
-    console.log(`[linkedin] Got ${posts.length} posts`);
-  } catch (err) {
-    // 403 = token lacks r_member_social scope; 426 = version expired.
-    // Either way, skip posts but still collect follower count.
-    console.warn(`[linkedin] Posts fetch failed (non-fatal): ${err.message}`);
-    console.warn('[linkedin] To read posts, re-authorize with r_member_social scope.');
-  }
-
-  for (const post of posts) {
-    // Post URN is at post.id for the Posts API (urn:li:share:... or urn:li:ugcPost:...).
-    const postUrn = post.id ?? post.urn ?? '';
-    if (!postUrn) {
-      console.warn('[linkedin] Post missing id/urn, skipping');
-      continue;
-    }
-
-    const analytics = await fetchPostAnalytics(token, postUrn);
-
-    // Determine content type: article vs standard post.
-    const hasArticle = !!(post.content?.article);
-    const contentType = hasArticle ? 'article' : 'post';
-
-    // Published time: Posts API returns createdAt as milliseconds epoch.
-    let publishedAt = null;
-    if (post.createdAt) {
-      publishedAt = new Date(post.createdAt).toISOString();
-    }
-
-    const record = {
-      platform: 'linkedin',
-      content_type: contentType,
-      post_id: postUrn,
-      post_url: post.content?.article?.source ?? null,
-      published_at: publishedAt,
-      metric_date: today,
-      impressions: analytics.impressions,
-      reach: 0,
-      likes: analytics.likes,
-      comments: analytics.comments,
-      shares: analytics.shares,
-      saves: 0,
-      clicks: analytics.clicks,
-      video_views: 0,
-      followers_delta: 0,
-      extra_json: JSON.stringify({
-        commentary_snippet: (post.commentary ?? '').slice(0, 120),
-        visibility: post.visibility ?? null,
-        lifecycle_state: post.lifecycleState ?? null,
-      }),
-      fetched_at: fetchedAt,
-    };
-
-    upsertMetric(db, record);
-    console.log(`[linkedin] Upserted metric for post ${postUrn} (${contentType})`);
-  }
-
-  // Follower / connection snapshot.
-  try {
-    console.log('[linkedin] Fetching follower/connection count…');
-    const followerCount = await fetchFollowerCount(token, personUrn);
-    upsertFollowerSnapshot(db, {
-      platform: 'linkedin',
-      follower_count: followerCount,
-      snapshot_date: today,
-    });
-    console.log(`[linkedin] Upserted follower snapshot: ${followerCount}`);
-  } catch (err) {
-    console.warn(`[linkedin] Follower snapshot error (non-fatal): ${err.message}`);
-  }
-
-  console.log('[linkedin] Poll complete.');
-}
-
-module.exports = {
-  fetchLinkedInPosts,
-  fetchPostAnalytics,
-  fetchLinkedInProfile,
-  fetchFollowerCount,
-  pollLinkedIn,
-};
-
-// Allow running directly: node scripts/social-analytics/pollers/linkedin.js
-if (require.main === module) {
-  (async () => {
-    const { initDb } = require('../store');
-    const db = initDb();
-    try {
-      await pollLinkedIn(db);
-    } catch (err) {
-      console.error('[linkedin] Fatal error:', err.message);
-      process.exit(1);
-    } finally {
-      db.close();
-    }
-  })();
-}