thumbgate 0.9.10

package/scripts/social-analytics/notify.js

@@ -0,0 +1,162 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+const DEFAULT_DIGEST_PATH = path.join(REPO_ROOT, '.artifacts', 'social', 'digests', 'digest.json');
+
+/**
+ * Sends a formatted digest to Slack via an incoming webhook.
+ * Skips silently if SLACK_WEBHOOK_URL is not set.
+ *
+ * @param {object} digest
+ * @returns {Promise<void>}
+ */
+async function sendSlackDigest(digest) {
+  const webhookUrl = process.env.SLACK_WEBHOOK_URL;
+
+  if (!webhookUrl) {
+    console.log('Slack webhook not configured, skipping notification');
+    return;
+  }
+
+  const { period, summary, top_content } = digest;
+
+  const followerLines = Object.entries(summary.follower_delta)
+    .map(([platform, delta]) => {
+      const sign = delta >= 0 ? '+' : '';
+      return `• ${platform}: ${sign}${delta}`;
+    })
+    .join('\n');
+
+  const topThree = (top_content || []).slice(0, 3);
+  const topLines = topThree.length
+    ? topThree
+        .map((item, idx) => {
+          const urlPart = item.post_url ? ` — <${item.post_url}|link>` : '';
+          return `${idx + 1}. *${item.platform}* \`${item.post_id}\` — ${item.total_engagement} eng / ${(item.impressions || 0).toLocaleString()} impressions${urlPart}`;
+        })
+        .join('\n')
+    : '_No content recorded._';
+
+  const payload = {
+    blocks: [
+      {
+        type: 'header',
+        text: {
+          type: 'plain_text',
+          text: `Weekly Social Digest (${period.start} → ${period.end})`,
+          emoji: false,
+        },
+      },
+      {
+        type: 'section',
+        text: {
+          type: 'mrkdwn',
+          text: [
+            `*Impressions:* ${summary.total_impressions.toLocaleString()}`,
+            `*Likes:* ${summary.total_likes.toLocaleString()}`,
+            `*Comments:* ${summary.total_comments.toLocaleString()}`,
+            `*Shares:* ${summary.total_shares.toLocaleString()}`,
+            `*Engagement Rate:* ${summary.engagement_rate}`,
+            '',
+            '*Follower Delta:*',
+            followerLines,
+          ].join('\n'),
+        },
+      },
+      {
+        type: 'section',
+        text: {
+          type: 'mrkdwn',
+          text: `*Top 3 Content:*\n${topLines}`,
+        },
+      },
+    ],
+  };
+
+  const response = await fetch(webhookUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(payload),
+  });
+
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(`Slack webhook failed: ${response.status} ${response.statusText} — ${body}`);
+  }
+
+  console.log(`Slack digest sent for period ${period.start} → ${period.end}`);
+}
+
+/**
+ * Returns a formatted string for terminal output. Always works — no Slack needed.
+ *
+ * @param {object} digest
+ * @returns {string}
+ */
+function formatDigestForConsole(digest) {
+  const { period, summary, top_content } = digest;
+
+  const lines = [];
+
+  lines.push('');
+  lines.push(`Weekly Social Digest: ${period.start} -> ${period.end}`);
+  lines.push('='.repeat(60));
+  lines.push('');
+  lines.push('SUMMARY');
+  lines.push(`  Impressions    : ${summary.total_impressions.toLocaleString()}`);
+  lines.push(`  Likes          : ${summary.total_likes.toLocaleString()}`);
+  lines.push(`  Comments       : ${summary.total_comments.toLocaleString()}`);
+  lines.push(`  Shares         : ${summary.total_shares.toLocaleString()}`);
+  lines.push(`  Engagement Rate: ${summary.engagement_rate}`);
+  lines.push('');
+  lines.push('FOLLOWER DELTA');
+  for (const [platform, delta] of Object.entries(summary.follower_delta)) {
+    const sign = delta >= 0 ? '+' : '';
+    lines.push(`  ${platform.padEnd(12)}: ${sign}${delta}`);
+  }
+  lines.push('');
+  lines.push('TOP CONTENT');
+
+  const topThree = (top_content || []).slice(0, 3);
+  if (topThree.length === 0) {
+    lines.push('  No content recorded in this period.');
+  } else {
+    topThree.forEach((item, idx) => {
+      lines.push(`  ${idx + 1}. [${item.platform}] ${item.post_id}`);
+      lines.push(`     Engagement: ${item.total_engagement}  Impressions: ${(item.impressions || 0).toLocaleString()}`);
+      if (item.post_url) {
+        lines.push(`     URL: ${item.post_url}`);
+      }
+    });
+  }
+
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+module.exports = {
+  sendSlackDigest,
+  formatDigestForConsole,
+};
+
+// Run as main: load latest digest and send.
+if (require.main === module) {
+  if (!fs.existsSync(DEFAULT_DIGEST_PATH)) {
+    console.error(`Digest file not found: ${DEFAULT_DIGEST_PATH}`);
+    console.error('Run scripts/social-analytics/digest.js first to generate it.');
+    process.exit(1);
+  }
+
+  const digest = JSON.parse(fs.readFileSync(DEFAULT_DIGEST_PATH, 'utf8'));
+
+  console.log(formatDigestForConsole(digest));
+
+  sendSlackDigest(digest).catch((err) => {
+    console.error('Failed to send Slack notification:', err.message);
+    process.exit(1);
+  });
+}

package/scripts/social-analytics/poll-all.js

@@ -0,0 +1,107 @@
+'use strict';
+
+require('dotenv').config({ path: require('node:path').resolve(__dirname, '..', '..', '.env') });
+const path = require('node:path');
+const fs = require('node:fs');
+
+// Load .env if available
+const envPath = path.resolve(__dirname, '..', '..', '.env');
+if (fs.existsSync(envPath)) {
+  const envContent = fs.readFileSync(envPath, 'utf8');
+  for (const line of envContent.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eqIdx = trimmed.indexOf('=');
+    if (eqIdx > 0) {
+      const key = trimmed.slice(0, eqIdx);
+      const value = trimmed.slice(eqIdx + 1);
+      if (!process.env[key]) process.env[key] = value;
+    }
+  }
+}
+
+const { initDb } = require('./store');
+
+const POLLERS = [
+  { name: 'github', module: './pollers/github', envRequired: ['GITHUB_TOKEN'] },
+  { name: 'instagram', module: './pollers/instagram', envRequired: ['INSTAGRAM_ACCESS_TOKEN', 'INSTAGRAM_USER_ID'] },
+  { name: 'tiktok', module: './pollers/tiktok', envRequired: ['TIKTOK_ACCESS_TOKEN'] },
+  { name: 'linkedin', module: './pollers/linkedin', envRequired: ['LINKEDIN_ACCESS_TOKEN', 'LINKEDIN_PERSON_URN'] },
+  { name: 'x', module: './pollers/x', envRequired: ['X_BEARER_TOKEN', 'X_USER_ID'] },
+  { name: 'reddit', module: './pollers/reddit', envRequired: ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET', 'REDDIT_USERNAME', 'REDDIT_PASSWORD'] },
+  { name: 'threads', module: './pollers/threads', envRequired: ['THREADS_ACCESS_TOKEN', 'THREADS_USER_ID'] },
+  { name: 'youtube', module: './pollers/youtube', envRequired: ['YOUTUBE_API_KEY', 'YOUTUBE_CHANNEL_ID'] },
+  { name: 'plausible', module: './pollers/plausible', envRequired: ['PLAUSIBLE_API_KEY', 'PLAUSIBLE_SITE_ID'] },
+  { name: 'zernio', module: './pollers/zernio', envRequired: ['ZERNIO_API_KEY'] },
+];
+
+function hasEnv(keys) {
+  return keys.every((k) => process.env[k]);
+}
+
+async function pollAll(options = {}) {
+  const db = initDb(options.dbPath);
+  const results = { succeeded: [], skipped: [], failed: [] };
+
+  for (const poller of POLLERS) {
+    if (!hasEnv(poller.envRequired)) {
+      console.log(`⏭  ${poller.name}: skipped (missing env: ${poller.envRequired.filter((k) => !process.env[k]).join(', ')})`);
+      results.skipped.push(poller.name);
+      continue;
+    }
+
+    try {
+      const mod = require(poller.module);
+      const fn = mod[`poll${poller.name.charAt(0).toUpperCase()}${poller.name.slice(1)}`]
+        || mod.pollGitHub || mod.pollInstagram || mod.pollTikTok
+        || mod.pollLinkedIn || mod.pollX || mod.pollReddit
+        || mod.pollThreads || mod.pollPlausible || mod.pollZernio;
+
+      if (!fn) {
+        console.log(`⚠  ${poller.name}: no poll function found in module`);
+        results.skipped.push(poller.name);
+        continue;
+      }
+
+      console.log(`🔄 ${poller.name}: polling...`);
+      await fn(db);
+      console.log(`✅ ${poller.name}: complete`);
+      results.succeeded.push(poller.name);
+    } catch (err) {
+      console.error(`❌ ${poller.name}: ${err.message}`);
+      results.failed.push({ name: poller.name, error: err.message });
+    }
+  }
+
+  db.close();
+  return results;
+}
+
+async function main() {
+  console.log('=== Social Analytics Poll All ===');
+  console.log(`Time: ${new Date().toISOString()}`);
+  console.log('');
+
+  const results = await pollAll();
+
+  console.log('');
+  console.log('=== Summary ===');
+  console.log(`Succeeded: ${results.succeeded.join(', ') || 'none'}`);
+  console.log(`Skipped:   ${results.skipped.join(', ') || 'none'}`);
+  console.log(`Failed:    ${results.failed.map((f) => f.name).join(', ') || 'none'}`);
+
+  // Exit non-zero only if nothing succeeded AND there were failures.
+  // Partial success (some pollers skipped/failed but at least one succeeded) is OK.
+  if (results.succeeded.length === 0 && results.failed.length > 0) {
+    process.exitCode = 1;
+  }
+}
+
+if (require.main === module) {
+  main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
+}
+
+module.exports = { pollAll, POLLERS };

package/scripts/social-analytics/pollers/github.js

@@ -0,0 +1,195 @@
+'use strict';
+
+/**
+ * github.js
+ * Polls GitHub REST API for repository traffic data.
+ *
+ * IMPORTANT: GitHub only retains traffic data for 14 days.
+ * This poller must run at least daily to avoid gaps.
+ *
+ * Required env vars:
+ *   GITHUB_TOKEN  — personal access token with repo scope (required)
+ *   GITHUB_REPO   — owner/repo slug (default: IgorGanapolsky/ThumbGate)
+ */
+
+const path = require('path');
+
+const GITHUB_API_BASE = 'https://api.github.com';
+const DEFAULT_REPO = 'IgorGanapolsky/ThumbGate';
+
+/**
+ * Build standard GitHub API request headers.
+ * @param {string} token
+ * @returns {Record<string, string>}
+ */
+function buildHeaders(token) {
+  return {
+    Authorization: `Bearer ${token}`,
+    Accept: 'application/vnd.github+json',
+    'X-GitHub-Api-Version': '2022-11-28',
+    'User-Agent': 'social-analytics-poller/1.0',
+  };
+}
+
+/**
+ * Fetch a single GitHub API endpoint and return parsed JSON.
+ * Throws on non-2xx responses.
+ * @param {string} url
+ * @param {string} token
+ * @returns {Promise<unknown>}
+ */
+async function ghFetch(url, token) {
+  const res = await fetch(url, { headers: buildHeaders(token) });
+  if (!res.ok) {
+    const body = await res.text().catch(() => '');
+    throw new Error(`GitHub API ${res.status} for ${url}: ${body}`);
+  }
+  return res.json();
+}
+
+/**
+ * Fetch all traffic data for a repository.
+ *
+ * @param {string} token  - GitHub personal access token
+ * @param {string} [repo] - owner/repo slug
+ * @returns {Promise<{
+ *   views: object,
+ *   clones: object,
+ *   referrers: object[],
+ *   repoStats: object
+ * }>}
+ */
+async function fetchGitHubTraffic(token, repo) {
+  if (!token) {
+    throw new Error('GITHUB_TOKEN is required');
+  }
+  const slug = repo || DEFAULT_REPO;
+  const base = `${GITHUB_API_BASE}/repos/${slug}`;
+
+  console.log(`[github] Fetching traffic for ${slug}`);
+
+  const [views, clones, referrers, repoStats] = await Promise.all([
+    ghFetch(`${base}/traffic/views`, token),
+    ghFetch(`${base}/traffic/clones`, token),
+    ghFetch(`${base}/traffic/popular/referrers`, token),
+    ghFetch(base, token),
+  ]);
+
+  console.log(
+    `[github] views=${views.count} uniques=${views.uniques} ` +
+      `clones=${clones.count} stars=${repoStats.stargazers_count} ` +
+      `forks=${repoStats.forks_count}`
+  );
+
+  return { views, clones, referrers, repoStats };
+}
+
+/**
+ * Main polling entry point.
+ *
+ * Fetches GitHub traffic, normalizes each daily view entry, and upserts
+ * into the analytics database. Also records a follower_snapshot using
+ * stargazers_count as the follower-count equivalent.
+ *
+ * @param {import('better-sqlite3').Database} db
+ * @returns {Promise<void>}
+ */
+async function pollGitHub(db) {
+  const token = process.env.GITHUB_TOKEN;
+  if (!token) {
+    throw new Error('GITHUB_TOKEN environment variable is required');
+  }
+  const repo = process.env.GITHUB_REPO || DEFAULT_REPO;
+
+  const { views, clones, referrers, repoStats } = await fetchGitHubTraffic(token, repo);
+
+  // Lazy-require sibling modules so they can be built/tested independently.
+  const { normalizeGitHubMetric: normalizeMetric } = require('../normalizer');
+  const { upsertMetric, upsertFollowerSnapshot } = require('../store');
+
+  const repoName = repo.split('/').pop();
+  const fetchedAt = new Date().toISOString();
+
+  // Build a date-keyed map of clone counts for merge into view records.
+  const clonesByDate = {};
+  if (Array.isArray(clones.views)) {
+    for (const entry of clones.views) {
+      const date = entry.timestamp.slice(0, 10);
+      clonesByDate[date] = (clonesByDate[date] || 0) + entry.count;
+    }
+  }
+
+  const upsertedDates = [];
+
+  for (const entry of views.views || []) {
+    const metricDate = entry.timestamp.slice(0, 10);
+    const cloneCount = clonesByDate[metricDate] || 0;
+
+    const record = {
+      platform: 'github',
+      content_type: 'repo',
+      post_id: repoName,
+      post_url: `https://github.com/${repo}`,
+      metric_date: metricDate,
+      impressions: entry.count,
+      reach: entry.uniques,
+      clicks: cloneCount,
+      likes: repoStats.stargazers_count,
+      shares: repoStats.forks_count,
+      comments: 0,
+      saves: 0,
+      video_views: 0,
+      followers_delta: 0,
+      extra_json: JSON.stringify({ referrers }),
+      fetched_at: fetchedAt,
+    };
+
+    upsertMetric(db, record);
+    upsertedDates.push(metricDate);
+  }
+
+  console.log(`[github] Upserted ${upsertedDates.length} daily metric records: ${upsertedDates.join(', ')}`);
+
+  // Record follower snapshot (stars as follower-count equivalent).
+  const snapshotDate = fetchedAt.slice(0, 10);
+  upsertFollowerSnapshot(db, {
+    platform: 'github',
+    follower_count: repoStats.stargazers_count,
+    snapshot_date: snapshotDate,
+  });
+
+  console.log(
+    `[github] Follower snapshot upserted: platform=github ` +
+      `follower_count=${repoStats.stargazers_count} date=${snapshotDate}`
+  );
+}
+
+module.exports = { fetchGitHubTraffic, pollGitHub };
+
+// ---------------------------------------------------------------------------
+// Stand-alone execution
+// ---------------------------------------------------------------------------
+if (require.main === module) {
+  const Database = require('better-sqlite3');
+  const schemaPath = path.join(__dirname, '..', 'db', 'schema.sql');
+  const dbPath = path.join(__dirname, '..', 'db', 'social-analytics.db');
+  const fs = require('fs');
+
+  const db = new Database(dbPath);
+  db.pragma('busy_timeout = 3000');
+  db.pragma('journal_mode = WAL');
+
+  const schema = fs.readFileSync(schemaPath, 'utf8');
+  db.exec(schema);
+
+  pollGitHub(db)
+    .then(() => {
+      console.log('[github] Poll complete.');
+      db.close();
+    })
+    .catch((err) => {
+      console.error('[github] Poll failed:', err.message);
+      db.close();
+      process.exit(1);
+    });
+}