thumbgate 1.23.1 → 1.23.2

package/scripts/telemetry-analytics.js

@@ -255,6 +255,149 @@ function inferTrafficChannel(raw = {}, referrerHost = null) {
   return 'referral';
 }
 
+function normalizeBoolean(value) {
+  if (value === true || value === false) return value;
+  const text = normalizeText(value, 32);
+  if (!text) return false;
+  return ['1', 'true', 'yes', 'y', 'bot'].includes(text.toLowerCase());
+}
+
+function includesAnyToken(values, tokens) {
+  const haystack = values
+    .map((value) => normalizeText(value, 512))
+    .filter(Boolean)
+    .join(' ')
+    .toLowerCase();
+  if (!haystack) return false;
+  return tokens.some((token) => haystack.includes(token));
+}
+
+function addAudienceReason(reasons, code) {
+  if (!reasons.includes(code)) reasons.push(code);
+}
+
+function classifyTelemetryAudience(entry = {}, raw = {}) {
+  const reasons = [];
+  const identityValues = [
+    raw.email,
+    raw.customerEmail,
+    raw.buyerEmail,
+    raw.userEmail,
+    entry.visitorId,
+    entry.sessionId,
+    entry.traceId,
+    entry.acquisitionId,
+    entry.installId,
+  ];
+  const attributionValues = [
+    entry.source,
+    entry.utmSource,
+    entry.utmMedium,
+    entry.utmCampaign,
+    entry.utmContent,
+    entry.creator,
+    entry.offerCode,
+    entry.campaignVariant,
+    entry.referrerHost,
+  ];
+  const eventValues = [
+    entry.eventType,
+    entry.event,
+    entry.page,
+    entry.landingPath,
+    entry.ctaId,
+    entry.reasonCode,
+    entry.reasonDetail,
+  ];
+  const userAgent = normalizeText(entry.userAgent || raw.userAgent, 512) || '';
+  const hostValues = [raw.host, raw.hostname, raw.origin, raw.url, raw.pageUrl, entry.referrerHost];
+
+  if (normalizeBoolean(entry.isBot || raw.isBot) || includesAnyToken([userAgent], [
+    'bot',
+    'crawler',
+    'spider',
+    'headless',
+    'playwright',
+    'puppeteer',
+    'curl/',
+    'wget/',
+    'lighthouse',
+  ])) {
+    addAudienceReason(reasons, 'bot_or_automation_user_agent');
+  }
+
+  if (includesAnyToken(identityValues, [
+    '@example.com',
+    'buyer@example.com',
+    'test@example.com',
+    'codex-verification',
+    'audit@thumbgate.ai',
+  ])) {
+    addAudienceReason(reasons, 'test_identity');
+  }
+
+  if (includesAnyToken([...attributionValues, ...eventValues], [
+    'codex',
+    'audit',
+    'verification',
+    'synthetic',
+    'smoke',
+    'probe',
+    'test',
+  ])) {
+    addAudienceReason(reasons, 'test_or_audit_attribution');
+  }
+
+  if (includesAnyToken(hostValues, [
+    'localhost',
+    '127.0.0.1',
+    '::1',
+    'thumbgate-production.up.railway.app',
+  ])) {
+    addAudienceReason(reasons, 'internal_host');
+  }
+
+  if (includesAnyToken([userAgent, ...identityValues], [
+    'codex',
+    'github-actions',
+    'node-fetch',
+    'thumbgate-analytics',
+    'thumbgate-verification',
+  ])) {
+    addAudienceReason(reasons, 'internal_operator_or_ci');
+  }
+
+  if (
+    (entry.clientType || entry.client) === 'unknown' &&
+    !pickFirstText(entry.visitorId, entry.sessionId, entry.acquisitionId, entry.installId, entry.traceId) &&
+    !userAgent &&
+    includesAnyToken([entry.source, entry.utmSource], ['direct', 'website'])
+  ) {
+    addAudienceReason(reasons, 'unknown_direct_no_identity');
+  }
+
+  const audience = reasons.includes('test_identity') || reasons.includes('test_or_audit_attribution')
+    ? 'test'
+    : (reasons.includes('bot_or_automation_user_agent')
+      ? 'bot'
+      : (
+        reasons.includes('internal_host') ||
+        reasons.includes('internal_operator_or_ci') ||
+        reasons.includes('unknown_direct_no_identity')
+          ? 'internal'
+          : 'external'
+      ));
+
+  return {
+    audience,
+    reasons,
+    isExternal: audience === 'external',
+    isInternal: audience === 'internal',
+    isTest: audience === 'test',
+    isBot: audience === 'bot',
+  };
+}
+
 function sanitizeTelemetryPayload(payload = {}, headers = {}) {
   const raw = payload && typeof payload === 'object' ? payload : {};
   const clientType = inferClientType(raw);
@@ -342,6 +485,15 @@ function sanitizeTelemetryPayload(payload = {}, headers = {}) {
     ),
   };
 
+  const audience = classifyTelemetryAudience(entry, raw);
+  entry.audience = audience.audience;
+  entry.trafficAudience = audience.audience;
+  entry.audienceReasons = audience.reasons;
+  entry.isExternal = audience.isExternal;
+  entry.isInternal = audience.isInternal;
+  entry.isTest = audience.isTest;
+  entry.isBotTraffic = audience.isBot;
+
   return entry;
 }
 
@@ -432,9 +584,162 @@ function summarizeRecentEvents(events) {
       community: entry.community || null,
       offerCode: entry.offerCode || null,
       campaignVariant: entry.campaignVariant || null,
+      audience: entry.audience || 'external',
+      audienceReasons: entry.audienceReasons || [],
     }));
 }
 
+function summarizeExternalTrafficQuality(events) {
+  const byAudience = {};
+  const byExclusionReason = {};
+  const externalVisitors = new Set();
+  const externalSessions = new Set();
+  const externalCheckoutStarters = new Set();
+  const pageViewsBySource = {};
+  const pageViewsByPath = {};
+  const pageViewsByTrafficChannel = {};
+  const checkoutStartsBySource = {};
+  const checkoutStartsByTrafficChannel = {};
+  const buyerLossReasons = {};
+  const visitorPaths = new Map();
+  let externalEvents = 0;
+  let excludedEvents = 0;
+  let externalWebEvents = 0;
+  let externalPageViews = 0;
+  let externalCtaClicks = 0;
+  let externalCheckoutStarts = 0;
+  let externalBuyerLossSignals = 0;
+
+  for (const entry of events) {
+    const audience = entry.audience || 'external';
+    incrementCounter(byAudience, audience);
+    if (audience !== 'external') {
+      excludedEvents += 1;
+      const reasons = Array.isArray(entry.audienceReasons) && entry.audienceReasons.length > 0
+        ? entry.audienceReasons
+        : [`${audience}_traffic`];
+      for (const reason of reasons) incrementCounter(byExclusionReason, reason);
+      continue;
+    }
+
+    externalEvents += 1;
+    if ((entry.clientType || entry.client) !== 'web') continue;
+
+    externalWebEvents += 1;
+    const visitorKey = pickFirstText(entry.visitorId, entry.installId, entry.sessionId);
+    if (visitorKey) externalVisitors.add(visitorKey);
+    if (entry.sessionId) externalSessions.add(entry.sessionId);
+
+    const eventType = entry.eventType || entry.event;
+    if (visitorKey) {
+      const pathRow = visitorPaths.get(visitorKey) || {
+        visitorKey,
+        firstSeenAt: entry.receivedAt || null,
+        lastSeenAt: entry.receivedAt || null,
+        source: entry.source || null,
+        trafficChannel: entry.trafficChannel || null,
+        events: [],
+        pages: [],
+        checkoutStarts: 0,
+      };
+      if (!pathRow.firstSeenAt || String(entry.receivedAt || '') < pathRow.firstSeenAt) {
+        pathRow.firstSeenAt = entry.receivedAt || null;
+      }
+      if (!pathRow.lastSeenAt || String(entry.receivedAt || '') > pathRow.lastSeenAt) {
+        pathRow.lastSeenAt = entry.receivedAt || null;
+      }
+      pathRow.events.push({
+        at: entry.receivedAt || null,
+        eventType,
+        page: entry.page || entry.landingPath || null,
+        ctaId: entry.ctaId || null,
+      });
+      if ((entry.page || entry.landingPath) && !pathRow.pages.includes(entry.page || entry.landingPath)) {
+        pathRow.pages.push(entry.page || entry.landingPath);
+      }
+      if (eventType === 'checkout_start' || eventType === 'checkout_bootstrap') {
+        pathRow.checkoutStarts += 1;
+      }
+      visitorPaths.set(visitorKey, pathRow);
+    }
+
+    if (eventType === 'landing_page_view') {
+      externalPageViews += 1;
+      incrementCounter(pageViewsBySource, entry.source);
+      incrementCounter(pageViewsByPath, entry.page);
+      incrementCounter(pageViewsByTrafficChannel, entry.trafficChannel);
+    }
+
+    if (isMarketingClickEvent(eventType)) {
+      externalCtaClicks += 1;
+    }
+
+    if (eventType === 'checkout_start' || eventType === 'checkout_bootstrap') {
+      externalCheckoutStarts += 1;
+      incrementCounter(checkoutStartsBySource, entry.source);
+      incrementCounter(checkoutStartsByTrafficChannel, entry.trafficChannel);
+      const starterKey = pickFirstText(
+        entry.acquisitionId,
+        entry.visitorId,
+        entry.sessionId,
+        entry.installId,
+        entry.traceId
+      );
+      if (starterKey) externalCheckoutStarters.add(starterKey);
+    }
+
+    if (eventType === 'checkout_cancelled' || eventType === 'checkout_abandoned' || eventType === 'reason_not_buying') {
+      externalBuyerLossSignals += 1;
+      incrementCounter(buyerLossReasons, entry.reasonCode);
+    }
+  }
+
+  const topPath = getTopCounterEntry(pageViewsByPath);
+  const topSource = getTopCounterEntry(pageViewsBySource);
+  const topTrafficChannel = getTopCounterEntry(pageViewsByTrafficChannel);
+  const topExclusionReason = getTopCounterEntry(byExclusionReason);
+
+  return {
+    rawEvents: events.length,
+    externalEvents,
+    excludedEvents,
+    internalEvents: byAudience.internal || 0,
+    testEvents: byAudience.test || 0,
+    botEvents: byAudience.bot || 0,
+    exclusionRate: safeRate(excludedEvents, events.length),
+    byAudience,
+    byExclusionReason,
+    topExclusionReason: topExclusionReason ? { key: topExclusionReason[0], count: topExclusionReason[1] } : null,
+    external: {
+      totalEvents: externalWebEvents,
+      uniqueVisitors: externalVisitors.size,
+      uniqueSessions: externalSessions.size,
+      uniqueCheckoutStarters: externalCheckoutStarters.size,
+      pageViews: externalPageViews,
+      ctaClicks: externalCtaClicks,
+      checkoutStarts: externalCheckoutStarts,
+      buyerLossSignals: externalBuyerLossSignals,
+      pageViewToCheckoutRate: safeRate(externalCheckoutStarts, externalPageViews),
+      visitorToCheckoutRate: safeRate(externalCheckoutStarts, externalVisitors.size),
+      bySource: pageViewsBySource,
+      byPath: pageViewsByPath,
+      byTrafficChannel: pageViewsByTrafficChannel,
+      checkoutStartsBySource,
+      checkoutStartsByTrafficChannel,
+      buyerLossReasons,
+      topSource: topSource ? { key: topSource[0], count: topSource[1] } : null,
+      topPath: topPath ? { key: topPath[0], count: topPath[1] } : null,
+      topTrafficChannel: topTrafficChannel ? { key: topTrafficChannel[0], count: topTrafficChannel[1] } : null,
+      visitorPaths: Array.from(visitorPaths.values())
+        .sort((a, b) => String(a.firstSeenAt || '').localeCompare(String(b.firstSeenAt || '')))
+        .slice(0, 50),
+    },
+    verdict: externalEvents > 0
+      ? (excludedEvents > externalEvents ? 'polluted_but_usable' : 'usable')
+      : (events.length > 0 ? 'polluted_no_external_signal' : 'missing'),
+  };
+}
+
 function getTelemetrySummary(feedbackDir, options = {}) {
   const analyticsWindow = resolveAnalyticsWindow(options);
   const telemetryLoadOptions = analyticsWindow.bounded
@@ -445,6 +750,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
     analyticsWindow,
     (entry) => entry && (entry.receivedAt || entry.timestamp)
   );
+  const trafficQuality = summarizeExternalTrafficQuality(events);
   const byClientType = {};
   const byEventType = {};
   const webVisitors = new Set();
@@ -788,6 +1094,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
     window: serializeAnalyticsWindow(analyticsWindow),
     totalEvents: events.length,
     latestSeenAt,
+    trafficQuality,
     byClientType,
     byEventType,
     conversionFunnel: {
@@ -963,6 +1270,8 @@ function getTelemetryAnalytics(feedbackDir, options = {}) {
     window: summary.window,
     totalEvents: summary.totalEvents,
     latestSeenAt: summary.latestSeenAt,
+    trafficQuality: summary.trafficQuality,
+    qualified: summary.trafficQuality.external,
     byClientType: summary.byClientType,
     byEventType: summary.byEventType,
     conversionFunnel: summary.conversionFunnel,
@@ -1142,6 +1451,7 @@ const appendTelemetryPing = appendTelemetryEvent;
 module.exports = {
   TELEMETRY_FILE_NAME,
   sanitizeTelemetryPayload,
+  classifyTelemetryAudience,
   appendTelemetryPing,
   appendTelemetryEvent,
   getTelemetrySourceDiagnostics,

package/scripts/visitor-journey.js

@@ -0,0 +1,172 @@
+'use strict';
+
+const STAGE_ORDER = [
+  'landing',
+  'pricing',
+  'checkout_viewed',
+  'email_submitted',
+  'stripe_redirect',
+  'purchase',
+];
+
+function parseTimestamp(row) {
+  const raw = row && (row.timestamp || row.receivedAt || row.ts);
+  const ms = raw ? Date.parse(raw) : NaN;
+  return Number.isFinite(ms) ? ms : null;
+}
+
+function pickText(...values) {
+  for (const value of values) {
+    const text = String(value || '').trim();
+    if (text) return text;
+  }
+  return '';
+}
+
+function extractSessionId(row) {
+  return pickText(
+    row.sessionId,
+    row.visitorSessionId,
+    row.visitor_session_id,
+    row.attribution && row.attribution.sessionId,
+    row.metadata && row.metadata.sessionId,
+    row.stripeSessionId,
+    row.traceId,
+    row.acquisitionId,
+  );
+}
+
+function extractVisitorId(row) {
+  return pickText(
+    row.visitorId,
+    row.visitor_id,
+    row.installId,
+    row.attribution && row.attribution.visitorId,
+    row.metadata && row.metadata.visitorId,
+  );
+}
+
+function extractPath(row) {
+  return pickText(
+    row.path,
+    row.page,
+    row.landingPath,
+    row.landing_path,
+    row.attribution && row.attribution.landingPath,
+  );
+}
+
+function classifyStage(row) {
+  const event = String(row.eventType || row.event || '').toLowerCase();
+  const stage = String(row.stage || '').toLowerCase();
+  const path = extractPath(row).toLowerCase();
+  const cta = String(row.ctaId || row.cta_id || '').toLowerCase();
+
+  if (/purchase|paid|checkout\.session\.completed/.test(event) || stage === 'purchase') return 'purchase';
+  if (/stripe_redirect|stripe redirect|redirect started/.test(event) || stage === 'stripe_redirect') return 'stripe_redirect';
+  if (/email_submitted|email submitted|checkout_interstitial_cta_clicked/.test(event)) return 'email_submitted';
+  if (/checkout/.test(path) || /checkout.*view|checkout pro viewed/.test(event)) return 'checkout_viewed';
+  if (/pricing/.test(path) || /pricing/.test(cta) || /pricing_cta/.test(event)) return 'pricing';
+  if (/landing|page_view|landing_page_view|discovery/.test(event) || stage === 'discovery') return 'landing';
+  return null;
+}
+
+function stageRank(stage) {
+  const index = STAGE_ORDER.indexOf(stage);
+  return index === -1 ? -1 : index;
+}
+
+function nextMissingStage(maxStage) {
+  const index = stageRank(maxStage);
+  if (index < 0) return 'unknown';
+  if (maxStage === 'purchase') return 'converted';
+  return STAGE_ORDER[index + 1] || 'unknown';
+}
+
+function buildVisitorJourneySummary({ telemetryRows = [], funnelRows = [], limit = 100 } = {}) {
+  const sessions = new Map();
+  const stageCounts = Object.fromEntries(STAGE_ORDER.map((stage) => [stage, 0]));
+
+  function ingest(row, source) {
+    const ts = parseTimestamp(row);
+    if (ts == null) return;
+    const sessionId = extractSessionId(row) || `anonymous:${extractVisitorId(row) || ts}`;
+    const visitorId = extractVisitorId(row) || null;
+    const stage = classifyStage(row);
+    const path = extractPath(row);
+    const existing = sessions.get(sessionId) || {
+      sessionId,
+      visitorId,
+      firstSeen: new Date(ts).toISOString(),
+      lastSeen: new Date(ts).toISOString(),
+      maxStage: null,
+      dropoffStage: 'unknown',
+      events: [],
+      paths: [],
+      ctas: [],
+      sources: [],
+      visitorType: row.visitorType || null,
+      utm: {},
+      referrerHost: pickText(row.referrerHost, row.referrer_host, row.referrer),
+    };
+
+    existing.visitorId = existing.visitorId || visitorId;
+    existing.firstSeen = new Date(Math.min(Date.parse(existing.firstSeen), ts)).toISOString();
+    existing.lastSeen = new Date(Math.max(Date.parse(existing.lastSeen), ts)).toISOString();
+    existing.sources.push(source);
+    if (stage) {
+      stageCounts[stage] += 1;
+      if (stageRank(stage) > stageRank(existing.maxStage)) existing.maxStage = stage;
+    }
+    if (path && !existing.paths.includes(path)) existing.paths.push(path);
+    const cta = pickText(row.ctaId, row.cta_id);
+    if (cta && !existing.ctas.includes(cta)) existing.ctas.push(cta);
+    for (const key of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term']) {
+      const value = pickText(row[key], row.attribution && row.attribution[key]);
+      if (value && !existing.utm[key]) existing.utm[key] = value;
+    }
+    existing.events.push({
+      timestamp: new Date(ts).toISOString(),
+      source,
+      eventType: pickText(row.eventType, row.event, row.stage, 'unknown'),
+      stage,
+      path: path || null,
+      ctaId: cta || null,
+    });
+    sessions.set(sessionId, existing);
+  }
+
+  for (const row of telemetryRows) ingest(row, 'telemetry');
+  for (const row of funnelRows) ingest(row, 'funnel');
+
+  const journeys = [...sessions.values()].map((session) => {
+    session.sources = [...new Set(session.sources)];
+    session.events.sort((a, b) => Date.parse(a.timestamp) - Date.parse(b.timestamp));
+    session.maxStage = session.maxStage || 'unknown';
+    session.dropoffStage = nextMissingStage(session.maxStage);
+    session.eventCount = session.events.length;
+    return session;
+  }).sort((a, b) => Date.parse(b.lastSeen) - Date.parse(a.lastSeen));
+
+  const dropoffCounts = {};
+  for (const journey of journeys) {
+    dropoffCounts[journey.dropoffStage] = (dropoffCounts[journey.dropoffStage] || 0) + 1;
+  }
+
+  return {
+    generatedAt: new Date().toISOString(),
+    sessionCount: journeys.length,
+    stageCounts,
+    dropoffCounts,
+    journeys: journeys.slice(0, Math.max(1, Math.min(Number(limit) || 100, 500))),
+    truncated: journeys.length > Math.max(1, Math.min(Number(limit) || 100, 500)),
+  };
+}
+
+module.exports = {
+  STAGE_ORDER,
+  buildVisitorJourneySummary,
+  classifyStage,
+  extractSessionId,
+  extractVisitorId,
+};