thumbgate 1.27.18 → 1.27.20

package/scripts/telemetry-analytics.js

@@ -75,6 +75,13 @@ function normalizeInteger(value) {
   return Number.isFinite(parsed) ? Math.trunc(parsed) : null;
 }
 
+function normalizeRate(value) {
+  if (value === undefined || value === null || value === '') return null;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return null;
+  return Math.max(0, Math.min(parsed, 1));
+}
+
 function safeRate(num, den) {
   if (!den) return 0;
   return Number((num / den).toFixed(4));
@@ -255,6 +262,149 @@ function inferTrafficChannel(raw = {}, referrerHost = null) {
   return 'referral';
 }
 
+function normalizeBoolean(value) {
+  if (value === true || value === false) return value;
+  const text = normalizeText(value, 32);
+  if (!text) return false;
+  return ['1', 'true', 'yes', 'y', 'bot'].includes(text.toLowerCase());
+}
+
+function includesAnyToken(values, tokens) {
+  const haystack = values
+    .map((value) => normalizeText(value, 512))
+    .filter(Boolean)
+    .join(' ')
+    .toLowerCase();
+  if (!haystack) return false;
+  return tokens.some((token) => haystack.includes(token));
+}
+
+function addAudienceReason(reasons, code) {
+  if (!reasons.includes(code)) reasons.push(code);
+}
+
+function classifyTelemetryAudience(entry = {}, raw = {}) {
+  const reasons = [];
+  const identityValues = [
+    raw.email,
+    raw.customerEmail,
+    raw.buyerEmail,
+    raw.userEmail,
+    entry.visitorId,
+    entry.sessionId,
+    entry.traceId,
+    entry.acquisitionId,
+    entry.installId,
+  ];
+  const attributionValues = [
+    entry.source,
+    entry.utmSource,
+    entry.utmMedium,
+    entry.utmCampaign,
+    entry.utmContent,
+    entry.creator,
+    entry.offerCode,
+    entry.campaignVariant,
+    entry.referrerHost,
+  ];
+  const eventValues = [
+    entry.eventType,
+    entry.event,
+    entry.page,
+    entry.landingPath,
+    entry.ctaId,
+    entry.reasonCode,
+    entry.reasonDetail,
+  ];
+  const userAgent = normalizeText(entry.userAgent || raw.userAgent, 512) || '';
+  const hostValues = [raw.host, raw.hostname, raw.origin, raw.url, raw.pageUrl, entry.referrerHost];
+
+  if (normalizeBoolean(entry.isBot || raw.isBot) || includesAnyToken([userAgent], [
+    'bot',
+    'crawler',
+    'spider',
+    'headless',
+    'playwright',
+    'puppeteer',
+    'curl/',
+    'wget/',
+    'lighthouse',
+  ])) {
+    addAudienceReason(reasons, 'bot_or_automation_user_agent');
+  }
+
+  if (includesAnyToken(identityValues, [
+    '@example.com',
+    'buyer@example.com',
+    'test@example.com',
+    'codex-verification',
+    'audit@thumbgate.ai',
+  ])) {
+    addAudienceReason(reasons, 'test_identity');
+  }
+
+  if (includesAnyToken([...attributionValues, ...eventValues], [
+    'codex',
+    'audit',
+    'verification',
+    'synthetic',
+    'smoke',
+    'probe',
+    'test',
+  ])) {
+    addAudienceReason(reasons, 'test_or_audit_attribution');
+  }
+
+  if (includesAnyToken(hostValues, [
+    'localhost',
+    '127.0.0.1',
+    '::1',
+    'thumbgate-production.up.railway.app',
+  ])) {
+    addAudienceReason(reasons, 'internal_host');
+  }
+
+  if (includesAnyToken([userAgent, ...identityValues], [
+    'codex',
+    'github-actions',
+    'node-fetch',
+    'thumbgate-analytics',
+    'thumbgate-verification',
+  ])) {
+    addAudienceReason(reasons, 'internal_operator_or_ci');
+  }
+
+  if (
+    (entry.clientType || entry.client) === 'unknown' &&
+    !pickFirstText(entry.visitorId, entry.sessionId, entry.acquisitionId, entry.installId, entry.traceId) &&
+    !userAgent &&
+    includesAnyToken([entry.source, entry.utmSource], ['direct', 'website'])
+  ) {
+    addAudienceReason(reasons, 'unknown_direct_no_identity');
+  }
+
+  const audience = reasons.includes('test_identity') || reasons.includes('test_or_audit_attribution')
+    ? 'test'
+    : (reasons.includes('bot_or_automation_user_agent')
+      ? 'bot'
+      : (
+        reasons.includes('internal_host') ||
+        reasons.includes('internal_operator_or_ci') ||
+        reasons.includes('unknown_direct_no_identity')
+          ? 'internal'
+          : 'external'
+      ));
+
+  return {
+    audience,
+    reasons,
+    isExternal: audience === 'external',
+    isInternal: audience === 'internal',
+    isTest: audience === 'test',
+    isBot: audience === 'bot',
+  };
+}
+
 function sanitizeTelemetryPayload(payload = {}, headers = {}) {
   const raw = payload && typeof payload === 'object' ? payload : {};
   const clientType = inferClientType(raw);
@@ -313,6 +463,12 @@ function sanitizeTelemetryPayload(payload = {}, headers = {}) {
     pipelineStatus: pickFirstText(raw.pipelineStatus, raw.workflowSprintStatus, raw.status),
     reasonCode,
     reasonDetail: pickFirstText(raw.reasonDetail, raw.reasonText, raw.otherReason, raw.notes),
+    integration: pickFirstText(raw.integration, raw.actionIntegration),
+    actionOperation: pickFirstText(raw.actionOperation, raw.operationId),
+    endpoint: pickFirstText(raw.endpoint, raw.apiPath),
+    decisionMode: pickFirstText(raw.decisionMode, raw.executionMode),
+    actionStatus: pickFirstText(raw.actionStatus, raw.status),
+    accepted: raw.accepted === undefined || raw.accepted === null ? null : Boolean(raw.accepted),
     pricingInterest: pickFirstText(raw.pricingInterest, raw.interestLevel),
     seoQuery: pickFirstText(raw.seoQuery, raw.query),
     seoSurface: pickFirstText(raw.seoSurface, raw.searchSurface, raw.surface),
@@ -331,11 +487,22 @@ function sanitizeTelemetryPayload(payload = {}, headers = {}) {
     httpStatus: normalizeInteger(raw.httpStatus),
     userAgent: pickFirstText(raw.userAgent, headers['user-agent']),
     isBot: pickFirstText(raw.isBot),
+    interstitialSampled: pickFirstText(raw.interstitialSampled),
+    interstitialSampleRate: normalizeRate(raw.interstitialSampleRate),
     attributionTagged: Boolean(
       pickFirstText(raw.utmSource, raw.utmMedium, raw.utmCampaign, raw.utmContent, raw.utmTerm)
     ),
   };
 
+  const audience = classifyTelemetryAudience(entry, raw);
+  entry.audience = audience.audience;
+  entry.trafficAudience = audience.audience;
+  entry.audienceReasons = audience.reasons;
+  entry.isExternal = audience.isExternal;
+  entry.isInternal = audience.isInternal;
+  entry.isTest = audience.isTest;
+  entry.isBotTraffic = audience.isBot;
+
   return entry;
 }
 
@@ -426,9 +593,162 @@ function summarizeRecentEvents(events) {
       community: entry.community || null,
       offerCode: entry.offerCode || null,
       campaignVariant: entry.campaignVariant || null,
+      audience: entry.audience || 'external',
+      audienceReasons: entry.audienceReasons || [],
     }));
 }
 
+function summarizeExternalTrafficQuality(events) {
+  const byAudience = {};
+  const byExclusionReason = {};
+  const externalVisitors = new Set();
+  const externalSessions = new Set();
+  const externalCheckoutStarters = new Set();
+  const pageViewsBySource = {};
+  const pageViewsByPath = {};
+  const pageViewsByTrafficChannel = {};
+  const checkoutStartsBySource = {};
+  const checkoutStartsByTrafficChannel = {};
+  const buyerLossReasons = {};
+  const visitorPaths = new Map();
+  let externalEvents = 0;
+  let excludedEvents = 0;
+  let externalWebEvents = 0;
+  let externalPageViews = 0;
+  let externalCtaClicks = 0;
+  let externalCheckoutStarts = 0;
+  let externalBuyerLossSignals = 0;
+
+  for (const entry of events) {
+    const audience = entry.audience || 'external';
+    incrementCounter(byAudience, audience);
+    if (audience !== 'external') {
+      excludedEvents += 1;
+      const reasons = Array.isArray(entry.audienceReasons) && entry.audienceReasons.length > 0
+        ? entry.audienceReasons
+        : [`${audience}_traffic`];
+      for (const reason of reasons) incrementCounter(byExclusionReason, reason);
+      continue;
+    }
+
+    externalEvents += 1;
+    if ((entry.clientType || entry.client) !== 'web') continue;
+
+    externalWebEvents += 1;
+    const visitorKey = pickFirstText(entry.visitorId, entry.installId, entry.sessionId);
+    if (visitorKey) externalVisitors.add(visitorKey);
+    if (entry.sessionId) externalSessions.add(entry.sessionId);
+
+    const eventType = entry.eventType || entry.event;
+    if (visitorKey) {
+      const pathRow = visitorPaths.get(visitorKey) || {
+        visitorKey,
+        firstSeenAt: entry.receivedAt || null,
+        lastSeenAt: entry.receivedAt || null,
+        source: entry.source || null,
+        trafficChannel: entry.trafficChannel || null,
+        events: [],
+        pages: [],
+        checkoutStarts: 0,
+      };
+      if (!pathRow.firstSeenAt || String(entry.receivedAt || '') < pathRow.firstSeenAt) {
+        pathRow.firstSeenAt = entry.receivedAt || null;
+      }
+      if (!pathRow.lastSeenAt || String(entry.receivedAt || '') > pathRow.lastSeenAt) {
+        pathRow.lastSeenAt = entry.receivedAt || null;
+      }
+      pathRow.events.push({
+        at: entry.receivedAt || null,
+        eventType,
+        page: entry.page || entry.landingPath || null,
+        ctaId: entry.ctaId || null,
+      });
+      if ((entry.page || entry.landingPath) && !pathRow.pages.includes(entry.page || entry.landingPath)) {
+        pathRow.pages.push(entry.page || entry.landingPath);
+      }
+      if (eventType === 'checkout_start' || eventType === 'checkout_bootstrap') {
+        pathRow.checkoutStarts += 1;
+      }
+      visitorPaths.set(visitorKey, pathRow);
+    }
+
+    if (eventType === 'landing_page_view') {
+      externalPageViews += 1;
+      incrementCounter(pageViewsBySource, entry.source);
+      incrementCounter(pageViewsByPath, entry.page);
+      incrementCounter(pageViewsByTrafficChannel, entry.trafficChannel);
+    }
+
+    if (isMarketingClickEvent(eventType)) {
+      externalCtaClicks += 1;
+    }
+
+    if (eventType === 'checkout_start' || eventType === 'checkout_bootstrap') {
+      externalCheckoutStarts += 1;
+      incrementCounter(checkoutStartsBySource, entry.source);
+      incrementCounter(checkoutStartsByTrafficChannel, entry.trafficChannel);
+      const starterKey = pickFirstText(
+        entry.acquisitionId,
+        entry.visitorId,
+        entry.sessionId,
+        entry.installId,
+        entry.traceId
+      );
+      if (starterKey) externalCheckoutStarters.add(starterKey);
+    }
+
+    if (eventType === 'checkout_cancelled' || eventType === 'checkout_abandoned' || eventType === 'reason_not_buying') {
+      externalBuyerLossSignals += 1;
+      incrementCounter(buyerLossReasons, entry.reasonCode);
+    }
+  }
+
+  const topPath = getTopCounterEntry(pageViewsByPath);
+  const topSource = getTopCounterEntry(pageViewsBySource);
+  const topTrafficChannel = getTopCounterEntry(pageViewsByTrafficChannel);
+  const topExclusionReason = getTopCounterEntry(byExclusionReason);
+
+  return {
+    rawEvents: events.length,
+    externalEvents,
+    excludedEvents,
+    internalEvents: byAudience.internal || 0,
+    testEvents: byAudience.test || 0,
+    botEvents: byAudience.bot || 0,
+    exclusionRate: safeRate(excludedEvents, events.length),
+    byAudience,
+    byExclusionReason,
+    topExclusionReason: topExclusionReason ? { key: topExclusionReason[0], count: topExclusionReason[1] } : null,
+    external: {
+      totalEvents: externalWebEvents,
+      uniqueVisitors: externalVisitors.size,
+      uniqueSessions: externalSessions.size,
+      uniqueCheckoutStarters: externalCheckoutStarters.size,
+      pageViews: externalPageViews,
+      ctaClicks: externalCtaClicks,
+      checkoutStarts: externalCheckoutStarts,
+      buyerLossSignals: externalBuyerLossSignals,
+      pageViewToCheckoutRate: safeRate(externalCheckoutStarts, externalPageViews),
+      visitorToCheckoutRate: safeRate(externalCheckoutStarts, externalVisitors.size),
+      bySource: pageViewsBySource,
+      byPath: pageViewsByPath,
+      byTrafficChannel: pageViewsByTrafficChannel,
+      checkoutStartsBySource,
+      checkoutStartsByTrafficChannel,
+      buyerLossReasons,
+      topSource: topSource ? { key: topSource[0], count: topSource[1] } : null,
+      topPath: topPath ? { key: topPath[0], count: topPath[1] } : null,
+      topTrafficChannel: topTrafficChannel ? { key: topTrafficChannel[0], count: topTrafficChannel[1] } : null,
+      visitorPaths: Array.from(visitorPaths.values())
+        .sort((a, b) => String(a.firstSeenAt || '').localeCompare(String(b.firstSeenAt || '')))
+        .slice(0, 50),
+    },
+    verdict: externalEvents > 0
+      ? (excludedEvents > externalEvents ? 'polluted_but_usable' : 'usable')
+      : (events.length > 0 ? 'polluted_no_external_signal' : 'missing'),
+  };
+}
+
 function getTelemetrySummary(feedbackDir, options = {}) {
   const analyticsWindow = resolveAnalyticsWindow(options);
   const telemetryLoadOptions = analyticsWindow.bounded
@@ -439,6 +759,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
     analyticsWindow,
     (entry) => entry && (entry.receivedAt || entry.timestamp)
   );
+  const trafficQuality = summarizeExternalTrafficQuality(events);
   const byClientType = {};
   const byEventType = {};
   const webVisitors = new Set();
@@ -526,6 +847,12 @@ function getTelemetrySummary(feedbackDir, options = {}) {
   let exitEngagementMsCount = 0;
   let exitScrollPercentTotal = 0;
   let exitScrollPercentCount = 0;
+  let chatgptActionCalls = 0;
+  let chatgptActionAccepted = 0;
+  const chatgptActionsByOperation = {};
+  const chatgptActionsByEndpoint = {};
+  const chatgptActionsByStatus = {};
+  const chatgptActionsByDecisionMode = {};
 
   for (const entry of events) {
     incrementCounter(byClientType, entry.clientType || entry.client || 'unknown');
@@ -714,6 +1041,18 @@ function getTelemetrySummary(feedbackDir, options = {}) {
       }
     }
 
+    if (
+      String(entry.eventType || entry.event || '').startsWith('chatgpt_action_') ||
+      entry.integration === 'chatgpt_gpt'
+    ) {
+      chatgptActionCalls += 1;
+      if (entry.accepted === true) chatgptActionAccepted += 1;
+      incrementCounter(chatgptActionsByOperation, entry.actionOperation);
+      incrementCounter(chatgptActionsByEndpoint, entry.endpoint);
+      incrementCounter(chatgptActionsByStatus, entry.actionStatus);
+      incrementCounter(chatgptActionsByDecisionMode, entry.decisionMode);
+    }
+
     if ((entry.clientType || entry.client) === 'cli') {
       if (entry.installId) cliInstalls.add(entry.installId);
       incrementCounter(cliByPlatform, entry.platform);
@@ -764,12 +1103,14 @@ function getTelemetrySummary(feedbackDir, options = {}) {
     window: serializeAnalyticsWindow(analyticsWindow),
     totalEvents: events.length,
     latestSeenAt,
+    trafficQuality,
     byClientType,
     byEventType,
     conversionFunnel: {
       landingViews: pageViews,
       installCopies,
       gptOpens,
+      gptActionCalls: chatgptActionCalls,
       checkoutStarts,
       checkoutInterstitialViews,
       checkoutInterstitialClicks,
@@ -777,6 +1118,7 @@ function getTelemetrySummary(feedbackDir, options = {}) {
       proConversions,
       landingToInstallCopyRate: safeRate(installCopies, pageViews),
       landingToGptOpenRate: safeRate(gptOpens, pageViews),
+      gptOpenToActionRate: safeRate(chatgptActionCalls, gptOpens),
       landingToCheckoutRate: safeRate(checkoutStarts, pageViews),
       checkoutInterstitialClickRate: safeRate(checkoutInterstitialClicks, checkoutInterstitialViews),
       checkoutInterstitialProConfirmRate: safeRate(checkoutInterstitialProConfirms, checkoutInterstitialViews),
@@ -835,6 +1177,17 @@ function getTelemetrySummary(feedbackDir, options = {}) {
       byPlatform: cliByPlatform,
       byVersion: cliByVersion,
     },
+    chatgpt: {
+      gptOpens,
+      actionCalls: chatgptActionCalls,
+      acceptedActionCalls: chatgptActionAccepted,
+      openToActionRate: safeRate(chatgptActionCalls, gptOpens),
+      acceptedActionRate: safeRate(chatgptActionAccepted, chatgptActionCalls),
+      byOperation: chatgptActionsByOperation,
+      byEndpoint: chatgptActionsByEndpoint,
+      byStatus: chatgptActionsByStatus,
+      byDecisionMode: chatgptActionsByDecisionMode,
+    },
     marketing: {
       pageViewsBySource,
       pageViewsByCampaign,
@@ -926,9 +1279,12 @@ function getTelemetryAnalytics(feedbackDir, options = {}) {
     window: summary.window,
     totalEvents: summary.totalEvents,
     latestSeenAt: summary.latestSeenAt,
+    trafficQuality: summary.trafficQuality,
+    qualified: summary.trafficQuality.external,
     byClientType: summary.byClientType,
     byEventType: summary.byEventType,
     conversionFunnel: summary.conversionFunnel,
+    chatgpt: summary.chatgpt,
     visitors: {
       totalEvents: summary.web.totalEvents,
       uniqueVisitors: summary.web.uniqueVisitors,
@@ -1104,6 +1460,7 @@ const appendTelemetryPing = appendTelemetryEvent;
 module.exports = {
   TELEMETRY_FILE_NAME,
   sanitizeTelemetryPayload,
+  classifyTelemetryAudience,
   appendTelemetryPing,
   appendTelemetryEvent,
   getTelemetrySourceDiagnostics,

package/scripts/thompson-sampling.js

@@ -302,24 +302,44 @@ function getCalibration(model) {
 // ---------------------------------------------------------------------------
 
 /**
- * Draw one sample from the Beta posterior for each category via the
- * Marsaglia-Tsang (2000) gamma ratio method. No external library needed.
+ * Return the Beta posterior parameters after applying Thompson temperature
+ * scaling. The posterior mean is preserved while precision changes:
+ * lower temperatures sharpen the posterior, higher temperatures flatten it.
  *
- * betaSample(alpha, beta) = gammaSample(alpha) / (gammaSample(alpha) + gammaSample(beta))
+ * @param {Object} params - Category posterior parameters
+ * @param {number} temperature - Scaling factor (default 1.0)
+ * @returns {{ alpha: number, beta: number }}
+ */
+function getTemperatureScaledPosteriorParams(params, temperature = 1.0) {
+  const T = Math.max(0.01, Number(temperature) || 1.0);
+  const invT = 1.0 / T;
+  return {
+    alpha: Math.max(params.alpha * invT, 0.01),
+    beta: Math.max(params.beta * invT, 0.01),
+  };
+}
+
+/**
+ * Draw one sample from the Beta posterior for each category.
+ * Supports temperature scaling to adjust exploitation vs exploration.
  *
- * This is the JS equivalent of Python's random.betavariate(alpha, beta).
- * Used for Thompson Sampling action selection (explore via uncertainty).
+ * Temperature (T):
+ *   T = 1.0 (default) — Standard Thompson Sampling.
+ *   T < 1.0 — Sharper distribution, favors high-reliability categories (exploit).
+ *   T > 1.0 — Flatter distribution, increases uncertainty and exploration.
+ *
+ * Implementation: Scales alpha and beta by 1/T.
  *
  * @param {Object} model - Model object containing categories
+ * @param {number} temperature - Scaling factor (default 1.0)
  * @returns {Object} Map of category → float sample in [0, 1]
  */
-function samplePosteriors(model) {
+function samplePosteriors(model, temperature = 1.0) {
   const samples = {};
+
   for (const [cat, params] of Object.entries(model.categories || {})) {
-    samples[cat] = betaSample(
-      Math.max(params.alpha, 0.01),
-      Math.max(params.beta, 0.01),
-    );
+    const { alpha, beta } = getTemperatureScaledPosteriorParams(params, temperature);
+    samples[cat] = betaSample(alpha, beta);
   }
   return samples;
 }
@@ -451,6 +471,7 @@ module.exports = {
   getReliability,
   isCalibrated,
   getCalibration,
+  getTemperatureScaledPosteriorParams,
   samplePosteriors,
   argmaxPosteriors,
   pickBestCategory,

package/scripts/thumbgate-bench.js

@@ -4,6 +4,7 @@
 const fs = require('node:fs');
 const os = require('node:os');
 const path = require('node:path');
+const { expandFixturePlaceholders } = require('./secret-fixture-tokens');
 
 const ROOT = path.join(__dirname, '..');
 const DEFAULT_SUITE_PATH = path.join(ROOT, 'bench', 'thumbgate-bench.json');
@@ -180,6 +181,20 @@ function assertObject(value, label) {
   }
 }
 
+function expandScenarioFixturePlaceholders(value) {
+  if (typeof value === 'string') return expandFixturePlaceholders(value);
+  if (Array.isArray(value)) return value.map(expandScenarioFixturePlaceholders);
+  if (value && typeof value === 'object') {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, nestedValue]) => [
+        key,
+        expandScenarioFixturePlaceholders(nestedValue),
+      ]),
+    );
+  }
+  return value;
+}
+
 function loadScenarioSuite(filePath = DEFAULT_SUITE_PATH) {
   const suite = readJson(filePath);
   assertObject(suite, 'Scenario suite');
@@ -202,7 +217,7 @@ function loadScenarioSuite(filePath = DEFAULT_SUITE_PATH) {
       throw new Error(`Scenario ${id} has invalid expectedDecision`);
     }
     return {
-      ...scenario,
+      ...expandScenarioFixturePlaceholders(scenario),
       id,
       unsafe: Boolean(scenario.unsafe),
       positivePattern: Boolean(scenario.positivePattern),

package/scripts/thumbgate-search.js

@@ -151,8 +151,67 @@ function sortResults(results) {
   });
 }
 
-function getFeedbackResults(query, limit, signal) {
-  const results = searchFeedbackLog(query, Math.max(limit * 3, limit));
+function extractFeedbackId(str) {
+  if (!str) return null;
+  const match = str.match(/fb[_-]\d+[_-][a-z0-9]+/i);
+  return match ? match[0].replace(/-/g, '_').toLowerCase() : null;
+}
+
+function deduplicateResults(results) {
+  const bestByFeedbackId = new Map();
+
+  for (const r of results) {
+    const feedId = extractFeedbackId(r.id || r.title || r.file || '');
+    if (feedId) {
+      const existing = bestByFeedbackId.get(feedId);
+      if (!existing) {
+        bestByFeedbackId.set(feedId, r);
+      } else {
+        const sourceOrder = { feedback: 4, contextfs: 3, prevention_rule: 2, document: 1 };
+        const existingOrder = sourceOrder[existing.source] || 0;
+        const currentOrder = sourceOrder[r.source] || 0;
+        if (currentOrder > existingOrder) {
+          bestByFeedbackId.set(feedId, r);
+        } else if (currentOrder === existingOrder && (r.score || 0) > (existing.score || 0)) {
+          bestByFeedbackId.set(feedId, r);
+        }
+      }
+    }
+  }
+
+  const finalResults = [];
+  const seenContent = new Set();
+  const seenIds = new Set();
+
+  for (const r of results) {
+    const feedId = extractFeedbackId(r.id || r.title || r.file || '');
+    let recordToUse = r;
+    if (feedId) {
+      recordToUse = bestByFeedbackId.get(feedId);
+      if (seenIds.has(recordToUse.id)) continue;
+    } else {
+      if (r.id && r.id !== 'null' && String(r.id).trim() !== '') {
+        if (seenIds.has(r.id)) continue;
+      }
+    }
+
+    const normTitle = String(recordToUse.title || '').trim().toLowerCase();
+    const normContext = String(recordToUse.context || '').trim().toLowerCase();
+    const contentKey = `${normTitle}|${normContext}`;
+    if (seenContent.has(contentKey)) continue;
+
+    if (recordToUse.id && recordToUse.id !== 'null' && String(recordToUse.id).trim() !== '') {
+      seenIds.add(recordToUse.id);
+    }
+    seenContent.add(contentKey);
+    finalResults.push(recordToUse);
+  }
+
+  return finalResults;
+}
+
+function getFeedbackResults(query, limit, signal, feedbackDir) {
+  const results = searchFeedbackLog(query, Math.max(limit * 3, limit), { feedbackDir });
   const normalizedSignal = normalizeSignal(signal);
   const filtered = normalizedSignal
     ? results.filter((record) => normalizeRecordSignal(record.signal) === normalizedSignal)
@@ -160,19 +219,19 @@ function getFeedbackResults(query, limit, signal) {
   return filtered.slice(0, limit).map(mapFeedbackResult);
 }
 
-function getContextResults(query, limit) {
-  return searchContextFs(query, limit).map(mapContextResult);
+function getContextResults(query, limit, feedbackDir) {
+  return searchContextFs(query, limit, { feedbackDir }).map(mapContextResult);
 }
 
-function getRuleResults(query, limit) {
-  return searchPreventionRulesSync(query, limit).map(mapRuleResult);
+function getRuleResults(query, limit, feedbackDir) {
+  return searchPreventionRulesSync(query, limit, { feedbackDir }).map(mapRuleResult);
 }
 
-function getDocumentResults(query, limit) {
-  return searchImportedDocuments({ query, limit }).map(mapDocumentResult);
+function getDocumentResults(query, limit, feedbackDir) {
+  return searchImportedDocuments({ query, limit, feedbackDir }).map(mapDocumentResult);
 }
 
-function searchThumbgate({ query, source = 'all', limit = 10, signal = null } = {}) {
+function searchThumbgate({ query, source = 'all', limit = 10, signal = null, feedbackDir = null } = {}) {
   const trimmedQuery = String(query || '').trim();
   if (!trimmedQuery) {
     throw new Error('query is required');
@@ -182,22 +241,29 @@ function searchThumbgate({ query, source = 'all', limit = 10, signal = null } =
   const normalizedSignal = normalizeSignal(signal);
   const normalizedLimit = normalizeLimit(limit);
 
+  const fetchLimit = Math.max(100, normalizedLimit * 5);
+
   let results = [];
   if (normalizedSource === 'feedback') {
-    results = getFeedbackResults(trimmedQuery, normalizedLimit, normalizedSignal);
+    const raw = getFeedbackResults(trimmedQuery, fetchLimit, normalizedSignal, feedbackDir);
+    results = deduplicateResults(raw).slice(0, normalizedLimit);
   } else if (normalizedSource === 'context') {
-    results = getContextResults(trimmedQuery, normalizedLimit);
+    const raw = getContextResults(trimmedQuery, fetchLimit, feedbackDir);
+    results = deduplicateResults(raw).slice(0, normalizedLimit);
   } else if (normalizedSource === 'rules') {
-    results = getRuleResults(trimmedQuery, normalizedLimit);
+    const raw = getRuleResults(trimmedQuery, fetchLimit, feedbackDir);
+    results = deduplicateResults(raw).slice(0, normalizedLimit);
   } else if (normalizedSource === 'documents') {
-    results = getDocumentResults(trimmedQuery, normalizedLimit);
+    const raw = getDocumentResults(trimmedQuery, fetchLimit, feedbackDir);
+    results = deduplicateResults(raw).slice(0, normalizedLimit);
   } else {
-    results = sortResults([
-      ...getFeedbackResults(trimmedQuery, normalizedLimit, normalizedSignal),
-      ...getContextResults(trimmedQuery, normalizedLimit),
-      ...getRuleResults(trimmedQuery, normalizedLimit),
-      ...getDocumentResults(trimmedQuery, normalizedLimit),
-    ]).slice(0, normalizedLimit);
+    const combined = [
+      ...getFeedbackResults(trimmedQuery, fetchLimit, normalizedSignal, feedbackDir),
+      ...getContextResults(trimmedQuery, fetchLimit, feedbackDir),
+      ...getRuleResults(trimmedQuery, fetchLimit, feedbackDir),
+      ...getDocumentResults(trimmedQuery, fetchLimit, feedbackDir),
+    ];
+    results = deduplicateResults(sortResults(combined)).slice(0, normalizedLimit);
   }
 
   return {