thumbgate 1.27.6 → 1.27.8

package/scripts/statusline-cache-path.js

@@ -1,7 +1,12 @@
 #!/usr/bin/env node
 'use strict';
 
-const path = require('path');
+const path = require('node:path');
+const os = require('node:os');
+const {
+  getAggregateStatuslineCachePath,
+  shouldAggregateFeedback,
+} = require('./feedback-aggregate');
 const {
   listFeedbackArtifactPaths,
   resolveFeedbackDir,
@@ -12,18 +17,28 @@ function unique(values = []) {
   return [...new Set(values.filter(Boolean).map((value) => path.resolve(value)))];
 }
 
+function hasIsolatedTempFeedbackDir(env = process.env) {
+  if (!env.THUMBGATE_FEEDBACK_DIR) return false;
+  try {
+    return path.resolve(env.THUMBGATE_FEEDBACK_DIR).startsWith(path.resolve(os.tmpdir()) + path.sep);
+  } catch {
+    return false;
+  }
+}
+
 function getStatuslineCacheCandidates(options = {}) {
   const env = options.env || process.env;
   const projectDir = resolveProjectDir({ cwd: options.cwd, env });
   const feedbackDir = resolveFeedbackDir({ projectDir, env });
 
   return unique([
+    shouldAggregateFeedback({ env }) && !hasIsolatedTempFeedbackDir(env) && getAggregateStatuslineCachePath({ env }),
     ...listFeedbackArtifactPaths('statusline_cache.json', { projectDir, env }),
     path.join(feedbackDir, 'statusline_cache.json'),
   ]);
 }
 
-if (require.main === module) {
+if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(__filename)) {
   process.stdout.write(JSON.stringify({ candidates: getStatuslineCacheCandidates() }));
 }
 

package/scripts/statusline-cache-read.js

@@ -0,0 +1,57 @@
+#!/usr/bin/env node
+'use strict';
+
+// Resolve the statusline cache to read for display.
+//
+// PRIOR BUG: the earlier version of this script "aggregated" by summing the
+// thumbs_up/down fields across every per-folder statusline_cache.json. That
+// double-counted, because the global aggregate cache at
+// ~/.thumbgate/statusline_cache.json is ITSELF already the cross-store sum
+// (written by feedback-aggregate.js / hook-thumbgate-cache-updater.js). Summing
+// the global aggregate plus per-folder caches counted every event twice or more
+// and produced bogus totals like 1152↑/747↓ when the true aggregate was 727/600.
+//
+// CORRECT BEHAVIOR: pick the highest-priority existing cache from the
+// candidate list (`statusline-cache-path.js` puts the canonical aggregate path
+// first when aggregation is enabled) and return its content unchanged. No
+// summing across files — the upstream aggregator already did that work.
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { getStatuslineCacheCandidates } = require('./statusline-cache-path');
+
+function readCacheFile(filePath) {
+  try {
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const parsed = JSON.parse(raw);
+    if (parsed && typeof parsed === 'object') return parsed;
+  } catch {
+    /* unreadable / unparseable caches are silently skipped */
+  }
+  return null;
+}
+
+function readResolvedStatuslineCache(options = {}) {
+  const candidates = getStatuslineCacheCandidates(options);
+  for (const candidate of candidates) {
+    if (!fs.existsSync(candidate)) continue;
+    const data = readCacheFile(candidate);
+    if (data) {
+      return { ...data, source: path.resolve(candidate) };
+    }
+  }
+  return null;
+}
+
+const _invokedDirectly =
+  process.argv[1] && path.resolve(process.argv[1]) === path.resolve(__filename);
+if (_invokedDirectly) {
+  const resolved = readResolvedStatuslineCache();
+  if (resolved) {
+    process.stdout.write(JSON.stringify(resolved));
+  }
+}
+
+module.exports = {
+  readResolvedStatuslineCache,
+};

package/scripts/statusline-local-stats.js

@@ -2,6 +2,10 @@
 'use strict';
 
 const { analyzeFeedback } = require('./feedback-loop');
+const {
+  computeAggregateFeedbackStats,
+  shouldAggregateFeedback,
+} = require('./feedback-aggregate');
 const { normalizeStatsPayload } = require('./hook-thumbgate-cache-updater');
 const { syncClaudeHistoryFeedback } = require('./claude-feedback-sync');
 const { resolveProjectDir } = require('./feedback-paths');
@@ -9,9 +13,13 @@ const { resolveProjectDir } = require('./feedback-paths');
 try {
   const projectDir = resolveProjectDir({ cwd: process.cwd(), env: process.env });
   syncClaudeHistoryFeedback({ projectDir });
-  const stats = analyzeFeedback();
+  const scope = String(process.env.THUMBGATE_STATUSLINE_SCOPE || 'global').toLowerCase();
+  const stats = scope !== 'project' && shouldAggregateFeedback({ env: process.env })
+    ? computeAggregateFeedbackStats({ projectDir, env: process.env })
+    : analyzeFeedback();
   const payload = {
     ...normalizeStatsPayload(stats),
+    aggregate: stats.aggregate || { enabled: false },
     updated_at: String(Math.floor(Date.now() / 1000)),
   };
   process.stdout.write(JSON.stringify(payload));

package/scripts/statusline-meta.js

@@ -2,7 +2,7 @@
 'use strict';
 
 const path = require('path');
-const { isProLicensed } = require('./license');
+const { isProLicensed, isValidKey } = require('./license');
 
 function getStatuslineMeta(options = {}) {
   const pkg = require(path.join(__dirname, '..', 'package.json'));
@@ -11,7 +11,10 @@ function getStatuslineMeta(options = {}) {
   const fs = require('fs');
   
   // Enterprise detection based on key prefix
-  let apiKey = env.THUMBGATE_API_KEY || env.THUMBGATE_OPERATOR_KEY || '';
+  let apiKey = env.THUMBGATE_OPERATOR_KEY || env.THUMBGATE_API_KEY || '';
+  if (apiKey && !isValidKey(apiKey)) {
+    apiKey = '';
+  }
   
   // Fallback to reading from disk if not in env
   if (!apiKey) {

package/scripts/statusline.sh

@@ -45,7 +45,19 @@ if [ -z "$THUMBGATE_CACHE" ]; then
 fi
 
 UP="0"; DOWN="0"; LESSONS="0"; TREND="?"; CACHE_TS="0"
-if [ -f "$THUMBGATE_CACHE" ]; then
+# Display reads aggregate across every per-folder cache so the statusline
+# reflects true totals, not whichever folder happens to be cwd. Writes still
+# target $THUMBGATE_CACHE (per-folder), so attribution is preserved.
+_RESOLVED_CACHE_JSON=$(node "${SCRIPT_DIR}/statusline-cache-read.js" 2>/dev/null)
+if [[ -n "$_RESOLVED_CACHE_JSON" ]]; then
+  eval "$(echo "$_RESOLVED_CACHE_JSON" | jq -r '
+    @sh "UP=\(.thumbs_up // "0")",
+    @sh "DOWN=\(.thumbs_down // "0")",
+    @sh "LESSONS=\(.lessons // "0")",
+    @sh "TREND=\(.trend // "?")",
+    @sh "CACHE_TS=\(.updated_at // "0")"
+  ' 2>/dev/null)"
+elif [[ -f "$THUMBGATE_CACHE" ]]; then
   eval "$(jq -r '
     @sh "UP=\(.thumbs_up // "0")",
     @sh "DOWN=\(.thumbs_down // "0")",

package/scripts/sync-telemetry-from-prod.js

@@ -0,0 +1,374 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * sync-telemetry-from-prod.js
+ *
+ * The agentic data pipeline (scripts/agentic-data-pipeline.js →
+ * scripts/telemetry-analytics.js) computes `get_business_metrics` from the LOCAL
+ * telemetry store at `<feedbackDir>/telemetry-pings.jsonl`. On a developer
+ * machine that store is near-empty, so metrics read uniqueVisitors:0 /
+ * checkoutStarts:0 even though real web traffic is flowing.
+ *
+ * The real web funnel lives on the PROD Railway volume and is exposed only via
+ * the operator-key-gated endpoint `GET /v1/telemetry/export`. Both stores use the
+ * SAME jsonl format/filename and matching fields (receivedAt/event/eventType/
+ * installId/visitorId/sessionId/traceId), so a sync is a clean append + dedupe.
+ *
+ * This script:
+ *   1. Resolves the operator (or admin) key from env or ~/.config/thumbgate/operator.json.
+ *   2. GETs /v1/telemetry/export?source=both with `Authorization: Bearer <key>`.
+ *   3. Merges/dedupes the returned `telemetry` rows into
+ *      `<feedbackDir>/telemetry-pings.jsonl` (dedupe by a stable sha256 of the
+ *      canonical row, so repeat runs never double-count).
+ *   4. Optionally (`--funnel`) persists the returned `funnel` rows into
+ *      `<feedbackDir>/funnel-events.jsonl` the same way.
+ *
+ * After a run, `get_business_metrics` reflects the real funnel because the
+ * pipeline reads the same feedback dir on each call.
+ *
+ * SECURITY: the operator key is never printed. Only its source is reported.
+ *
+ * Usage:
+ *   node scripts/sync-telemetry-from-prod.js [--since-days=30] [--since=<iso>]
+ *        [--limit=10000] [--source=both|telemetry|funnel] [--funnel]
+ *        [--base-url=<url>] [--feedback-dir=<dir>] [--json] [--dry-run]
+ */
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const crypto = require('node:crypto');
+
+const { getFeedbackPaths } = require('./feedback-loop');
+
+const DEFAULT_BASE_URL = 'https://thumbgate-production.up.railway.app';
+const OPERATOR_CONFIG_PATH = path.join(os.homedir(), '.config', 'thumbgate', 'operator.json');
+const TELEMETRY_FILE_NAME = 'telemetry-pings.jsonl';
+const FUNNEL_FILE_NAME = 'funnel-events.jsonl';
+const EXPORT_PATH = '/v1/telemetry/export';
+const HARD_LIMIT = 10000; // server clamps each stream to this
+const MS_PER_DAY = 24 * 60 * 60 * 1000;
+
+// Configure fetch proxy when running behind a corporate/sandbox proxy. Mirrors
+// scripts/operational-summary.js so hosted reads behave the same everywhere.
+(function configureProxy() {
+  const proxyUrl = process.env.HTTPS_PROXY || process.env.https_proxy
+    || process.env.HTTP_PROXY || process.env.http_proxy;
+  if (!proxyUrl) return;
+  try {
+    const { ProxyAgent, setGlobalDispatcher } = require('undici');
+    setGlobalDispatcher(new ProxyAgent(proxyUrl));
+  } catch {
+    // undici not available — fetch will use the default dispatcher.
+  }
+}());
+
+function normalizeText(value) {
+  if (value === undefined || value === null) return null;
+  const text = String(value).trim();
+  return text || null;
+}
+
+function loadOperatorConfig(configPath = OPERATOR_CONFIG_PATH) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    return {
+      operatorKey: normalizeText(parsed.operatorKey),
+      baseUrl: normalizeText(parsed.baseUrl),
+    };
+  } catch {
+    return { operatorKey: null, baseUrl: null };
+  }
+}
+
+/**
+ * Resolve the credential + base URL the same way scripts/operational-summary.js
+ * does: env THUMBGATE_OPERATOR_KEY > operator.json operatorKey > env
+ * THUMBGATE_API_KEY (admin keys are an accepted alternate auth path on the
+ * export endpoint). `source` records WHERE the key came from for diagnostics —
+ * the key value itself is never returned to callers that print it.
+ */
+function resolveOperatorConfig(options = {}) {
+  const operatorConfig = loadOperatorConfig(options.configPath);
+  let apiKey = normalizeText(process.env.THUMBGATE_OPERATOR_KEY);
+  let keySource = apiKey ? 'env:THUMBGATE_OPERATOR_KEY' : null;
+  if (!apiKey && operatorConfig.operatorKey) {
+    apiKey = operatorConfig.operatorKey;
+    keySource = 'operator.json';
+  }
+  if (!apiKey) {
+    apiKey = normalizeText(process.env.THUMBGATE_API_KEY);
+    keySource = apiKey ? 'env:THUMBGATE_API_KEY' : null;
+  }
+  const baseUrl = normalizeText(options.baseUrl)
+    || normalizeText(process.env.THUMBGATE_SYNC_BASE_URL)
+    || normalizeText(process.env.THUMBGATE_BILLING_API_BASE_URL)
+    || operatorConfig.baseUrl
+    || DEFAULT_BASE_URL;
+  return { apiKey, keySource, baseUrl };
+}
+
+// Deterministic, LOCALE-INDEPENDENT key comparator (UTF-16 code-unit order).
+// Do NOT switch this to String.prototype.localeCompare (what SonarCloud S2871
+// suggests): localeCompare is locale-sensitive, so it would make the canonical
+// serialization — and therefore the dedupe hash below — differ across machines
+// and CI runners, silently breaking cross-environment dedupe.
+function compareKeys(a, b) {
+  if (a < b) return -1;
+  if (a > b) return 1;
+  return 0;
+}
+
+// Recursively sort object keys so structurally-identical rows serialize
+// identically regardless of original key order.
+function canonicalize(value) {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (value && typeof value === 'object') {
+    const out = {};
+    for (const key of Object.keys(value).sort(compareKeys)) out[key] = canonicalize(value[key]);
+    return out;
+  }
+  return value;
+}
+
+// Stable dedupe id. sha256 (not sha1/md5 — SonarCloud S4790) over the canonical
+// JSON of the whole row: identical rows → identical id across runs.
+function stableRowId(row) {
+  return crypto.createHash('sha256').update(JSON.stringify(canonicalize(row))).digest('hex');
+}
+
+// The export endpoint returns each stream as { rows: [...] }; older/simplified
+// shapes may return a bare array. Accept both.
+function extractRows(streamValue) {
+  if (Array.isArray(streamValue)) return streamValue;
+  if (streamValue && Array.isArray(streamValue.rows)) return streamValue.rows;
+  return [];
+}
+
+/**
+ * Fetch the telemetry export. `fetchImpl` is injectable so tests run without
+ * network or secrets. Auth is `Authorization: Bearer <key>` — the server's
+ * extractApiKey reads Bearer / x-api-key, NOT x-operator-key.
+ */
+async function fetchTelemetryExport(params = {}) {
+  const {
+    baseUrl,
+    operatorKey,
+    since,
+    limit = HARD_LIMIT,
+    source = 'both',
+    fetchImpl = globalThis.fetch,
+  } = params;
+  if (!operatorKey) {
+    const err = new Error('No operator/admin key available for telemetry export.');
+    err.code = 'no_key';
+    throw err;
+  }
+  if (typeof fetchImpl !== 'function') {
+    const err = new Error('No fetch implementation available (Node >=18 or inject fetchImpl).');
+    err.code = 'no_fetch';
+    throw err;
+  }
+  const url = new URL(EXPORT_PATH, baseUrl || DEFAULT_BASE_URL);
+  url.searchParams.set('source', source);
+  url.searchParams.set('limit', String(Math.min(Math.max(1, Number(limit) || HARD_LIMIT), HARD_LIMIT)));
+  if (since) url.searchParams.set('since', since);
+
+  const response = await fetchImpl(url, {
+    method: 'GET',
+    headers: {
+      authorization: `Bearer ${operatorKey}`,
+      accept: 'application/json',
+    },
+  });
+  if (!response.ok) {
+    const bodyText = await response.text().catch(() => '');
+    const err = new Error(`Telemetry export failed: HTTP ${response.status} ${bodyText.slice(0, 200)}`);
+    err.code = 'http_error';
+    err.status = response.status;
+    throw err;
+  }
+  return response.json();
+}
+
+// Read every stable id already present in a jsonl ledger (skips unparseable lines).
+function readExistingIds(filePath) {
+  const ids = new Set();
+  if (!fs.existsSync(filePath)) return ids;
+  const text = fs.readFileSync(filePath, 'utf8');
+  for (const line of text.split('\n')) {
+    if (!line.trim()) continue;
+    try {
+      ids.add(stableRowId(JSON.parse(line)));
+    } catch {
+      // Tolerate malformed historical lines.
+    }
+  }
+  return ids;
+}
+
+/**
+ * Append only rows whose stable id isn't already present. Pure aside from the
+ * single append; safe to re-run. Returns { added, skipped }.
+ */
+function mergeRowsIntoLedger(filePath, rows, options = {}) {
+  const existing = options.existingIds || readExistingIds(filePath);
+  const toAppend = [];
+  let skipped = 0;
+  for (const row of rows) {
+    if (!row || typeof row !== 'object') { skipped += 1; continue; }
+    const id = stableRowId(row);
+    if (existing.has(id)) { skipped += 1; continue; }
+    existing.add(id);
+    toAppend.push(JSON.stringify(row));
+  }
+  if (toAppend.length && !options.dryRun) {
+    fs.mkdirSync(path.dirname(filePath), { recursive: true });
+    fs.appendFileSync(filePath, toAppend.map((l) => `${l}\n`).join(''), 'utf8');
+  }
+  return { added: toAppend.length, skipped };
+}
+
+function resolveFeedbackDir(options = {}) {
+  if (options.feedbackDir) return path.resolve(options.feedbackDir);
+  return getFeedbackPaths().FEEDBACK_DIR;
+}
+
+// Resolve the `since` ISO timestamp: an explicit --since wins; otherwise derive
+// it from --since-days relative to nowMs (test-injectable) or the current time.
+function computeSinceIso(options = {}) {
+  if (options.since) return options.since;
+  if (!Number.isFinite(options.sinceDays)) return undefined;
+  const baseMs = Number.isFinite(options.nowMs) ? options.nowMs : Date.now();
+  return new Date(baseMs - options.sinceDays * MS_PER_DAY).toISOString();
+}
+
+/**
+ * End-to-end sync: fetch export → merge telemetry (always) and funnel (opt-in).
+ * Returns a summary object with counts and resolved paths (no secrets).
+ */
+async function syncTelemetryFromProd(options = {}) {
+  const config = resolveOperatorConfig(options);
+  const feedbackDir = resolveFeedbackDir(options);
+  const source = options.source || (options.funnel ? 'both' : 'telemetry');
+  const sinceIso = computeSinceIso(options);
+
+  const payload = await fetchTelemetryExport({
+    baseUrl: config.baseUrl,
+    operatorKey: config.apiKey,
+    since: sinceIso,
+    limit: options.limit || HARD_LIMIT,
+    source,
+    fetchImpl: options.fetchImpl,
+  });
+
+  const telemetryRows = extractRows(payload.telemetry);
+  const telemetryPath = path.join(feedbackDir, TELEMETRY_FILE_NAME);
+  const telemetryResult = mergeRowsIntoLedger(telemetryPath, telemetryRows, { dryRun: options.dryRun });
+
+  const summary = {
+    feedbackDir,
+    baseUrl: config.baseUrl,
+    keySource: config.keySource,
+    source,
+    since: payload.since || sinceIso || null,
+    dryRun: !!options.dryRun,
+    telemetry: {
+      path: telemetryPath,
+      fetched: telemetryRows.length,
+      added: telemetryResult.added,
+      skipped: telemetryResult.skipped,
+    },
+    funnel: null,
+  };
+
+  if (options.funnel) {
+    const funnelRows = extractRows(payload.funnel);
+    const funnelPath = path.join(feedbackDir, FUNNEL_FILE_NAME);
+    const funnelResult = mergeRowsIntoLedger(funnelPath, funnelRows, { dryRun: options.dryRun });
+    summary.funnel = {
+      path: funnelPath,
+      fetched: funnelRows.length,
+      added: funnelResult.added,
+      skipped: funnelResult.skipped,
+    };
+  }
+
+  return summary;
+}
+
+function parseArgs(argv) {
+  const opts = { sinceDays: 30, funnel: false, json: false, dryRun: false };
+  for (const arg of argv) {
+    if (arg === '--funnel') opts.funnel = true;
+    else if (arg === '--json') opts.json = true;
+    else if (arg === '--dry-run') opts.dryRun = true;
+    else if (arg.startsWith('--since-days=')) opts.sinceDays = Number(arg.split('=')[1]);
+    else if (arg.startsWith('--since=')) opts.since = arg.split('=').slice(1).join('=');
+    else if (arg.startsWith('--limit=')) opts.limit = Number(arg.split('=')[1]);
+    else if (arg.startsWith('--source=')) opts.source = arg.split('=')[1];
+    else if (arg.startsWith('--base-url=')) opts.baseUrl = arg.split('=').slice(1).join('=');
+    else if (arg.startsWith('--feedback-dir=')) opts.feedbackDir = arg.split('=').slice(1).join('=');
+  }
+  if (!Number.isFinite(opts.sinceDays)) opts.sinceDays = 30;
+  return opts;
+}
+
+async function main() {
+  const opts = parseArgs(process.argv.slice(2));
+  const preflight = resolveOperatorConfig(opts);
+  if (!preflight.apiKey) {
+    console.error(
+      'No operator/admin key found. Set THUMBGATE_OPERATOR_KEY (or THUMBGATE_API_KEY), '
+      + 'or add operatorKey to ~/.config/thumbgate/operator.json. '
+      + 'The key lives in Railway prod env as THUMBGATE_OPERATOR_KEY.'
+    );
+    process.exitCode = 1;
+    return;
+  }
+  try {
+    const summary = await syncTelemetryFromProd(opts);
+    if (opts.json) {
+      console.log(JSON.stringify(summary, null, 2));
+    } else {
+      const t = summary.telemetry;
+      console.log(`✅ Telemetry sync${summary.dryRun ? ' (dry-run)' : ''} complete`);
+      console.log(`   source=${summary.source} since=${summary.since || 'server-default'} via ${summary.keySource}`);
+      console.log(`   telemetry: ${t.fetched} fetched → ${t.added} added, ${t.skipped} already present`);
+      console.log(`   → ${t.path}`);
+      if (summary.funnel) {
+        const f = summary.funnel;
+        console.log(`   funnel: ${f.fetched} fetched → ${f.added} added, ${f.skipped} already present`);
+        console.log(`   → ${f.path}`);
+      }
+    }
+  } catch (err) {
+    console.error(`❌ Telemetry sync failed: ${err.message}`);
+    process.exitCode = 1;
+  }
+}
+
+// Path-based entrypoint check (require.main === module is flagged by SonarCloud S3403).
+if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(__filename)) {
+  main();
+}
+
+module.exports = {
+  DEFAULT_BASE_URL,
+  TELEMETRY_FILE_NAME,
+  FUNNEL_FILE_NAME,
+  HARD_LIMIT,
+  canonicalize,
+  stableRowId,
+  extractRows,
+  loadOperatorConfig,
+  resolveOperatorConfig,
+  fetchTelemetryExport,
+  readExistingIds,
+  mergeRowsIntoLedger,
+  resolveFeedbackDir,
+  computeSinceIso,
+  syncTelemetryFromProd,
+  parseArgs,
+};

package/scripts/telemetry-analytics.js

@@ -75,6 +75,13 @@ function normalizeInteger(value) {
   return Number.isFinite(parsed) ? Math.trunc(parsed) : null;
 }
 
+function normalizeRate(value) {
+  if (value === undefined || value === null || value === '') return null;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return null;
+  return Math.max(0, Math.min(parsed, 1));
+}
+
 function safeRate(num, den) {
   if (!den) return 0;
   return Number((num / den).toFixed(4));
@@ -480,6 +487,8 @@ function sanitizeTelemetryPayload(payload = {}, headers = {}) {
     httpStatus: normalizeInteger(raw.httpStatus),
     userAgent: pickFirstText(raw.userAgent, headers['user-agent']),
     isBot: pickFirstText(raw.isBot),
+    interstitialSampled: pickFirstText(raw.interstitialSampled),
+    interstitialSampleRate: normalizeRate(raw.interstitialSampleRate),
     attributionTagged: Boolean(
       pickFirstText(raw.utmSource, raw.utmMedium, raw.utmCampaign, raw.utmContent, raw.utmTerm)
     ),