thumbgate 1.35.0 → 1.37.1

package/scripts/telemetry-analytics.js

@@ -13,6 +13,7 @@ const {
   getFallbackFeedbackDir,
   resolveFallbackArtifactPath,
 } = require('./feedback-paths');
+const { readTextTail } = require('./fs-utils');
 
 const TELEMETRY_FILE_NAME = 'telemetry-pings.jsonl';
 const MARKETING_CLICK_EVENT_TYPES = new Set([
@@ -531,7 +532,18 @@ function sanitizeTelemetryPayload(payload = {}, headers = {}) {
   return entry;
 }
 
+function headerHasPrivacyOptOut(headers = {}) {
+  const normalized = {};
+  for (const [key, value] of Object.entries(headers || {})) {
+    normalized[String(key).toLowerCase()] = String(value == null ? '' : value).trim();
+  }
+  return normalized['sec-gpc'] === '1' || normalized.dnt === '1';
+}
+
 function appendTelemetryEvent(feedbackDir, payload = {}, headers = {}) {
+  if (headerHasPrivacyOptOut(headers)) {
+    return null;
+  }
   const entry = sanitizeTelemetryPayload(payload, headers);
   const telemetryPath = getTelemetryPath(feedbackDir);
   fs.mkdirSync(path.dirname(telemetryPath), { recursive: true });
@@ -541,31 +553,36 @@ function appendTelemetryEvent(feedbackDir, payload = {}, headers = {}) {
 
 const DEFAULT_BOUNDED_TELEMETRY_TAIL_BYTES = 8 * 1024 * 1024;
 
-function readTelemetryText(filePath, options = {}) {
-  if (!fs.existsSync(filePath)) return '';
-  const maxBytes = Number(options.maxBytes || 0);
-  if (maxBytes > 0) {
-    const stats = fs.statSync(filePath);
-    if (stats.size > maxBytes) {
-      const fd = fs.openSync(filePath, 'r');
-      try {
-        const buffer = Buffer.alloc(maxBytes);
-        fs.readSync(fd, buffer, 0, maxBytes, stats.size - maxBytes);
-        const text = buffer.toString('utf-8');
-        const firstNewline = text.indexOf('\n');
-        return firstNewline >= 0 ? text.slice(firstNewline + 1) : text;
-      } finally {
-        fs.closeSync(fd);
-      }
-    }
+// Delegates to readTextTail(), which applies an UNCONDITIONAL full-read ceiling
+// and drops the partial first line after a mid-file seek.
+//
+// WHY: this function used to fall through to a full fs.readFileSync() whenever
+// the caller passed no maxBytes. getTelemetrySummary() only supplies maxBytes for
+// a `bounded` analytics window, so the default /v1/dashboard request read the
+// whole telemetry log. Once production telemetry passed V8's max string length
+// that threw
+//   "Cannot create a string longer than 0x1fffffe8 characters"
+// which escaped generateDashboard() and 503'd /v1/dashboard ("Dashboard data too
+// large") on every deploy-verification run. A size ceiling must not be opt-in.
+function readTelemetryTextWithMeta(filePath, options = {}) {
+  if (!fs.existsSync(filePath)) return { text: '', truncated: false, size: 0 };
+  try {
+    const tail = readTextTail(filePath, Number(options.maxBytes || 0));
+    return { text: tail.text, truncated: Boolean(tail.truncated), size: tail.size || 0 };
+  } catch {
+    // A single unreadable/oversized telemetry log must never take down the
+    // whole dashboard. Degrade to "no telemetry", not to a 503 — but say so:
+    // an unreadable source is a hole in the data, not an empty data set.
+    return { text: '', truncated: true, size: 0, unreadable: true };
   }
-  return fs.readFileSync(filePath, 'utf-8');
 }
 
-function loadTelemetryEventsFromPath(filePath, options = {}) {
-  const raw = readTelemetryText(filePath, options).trim();
-  if (!raw) return [];
-  return raw
+function readTelemetryText(filePath, options = {}) {
+  return readTelemetryTextWithMeta(filePath, options).text;
+}
+
+function parseTelemetryLines(raw) {
+  return String(raw || '')
     .split('\n')
     .map((line) => {
       try {
@@ -582,13 +599,26 @@ function loadTelemetryEventsFromPath(filePath, options = {}) {
     .filter(Boolean);
 }
 
-function loadTelemetryEvents(feedbackDir, options = {}) {
+function loadTelemetryEventsFromPath(filePath, options = {}) {
+  const raw = readTelemetryText(filePath, options).trim();
+  if (!raw) return [];
+  return parseTelemetryLines(raw);
+}
+
+// Same merge as loadTelemetryEvents(), but reports whether any source was read
+// only as a tail. Callers that label a result "lifetime" MUST consult this:
+// a tail-read source means the counts below it are partial, not lifetime.
+function loadTelemetryEventsWithMeta(feedbackDir, options = {}) {
   const diagnostics = getTelemetrySourceDiagnostics(feedbackDir);
   const merged = [];
   const seen = new Set();
+  const truncatedPaths = [];
 
   for (const filePath of diagnostics.activePaths) {
-    const rows = loadTelemetryEventsFromPath(filePath, options);
+    const read = readTelemetryTextWithMeta(filePath, options);
+    if (read.truncated) truncatedPaths.push(filePath);
+    const raw = read.text.trim();
+    const rows = raw ? parseTelemetryLines(raw) : [];
     for (const row of rows) {
       const key = JSON.stringify(row);
       if (seen.has(key)) continue;
@@ -597,7 +627,15 @@ function loadTelemetryEvents(feedbackDir, options = {}) {
     }
   }
 
-  return merged;
+  return {
+    events: merged,
+    truncated: truncatedPaths.length > 0,
+    truncatedPaths,
+  };
+}
+
+function loadTelemetryEvents(feedbackDir, options = {}) {
+  return loadTelemetryEventsWithMeta(feedbackDir, options).events;
 }
 
 function summarizeRecentEvents(events) {
@@ -779,8 +817,12 @@ function getTelemetrySummary(feedbackDir, options = {}) {
   const telemetryLoadOptions = analyticsWindow.bounded
     ? { maxBytes: Number(options.telemetryTailBytes || DEFAULT_BOUNDED_TELEMETRY_TAIL_BYTES) }
     : {};
+  // readTextTail() applies an unconditional ceiling even for the unbounded
+  // ("lifetime") window, so a large enough telemetry log is read as a tail.
+  // Carry that fact through instead of presenting a tail as complete history.
+  const telemetryRead = loadTelemetryEventsWithMeta(feedbackDir, telemetryLoadOptions);
   const events = filterEntriesForWindow(
-    loadTelemetryEvents(feedbackDir, telemetryLoadOptions),
+    telemetryRead.events,
     analyticsWindow,
     (entry) => entry && (entry.receivedAt || entry.timestamp)
   );
@@ -1129,7 +1171,19 @@ function getTelemetrySummary(feedbackDir, options = {}) {
   const proConversions = checkoutPaidConfirmations;
 
   return {
-    window: serializeAnalyticsWindow(analyticsWindow),
+    window: {
+      ...serializeAnalyticsWindow(analyticsWindow),
+      // A truncated source means these counts cover only the newest slice of
+      // the log, so `lifetime` would be a lie. Consumers that present
+      // acquisition/conversion rates must check `complete`.
+      truncated: telemetryRead.truncated,
+      complete: !telemetryRead.truncated,
+    },
+    telemetrySource: {
+      truncated: telemetryRead.truncated,
+      truncatedPaths: telemetryRead.truncatedPaths,
+      tailBytes: Number(telemetryLoadOptions.maxBytes) || null,
+    },
     totalEvents: events.length,
     latestSeenAt,
     trafficQuality,
@@ -1493,10 +1547,13 @@ module.exports = {
   TELEMETRY_FILE_NAME,
   sanitizeTelemetryPayload,
   classifyTelemetryAudience,
+  headerHasPrivacyOptOut,
   appendTelemetryPing,
   appendTelemetryEvent,
   getTelemetrySourceDiagnostics,
   loadTelemetryEvents,
+  loadTelemetryEventsWithMeta,
   getTelemetryAnalytics,
+  getTelemetrySummary,
   inferTrafficChannel,
 };

package/scripts/temporal-decay-weighting.js

@@ -0,0 +1,138 @@
+/**
+ * Temporal Decay Function for Lesson Retrieval Scoring
+ *
+ * Inspired by episode #1017: "The RAG Mistake Almost Every Team Is Making"
+ * Key insight: Traditional vector memory is insufficient. We need temporal filtering
+ * to avoid outdated information polluting retrieval results.
+ *
+ * This module adds:
+ * - Temporal decay weighting in retrieval scoring
+ * - Configurable TTL for lesson relevance
+ * - Half-life-based score degradation
+ */
+
+const DEFAULT_HALF_LIFE_MS = 30 * 24 * 60 * 60 * 1000;
+
+/**
+ * Coerce a lesson timestamp into a Date without stringifying epoch numbers
+ * (String(1785542400000) is not a valid Date input).
+ *
+ * @param {Date|string|number|null|undefined} lessonTimestamp
+ * @returns {Date}
+ */
+function coerceLessonDate(lessonTimestamp) {
+  if (lessonTimestamp instanceof Date) {
+    return lessonTimestamp;
+  }
+  if (typeof lessonTimestamp === 'number' && Number.isFinite(lessonTimestamp)) {
+    return new Date(lessonTimestamp);
+  }
+  if (typeof lessonTimestamp === 'string' && lessonTimestamp.trim()) {
+    const asNumber = Number(lessonTimestamp);
+    if (Number.isFinite(asNumber) && /^-?\d+(\.\d+)?$/.test(lessonTimestamp.trim())) {
+      return new Date(asNumber);
+    }
+    return new Date(lessonTimestamp);
+  }
+  return new Date(NaN);
+}
+
+/**
+ * Apply temporal decay to an embedding score based on lesson age.
+ *
+ * @param {number} rawScore - Raw cosine similarity score (0-1)
+ * @param {Date|string|number} lessonTimestamp - When the lesson was recorded
+ * @param {number} halfLifeMs - Half-life of relevance (default: 30 days)
+ * @param {boolean} activeMode - Active investigation mode uses a longer half-life
+ *   (slower decay) without boosting scores above the raw value
+ * @returns {number} Decayed score
+ */
+function applyTemporalDecay(rawScore, lessonTimestamp, halfLifeMs = DEFAULT_HALF_LIFE_MS, activeMode = false) {
+  if (!rawScore || rawScore <= 0) return rawScore;
+
+  if (!Number.isFinite(halfLifeMs) || halfLifeMs <= 0) {
+    throw new TypeError('halfLifeMs must be a positive finite number');
+  }
+
+  const lessonDate = coerceLessonDate(lessonTimestamp);
+
+  // Handle invalid timestamps (e.g., from git metadata like "2026-07-31T19:48:50+02:00")
+  if (isNaN(lessonDate.getTime())) {
+    return rawScore * 0.1; // Penalize stale/untraceable lessons heavily
+  }
+
+  const ageMs = Math.max(0, Date.now() - lessonDate.getTime());
+  // Longer half-life = slower decay; never multiply the score itself.
+  const effectiveHalfLifeMs = activeMode ? halfLifeMs * 1.5 : halfLifeMs;
+  const decayFactor = Math.exp((-Math.log(2) * ageMs) / effectiveHalfLifeMs);
+
+  return rawScore * decimal(decayFactor, 4);
+}
+
+/**
+ * Apply combined scoring: base similarity + metadata boost + temporal decay
+ */
+function computeContextualScore(rawScore, lesson, config = {}) {
+  let totalScore = rawScore;
+
+  const safeLesson = lesson || {};
+
+  // Metadata filters from podcast: tag-based relevance boosting
+  if (config.metadataFilters) {
+    const requiredTags = Array.isArray(config.metadataFilters.tags) ? config.metadataFilters.tags : [];
+    const hasTags =
+      Array.isArray(safeLesson.tags) && requiredTags.every((tag) => safeLesson.tags.includes(String(tag)));
+
+    totalScore = hasTags ? totalScore * 1.2 : Math.max(totalScore, rawScore); // Bonus for tag match
+  }
+
+  // Temporal decay always applies (including when no reranker is configured).
+  const timestamp =
+    safeLesson.timestamp != null && safeLesson.timestamp !== ''
+      ? safeLesson.timestamp
+      : safeLesson.receivedAt != null && safeLesson.receivedAt !== ''
+        ? safeLesson.receivedAt
+        : safeLesson.created_at != null && safeLesson.created_at !== ''
+          ? safeLesson.created_at
+          : undefined;
+
+  totalScore = applyTemporalDecay(
+    totalScore,
+    timestamp,
+    config.halfLifeMs || DEFAULT_HALF_LIFE_MS,
+    Boolean(config.activeMode)
+  );
+
+  // Reranker boost (cross-encoder or similar rerank module): if score > threshold, apply non-linear boost
+  if (totalScore > config.rerankThreshold && config.rrfBoost) {
+    totalScore = rrfDecay(totalScore * config.rrfBoost, config.rrfPenalty);
+  }
+
+  return decimal(totalScore, 4);
+}
+
+/**
+ * Re-Rank Decay (RRF-style) to prevent multiple hits from the same topic
+ * Episode insight: "post-retrieval semantic filtering" reduces duplication
+ */
+function rrfDecay(score, penalty = 0.3) {
+  if (!score || score <= 1) return score;
+  return decimal((score - penalty) / (2 - penalty), 4);
+}
+
+/**
+ * Format a number to fixed decimal places
+ */
+function decimal(num, places) {
+  const mult = Math.pow(10, places);
+  return Math.round(num * mult) / mult;
+}
+
+// Export for modular use in retrieval pipelines
+module.exports = {
+  DEFAULT_HALF_LIFE_MS,
+  applyTemporalDecay,
+  coerceLessonDate,
+  computeContextualScore,
+  rrfDecay,
+};

package/scripts/test-all.js

@@ -0,0 +1,165 @@
+#!/usr/bin/env node
+/**
+ * test-all — run every `test:*` npm script and AGGREGATE the results.
+ *
+ * WHY THIS EXISTS
+ * ---------------
+ * `npm test` is 360 sub-commands joined with `&&`. Two consequences, both bad:
+ *
+ *   1. `&&` short-circuits. The first failing suite hides the other 359, so CI
+ *      can report "something failed" but never "these 12 failed". Every red run
+ *      costs a full re-run per fix.
+ *   2. Membership in that chain is hand-maintained, so suites drift out of it.
+ *      Measured 2026-08-27 on d0bb3768: 405 `test:*` scripts defined, 359 in the
+ *      chain, **46 never executed** — including test:redteam,
+ *      test:stealth-memory-injection, test:mcp-policy,
+ *      test:reward-hacking-guardrails and test:proactive-agent-eval-guardrails.
+ *      All six were run by hand and PASS (33 assertions). They were green and
+ *      guarding nothing.
+ *
+ * This runner discovers suites instead of listing them, so a new `test:*` script
+ * is covered the moment it exists. It runs everything, then reports.
+ *
+ * USAGE
+ *   node scripts/test-all.js              run all, aggregate, exit 1 if any fail
+ *   node scripts/test-all.js --list       print the discovered suites, run none
+ *   node scripts/test-all.js --orphans    print suites missing from `npm test`
+ *   node scripts/test-all.js --jobs=4     concurrency (default: cpus-1, min 1)
+ *   node scripts/test-all.js --filter=re  only suites whose name matches
+ */
+'use strict';
+
+const { spawn } = require('node:child_process');
+const os = require('node:os');
+const path = require('node:path');
+const fs = require('node:fs');
+
+const ROOT = path.resolve(__dirname, '..');
+const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
+
+const argv = process.argv.slice(2);
+const flag = (name) => argv.includes(`--${name}`);
+const value = (name, dflt) => {
+  const hit = argv.find((a) => a.startsWith(`--${name}=`));
+  return hit ? hit.slice(name.length + 3) : dflt;
+};
+
+const SKIP_SELF = new Set([
+  // shells out to the whole suite
+  'test:coverage',
+  // this runner itself — including it would recurse
+  'test:all',
+  // diagnostic: exits 1 when the hand-maintained chain has drift
+  'test:orphans',
+]);
+
+const ALL = Object.keys(pkg.scripts)
+  .filter((n) => n.startsWith('test:'))
+  .filter((n) => !SKIP_SELF.has(n))
+  // Suite names are ASCII identifiers (`test:foo-bar`), so a byte-wise
+  // comparator is deterministic across locales — unlike bare `.sort()`.
+  .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0));
+
+const chained = new Set(
+  String(pkg.scripts.test || '')
+    .split('&&')
+    .map((s) => s.trim())
+    .filter((s) => s.startsWith('npm run '))
+    .map((s) => s.replace('npm run ', '').trim())
+);
+const ORPHANS = ALL.filter((n) => !chained.has(n));
+
+const filter = value('filter', null);
+// `--filter` is a literal substring, never a regex. Building a RegExp from a
+// command-line argument is a regex-injection sink (CodeQL js/regex-injection)
+// and buys nothing: suite names are plain `test:foo-bar` identifiers.
+const suites = filter ? ALL.filter((n) => n.includes(filter)) : ALL;
+
+if (flag('list')) {
+  suites.forEach((s) => console.log(s));
+  process.exit(0);
+}
+if (flag('orphans')) {
+  console.log(`${ORPHANS.length} test:* scripts are NOT in the \`npm test\` chain:\n`);
+  ORPHANS.forEach((s) => console.log('  ' + s));
+  process.exit(ORPHANS.length ? 1 : 0);
+}
+
+const JOBS = Math.max(1, Number.parseInt(value('jobs', String(Math.max(1, os.cpus().length - 1))), 10) || 1);
+
+function run(name) {
+  return new Promise((resolve) => {
+    const started = Date.now();
+    // Invoking `npm` by name is intentional: this runner executes inside a
+    // checkout where npm must come from the user's own PATH (brew/apt/scoop/
+    // Volta/nvm all install it in different places). The command name is a
+    // hard-coded literal and args is an array, so spawn never uses a shell.
+    const child = spawn('npm', ['run', '--silent', name], { // NOSONAR javascript:S4036
+      cwd: ROOT,
+      env: { ...process.env, NO_COLOR: '1' },
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    let out = '';
+    child.stdout.on('data', (d) => (out += d));
+    child.stderr.on('data', (d) => (out += d));
+    child.on('error', (err) =>
+      resolve({ name, code: -1, ms: Date.now() - started, out: String(err?.message) })
+    );
+    child.on('close', (code) => resolve({ name, code, ms: Date.now() - started, out }));
+  });
+}
+
+const num = (text, re) => {
+  const m = text.match(re);
+  return m ? Number(m[1]) : 0;
+};
+
+(async () => {
+  const started = Date.now();
+  const results = [];
+  let cursor = 0;
+
+  console.log(
+    `test-all: ${suites.length} suites, ${JOBS} parallel` +
+      (ORPHANS.length ? `  (${ORPHANS.length} of these are absent from \`npm test\`)` : '')
+  );
+
+  const worker = async () => {
+    while (cursor < suites.length) {
+      const name = suites[cursor++];
+      const r = await run(name);
+      results.push(r);
+      const mark = r.code === 0 ? 'ok  ' : 'FAIL';
+      const tag = chained.has(name) ? '   ' : '[+]'; // [+] = previously never run
+      process.stdout.write(
+        `  ${mark} ${tag} ${name.padEnd(48)} ${String(Math.round(r.ms / 100) / 10).padStart(6)}s\n`
+      );
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(JOBS, suites.length) }, worker));
+
+  const failed = results.filter((r) => r.code !== 0);
+  const assertPass = results.reduce((a, r) => a + num(r.out, /^# pass (\d+)/m), 0);
+  const assertFail = results.reduce((a, r) => a + num(r.out, /^# fail (\d+)/m), 0);
+
+  console.log('\n' + '='.repeat(62));
+  console.log(`suites   : ${results.length - failed.length}/${results.length} passed`);
+  console.log(`asserts  : ${assertPass} passed, ${assertFail} failed`);
+  console.log(`wall     : ${Math.round((Date.now() - started) / 1000)}s`);
+
+  if (failed.length) {
+    // The whole point: report EVERY failure, not just the first.
+    console.log(`\n${failed.length} FAILING SUITE(S):`);
+    for (const f of failed) {
+      const first =
+        f.out
+          .split('\n')
+          // `^` intentionally binds only to the "not ok " prefix; Error /
+          // AssertionError / failed may appear anywhere on the line. The
+          // grouping makes that precedence explicit (Sonar S5850).
+          .find((l) => /^(not ok )|Error|AssertionError|failed/i.test(l)) || '(no summary line)';
+      console.log(`  ${f.name}  exit=${f.code}\n      ${first.trim().slice(0, 140)}`);
+    }
+  }
+  process.exit(failed.length ? 1 : 0);
+})();

package/scripts/token-savings.js

@@ -168,8 +168,47 @@ function formatTokens(count) {
   return String(Math.round(n));
 }
 
+// Eye on AI / Trustwise (Manoj Saxena, public remarks): one user turn can
+// fan out into 20–50 downstream tool calls (~40x tokens vs a simple prompt).
+// These are modeled bounds for quoting — not ThumbGate production telemetry.
+const AGENTIC_FANOUT_ACTIONS_LOW = 20;
+const AGENTIC_FANOUT_ACTIONS_HIGH = 50;
+const AGENTIC_TOKEN_MULTIPLIER_CAP = 40;
+
+function estimateAgenticFanoutSpend(input = {}) {
+  const promptIn = clampNumber(input.promptInputTokens, 0);
+  const promptOut = clampNumber(input.promptOutputTokens, 0);
+  const requested = input.downstreamActions === undefined
+    ? AGENTIC_FANOUT_ACTIONS_LOW
+    : clampNumber(input.downstreamActions, AGENTIC_FANOUT_ACTIONS_LOW);
+  const actions = Math.min(Math.max(requested, 1), AGENTIC_FANOUT_ACTIONS_HIGH);
+  const prices = blendedPricePer1M(
+    input.modelMix || DEFAULT_MODEL_MIX,
+    input.modelPrices || DEFAULT_MODEL_PRICES,
+  );
+  const simpleUsd = (promptIn * prices.input + promptOut * prices.output) / 1e6;
+  const perActionOut = Math.max(promptOut, DEFAULT_AVG_OUTPUT_TOKENS_PER_BLOCK);
+  const fanoutUsd = actions * ((promptIn * prices.input + perActionOut * prices.output) / 1e6);
+  const rawRatio = simpleUsd > 0 ? fanoutUsd / simpleUsd : 0;
+  const modeledMultiplier = Math.min(rawRatio, AGENTIC_TOKEN_MULTIPLIER_CAP);
+  const agenticFanoutUsd = simpleUsd > 0
+    ? simpleUsd * modeledMultiplier
+    : fanoutUsd;
+  return {
+    modeledNotMeasured: true,
+    source: 'eye-on-ai-trustwise-fanout-bounds',
+    affiliation: 'Inspired by public Eye on AI conversation with Manoj Saxena / Trustwise. Not affiliated. Not their telemetry.',
+    downstreamActions: actions,
+    simplePromptUsd: simpleUsd,
+    agenticFanoutUsd,
+    modeledMultiplier,
+    multiplierCap: AGENTIC_TOKEN_MULTIPLIER_CAP,
+  };
+}
+
 module.exports = {
   computeTokenSavings,
+  estimateAgenticFanoutSpend,
   formatDollars,
   formatTokens,
   blendedPricePer1M,
@@ -177,4 +216,7 @@ module.exports = {
   DEFAULT_MODEL_MIX,
   DEFAULT_AVG_INPUT_TOKENS_PER_BLOCK,
   DEFAULT_AVG_OUTPUT_TOKENS_PER_BLOCK,
+  AGENTIC_FANOUT_ACTIONS_LOW,
+  AGENTIC_FANOUT_ACTIONS_HIGH,
+  AGENTIC_TOKEN_MULTIPLIER_CAP,
 };