thumbgate 1.27.6 → 1.27.7

package/scripts/dashboard.js

@@ -49,6 +49,10 @@ const {
   readDecisionLog,
 } = require('./decision-journal');
 const { analyzeFeedback } = require('./feedback-loop');
+const {
+  collectAggregateLogEntries,
+  shouldAggregateFeedback,
+} = require('./feedback-aggregate');
 
 const PROJECT_ROOT = path.join(__dirname, '..');
 const DEFAULT_GATES_PATH = path.join(PROJECT_ROOT, 'config', 'gates', 'default.json');
@@ -1545,6 +1549,10 @@ function resolveTeamWindowHours(analyticsWindow) {
 // ---------------------------------------------------------------------------
 
 function collectAllFeedbackEntries(feedbackDir) {
+  if (shouldAggregateFeedback()) {
+    return collectAggregateLogEntries('feedback-log.jsonl', { feedbackDir }).entries;
+  }
+
   const entries = [];
   const seen = new Set();
 

package/scripts/export-databricks-bundle.js

@@ -6,6 +6,7 @@ const path = require('path');
 
 const { getFeedbackPaths } = require('./feedback-loop');
 const { ensureDir } = require('./fs-utils');
+const { redactSecretsDeep } = require('./secret-redaction');
 
 const PROJECT_ROOT = path.join(__dirname, '..');
 const DEFAULT_PROOF_DIR = process.env.THUMBGATE_PROOF_DIR
@@ -47,7 +48,10 @@ function readJSON(filePath) {
 }
 
 function writeJSONL(filePath, rows) {
-  const content = rows.map((row) => JSON.stringify(row)).join('\n');
+  // Redact secrets from every bundle row — this is the single choke point for all bundle tables
+  // (feedback_events, memory_records, sequences, attributions, proof_reports). A shared/published
+  // dataset must never ship a captured credential. See scripts/secret-redaction.js.
+  const content = rows.map((row) => JSON.stringify(redactSecretsDeep(row))).join('\n');
   fs.writeFileSync(filePath, content ? `${content}\n` : '');
 }
 

package/scripts/export-dpo-pairs.js

@@ -9,6 +9,7 @@ const fs = require('fs');
 const path = require('path');
 const { traceForDpoPair, aggregateTraces } = require('./code-reasoning');
 const { resolveFeedbackDir } = require('./feedback-paths');
+const { redactSecretsDeep } = require('./secret-redaction');
 
 const DEFAULT_LOCAL_MEMORY_LOG = path.join(resolveFeedbackDir(), 'memory-log.jsonl');
 
@@ -201,14 +202,18 @@ function exportDpoFromMemories(memories) {
     },
   }));
 
+  // Redact secrets before the pairs leave this module — they are derived from memory content and
+  // are shipped to disk here AND consumed by export-hf-dataset.js. See scripts/secret-redaction.js.
+  const redactedPairs = pairsWithTraces.map((pair) => redactSecretsDeep(pair));
+
   return {
-    pairs: pairsWithTraces,
+    pairs: redactedPairs,
     unpairedErrors: result.unpairedErrors,
     unpairedLearnings: result.unpairedLearnings,
     errors,
     learnings,
     reasoning,
-    jsonl: toJSONL(pairsWithTraces),
+    jsonl: toJSONL(redactedPairs),
   };
 }
 

package/scripts/feedback-aggregate.js

@@ -0,0 +1,281 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const {
+  getFallbackFeedbackDir,
+  getGlobalFeedbackDir,
+  getHomeDir,
+  getLegacyFeedbackDir,
+  getThumbgateFeedbackDir,
+  resolveFeedbackDir,
+  resolveProjectDir,
+} = require('./feedback-paths');
+const { readJsonl } = require('./fs-utils');
+
+const FEEDBACK_LOG = 'feedback-log.jsonl';
+const MEMORY_LOG = 'memory-log.jsonl';
+const STATUSLINE_CACHE = 'statusline_cache.json';
+
+function truthyDisabled(value) {
+  return value === '0' || value === 'false' || value === 'local';
+}
+
+function shouldAggregateFeedback(options = {}) {
+  const env = options.env || process.env;
+  return !truthyDisabled(String(env.THUMBGATE_STATUSLINE_AGGREGATE || env.THUMBGATE_AGGREGATE_FEEDBACK || '1').toLowerCase());
+}
+
+function normalizePath(candidate) {
+  if (!candidate) return null;
+  try {
+    return path.resolve(String(candidate));
+  } catch {
+    return null;
+  }
+}
+
+function uniquePaths(values = []) {
+  const seen = new Set();
+  const out = [];
+  for (const value of values) {
+    const resolved = normalizePath(value);
+    if (!resolved || seen.has(resolved)) continue;
+    seen.add(resolved);
+    out.push(resolved);
+  }
+  return out;
+}
+
+function safeExists(candidate) {
+  try {
+    return Boolean(candidate && fs.existsSync(candidate));
+  } catch {
+    return false;
+  }
+}
+
+function immediateChildDirs(parentDir) {
+  try {
+    return fs.readdirSync(parentDir, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => path.join(parentDir, entry.name));
+  } catch {
+    return [];
+  }
+}
+
+function ancestorProjectFeedbackDirs(projectDir, options = {}) {
+  const home = normalizePath(getHomeDir(options));
+  const start = normalizePath(projectDir);
+  if (!start) return [];
+
+  const dirs = [];
+  let cursor = start;
+  while (cursor && cursor !== path.dirname(cursor)) {
+    dirs.push(
+      path.join(cursor, '.thumbgate'),
+      path.join(cursor, '.thumbgate-compat'),
+      path.join(cursor, '.claude', 'memory', 'feedback')
+    );
+    if (home && cursor === home) break;
+    const next = path.dirname(cursor);
+    if (next === cursor) break;
+    cursor = next;
+  }
+  return dirs;
+}
+
+function listFeedbackStoreDirs(options = {}) {
+  const env = options.env || process.env;
+  const projectDir = resolveProjectDir({ cwd: options.cwd, env, projectDir: options.projectDir });
+  const home = getHomeDir({ env });
+  const homeThumbgate = path.join(home, '.thumbgate');
+  const projectsDir = path.join(homeThumbgate, 'projects');
+  const explicitRoots = String(env.THUMBGATE_AGGREGATE_ROOTS || '')
+    .split(path.delimiter)
+    .map((value) => value.trim())
+    .filter(Boolean);
+  const explicitFeedbackDir = options.feedbackDir || env.THUMBGATE_FEEDBACK_DIR;
+  const normalizedExplicitFeedbackDir = normalizePath(explicitFeedbackDir);
+
+  if (
+    normalizedExplicitFeedbackDir &&
+    normalizedExplicitFeedbackDir.startsWith(normalizePath(os.tmpdir()) + path.sep) &&
+    explicitRoots.length === 0
+  ) {
+    return uniquePaths([explicitFeedbackDir])
+      .filter((dir) => safeExists(path.join(dir, FEEDBACK_LOG)) || safeExists(path.join(dir, MEMORY_LOG)) || safeExists(path.join(dir, STATUSLINE_CACHE)));
+  }
+
+  return uniquePaths([
+    explicitFeedbackDir,
+    resolveFeedbackDir({ projectDir, env }),
+    getThumbgateFeedbackDir({ projectDir, env }),
+    getFallbackFeedbackDir({ projectDir, env }),
+    getLegacyFeedbackDir({ projectDir, env }),
+    getGlobalFeedbackDir({ projectDir, env }),
+    homeThumbgate,
+    ...immediateChildDirs(projectsDir),
+    ...ancestorProjectFeedbackDirs(projectDir, { env }),
+    ...explicitRoots,
+  ]).filter((dir) => safeExists(path.join(dir, FEEDBACK_LOG)) || safeExists(path.join(dir, MEMORY_LOG)) || safeExists(path.join(dir, STATUSLINE_CACHE)));
+}
+
+function normalizeSignal(signal) {
+  const raw = String(signal || '').toLowerCase();
+  if (raw === 'positive' || raw === 'up' || raw === 'thumbs_up') return 'positive';
+  if (raw === 'negative' || raw === 'down' || raw === 'thumbs_down') return 'negative';
+  return raw || null;
+}
+
+function stableEntryKey(entry = {}, source = {}) {
+  const id = entry.id || entry.feedbackId || entry.sourceFeedbackId;
+  if (id) return `id:${id}`;
+  const material = JSON.stringify({
+    sourcePath: source.logPath || '',
+    sourceIndex: Number.isFinite(source.index) ? source.index : -1,
+    entry,
+  });
+  return `sha:${crypto.createHash('sha256').update(material).digest('hex')}`;
+}
+
+function sortByTimestamp(entries) {
+  return entries.sort((a, b) => {
+    const at = a.timestamp ? Date.parse(a.timestamp) : 0;
+    const bt = b.timestamp ? Date.parse(b.timestamp) : 0;
+    return (Number.isFinite(at) ? at : 0) - (Number.isFinite(bt) ? bt : 0);
+  });
+}
+
+function collectAggregateLogEntries(fileName, options = {}) {
+  const stores = listFeedbackStoreDirs(options);
+  const seen = new Set();
+  const entries = [];
+
+  for (const dir of stores) {
+    const logPath = path.join(dir, fileName);
+    if (!safeExists(logPath)) continue;
+    const rows = readJsonl(logPath, { maxLines: 0 }) || [];
+    for (let index = 0; index < rows.length; index += 1) {
+      const rawEntry = rows[index];
+      const entry = { ...rawEntry };
+      if (entry.signal || entry.feedback) entry.signal = normalizeSignal(entry.signal || entry.feedback);
+      const key = stableEntryKey(entry, { logPath, index });
+      if (seen.has(key)) continue;
+      seen.add(key);
+      entries.push({ ...entry, sourceFeedbackDir: dir });
+    }
+  }
+
+  return {
+    entries: sortByTimestamp(entries),
+    stores,
+  };
+}
+
+function summarizeFeedbackEntries(entries) {
+  let totalPositive = 0;
+  let totalNegative = 0;
+  let rubricSamples = 0;
+
+  for (const entry of entries) {
+    if (entry.signal === 'positive') totalPositive += 1;
+    if (entry.signal === 'negative') totalNegative += 1;
+    if (entry.rubric && entry.rubric.weightedScore != null) rubricSamples += 1;
+  }
+
+  return { totalPositive, totalNegative, rubricSamples };
+}
+
+function createRateWindows(total, totalPositive, approvalRate) {
+  return {
+    '7d': { total: 0, positive: 0, rate: 0 },
+    '30d': { total: 0, positive: 0, rate: 0 },
+    lifetime: { total, positive: totalPositive, rate: approvalRate },
+  };
+}
+
+function applyEntryToRateWindow(window, entry) {
+  window.total += 1;
+  if (entry.signal === 'positive') window.positive += 1;
+}
+
+function updateRateWindows(windows, entries, now = Date.now()) {
+  const sevenDays = 7 * 24 * 60 * 60 * 1000;
+  const thirtyDays = 30 * 24 * 60 * 60 * 1000;
+
+  for (const entry of entries) {
+    const ts = entry.timestamp ? Date.parse(entry.timestamp) : Number.NaN;
+    if (!Number.isFinite(ts)) continue;
+    const age = now - ts;
+    if (age <= sevenDays) applyEntryToRateWindow(windows['7d'], entry);
+    if (age <= thirtyDays) applyEntryToRateWindow(windows['30d'], entry);
+  }
+}
+
+function finalizeRateWindows(windows) {
+  for (const key of ['7d', '30d']) {
+    const window = windows[key];
+    window.rate = 0;
+    if (window.total > 0) {
+      window.rate = Math.round((window.positive / window.total) * 1000) / 1000;
+    }
+  }
+}
+
+function trendFromRateWindows(windows) {
+  const hasTrendData = windows['7d'].total > 0 && windows['30d'].total > 0;
+  if (hasTrendData) {
+    if (windows['7d'].rate > windows['30d'].rate + 0.05) return 'improving';
+    if (windows['7d'].rate < windows['30d'].rate - 0.05) return 'degrading';
+  }
+  return 'stable';
+}
+
+function computeAggregateFeedbackStats(options = {}) {
+  const { entries, stores } = collectAggregateLogEntries(FEEDBACK_LOG, options);
+  const memory = collectAggregateLogEntries(MEMORY_LOG, options);
+  const { totalPositive, totalNegative, rubricSamples } = summarizeFeedbackEntries(entries);
+  const total = totalPositive + totalNegative;
+  const approvalRate = total > 0 ? Math.round((totalPositive / total) * 1000) / 1000 : 0;
+  const windows = createRateWindows(total, totalPositive, approvalRate);
+  updateRateWindows(windows, entries);
+  finalizeRateWindows(windows);
+  const trend = trendFromRateWindows(windows);
+
+  return {
+    total,
+    totalPositive,
+    totalNegative,
+    approvalRate,
+    recentRate: windows['7d'].rate || approvalRate,
+    trend,
+    windows,
+    rubric: { samples: rubricSamples || memory.entries.length, blockedPromotions: 0, failingCriteria: {} },
+    aggregate: {
+      enabled: true,
+      stores: stores.length,
+      feedbackLogPaths: stores.map((dir) => path.join(dir, FEEDBACK_LOG)),
+      memoryStores: memory.stores.length,
+    },
+  };
+}
+
+function getAggregateStatuslineCachePath(options = {}) {
+  const env = options.env || process.env;
+  if (env.THUMBGATE_STATUSLINE_CACHE) return path.resolve(env.THUMBGATE_STATUSLINE_CACHE);
+  return path.join(getHomeDir({ env }), '.thumbgate', STATUSLINE_CACHE);
+}
+
+module.exports = {
+  collectAggregateLogEntries,
+  computeAggregateFeedbackStats,
+  getAggregateStatuslineCachePath,
+  listFeedbackStoreDirs,
+  normalizeSignal,
+  shouldAggregateFeedback,
+};

package/scripts/feedback-loop.js

@@ -1499,10 +1499,44 @@ function captureFeedback(params) {
             totalGates: promoteResult.totalGates,
           });
         } catch { /* activation telemetry is non-critical */ }
+
+        // Trigger Self-Harness Optimizer to propagate the new rules to prompt files & validate
+        try {
+          const { fork } = require('child_process');
+          const localOptimizerPath = path.join(process.cwd(), 'scripts', 'self-harness-optimizer.js');
+          const packageOptimizerPath = path.join(__dirname, 'self-harness-optimizer.js');
+          
+          if (fs.existsSync(localOptimizerPath)) {
+            fork(localOptimizerPath, [], { stdio: 'ignore', detached: true }).unref();
+          } else if (fs.existsSync(packageOptimizerPath)) {
+            fork(packageOptimizerPath, [], { stdio: 'ignore', detached: true }).unref();
+          }
+        } catch (err) {
+          console.error('Failed to trigger self-harness optimizer:', err);
+        }
       }
     } catch { /* Gate promotion is non-critical */ }
   }
 
+  // Auto-export to Obsidian if configured (deferred but tracked)
+  if (process.env.THUMBGATE_OBSIDIAN_VAULT_PATH) {
+    const exportPromise = new Promise((resolve) => {
+      setImmediate(() => {
+        try {
+          const { exportAll } = require('./obsidian-export');
+          exportAll({
+            feedbackDir: FEEDBACK_DIR,
+            outputDir: process.env.THUMBGATE_OBSIDIAN_VAULT_PATH,
+          });
+        } catch (_err) {
+          // Non-critical, do not crash feedback loop
+        }
+        resolve();
+      });
+    });
+    trackBackgroundSideEffect(exportPromise);
+  }
+
   // --- Deferred side-effects (contextFs, RLAIF — non-critical, potentially slow) ---
   setImmediate(() => {
     try {

package/scripts/filesystem-search.js

@@ -53,10 +53,31 @@ function listJsonFiles(dirPath) {
   return results;
 }
 
+const STOPWORDS = new Set([
+  'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'arent', 'as', 'at',
+  'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by',
+  'cant', 'cannot', 'could', 'couldnt', 'did', 'didnt', 'do', 'does', 'doesnt', 'doing', 'dont', 'down', 'during',
+  'each', 'few', 'for', 'from', 'further', 'had', 'hadnt', 'has', 'hasnt', 'have', 'havent', 'having',
+  'he', 'hed', 'hell', 'hes', 'her', 'here', 'heres', 'hers', 'herself', 'him', 'himself', 'his',
+  'how', 'hows', 'i', 'id', 'ill', 'im', 'ive', 'if', 'in', 'into', 'is', 'isnt', 'it', 'its', 'itself',
+  'lets', 'me', 'more', 'most', 'mustnt', 'my', 'myself',
+  'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own',
+  'same', 'shant', 'she', 'shed', 'shell', 'shes', 'should', 'shouldnt', 'so', 'some', 'such',
+  'than', 'that', 'thats', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'theres', 'these', 'they', 'theyd', 'theyll', 'theyre', 'theyve', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very',
+  'was', 'wasnt', 'we', 'wed', 'well', 'were', 'weve', 'werent', 'what', 'whats', 'when', 'whens', 'where', 'wheres', 'which', 'while', 'who', 'whos', 'whom', 'why', 'whys',
+  'with', 'wont', 'would', 'wouldnt', 'you', 'youd', 'youll', 'youre', 'youve', 'your', 'yours', 'yourself', 'yourselves'
+]);
+
 function tokenize(text) {
   return String(text || '').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
 }
 
+function getSearchTokens(queryText) {
+  const allTokens = tokenize(queryText);
+  const contentTokens = allTokens.filter(t => !STOPWORDS.has(t));
+  return contentTokens.length > 0 ? contentTokens : allTokens;
+}
+
 function unique(arr) {
   return [...new Set(arr)];
 }
@@ -82,7 +103,7 @@ function substringBoost(query, text) {
   const q = query.toLowerCase();
   const t = text.toLowerCase();
   if (t.includes(q)) return 0.3;
-  const words = q.split(/\s+/).filter((w) => w.length > 2);
+  const words = q.split(/\s+/).filter((w) => w.length > 2 && !STOPWORDS.has(w));
   const matched = words.filter((w) => t.includes(w)).length;
   return words.length > 0 ? (matched / words.length) * 0.2 : 0;
 }
@@ -117,8 +138,12 @@ function scoreRecord(queryTokens, queryText, record) {
   const recency = recencyScore(record.timestamp);
   const signalBoost = record.signal === 'down' ? 0.05 : 0;
 
+  const matchScore = jaccard + substr;
+  const isWildcard = !queryText || queryText === '*';
+  const score = isWildcard ? (recency + signalBoost || 0.01) : (matchScore > 0 ? matchScore + recency + signalBoost : 0);
+
   return {
-    score: jaccard + substr + recency + signalBoost,
+    score: Number(score.toFixed(4)),
     record,
     matchedTokens: unique(queryTokens).filter((t) => new Set(recordTokens).has(t)),
   };
@@ -129,7 +154,7 @@ function scoreRecord(queryTokens, queryText, record) {
 // ---------------------------------------------------------------------------
 
 function searchFeedbackLog(queryText, limit = 5, options = {}) {
-  const logPath = path.join(getFeedbackDir(), 'feedback-log.jsonl');
+  const logPath = path.join(options.feedbackDir || getFeedbackDir(), 'feedback-log.jsonl');
   let records = readJsonl(logPath);
 
   // SQLite fallback: if JSONL is empty/tiny, pull records from the lesson DB
@@ -156,7 +181,7 @@ function searchFeedbackLog(queryText, limit = 5, options = {}) {
 
   // Wildcard query: return all records sorted by recency
   const isWildcard = queryText === '*' || queryText === '';
-  const queryTokens = isWildcard ? [] : tokenize(queryText);
+  const queryTokens = isWildcard ? [] : getSearchTokens(queryText);
 
   let scored = isWildcard
     ? records.map((r) => ({ score: recencyScore(r.timestamp) || 0.01, record: r, matchedTokens: [] }))
@@ -186,9 +211,9 @@ function searchFeedbackLog(queryText, limit = 5, options = {}) {
 }
 
 function searchContextFs(queryText, limit = 5, options = {}) {
-  const contextDir = getContextFsDir();
+  const contextDir = options.contextDir || (options.feedbackDir ? path.join(options.feedbackDir, 'contextfs') : getContextFsDir());
   const namespaces = options.namespaces || ['memory/error', 'memory/learning', 'rules', 'raw_history'];
-  const queryTokens = tokenize(queryText);
+  const queryTokens = getSearchTokens(queryText);
   const scored = [];
 
   for (const ns of namespaces) {
@@ -221,12 +246,12 @@ function searchContextFs(queryText, limit = 5, options = {}) {
     }));
 }
 
-function searchPreventionRules(queryText, limit = 5) {
-  const rulesPath = path.join(getFeedbackDir(), 'prevention-rules.md');
+function searchPreventionRules(queryText, limit = 5, options = {}) {
+  const rulesPath = path.join(options.feedbackDir || getFeedbackDir(), 'prevention-rules.md');
   if (!fs.existsSync(rulesPath)) return [];
 
   const content = fs.readFileSync(rulesPath, 'utf-8');
-  const queryTokens = tokenize(queryText);
+  const queryTokens = getSearchTokens(queryText);
   const blocks = content.split(/^#{1,3}\s+/m).filter(Boolean);
 
   return blocks
@@ -252,7 +277,7 @@ function searchPreventionRules(queryText, limit = 5) {
 function searchAll(queryText, limit = 10, options = {}) {
   const feedbackResults = searchFeedbackLog(queryText, limit, options);
   const contextResults = searchContextFs(queryText, limit, options);
-  const ruleResults = searchPreventionRules(queryText, limit);
+  const ruleResults = searchPreventionRules(queryText, limit, options);
 
   const merged = [
     ...feedbackResults.map((r) => ({ ...r, _source_type: 'feedback' })),