thumbgate 1.27.18 → 1.27.19

package/scripts/filesystem-search.js

@@ -53,10 +53,31 @@ function listJsonFiles(dirPath) {
   return results;
 }
 
+const STOPWORDS = new Set([
+  'a', 'about', 'above', 'after', 'again', 'against', 'all', 'am', 'an', 'and', 'any', 'are', 'arent', 'as', 'at',
+  'be', 'because', 'been', 'before', 'being', 'below', 'between', 'both', 'but', 'by',
+  'cant', 'cannot', 'could', 'couldnt', 'did', 'didnt', 'do', 'does', 'doesnt', 'doing', 'dont', 'down', 'during',
+  'each', 'few', 'for', 'from', 'further', 'had', 'hadnt', 'has', 'hasnt', 'have', 'havent', 'having',
+  'he', 'hed', 'hell', 'hes', 'her', 'here', 'heres', 'hers', 'herself', 'him', 'himself', 'his',
+  'how', 'hows', 'i', 'id', 'ill', 'im', 'ive', 'if', 'in', 'into', 'is', 'isnt', 'it', 'its', 'itself',
+  'lets', 'me', 'more', 'most', 'mustnt', 'my', 'myself',
+  'no', 'nor', 'not', 'of', 'off', 'on', 'once', 'only', 'or', 'other', 'ought', 'our', 'ours', 'ourselves', 'out', 'over', 'own',
+  'same', 'shant', 'she', 'shed', 'shell', 'shes', 'should', 'shouldnt', 'so', 'some', 'such',
+  'than', 'that', 'thats', 'the', 'their', 'theirs', 'them', 'themselves', 'then', 'there', 'theres', 'these', 'they', 'theyd', 'theyll', 'theyre', 'theyve', 'this', 'those', 'through', 'to', 'too', 'under', 'until', 'up', 'very',
+  'was', 'wasnt', 'we', 'wed', 'well', 'were', 'weve', 'werent', 'what', 'whats', 'when', 'whens', 'where', 'wheres', 'which', 'while', 'who', 'whos', 'whom', 'why', 'whys',
+  'with', 'wont', 'would', 'wouldnt', 'you', 'youd', 'youll', 'youre', 'youve', 'your', 'yours', 'yourself', 'yourselves'
+]);
+
 function tokenize(text) {
   return String(text || '').toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
 }
 
+function getSearchTokens(queryText) {
+  const allTokens = tokenize(queryText);
+  const contentTokens = allTokens.filter(t => !STOPWORDS.has(t));
+  return contentTokens.length > 0 ? contentTokens : allTokens;
+}
+
 function unique(arr) {
   return [...new Set(arr)];
 }
@@ -82,7 +103,7 @@ function substringBoost(query, text) {
   const q = query.toLowerCase();
   const t = text.toLowerCase();
   if (t.includes(q)) return 0.3;
-  const words = q.split(/\s+/).filter((w) => w.length > 2);
+  const words = q.split(/\s+/).filter((w) => w.length > 2 && !STOPWORDS.has(w));
   const matched = words.filter((w) => t.includes(w)).length;
   return words.length > 0 ? (matched / words.length) * 0.2 : 0;
 }
@@ -117,8 +138,12 @@ function scoreRecord(queryTokens, queryText, record) {
   const recency = recencyScore(record.timestamp);
   const signalBoost = record.signal === 'down' ? 0.05 : 0;
 
+  const matchScore = jaccard + substr;
+  const isWildcard = !queryText || queryText === '*';
+  const score = isWildcard ? (recency + signalBoost || 0.01) : (matchScore > 0 ? matchScore + recency + signalBoost : 0);
+
   return {
-    score: jaccard + substr + recency + signalBoost,
+    score: Number(score.toFixed(4)),
     record,
     matchedTokens: unique(queryTokens).filter((t) => new Set(recordTokens).has(t)),
   };
@@ -129,7 +154,7 @@ function scoreRecord(queryTokens, queryText, record) {
 // ---------------------------------------------------------------------------
 
 function searchFeedbackLog(queryText, limit = 5, options = {}) {
-  const logPath = path.join(getFeedbackDir(), 'feedback-log.jsonl');
+  const logPath = path.join(options.feedbackDir || getFeedbackDir(), 'feedback-log.jsonl');
   let records = readJsonl(logPath);
 
   // SQLite fallback: if JSONL is empty/tiny, pull records from the lesson DB
@@ -156,7 +181,7 @@ function searchFeedbackLog(queryText, limit = 5, options = {}) {
 
   // Wildcard query: return all records sorted by recency
   const isWildcard = queryText === '*' || queryText === '';
-  const queryTokens = isWildcard ? [] : tokenize(queryText);
+  const queryTokens = isWildcard ? [] : getSearchTokens(queryText);
 
   let scored = isWildcard
     ? records.map((r) => ({ score: recencyScore(r.timestamp) || 0.01, record: r, matchedTokens: [] }))
@@ -186,9 +211,9 @@ function searchFeedbackLog(queryText, limit = 5, options = {}) {
 }
 
 function searchContextFs(queryText, limit = 5, options = {}) {
-  const contextDir = getContextFsDir();
+  const contextDir = options.contextDir || (options.feedbackDir ? path.join(options.feedbackDir, 'contextfs') : getContextFsDir());
   const namespaces = options.namespaces || ['memory/error', 'memory/learning', 'rules', 'raw_history'];
-  const queryTokens = tokenize(queryText);
+  const queryTokens = getSearchTokens(queryText);
   const scored = [];
 
   for (const ns of namespaces) {
@@ -221,12 +246,12 @@ function searchContextFs(queryText, limit = 5, options = {}) {
     }));
 }
 
-function searchPreventionRules(queryText, limit = 5) {
-  const rulesPath = path.join(getFeedbackDir(), 'prevention-rules.md');
+function searchPreventionRules(queryText, limit = 5, options = {}) {
+  const rulesPath = path.join(options.feedbackDir || getFeedbackDir(), 'prevention-rules.md');
   if (!fs.existsSync(rulesPath)) return [];
 
   const content = fs.readFileSync(rulesPath, 'utf-8');
-  const queryTokens = tokenize(queryText);
+  const queryTokens = getSearchTokens(queryText);
   const blocks = content.split(/^#{1,3}\s+/m).filter(Boolean);
 
   return blocks
@@ -252,7 +277,7 @@ function searchPreventionRules(queryText, limit = 5) {
 function searchAll(queryText, limit = 10, options = {}) {
   const feedbackResults = searchFeedbackLog(queryText, limit, options);
   const contextResults = searchContextFs(queryText, limit, options);
-  const ruleResults = searchPreventionRules(queryText, limit);
+  const ruleResults = searchPreventionRules(queryText, limit, options);
 
   const merged = [
     ...feedbackResults.map((r) => ({ ...r, _source_type: 'feedback' })),

package/scripts/gate-stats.js

@@ -9,6 +9,7 @@ const { sequencePathFor } = require('./risk-scorer');
 
 const PROJECT_ROOT = path.join(__dirname, '..');
 const MANUAL_GATES_PATH = path.join(PROJECT_ROOT, 'config', 'gates', 'default.json');
+const STATS_PATH = path.join(process.env.HOME || '/tmp', '.thumbgate', 'gate-stats.json');
 
 function loadGatesFile(filePath) {
   if (!fs.existsSync(filePath)) return [];
@@ -65,6 +66,15 @@ function calculateStats() {
   // sample can produce a misleading 0.0% floor.
   const bayesErrorRate = tryComputeBayesErrorRate();
 
+  // Calibration: per-gate assessment of whether block actions are well-supported
+  // by negative feedback, or potentially over/under-blocking without confirmation.
+  const calibration = computeCalibration(allGates);
+
+  // First-time fix rate: 1 - (recurringBlocks / totalBlocksAndWarns)
+  // Measures how often a single gate fire resolves the issue vs the agent retrying.
+  // Returns null when there is no recorded block/warn data yet.
+  const firstTimeFixRate = computeFirstTimeFixRate();
+
   return {
     totalGates: allGates.length,
     manualGates: manualGates.length,
@@ -77,10 +87,70 @@ function calculateStats() {
     lastPromotion,
     estimatedHoursSaved,
     bayesErrorRate,
+    calibration,
+    firstTimeFixRate,
     gates: allGates,
   };
 }
 
+/**
+ * Assess each gate's calibration by comparing block occurrences to confirmed
+ * negative feedback counts. A gate with many blocks but no confirming negative
+ * feedback may be over-blocking; one with matching feedback is well-calibrated.
+ *
+ * @param {Array} gates - Combined array of manual + auto-promoted gate objects
+ * @returns {Array<{gateId: string, occurrences: number, action: string, calibrationNote: string}>}
+ */
+function computeCalibration(gates) {
+  const calibration = [];
+  for (const gate of gates || []) {
+    if (!gate || !gate.id) continue;
+    const occurrences = Number(gate.occurrences || 0);
+    const action = gate.action || 'unknown';
+    // Only annotate gates with recorded occurrence data
+    if (occurrences === 0) continue;
+
+    if (action === 'block') {
+      const confirmedNegative = Number(gate.confirmedNegative || gate.negativeCount || 0);
+      let calibrationNote;
+      if (occurrences > 10 && confirmedNegative === 0) {
+        calibrationNote = `over-blocking (${occurrences} blocks, 0 confirmed)`;
+      } else if (confirmedNegative > 0) {
+        calibrationNote = `well-calibrated (${occurrences} blocks, ${confirmedNegative} confirmed)`;
+      } else {
+        // Low occurrence count with no feedback — not enough data yet
+        calibrationNote = `insufficient data (${occurrences} blocks, 0 confirmed)`;
+      }
+      calibration.push({ gateId: gate.id, occurrences, action, calibrationNote });
+    }
+  }
+  return calibration;
+}
+
+/**
+ * Compute the first-time fix rate from the persisted gate-stats.json file.
+ *
+ * firstTimeFixRate = 1 - (recurringBlocks / totalBlocksAndWarns)
+ *
+ * Returns null when there are no recorded block/warn events yet.
+ * Returns a number in [0, 1] otherwise, where 1.0 means every gate fire
+ * was a first-time occurrence and 0.0 means every gate fired at least twice.
+ */
+function computeFirstTimeFixRate() {
+  try {
+    if (!fs.existsSync(STATS_PATH)) return null;
+    const raw = fs.readFileSync(STATS_PATH, 'utf8');
+    const data = JSON.parse(raw);
+    const totalBlocksAndWarns = (data.blocked || 0) + (data.warned || 0);
+    if (totalBlocksAndWarns === 0) return null;
+    const recurring = data.recurringBlocks || 0;
+    const rate = 1 - (recurring / totalBlocksAndWarns);
+    return Math.max(0, Math.min(1, rate));
+  } catch {
+    return null;
+  }
+}
+
 function tryComputeBayesErrorRate() {
   try {
     const seqPath = sequencePathFor();
@@ -142,6 +212,13 @@ function formatStats(stats) {
   lines.push(`  Last promotion: ${formatLastPromotion(stats.lastPromotion)}`);
   lines.push(`  Estimated time saved: ~${stats.estimatedHoursSaved} hours`);
   lines.push(`  Bayes error rate: ${formatBayesErrorRate(stats.bayesErrorRate)}`);
+  lines.push(`  First-time fix rate: ${formatFirstTimeFixRate(stats.firstTimeFixRate)}`);
+  if (Array.isArray(stats.calibration) && stats.calibration.length > 0) {
+    lines.push('Calibration:');
+    for (const entry of stats.calibration) {
+      lines.push(`  - ${entry.gateId}: ${entry.calibrationNote}`);
+    }
+  }
   return lines.join('\n');
 }
 
@@ -153,6 +230,14 @@ function formatBayesErrorRate(rate) {
   return `${pct}% — high irreducible error; the feature set can't discriminate`;
 }
 
+function formatFirstTimeFixRate(rate) {
+  if (rate === null || rate === undefined) return 'n/a (no blocks or warns recorded yet)';
+  const pct = (rate * 100).toFixed(1);
+  if (rate >= 0.95) return `${pct}% — agents fix issues on first block (excellent)`;
+  if (rate >= 0.80) return `${pct}% — most blocks resolved first time (good)`;
+  return `${pct}% — recurring blocks detected; agents may be ignoring gate feedback`;
+}
+
 if (require.main === module) {
   try {
     const stats = calculateStats();
@@ -168,7 +253,11 @@ module.exports = {
   formatStats,
   formatLastPromotion,
   formatBayesErrorRate,
+  formatFirstTimeFixRate,
+  computeFirstTimeFixRate,
   loadGatesFile,
   tryComputeBayesErrorRate,
+  computeCalibration,
   MANUAL_GATES_PATH,
+  STATS_PATH,
 };