thumbgate 0.9.10

package/scripts/export-databricks-bundle.js

@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const { getFeedbackPaths } = require('./feedback-loop');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const DEFAULT_PROOF_DIR = process.env.THUMBGATE_PROOF_DIR
+  || path.join(PROJECT_ROOT, 'proof');
+
+function parseArgs(argv) {
+  const args = {};
+  argv.forEach((arg) => {
+    if (!arg.startsWith('--')) return;
+    const [key, ...rest] = arg.slice(2).split('=');
+    args[key] = rest.length ? rest.join('=') : true;
+  });
+  return args;
+}
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+function readJSONL(filePath) {
+  if (!fs.existsSync(filePath)) return [];
+  const raw = fs.readFileSync(filePath, 'utf8').trim();
+  if (!raw) return [];
+  return raw
+    .split('\n')
+    .map((line) => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return null;
+      }
+    })
+    .filter(Boolean);
+}
+
+function readJSON(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function writeJSONL(filePath, rows) {
+  const content = rows.map((row) => JSON.stringify(row)).join('\n');
+  fs.writeFileSync(filePath, content ? `${content}\n` : '');
+}
+
+function getDefaultFeedbackDir() {
+  return getFeedbackPaths().FEEDBACK_DIR;
+}
+
+function toBundleRelativePath(...segments) {
+  return path.posix.join(...segments);
+}
+
+function normalizeBundleRelativePath(relativePath) {
+  return String(relativePath || '').replace(/\\/g, '/');
+}
+
+function walkJsonFiles(dirPath, acc = []) {
+  if (!fs.existsSync(dirPath)) return acc;
+  const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dirPath, entry.name);
+    if (entry.isDirectory()) {
+      walkJsonFiles(fullPath, acc);
+    } else if (entry.isFile() && entry.name.endsWith('.json')) {
+      acc.push(fullPath);
+    }
+  }
+  return acc;
+}
+
+function annotateRows(rows, dataset, sourceFile, exportedAt) {
+  return rows.map((row, index) => ({
+    bundleDataset: dataset,
+    bundleRowNumber: index + 1,
+    bundleExportedAt: exportedAt,
+    bundleSourceFile: sourceFile,
+    ...row,
+  }));
+}
+
+function collectProofReports(proofDir, exportedAt) {
+  return walkJsonFiles(proofDir)
+    .map((filePath, index) => ({
+      bundleDataset: 'proof_reports',
+      bundleRowNumber: index + 1,
+      bundleExportedAt: exportedAt,
+      reportId: path.basename(filePath, '.json'),
+      reportCategory: path.basename(path.dirname(filePath)),
+      reportPath: normalizeBundleRelativePath(path.relative(proofDir, filePath)),
+      report: readJSON(filePath),
+    }))
+    .filter((row) => row.report);
+}
+
+function buildSqlTemplate(manifest) {
+  const lines = [
+    '-- Databricks bootstrap for the exported analytics bundle.',
+    '-- Replace __CATALOG__, __SCHEMA__, and __BUNDLE_ROOT__ before running.',
+    '',
+    'CREATE SCHEMA IF NOT EXISTS __CATALOG__.__SCHEMA__;',
+    '',
+  ];
+
+  for (const table of manifest.tables) {
+    lines.push(`CREATE OR REPLACE TABLE __CATALOG__.__SCHEMA__.${table.tableName} AS`);
+    lines.push('SELECT *, _metadata.file_path AS source_file');
+    lines.push(`FROM read_files('__BUNDLE_ROOT__/${normalizeBundleRelativePath(table.relativePath)}', format => 'json');`);
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+function timestampSlug() {
+  return new Date().toISOString().replace(/[:.]/g, '-');
+}
+
+function exportDatabricksBundle(feedbackDir = getDefaultFeedbackDir(), outputPath, options = {}) {
+  const resolvedFeedbackDir = path.resolve(feedbackDir || getDefaultFeedbackDir());
+  const resolvedProofDir = path.resolve(options.proofDir || DEFAULT_PROOF_DIR);
+  const exportedAt = new Date().toISOString();
+  const bundlePath = path.resolve(outputPath || path.join(
+    resolvedFeedbackDir,
+    'analytics',
+    `databricks-${timestampSlug()}`
+  ));
+  const tablesDir = path.join(bundlePath, 'tables');
+  ensureDir(tablesDir);
+
+  const datasets = [
+    {
+      tableName: 'feedback_events',
+      sourcePath: path.join(resolvedFeedbackDir, 'feedback-log.jsonl'),
+      description: 'Raw ThumbGate feedback events from feedback-log.jsonl',
+    },
+    {
+      tableName: 'memory_records',
+      sourcePath: path.join(resolvedFeedbackDir, 'memory-log.jsonl'),
+      description: 'Promoted learning and mistake memories from memory-log.jsonl',
+    },
+    {
+      tableName: 'feedback_sequences',
+      sourcePath: path.join(resolvedFeedbackDir, 'feedback-sequences.jsonl'),
+      description: 'Sequence-model training rows derived from accepted feedback',
+    },
+    {
+      tableName: 'feedback_attributions',
+      sourcePath: path.join(resolvedFeedbackDir, 'attributed-feedback.jsonl'),
+      description: 'Tool-call attribution rows for negative feedback events',
+    },
+  ];
+
+  const tables = datasets.map((dataset) => {
+    const rows = annotateRows(
+      readJSONL(dataset.sourcePath),
+      dataset.tableName,
+      path.basename(dataset.sourcePath),
+      exportedAt,
+    );
+    const fileName = `${dataset.tableName}.jsonl`;
+    const relativePath = toBundleRelativePath('tables', fileName);
+    writeJSONL(path.join(tablesDir, fileName), rows);
+    return {
+      tableName: dataset.tableName,
+      relativePath,
+      rowCount: rows.length,
+      description: dataset.description,
+    };
+  });
+
+  const proofRows = collectProofReports(resolvedProofDir, exportedAt);
+  const proofRelativePath = toBundleRelativePath('tables', 'proof_reports.jsonl');
+  writeJSONL(path.join(tablesDir, 'proof_reports.jsonl'), proofRows);
+  tables.push({
+    tableName: 'proof_reports',
+    relativePath: proofRelativePath,
+    rowCount: proofRows.length,
+    description: 'Machine-readable proof artifacts discovered under proof/**/*.json',
+  });
+
+  const manifest = {
+    format: 'databricks-analytics-bundle',
+    version: 1,
+    exportedAt,
+    bundlePath,
+    feedbackDir: resolvedFeedbackDir,
+    proofDir: resolvedProofDir,
+    placeholders: {
+      catalog: '__CATALOG__',
+      schema: '__SCHEMA__',
+      bundleRoot: '__BUNDLE_ROOT__',
+    },
+    tables,
+  };
+
+  const manifestPath = path.join(bundlePath, 'manifest.json');
+  const sqlTemplatePath = path.join(bundlePath, 'load_databricks.sql');
+  fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
+  fs.writeFileSync(sqlTemplatePath, buildSqlTemplate(manifest) + '\n');
+
+  return {
+    bundlePath,
+    manifestPath,
+    sqlTemplatePath,
+    tableCount: tables.length,
+    totalRows: tables.reduce((sum, table) => sum + table.rowCount, 0),
+    tables,
+  };
+}
+
+module.exports = {
+  DEFAULT_PROOF_DIR,
+  buildSqlTemplate,
+  collectProofReports,
+  exportDatabricksBundle,
+  getDefaultFeedbackDir,
+  readJSONL,
+  toBundleRelativePath,
+};
+
+if (require.main === module) {
+  const args = parseArgs(process.argv.slice(2));
+  const result = exportDatabricksBundle(
+    args['feedback-dir'],
+    args.output,
+    { proofDir: args['proof-dir'] }
+  );
+  console.log(JSON.stringify(result, null, 2));
+}

package/scripts/export-dpo-pairs.js

@@ -0,0 +1,345 @@
+#!/usr/bin/env node
+/**
+ * DPO Preference Pair Exporter
+ *
+ * Transforms error + learning memories into DPO JSONL triples.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { traceForDpoPair, aggregateTraces } = require('./code-reasoning');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const FEEDBACK_DIR = process.env.THUMBGATE_FEEDBACK_DIR || path.join(PROJECT_ROOT, '.claude', 'memory', 'feedback');
+const DEFAULT_LOCAL_MEMORY_LOG = path.join(FEEDBACK_DIR, 'memory-log.jsonl');
+
+function readJSONL(filePath) {
+  if (!fs.existsSync(filePath)) return [];
+  const raw = fs.readFileSync(filePath, 'utf-8').trim();
+  if (!raw) return [];
+  return raw
+    .split('\n')
+    .map((line) => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return null;
+      }
+    })
+    .filter(Boolean);
+}
+
+function extractDomainKeys(memory) {
+  const keys = new Set();
+  const genericTags = new Set(['feedback', 'positive', 'negative']);
+
+  if (Array.isArray(memory.tags)) {
+    for (const tag of memory.tags) {
+      if (!genericTags.has(tag)) keys.add(tag);
+    }
+  }
+
+  const titleWords = (memory.title || '')
+    .replace(/^(MISTAKE|SUCCESS|ERROR|LEARNING|PREFERENCE):\s*/i, '')
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((w) => w.length > 3)
+    .slice(0, 3);
+
+  for (const word of titleWords) keys.add(word);
+
+  return Array.from(keys);
+}
+
+function domainOverlap(keysA, keysB) {
+  const setB = new Set(keysB);
+  let overlap = 0;
+  for (const key of keysA) {
+    if (setB.has(key)) overlap++;
+  }
+  return overlap;
+}
+
+function inferPrompt(error, learning) {
+  const shared = (error.tags || []).filter((t) => (learning.tags || []).includes(t));
+  if (shared.length > 0) {
+    return `Task domain: ${shared.join(', ')}. How should the agent handle this scenario?`;
+  }
+
+  const scenario = (error.title || '').replace(/^(MISTAKE|ERROR):\s*/i, '').trim();
+  if (scenario) return `Scenario: ${scenario}. What is the better response?`;
+  return 'How should the agent respond in this situation?';
+}
+
+function getRubricWeightedScore(memory) {
+  if (!memory || !memory.rubricSummary) return null;
+  const score = Number(memory.rubricSummary.weightedScore);
+  if (!Number.isFinite(score)) return null;
+  return score;
+}
+
+function buildRubricDelta(error, learning) {
+  const errorScore = getRubricWeightedScore(error);
+  const learningScore = getRubricWeightedScore(learning);
+  if (errorScore == null && learningScore == null) return null;
+  const delta = (learningScore != null && errorScore != null)
+    ? Math.round((learningScore - errorScore) * 1000) / 1000
+    : null;
+  return {
+    learningWeightedScore: learningScore,
+    errorWeightedScore: errorScore,
+    weightedDelta: delta,
+    errorFailingCriteria: error && error.rubricSummary ? error.rubricSummary.failingCriteria || [] : [],
+    learningFailingCriteria: learning && learning.rubricSummary ? learning.rubricSummary.failingCriteria || [] : [],
+  };
+}
+
+/**
+ * Find distractor errors for a learning: same-domain errors that are NOT
+ * the best match but look similar (high overlap, low rubric delta).
+ * These are harder negatives that train retrieval precision.
+ */
+function findDistractors(learning, errorKeys, usedErrorId, maxDistractors) {
+  const learningKeys = extractDomainKeys(learning);
+  return errorKeys
+    .filter((err) => err.memory.id !== usedErrorId)
+    .map((err) => {
+      const overlap = domainOverlap(err.keys, learningKeys);
+      return { memory: err.memory, overlap };
+    })
+    .filter((d) => d.overlap > 0)
+    .sort((a, b) => b.overlap - a.overlap)
+    .slice(0, maxDistractors)
+    .map((d) => ({
+      id: d.memory.id,
+      title: d.memory.title,
+      content: d.memory.content,
+      overlap: d.overlap,
+    }));
+}
+
+function buildDpoPairs(errors, learnings, { maxDistractors = 2 } = {}) {
+  const pairs = [];
+  const usedErrors = new Set();
+  const usedLearnings = new Set();
+
+  const errorKeys = errors.map((e) => ({ memory: e, keys: extractDomainKeys(e) }));
+  const learningKeys = learnings.map((l) => ({ memory: l, keys: extractDomainKeys(l) }));
+
+  for (const err of errorKeys) {
+    let best = null;
+    let bestScore = 0;
+    let bestOverlap = 0;
+
+    for (const learn of learningKeys) {
+      if (usedLearnings.has(learn.memory.id)) continue;
+      const overlap = domainOverlap(err.keys, learn.keys);
+      const rubric = buildRubricDelta(err.memory, learn.memory);
+      const rubricDelta = rubric && rubric.weightedDelta != null ? rubric.weightedDelta : 0;
+      const score = overlap + Math.max(0, rubricDelta);
+      if (score > bestScore) {
+        best = learn;
+        bestScore = score;
+        bestOverlap = overlap;
+      }
+    }
+
+    if (best && bestScore > 0 && bestOverlap > 0) {
+      const distractors = findDistractors(best.memory, errorKeys, err.memory.id, maxDistractors);
+      pairs.push({
+        prompt: inferPrompt(err.memory, best.memory),
+        chosen: best.memory.content,
+        rejected: err.memory.content,
+        distractors: distractors.length > 0 ? distractors : undefined,
+        metadata: {
+          errorId: err.memory.id,
+          learningId: best.memory.id,
+          matchScore: bestScore,
+          overlapScore: domainOverlap(err.keys, best.keys),
+          matchedKeys: err.keys.filter((k) => best.keys.includes(k)),
+          errorTitle: err.memory.title,
+          learningTitle: best.memory.title,
+          rubric: buildRubricDelta(err.memory, best.memory),
+          distractorCount: distractors.length,
+        },
+      });
+      usedErrors.add(err.memory.id);
+      usedLearnings.add(best.memory.id);
+    }
+  }
+
+  return {
+    pairs,
+    unpairedErrors: errors.filter((e) => !usedErrors.has(e.id)),
+    unpairedLearnings: learnings.filter((l) => !usedLearnings.has(l.id)),
+  };
+}
+
+function toJSONL(pairs) {
+  return `${pairs.map((p) => JSON.stringify(p)).join('\n')}\n`;
+}
+
+function exportDpoFromMemories(memories) {
+  const errors = memories.filter((m) => m.category === 'error');
+  const learnings = memories.filter((m) => m.category === 'learning');
+  const result = buildDpoPairs(errors, learnings);
+
+  const traces = result.pairs.map((pair) => traceForDpoPair(pair));
+  const reasoning = aggregateTraces(traces);
+
+  const pairsWithTraces = result.pairs.map((pair, i) => ({
+    ...pair,
+    metadata: {
+      ...pair.metadata,
+      reasoningTrace: {
+        traceId: traces[i].traceId,
+        confidence: traces[i].summary.confidence,
+        passed: traces[i].summary.passed,
+        verified: traces[i].summary.verified,
+        refuted: traces[i].summary.refuted,
+        edgeCases: traces[i].edgeCases,
+      },
+    },
+  }));
+
+  return {
+    pairs: pairsWithTraces,
+    unpairedErrors: result.unpairedErrors,
+    unpairedLearnings: result.unpairedLearnings,
+    errors,
+    learnings,
+    reasoning,
+    jsonl: toJSONL(pairsWithTraces),
+  };
+}
+
+function parseArgs(argv) {
+  const args = {};
+  argv.forEach((arg) => {
+    if (!arg.startsWith('--')) return;
+    const [key, ...rest] = arg.slice(2).split('=');
+    args[key] = rest.length ? rest.join('=') : true;
+  });
+  return args;
+}
+
+function runCli() {
+  const args = parseArgs(process.argv.slice(2));
+
+  if (args.test) {
+    runTests();
+    return;
+  }
+
+  let memories = [];
+
+  if (args.input) {
+    const raw = fs.readFileSync(args.input, 'utf-8');
+    const parsed = JSON.parse(raw);
+    memories = Array.isArray(parsed) ? parsed : parsed.memories || [];
+  } else if (args['from-local']) {
+    memories = readJSONL(DEFAULT_LOCAL_MEMORY_LOG);
+  } else {
+    console.error('Provide --input=<path-to-json> or --from-local');
+    process.exit(1);
+  }
+
+  const result = exportDpoFromMemories(memories);
+  const jsonl = result.jsonl;
+
+  if (args.output) {
+    fs.writeFileSync(args.output, jsonl);
+    console.error(`Wrote ${result.pairs.length} DPO pairs to ${args.output}`);
+  } else {
+    process.stdout.write(jsonl);
+  }
+
+  console.error(`Errors=${result.errors.length} Learnings=${result.learnings.length} Pairs=${result.pairs.length}`);
+  console.error(`Unpaired errors=${result.unpairedErrors.length} Unpaired learnings=${result.unpairedLearnings.length}`);
+}
+
+function runTests() {
+  let passed = 0;
+  let failed = 0;
+
+  function assert(condition, name) {
+    if (condition) {
+      passed++;
+      console.log(`  PASS ${name}`);
+    } else {
+      failed++;
+      console.log(`  FAIL ${name}`);
+    }
+  }
+
+  console.log('\nexport-dpo-pairs.js tests\n');
+
+  const errors = [
+    {
+      id: 1,
+      title: 'MISTAKE: Claimed done with no test proof',
+      content: 'Claimed completion without running tests.',
+      category: 'error',
+      tags: ['verification', 'feedback'],
+      rubricSummary: {
+        weightedScore: 0.32,
+        failingCriteria: ['verification_evidence'],
+        failingGuardrails: ['testsPassed'],
+      },
+    },
+    {
+      id: 2,
+      title: 'MISTAKE: Generic mismatch',
+      content: 'No matching learning memory for this domain.',
+      category: 'error',
+      tags: ['unique-tag'],
+    },
+  ];
+
+  const learnings = [
+    {
+      id: 10,
+      title: 'SUCCESS: Always run tests before completion claims',
+      content: 'Run tests and include output before saying complete.',
+      category: 'learning',
+      tags: ['verification', 'feedback'],
+      rubricSummary: {
+        weightedScore: 0.89,
+        failingCriteria: [],
+        failingGuardrails: [],
+      },
+    },
+  ];
+
+  const result = buildDpoPairs(errors, learnings);
+  assert(result.pairs.length === 1, 'one pair built from overlapping domain keys');
+  assert(result.unpairedErrors.length === 1, 'unpaired error left when no match exists');
+  assert(result.unpairedLearnings.length === 0, 'no unpaired learnings');
+
+  const jsonl = toJSONL(result.pairs);
+  assert(jsonl.endsWith('\n'), 'JSONL output ends with newline');
+
+  const parsed = JSON.parse(jsonl.trim());
+  assert(parsed.prompt.includes('verification'), 'inferred prompt includes shared domain');
+  assert(parsed.metadata.rubric.weightedDelta > 0, 'rubric delta metadata is attached');
+
+  console.log(`\nResults: ${passed} passed, ${failed} failed\n`);
+  process.exit(failed > 0 ? 1 : 0);
+}
+
+module.exports = {
+  readJSONL,
+  extractDomainKeys,
+  domainOverlap,
+  inferPrompt,
+  buildRubricDelta,
+  findDistractors,
+  buildDpoPairs,
+  toJSONL,
+  exportDpoFromMemories,
+  DEFAULT_LOCAL_MEMORY_LOG,
+};
+
+if (require.main === module) {
+  runCli();
+}