thumbgate 0.9.9

package/scripts/experiment-tracker.js

@@ -0,0 +1,267 @@
+'use strict';
+/**
+ * Experiment Tracker (AUTORESEARCH-01)
+ *
+ * Tracks autonomous iteration experiments inspired by Karpathy's autoresearch.
+ * Each experiment = a config mutation + test run + measurable score.
+ * Keeps/discards based on whether score improves over baseline.
+ *
+ * Persists experiments to .thumbgate/experiments.jsonl and writes a progress
+ * summary to .thumbgate/experiment-progress.json.
+ *
+ * Zero external dependencies — uses only node:* and existing project modules.
+ *
+ * Exports: createExperiment, recordResult, getProgress, getBestExperiment,
+ *          loadExperiments, EXPERIMENT_LOG_PATH
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { getFeedbackPaths, readJSONL } = require('./feedback-loop');
+
+// ---------------------------------------------------------------------------
+// Paths
+// ---------------------------------------------------------------------------
+
+function getExperimentPaths() {
+  const { FEEDBACK_DIR } = getFeedbackPaths();
+  return {
+    logPath: path.join(FEEDBACK_DIR, 'experiments.jsonl'),
+    progressPath: path.join(FEEDBACK_DIR, 'experiment-progress.json'),
+  };
+}
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+function appendJSONL(filePath, record) {
+  ensureDir(path.dirname(filePath));
+  fs.appendFileSync(filePath, `${JSON.stringify(record)}\n`);
+}
+
+// ---------------------------------------------------------------------------
+// Experiment Lifecycle
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a new experiment record. Does NOT execute anything — just records
+ * the intent so the runner can fill in results later.
+ *
+ * @param {object} params
+ * @param {string} params.name - Human-readable experiment name
+ * @param {string} params.hypothesis - What change is being tested
+ * @param {string} params.mutationType - Category of mutation (config|prompt|code|threshold)
+ * @param {object} [params.mutation] - The actual mutation applied (key/value diff)
+ * @param {string} [params.branch] - Git branch name for this experiment
+ * @returns {object} experiment record with id and status='pending'
+ */
+function createExperiment(params) {
+  if (!params || !params.name || !params.hypothesis) {
+    throw new Error('Experiment requires name and hypothesis');
+  }
+
+  const validMutationTypes = ['config', 'prompt', 'code', 'threshold'];
+  const mutationType = params.mutationType || 'config';
+  if (!validMutationTypes.includes(mutationType)) {
+    throw new Error(`Invalid mutationType "${mutationType}". Must be one of: ${validMutationTypes.join(', ')}`);
+  }
+
+  const experiment = {
+    id: `exp_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+    name: params.name,
+    hypothesis: params.hypothesis,
+    mutationType,
+    mutation: params.mutation || null,
+    branch: params.branch || null,
+    status: 'pending',
+    createdAt: new Date().toISOString(),
+    completedAt: null,
+    baseline: null,
+    result: null,
+    score: null,
+    kept: null,
+    reason: null,
+  };
+
+  const { logPath } = getExperimentPaths();
+  appendJSONL(logPath, experiment);
+  return experiment;
+}
+
+/**
+ * Record the result of a completed experiment.
+ *
+ * @param {object} params
+ * @param {string} params.experimentId - ID from createExperiment
+ * @param {number} params.score - Measured score (higher = better)
+ * @param {number} params.baseline - Baseline score to compare against
+ * @param {boolean} [params.testsPassed] - Whether the test suite passed
+ * @param {object} [params.metrics] - Additional metrics (coverage, duration, etc.)
+ * @returns {object} Updated experiment with kept/discarded decision
+ */
+function recordResult(params) {
+  if (!params || !params.experimentId) {
+    throw new Error('recordResult requires experimentId');
+  }
+  if (typeof params.score !== 'number' || typeof params.baseline !== 'number') {
+    throw new Error('recordResult requires numeric score and baseline');
+  }
+
+  const { logPath } = getExperimentPaths();
+  const experiments = loadExperiments();
+  const experiment = experiments.find(e => e.id === params.experimentId);
+
+  if (!experiment) {
+    throw new Error(`Experiment ${params.experimentId} not found`);
+  }
+
+  const improved = params.score > params.baseline;
+  const testsPassed = params.testsPassed !== false;
+  const kept = improved && testsPassed;
+
+  const result = {
+    ...experiment,
+    status: 'completed',
+    completedAt: new Date().toISOString(),
+    baseline: params.baseline,
+    score: params.score,
+    delta: params.score - params.baseline,
+    testsPassed,
+    metrics: params.metrics || null,
+    kept,
+    reason: !testsPassed
+      ? 'Tests failed — discarded'
+      : improved
+        ? `Score improved by ${(params.score - params.baseline).toFixed(4)}`
+        : `Score did not improve (${params.score} <= ${params.baseline})`,
+  };
+
+  appendJSONL(logPath, result);
+  updateProgress();
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// Progress Tracking
+// ---------------------------------------------------------------------------
+
+/**
+ * Load all experiment records from the JSONL log.
+ * @returns {object[]}
+ */
+function loadExperiments() {
+  const { logPath } = getExperimentPaths();
+  return readJSONL(logPath);
+}
+
+/**
+ * Recompute and persist experiment progress summary.
+ * @returns {object} progress summary
+ */
+function updateProgress() {
+  const experiments = loadExperiments();
+  const completed = experiments.filter(e => e.status === 'completed');
+  const kept = completed.filter(e => e.kept === true);
+  const discarded = completed.filter(e => e.kept === false);
+  const pending = experiments.filter(e => e.status === 'pending');
+
+  const bestExperiment = kept.length > 0
+    ? kept.reduce((best, e) => (e.delta || 0) > (best.delta || 0) ? e : best, kept[0])
+    : null;
+
+  const progress = {
+    totalExperiments: experiments.length,
+    completed: completed.length,
+    kept: kept.length,
+    discarded: discarded.length,
+    pending: pending.length,
+    keepRate: completed.length > 0
+      ? (kept.length / completed.length * 100).toFixed(1)
+      : '0.0',
+    bestExperiment: bestExperiment
+      ? { id: bestExperiment.id, name: bestExperiment.name, delta: bestExperiment.delta }
+      : null,
+    lastUpdated: new Date().toISOString(),
+  };
+
+  const { progressPath } = getExperimentPaths();
+  ensureDir(path.dirname(progressPath));
+  fs.writeFileSync(progressPath, JSON.stringify(progress, null, 2) + '\n');
+  return progress;
+}
+
+/**
+ * Get current experiment progress.
+ * @returns {object} progress summary
+ */
+function getProgress() {
+  const { progressPath } = getExperimentPaths();
+  if (fs.existsSync(progressPath)) {
+    try {
+      return JSON.parse(fs.readFileSync(progressPath, 'utf-8'));
+    } catch {
+      // Fall through to recompute
+    }
+  }
+  return updateProgress();
+}
+
+/**
+ * Get the best-performing kept experiment.
+ * @returns {object|null}
+ */
+function getBestExperiment() {
+  const experiments = loadExperiments();
+  const kept = experiments.filter(e => e.kept === true);
+  if (kept.length === 0) return null;
+  return kept.reduce((best, e) => (e.delta || 0) > (best.delta || 0) ? e : best, kept[0]);
+}
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+if (require.main === module) {
+  const args = {};
+  process.argv.slice(2).forEach(arg => {
+    if (!arg.startsWith('--')) return;
+    const [key, ...rest] = arg.slice(2).split('=');
+    args[key] = rest.length > 0 ? rest.join('=') : true;
+  });
+
+  if (args.progress) {
+    console.log(JSON.stringify(getProgress(), null, 2));
+  } else if (args.best) {
+    const best = getBestExperiment();
+    console.log(best ? JSON.stringify(best, null, 2) : 'No kept experiments yet.');
+  } else if (args.list) {
+    const exps = loadExperiments().filter(e => e.status === 'completed');
+    console.log(`${exps.length} completed experiments (${exps.filter(e => e.kept).length} kept)`);
+    exps.slice(-10).forEach(e => {
+      const icon = e.kept ? '✓' : '✗';
+      console.log(`  ${icon} ${e.name} — delta: ${(e.delta || 0).toFixed(4)}`);
+    });
+  } else {
+    console.log(`Usage:
+  node scripts/experiment-tracker.js --progress
+  node scripts/experiment-tracker.js --best
+  node scripts/experiment-tracker.js --list`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+
+module.exports = {
+  createExperiment,
+  recordResult,
+  getProgress,
+  getBestExperiment,
+  loadExperiments,
+  updateProgress,
+  getExperimentPaths,
+};

package/scripts/export-databricks-bundle.js

@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const { getFeedbackPaths } = require('./feedback-loop');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const DEFAULT_PROOF_DIR = process.env.THUMBGATE_PROOF_DIR
+  || path.join(PROJECT_ROOT, 'proof');
+
+function parseArgs(argv) {
+  const args = {};
+  argv.forEach((arg) => {
+    if (!arg.startsWith('--')) return;
+    const [key, ...rest] = arg.slice(2).split('=');
+    args[key] = rest.length ? rest.join('=') : true;
+  });
+  return args;
+}
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+function readJSONL(filePath) {
+  if (!fs.existsSync(filePath)) return [];
+  const raw = fs.readFileSync(filePath, 'utf8').trim();
+  if (!raw) return [];
+  return raw
+    .split('\n')
+    .map((line) => {
+      try {
+        return JSON.parse(line);
+      } catch {
+        return null;
+      }
+    })
+    .filter(Boolean);
+}
+
+function readJSON(filePath) {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+  } catch {
+    return null;
+  }
+}
+
+function writeJSONL(filePath, rows) {
+  const content = rows.map((row) => JSON.stringify(row)).join('\n');
+  fs.writeFileSync(filePath, content ? `${content}\n` : '');
+}
+
+function getDefaultFeedbackDir() {
+  return getFeedbackPaths().FEEDBACK_DIR;
+}
+
+function toBundleRelativePath(...segments) {
+  return path.posix.join(...segments);
+}
+
+function normalizeBundleRelativePath(relativePath) {
+  return String(relativePath || '').replace(/\\/g, '/');
+}
+
+function walkJsonFiles(dirPath, acc = []) {
+  if (!fs.existsSync(dirPath)) return acc;
+  const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = path.join(dirPath, entry.name);
+    if (entry.isDirectory()) {
+      walkJsonFiles(fullPath, acc);
+    } else if (entry.isFile() && entry.name.endsWith('.json')) {
+      acc.push(fullPath);
+    }
+  }
+  return acc;
+}
+
+function annotateRows(rows, dataset, sourceFile, exportedAt) {
+  return rows.map((row, index) => ({
+    bundleDataset: dataset,
+    bundleRowNumber: index + 1,
+    bundleExportedAt: exportedAt,
+    bundleSourceFile: sourceFile,
+    ...row,
+  }));
+}
+
+function collectProofReports(proofDir, exportedAt) {
+  return walkJsonFiles(proofDir)
+    .map((filePath, index) => ({
+      bundleDataset: 'proof_reports',
+      bundleRowNumber: index + 1,
+      bundleExportedAt: exportedAt,
+      reportId: path.basename(filePath, '.json'),
+      reportCategory: path.basename(path.dirname(filePath)),
+      reportPath: normalizeBundleRelativePath(path.relative(proofDir, filePath)),
+      report: readJSON(filePath),
+    }))
+    .filter((row) => row.report);
+}
+
+function buildSqlTemplate(manifest) {
+  const lines = [
+    '-- Databricks bootstrap for the exported analytics bundle.',
+    '-- Replace __CATALOG__, __SCHEMA__, and __BUNDLE_ROOT__ before running.',
+    '',
+    'CREATE SCHEMA IF NOT EXISTS __CATALOG__.__SCHEMA__;',
+    '',
+  ];
+
+  for (const table of manifest.tables) {
+    lines.push(`CREATE OR REPLACE TABLE __CATALOG__.__SCHEMA__.${table.tableName} AS`);
+    lines.push('SELECT *, _metadata.file_path AS source_file');
+    lines.push(`FROM read_files('__BUNDLE_ROOT__/${normalizeBundleRelativePath(table.relativePath)}', format => 'json');`);
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+function timestampSlug() {
+  return new Date().toISOString().replace(/[:.]/g, '-');
+}
+
+function exportDatabricksBundle(feedbackDir = getDefaultFeedbackDir(), outputPath, options = {}) {
+  const resolvedFeedbackDir = path.resolve(feedbackDir || getDefaultFeedbackDir());
+  const resolvedProofDir = path.resolve(options.proofDir || DEFAULT_PROOF_DIR);
+  const exportedAt = new Date().toISOString();
+  const bundlePath = path.resolve(outputPath || path.join(
+    resolvedFeedbackDir,
+    'analytics',
+    `databricks-${timestampSlug()}`
+  ));
+  const tablesDir = path.join(bundlePath, 'tables');
+  ensureDir(tablesDir);
+
+  const datasets = [
+    {
+      tableName: 'feedback_events',
+      sourcePath: path.join(resolvedFeedbackDir, 'feedback-log.jsonl'),
+      description: 'Raw ThumbGate feedback events from feedback-log.jsonl',
+    },
+    {
+      tableName: 'memory_records',
+      sourcePath: path.join(resolvedFeedbackDir, 'memory-log.jsonl'),
+      description: 'Promoted learning and mistake memories from memory-log.jsonl',
+    },
+    {
+      tableName: 'feedback_sequences',
+      sourcePath: path.join(resolvedFeedbackDir, 'feedback-sequences.jsonl'),
+      description: 'Sequence-model training rows derived from accepted feedback',
+    },
+    {
+      tableName: 'feedback_attributions',
+      sourcePath: path.join(resolvedFeedbackDir, 'attributed-feedback.jsonl'),
+      description: 'Tool-call attribution rows for negative feedback events',
+    },
+  ];
+
+  const tables = datasets.map((dataset) => {
+    const rows = annotateRows(
+      readJSONL(dataset.sourcePath),
+      dataset.tableName,
+      path.basename(dataset.sourcePath),
+      exportedAt,
+    );
+    const fileName = `${dataset.tableName}.jsonl`;
+    const relativePath = toBundleRelativePath('tables', fileName);
+    writeJSONL(path.join(tablesDir, fileName), rows);
+    return {
+      tableName: dataset.tableName,
+      relativePath,
+      rowCount: rows.length,
+      description: dataset.description,
+    };
+  });
+
+  const proofRows = collectProofReports(resolvedProofDir, exportedAt);
+  const proofRelativePath = toBundleRelativePath('tables', 'proof_reports.jsonl');
+  writeJSONL(path.join(tablesDir, 'proof_reports.jsonl'), proofRows);
+  tables.push({
+    tableName: 'proof_reports',
+    relativePath: proofRelativePath,
+    rowCount: proofRows.length,
+    description: 'Machine-readable proof artifacts discovered under proof/**/*.json',
+  });
+
+  const manifest = {
+    format: 'databricks-analytics-bundle',
+    version: 1,
+    exportedAt,
+    bundlePath,
+    feedbackDir: resolvedFeedbackDir,
+    proofDir: resolvedProofDir,
+    placeholders: {
+      catalog: '__CATALOG__',
+      schema: '__SCHEMA__',
+      bundleRoot: '__BUNDLE_ROOT__',
+    },
+    tables,
+  };
+
+  const manifestPath = path.join(bundlePath, 'manifest.json');
+  const sqlTemplatePath = path.join(bundlePath, 'load_databricks.sql');
+  fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
+  fs.writeFileSync(sqlTemplatePath, buildSqlTemplate(manifest) + '\n');
+
+  return {
+    bundlePath,
+    manifestPath,
+    sqlTemplatePath,
+    tableCount: tables.length,
+    totalRows: tables.reduce((sum, table) => sum + table.rowCount, 0),
+    tables,
+  };
+}
+
+module.exports = {
+  DEFAULT_PROOF_DIR,
+  buildSqlTemplate,
+  collectProofReports,
+  exportDatabricksBundle,
+  getDefaultFeedbackDir,
+  readJSONL,
+  toBundleRelativePath,
+};
+
+if (require.main === module) {
+  const args = parseArgs(process.argv.slice(2));
+  const result = exportDatabricksBundle(
+    args['feedback-dir'],
+    args.output,
+    { proofDir: args['proof-dir'] }
+  );
+  console.log(JSON.stringify(result, null, 2));
+}