thumbgate 1.34.3 → 1.37.1

package/scripts/codex-runbook-flywheel.js

@@ -0,0 +1,318 @@
+'use strict';
+
+/**
+ * Codex Runbook Flywheel — ThumbGate steal of OpenAI's "Automating repetitive
+ * work at OpenAI with Codex" workflow (developers.openai.com blog).
+ *
+ * The episode's loop, verbatim in spirit:
+ *   run the workflow -> document it (commands, results, dead ends,
+ *   decisions) -> the next run reuses what earlier runs learned.
+ *
+ * Four enforcement primitives, all deterministic:
+ *
+ *   1. Plan-before-act. Codex writes the plan into the notebook and WAITS
+ *      for approval before executing. -> newRunbook() starts at 'plan';
+ *      execute() refuses to run without an approved plan.
+ *
+ *   2. Consequential choices need human judgment. Automatic approval review
+ *      handles eligible actions WITHOUT changing permission boundaries.
+ *      -> autoReview() marks only bounded, reversible actions eligible;
+ *      consequential ones stay on the human queue.
+ *
+ *   3. Capture decisions that would otherwise vanish into chat history:
+ *      which option was chosen, why, what to do differently next time.
+ *      -> captureDecision() appends to a per-workflow decision log.
+ *
+ *   4. Cheap discovery for the next run: a companion index over past
+ *      runbooks (the *.index.md analog), searchable by workflow name.
+ *      -> buildIndex() / discoverContext().
+ */
+
+const RUN_STATES = Object.freeze(['plan', 'approved', 'running', 'done', 'blocked']);
+
+const CONSEQUENTIAL = Object.freeze([
+  'payment', 'external-email', 'production-deploy', 'delete', 'permission-change', 'publish',
+]);
+
+const ALLOWED_ACTIONS = Object.freeze([
+  'read-file', 'list-files', 'grep', 'search', 'read-notes',
+  'list-models', 'estimate-cost', 'check-quota',
+]);
+
+/**
+ * Create a runbook. Starts at the plan stage — execution is impossible until
+ * a human approves. Mirrors "wait for me to review and approve the plan."
+ */
+function newRunbook(workflow, goal) {
+  if (!workflow || !goal) {
+    throw new Error('runbook needs a workflow name and a goal');
+  }
+  return {
+    id: require('crypto').randomUUID(),
+    workflow,
+    goal,
+    state: 'plan',
+    plan: [],
+    decisions: [],
+    steps: [],
+    deadEnds: [],
+    createdAt: new Date().toISOString(),
+  };
+}
+
+/**
+ * Normalize a step to its stable identifier. String steps are used directly.
+ * Object steps must have an `id` property; otherwise approval is rejected.
+ * This prevents the approved-snapshot and executeStep from disagreeing due
+ * to object reference identity.
+ */
+function stepId(step) {
+  if (typeof step === 'string') return step.trim() || null;
+  if (step && typeof step === 'object' && typeof step.id === 'string' && step.id.trim()) {
+    return step.id.trim();
+  }
+  return null;
+}
+
+/**
+ * Record the plan and approve it. Approval is an explicit human act.
+ * Steps are normalized to immutable identifiers so that executeStep can
+ * match against the approved plan without relying on object reference identity.
+ */
+function approvePlan(runbook, plan, approver) {
+  if (runbook.state !== 'plan') {
+    return { ok: false, reason: `cannot approve from state "${runbook.state}"` };
+  }
+  if (!Array.isArray(plan) || plan.length === 0) {
+    return { ok: false, reason: 'an empty plan cannot be approved' };
+  }
+  if (!approver || typeof approver !== 'string' || !approver.trim()) {
+    return { ok: false, reason: 'approval requires a non-empty, trimmed string approver' };
+  }
+  // Normalize each step to a stable identifier. Object entries without a
+  // usable `id` are rejected rather than silently cloned by reference.
+  const normalized = [];
+  for (const s of plan) {
+    const id = stepId(s);
+    if (id === null) {
+      return { ok: false, reason: `step ${JSON.stringify(s)} has no stable id — reject or assign one` };
+    }
+    normalized.push(id);
+  }
+  runbook.approvedBy = approver.trim();
+  runbook.approvedPlan = Object.freeze(normalized);
+  runbook.state = 'approved';
+  runbook.approvedAt = new Date().toISOString();
+  return { ok: true, state: runbook.state };
+}
+
+/**
+ * Automatic approval review for individual actions. Uses an explicit
+ * allowlist of safe action types — fail closed for anything unlisted.
+ */
+function autoReview(action) {
+  if (!action || !action.type || typeof action.type !== 'string') {
+    return { eligible: false, reason: 'missing or invalid action type — human approval required' };
+  }
+  if (!ALLOWED_ACTIONS.includes(action.type)) {
+    return { eligible: false, reason: `"${action.type}" is unlisted — human approval required` };
+  }
+  if (CONSEQUENTIAL.includes(action.type)) {
+    return { eligible: false, reason: `"${action.type}" is consequential — human approval required` };
+  }
+  if (action.irreversible) {
+    return { eligible: false, reason: 'irreversible actions are never auto-approved' };
+  }
+  return { eligible: true, reason: 'bounded and reversible — auto-review passes' };
+}
+
+/**
+ * Execute one step. Refuses unless the plan was approved.
+ */
+function executeStep(runbook, step, outcome) {
+  if (runbook.state !== 'approved' && runbook.state !== 'running') {
+    return { ok: false, reason: `execution refused — runbook state is "${runbook.state}", not approved` };
+  }
+  if (!Array.isArray(runbook.approvedPlan)) {
+    return { ok: false, reason: 'execution refused — no approved plan snapshot on record' };
+  }
+  if (!runbook.approvedPlan.includes(stepId(step))) {
+    return { ok: false, reason: `execution refused — "${step}" is not in the approved plan` };
+  }
+  runbook.state = 'running';
+  runbook.steps.push({ step, outcome: outcome || 'ok', at: new Date().toISOString() });
+  return { ok: true, executed: runbook.steps.length };
+}
+
+/**
+ * Capture a decision that would otherwise disappear into chat history.
+ */
+function captureDecision(runbook, decision) {
+  if (!decision || !decision.choice || !decision.reason) {
+    return { ok: false, reason: 'a decision needs both a choice and a reason' };
+  }
+  runbook.decisions.push({
+    choice: decision.choice,
+    reason: decision.reason,
+    nextTime: decision.nextTime || null,
+    at: new Date().toISOString(),
+  });
+  return { ok: true, decisions: runbook.decisions.length };
+}
+
+/**
+ * Mark a dead end — the article documents these on purpose so the next run
+ * doesn't repeat them.
+ */
+function recordDeadEnd(runbook, deadEnd) {
+  runbook.deadEnds.push({ deadEnd, at: new Date().toISOString() });
+  return { deadEnds: runbook.deadEnds.length };
+}
+
+/**
+ * Close the runbook. Only a running/approved runbook can close, and only
+ * with at least one recorded step.
+ */
+function closeRunbook(runbook) {
+  if (runbook.state !== 'approved' && runbook.state !== 'running') {
+    return { ok: false, reason: `cannot close from state "${runbook.state}"` };
+  }
+  if (runbook.steps.length === 0) {
+    return { ok: false, reason: 'cannot close a runbook with no recorded steps' };
+  }
+  runbook.state = 'done';
+  runbook.completedAt = new Date().toISOString();
+  return { ok: true, state: 'done' };
+}
+
+/**
+ * Build the companion index over completed runbooks (the *.index.md analog).
+ * Full decision and dead-end records are stored alongside counts so later
+ * runs can reuse the captured guidance.
+ */
+function buildIndex(runbooks) {
+  return (runbooks || [])
+    .filter((r) => r.state === 'done')
+    .map((r) => ({
+      workflow: r.workflow,
+      goal: r.goal,
+      steps: r.steps.length,
+      decisions: r.decisions.length,
+      deadEnds: r.deadEnds.length,
+      decisionsLog: r.decisions.map((d) => ({ ...d })),
+      deadEndsLog: r.deadEnds.map((d) => ({ ...d })),
+      completedAt: r.completedAt,
+    }));
+}
+
+/**
+ * Discover prior context for a workflow — what earlier runs learned.
+ * Returns the full decision and dead-end records so callers can act on
+ * the captured guidance, not just counts.
+ */
+function discoverContext(index, workflow) {
+  const prior = (index || []).filter((e) => e.workflow === workflow);
+  return {
+    workflow,
+    priorRuns: prior.length,
+    totalDecisions: prior.reduce((n, e) => n + e.decisions, 0),
+    totalDeadEnds: prior.reduce((n, e) => n + e.deadEnds, 0),
+    decisions: prior.flatMap((e) => (e.decisionsLog || []).map((d) => ({ ...d }))),
+    deadEnds: prior.flatMap((e) => (e.deadEndsLog || []).map((d) => ({ ...d }))),
+    reusable: prior.length > 0,
+  };
+}
+
+function isCliEntrypoint() {
+  return require.main === module;
+}
+
+function parseCliArgs(argv) {
+  const args = argv.slice(2);
+  const mode = { dryRun: false, solve: false };
+  for (const arg of args) {
+    if (arg === '--dry-run') mode.dryRun = true;
+    else if (arg === '--solve') mode.solve = true;
+  }
+  return mode;
+}
+
+function main() {
+  const { dryRun, solve } = parseCliArgs(process.argv);
+  const rb = newRunbook('model-eval', 'Run the evaluation against the current model');
+
+  const premature = executeStep(rb, 'run eval'); // must refuse
+  approvePlan(rb, ['review previous run', 'write plan', 'run eval', 'document'], 'igor');
+  const review = { auto: autoReview({ type: 'read-file' }), blocked: autoReview({ type: 'production-deploy' }) };
+
+  if (dryRun) {
+    // Dry-run mode: do not record execution steps or complete the runbook.
+    // Surface the plan and auto-review decisions only.
+    process.stdout.write(JSON.stringify({
+      mode: 'dry-run',
+      honesty: 'deterministic model of the OpenAI Codex+Runme runbook flywheel',
+      source: 'https://developers.openai.com/blog/automating-repetitive-work-at-openai-with-codex',
+      prematureExecution: premature,
+      autoReview: review,
+      approvedPlan: rb.approvedPlan,
+      state: rb.state,
+    }, null, 2) + '\n');
+    return;
+  }
+
+  if (solve) {
+    // Solve mode: execute the approved plan end-to-end.
+    return runSolve(rb, review, premature);
+  }
+
+  // Default mode: execute the approved plan end-to-end.
+  runSolve(rb, review, premature);
+}
+
+function runSolve(rb, review, premature) {
+  executeStep(rb, 'review previous run', 'ok');
+  executeStep(rb, 'run eval', 'ok');
+  captureDecision(rb, {
+    choice: 'reuse existing eval cluster',
+    reason: 'quota exhausted on new provisioning',
+    nextTime: 'check quota before provisioning',
+  });
+  recordDeadEnd(rb, 'new cluster provisioning — quota exhausted');
+  closeRunbook(rb);
+
+  const index = buildIndex([rb]);
+  const context = discoverContext(index, 'model-eval');
+
+  process.stdout.write(JSON.stringify({
+    mode: 'solve',
+    honesty: 'deterministic model of the OpenAI Codex+Runme runbook flywheel',
+    source: 'https://developers.openai.com/blog/automating-repetitive-work-at-openai-with-codex',
+    prematureExecution: premature,
+    autoReview: review,
+    finalState: rb.state,
+    decisions: rb.decisions.length,
+    deadEnds: rb.deadEnds.length,
+    decisionsLog: rb.decisions,
+    deadEndsLog: rb.deadEnds,
+    index, context,
+  }, null, 2) + '\n');
+}
+
+if (isCliEntrypoint()) main();
+
+module.exports = {
+  RUN_STATES,
+  CONSEQUENTIAL,
+  ALLOWED_ACTIONS,
+  stepId,
+  newRunbook,
+  approvePlan,
+  autoReview,
+  executeStep,
+  captureDecision,
+  recordDeadEnd,
+  closeRunbook,
+  buildIndex,
+  discoverContext,
+  isCliEntrypoint,
+};

package/scripts/context-footprint.js

@@ -141,6 +141,168 @@ function buildFeedbackContextFootprintReport(entries = [], anchors = [], options
   };
 }
 
+function renderSymbolicTaskCanvas(options = {}) {
+  const activeTask = options.activeTask || 'Execute Task';
+  const milestones = Array.isArray(options.milestones) ? options.milestones : [];
+  const blockers = Array.isArray(options.blockers) ? options.blockers : [];
+
+  let mermaidDiagram = 'graph TD\n';
+  mermaidDiagram += `  Start["Task Goal: ${activeTask}"]\n`;
+
+  if (milestones.length === 0) {
+    mermaidDiagram += '  Start --> Step1["In Progress"]\n';
+  } else {
+    let prevId = 'Start';
+    milestones.forEach((m, idx) => {
+      const stepId = `Step${idx + 1}`;
+      const status = m.status === 'completed' ? ' [DONE]' : m.status === 'blocked' ? ' [BLOCKED]' : ' [IN_PROGRESS]';
+      const label = `${m.name || m.title || `Milestone ${idx + 1}`}${status}`;
+      mermaidDiagram += `  ${prevId} --> ${stepId}["${label.replace(/"/g, "'")}"]\n`;
+      prevId = stepId;
+    });
+  }
+
+  if (blockers.length > 0) {
+    blockers.forEach((b, idx) => {
+      mermaidDiagram += `  Blocker${idx + 1}["BLOCKED: ${String(b).replace(/"/g, "'")}"] -.-> Start\n`;
+    });
+  }
+
+  const canvasText = [
+    `# Symbolic Task Canvas: ${activeTask}`,
+    `**Milestones:** ${milestones.length}`,
+    `**Blockers:** ${blockers.length}`,
+    '```mermaid',
+    mermaidDiagram.trim(),
+    '```',
+  ].join('\n');
+
+  return {
+    canvasText,
+    mermaidDiagram,
+    activeTask,
+    milestoneCount: milestones.length,
+    blockerCount: blockers.length,
+  };
+}
+
+function compactSymbolicTaskCanvas(entries = [], options = {}) {
+  const safeEntries = Array.isArray(entries) ? entries : [];
+  const activeTask = options.activeTask || 'Active Session Task';
+  const milestones = [];
+  const blockers = [];
+
+  for (const entry of safeEntries) {
+    if (!entry) continue;
+    const text = typeof entry === 'string' ? entry : JSON.stringify(entry);
+    if (/block|fail|error/i.test(text)) {
+      blockers.push(text.slice(0, 80));
+    } else {
+      milestones.push({ name: text.slice(0, 60), status: /done|pass|success/i.test(text) ? 'completed' : 'in_progress' });
+    }
+  }
+
+  const canvas = renderSymbolicTaskCanvas({ activeTask, milestones: milestones.slice(0, 5), blockers: blockers.slice(0, 3) });
+  const footprint = measureFootprint(safeEntries, canvas.canvasText, {
+    targetReduction: options.targetReduction || 0.5,
+    charsPerToken: options.charsPerToken,
+  });
+
+  return {
+    kind: 'symbolic-task-canvas',
+    strategy: 'symbolic-state-offloading',
+    canvas,
+    footprint,
+  };
+}
+
+function isPositiveInt(value) {
+  const n = Number(value);
+  return Number.isInteger(n) && n > 0;
+}
+
+function buildMatryoshkaEmbeddingReport(options = {}) {
+  const defaults = {
+    fullDimension: 1536,
+    targetDimensions: [768, 512, 256, 128],
+    itemCount: 1000,
+  };
+
+  const fullDimensionRaw = options.fullDimension == null ? defaults.fullDimension : options.fullDimension;
+  if (!isPositiveInt(fullDimensionRaw)) {
+    throw new Error('matryoshkaOptions.fullDimension must be a positive integer');
+  }
+  const fullDimension = Number(fullDimensionRaw);
+
+  const itemsRaw = options.itemCount == null ? defaults.itemCount : options.itemCount;
+  if (!isPositiveInt(itemsRaw)) {
+    throw new Error('matryoshkaOptions.itemCount must be a positive integer');
+  }
+  const items = Number(itemsRaw);
+
+  let targetDimensions;
+  if (Array.isArray(options.targetDimensions) && options.targetDimensions.length > 0) {
+    targetDimensions = options.targetDimensions.map(Number);
+    for (const dim of targetDimensions) {
+      if (!isPositiveInt(dim)) {
+        throw new Error('matryoshkaOptions.targetDimensions must be positive integers');
+      }
+      if (dim > fullDimension) {
+        throw new Error('matryoshkaOptions.targetDimensions must be <= fullDimension');
+      }
+    }
+  } else {
+    targetDimensions = defaults.targetDimensions.filter((dim) => dim <= fullDimension);
+  }
+
+  if (targetDimensions.length === 0) {
+    throw new Error('matryoshkaOptions.targetDimensions must include at least one valid dimension');
+  }
+
+  const baselineBytes = items * fullDimension * 4;
+  const qualityVerified = options.qualityVerified === true
+    && options.behaviorPreserved === true
+    && Number.isFinite(Number(options.accuracyLossEstimatePct));
+
+  const tiers = targetDimensions.map((dim) => {
+    const dimBytes = items * dim * 4;
+    const reductionRatio = baselineBytes > 0 ? (baselineBytes - dimBytes) / baselineBytes : 0;
+    return {
+      dimension: dim,
+      bytesPerItem: dim * 4,
+      totalBytes: dimBytes,
+      reductionPercent: Number((reductionRatio * 100).toFixed(1)),
+      estimatedMemorySavingsBytes: Math.max(0, baselineBytes - dimBytes),
+      recommendedUsage:
+        dim <= 256
+          ? 'Fast coarse vector filtering & initial top-K candidate pooling'
+          : 'High-precision re-ranking & final gate evaluation',
+    };
+  });
+
+  return {
+    kind: 'matryoshka-embedding-compaction',
+    strategy: 'matryoshka-representation-learning-mrl',
+    fullDimension,
+    itemCount: items,
+    baselineTotalBytes: baselineBytes,
+    qualityContract: qualityVerified
+      ? {
+        behaviorPreserved: true,
+        accuracyLossEstimatePct: `${Number(options.accuracyLossEstimatePct)}%`,
+        verified: true,
+        reason: 'Provider-specific golden evaluation met required recall/precision thresholds.',
+      }
+      : {
+        behaviorPreserved: false,
+        accuracyLossEstimatePct: 'unverified',
+        verified: false,
+        reason: 'Quality unverified: no provider-specific Matryoshka golden evaluation evidence was supplied. Do not treat truncation as behavior-preserving until deterministic RAG eval meets required thresholds.',
+      },
+    tiers,
+  };
+}
+
 function buildContextFootprintReport(options = {}) {
   const targetReduction = normalizeRatio(options.targetReduction);
   const report = {
@@ -152,6 +314,8 @@ function buildContextFootprintReport(options = {}) {
       'Use construct_context_pack with maxChars before injecting lessons into a model prompt.',
       'Keep gate ids, proof URLs, and anchor lessons stable so compaction does not hide evidence.',
       'Track estimated token savings beside every optimized context path.',
+      'Offload verbose raw logs into a Symbolic Task Canvas (Mermaid state graph) to save ~60% tokens.',
+      'Truncate Matryoshka vector embeddings (1536d -> 256d) for fast coarse filtering, reserving full dimensions for final re-ranking.',
     ],
   };
 
@@ -177,6 +341,23 @@ function buildContextFootprintReport(options = {}) {
     );
   }
 
+  if (options.symbolicCanvas || Array.isArray(options.symbolicEntries)) {
+    report.symbolicTaskCanvas = compactSymbolicTaskCanvas(
+      options.symbolicEntries || options.entries || [],
+      {
+        activeTask: options.activeTask,
+        targetReduction,
+        charsPerToken: options.charsPerToken,
+      },
+    );
+  }
+
+  if (options.matryoshkaEmbedding || options.matryoshkaOptions) {
+    report.matryoshkaEmbedding = buildMatryoshkaEmbeddingReport(
+      typeof options.matryoshkaOptions === 'object' ? options.matryoshkaOptions : {},
+    );
+  }
+
   return report;
 }
 
@@ -187,5 +368,10 @@ module.exports = {
   measureFootprint,
   buildMcpToolFootprintReport,
   buildFeedbackContextFootprintReport,
+  renderSymbolicTaskCanvas,
+  compactSymbolicTaskCanvas,
+  buildMatryoshkaEmbeddingReport,
   buildContextFootprintReport,
 };
+
+