thumbgate 0.9.9

package/scripts/internal-agent-bootstrap.js

@@ -0,0 +1,490 @@
+#!/usr/bin/env node
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const crypto = require('crypto');
+const { execFileSync } = require('child_process');
+
+const {
+  ensureContextFs,
+  constructContextPack,
+  recordProvenance,
+} = require('./contextfs');
+const { planIntent } = require('./intent-router');
+const { formatCodeGraphRecallSection } = require('./codegraph-context');
+
+const KNOWN_SOURCES = new Set(['github', 'slack', 'linear', 'api', 'cli']);
+const DEFAULT_SOURCE = 'api';
+const DEFAULT_SANDBOX_ROOT = path.join(os.tmpdir(), 'rlhf-internal-agent-sandboxes');
+
+function normalizeText(value) {
+  if (value === undefined || value === null) return '';
+  return String(value).trim();
+}
+
+function slugify(value) {
+  return String(value || 'item')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 96) || 'item';
+}
+
+function uniqueStrings(values = []) {
+  return Array.from(new Set(
+    values
+      .map((value) => normalizeText(value))
+      .filter(Boolean),
+  ));
+}
+
+function hashFragment(values = []) {
+  return crypto
+    .createHash('sha1')
+    .update(values.filter(Boolean).join('|'))
+    .digest('hex')
+    .slice(0, 12);
+}
+
+function ensureGitRepo(repoPath) {
+  if (!repoPath) return null;
+  const resolved = path.resolve(repoPath);
+  const target = fs.existsSync(resolved) && fs.statSync(resolved).isDirectory()
+    ? resolved
+    : path.dirname(resolved);
+  try {
+    return execFileSync('git', ['-C', target, 'rev-parse', '--show-toplevel'], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'ignore'],
+    }).trim();
+  } catch (err) {
+    throw new Error(`repoPath must point to a git repository: ${resolved}`);
+  }
+}
+
+function normalizeConversation(entries = []) {
+  if (!Array.isArray(entries)) return [];
+  return entries
+    .map((entry) => {
+      if (typeof entry === 'string') {
+        return {
+          author: null,
+          text: normalizeText(entry),
+          timestamp: null,
+        };
+      }
+      if (!entry || typeof entry !== 'object') return null;
+      return {
+        author: normalizeText(entry.author || entry.user || entry.name) || null,
+        text: normalizeText(entry.text || entry.body || entry.message),
+        timestamp: normalizeText(entry.timestamp || entry.createdAt || entry.updatedAt) || null,
+      };
+    })
+    .filter((entry) => entry && entry.text);
+}
+
+function inferIntentId({ task, context }) {
+  const text = `${normalizeText(task.title)} ${normalizeText(task.body)} ${normalizeText(context)}`.toLowerCase();
+  if (/\b(incident|postmortem|outage|retro)\b/.test(text)) {
+    return 'incident_postmortem';
+  }
+  return 'improve_response_quality';
+}
+
+function normalizeSource(source) {
+  const normalized = normalizeText(source).toLowerCase();
+  return KNOWN_SOURCES.has(normalized) ? normalized : DEFAULT_SOURCE;
+}
+
+function deriveThreadId(invocation) {
+  const explicit = normalizeText(invocation.thread.id)
+    || normalizeText(invocation.trigger.id)
+    || normalizeText(invocation.task.number)
+    || normalizeText(invocation.task.title);
+
+  if (explicit) {
+    return `${invocation.source}-${slugify(explicit)}`;
+  }
+
+  return `${invocation.source}-${hashFragment([
+    invocation.source,
+    invocation.trigger.type,
+    invocation.context,
+    invocation.task.title,
+    invocation.task.body,
+  ])}`;
+}
+
+function normalizeInvocation(input = {}) {
+  const source = normalizeSource(input.source);
+  const trigger = {
+    type: normalizeText(input.trigger && input.trigger.type) || `${source}_event`,
+    id: normalizeText(input.trigger && input.trigger.id) || null,
+    url: normalizeText(input.trigger && input.trigger.url) || null,
+    actor: normalizeText(input.trigger && input.trigger.actor) || null,
+  };
+  const thread = {
+    id: normalizeText(input.thread && input.thread.id) || null,
+    title: normalizeText(input.thread && input.thread.title) || null,
+    url: normalizeText(input.thread && input.thread.url) || null,
+  };
+  const task = {
+    title: normalizeText(input.task && input.task.title) || thread.title || null,
+    body: normalizeText(input.task && input.task.body) || null,
+    number: normalizeText(input.task && input.task.number) || null,
+    labels: uniqueStrings(Array.isArray(input.task && input.task.labels) ? input.task.labels : []),
+    branch: normalizeText(input.task && input.task.branch) || null,
+  };
+  const context = normalizeText(input.context);
+  const repoPath = input.repoPath ? ensureGitRepo(input.repoPath) : null;
+  const threadId = deriveThreadId({ source, trigger, thread, task, context });
+  const intentId = normalizeText(input.intentId) || inferIntentId({ task, context });
+  const prepareSandbox = input.prepareSandbox !== false && Boolean(repoPath);
+  const sandboxRoot = path.resolve(input.sandboxRoot || process.env.THUMBGATE_AGENT_SANDBOX_ROOT || DEFAULT_SANDBOX_ROOT);
+
+  return {
+    source,
+    trigger,
+    thread: {
+      ...thread,
+      id: thread.id || threadId,
+    },
+    threadId,
+    task,
+    comments: normalizeConversation(input.comments),
+    messages: normalizeConversation(input.messages),
+    context,
+    repoPath,
+    prepareSandbox,
+    sandboxRoot,
+    intentId,
+    mcpProfile: normalizeText(input.mcpProfile) || undefined,
+    partnerProfile: normalizeText(input.partnerProfile) || undefined,
+    delegationMode: normalizeText(input.delegationMode) || 'auto',
+    approved: input.approved === true,
+  };
+}
+
+function buildStartupContext(invocation) {
+  const sections = [];
+
+  sections.push({
+    title: 'Trigger',
+    lines: [
+      `Source: ${invocation.source}`,
+      `Trigger type: ${invocation.trigger.type}`,
+      `Thread ID: ${invocation.threadId}`,
+      invocation.trigger.actor ? `Actor: ${invocation.trigger.actor}` : null,
+      invocation.trigger.url ? `Trigger URL: ${invocation.trigger.url}` : null,
+      invocation.thread.url ? `Thread URL: ${invocation.thread.url}` : null,
+    ].filter(Boolean),
+  });
+
+  sections.push({
+    title: 'Task',
+    lines: [
+      invocation.task.title ? `Title: ${invocation.task.title}` : null,
+      invocation.task.number ? `Number: ${invocation.task.number}` : null,
+      invocation.task.labels.length > 0 ? `Labels: ${invocation.task.labels.join(', ')}` : null,
+      invocation.task.body ? `Body: ${invocation.task.body}` : null,
+    ].filter(Boolean),
+  });
+
+  if (invocation.comments.length > 0) {
+    sections.push({
+      title: 'Thread History',
+      lines: invocation.comments.map((entry) => {
+        const prefix = entry.author ? `${entry.author}: ` : '';
+        return `${prefix}${entry.text}`;
+      }),
+    });
+  }
+
+  if (invocation.messages.length > 0) {
+    sections.push({
+      title: 'Conversation',
+      lines: invocation.messages.map((entry) => {
+        const prefix = entry.author ? `${entry.author}: ` : '';
+        return `${prefix}${entry.text}`;
+      }),
+    });
+  }
+
+  if (invocation.context) {
+    sections.push({
+      title: 'Operator Context',
+      lines: [invocation.context],
+    });
+  }
+
+  if (invocation.repoPath) {
+    sections.push({
+      title: 'Workspace',
+      lines: [`Repository: ${invocation.repoPath}`],
+    });
+  }
+
+  const text = sections
+    .filter((section) => section.lines.length > 0)
+    .map((section) => `## ${section.title}\n${section.lines.join('\n')}`)
+    .join('\n\n');
+
+  return {
+    text,
+    sections,
+  };
+}
+
+function buildRecallQuery(invocation) {
+  return [
+    invocation.task.title,
+    invocation.task.body,
+    invocation.context,
+    ...invocation.comments.slice(0, 3).map((entry) => entry.text),
+    ...invocation.messages.slice(0, 3).map((entry) => entry.text),
+  ]
+    .filter(Boolean)
+    .join(' ')
+    .slice(0, 600);
+}
+
+function ensureWorktreeSandbox({ repoPath, sandboxRoot, threadId }) {
+  if (!repoPath) {
+    return {
+      ready: false,
+      kind: 'none',
+      path: null,
+      reused: false,
+      baseRef: null,
+    };
+  }
+
+  const repoName = slugify(path.basename(repoPath));
+  const sandboxPath = path.join(path.resolve(sandboxRoot), repoName, slugify(threadId));
+  fs.mkdirSync(path.dirname(sandboxPath), { recursive: true });
+
+  if (fs.existsSync(path.join(sandboxPath, '.git'))) {
+    return {
+      ready: true,
+      kind: 'git_worktree',
+      path: sandboxPath,
+      reused: true,
+      baseRef: 'HEAD',
+    };
+  }
+
+  if (fs.existsSync(sandboxPath) && fs.readdirSync(sandboxPath).length > 0) {
+    throw new Error(`Sandbox path already exists and is not a ThumbGate worktree: ${sandboxPath}`);
+  }
+
+  execFileSync('git', ['-C', repoPath, 'worktree', 'add', '--detach', sandboxPath, 'HEAD'], {
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+
+  return {
+    ready: true,
+    kind: 'git_worktree',
+    path: sandboxPath,
+    reused: false,
+    baseRef: 'HEAD',
+  };
+}
+
+function summarizePack(pack) {
+  return {
+    packId: pack.packId,
+    itemCount: Array.isArray(pack.items) ? pack.items.length : 0,
+    cache: pack.cache || { hit: false },
+    visibleTitles: pack.visibility && Array.isArray(pack.visibility.visibleTitles)
+      ? pack.visibility.visibleTitles.slice()
+      : [],
+  };
+}
+
+function buildReviewerLane(plan) {
+  if (!plan) {
+    return {
+      enabled: false,
+      delegateProfile: null,
+      executionMode: null,
+      verificationMode: null,
+      requiredChecks: [],
+      requiredEvidence: [],
+    };
+  }
+
+  const requiredChecks = plan.handoffContract && Array.isArray(plan.handoffContract.requiredChecks)
+    ? plan.handoffContract.requiredChecks.slice()
+    : (plan.partnerStrategy && Array.isArray(plan.partnerStrategy.recommendedChecks)
+      ? plan.partnerStrategy.recommendedChecks.slice()
+      : []);
+
+  const requiredEvidence = plan.handoffContract && Array.isArray(plan.handoffContract.requiredEvidence)
+    ? plan.handoffContract.requiredEvidence.slice()
+    : ['tests', 'proof', 'diff_review'];
+
+  return {
+    enabled: plan.delegationEligible === true,
+    delegateProfile: plan.delegateProfile || null,
+    executionMode: plan.executionMode || null,
+    verificationMode: plan.partnerStrategy ? plan.partnerStrategy.verificationMode : null,
+    requiredChecks,
+    requiredEvidence,
+  };
+}
+
+function buildMiddlewarePlan({ sandbox, recallPack, plan, reviewerLane }) {
+  return [
+    {
+      step: 'trigger_normalization',
+      status: 'ready',
+      details: {},
+    },
+    {
+      step: 'startup_context_hydration',
+      status: 'ready',
+      details: {},
+    },
+    {
+      step: 'context_pack_construction',
+      status: 'ready',
+      details: summarizePack(recallPack),
+    },
+    {
+      step: 'sandbox_preparation',
+      status: sandbox.ready ? 'ready' : 'skipped',
+      details: {
+        kind: sandbox.kind,
+        path: sandbox.path,
+        reused: sandbox.reused,
+      },
+    },
+    {
+      step: 'intent_planning',
+      status: 'ready',
+      details: {
+        intentId: plan.intent.id,
+        status: plan.status,
+        executionMode: plan.executionMode,
+      },
+    },
+    {
+      step: 'reviewer_lane',
+      status: reviewerLane.enabled ? 'ready' : 'optional',
+      details: {
+        enabled: reviewerLane.enabled,
+        delegateProfile: reviewerLane.delegateProfile,
+        verificationMode: reviewerLane.verificationMode,
+      },
+    },
+    {
+      step: 'proof_gate',
+      status: 'ready',
+      details: {
+        requiredChecks: reviewerLane.requiredChecks,
+        requiredEvidence: reviewerLane.requiredEvidence,
+      },
+    },
+    {
+      step: 'feedback_capture',
+      status: 'ready',
+      details: {
+        tool: 'capture_feedback',
+      },
+    },
+  ];
+}
+
+function bootstrapInternalAgent(options = {}) {
+  const invocation = normalizeInvocation(options);
+  const startupContext = buildStartupContext(invocation);
+
+  ensureContextFs();
+  const recallPack = constructContextPack({
+    query: buildRecallQuery(invocation),
+    maxItems: 6,
+    maxChars: 5000,
+  });
+
+  const sandbox = invocation.prepareSandbox
+    ? ensureWorktreeSandbox({
+      repoPath: invocation.repoPath,
+      sandboxRoot: invocation.sandboxRoot,
+      threadId: invocation.threadId,
+    })
+    : {
+      ready: false,
+      kind: 'none',
+      path: null,
+      reused: false,
+      baseRef: null,
+    };
+
+  const plan = planIntent({
+    intentId: invocation.intentId,
+    context: startupContext.text,
+    mcpProfile: invocation.mcpProfile,
+    partnerProfile: invocation.partnerProfile,
+    delegationMode: invocation.delegationMode,
+    approved: invocation.approved,
+    repoPath: sandbox.path || invocation.repoPath || undefined,
+  });
+
+  const reviewerLane = buildReviewerLane(plan);
+  const codeGraphSection = formatCodeGraphRecallSection(plan.codegraphImpact);
+  const middlewarePlan = buildMiddlewarePlan({
+    sandbox,
+    recallPack,
+    plan,
+    reviewerLane,
+  });
+
+  const payload = {
+    generatedAt: new Date().toISOString(),
+    invocation: {
+      source: invocation.source,
+      trigger: invocation.trigger,
+      thread: invocation.thread,
+      threadId: invocation.threadId,
+      task: invocation.task,
+      repoPath: invocation.repoPath,
+      prepareSandbox: invocation.prepareSandbox,
+      intentId: invocation.intentId,
+      mcpProfile: invocation.mcpProfile || 'default',
+      delegationMode: invocation.delegationMode,
+    },
+    startupContext,
+    recallPack: summarizePack(recallPack),
+    sandbox,
+    intentPlan: plan,
+    reviewerLane,
+    middlewarePlan,
+    codeGraph: {
+      enabled: Boolean(codeGraphSection),
+      section: codeGraphSection,
+    },
+  };
+
+  recordProvenance({
+    type: 'internal_agent_bootstrap_created',
+    source: invocation.source,
+    threadId: invocation.threadId,
+    sandboxPath: sandbox.path,
+    intentId: plan.intent.id,
+    delegationEligible: plan.delegationEligible,
+    packId: recallPack.packId,
+  });
+
+  return payload;
+}
+
+module.exports = {
+  DEFAULT_SANDBOX_ROOT,
+  normalizeInvocation,
+  buildStartupContext,
+  ensureWorktreeSandbox,
+  buildMiddlewarePlan,
+  bootstrapInternalAgent,
+};

package/scripts/jsonl-watcher.js

@@ -0,0 +1,155 @@
+#!/usr/bin/env node
+/**
+ * JSONL File Watcher
+ *
+ * Watches feedback-log.jsonl for new entries written by external sources
+ * (e.g., Amp plugin bridge) and routes them through captureFeedback()
+ * for full pipeline processing: memory promotion, vector indexing,
+ * sequence tracking, and DPO export eligibility.
+ *
+ * Usage:
+ *   node scripts/jsonl-watcher.js                    # watch mode
+ *   node scripts/jsonl-watcher.js --once             # process pending, exit
+ *   node scripts/jsonl-watcher.js --source amp-plugin-bridge  # filter by source
+ *
+ * The watcher tracks its position via .thumbgate/.watcher-offset to avoid
+ * reprocessing entries on restart.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { captureFeedback, getFeedbackPaths } = require('./feedback-loop');
+
+const POLL_INTERVAL_MS = 2000;
+const WATCHER_SOURCE_TAG = 'watcher-ingested';
+
+// Use stderr for logging so stdout stays clean for MCP JSON-RPC when co-hosted
+const log = (...args) => process.stderr.write(`${args.join(' ')}\n`);
+
+function getOffsetPath(feedbackDir) {
+  return path.join(feedbackDir, '.watcher-offset');
+}
+
+function readOffset(feedbackDir) {
+  const offsetPath = getOffsetPath(feedbackDir);
+  try {
+    return parseInt(fs.readFileSync(offsetPath, 'utf-8').trim(), 10) || 0;
+  } catch {
+    return 0;
+  }
+}
+
+function writeOffset(feedbackDir, offset) {
+  fs.writeFileSync(getOffsetPath(feedbackDir), String(offset) + '\n');
+}
+
+function parseEntry(line) {
+  try {
+    return JSON.parse(line);
+  } catch {
+    return null;
+  }
+}
+
+function shouldIngest(entry, sourceFilter) {
+  // Only ingest entries from external sources (not from captureFeedback itself)
+  if (!entry || !entry.source) return false;
+  if (sourceFilter && entry.source !== sourceFilter) return false;
+  // Skip entries already ingested by the watcher
+  if (entry.tags && entry.tags.includes(WATCHER_SOURCE_TAG)) return false;
+  return true;
+}
+
+function ingestEntry(entry) {
+  const signal = entry.signal === 'positive' ? 'up' : entry.signal === 'negative' ? 'down' : entry.signal;
+
+  const result = captureFeedback({
+    signal,
+    context: entry.context || '',
+    whatWentWrong: entry.whatWentWrong || undefined,
+    whatToChange: entry.whatToChange || undefined,
+    whatWorked: entry.whatWorked || undefined,
+    tags: [...(entry.tags || []), WATCHER_SOURCE_TAG, `bridged-from:${entry.source}`],
+    skill: entry.skill || undefined,
+  });
+
+  return result;
+}
+
+function processNewEntries(feedbackDir, feedbackLogPath, sourceFilter) {
+  if (!fs.existsSync(feedbackLogPath)) return { processed: 0, promoted: 0 };
+
+  const content = fs.readFileSync(feedbackLogPath, 'utf-8');
+  const lines = content.split('\n').filter(Boolean);
+  const offset = readOffset(feedbackDir);
+
+  if (offset >= lines.length) return { processed: 0, promoted: 0 };
+
+  const newLines = lines.slice(offset);
+  let processed = 0;
+  let promoted = 0;
+
+  for (const line of newLines) {
+    const entry = parseEntry(line);
+    if (shouldIngest(entry, sourceFilter)) {
+      const result = ingestEntry(entry);
+      processed++;
+      if (result && result.accepted) promoted++;
+    }
+  }
+
+  writeOffset(feedbackDir, lines.length);
+  return { processed, promoted };
+}
+
+function watch(sourceFilter) {
+  const { FEEDBACK_DIR, FEEDBACK_LOG_PATH } = getFeedbackPaths();
+
+  // Initialize offset to current end of file to avoid reprocessing history
+  if (!fs.existsSync(getOffsetPath(FEEDBACK_DIR))) {
+    if (fs.existsSync(FEEDBACK_LOG_PATH)) {
+      const lines = fs.readFileSync(FEEDBACK_LOG_PATH, 'utf-8').split('\n').filter(Boolean);
+      writeOffset(FEEDBACK_DIR, lines.length);
+    } else {
+      writeOffset(FEEDBACK_DIR, 0);
+    }
+    log(`[jsonl-watcher] Initialized offset at current end of file`);
+  }
+
+  log(`[jsonl-watcher] Watching ${FEEDBACK_LOG_PATH}`);
+  if (sourceFilter) log(`[jsonl-watcher] Filtering source: ${sourceFilter}`);
+
+  setInterval(() => {
+    try {
+      const result = processNewEntries(FEEDBACK_DIR, FEEDBACK_LOG_PATH, sourceFilter);
+      if (result.processed > 0) {
+        log(`[jsonl-watcher] Ingested ${result.processed} entries (${result.promoted} promoted)`);
+      }
+    } catch (err) {
+      log(`[jsonl-watcher] Poll error (non-fatal): ${err.message}`);
+    }
+  }, POLL_INTERVAL_MS);
+}
+
+function once(sourceFilter) {
+  const { FEEDBACK_DIR, FEEDBACK_LOG_PATH } = getFeedbackPaths();
+  const result = processNewEntries(FEEDBACK_DIR, FEEDBACK_LOG_PATH, sourceFilter);
+  log(`[jsonl-watcher] Processed ${result.processed} entries (${result.promoted} promoted)`);
+  return result;
+}
+
+// CLI
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const isOnce = args.includes('--once');
+  const sourceIdx = args.indexOf('--source');
+  const sourceFilter = sourceIdx >= 0 ? args[sourceIdx + 1] : undefined;
+
+  if (isOnce) {
+    once(sourceFilter);
+  } else {
+    watch(sourceFilter);
+  }
+}
+
+module.exports = { processNewEntries, ingestEntry, shouldIngest, watch, once };