rlhf-feedback-loop 0.6.9 → 0.6.11

package/README.md

@@ -1,6 +1,10 @@
 # RLHF-Ready Feedback Loop — Agentic Control Plane & Context Engineering Studio
 
 [![CI](https://github.com/IgorGanapolsky/rlhf-feedback-loop/actions/workflows/ci.yml/badge.svg)](https://github.com/IgorGanapolsky/rlhf-feedback-loop/actions/workflows/ci.yml)
+[![Self-Healing](https://github.com/IgorGanapolsky/rlhf-feedback-loop/actions/workflows/self-healing-monitor.yml/badge.svg)](https://github.com/IgorGanapolsky/rlhf-feedback-loop/actions/workflows/self-healing-monitor.yml)
+[![npm](https://img.shields.io/npm/v/rlhf-feedback-loop)](https://www.npmjs.com/package/rlhf-feedback-loop)
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE)
+[![Node](https://img.shields.io/badge/node-%3E%3D18.18.0-brightgreen)](package.json)
 [![Marketplace Ready](https://img.shields.io/badge/Anthropic_Marketplace-Ready-blue)](docs/ANTHROPIC_MARKETPLACE_STRATEGY.md)
 [![GEO Optimized](https://img.shields.io/badge/GEO-optimized-orange)](docs/geo-strategy-for-ai-agents.md)
 
@@ -84,7 +88,7 @@ All data stored locally as **JSONL** files — fully transparent, fully portable
 
 The open-source package is fully functional and free forever. Cloud Pro is for teams that don't want to self-host.
 
-| | Open Source | Cloud Pro ($49/mo) |
+| | Open Source | Cloud Pro (Founding price: $10/mo) |
 |---|---|---|
 | Feedback capture | Local MCP server | Hosted HTTPS API |
 | Storage | Your machine | Managed cloud |
@@ -94,7 +98,7 @@ The open-source package is fully functional and free forever. Cloud Pro is for t
 | Support | GitHub Issues | Email |
 | Uptime | You manage | We manage (99.9% SLA) |
 
-[Get Cloud Pro](https://buy.stripe.com/bJe14neyU4r4f0leOD3sI02) | [Live API](https://rlhf-feedback-loop-710216278770.us-central1.run.app) | [Verification Evidence](docs/VERIFICATION_EVIDENCE.md)
+[Get Cloud Pro ($10/mo)](https://buy.stripe.com/bJe14neyU4r4f0leOD3sI02) | [Live API](https://rlhf-feedback-loop-710216278770.us-central1.run.app) | [Verification Evidence](docs/VERIFICATION_EVIDENCE.md)
 
 ## Deep Dive
 

package/adapters/mcp/server-stdio.js

@@ -655,12 +655,22 @@ async function onData(chunk) {
 function startStdioServer() {
   if (stdioStarted) return;
   stdioStarted = true;
+
+  // Keep the process alive even if stdin closes (prevents premature exit
+  // when launched by MCP clients like Claude Code, Codex, Gemini CLI).
+  const keepAlive = setInterval(() => {}, 60_000);
+
+  process.stdin.resume();
   process.stdin.on('data', (chunk) => {
     onData(chunk).catch((err) => {
       const transport = err && err.transport === 'ndjson' ? 'ndjson' : 'framed';
       writeMessage({ jsonrpc: '2.0', id: null, error: { code: -32603, message: err.message } }, transport);
     });
   });
+  process.stdin.on('end', () => {
+    // stdin closed — clean up and exit gracefully
+    clearInterval(keepAlive);
+  });
 }
 
 module.exports = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rlhf-feedback-loop",
-  "version": "0.6.9",
+  "version": "0.6.11",
   "description": "RLHF-ready human feedback capture and DPO data pipeline for AI agents. Optimize agentic reliability with Feedback-Driven Development (FDD): capture preference signals, enforce guardrails, and export training pairs for downstream optimization.",
   "homepage": "https://github.com/IgorGanapolsky/rlhf-feedback-loop#readme",
   "repository": {
@@ -33,7 +33,7 @@
     "test:schema": "node scripts/feedback-schema.js --test",
     "test:loop": "node scripts/feedback-loop.js --test",
     "test:dpo": "node scripts/export-dpo-pairs.js --test",
-    "test:api": "node --test tests/api-server.test.js tests/api-auth-config.test.js tests/mcp-server.test.js tests/adapters.test.js tests/openapi-parity.test.js tests/budget-guard.test.js tests/contextfs.test.js tests/mcp-policy.test.js tests/subagent-profiles.test.js tests/intent-router.test.js tests/rubric-engine.test.js tests/self-healing-check.test.js tests/self-heal.test.js tests/feedback-schema.test.js tests/thompson-sampling.test.js tests/feedback-sequences.test.js tests/diversity-tracking.test.js tests/vector-store.test.js tests/feedback-attribution.test.js tests/hybrid-feedback-context.test.js tests/loop-closure.test.js tests/code-reasoning.test.js tests/feedback-loop.test.js tests/feedback-inbox-read.test.js tests/feedback-to-memory.test.js",
+    "test:api": "node --test tests/api-server.test.js tests/api-auth-config.test.js tests/mcp-server.test.js tests/adapters.test.js tests/openapi-parity.test.js tests/budget-guard.test.js tests/contextfs.test.js tests/mcp-policy.test.js tests/subagent-profiles.test.js tests/intent-router.test.js tests/rubric-engine.test.js tests/self-healing-check.test.js tests/self-heal.test.js tests/feedback-schema.test.js tests/thompson-sampling.test.js tests/feedback-sequences.test.js tests/diversity-tracking.test.js tests/vector-store.test.js tests/feedback-attribution.test.js tests/hybrid-feedback-context.test.js tests/loop-closure.test.js tests/code-reasoning.test.js tests/feedback-loop.test.js tests/feedback-inbox-read.test.js tests/feedback-to-memory.test.js tests/test-coverage.test.js tests/version-metadata.test.js",
     "test:proof": "node --test --test-concurrency=1 tests/prove-adapters.test.js tests/prove-automation.test.js tests/prove-attribution.test.js tests/prove-lancedb.test.js tests/prove-data-quality.test.js tests/prove-intelligence.test.js tests/prove-loop-closure.test.js tests/prove-subway-upgrades.test.js tests/prove-training-export.test.js",
     "test:rlaif": "node --test tests/rlaif-self-audit.test.js tests/dpo-optimizer.test.js tests/meta-policy.test.js",
     "test:attribution": "node --test tests/feedback-attribution.test.js tests/hybrid-feedback-context.test.js",
@@ -43,6 +43,7 @@
     "test:deployment": "node --test tests/deployment.test.js",
     "test:billing": "node --test tests/billing.test.js",
     "test:cli": "node --test tests/cli.test.js",
+    "test:coverage": "node scripts/test-coverage.js",
     "start:api": "node src/api/server.js",
     "start:mcp": "node adapters/mcp/server-stdio.js",
     "feedback:capture": "node .claude/scripts/feedback/capture-feedback.js",
@@ -75,7 +76,9 @@
     "ml:train": "python3 scripts/train_from_feedback.py --train",
     "ml:incremental": "python3 scripts/train_from_feedback.py --incremental",
     "ml:reliability": "python3 scripts/train_from_feedback.py --reliability",
-    "ml:sample": "python3 scripts/train_from_feedback.py --sample"
+    "ml:sample": "python3 scripts/train_from_feedback.py --sample",
+    "adk:consolidate": "node scripts/adk-consolidator.js",
+    "adk:watch": "node scripts/adk-consolidator.js --watch"
   },
   "keywords": [
     "rlhf",
@@ -115,10 +118,10 @@
     "node": ">=18.18.0"
   },
   "dependencies": {
+    "@google/genai": "^1.44.0",
     "@huggingface/transformers": "^3.8.1",
     "@lancedb/lancedb": "^0.26.2",
-    "apache-arrow": "^18.1.0",
-    "stripe": "^20.4.1"
+    "apache-arrow": "^18.1.0"
   },
   "mcpName": "io.github.IgorGanapolsky/rlhf-feedback-loop"
 }

package/scripts/adk-consolidator.js

@@ -0,0 +1,173 @@
+#!/usr/bin/env node
+/**
+ * Agent Development Kit (ADK) Memory Consolidator
+ * 
+ * 'Always-On' background service that reads disparate feedback logs and uses 
+ * Gemini (Flash-Lite/Flash) to actively consolidate, compress, and dream up 
+ * generalized prevention rules. This moves the system from 'passive logging' 
+ * to 'active semantic memory consolidation'.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { GoogleGenAI } = require('@google/genai');
+
+const PROJECT_ROOT = path.join(__dirname, '..');
+const { getFeedbackPaths, readJSONL } = require('./feedback-loop');
+
+// Keep track of the last processed ID to avoid re-consolidating the exact same logs
+const STATE_FILE = process.env.ADK_STATE_FILE || path.join(PROJECT_ROOT, '.rlhf', 'adk-state.json');
+
+function ensureDir(dirPath) {
+  if (!fs.existsSync(dirPath)) {
+    fs.mkdirSync(dirPath, { recursive: true });
+  }
+}
+
+function loadState() {
+  if (fs.existsSync(STATE_FILE)) {
+    try {
+      return JSON.parse(fs.readFileSync(STATE_FILE, 'utf-8'));
+    } catch {
+      return { lastProcessedFeedbackId: null };
+    }
+  }
+  return { lastProcessedFeedbackId: null };
+}
+
+function saveState(state) {
+  ensureDir(path.dirname(STATE_FILE));
+  fs.writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
+}
+
+async function consolidateMemory() {
+  const apiKey = process.env.GEMINI_API_KEY;
+  if (!apiKey) {
+    console.warn('[ADK Consolidator] GEMINI_API_KEY is not set. Skipping active consolidation.');
+    return;
+  }
+
+  const ai = new GoogleGenAI({ apiKey });
+  const paths = getFeedbackPaths();
+  const state = loadState();
+
+  const allLogs = readJSONL(paths.FEEDBACK_LOG_PATH);
+  
+  if (allLogs.length === 0) {
+    console.log('[ADK Consolidator] No logs to consolidate.');
+    return;
+  }
+
+  // Find where we left off
+  let newLogs = [];
+  if (state.lastProcessedFeedbackId) {
+    const lastIdx = allLogs.findIndex(l => l.id === state.lastProcessedFeedbackId);
+    if (lastIdx !== -1) {
+      newLogs = allLogs.slice(lastIdx + 1);
+    } else {
+      // If we can't find it (log rotation?), just take the last 50
+      newLogs = allLogs.slice(-50);
+    }
+  } else {
+    // First time running, process up to last 50 entries
+    newLogs = allLogs.slice(-50);
+  }
+
+  if (newLogs.length === 0) {
+    console.log('[ADK Consolidator] No new logs since last consolidation cycle.');
+    return;
+  }
+
+  console.log(`[ADK Consolidator] Found ${newLogs.length} new feedback events. Activating Gemini for semantic consolidation...`);
+
+  const prompt = `
+You are the Agent Development Kit (ADK) 'Always-On' Memory Consolidator.
+Your job is to read the raw, disparate feedback logs of an AI agent and synthesize them into high-level, generalized prevention rules and learned intuitions.
+Unlike standard systems that just count regex matches, you must semantically connect different failures (e.g., an API timeout and a missing import might both stem from 'rushing execution without verifying environment').
+
+Here are the latest feedback events (JSON):
+${JSON.stringify(newLogs.map(l => ({ signal: l.signal, context: l.context, tags: l.tags, whatWentWrong: l.whatWentWrong, whatWorked: l.whatWorked })), null, 2)}
+
+Existing Prevention Rules (if any):
+${fs.existsSync(paths.PREVENTION_RULES_PATH) ? fs.readFileSync(paths.PREVENTION_RULES_PATH, 'utf-8').slice(0, 2000) : 'None yet.'}
+
+Output ONLY a valid JSON object with the following structure, representing the new synthesized insights:
+{
+  "consolidatedInsights": [
+    {
+      "pattern": "Description of the underlying behavioral flaw or success pattern you detected.",
+      "rule": "A clear, actionable directive starting with 'ALWAYS' or 'NEVER' that should be added to prevention rules.",
+      "severity": "critical|high|medium|low"
+    }
+  ],
+  "reasoning": "A short summary of how you connected the dots between these logs."
+}
+`;
+
+  try {
+    // We use gemini-2.5-flash as the proxy for Flash-Lite/Flash efficiency
+    const response = await ai.models.generateContent({
+      model: 'gemini-2.5-flash',
+      contents: prompt,
+      config: {
+        responseMimeType: "application/json",
+      }
+    });
+
+    const result = JSON.parse(response.text);
+    console.log(`[ADK Consolidator] Consolidation complete. Reasoning: ${result.reasoning}`);
+
+    if (result.consolidatedInsights && result.consolidatedInsights.length > 0) {
+      appendRules(result.consolidatedInsights, paths.PREVENTION_RULES_PATH);
+    }
+
+    // Update state
+    state.lastProcessedFeedbackId = newLogs[newLogs.length - 1].id;
+    saveState(state);
+
+  } catch (err) {
+    console.error('[ADK Consolidator] Consolidation failed:', err.message);
+  }
+}
+
+function appendRules(insights, rulesPath) {
+  let existingContent = '';
+  if (fs.existsSync(rulesPath)) {
+    existingContent = fs.readFileSync(rulesPath, 'utf-8');
+  } else {
+    existingContent = '# Prevention Rules\n\nGenerated from active semantic memory consolidation.\n\n';
+  }
+
+  let newRulesBlock = '\n## ADK Semantic Consolidations\n';
+  const timestamp = new Date().toISOString();
+  insights.forEach(insight => {
+    newRulesBlock += `- [${insight.severity.toUpperCase()}] **${insight.pattern}**\n  - Rule: ${insight.rule} *(Consolidated at ${timestamp})*\n`;
+  });
+
+  const updatedContent = existingContent + newRulesBlock;
+  ensureDir(path.dirname(rulesPath));
+  fs.writeFileSync(rulesPath, updatedContent);
+  console.log(`[ADK Consolidator] Appended ${insights.length} new consolidated rules to ${rulesPath}`);
+}
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const isWatchMode = args.includes('--watch');
+
+  if (isWatchMode) {
+    console.log('[ADK Consolidator] Started in Always-On Watch Mode (interval: 5 minutes)');
+    consolidateMemory(); // Run once immediately
+    setInterval(() => {
+      consolidateMemory();
+    }, 5 * 60 * 1000); // Check every 5 minutes
+  } else {
+    consolidateMemory().then(() => {
+      console.log('[ADK Consolidator] Cycle finished.');
+      process.exit(0);
+    });
+  }
+}
+
+module.exports = { consolidateMemory };

package/scripts/billing.js

@@ -26,7 +26,7 @@ const crypto = require('crypto');
 const STRIPE_SECRET_KEY = process.env.STRIPE_SECRET_KEY || '';
 const STRIPE_WEBHOOK_SECRET = process.env.STRIPE_WEBHOOK_SECRET || '';
 const GITHUB_MARKETPLACE_WEBHOOK_SECRET = process.env.GITHUB_MARKETPLACE_WEBHOOK_SECRET || '';
-const STRIPE_PRICE_ID = process.env.STRIPE_PRICE_ID || 'price_cloud_pro_49_monthly';
+const STRIPE_PRICE_ID = process.env.STRIPE_PRICE_ID || '';
 
 const API_KEYS_PATH = process.env._TEST_API_KEYS_PATH || path.resolve(
   __dirname,
@@ -371,6 +371,10 @@ async function createCheckoutSession({ successUrl, cancelUrl, customerEmail, ins
     };
   }
 
+  if (!STRIPE_PRICE_ID) {
+    throw new Error('STRIPE_PRICE_ID not configured');
+  }
+
   const params = {
     mode: 'subscription',
     line_items: [

package/scripts/code-reasoning.js

@@ -71,12 +71,33 @@ function addEdgeCase(trace, description) {
   return trace;
 }
 
+function computeControllability(trace) {
+  const steps = trace.steps;
+  const edgeCases = trace.edgeCases;
+  if (steps.length === 0) return { score: 0, flags: ['empty_trace'] };
+
+  const flags = [];
+  const allVerified = steps.every((s) => s.verdict === 'verified');
+  const allSameEvidence = new Set(steps.map((s) => s.evidence)).size === 1 && steps.length > 1;
+  const shortEvidence = steps.filter((s) => s.evidence.length < 10).length;
+  const noEdgeCases = edgeCases.length === 0;
+
+  if (allVerified && steps.length > 2) flags.push('all_verified');
+  if (allSameEvidence) flags.push('identical_evidence');
+  if (shortEvidence > steps.length / 2) flags.push('thin_evidence');
+  if (noEdgeCases && steps.length > 1) flags.push('no_edge_cases');
+
+  const score = Math.round((flags.length / 4) * 1000) / 1000;
+  return { score, flags };
+}
+
 function finalizeTrace(trace, { confidenceThreshold = DEFAULT_CONFIDENCE_THRESHOLD } = {}) {
   const totalSteps = trace.steps.length;
   const verified = trace.steps.filter((s) => s.verdict === 'verified').length;
   const unverified = trace.steps.filter((s) => s.verdict === 'unverified').length;
   const refuted = trace.steps.filter((s) => s.verdict === 'refuted').length;
   const confidence = totalSteps > 0 ? Math.round((verified / totalSteps) * 1000) / 1000 : 0;
+  const ctrl = computeControllability(trace);
 
   trace.summary = {
     totalSteps,
@@ -85,6 +106,8 @@ function finalizeTrace(trace, { confidenceThreshold = DEFAULT_CONFIDENCE_THRESHO
     refuted,
     confidence,
     passed: confidence >= confidenceThreshold && refuted === 0,
+    controllability: ctrl.score,
+    controllabilityFlags: ctrl.flags,
   };
 
   return trace;
@@ -291,6 +314,7 @@ function aggregateTraces(traces) {
     refuted,
     averageConfidence: avgConfidence,
     allPassed: passedTraces === totalTraces,
+    flaggedTraces: traces.filter((t) => t.summary && t.summary.controllability > 0.5).length,
   };
 }
 
@@ -298,6 +322,7 @@ module.exports = {
   createTrace,
   addStep,
   addEdgeCase,
+  computeControllability,
   finalizeTrace,
   traceForSelfHealFix,
   traceForDpoPair,
@@ -305,4 +330,4 @@ module.exports = {
   aggregateTraces,
   DEFAULT_CONFIDENCE_THRESHOLD,
 };
-// test coverage: 573 tests
+// Tests cover this module through the node:test suite; avoid hardcoding counts here.

package/scripts/context-engine.js

@@ -273,7 +273,7 @@ function routeQuery(query, indexPath, topN) {
     index = JSON.parse(fs.readFileSync(idxPath, 'utf-8'));
   } catch {
     // Index doesn't exist — build it on the fly
-    index = buildKnowledgeIndex();
+    index = buildKnowledgeIndex(undefined, idxPath);
   }
 
   const queryTokens = query
@@ -312,9 +312,10 @@ function routeQuery(query, indexPath, topN) {
  * @param {string} query - The original query
  * @param {string[]} retrievedDocs - Filenames of retrieved docs
  * @param {string[]} expectedTopics - Expected topic keywords to match against
+ * @param {string} [logPath] - Optional path for the quality log
  * @returns {{ precision: number, recall: number, f1: number, query: string, timestamp: string }}
  */
-function scoreRetrievalQuality(query, retrievedDocs, expectedTopics) {
+function scoreRetrievalQuality(query, retrievedDocs, expectedTopics, logPath) {
   if (!retrievedDocs.length || !expectedTopics.length) {
     const result = {
       query,
@@ -325,7 +326,7 @@ function scoreRetrievalQuality(query, retrievedDocs, expectedTopics) {
       expectedCount: expectedTopics.length,
       timestamp: new Date().toISOString(),
     };
-    logQualityResult(result);
+    logQualityResult(result, logPath);
     return result;
   }
 
@@ -369,7 +370,7 @@ function scoreRetrievalQuality(query, retrievedDocs, expectedTopics) {
     timestamp: new Date().toISOString(),
   };
 
-  logQualityResult(result);
+  logQualityResult(result, logPath);
   return result;
 }
 

package/scripts/contextfs.js

@@ -218,6 +218,7 @@ function writeContextObject({ namespace, title, content, tags = [], source, ttl
   };
 
   writeJson(filePath, doc);
+  indexContextObject(doc, filePath);
 
   recordProvenance({
     type: 'context_object_created',
@@ -355,6 +356,130 @@ function scoreDocument(doc, queryTokens) {
   return score;
 }
 
+/* ── Memex-style Indexed Memory ────────────────────────────────── */
+
+const MEMEX_INDEX_FILE = 'memex-index.jsonl';
+
+function getMemexIndexPath() {
+  return path.join(CONTEXTFS_ROOT, NAMESPACES.provenance, MEMEX_INDEX_FILE);
+}
+
+function buildIndexEntry(doc, filePath) {
+  return {
+    id: doc.id,
+    namespace: doc.namespace || '',
+    title: doc.title || '',
+    tags: doc.tags || [],
+    digest: String(doc.content || '').slice(0, 120),
+    createdAt: doc.createdAt || nowIso(),
+    stableRef: filePath,
+  };
+}
+
+function indexContextObject(doc, filePath) {
+  const entry = buildIndexEntry(doc, filePath);
+  appendJsonl(getMemexIndexPath(), entry);
+  return entry;
+}
+
+function loadMemexIndex() {
+  return readJsonl(getMemexIndexPath());
+}
+
+function dereferenceEntry(entry) {
+  if (!entry || !entry.stableRef) return null;
+  try {
+    return JSON.parse(fs.readFileSync(entry.stableRef, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+function searchMemexIndex({ query = '', maxResults = 10, namespaces = [] } = {}) {
+  const index = loadMemexIndex();
+  const tokens = tokenizeQuery(query);
+  const nsFilter = namespaces.length > 0 ? new Set(normalizeNamespaces(namespaces)) : null;
+
+  const scored = index
+    .filter((entry) => !nsFilter || nsFilter.has(entry.namespace))
+    .map((entry) => {
+      const haystack = `${entry.title} ${entry.digest} ${(entry.tags || []).join(' ')}`.toLowerCase();
+      let score = 0;
+      tokens.forEach((t) => { if (t.length > 2 && haystack.includes(t)) score += 3; });
+      if (entry.namespace.includes('memory/error')) score += 1;
+      if (entry.namespace.includes('memory/learning')) score += 1;
+      if (entry.createdAt) {
+        const hours = (Date.now() - new Date(entry.createdAt).getTime()) / 3_600_000;
+        if (Number.isFinite(hours)) {
+          if (hours < 24) score += 2;
+          else if (hours < 168) score += 1;
+        }
+      }
+      return { entry, score };
+    })
+    .filter((x) => x.score > 0)
+    .sort((a, b) => b.score - a.score)
+    .slice(0, maxResults);
+
+  return scored.map((x) => ({ ...x.entry, _score: x.score }));
+}
+
+function constructMemexPack({ query = '', maxItems = 8, maxChars = 6000, namespaces = [] } = {}) {
+  const normalizedNamespaces = normalizeNamespaces(namespaces);
+  const hits = searchMemexIndex({ query, maxResults: maxItems * 2, namespaces: normalizedNamespaces });
+
+  const items = [];
+  let usedChars = 0;
+  const dereferenced = [];
+
+  for (const hit of hits) {
+    if (items.length >= maxItems) break;
+    const full = dereferenceEntry(hit);
+    if (!full) continue;
+
+    const snippet = `${full.title}\n${full.content || ''}`;
+    if (usedChars + snippet.length > maxChars) continue;
+
+    items.push({
+      id: full.id,
+      namespace: hit.namespace,
+      title: full.title,
+      content: full.content,
+      tags: full.tags || [],
+      score: hit._score,
+    });
+    usedChars += snippet.length;
+    dereferenced.push(hit.id);
+  }
+
+  const packId = `memex_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+  const pack = {
+    packId,
+    query,
+    maxItems,
+    maxChars,
+    usedChars,
+    namespaces: normalizedNamespaces,
+    createdAt: nowIso(),
+    items,
+    indexHits: hits.length,
+    dereferencedCount: dereferenced.length,
+    cache: { hit: false },
+  };
+
+  appendJsonl(path.join(CONTEXTFS_ROOT, NAMESPACES.provenance, 'packs.jsonl'), pack);
+  recordProvenance({
+    type: 'memex_pack_constructed',
+    packId,
+    query,
+    indexHits: hits.length,
+    dereferencedCount: dereferenced.length,
+    usedChars,
+  });
+
+  return pack;
+}
+
 function constructContextPack({ query = '', maxItems = 8, maxChars = 6000, namespaces = [] } = {}) {
   const normalizedNamespaces = normalizeNamespaces(namespaces);
   const tokens = tokenizeQuery(query);
@@ -505,6 +630,11 @@ module.exports = {
   querySimilarity,
   findSemanticCacheHit,
   getSemanticCacheConfig,
+  buildIndexEntry,
+  loadMemexIndex,
+  dereferenceEntry,
+  searchMemexIndex,
+  constructMemexPack,
 };
 
 if (require.main === module) {