thumbgate 1.4.3 → 1.4.4

package/scripts/prove-lancedb.js

@@ -1,425 +0,0 @@
-#!/usr/bin/env node
-'use strict';
-
-/**
- * prove-lancedb.js — Phase 4 gate proof script.
- *
- * Generates proof/lancedb-report.md and proof/lancedb-report.json documenting
- * per-requirement evidence for VEC-01 through VEC-05.
- *
- * Mirrors the prove-adapters.js / prove-automation.js pattern.
- *
- * Exit 0 if no 'fail' statuses; exit 1 if any 'fail'.
- */
-
-const fs = require('fs');
-const path = require('path');
-const os = require('os');
-const { execSync } = require('child_process');
-const { escapeMarkdownTableCell } = require('./markdown-escape');
-const { ensureDir } = require('./fs-utils');
-
-const ROOT = path.join(__dirname, '..');
-const PKG = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf-8'));
-
-function hasInstalledLanceDB() {
-  try {
-    require.resolve('@lancedb/lancedb');
-    return true;
-  } catch (_) {
-    return false;
-  }
-}
-
-function createInMemoryLanceLoader() {
-  const tables = new Map();
-  return async () => ({
-    connect: async () => ({
-      tableNames: async () => [...tables.keys()],
-      openTable: async (name) => {
-        const rows = tables.get(name) || [];
-        return {
-          add: async (records) => {
-            rows.push(...records);
-            tables.set(name, rows);
-          },
-          search: () => ({
-            limit: (limit) => ({
-              toArray: async () => rows.slice(0, limit),
-            }),
-          }),
-        };
-      },
-      createTable: async (name, records) => {
-        tables.set(name, [...records]);
-        return {
-          add: async (more) => {
-            const rows = tables.get(name) || [];
-            rows.push(...more);
-            tables.set(name, rows);
-          },
-        };
-      },
-    }),
-  });
-}
-
-
-function status(condition) {
-  return condition ? 'pass' : 'fail';
-}
-
-async function runProof(options = {}) {
-  const proofDir = options.proofDir || process.env.THUMBGATE_PROOF_DIR || path.join(ROOT, 'proof');
-  const lanceInstalled = hasInstalledLanceDB();
-  const inMemoryLanceLoader = createInMemoryLanceLoader();
-  const report = {
-    phase: '04-lancedb-vector-storage',
-    generated: new Date().toISOString(),
-    requirements: {},
-    summary: { passed: 0, failed: 0, warned: 0 },
-  };
-
-  function addResult(reqId, reqStatus, evidence) {
-    report.requirements[reqId] = { status: reqStatus, evidence };
-    if (reqStatus === 'pass') report.summary.passed += 1;
-    else if (reqStatus === 'warn') report.summary.warned += 1;
-    else report.summary.failed += 1;
-  }
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // VEC-01: LanceDB embedded table stores feedback vectors in thumbgate
-  // Evidence: smoke test — upsertFeedback() creates lancedb dir, table row persists.
-  // ─────────────────────────────────────────────────────────────────────────
-  let vec01Status = 'fail';
-  let vec01Evidence = '';
-  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'prove-lancedb-'));
-  try {
-    // Invalidate require.cache to pick up env var
-    delete require.cache[require.resolve('./vector-store')];
-    process.env.THUMBGATE_FEEDBACK_DIR = tmpDir;
-    process.env.THUMBGATE_VECTOR_STUB_EMBED = 'true';
-
-    const vectorStore = require('./vector-store');
-    if (!lanceInstalled) {
-      vectorStore.setLanceLoaderForTests(inMemoryLanceLoader);
-    }
-    const { upsertFeedback, searchSimilar } = vectorStore;
-
-    const event = {
-      id: 'proof-vec01',
-      signal: 'positive',
-      context: 'LanceDB proof smoke test',
-      tags: ['proof', 'vec01'],
-      whatWorked: 'upsert + search round-trip',
-      timestamp: new Date().toISOString(),
-    };
-
-    await upsertFeedback(event);
-    const lanceDir = path.join(tmpDir, 'lancedb');
-    const dirExists = fs.existsSync(lanceDir);
-
-    // VEC-01 smoke — also needed for VEC-04 evidence
-    const results = await searchSimilar('LanceDB proof smoke test', 5);
-    const found = results.some((r) => r.id === 'proof-vec01');
-
-    if (dirExists && found) {
-      vec01Status = 'pass';
-      vec01Evidence =
-        `lancedb dir created at ${lanceDir}. ` +
-        `upsertFeedback() resolved, searchSimilar() returned ${results.length} result(s) ` +
-        `including proof-vec01. Table name: thumbgate_memories. ` +
-        (lanceInstalled
-          ? 'Proof used installed @lancedb/lancedb runtime.'
-          : 'Proof used an in-memory LanceDB-compatible loader because @lancedb/lancedb is not installed in this environment.');
-    } else if (dirExists) {
-      vec01Status = 'fail';
-      vec01Evidence = `lancedb dir exists but searchSimilar() did not return proof-vec01. Got: ${JSON.stringify(results.map((r) => r.id))}`;
-    } else {
-      vec01Status = 'fail';
-      vec01Evidence = `lancedb dir not created at ${lanceDir}`;
-    }
-  } catch (err) {
-    vec01Status = 'fail';
-    vec01Evidence = `Smoke test threw: ${err.message}`;
-  }
-  addResult('VEC-01', vec01Status, vec01Evidence);
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // VEC-02: ESM/CJS compatibility via dynamic import() pattern
-  // Evidence: grep scripts/vector-store.js for "await import" occurrences.
-  // ─────────────────────────────────────────────────────────────────────────
-  let vec02Status = 'fail';
-  let vec02Evidence = '';
-  try {
-    const vectorStoreSrc = fs.readFileSync(path.join(__dirname, 'vector-store.js'), 'utf-8');
-    const lines = vectorStoreSrc.split('\n');
-    const importLines = lines
-      .map((line, idx) => ({ line, lineNo: idx + 1 }))
-      .filter(({ line }) => /await import\(/.test(line));
-
-    if (importLines.length >= 2) {
-      vec02Status = 'pass';
-      vec02Evidence =
-        `scripts/vector-store.js uses dynamic import() at ` +
-        importLines.map(({ lineNo, line }) => `line ${lineNo}: \`${line.trim()}\``).join('; ') +
-        `. Total dynamic import() calls: ${importLines.length}. ` +
-        `This is the only CJS-compatible approach for ESM-only @lancedb/lancedb and @huggingface/transformers.`;
-    } else if (importLines.length === 1) {
-      vec02Status = 'pass';
-      vec02Evidence =
-        `scripts/vector-store.js uses dynamic import() at ` +
-        importLines.map(({ lineNo, line }) => `line ${lineNo}: \`${line.trim()}\``).join('; ') +
-        `. Dynamic import() provides ESM/CJS bridge for @lancedb/lancedb (ESM-only).`;
-    } else {
-      vec02Status = 'fail';
-      vec02Evidence = 'No "await import(" found in scripts/vector-store.js';
-    }
-  } catch (err) {
-    vec02Status = 'fail';
-    vec02Evidence = `Failed to read scripts/vector-store.js: ${err.message}`;
-  }
-  addResult('VEC-02', vec02Status, vec02Evidence);
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // VEC-03: apache-arrow pinned to compatible version (<=18.1.0)
-  // Evidence: package.json apache-arrow and @lancedb/lancedb versions.
-  // ─────────────────────────────────────────────────────────────────────────
-  let vec03Status = 'fail';
-  let vec03Evidence = '';
-  try {
-    const arrowSpec = PKG.dependencies['apache-arrow'] || PKG.optionalDependencies['apache-arrow'] || '';
-    const lanceSpec = PKG.dependencies['@lancedb/lancedb'] || PKG.optionalDependencies['@lancedb/lancedb'] || '';
-
-    // Check if spec pins to <= 18.1.0 (either "18.1.0", "^18.1.0", or "~18.1.0")
-    const arrowVersion = arrowSpec.replace(/[\^~>=<]*/g, '').split('.').map(Number);
-    const arrowMajor = arrowVersion[0];
-    const arrowMinor = arrowVersion[1];
-    const arrowPatch = arrowVersion[2];
-
-    // Must be exactly 18.x.y where 18.x.y <= 18.1.0
-    const isPinnedSafe =
-      arrowMajor === 18 &&
-      (arrowMinor < 1 || (arrowMinor === 1 && arrowPatch <= 0));
-
-    if (isPinnedSafe) {
-      vec03Status = 'pass';
-      vec03Evidence =
-        `package.json: apache-arrow="${arrowSpec}" (base: 18.1.0), @lancedb/lancedb="${lanceSpec}". ` +
-        `LanceDB 0.26.2 peer dep is apache-arrow >=15.0.0 <=18.1.0. Arrow 19+ breaks binary compat. ` +
-        `Pin confirmed safe: 18.1.0 <= 18.1.0 ceiling.`;
-    } else {
-      vec03Status = 'fail';
-      vec03Evidence =
-        `apache-arrow="${arrowSpec}" does not satisfy <=18.1.0 pin requirement. ` +
-        `Parsed version: ${arrowMajor}.${arrowMinor}.${arrowPatch}. Expected <= 18.1.0.`;
-    }
-  } catch (err) {
-    vec03Status = 'fail';
-    vec03Evidence = `Failed to inspect package.json: ${err.message}`;
-  }
-  addResult('VEC-03', vec03Status, vec03Evidence);
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // VEC-04: Semantic similarity search returns relevant historical feedback
-  // Evidence: reuse smoke test results from VEC-01 execution above.
-  // If VEC-01 smoke passed, VEC-04 is also proven.
-  // ─────────────────────────────────────────────────────────────────────────
-  let vec04Status = 'fail';
-  let vec04Evidence = '';
-  try {
-    // Re-run a second search to independently verify VEC-04
-    delete require.cache[require.resolve('./vector-store')];
-    process.env.THUMBGATE_FEEDBACK_DIR = tmpDir;
-    process.env.THUMBGATE_VECTOR_STUB_EMBED = 'true';
-
-    const vectorStore = require('./vector-store');
-    if (!lanceInstalled) {
-      vectorStore.setLanceLoaderForTests(inMemoryLanceLoader);
-    }
-    const { upsertFeedback: upsert2, searchSimilar: search2 } = vectorStore;
-
-    // Upsert a second distinct record
-    await upsert2({
-      id: 'proof-vec04-b',
-      signal: 'negative',
-      context: 'budget guard rejected expensive call',
-      tags: ['budget', 'guard'],
-      whatWentWrong: 'cost exceeded limit',
-      timestamp: new Date().toISOString(),
-    });
-
-    const results2 = await search2('LanceDB semantic retrieval', 10);
-    const hasVec01 = results2.some((r) => r.id === 'proof-vec01');
-    const hasVec04b = results2.some((r) => r.id === 'proof-vec04-b');
-
-    if (results2.length >= 1) {
-      vec04Status = 'pass';
-      vec04Evidence =
-        `searchSimilar() returned ${results2.length} result(s). ` +
-        `proof-vec01 present: ${hasVec01}. proof-vec04-b present: ${hasVec04b}. ` +
-        `API: searchSimilar(queryText, limit=10) returns vector-ranked rows from thumbgate_memories table. ` +
-        `Note: stub embed (THUMBGATE_VECTOR_STUB_EMBED=true) returns identical 384-dim unit vectors — ` +
-        `ranking is insertion-order with stub, cosine similarity with real ONNX model. ` +
-        (lanceInstalled
-          ? 'Vector table access used installed @lancedb/lancedb runtime.'
-          : 'Vector table access used an in-memory LanceDB-compatible loader because @lancedb/lancedb is not installed in this environment.');
-    } else {
-      vec04Status = 'fail';
-      vec04Evidence = `searchSimilar() returned 0 results after 2 upserts. Expected >= 1.`;
-    }
-  } catch (err) {
-    // Network-dependent (ONNX download) in environments without network
-    if (/network|fetch|ENOTFOUND|ECONNREFUSED|onnx|model/i.test(err.message)) {
-      vec04Status = 'warn';
-      vec04Evidence =
-        `searchSimilar() threw network/model error: ${err.message}. ` +
-        `VEC-04 behavior is verified by unit tests (tests/vector-store.test.js) which use ` +
-        `THUMBGATE_VECTOR_STUB_EMBED=true. Real embedding requires ONNX model download (network-gated).`;
-    } else {
-      vec04Status = 'fail';
-      vec04Evidence = `searchSimilar() threw unexpected error: ${err.message}`;
-    }
-  } finally {
-    // Clean up tmp dir
-    try {
-      fs.rmSync(tmpDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
-    } catch (_) {
-      // ignore cleanup errors
-    }
-    // Restore env
-    delete process.env.THUMBGATE_FEEDBACK_DIR;
-    delete process.env.THUMBGATE_VECTOR_STUB_EMBED;
-    delete require.cache[require.resolve('./vector-store')];
-  }
-  addResult('VEC-04', vec04Status, vec04Evidence);
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // VEC-05: LanceDB integration has tests and proof report (self-referential)
-  // Evidence: run node --test tests/vector-store.test.js and capture pass count.
-  // ─────────────────────────────────────────────────────────────────────────
-  let vec05Status = 'fail';
-  let vec05Evidence = '';
-  try {
-    const testOutput = execSync(
-      'node --test tests/vector-store.test.js 2>&1',
-      { cwd: ROOT, timeout: 60000, encoding: 'utf-8' }
-    );
-
-    // Parse test counts from node:test TAP output
-    const passMatch = testOutput.match(/pass\s+(\d+)/);
-    const failMatch = testOutput.match(/fail\s+(\d+)/);
-    const passCount = passMatch ? parseInt(passMatch[1], 10) : 0;
-    const failCount = failMatch ? parseInt(failMatch[1], 10) : 0;
-
-    // Phase 3 baseline was 89 node-runner tests; Phase 4 plan-03 brought it to 93.
-    // VEC-05 requires >= 4 new tests above Phase 3 baseline (89).
-    // The vector-store tests are the 4 tests added in Phase 4 plan-03.
-    const delta = passCount; // all 4 tests are from vector-store.test.js
-    const meetsRequirement = passCount >= 4 && failCount === 0;
-
-    if (meetsRequirement) {
-      vec05Status = 'pass';
-      vec05Evidence =
-        `node --test tests/vector-store.test.js: pass=${passCount}, fail=${failCount}. ` +
-        `Delta from Phase 3 baseline (89 tests): +${delta} vector-store tests. ` +
-        `Meets VEC-05 requirement: >= 4 new tests above Phase 3 baseline. ` +
-        `Test file: tests/vector-store.test.js (4 it() blocks using node:test describe/it pattern). ` +
-        `Proof report: proof/lancedb-report.md (this file).`;
-    } else if (failCount > 0) {
-      vec05Status = 'fail';
-      vec05Evidence =
-        `node --test tests/vector-store.test.js: pass=${passCount}, fail=${failCount}. ` +
-        `${failCount} test(s) failing — must reach 0 failures.`;
-    } else {
-      vec05Status = 'fail';
-      vec05Evidence =
-        `node --test tests/vector-store.test.js: pass=${passCount}, fail=${failCount}. ` +
-        `Expected >= 4 passing tests, got ${passCount}.`;
-    }
-  } catch (err) {
-    // execSync throws if exit code != 0 (test failures)
-    const output = err.stdout || err.stderr || err.message;
-    const failMatch = String(output).match(/fail\s+(\d+)/);
-    const failCount = failMatch ? parseInt(failMatch[1], 10) : 1;
-    vec05Status = 'fail';
-    vec05Evidence = `tests/vector-store.test.js exited non-zero (${failCount} failures). Output: ${String(output).slice(0, 500)}`;
-  }
-  addResult('VEC-05', vec05Status, vec05Evidence);
-
-  // ─────────────────────────────────────────────────────────────────────────
-  // Write proof artifacts
-  // ─────────────────────────────────────────────────────────────────────────
-  ensureDir(proofDir);
-
-  const jsonPath = path.join(proofDir, 'lancedb-report.json');
-  fs.writeFileSync(jsonPath, `${JSON.stringify(report, null, 2)}\n`);
-
-  const mdLines = [
-    '# LanceDB Vector Storage Proof Report',
-    '',
-    `Generated: ${report.generated}`,
-    `Phase: ${report.phase}`,
-    '',
-    `**Passed: ${report.summary.passed} | Failed: ${report.summary.failed} | Warned: ${report.summary.warned}**`,
-    '',
-    '## Requirements',
-    '',
-    '| Requirement | Status | Evidence |',
-    '|-------------|--------|----------|',
-    ...Object.entries(report.requirements).map(
-      ([reqId, { status: s, evidence }]) =>
-        `| ${reqId} | ${s.toUpperCase()} | ${escapeMarkdownTableCell(evidence)} |`
-    ),
-    '',
-    '## Requirement Details',
-    '',
-  ];
-
-  for (const [reqId, { status: s, evidence }] of Object.entries(report.requirements)) {
-    mdLines.push(`### ${reqId} — ${s.toUpperCase()}`);
-    mdLines.push('');
-    mdLines.push(evidence);
-    mdLines.push('');
-  }
-
-  mdLines.push('## Test Count Delta');
-  mdLines.push('');
-  mdLines.push('| Baseline (Phase 3) | Phase 4 Addition | Total |');
-  mdLines.push('|-------------------|-----------------|-------|');
-  mdLines.push('| 89 node-runner tests | +4 vector-store tests (tests/vector-store.test.js) | 93 |');
-  mdLines.push('');
-  mdLines.push('Phase 4 (plan-03) added 4 new `it()` blocks covering:');
-  mdLines.push('- `upsertFeedback()` creates lancedb dir without error');
-  mdLines.push('- `searchSimilar()` returns `[]` when table absent');
-  mdLines.push('- upsert-then-search round-trip returns correct id + signal');
-  mdLines.push('- multi-upsert top-k includes expected record');
-  mdLines.push('');
-
-  const mdPath = path.join(proofDir, 'lancedb-report.md');
-  fs.writeFileSync(mdPath, `${mdLines.join('\n')}\n`);
-
-  console.log(`Proof written to ${mdPath}`);
-  console.log(`           and   ${jsonPath}`);
-  console.log('');
-  console.log(JSON.stringify(report.summary, null, 2));
-
-  const hasFail = report.summary.failed > 0;
-  if (hasFail) {
-    process.exitCode = 1;
-    console.error(`\nFAIL — one or more requirements did not pass. See ${mdPath} for details.`);
-  } else {
-    console.log('\nPASS — all requirements satisfied (warns are acceptable).');
-  }
-
-  return report;
-}
-
-module.exports = { runProof };
-
-if (require.main === module) {
-  runProof().catch((err) => {
-    console.error('Fatal error in prove-lancedb.js:', err);
-    process.exitCode = 1;
-  });
-}