thumbgate 1.30.0 → 1.34.0

package/scripts/document-intake.js

@@ -169,7 +169,7 @@ function writeJsonl(filePath, records) {
 }
 
 function normalizeText(value) {
-  const withoutBom = String(value || '').split('\uFEFF').join('');
+  const withoutBom = String(value || '').replaceAll('\uFEFF', '');
   const normalizedNewlines = normalizeNewlines(withoutBom);
   const trimmedLines = normalizedNewlines
     .split('\n')
@@ -189,6 +189,70 @@ function normalizeTags(tags) {
     .filter(Boolean)));
 }
 
+function normalizeAccessIdentifier(value) {
+  const normalized = String(value || '').trim();
+  if (!normalized) return null;
+  return normalized.slice(0, 160);
+}
+
+function normalizeAccessContext(context = {}) {
+  const source = context && typeof context === 'object' ? context : {};
+  return {
+    tenantId: normalizeAccessIdentifier(source.tenantId),
+    principalId: normalizeAccessIdentifier(source.principalId),
+    isAdmin: source.isAdmin === true,
+  };
+}
+
+function buildDocumentAccess(options = {}) {
+  const context = normalizeAccessContext(options.accessContext);
+  if (!context.tenantId && !context.principalId) return null;
+  const visibility = options.visibility === 'private' ? 'private' : 'tenant';
+  const allowedPrincipals = safeArray(options.allowedPrincipals)
+    .map(normalizeAccessIdentifier)
+    .filter(Boolean);
+  return {
+    tenantId: context.tenantId,
+    ownerId: context.principalId,
+    visibility,
+    allowedPrincipals: [...new Set(allowedPrincipals)],
+  };
+}
+
+/**
+ * Fail-closed authorization for protected imported documents.
+ *
+ * Legacy local documents without access metadata stay readable for backwards
+ * compatibility. Once a document carries a tenant, owner, or principal ACL,
+ * callers must present a matching authorization context on every read path.
+ */
+function documentAccessAllowed(document = {}, accessContext = {}) {
+  const access = document.access;
+  const protectedDocument = access && typeof access === 'object' && (
+    normalizeAccessIdentifier(access.tenantId)
+    || normalizeAccessIdentifier(access.ownerId)
+    || safeArray(access.allowedPrincipals).length > 0
+  );
+  if (!protectedDocument) return true;
+
+  const caller = normalizeAccessContext(accessContext);
+  if (caller.isAdmin) return true;
+
+  const tenantId = normalizeAccessIdentifier(access.tenantId);
+  if (tenantId && (!caller.tenantId || caller.tenantId !== tenantId)) return false;
+
+  if (access.visibility === 'tenant') {
+    return Boolean(caller.tenantId && caller.tenantId === tenantId);
+  }
+
+  if (!caller.principalId) return false;
+  const allowed = new Set([
+    normalizeAccessIdentifier(access.ownerId),
+    ...safeArray(access.allowedPrincipals).map(normalizeAccessIdentifier),
+  ].filter(Boolean));
+  return allowed.has(caller.principalId);
+}
+
 function normalizeNewlines(value) {
   let result = '';
   const text = String(value || '');
@@ -716,6 +780,7 @@ function buildDocumentSummary(document) {
     proposalCount: safeArray(document.proposals).length,
     matchedTemplateIds: safeArray(document.matchedTemplateIds),
     fingerprint: document.fingerprint,
+    access: document.access || null,
   };
 }
 
@@ -723,7 +788,8 @@ function readImportedDocument(documentId, options = {}) {
   const filePath = getDocumentPath(String(documentId || '').trim(), options);
   if (!fs.existsSync(filePath)) return null;
   try {
-    return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+    const document = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+    return documentAccessAllowed(document, options.accessContext) ? document : null;
   } catch {
     return null;
   }
@@ -739,6 +805,7 @@ function listImportedDocuments(options = {}) {
   const documents = readJsonl(catalogPath);
 
   const filtered = documents.filter((document) => {
+    if (!documentAccessAllowed(document, options.accessContext)) return false;
     const tags = safeArray(document.tags).map((tag) => String(tag).toLowerCase());
     const matchedTemplateIds = safeArray(document.matchedTemplateIds).map((tag) => String(tag).toLowerCase());
     if (requestedTag && !tags.includes(requestedTag) && !matchedTemplateIds.includes(requestedTag)) {
@@ -765,10 +832,11 @@ function persistDocument(document, options = {}) {
   const paths = getDocumentStorePaths(options);
   ensureDir(paths.documentsDir);
   writeJson(getDocumentPath(document.documentId, options), document);
-  const summaries = listImportedDocuments({
-    ...options,
-    limit: MAX_SEARCH_SCAN,
-  }).documents.filter((entry) => entry.documentId !== document.documentId);
+  // Persistence must retain entries the current principal cannot read; using
+  // the public list function here would silently delete other principals'
+  // catalog rows during an import.
+  const summaries = readJsonl(paths.catalogPath)
+    .filter((entry) => entry.documentId !== document.documentId);
   const nextSummaries = [
     buildDocumentSummary(document),
     ...summaries,
@@ -849,6 +917,188 @@ function searchImportedDocuments(options = {}) {
   return docs.slice(0, limit);
 }
 
+function importedDocumentMatchesFilters(document, filters = {}) {
+  if (!filters || typeof filters !== 'object') return true;
+  const normalize = (value) => (
+    (Array.isArray(value) ? value : value == null ? [] : [value])
+      .map((item) => String(item).trim().toLowerCase())
+      .filter(Boolean)
+  );
+  for (const [field, actualValue] of [
+    ['sourceFormat', document.sourceFormat],
+    ['sourceType', document.sourceType],
+  ]) {
+    const required = normalize(filters[field]);
+    if (required.length > 0 && !required.includes(String(actualValue || '').toLowerCase())) {
+      return false;
+    }
+  }
+  const requiredTags = normalize(filters.tags);
+  if (requiredTags.length > 0) {
+    const actualTags = normalize(document.tags);
+    const matches = filters.requireAllTags === false
+      ? requiredTags.some((tag) => actualTags.includes(tag))
+      : requiredTags.every((tag) => actualTags.includes(tag));
+    if (!matches) return false;
+  }
+  return true;
+}
+
+/**
+ * Production document retrieval: rank child chunks, then hydrate their parent
+ * documents. Whole-file results stay bounded while evidence can come from any
+ * position in a long document.
+ */
+async function searchImportedDocumentsAsync(options = {}) {
+  const query = String(options.query || '').trim();
+  if (!query) throw new Error('query is required');
+  const limit = Number.isFinite(Number(options.limit))
+    ? Math.max(1, Math.min(50, Number(options.limit)))
+    : 10;
+  const documents = listImportedDocuments({
+    ...options,
+    limit: MAX_SEARCH_SCAN,
+    query: '',
+  }).documents
+    .map((summary) => readImportedDocument(summary.documentId, options))
+    .filter(Boolean)
+    .filter((document) => importedDocumentMatchesFilters(
+      document,
+      options.metadataFilters || options.filters,
+    ));
+  if (documents.length === 0) return [];
+
+  const { runDocumentPipeline } = require('./rag-document-pipeline');
+  const { pragmaticHybridSearch } = require('./pragmatic-hybrid-search');
+  const { buildQueryPlan, reciprocalRankFusion } = require('./lesson-retrieval');
+  const pipeline = runDocumentPipeline(documents.map((document) => ({
+    type: 'text',
+    id: document.documentId,
+    title: document.title,
+    content: document.content,
+    tags: document.tags,
+    source: 'imported_document',
+    metadata: {
+      documentId: document.documentId,
+      sourceFormat: document.sourceFormat,
+      sourceType: document.sourceType,
+      importedAt: document.importedAt,
+    },
+  })), {
+    maxChars: options.maxChunkChars || 900,
+    overlap: options.chunkOverlap || 120,
+  });
+  const chunks = pipeline.chunks;
+  if (chunks.length === 0) return [];
+
+  const queryPlan = await buildQueryPlan(query, options);
+  const queryVariants = queryPlan.variants;
+  let denseRankedIds = [];
+  let semanticProvider = null;
+  try {
+    const embeddingIndex = require('./lesson-embedding-index');
+    if (options.embedder || embeddingIndex.isEmbedderAvailable()) {
+      const denseLists = [];
+      for (const variant of queryVariants) {
+        const dense = await embeddingIndex.semanticRank(variant, chunks, {
+          feedbackDir: options.feedbackDir,
+          embedder: options.embedder,
+          embedderId: options.embedderId,
+          cacheFile: 'document-chunk-embeddings.json',
+          // `chunks` is metadata-filtered. Pruning this shared cache against a
+          // subset would make alternating filters evict and re-embed each other.
+          pruneCache: false,
+        });
+        const topScore = dense[0]?.score ?? 0;
+        const minimum = Math.max(
+          Number(options.minSemanticScore) || 0.15,
+          topScore - (Number(options.semanticScoreWindow) || 0.2),
+        );
+        denseLists.push(
+          dense.filter((entry) => entry.score >= minimum).map((entry) => entry.id),
+        );
+      }
+      denseRankedIds = reciprocalRankFusion(denseLists).map((entry) => entry.id);
+      semanticProvider = options.embedderId || 'configured';
+    }
+  } catch {
+    denseRankedIds = [];
+  }
+
+  const { results: rankedChunks, meta } = pragmaticHybridSearch({
+    corpus: chunks,
+    query,
+    toolName: options.toolName || 'Read',
+    options: {
+      topK: Math.min(chunks.length, Math.max(limit * 4, 20)),
+      pool: Math.min(chunks.length, Math.max(limit * 10, 50)),
+      queryVariants,
+      denseRankedIds,
+      diversify: false,
+    },
+  });
+  const lexicalEvidenceTokens = Array.from(new Set(
+    queryVariants.flatMap((variant) => tokenize(variant)),
+  ));
+  const denseEvidenceIds = new Set(denseRankedIds);
+  const byDocumentId = new Map(documents.map((document) => [document.documentId, document]));
+  const grouped = new Map();
+  for (const chunk of rankedChunks) {
+    const documentId = chunk.metadata?.documentId;
+    if (!documentId || !byDocumentId.has(documentId)) continue;
+    const lexicalHaystack = [
+      chunk.title,
+      chunk.content,
+      safeArray(chunk.tags).join(' '),
+    ].join(' ').toLowerCase();
+    const hasQueryOverlap = lexicalEvidenceTokens.some((token) => lexicalHaystack.includes(token));
+    const hasDenseEvidence = denseEvidenceIds.has(chunk.id);
+    // A synthetic tool name such as `Read` can contribute to scoreRelevance,
+    // but it is not query evidence. Require lexical overlap or a dense hit.
+    if (!hasQueryOverlap && !hasDenseEvidence) continue;
+    const score = Number(chunk.rerankedScore ?? chunk.relevanceScore ?? 0);
+    const group = grouped.get(documentId) || { score, chunks: [] };
+    group.score = Math.max(group.score, score);
+    if (group.chunks.length < 3) {
+      group.chunks.push({
+        chunkId: chunk.id,
+        chunkIndex: chunk.metadata?.chunkIndex ?? 0,
+        startChar: chunk.metadata?.startChar ?? null,
+        endChar: chunk.metadata?.endChar ?? null,
+        content: String(chunk.content || '').slice(0, 700),
+        score: Number(score.toFixed(4)),
+      });
+    }
+    grouped.set(documentId, group);
+  }
+
+  const queryTokens = tokenize(query);
+  return [...grouped.entries()]
+    .sort((left, right) => right[1].score - left[1].score)
+    .slice(0, limit)
+    .map(([documentId, group]) => {
+      const document = byDocumentId.get(documentId);
+      return {
+        ...document,
+        excerpt: group.chunks[0]?.content || document.excerpt,
+        _score: Number(group.score.toFixed(4)),
+        _matchedTokens: queryTokens.filter((token) => (
+          group.chunks.some((chunk) => chunk.content.toLowerCase().includes(token))
+        )),
+        _matchedChunks: group.chunks,
+        _retrieval: {
+          strategy: meta.strategy,
+          parentChild: true,
+          chunkCount: chunks.length,
+          queryVariants,
+          queryTransformation: queryPlan,
+          densePool: meta.densePool,
+          semanticProvider,
+        },
+      };
+    });
+}
+
 function importDocument(options = {}) {
   const hasFilePath = Boolean(options.filePath);
   const hasContent = typeof options.content === 'string' && options.content.trim().length > 0;
@@ -881,10 +1131,30 @@ function importDocument(options = {}) {
     normalizedContent,
     sourceFormat,
   });
+  const access = buildDocumentAccess(options);
   const fingerprint = sha256(`${title}\n${normalizedContent}`);
+  // Protected documents need a storage identity that includes their owning
+  // scope. Otherwise identical content imported by two private principals (or
+  // as both tenant-visible and private) would overwrite the first document and
+  // silently replace its ACL. Keep legacy local IDs unchanged.
+  const storageScope = !access
+    ? null
+    : access.visibility === 'private'
+      ? JSON.stringify({
+        tenantId: access.tenantId,
+        visibility: 'private',
+        ownerId: access.ownerId,
+      })
+      : JSON.stringify({
+        tenantId: access.tenantId,
+        visibility: 'tenant',
+      });
+  const storageFingerprint = storageScope
+    ? sha256(`${fingerprint}\n${storageScope}`)
+    : fingerprint;
   const importedAt = nowIso();
   const sourceName = sourcePath ? path.basename(sourcePath) : null;
-  const documentId = `doc_${slugify(title || sourceName || 'document').slice(0, 24) || 'document'}_${fingerprint.slice(0, 12)}`;
+  const documentId = `doc_${slugify(title || sourceName || 'document').slice(0, 24) || 'document'}_${storageFingerprint.slice(0, 12)}`;
   const document = {
     documentId,
     title,
@@ -902,6 +1172,7 @@ function importDocument(options = {}) {
     lineCount: normalizedContent.split('\n').filter(Boolean).length,
     headings: extractHeadings(normalizedContent),
   };
+  if (access) document.access = access;
   document.proposals = options.proposeGates === false
     ? []
     : proposeGatesFromDocument(document, options);
@@ -924,4 +1195,9 @@ module.exports = {
   proposeGatesFromDocument,
   readImportedDocument,
   searchImportedDocuments,
+  searchImportedDocumentsAsync,
+  importedDocumentMatchesFilters,
+  buildDocumentAccess,
+  documentAccessAllowed,
+  normalizeAccessContext,
 };

package/scripts/eval-quality-suite.js

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Unified bounded offline evaluation suite:
+ *   1) IR ranking — Recall@k / MRR / nDCG / Precision@k (retrieval-ranking-golden)
+ *   2) Generation quality — faithfulness / groundedness / answer_relevance (offline)
+ *
+ * Exit 0 only when both floors pass. Offline by default (no API key required).
+ *
+ *   npm run eval:quality
+ *   node scripts/eval-quality-suite.js --json
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+const {
+  evaluateRankingGolden,
+  formatRankingReport,
+} = require('./retrieval-ranking-eval');
+const {
+  evaluateGenerationGolden,
+  METRICS_VERSION,
+} = require('./ragas-style-metrics');
+
+const ROOT = path.join(__dirname, '..');
+const GEN_GOLDEN = path.join(ROOT, 'config', 'evals', 'generation-quality-golden.json');
+const REPORT_MD = path.join(ROOT, 'reports', 'eval-quality-suite.md');
+const REPORT_JSON = path.join(ROOT, 'reports', 'eval-quality-suite.json');
+const SUITE_VERSION = '2026-07-31.a-plus.1';
+
+function loadGenerationGolden() {
+  return JSON.parse(fs.readFileSync(GEN_GOLDEN, 'utf8'));
+}
+
+/**
+ * Drop expectFail / known-bad demos from floor means (still reported in rows).
+ */
+function filterGenerationCases(golden) {
+  const cases = (golden.cases || []).filter((c) => !c.expectFail);
+  return { ...golden, cases };
+}
+
+function runSuite(options = {}) {
+  const ranking = evaluateRankingGolden({
+    goldenPath: options.rankingGoldenPath,
+    thresholds: options.rankingThresholds,
+  });
+
+  const genGoldenRaw = options.generationGolden || loadGenerationGolden();
+  const genGolden = filterGenerationCases(genGoldenRaw);
+  const generation = evaluateGenerationGolden(genGolden, {
+    thresholds: options.generationThresholds,
+  });
+
+  // Also score known-bad cases for diagnostics (must score worse on faithfulness)
+  const badCases = (genGoldenRaw.cases || []).filter((c) => c.expectFail || c.id === 'ungrounded-contradiction');
+  const badScores = badCases.map((c) => require('./ragas-style-metrics').scoreGenerationCase(c));
+
+  const failures = [
+    ...(ranking.passed ? [] : ranking.failures.map((f) => `ranking: ${f}`)),
+    ...(generation.passed ? [] : generation.failures.map((f) => `generation: ${f}`)),
+  ];
+
+  // Integrity: bad contradiction sample should not beat good faithfulness mean
+  if (badScores.length > 0 && generation.summary.faithfulness > 0) {
+    const badFaith = badScores.reduce((s, r) => s + r.faithfulness, 0) / badScores.length;
+    if (badFaith >= generation.summary.faithfulness) {
+      failures.push(
+        `integrity: bad-case faithfulness ${badFaith.toFixed(3)} >= good mean ${generation.summary.faithfulness}`,
+      );
+    }
+  }
+
+  const report = {
+    suiteVersion: SUITE_VERSION,
+    metricsVersion: METRICS_VERSION,
+    passed: failures.length === 0,
+    failures,
+    ranking: {
+      passed: ranking.passed,
+      failures: ranking.failures,
+      summary: ranking.summary,
+      thresholds: ranking.thresholds,
+      queryCount: ranking.perQuery?.length || 0,
+    },
+    generation: {
+      passed: generation.passed,
+      failures: generation.failures,
+      summary: generation.summary,
+      thresholds: generation.thresholds,
+      caseCount: generation.rows?.length || 0,
+      badCaseDiagnostics: badScores,
+    },
+    grades: {
+      recallAtK: ranking.passed ? 'A+' : 'fail',
+      mrr: ranking.passed ? 'A+' : 'fail',
+      ndcg: ranking.passed ? 'A+' : 'fail',
+      precisionAtK: ranking.passed ? 'A+' : 'fail',
+      faithfulness: generation.passed ? 'A+' : 'fail',
+      groundedness: generation.passed ? 'A+' : 'fail',
+      answerRelevance: generation.passed ? 'A+' : 'fail',
+      overall: failures.length === 0 ? 'A+' : 'fail',
+    },
+  };
+
+  return { report, ranking, generation };
+}
+
+function formatMarkdown(report) {
+  const r = report.ranking.summary || {};
+  const g = report.generation.summary || {};
+  const lines = [
+    '# Evaluation quality suite (bounded offline)',
+    '',
+    `Suite version: \`${report.suiteVersion}\``,
+    `Overall: **${report.passed ? 'PASS' : 'FAIL'}** · grade **${report.grades.overall}**`,
+    '',
+    '## IR ranking (Recall@k / MRR / nDCG / Precision@k)',
+    '',
+    `| Metric | Value |`,
+    `|--------|------:|`,
+    `| Queries | ${report.ranking.queryCount} |`,
+    `| MRR | ${((r.mrr || 0) * 100).toFixed(1)}% |`,
+    `| Recall@5 | ${((r['recall@5'] || 0) * 100).toFixed(1)}% |`,
+    `| Precision@5 | ${((r['precision@5'] || 0) * 100).toFixed(1)}% |`,
+    `| nDCG@5 | ${((r['ndcg@5'] || 0) * 100).toFixed(1)}% |`,
+    `| Ranking gate | ${report.ranking.passed ? 'PASS' : 'FAIL'} |`,
+    '',
+    '## Generation quality (offline Ragas-style)',
+    '',
+    `| Metric | Value |`,
+    `|--------|------:|`,
+    `| Cases | ${report.generation.caseCount} |`,
+    `| Faithfulness | ${((g.faithfulness || 0) * 100).toFixed(1)}% |`,
+    `| Groundedness | ${((g.groundedness || 0) * 100).toFixed(1)}% |`,
+    `| Answer relevance | ${((g.answer_relevance || 0) * 100).toFixed(1)}% |`,
+    `| Context recall | ${((g.context_recall || 0) * 100).toFixed(1)}% |`,
+    `| Context precision | ${((g.context_precision || 0) * 100).toFixed(1)}% |`,
+    `| Generation gate | ${report.generation.passed ? 'PASS' : 'FAIL'} |`,
+    '',
+    '## Grades',
+    '',
+    Object.entries(report.grades).map(([k, v]) => `- **${k}**: ${v}`).join('\n'),
+    '',
+  ];
+  if (report.failures.length) {
+    lines.push('## Failures', '', ...report.failures.map((f) => `- ${f}`), '');
+  }
+  lines.push(
+    '## Honesty',
+    '',
+    '- IR metrics use graded qrels on a self-contained golden corpus.',
+    '- Generation metrics are offline lexical/claim proxies (not neural Ragas).',
+    '- Optional LLM judges may refine scores but cannot invent a pass when floors fail.',
+    '',
+  );
+  return lines.join('\n');
+}
+
+function writeReports(report) {
+  fs.mkdirSync(path.dirname(REPORT_MD), { recursive: true });
+  fs.writeFileSync(REPORT_MD, formatMarkdown(report));
+  fs.writeFileSync(REPORT_JSON, JSON.stringify(report, null, 2) + '\n');
+}
+
+function main() {
+  const json = process.argv.includes('--json');
+  const { report, ranking } = runSuite();
+  writeReports(report);
+
+  if (json) {
+    console.log(JSON.stringify(report, null, 2));
+  } else {
+    console.log(formatMarkdown(report));
+    // Also print ranking detail when verbose
+    if (process.argv.includes('--verbose') && ranking) {
+      console.log('\n' + formatRankingReport(ranking));
+    }
+    console.error(report.passed
+      ? 'eval-quality-suite: PASS (bounded offline floors met)'
+      : `eval-quality-suite: FAIL — ${report.failures.join('; ')}`);
+  }
+  process.exit(report.passed ? 0 : 1);
+}
+
+function isCliEntrypoint(argv = process.argv) {
+  return Boolean(argv[1]) && path.resolve(argv[1]) === path.resolve(__filename);
+}
+
+if (isCliEntrypoint()) {
+  main();
+}
+
+module.exports = {
+  SUITE_VERSION,
+  runSuite,
+  formatMarkdown,
+  loadGenerationGolden,
+  GEN_GOLDEN,
+  REPORT_MD,
+  REPORT_JSON,
+  isCliEntrypoint,
+};