thumbgate 0.9.10

package/scripts/hallucination-detector.js

@@ -0,0 +1,226 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Hallucination Detector — claim verification, confidence-weighted gates,
+ * retrieval-grounded verification.
+ *
+ * Turns ThumbGate from "block known-bad patterns" into "detect and block
+ * hallucinated claims" using 3 techniques from hallucination detection research.
+ */
+
+const { constructContextPack } = require('./contextfs');
+const { matchSkillPacks } = require('./skill-packs');
+
+// ---------------------------------------------------------------------------
+// 1. Claim Decomposition & Verification
+// ---------------------------------------------------------------------------
+
+const CLAIM_PATTERNS = [
+  { pattern: /\b(?:deployed|shipped|live|released)\b/i, type: 'deployment', verifyWith: ['health_check', 'version_match'] },
+  { pattern: /\b(?:tests?\s+pass|all\s+tests?\s+(?:pass|green))\b/i, type: 'test_result', verifyWith: ['test_output', 'exit_code'] },
+  { pattern: /\b(?:merged|PR\s+merged)\b/i, type: 'pr_merge', verifyWith: ['pr_state', 'ci_status'] },
+  { pattern: /\b(?:fixed|resolved|bug\s+fix)\b/i, type: 'fix_claim', verifyWith: ['test_evidence', 'reproduction_check'] },
+  { pattern: /\b(?:published|npm\s+publish)\b/i, type: 'publish', verifyWith: ['registry_check', 'version_match'] },
+  { pattern: /\b(?:no\s+(?:errors?|failures?|issues?))\b/i, type: 'clean_state', verifyWith: ['log_check', 'status_check'] },
+];
+
+/**
+ * Decompose agent output into verifiable sub-claims.
+ * Returns array of { claim, type, verifyWith, text }.
+ */
+function decomposeClaims(agentOutput) {
+  const text = String(agentOutput || '');
+  if (!text.trim()) return [];
+
+  const claims = [];
+  for (const cp of CLAIM_PATTERNS) {
+    cp.pattern.lastIndex = 0;
+    const matches = text.match(cp.pattern);
+    if (matches) {
+      for (const match of matches) {
+        // Extract surrounding sentence for context
+        const idx = text.indexOf(match);
+        const start = Math.max(0, text.lastIndexOf('.', idx) + 1);
+        const end = text.indexOf('.', idx + match.length);
+        const sentence = text.slice(start, end > idx ? end + 1 : undefined).trim().slice(0, 200);
+
+        claims.push({
+          claim: match,
+          type: cp.type,
+          verifyWith: cp.verifyWith,
+          context: sentence,
+        });
+      }
+    }
+  }
+
+  return claims;
+}
+
+/**
+ * Check a decomposed claim against available evidence.
+ * Evidence is a map of { evidence_type: boolean_or_string }.
+ */
+function verifyClaim(claim, evidence) {
+  const missing = [];
+  const verified = [];
+
+  for (const req of claim.verifyWith) {
+    if (evidence[req] === true || (typeof evidence[req] === 'string' && evidence[req].length > 0)) {
+      verified.push(req);
+    } else {
+      missing.push(req);
+    }
+  }
+
+  const isVerified = missing.length === 0;
+  return {
+    claim: claim.claim,
+    type: claim.type,
+    verified: isVerified,
+    verifiedEvidence: verified,
+    missingEvidence: missing,
+    confidence: claim.verifyWith.length > 0 ? Math.round((verified.length / claim.verifyWith.length) * 100) : 0,
+    verdict: isVerified ? 'grounded' : missing.length === claim.verifyWith.length ? 'hallucination' : 'partial',
+  };
+}
+
+// ---------------------------------------------------------------------------
+// 2. Confidence-Weighted Gate Decisions
+// ---------------------------------------------------------------------------
+
+const CONFIDENCE_THRESHOLDS = {
+  none: { action: 'block', minSamples: 0, maxSamples: 0 },
+  low: { action: 'block', minSamples: 1, maxSamples: 4 },
+  medium: { action: 'warn', minSamples: 5, maxSamples: 19 },
+  high: { action: 'allow', minSamples: 20, maxSamples: Infinity },
+};
+
+/**
+ * Determine gate action based on Thompson Sampling confidence tier.
+ * Low confidence = stricter (block), high confidence = lenient (allow).
+ */
+function confidenceWeightedDecision({ confidence, reliability, samples }) {
+  let tier = 'none';
+  const s = samples || 0;
+
+  if (s === 0) tier = 'none';
+  else if (s <= 4) tier = 'low';
+  else if (s <= 19) tier = 'medium';
+  else tier = 'high';
+
+  const threshold = CONFIDENCE_THRESHOLDS[tier];
+  const rel = typeof reliability === 'number' ? reliability : 0.5;
+
+  // Override: even high-confidence, if reliability < 0.3 → block
+  let action = threshold.action;
+  if (rel < 0.3) action = 'block';
+  else if (rel < 0.5 && tier === 'high') action = 'warn';
+
+  return {
+    tier,
+    action,
+    reliability: Math.round(rel * 1000) / 1000,
+    samples: s,
+    reasoning: `${tier} confidence (${s} samples, ${Math.round(rel * 100)}% reliability) → ${action}`,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// 3. Retrieval-Grounded Verification
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if a proposed action contradicts recalled prevention rules.
+ * Retrieves relevant context and scans for contradictions.
+ *
+ * Returns { grounded, contradictions, relevantRules, groundingScore }.
+ */
+function retrievalGroundedCheck(proposedAction, { maxItems = 5, maxChars = 3000 } = {}) {
+  const actionText = String(proposedAction || '').toLowerCase();
+  if (!actionText.trim()) return { grounded: true, contradictions: [], relevantRules: [], groundingScore: 100 };
+
+  // Retrieve relevant context
+  let pack;
+  try {
+    pack = constructContextPack({ query: proposedAction, maxItems, maxChars, namespaces: ['rules', 'memoryError'] });
+  } catch {
+    return { grounded: true, contradictions: [], relevantRules: [], groundingScore: 100 };
+  }
+
+  const contradictions = [];
+  const relevantRules = [];
+
+  for (const item of pack.items) {
+    const content = ((item.structuredContext && item.structuredContext.rawContent) || '').toLowerCase();
+    const title = (item.title || '').toLowerCase();
+
+    // Check for NEVER/ALWAYS rules that contradict the action
+    const neverMatches = content.match(/never\s+(.{10,80})/gi) || [];
+    for (const neverRule of neverMatches) {
+      const ruleAction = neverRule.replace(/^never\s+/i, '').trim();
+      // Check if the proposed action contains what the rule says never to do
+      const ruleTokens = ruleAction.split(/\s+/).filter((t) => t.length > 3);
+      const matchCount = ruleTokens.filter((t) => actionText.includes(t)).length;
+      if (matchCount >= 2) {
+        contradictions.push({
+          rule: neverRule.trim(),
+          source: item.title,
+          matchStrength: Math.round((matchCount / ruleTokens.length) * 100),
+        });
+      }
+    }
+
+    // Track all relevant rules
+    if (item.score > 0) {
+      relevantRules.push({ title: item.title, score: item.score, namespace: item.namespace });
+    }
+  }
+
+  const groundingScore = contradictions.length === 0 ? 100 : Math.max(0, 100 - contradictions.length * 25);
+
+  return {
+    grounded: contradictions.length === 0,
+    contradictions,
+    relevantRules,
+    groundingScore,
+    packItemCount: pack.items.length,
+  };
+}
+
+/**
+ * Full hallucination check: decompose claims + verify + ground against rules.
+ * Returns comprehensive report.
+ */
+function fullHallucinationCheck(agentOutput, evidence = {}) {
+  const claims = decomposeClaims(agentOutput);
+  const claimResults = claims.map((c) => verifyClaim(c, evidence));
+  const grounding = retrievalGroundedCheck(agentOutput);
+
+  const verifiedCount = claimResults.filter((r) => r.verified).length;
+  const hallucinationCount = claimResults.filter((r) => r.verdict === 'hallucination').length;
+  const totalClaims = claimResults.length;
+
+  return {
+    claims: claimResults,
+    grounding,
+    summary: {
+      totalClaims,
+      verified: verifiedCount,
+      hallucinated: hallucinationCount,
+      partial: totalClaims - verifiedCount - hallucinationCount,
+      claimPassRate: totalClaims > 0 ? Math.round((verifiedCount / totalClaims) * 1000) / 10 : 100,
+      groundingScore: grounding.groundingScore,
+      overallVerdict: hallucinationCount > 0 ? 'hallucination_detected' : (grounding.grounded ? 'grounded' : 'contradiction_detected'),
+    },
+    checkedAt: new Date().toISOString(),
+  };
+}
+
+module.exports = {
+  CLAIM_PATTERNS, CONFIDENCE_THRESHOLDS,
+  decomposeClaims, verifyClaim,
+  confidenceWeightedDecision,
+  retrievalGroundedCheck, fullHallucinationCheck,
+};

package/scripts/hf-papers.js

@@ -0,0 +1,317 @@
+'use strict';
+
+const { URL, URLSearchParams } = require('node:url');
+const {
+  NAMESPACES,
+  upsertContextObject,
+  recordProvenance,
+  constructTemplatedPack,
+} = require('./contextfs');
+
+const DEFAULT_HF_PAPERS_API_BASE = process.env.HF_PAPERS_API_BASE || 'https://huggingface.co/api';
+const DEFAULT_LIMIT = 5;
+
+function normalizeAuthors(authors) {
+  if (!Array.isArray(authors)) return [];
+  return authors
+    .map((author) => {
+      if (typeof author === 'string') return author.trim();
+      if (author && typeof author.name === 'string') return author.name.trim();
+      return '';
+    })
+    .filter(Boolean);
+}
+
+function normalizeTags(tags) {
+  if (!Array.isArray(tags)) return [];
+  return [...new Set(tags
+    .map((tag) => {
+      if (typeof tag === 'string') return tag.trim();
+      if (tag && typeof tag.label === 'string') return tag.label.trim();
+      if (tag && typeof tag.name === 'string') return tag.name.trim();
+      return '';
+    })
+    .filter(Boolean))];
+}
+
+function normalizePaper(record = {}) {
+  const paper = record && typeof record.paper === 'object' ? record.paper : record;
+  const paperId = String(
+    paper.id
+      || paper.paper_id
+      || paper.paperId
+      || paper.arxiv_id
+      || paper.arxivId
+      || record.id
+      || record.paper_id
+      || record.paperId
+      || record.arxiv_id
+      || record.arxivId
+      || ''
+  ).trim();
+  const title = String(
+    paper.title
+      || record.title
+      || (paperId ? `Paper ${paperId}` : 'Untitled paper')
+  ).trim();
+  const summary = String(
+    paper.summary
+      || paper.abstract
+      || record.summary
+      || record.abstract
+      || ''
+  ).trim();
+  const url = String(
+    paper.url
+      || paper.paper_url
+      || record.url
+      || record.paper_url
+      || (paperId ? `https://arxiv.org/abs/${paperId}` : '')
+  ).trim();
+
+  return {
+    paperId,
+    title,
+    summary,
+    url: url || null,
+    authors: normalizeAuthors(paper.authors || record.authors),
+    tags: normalizeTags(paper.tags || paper.categories || record.tags || record.categories),
+    publishedAt: paper.publishedAt || paper.published_at || record.publishedAt || record.published_at || null,
+    source: 'huggingface-papers',
+  };
+}
+
+function extractPaperItems(payload) {
+  if (Array.isArray(payload)) return payload;
+  if (!payload || typeof payload !== 'object') return [];
+  if (Array.isArray(payload.papers)) return payload.papers;
+  if (Array.isArray(payload.items)) return payload.items;
+  if (Array.isArray(payload.results)) return payload.results;
+  if (Array.isArray(payload.dailyPapers)) return payload.dailyPapers;
+  if (payload.paper && typeof payload.paper === 'object') return [payload.paper];
+  return [];
+}
+
+function buildSearchUrls({ query, limit = DEFAULT_LIMIT, baseUrl = DEFAULT_HF_PAPERS_API_BASE }) {
+  const normalizedBase = String(baseUrl || DEFAULT_HF_PAPERS_API_BASE).replace(/\/+$/, '');
+  const routes = [
+    ['/daily_papers', { query, limit: String(limit) }],
+    ['/papers/search', { q: query, limit: String(limit) }],
+    ['/papers', { query, limit: String(limit) }],
+  ];
+
+  return routes.map(([pathname, params]) => {
+    const url = new URL(`${normalizedBase}${pathname}`);
+    url.search = new URLSearchParams(params).toString();
+    return url.toString();
+  });
+}
+
+async function readJson(url, fetchImpl = global.fetch) {
+  if (typeof fetchImpl !== 'function') {
+    throw new Error('A fetch implementation is required');
+  }
+
+  const response = await fetchImpl(url, {
+    headers: {
+      accept: 'application/json',
+    },
+  });
+
+  if (!response.ok) {
+    const error = new Error(`HF papers request failed: ${response.status} ${response.statusText}`);
+    error.status = response.status;
+    throw error;
+  }
+
+  return response.json();
+}
+
+async function searchPapers({
+  query,
+  limit = DEFAULT_LIMIT,
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+} = {}) {
+  const normalizedQuery = String(query || '').trim();
+  if (!normalizedQuery) {
+    throw new Error('searchPapers requires query');
+  }
+
+  const urls = buildSearchUrls({
+    query: normalizedQuery,
+    limit: Math.max(1, Number(limit) || DEFAULT_LIMIT),
+    baseUrl,
+  });
+
+  let lastError = null;
+  for (const url of urls) {
+    try {
+      const payload = await readJson(url, fetchImpl);
+      const papers = extractPaperItems(payload)
+        .map(normalizePaper)
+        .filter((paper) => paper.paperId || paper.title);
+
+      if (papers.length > 0) {
+        return papers.slice(0, limit);
+      }
+    } catch (error) {
+      lastError = error;
+      if (error && error.status === 404) {
+        continue;
+      }
+    }
+  }
+
+  if (lastError) throw lastError;
+  return [];
+}
+
+function paperToMarkdown(paper) {
+  const normalized = normalizePaper(paper);
+  const lines = [
+    `# ${normalized.title}`,
+    '',
+    `Paper ID: ${normalized.paperId || 'unknown'}`,
+    `Source: ${normalized.source}`,
+  ];
+
+  if (normalized.url) {
+    lines.push(`URL: ${normalized.url}`);
+  }
+  if (normalized.publishedAt) {
+    lines.push(`Published: ${normalized.publishedAt}`);
+  }
+  if (normalized.authors.length > 0) {
+    lines.push(`Authors: ${normalized.authors.join(', ')}`);
+  }
+  if (normalized.tags.length > 0) {
+    lines.push(`Tags: ${normalized.tags.join(', ')}`);
+  }
+
+  lines.push('', '## Abstract', '', normalized.summary || 'No abstract available.', '');
+  return lines.join('\n');
+}
+
+function buildCitation(paper) {
+  return {
+    paperId: paper.paperId || null,
+    title: paper.title,
+    url: paper.url,
+  };
+}
+
+function ingestNormalizedPapers(papers, query) {
+  const ingested = papers.map((paper) => {
+    const normalizedTags = [...new Set([
+      'research',
+      'paper',
+      'hf-papers',
+      ...paper.tags.map((tag) => String(tag)),
+    ])].sort();
+
+    return upsertContextObject({
+      namespace: NAMESPACES.research,
+      title: `Paper: ${paper.title}`,
+      content: paperToMarkdown(paper),
+      tags: normalizedTags,
+      source: 'hf-papers',
+      metadata: {
+        provider: 'huggingface',
+        paperId: paper.paperId || null,
+        url: paper.url,
+        authors: paper.authors,
+        publishedAt: paper.publishedAt,
+        query,
+      },
+    });
+  });
+
+  recordProvenance({
+    type: 'hf_papers_ingested',
+    query,
+    count: ingested.length,
+    dedupedCount: ingested.filter((entry) => entry.deduped).length,
+    paperIds: papers.map((paper) => paper.paperId).filter(Boolean),
+  });
+
+  return ingested;
+}
+
+async function ingestPaperSearch({
+  query,
+  limit = DEFAULT_LIMIT,
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+  searchPapersImpl = searchPapers,
+} = {}) {
+  const papers = await searchPapersImpl({
+    query,
+    limit,
+    baseUrl,
+    fetchImpl,
+  });
+  const ingested = ingestNormalizedPapers(papers, query);
+
+  return {
+    query,
+    limit,
+    papers,
+    ingested,
+  };
+}
+
+async function buildResearchBrief({
+  query,
+  limit = DEFAULT_LIMIT,
+  template = 'research-brief',
+  baseUrl = DEFAULT_HF_PAPERS_API_BASE,
+  fetchImpl = global.fetch,
+  searchPapersImpl = searchPapers,
+} = {}) {
+  const result = await ingestPaperSearch({
+    query,
+    limit,
+    baseUrl,
+    fetchImpl,
+    searchPapersImpl,
+  });
+  const pack = constructTemplatedPack({ template, query });
+  const citations = result.papers.map(buildCitation);
+  const brief = pack.items
+    .map((item, index) => {
+      const digest = String(item.structuredContext && item.structuredContext.rawContent || '')
+        .split('\n')
+        .slice(0, 6)
+        .join(' ')
+        .trim();
+      return `${index + 1}. ${item.title} ${digest}`.trim();
+    })
+    .join('\n');
+
+  return {
+    query,
+    limit,
+    source: 'huggingface-papers',
+    template,
+    ingestedCount: result.ingested.length,
+    packId: pack.packId,
+    citations,
+    brief,
+    pack,
+  };
+}
+
+module.exports = {
+  DEFAULT_HF_PAPERS_API_BASE,
+  buildResearchBrief,
+  buildSearchUrls,
+  extractPaperItems,
+  ingestNormalizedPapers,
+  ingestPaperSearch,
+  normalizeAuthors,
+  normalizePaper,
+  normalizeTags,
+  paperToMarkdown,
+  searchPapers,
+};