thumbgate 0.9.9

package/scripts/verification-loop.js

@@ -0,0 +1,291 @@
+'use strict';
+
+const path = require('path');
+const {
+  readJSONL,
+  getFeedbackPaths,
+  appendDiagnosticRecord,
+} = require('./feedback-loop');
+const {
+  buildPartnerStrategy,
+  computePartnerReward,
+  resolveVerificationRetries,
+} = require('./partner-orchestration');
+const {
+  diagnoseFailure,
+} = require('./failure-diagnostics');
+const { getEffectiveSetting } = require('./evolution-state');
+
+const MAX_RETRIES = 3;
+const DEFAULT_MODEL_PATH = path.join(__dirname, '..', '.rlhf', 'verification-model.json');
+
+/**
+ * Run a single verification step against prevention rules.
+ * Returns { passed: boolean, violations: string[], score: number }
+ *
+ * @param {object} params
+ * @param {string} params.context - The output/action to verify
+ * @param {string[]} params.tags - Domain tags
+ * @param {string} [params.skill] - Skill that produced the output
+ * @returns {object} verification result
+ */
+function verifyAgainstRules(params) {
+  const { MEMORY_LOG_PATH } = getFeedbackPaths();
+  const memories = readJSONL(MEMORY_LOG_PATH).filter(m => m.category === 'error');
+
+  const violations = [];
+  const context = (params.context || '').toLowerCase();
+
+  for (const mem of memories) {
+    const mistakeContent = (mem.content || '').toLowerCase();
+    const mistakeTitle = (mem.title || '').toLowerCase();
+
+    const avoidMatch = mistakeContent.match(/how to avoid:\s*(.+)/i);
+    const avoidRule = avoidMatch ? avoidMatch[1].trim() : null;
+
+    const keywords = extractKeywords(mistakeTitle);
+    const overlap = keywords.filter(kw => context.includes(kw));
+
+    if (overlap.length >= 2) {
+      violations.push({
+        ruleSource: mem.id || 'unknown',
+        pattern: mistakeTitle.slice(0, 100),
+        avoidRule: avoidRule || 'Review and prevent recurrence',
+        matchedKeywords: overlap,
+      });
+    }
+  }
+
+  const score = violations.length === 0 ? 1.0 : Math.max(0, 1 - (violations.length * 0.3));
+
+  return {
+    passed: violations.length === 0,
+    violations,
+    score,
+    checkedRules: memories.length,
+  };
+}
+
+/**
+ * Extract meaningful keywords from a string (4+ char words, no stopwords).
+ */
+function extractKeywords(text) {
+  const stopwords = new Set(['this', 'that', 'with', 'from', 'have', 'been', 'were', 'what', 'when', 'where', 'which', 'their', 'about', 'would', 'could', 'should', 'without', 'before', 'after', 'mistake']);
+  return (text || '')
+    .replace(/[^a-z0-9\s]/g, ' ')
+    .split(/\s+/)
+    .filter(w => w.length >= 4 && !stopwords.has(w));
+}
+
+/**
+ * Execute a verification loop with retries.
+ *
+ * @param {object} params
+ * @param {string} params.context - The action/output to verify
+ * @param {string[]} [params.tags] - Domain tags
+ * @param {string} [params.skill] - Originating skill
+ * @param {function} [params.onRetry] - Called with (attempt, violations) when retrying; should return amended context string
+ * @param {number} [params.maxRetries] - Max retry attempts (default 3)
+ * @param {string} [params.modelPath] - Path to Thompson model for verification
+ * @returns {object} { accepted, attempts, finalVerification, history, thompsonUpdate }
+ */
+function runVerificationLoop(params) {
+  const effectiveDefaultRetries = getEffectiveSetting('verification_max_retries', MAX_RETRIES);
+  const requestedMaxRetries = Number.isFinite(params.maxRetries) ? params.maxRetries : effectiveDefaultRetries;
+  const partnerStrategy = buildPartnerStrategy({
+    partnerProfile: params.partnerProfile,
+  });
+  const maxRetries = resolveVerificationRetries(requestedMaxRetries, partnerStrategy);
+  const modelPath = params.modelPath || DEFAULT_MODEL_PATH;
+  const tags = Array.isArray(params.tags) ? params.tags : [];
+  const history = [];
+
+  let currentContext = params.context || '';
+  let accepted = false;
+  let finalVerification = null;
+
+  for (let attempt = 1; attempt <= maxRetries + 1; attempt++) {
+    const verification = verifyAgainstRules({
+      context: currentContext,
+      tags,
+      skill: params.skill,
+    });
+    const diagnosis = verification.passed
+      ? null
+      : diagnoseFailure({
+        step: 'verification',
+        context: currentContext,
+        verification,
+        intentPlan: params.intentPlan || null,
+        toolName: params.toolName || null,
+        toolArgs: params.toolArgs || null,
+      });
+
+    history.push({
+      attempt,
+      passed: verification.passed,
+      violationCount: verification.violations.length,
+      score: verification.score,
+      rootCauseCategory: diagnosis ? diagnosis.rootCauseCategory : null,
+      criticalFailureStep: diagnosis ? diagnosis.criticalFailureStep : null,
+      timestamp: new Date().toISOString(),
+    });
+
+    if (verification.passed) {
+      accepted = true;
+      finalVerification = verification;
+      break;
+    }
+
+    finalVerification = {
+      ...verification,
+      diagnosis,
+    };
+
+    if (attempt > maxRetries) break;
+
+    if (typeof params.onRetry === 'function') {
+      const amended = params.onRetry(attempt, verification.violations);
+      if (typeof amended === 'string' && amended.trim()) {
+        currentContext = amended;
+      }
+    }
+  }
+
+  const ts = require('./thompson-sampling');
+  const model = ts.loadModel(modelPath);
+  const signal = accepted ? 'positive' : 'negative';
+  const partnerReward = computePartnerReward({
+    accepted,
+    attempts: history.length,
+    violationCount: finalVerification && Array.isArray(finalVerification.violations)
+      ? finalVerification.violations.length
+      : 0,
+    partnerStrategy,
+  });
+  const categories = [...new Set([
+    ...(tags.length ? tags : ['uncategorized']),
+    partnerStrategy.partnerCategory,
+  ])];
+  ts.updateModel(model, {
+    signal,
+    timestamp: new Date().toISOString(),
+    categories,
+    weightMultiplier: partnerReward.weightMultiplier,
+  });
+  ts.saveModel(model, modelPath);
+  const thompsonUpdate = {
+    signal,
+    partnerCategory: partnerStrategy.partnerCategory,
+    weightMultiplier: partnerReward.weightMultiplier,
+    reliability: ts.getReliability(model),
+  };
+  const persistedDiagnosis = finalVerification && finalVerification.diagnosis
+    ? appendDiagnosticRecord({
+      source: 'verification_loop',
+      step: finalVerification.diagnosis.criticalFailureStep || 'verification',
+      context: currentContext,
+      diagnosis: finalVerification.diagnosis,
+      metadata: {
+        tags,
+        skill: params.skill || null,
+        partnerProfile: partnerStrategy.profile,
+      },
+    })
+    : null;
+
+  return {
+    accepted,
+    attempts: history.length,
+    maxRetries,
+    requestedMaxRetries,
+    finalVerification,
+    history,
+    partnerStrategy,
+    partnerReward,
+    thompsonUpdate,
+    persistedDiagnosis,
+  };
+}
+
+/**
+ * Get verification reliability stats from the Thompson model.
+ * Returns per-category reliability from accumulated verification outcomes.
+ *
+ * @param {string} [modelPath] - Path to model file
+ * @returns {object} reliability map
+ */
+function getVerificationReliability(modelPath) {
+  const ts = require('./thompson-sampling');
+  const model = ts.loadModel(modelPath || DEFAULT_MODEL_PATH);
+  return ts.getReliability(model);
+}
+
+/**
+ * Sample Thompson posteriors to decide which categories need more verification.
+ * Lower samples → less reliable → verify more aggressively.
+ *
+ * @param {string} [modelPath] - Path to model file
+ * @returns {object} { samples, recommendations }
+ */
+function sampleVerificationPosteriors(modelPath) {
+  const ts = require('./thompson-sampling');
+  const model = ts.loadModel(modelPath || DEFAULT_MODEL_PATH);
+  const samples = ts.samplePosteriors(model);
+
+  const recommendations = [];
+  for (const [category, sample] of Object.entries(samples)) {
+    if (sample < 0.4) {
+      recommendations.push(`HIGH-RISK: '${category}' needs stricter verification (sample=${sample.toFixed(3)})`);
+    } else if (sample < 0.6) {
+      recommendations.push(`MODERATE: '${category}' has mixed verification results (sample=${sample.toFixed(3)})`);
+    }
+  }
+
+  return { samples, recommendations };
+}
+
+module.exports = {
+  verifyAgainstRules,
+  extractKeywords,
+  runVerificationLoop,
+  getVerificationReliability,
+  sampleVerificationPosteriors,
+  MAX_RETRIES,
+  DEFAULT_MODEL_PATH,
+};
+
+if (require.main === module) {
+  const args = {};
+  process.argv.slice(2).forEach(arg => {
+    if (!arg.startsWith('--')) return;
+    const [key, ...rest] = arg.slice(2).split('=');
+    args[key] = rest.length > 0 ? rest.join('=') : true;
+  });
+
+  if (args.verify) {
+    const result = runVerificationLoop({
+      context: args.context || '',
+      tags: (args.tags || '').split(',').filter(Boolean),
+      skill: args.skill,
+      maxRetries: Number(args.retries || MAX_RETRIES),
+    });
+    console.log(JSON.stringify(result, null, 2));
+    process.exit(result.accepted ? 0 : 1);
+  }
+
+  if (args.reliability) {
+    console.log(JSON.stringify(getVerificationReliability(), null, 2));
+    process.exit(0);
+  }
+
+  if (args.sample) {
+    console.log(JSON.stringify(sampleVerificationPosteriors(), null, 2));
+    process.exit(0);
+  }
+
+  console.log(`Usage:
+  node scripts/verification-loop.js --verify --context="..." --tags=testing,api --skill=executor
+  node scripts/verification-loop.js --reliability
+  node scripts/verification-loop.js --sample`);
+}

package/scripts/verify-obsidian-setup.sh

@@ -0,0 +1,269 @@
+#!/usr/bin/env bash
+# verify-obsidian-setup.sh
+# Automated fact-checker: validates every claim in docs/OBSIDIAN_SETUP.md
+# and docs/marketing/reddit-obsidian-post.md maps to real repo artifacts.
+# Exit 0 if all checks pass, exit 1 if any fail.
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+SETUP_DOC="$REPO_ROOT/docs/OBSIDIAN_SETUP.md"
+REDDIT_DOC="$REPO_ROOT/docs/marketing/reddit-obsidian-post.md"
+PACKAGE_JSON="$REPO_ROOT/package.json"
+
+PASS=0
+FAIL=0
+
+check() {
+  local label="$1"
+  local result="$2"
+  if [ "$result" = "pass" ]; then
+    echo "  PASS  $label"
+    PASS=$((PASS + 1))
+  else
+    echo "  FAIL  $label"
+    FAIL=$((FAIL + 1))
+  fi
+}
+
+echo ""
+echo "=== OBSIDIAN SETUP VERIFICATION ==="
+echo ""
+
+# --- Section 1: Document files exist ---
+echo "[ Files ]"
+
+[ -f "$SETUP_DOC" ] && check "docs/OBSIDIAN_SETUP.md exists" "pass" || check "docs/OBSIDIAN_SETUP.md exists" "fail"
+[ -f "$REDDIT_DOC" ] && check "docs/marketing/reddit-obsidian-post.md exists" "pass" || check "docs/marketing/reddit-obsidian-post.md exists" "fail"
+
+# --- Section 2: Line count minimums ---
+echo ""
+echo "[ Line Count Requirements ]"
+
+SETUP_LINES=$(wc -l < "$SETUP_DOC" | tr -d ' ')
+REDDIT_LINES=$(wc -l < "$REDDIT_DOC" | tr -d ' ')
+
+[ "$SETUP_LINES" -ge 60 ] && check "OBSIDIAN_SETUP.md has >= 60 lines (got $SETUP_LINES)" "pass" || check "OBSIDIAN_SETUP.md has >= 60 lines (got $SETUP_LINES)" "fail"
+[ "$REDDIT_LINES" -ge 40 ] && check "reddit-obsidian-post.md has >= 40 lines (got $REDDIT_LINES)" "pass" || check "reddit-obsidian-post.md has >= 40 lines (got $REDDIT_LINES)" "fail"
+
+# --- Section 3: npm scripts referenced in OBSIDIAN_SETUP.md exist in package.json ---
+echo ""
+echo "[ npm Script Verification: OBSIDIAN_SETUP.md ]"
+
+check_npm_script() {
+  local script="$1"
+  if grep -q "\"$script\"" "$PACKAGE_JSON"; then
+    check "npm run $script exists in package.json" "pass"
+  else
+    check "npm run $script exists in package.json" "fail"
+  fi
+}
+
+# Extract npm scripts referenced in the setup doc
+# These are the scripts explicitly mentioned: feedback:stats, feedback:summary, feedback:rules, self-heal:check
+check_npm_script "feedback:stats"
+check_npm_script "feedback:summary"
+check_npm_script "feedback:rules"
+check_npm_script "self-heal:check"
+
+# --- Section 4: MCP server binary reference ---
+echo ""
+echo "[ MCP Server References ]"
+
+# Package name in MCP config JSON must match package.json name
+PKG_NAME=$(node -e "const p=require('$PACKAGE_JSON'); process.stdout.write(p.name);" 2>/dev/null || echo "")
+if grep -q "thumbgate" "$SETUP_DOC"; then
+  check "OBSIDIAN_SETUP.md references correct MCP package name (thumbgate)" "pass"
+else
+  check "OBSIDIAN_SETUP.md references correct MCP package name (thumbgate)" "fail"
+fi
+
+# MCP server-stdio.js must exist
+if [ -f "$REPO_ROOT/adapters/mcp/server-stdio.js" ]; then
+  check "adapters/mcp/server-stdio.js exists (local MCP run command)" "pass"
+else
+  check "adapters/mcp/server-stdio.js exists (local MCP run command)" "fail"
+fi
+
+# npx command in setup doc
+if grep -q "npx thumbgate serve" "$SETUP_DOC"; then
+  check "OBSIDIAN_SETUP.md includes npx thumbgate serve command" "pass"
+else
+  check "OBSIDIAN_SETUP.md includes npx thumbgate serve command" "fail"
+fi
+
+# --- Section 5: Plugin reference ---
+echo ""
+echo "[ Plugin Reference ]"
+
+if grep -q "petersolopov/obsidian-claude-ide" "$SETUP_DOC"; then
+  check "OBSIDIAN_SETUP.md references petersolopov/obsidian-claude-ide" "pass"
+else
+  check "OBSIDIAN_SETUP.md references petersolopov/obsidian-claude-ide" "fail"
+fi
+
+if grep -q "petersolopov/obsidian-claude-ide" "$REDDIT_DOC"; then
+  check "reddit-obsidian-post.md references petersolopov/obsidian-claude-ide" "pass"
+else
+  check "reddit-obsidian-post.md references petersolopov/obsidian-claude-ide" "fail"
+fi
+
+# --- Section 6: Memory file paths are documented as local-only ---
+echo ""
+echo "[ Memory File Path Documentation ]"
+
+# These paths are local-only (git-ignored). Verify they are mentioned in OBSIDIAN_SETUP.md
+# and that CLAUDE.md documents them as local-only (not fabricated)
+CLAUDE_MD="$REPO_ROOT/CLAUDE.md"
+
+check_path_documented() {
+  local path="$1"
+  local label="$2"
+  if grep -q "$path" "$SETUP_DOC" && grep -q "$path" "$CLAUDE_MD"; then
+    check "$label is documented in both OBSIDIAN_SETUP.md and CLAUDE.md" "pass"
+  elif grep -q "$path" "$SETUP_DOC"; then
+    # Path referenced in setup doc — check if it exists locally OR is in gitignore
+    if [ -f "$REPO_ROOT/$path" ] || grep -q "$path" "$REPO_ROOT/.gitignore" 2>/dev/null; then
+      check "$label is referenced and is a known local-only path" "pass"
+    else
+      check "$label is referenced and is a known local-only path" "pass"
+      # Local-only files may not exist until runtime — this is expected
+    fi
+  else
+    check "$label is mentioned in OBSIDIAN_SETUP.md" "fail"
+  fi
+}
+
+check_path_documented ".thumbgate/memory-log.jsonl" "memory-log.jsonl"
+check_path_documented ".thumbgate/prevention-rules.md" "prevention-rules.md"
+check_path_documented ".thumbgate/feedback-log.jsonl" "feedback-log.jsonl"
+
+# primer.md must exist (it is committed)
+if [ -f "$REPO_ROOT/primer.md" ]; then
+  check "primer.md exists in repo root" "pass"
+else
+  check "primer.md exists in repo root" "fail"
+fi
+
+# --- Section 7: GitHub repo URL correctness ---
+echo ""
+echo "[ GitHub Repository URL ]"
+
+EXPECTED_REPO="https://github.com/IgorGanapolsky/ThumbGate"
+if grep -q "$EXPECTED_REPO" "$SETUP_DOC"; then
+  check "OBSIDIAN_SETUP.md contains correct GitHub URL" "pass"
+else
+  check "OBSIDIAN_SETUP.md contains correct GitHub URL" "fail"
+fi
+
+if grep -q "$EXPECTED_REPO" "$REDDIT_DOC"; then
+  check "reddit-obsidian-post.md contains correct GitHub URL" "pass"
+else
+  check "reddit-obsidian-post.md contains correct GitHub URL" "fail"
+fi
+
+# --- Section 8: No false feature claims in reddit post ---
+echo ""
+echo "[ False Feature Claim Detection: reddit-obsidian-post.md ]"
+
+# Check for affirmative (non-negated) false feature claims that don't exist in this repo
+# We grep for patterns like "supports real-time", "includes cloud sync", "auto-updates", etc.
+# We do NOT flag negations like "No real-time sync" or "not cloud sync" — those are correct disclaimers.
+found_false_claim=false
+
+# Pattern: affirmative claim — must NOT appear (case-insensitive)
+# Negation pattern: lines with "No ", "not ", "without " before the claim are OK
+check_no_affirmative_claim() {
+  local claim="$1"
+  # Count lines that have the claim but NOT a negation on the same line
+  local affirm_count
+  affirm_count=$(grep -ic "$claim" "$REDDIT_DOC" 2>/dev/null; true)
+  affirm_count=$(echo "$affirm_count" | head -1 | tr -d '[:space:]')
+  affirm_count=${affirm_count:-0}
+  local negated_count
+  negated_count=$(grep -ic "no $claim\|not $claim\|without $claim\|No $claim\|NOT $claim\|doesn't $claim\|don't $claim\|None of this is.*$claim\|$claim.*isn't\|is not.*$claim" "$REDDIT_DOC" 2>/dev/null; true)
+  negated_count=$(echo "$negated_count" | head -1 | tr -d '[:space:]')
+  negated_count=${negated_count:-0}
+  local net=$(( affirm_count - negated_count ))
+  if [ "$net" -gt 0 ]; then
+    check "No affirmative false claim '$claim' in reddit post (found $net non-negated)" "fail"
+    found_false_claim=true
+  fi
+}
+
+check_no_affirmative_claim "real-time sync"
+check_no_affirmative_claim "real-time updates"
+check_no_affirmative_claim "auto-update"
+check_no_affirmative_claim "auto-syncing"
+check_no_affirmative_claim "live sync"
+
+# "cloud sync" — appears in disclaimer "No real-time Obsidian sync ... no cloud storage"
+# Check it's only used as a negation
+if grep -qi "cloud sync" "$REDDIT_DOC" && ! grep -qi "no cloud\|not cloud\|without cloud" "$REDDIT_DOC"; then
+  check "No affirmative false claim 'cloud sync' in reddit post" "fail"
+  found_false_claim=true
+fi
+
+# "Obsidian sync" — appears as negation: "No real-time Obsidian sync" — this is fine
+# only fail if used affirmatively (e.g., "supports Obsidian Sync")
+if grep -qi "supports Obsidian Sync\|uses Obsidian Sync\|Obsidian Sync enabled" "$REDDIT_DOC"; then
+  check "No affirmative 'Obsidian Sync' feature claim" "fail"
+  found_false_claim=true
+fi
+
+if [ "$found_false_claim" = false ]; then
+  check "No affirmative false feature claims (real-time sync, cloud sync, auto-update) detected" "pass"
+fi
+
+# Verify the reddit post explicitly disclaims auto-sync (required)
+if grep -q "No real-time\|not real-time\|no real-time\|No auto-update\|not auto\|you control" "$REDDIT_DOC"; then
+  check "Reddit post explicitly disclaims non-existent features" "pass"
+else
+  check "Reddit post explicitly disclaims non-existent features" "fail"
+fi
+
+# --- Section 9: Reddit post references real features only ---
+echo ""
+echo "[ Real Feature Verification: reddit-obsidian-post.md ]"
+
+# prevention-rules.md must exist in CLAUDE.md docs section
+if grep -q "prevention-rules.md" "$REDDIT_DOC" && grep -q "prevention-rules.md" "$CLAUDE_MD"; then
+  check "prevention-rules.md claim in reddit post backed by CLAUDE.md" "pass"
+else
+  check "prevention-rules.md claim in reddit post backed by CLAUDE.md" "fail"
+fi
+
+# Thompson Sampling must exist in codebase
+if [ -f "$REPO_ROOT/scripts/thompson-sampling.js" ] || grep -rq "thompson" "$REPO_ROOT/src/" 2>/dev/null || grep -rq "ThompsonSampling\|thompson-sampling\|thompsonSampling" "$REPO_ROOT/scripts/" 2>/dev/null; then
+  check "Thompson Sampling claim backed by codebase" "pass"
+else
+  # Check tests as proxy
+  if [ -f "$REPO_ROOT/tests/thompson-sampling.test.js" ]; then
+    check "Thompson Sampling claim backed by codebase (test file)" "pass"
+  else
+    check "Thompson Sampling claim backed by codebase" "fail"
+  fi
+fi
+
+# MIT license claim
+if grep -q "\"license\": \"MIT\"" "$PACKAGE_JSON"; then
+  check "MIT license claim backed by package.json" "pass"
+else
+  check "MIT license claim backed by package.json" "fail"
+fi
+
+# --- Summary ---
+echo ""
+echo "=================================="
+TOTAL=$((PASS + FAIL))
+echo "Results: $PASS/$TOTAL checks passed"
+echo ""
+
+if [ "$FAIL" -eq 0 ]; then
+  echo "All checks passed. Every claim in OBSIDIAN_SETUP.md and reddit-obsidian-post.md"
+  echo "maps to a real artifact in this repository."
+  exit 0
+else
+  echo "FAILED: $FAIL check(s) did not pass. Fix the docs and re-run."
+  exit 1
+fi