truthguard-ai 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 TruthGuard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,363 @@
+# TruthGuard
+
+**Standardized grounding validation for tool-calling AI agents.**
+
+> Detect when an agent's response contradicts the data returned by the tools it called — deterministically, without LLM-as-judge overhead.
+
+[![npm version](https://img.shields.io/npm/v/truthguard.svg)](https://www.npmjs.com/package/truthguard)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+---
+
+## The Problem
+
+Most "hallucinations" in tool-calling agents are **grounding failures** — the agent calls a tool, gets accurate data, and then ignores it, miscalculates, or fabricates from empty results. The source of truth is already in the trace.
+
+## The Solution
+
+TruthGuard extracts factual claims from the agent's response, cross-references them against tool outputs, and reports grounding failures with standardized codes — like OBD diagnostic codes for AI.
+
+```
+npm install truthguard
+```
+
+**Zero LLM calls.** Deterministic regex extraction + fuzzy matching. 30 detection rules across 4 categories. Runs in <50ms.
+
+---
+
+## Quick Start — 3 Minutes
+
+### 1. Evaluate a trace
+
+```typescript
+import { TraceBuilder, GroundingEngine, generateReport } from 'truthguard';
+
+const trace = new TraceBuilder({ traceId: 'run-001' })
+  .addUserInput('How many employees are on leave today?')
+  .addToolCall('getLeaveRecords', { date: '2024-03-15' })
+  .addToolOutput('getLeaveRecords', [
+    { employeeId: 'E01', name: 'Ana Jovic', status: 'on_leave' },
+    { employeeId: 'E02', name: 'Ivan Petrovic', status: 'on_leave' },
+  ])
+  .addFinalResponse('There are 3 employees on leave today.')  // ← Bug: says 3, data shows 2
+  .build();
+
+const engine = new GroundingEngine();
+const report = engine.evaluate(trace);
+
+console.log(report.groundingScore);       // 0.5
+console.log(report.detectedFailures[0]);  // { type: 'grounding.data_ignored', severity: 'high' }
+
+const { text } = generateReport(report);
+console.log(text);
+```
+
+### 2. Add a CI quality gate
+
+```typescript
+import { loadDataset, runDataset, evaluateGate, loadGateConfig } from 'truthguard';
+
+const entries = loadDataset('./test-cases.jsonl');
+const result = runDataset(entries);
+const gate = loadGateConfig('.ai-rcp-gate.yml');
+const verdict = evaluateGate(result, gate);
+
+if (!verdict.pass) {
+  console.error(verdict.report);
+  process.exit(1);
+}
+```
+
+### 3. Monitor in production (proxy mode)
+
+Works with **any language** — PHP, Python, Go, Java, Ruby, C#:
+
+```bash
+npx truthguard observe --port 3001
+```
+
+Change your AI base URL:
+```php
+// Before: ANTHROPIC_BASE_URL=https://api.anthropic.com
+// After:
+ANTHROPIC_BASE_URL=http://localhost:3001/proxy/anthropic
+```
+
+Your app works exactly the same. TruthGuard transparently proxies requests and evaluates grounding in the background.
+
+---
+
+## Detection Rules (30)
+
+### Grounding (16 rules)
+
+| Code | Description |
+|------|-------------|
+| `empty_fabrication` | Tool returned `[]`, agent fabricated results |
+| `no_tool_call` | Factual question answered without calling any tool |
+| `math_error` | Incorrect calculation from correct tool data |
+| `data_ignored` | Tool data altered or ignored in response |
+| `wrong_query` | Tool called with incorrect parameters |
+| `entity_mismatch` | Agent mixed up entities from results |
+| `hallucinated_entity` | Agent invented entity not in tool data |
+| `partial_answer` | Only part of the question answered |
+| `question_not_answered` | Core question not addressed |
+| `selective_omission` | Some tool results selectively excluded |
+| `tool_error_ignored` | Tool error not handled |
+| `stale_knowledge` | Used outdated data instead of tool results |
+| `incomplete_response` | Empty or fallback response despite having data |
+| `irrelevant_context` | Used unrelated data from different context |
+| `contradictory_claims` | Response contains self-contradicting statements |
+| `unverified_value` | Factual values with no tool data to verify against |
+
+### Orchestration (8 rules)
+
+| Code | Description |
+|------|-------------|
+| `malformed_tool_input` | Bad parameter format in tool call |
+| `raw_output_leak` | XML/JSON markup leaked into response |
+| `intermediate_response_leak` | "Let me check..." text shown to user |
+| `excessive_tool_calls` | Redundant repeated tool invocations |
+| `token_limit_truncation` | Response cut off by token limit |
+| `rate_limit_degradation` | Quality degraded due to rate limiting |
+| `quota_exhaustion` | API quota exceeded |
+| `model_fallback` | Unexpected model fallback |
+
+### Reasoning (4) & Safety (2)
+
+`scope_mismatch`, `overconfident_language`, `language_mismatch`, `duplicate_user_input`, `prompt_leak`, `sensitive_data_exposure`
+
+---
+
+## Features
+
+### Diagnostic Advisor
+
+Every detected failure includes root cause analysis, evidence from the trace, and two remediation paths:
+
+```typescript
+import { generateAdvisorReport, formatAdvisorReport } from 'truthguard';
+
+const advisor = generateAdvisorReport(report, trace);
+console.log(formatAdvisorReport(advisor));
+// REPAIR ORDER:
+//   1. Fix grounding.no_tool_call (root cause)
+//   2. Fix grounding.unverified_value (likely resolves after #1)
+// PROMPT HINT: "Always call the relevant tool before answering factual questions"
+// CODE GUARD:  if (!trace.hasToolCall()) return forceToolCall(query);
+```
+
+### Policy Engine
+
+Configure per-failure actions — block, warn, or observe:
+
+```typescript
+import { wrapOpenAI, GroundingError } from 'truthguard';
+import OpenAI from 'openai';
+
+const openai = wrapOpenAI(new OpenAI(), {
+  mode: 'block',
+  threshold: 0.85,
+  policy: {
+    rules: {
+      'grounding.empty_fabrication': 'block',
+      'grounding.math_error': 'warn',
+      'reasoning.overconfident_language': 'observe',
+    },
+  },
+});
+```
+
+### Baseline Regression Detection
+
+```typescript
+import { createSnapshot, saveBaseline, loadBaseline, compareToBaseline } from 'truthguard';
+
+// Save after a known-good run
+const snapshot = createSnapshot(result, 'v1.2-main');
+saveBaseline('.ai-rcp-baseline.json', snapshot);
+
+// Compare after changes
+const comparison = compareToBaseline(newResult, snapshot);
+if (!comparison.withinTolerance) {
+  console.error('Regression detected:', comparison.report);
+}
+```
+
+### MCP Server (VS Code, Cursor)
+
+Use TruthGuard directly from your IDE — no terminal needed.
+
+**Setup (one time):**
+1. In VS Code: `Ctrl+Shift+P` → **"MCP: Open User Configuration"**
+2. Add this to `mcp.json`:
+
+```json
+{
+  "servers": {
+    "truthguard": {
+      "type": "stdio",
+      "command": "npx",
+      "args": ["-y", "truthguard", "mcp"]
+    }
+  }
+}
+```
+
+3. Restart VS Code
+
+**Usage:** In Copilot Chat, say: *"Call truthguard verify_response with this trace: {...}"*
+
+8 tools available: `verify_response`, `quick_check`, `check_trace_quality`, `list_rules`, `get_failure_info`, `evaluate_with_policy`, `get_live_traces`, `get_trace_report`
+
+The last two tools bridge proxy results to your IDE — ask Copilot *"Call get_live_traces"* to see recent production evaluations.
+
+Full setup guide: [docs/getting-started.md](docs/getting-started.md#ide--mcp-server-vs-code-cursor)
+
+### Express Middleware
+
+```typescript
+import express from 'express';
+import { groundingMiddleware, FileStore } from 'truthguard';
+
+const app = express();
+app.post('/api/chat', groundingMiddleware({
+  mode: 'warn',
+  store: new FileStore('./traces/grounding.jsonl'),
+  extractTrace: (req, res, body) => body.trace,
+}));
+```
+
+---
+
+## CLI
+
+```bash
+npx truthguard debug trace.json              # Evaluate one trace
+npx truthguard run dataset.jsonl             # Batch dataset evaluation
+npx truthguard run dataset.jsonl --gate gate.yml  # CI quality gate
+npx truthguard observe --port 3001           # Start observe server + proxy
+```
+
+---
+
+## CI/CD Integration
+
+### GitHub Actions
+
+```yaml
+# .github/workflows/truthguard-gate.yml
+name: TruthGuard Quality Gate
+on: [push, pull_request]
+
+jobs:
+  grounding-gate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - run: npm ci
+      - run: npx truthguard run test-cases.jsonl --gate .ai-rcp-gate.yml
+```
+
+### Gate config (`.ai-rcp-gate.yml`)
+
+```yaml
+name: "Grounding Quality Gate"
+assertions:
+  - metric: grounding_score
+    operator: ">="
+    threshold: 0.90
+  - metric: failure_count
+    operator: "<="
+    threshold: 0
+  - metric: pass_rate
+    operator: ">="
+    threshold: 1.0
+```
+
+---
+
+## How It Works
+
+```
+Agent Response → Claim Extraction → Matcher → Rules → Report
+                  (regex: numbers,   (numeric,  (30      (score,
+                   dates, names,      count,     rules)   failures,
+                   counts)            date,               advisor)
+                                      name)
+```
+
+1. **Extract** factual claims from the agent's text response (numbers, dates, names, counts)
+2. **Match** each claim against values in tool outputs (with configurable tolerances)
+3. **Detect** failure patterns using 30 rules across 4 categories
+4. **Score** grounding quality with severity-weighted formula
+5. **Diagnose** root causes and suggest repair sequence
+
+**No LLM calls.** 100% deterministic. ~55% claim coverage (numbers, dates, names, counts). L2 structured matching (booleans, enums, key-values) extends to ~70-75%.
+
+---
+
+## Configurable Tolerances
+
+```yaml
+# .ai-rcp.yml
+tolerances:
+  numeric:
+    relative_tolerance: 0.05    # ±5% for numbers
+    rounding_allowed: true
+  count:
+    exact_match: true
+  date:
+    exact_match: true
+  name:
+    fuzzy_match: true
+    threshold: 0.85             # Jaro-Winkler similarity
+```
+
+---
+
+## Language Support
+
+- **Claim extraction:** Numbers, dates (7 formats incl. European DD.MM.YYYY), Serbian months (januar–decembar), relative dates (yesterday/juče, pre N dana)
+- **Unit conversion:** 13 languages (EN, SR, ES, FR, PT, RU, HI, AR, BN, ZH, JA...)
+- **Vague qualifier guard:** English + Serbian (oko, otprilike, negde)
+- **Name matching:** Diacritics-aware (ć→c, š→s) via Jaro-Winkler
+
+---
+
+## Architecture
+
+```
+src/
+├── Trace/        TraceBuilder SDK + multi-turn support
+├── Claims/       Claim extraction (regex, multilingual)
+├── Matchers/     Numeric, count, date, name matchers
+├── Rules/        30 detection rules (4 categories)
+├── Grounding/    Engine orchestration + entity-aware grounding
+├── Advisor/      Diagnostic advisor (RCA, repair sequence, hints)
+├── Registry/     Failure registry (severity, suppression graph)
+├── Policy/       Per-failure enforcement (block/warn/observe)
+├── Reports/      JSON + text report generators
+├── Config/       YAML tolerance configuration
+├── Gate/         CI/CD quality gate
+├── Baseline/     Snapshot regression detection
+├── Runner/       JSONL dataset batch evaluation
+├── Mode/         Pipeline (debug/ci/observe/warn/block)
+├── Store/        FileStore + InMemoryStore
+├── Alerting/     Console, Webhook, Callback dispatchers
+├── Middleware/    Express middleware factory
+├── SDK/          OpenAI wrapper (auto trace capture)
+├── Proxy/        Transparent AI API proxy builders
+├── MCP/          MCP Server (8 IDE tools)
+├── L2/           Structured context matching (boolean, enum, key-value)
+└── cli/          CLI commands
+```
+
+---
+
+## License
+
+MIT

package/dist/Advisor/index.d.ts

@@ -0,0 +1,78 @@
+/**
+ * TruthGuard Diagnostic Advisor
+ *
+ * Generates actionable diagnostic advice for detected failures.
+ * Two types of hints per failure:
+ *   - prompt_hint: suggest a prompt change (fast but unreliable — LLM may ignore)
+ *   - code_guard:  suggest a programmatic fix (slower to implement but deterministic)
+ *
+ * Does NOT auto-fix or generate patch-ready code snippets with "confidence" scores.
+ * Instead, gives honest, evidence-based guidance and re-evaluates after the fix.
+ */
+import type { FailureType, FailureSeverity, GroundingReport, Trace } from '../types';
+/** A single remediation hint — either prompt-level or code-level. */
+export interface RemediationHint {
+    /** Whether this is a prompt-level or code-level suggestion. */
+    type: 'prompt_hint' | 'code_guard';
+    /** What direction to take. */
+    direction: string;
+    /** A concrete example of the fix. */
+    example: string;
+    /** Why this hint might not be sufficient. */
+    caveat: string;
+}
+/** Evidence extracted from the trace proving the failure. */
+export interface DiagnosisEvidence {
+    /** What the trace showed. */
+    observation: string;
+    /** Where in the trace this was found (step ID or description). */
+    source: string;
+}
+/** Full diagnostic advice for a single detected failure. */
+export interface DiagnosticAdvice {
+    /** The failure type this advice addresses. */
+    failureType: FailureType;
+    /** Severity from the detection. */
+    severity: FailureSeverity;
+    /** One-line summary of the problem. */
+    what: string;
+    /** Explanation of why this happened. */
+    why: string;
+    /** Evidence from the trace proving the failure. */
+    evidence: DiagnosisEvidence[];
+    /** Remediation hints — always includes both prompt_hint and code_guard. */
+    hints: RemediationHint[];
+}
+/** Result of running the diagnostic advisor on a full grounding report. */
+export interface AdvisorReport {
+    /** The trace that was evaluated. */
+    traceId: string;
+    /** Total detected failures that were diagnosed. */
+    totalDiagnosed: number;
+    /** Diagnostic advice per detected failure. */
+    advice: DiagnosticAdvice[];
+    /** Ordered repair steps derived from failure dependencies. */
+    repairSequence?: {
+        type: FailureType;
+        reason: string;
+    }[];
+    /** High-level synthesized advice when multiple failures share a common root cause. */
+    synthesizedAdvice?: string;
+}
+/**
+ * Generate a full diagnostic advisor report from a grounding report.
+ *
+ * Only advises on primary and secondary failures (not suppressed).
+ * Hypotheses (low-confidence) are excluded — they need human review, not auto-advice.
+ */
+export declare function generateAdvisorReport(report: GroundingReport, trace: Trace): AdvisorReport;
+/**
+ * Generate diagnostic advice for a single failure type.
+ * Useful for looking up hints without running full evaluation.
+ */
+export declare function getHintsForFailureType(failureType: FailureType): RemediationHint[];
+/**
+ * Format the advisor report as human-readable text.
+ */
+export declare function formatAdvisorReport(advisor: AdvisorReport): string;
+//# sourceMappingURL=index.d.ts.map

package/dist/Advisor/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/Advisor/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,eAAe,EAEf,eAAe,EACf,KAAK,EAEN,MAAM,UAAU,CAAC;AAOlB,qEAAqE;AACrE,MAAM,WAAW,eAAe;IAC9B,+DAA+D;IAC/D,IAAI,EAAE,aAAa,GAAG,YAAY,CAAC;IACnC,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,6DAA6D;AAC7D,MAAM,WAAW,iBAAiB;IAChC,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,4DAA4D;AAC5D,MAAM,WAAW,gBAAgB;IAC/B,8CAA8C;IAC9C,WAAW,EAAE,WAAW,CAAC;IACzB,mCAAmC;IACnC,QAAQ,EAAE,eAAe,CAAC;IAC1B,uCAAuC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,GAAG,EAAE,MAAM,CAAC;IACZ,mDAAmD;IACnD,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,2EAA2E;IAC3E,KAAK,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED,2EAA2E;AAC3E,MAAM,WAAW,aAAa;IAC5B,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,8CAA8C;IAC9C,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,8DAA8D;IAC9D,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,WAAW,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACzD,sFAAsF;IACtF,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAghBD;;;;;GAKG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,eAAe,EACvB,KAAK,EAAE,KAAK,GACX,aAAa,CAyBf;AAwFD;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,WAAW,GAAG,eAAe,EAAE,CAMlF;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAsDlE"}