transparent-confidence 0.1.0

package/dist/index.js

@@ -0,0 +1,533 @@
+// src/dimensions/authority.ts
+var MAX = 20;
+var DEFAULT_TIERS = [
+  { name: "Primary", rank: 10 },
+  { name: "Secondary", rank: 20 },
+  { name: "Supporting", rank: 30 }
+];
+function scoreAuthority(inputs, config) {
+  const { candidates } = inputs;
+  const tiers = config.authority?.tiers ?? DEFAULT_TIERS;
+  const parts = [];
+  if (candidates.length === 0) {
+    return {
+      raw: 0,
+      max: MAX,
+      normalized: 0,
+      explanation: "No candidates retrieved \u2014 authority cannot be evaluated."
+    };
+  }
+  const effectiveRanks = candidates.map((c) => resolveRank(c.authorityRank, c.documentType, tiers));
+  const minRank = Math.min(...effectiveRanks);
+  let base;
+  let tierName;
+  if (minRank <= 10) {
+    base = 18;
+    tierName = tiers.find((t) => t.rank <= 10)?.name ?? "Primary";
+    parts.push(`Answer grounded in ${tierName} tier documents (rank \u2264 10) \u2014 highest authority.`);
+  } else if (minRank <= 20) {
+    base = 13;
+    tierName = tiers.find((t) => t.rank > 10 && t.rank <= 20)?.name ?? "Secondary";
+    parts.push(`Answer grounded in ${tierName} tier documents (rank \u2264 20).`);
+  } else if (minRank <= 30) {
+    base = 7;
+    tierName = tiers.find((t) => t.rank > 20 && t.rank <= 30)?.name ?? "Supporting";
+    parts.push(`Answer grounded in ${tierName} tier documents (rank \u2264 30) \u2014 lower authority.`);
+  } else {
+    base = 2;
+    parts.push("Answer grounded in unclassified or minimal-authority documents.");
+  }
+  let bonus = 0;
+  if (candidates.some((c) => c.isAmendment === true)) {
+    bonus += 1;
+    parts.push(
+      "Amendment sections included \u2014 amended version controls over original language (+1)."
+    );
+  }
+  const uniqueRankTiers = new Set(effectiveRanks.map((r) => rankBucket(r)));
+  if (uniqueRankTiers.size > 1) {
+    bonus += 1;
+    parts.push("Multiple authority tiers consulted (+1).");
+  }
+  const raw = Math.min(MAX, base + bonus);
+  return {
+    raw,
+    max: MAX,
+    normalized: Math.round(raw / MAX * 100),
+    explanation: parts.join(" ")
+  };
+}
+function resolveRank(authorityRank, documentType, tiers) {
+  if (authorityRank !== void 0) return authorityRank;
+  if (!documentType) return 99;
+  const lower = documentType.toLowerCase();
+  for (const tier of tiers) {
+    if (tier.keywords?.some((kw) => lower.includes(kw.toLowerCase()))) {
+      return tier.rank;
+    }
+  }
+  return 99;
+}
+function rankBucket(rank) {
+  if (rank <= 10) return "tier1";
+  if (rank <= 20) return "tier2";
+  if (rank <= 30) return "tier3";
+  return "unclassified";
+}
+
+// src/dimensions/consistency.ts
+var MAX2 = 10;
+function scoreConsistency(inputs) {
+  const { candidates } = inputs;
+  const parts = [];
+  if (candidates.length === 0) {
+    return {
+      raw: 0,
+      max: MAX2,
+      normalized: 0,
+      explanation: "No candidates retrieved \u2014 evidence consistency cannot be evaluated."
+    };
+  }
+  let variancePts;
+  if (candidates.length === 1) {
+    variancePts = 4;
+    parts.push("Single candidate retrieved \u2014 variance unmeasurable, neutral score assigned.");
+  } else {
+    const scores = candidates.map((c) => c.combinedScore);
+    const sd = stdDev(scores);
+    if (sd < 0.1) {
+      variancePts = 8;
+      parts.push(`Score std dev ${sd.toFixed(3)} \u2014 very tight retrieval consistency.`);
+    } else if (sd < 0.2) {
+      variancePts = 6;
+      parts.push(`Score std dev ${sd.toFixed(3)} \u2014 good retrieval consistency.`);
+    } else if (sd < 0.3) {
+      variancePts = 4;
+      parts.push(`Score std dev ${sd.toFixed(3)} \u2014 moderate retrieval consistency.`);
+    } else {
+      variancePts = 2;
+      parts.push(`Score std dev ${sd.toFixed(3)} \u2014 scattered retrieval scores.`);
+    }
+  }
+  let conflictAdj;
+  if (inputs.conflictingCandidateCount !== void 0) {
+    if (inputs.conflictingCandidateCount === 0) {
+      conflictAdj = 2;
+      parts.push("No conflicting candidates \u2014 full agreement (+2).");
+    } else if (inputs.conflictingCandidateCount === 1) {
+      conflictAdj = 0;
+      parts.push("1 conflicting candidate detected.");
+    } else {
+      conflictAdj = -2;
+      parts.push(`${inputs.conflictingCandidateCount} conflicting candidates detected (\u22122).`);
+    }
+  } else if (inputs.hasConflict === true) {
+    conflictAdj = -2;
+    parts.push("Conflicting information detected across candidates (\u22122).");
+  } else {
+    conflictAdj = 2;
+    parts.push("No conflict detected (+2).");
+  }
+  const raw = Math.max(0, Math.min(MAX2, variancePts + conflictAdj));
+  return {
+    raw,
+    max: MAX2,
+    normalized: Math.round(raw / MAX2 * 100),
+    explanation: parts.join(" ")
+  };
+}
+function stdDev(values) {
+  const n = values.length;
+  if (n < 2) return 0;
+  const mean = values.reduce((sum, v) => sum + v, 0) / n;
+  const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / n;
+  return Math.sqrt(variance);
+}
+
+// src/dimensions/corpus.ts
+var MAX3 = 15;
+function scoreCorpus(inputs, config) {
+  const expectedDocCount = config.corpus?.expectedDocCount ?? 5;
+  const { corpusDocCount, missingRelevantType } = inputs;
+  const parts = [];
+  if (corpusDocCount === void 0) {
+    return {
+      raw: 0,
+      max: MAX3,
+      normalized: 0,
+      explanation: "corpusDocCount not provided. Supply the number of loaded document types to enable corpus scoring."
+    };
+  }
+  const ratio = Math.min(corpusDocCount, expectedDocCount) / expectedDocCount;
+  let base;
+  if (ratio >= 1) {
+    base = 15;
+    parts.push(`All ${expectedDocCount} expected document types present.`);
+  } else if (ratio >= 0.8) {
+    base = 12;
+    parts.push(`${corpusDocCount} of ${expectedDocCount} expected document types present.`);
+  } else if (ratio >= 0.6) {
+    base = 9;
+    parts.push(`${corpusDocCount} of ${expectedDocCount} expected document types present.`);
+  } else if (ratio >= 0.4) {
+    base = 5;
+    parts.push(`${corpusDocCount} of ${expectedDocCount} expected document types present.`);
+  } else if (ratio >= 0.2) {
+    base = 2;
+    parts.push(`Only ${corpusDocCount} of ${expectedDocCount} expected document types present.`);
+  } else {
+    base = 0;
+    parts.push(`Corpus is empty or nearly empty (${corpusDocCount} of ${expectedDocCount} types).`);
+  }
+  let score = base;
+  if (missingRelevantType === true) {
+    score = Math.max(0, score - 3);
+    parts.push(
+      "A document type directly relevant to this question appears to be missing \u2014 answer may be incomplete (\u22123)."
+    );
+  }
+  const raw = Math.max(0, Math.min(MAX3, score));
+  return {
+    raw,
+    max: MAX3,
+    normalized: Math.round(raw / MAX3 * 100),
+    explanation: parts.join(" ")
+  };
+}
+
+// src/dimensions/freshness.ts
+var MAX4 = 15;
+var DEFAULTS = {
+  maxAgeForFullScore: 90,
+  penaltyPerMonth: 1.5,
+  hardCutoffAge: 730
+};
+function scoreFreshness(inputs, config) {
+  const cfg = {
+    maxAgeForFullScore: config.freshness?.maxAgeForFullScore ?? DEFAULTS.maxAgeForFullScore,
+    penaltyPerMonth: config.freshness?.penaltyPerMonth ?? DEFAULTS.penaltyPerMonth,
+    hardCutoffAge: config.freshness?.hardCutoffAge ?? DEFAULTS.hardCutoffAge
+  };
+  const datedCandidates = inputs.candidates.filter((c) => c.lastUpdated instanceof Date);
+  if (datedCandidates.length === 0) {
+    return {
+      raw: 0,
+      max: MAX4,
+      normalized: 0,
+      explanation: "No lastUpdated dates provided on candidates. Supply Candidate.lastUpdated to enable freshness scoring."
+    };
+  }
+  const now = Date.now();
+  const agesInDays = datedCandidates.map((c) => (now - c.lastUpdated.getTime()) / (1e3 * 60 * 60 * 24)).sort((a, b) => a - b);
+  const medianAge = median(agesInDays);
+  const parts = [];
+  parts.push(`Median document age: ${Math.round(medianAge)} days.`);
+  if (medianAge >= cfg.hardCutoffAge) {
+    return {
+      raw: 0,
+      max: MAX4,
+      normalized: 0,
+      explanation: `${parts.join(" ")} Documents exceed the ${cfg.hardCutoffAge}-day cutoff \u2014 freshness score is 0.`
+    };
+  }
+  if (medianAge <= cfg.maxAgeForFullScore) {
+    parts.push(`Within the ${cfg.maxAgeForFullScore}-day full-score window.`);
+    return {
+      raw: MAX4,
+      max: MAX4,
+      normalized: 100,
+      explanation: parts.join(" ")
+    };
+  }
+  const daysOverWindow = medianAge - cfg.maxAgeForFullScore;
+  const monthsOver = daysOverWindow / 30;
+  const penalty = monthsOver * cfg.penaltyPerMonth;
+  const raw = Math.max(0, Math.min(MAX4, MAX4 - penalty));
+  const rawRounded = Math.round(raw * 10) / 10;
+  parts.push(
+    `${Math.round(daysOverWindow)} days beyond the ${cfg.maxAgeForFullScore}-day window \u2014 penalty: ${penalty.toFixed(1)} pts.`
+  );
+  return {
+    raw: Math.round(rawRounded),
+    max: MAX4,
+    normalized: Math.round(rawRounded / MAX4 * 100),
+    explanation: parts.join(" ")
+  };
+}
+function median(sorted) {
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0 ? ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2 : sorted[mid] ?? 0;
+}
+
+// src/dimensions/grounding.ts
+var MAX5 = 30;
+var COMPLEXITY_CEILINGS = {
+  direct: 30,
+  inferential: 24,
+  "multi-hop": 18,
+  comparative: 16
+};
+function scoreGrounding(inputs) {
+  const parts = [];
+  if (inputs.documentsSilent === true) {
+    return {
+      raw: 0,
+      max: MAX5,
+      normalized: 0,
+      explanation: "Source documents do not address this question."
+    };
+  }
+  let score;
+  if (inputs.confidenceLevel === "high" && !inputs.ambiguityNotes) {
+    score = 30;
+    parts.push("Source text directly and unambiguously answers the question.");
+  } else if (inputs.confidenceLevel === "high") {
+    score = 21;
+    parts.push(
+      `Source text addresses the question but contains ambiguity: ${inputs.ambiguityNotes}`
+    );
+  } else if (inputs.confidenceLevel === "medium") {
+    score = 13;
+    if (inputs.ambiguityNotes) {
+      parts.push(
+        `Answer is inferrable from source text but not explicit. Ambiguity: ${inputs.ambiguityNotes}`
+      );
+    } else {
+      parts.push("Answer is inferrable from source text but not explicitly stated.");
+    }
+  } else {
+    score = 5;
+    parts.push(
+      "Source documents are insufficient, conflicting, or ambiguous on this question. Answer reflects best available inference."
+    );
+  }
+  if (inputs.requiresExpertReview === true) {
+    score -= 3;
+    parts.push("Expert review recommended before acting on this answer (\u22123).");
+  }
+  if (inputs.externalConstraintNote) {
+    score -= 2;
+    parts.push(
+      `External constraint may affect this answer: ${inputs.externalConstraintNote} (\u22122).`
+    );
+  }
+  if (inputs.hasConflict === true) {
+    score -= 5;
+    parts.push("Conflicting information detected across retrieved sections (\u22125).");
+  }
+  score = Math.max(0, score);
+  if (inputs.queryComplexity && inputs.queryComplexity !== "direct") {
+    const ceiling = COMPLEXITY_CEILINGS[inputs.queryComplexity];
+    if (score > ceiling) {
+      score = ceiling;
+      parts.push(`Score capped at ${ceiling} for ${inputs.queryComplexity} question type.`);
+    }
+  }
+  if (inputs.faithfulnessScore !== void 0) {
+    const fs = inputs.faithfulnessScore;
+    if (fs < 0.5) {
+      score = Math.max(0, score - 12);
+      parts.push(
+        `Faithfulness score ${fs.toFixed(2)} indicates significant hallucination risk (\u221212).`
+      );
+    } else if (fs < 0.7) {
+      score = Math.max(0, score - 7);
+      parts.push(`Faithfulness score ${fs.toFixed(2)} indicates moderate hallucination risk (\u22127).`);
+    } else if (fs < 0.9) {
+      score = Math.max(0, score - 3);
+      parts.push(`Faithfulness score ${fs.toFixed(2)} indicates minor hallucination risk (\u22123).`);
+    }
+  }
+  if (inputs.citationCount !== void 0 && inputs.citationCount > 0) {
+    const bonus = inputs.citationCount >= 3 ? 2 : inputs.citationCount === 2 ? 1 : 0;
+    if (bonus > 0) {
+      score = Math.min(MAX5, score + bonus);
+      parts.push(`${inputs.citationCount} sections explicitly cited in answer (+${bonus}).`);
+    }
+  }
+  const raw = Math.max(0, Math.min(MAX5, score));
+  return {
+    raw,
+    max: MAX5,
+    normalized: normalize(raw, MAX5),
+    explanation: parts.join(" ")
+  };
+}
+function normalize(raw, max) {
+  return Math.round(raw / max * 100);
+}
+
+// src/dimensions/retrieval.ts
+var MAX6 = 25;
+function scoreRetrieval(inputs) {
+  const { candidates } = inputs;
+  const parts = [];
+  if (candidates.length === 0) {
+    return {
+      raw: 0,
+      max: MAX6,
+      normalized: 0,
+      explanation: "No candidates were retrieved."
+    };
+  }
+  const confirmed = candidates.filter(
+    (c) => Object.values(c.retrievalScores).filter((s) => s > 0).length >= 2
+  );
+  let agreementPts;
+  if (confirmed.length >= 3) {
+    agreementPts = 15;
+    parts.push(`${confirmed.length} candidates confirmed by 2+ retrieval methods.`);
+  } else if (confirmed.length === 2) {
+    agreementPts = 12;
+    parts.push("2 candidates confirmed by 2+ retrieval methods.");
+  } else if (confirmed.length === 1) {
+    agreementPts = 8;
+    parts.push("1 candidate confirmed by 2+ retrieval methods.");
+  } else {
+    agreementPts = 3;
+    parts.push(
+      "No candidates confirmed by multiple retrieval methods \u2014 single-path retrieval only."
+    );
+  }
+  const sorted = [...candidates].sort((a, b) => b.combinedScore - a.combinedScore);
+  const top3 = sorted.slice(0, 3);
+  const effectiveScores = top3.map(
+    (c) => c.extractionQuality !== void 0 ? c.combinedScore * c.extractionQuality : c.combinedScore
+  );
+  const avgScore = effectiveScores.reduce((sum, s) => sum + s, 0) / effectiveScores.length;
+  let magnitudePts;
+  if (avgScore >= 0.8) {
+    magnitudePts = 8;
+  } else if (avgScore >= 0.65) {
+    magnitudePts = 6;
+  } else if (avgScore >= 0.5) {
+    magnitudePts = 4;
+  } else if (avgScore >= 0.35) {
+    magnitudePts = 2;
+  } else {
+    magnitudePts = 0;
+  }
+  parts.push(`Top-3 effective score avg: ${avgScore.toFixed(2)}.`);
+  if (top3.some((c) => c.extractionQuality !== void 0)) {
+    parts.push("Extraction quality applied as score multiplier.");
+  }
+  const withIds = candidates.filter((c) => c.documentId !== void 0);
+  const uniqueDocIds = new Set(withIds.map((c) => c.documentId));
+  let diversityPts = 0;
+  if (withIds.length > 0) {
+    if (uniqueDocIds.size >= 3) {
+      diversityPts = 3;
+      parts.push(`${uniqueDocIds.size} distinct source documents.`);
+    } else if (uniqueDocIds.size === 2) {
+      diversityPts = 1;
+      parts.push("2 distinct source documents.");
+    } else {
+      parts.push("All candidates from a single source document.");
+    }
+  }
+  let breadthPts = 0;
+  if (candidates.length >= 5) {
+    breadthPts = 2;
+  } else if (candidates.length >= 3) {
+    breadthPts = 1;
+  }
+  parts.push(`${candidates.length} total candidates.`);
+  const raw = Math.min(MAX6, agreementPts + magnitudePts + diversityPts + breadthPts);
+  return {
+    raw,
+    max: MAX6,
+    normalized: Math.round(raw / MAX6 * 100),
+    explanation: parts.join(" ")
+  };
+}
+
+// src/labels.ts
+function deriveLabel(score) {
+  if (score >= 85) return { label: "Strong", color: "green" };
+  if (score >= 65) return { label: "Moderate", color: "amber" };
+  if (score >= 40) return { label: "Limited", color: "orange" };
+  return { label: "Insufficient", color: "red" };
+}
+function deriveTier1(raw, max, notAddressed = false) {
+  if (notAddressed) {
+    return { score: 0, label: "Not Addressed", color: "gray" };
+  }
+  if (max <= 0) return null;
+  const score = Math.round(raw / max * 100);
+  const { label, color } = deriveLabel(score);
+  return { score, label, color };
+}
+function deriveTier2(raw, max) {
+  if (max <= 0) return null;
+  const score = Math.round(raw / max * 100);
+  if (score >= 85) return { score, label: "Complete", color: "green" };
+  if (score >= 65) return { score, label: "Good", color: "amber" };
+  if (score >= 40) return { score, label: "Partial", color: "orange" };
+  return { score, label: "Thin", color: "red" };
+}
+
+// src/normalize.ts
+function normalize2(raw, max) {
+  if (max <= 0) return 0;
+  return Math.round(Math.max(0, Math.min(100, raw / max * 100)));
+}
+
+// src/scorer.ts
+var CORE_MAX = 65;
+var AUTHORITY_MAX = 20;
+var CORPUS_MAX = 15;
+var FRESHNESS_MAX = 15;
+function computeConfidence(inputs, config = {}) {
+  const hasAuthority = config.authority !== void 0;
+  const hasCorpus = config.corpus !== void 0;
+  const hasFreshness = config.freshness !== void 0;
+  const grounding = scoreGrounding(inputs);
+  const retrieval = scoreRetrieval(inputs);
+  const consistency = scoreConsistency(inputs);
+  const authority = hasAuthority ? scoreAuthority(inputs, config) : void 0;
+  const corpus = hasCorpus ? scoreCorpus(inputs, config) : void 0;
+  const freshness = hasFreshness ? scoreFreshness(inputs, config) : void 0;
+  const activeExtensions = [];
+  if (hasAuthority) activeExtensions.push("authority");
+  if (hasCorpus) activeExtensions.push("corpus");
+  if (hasFreshness) activeExtensions.push("freshness");
+  const maxPossible = CORE_MAX + (hasAuthority ? AUTHORITY_MAX : 0) + (hasCorpus ? CORPUS_MAX : 0) + (hasFreshness ? FRESHNESS_MAX : 0);
+  const rawTotal = grounding.raw + retrieval.raw + consistency.raw + (authority?.raw ?? 0) + (corpus?.raw ?? 0) + (freshness?.raw ?? 0);
+  const total = normalize2(rawTotal, maxPossible);
+  const { label, color: labelColor } = deriveLabel(total);
+  const tier1Raw = grounding.raw + retrieval.raw + consistency.raw + (authority?.raw ?? 0);
+  const tier1Max = CORE_MAX + (hasAuthority ? AUTHORITY_MAX : 0);
+  const tier1 = deriveTier1(tier1Raw, tier1Max, inputs.documentsSilent === true);
+  const tier2Max = (hasCorpus ? CORPUS_MAX : 0) + (hasFreshness ? FRESHNESS_MAX : 0);
+  const tier2Raw = (corpus?.raw ?? 0) + (freshness?.raw ?? 0);
+  const tier2 = deriveTier2(tier2Raw, tier2Max);
+  return {
+    total,
+    label,
+    labelColor,
+    tier1,
+    tier2,
+    dimensions: {
+      grounding,
+      retrieval,
+      consistency,
+      ...authority !== void 0 && { authority },
+      ...corpus !== void 0 && { corpus },
+      ...freshness !== void 0 && { freshness }
+    },
+    meta: {
+      rawTotal,
+      maxPossible,
+      activeExtensions
+    }
+  };
+}
+function createScorer(config) {
+  return {
+    compute: (inputs) => computeConfidence(inputs, config)
+  };
+}
+export {
+  computeConfidence,
+  createScorer
+};

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "transparent-confidence",
+  "version": "0.1.0",
+  "description": "Explainable confidence scoring for RAG system answers — structured 0–100 scorecard across retrieval, grounding, and consistency dimensions",
+  "author": "Eric Tetzlaff",
+  "license": "Apache-2.0",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "CHANGELOG.md"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "keywords": [
+    "rag",
+    "retrieval-augmented-generation",
+    "confidence",
+    "scoring",
+    "llm",
+    "ai",
+    "explainability",
+    "evaluation",
+    "ragas",
+    "vector-search"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/emtcmca/transparent-confidence.git"
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm,cjs --dts --out-dir dist",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "coverage": "vitest run --coverage",
+    "lint": "biome check src/",
+    "lint:fix": "biome check --write src/",
+    "typecheck": "tsc --noEmit"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.4.16",
+    "@types/node": "^25.9.2",
+    "@vitest/coverage-v8": "^4.1.8",
+    "tsup": "^8.5.1",
+    "tsx": "^4.22.4",
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.8"
+  }
+}