rag-eval-node-ts 0.3.0

package/dist/types.d.ts

@@ -0,0 +1,169 @@
+/** The atomic unit of RAG evaluation. All metrics operate on this structure. */
+export interface EvalSample {
+    /** The user's question or query. */
+    question: string;
+    /** The generated answer from the RAG pipeline. */
+    answer: string;
+    /**
+     * The retrieved context chunks provided to the LLM.
+     * Order matters for contextPrecision: index 0 is the highest-ranked chunk.
+     */
+    contexts: string[];
+    /**
+     * The reference (ground truth) answer.
+     * Required for: contextPrecision, contextRecall, answerCorrectness.
+     * Optional for all other metrics.
+     */
+    groundTruth?: string;
+    /** Optional identifier for tracking in batch results and reports. */
+    id?: string;
+    /** Optional metadata for grouping, filtering, or annotation. */
+    metadata?: Record<string, unknown>;
+}
+export type MetricId = 'faithfulness' | 'answerRelevance' | 'contextPrecision' | 'contextRecall' | 'contextRelevance' | 'answerCorrectness' | 'hallucinationRate';
+export type EvaluationMode = 'heuristic' | 'llm' | 'hybrid';
+/** A specific finding detected during metric computation. */
+export interface EvalSignal {
+    /** Machine-readable identifier for this signal type. */
+    id: string;
+    /** Which metric produced this signal. */
+    metricId: MetricId;
+    /** Severity level of the signal. */
+    severity: 'info' | 'warning' | 'critical';
+    /** Human-readable description of the finding. */
+    message: string;
+    /**
+     * The specific text (claim, sentence, context chunk excerpt) that triggered this signal.
+     */
+    evidence?: string;
+    /** Index of the context chunk involved, if applicable. */
+    contextChunkIndex?: number;
+}
+/** The result of computing one metric on one EvalSample. */
+export interface MetricResult {
+    metricId: MetricId;
+    /**
+     * The 0-1 score. null if the metric could not be computed
+     * (missing required inputs, all LLM calls failed).
+     */
+    score: number | null;
+    /** The mode used to compute this metric (heuristic or llm, not hybrid). */
+    mode: 'heuristic' | 'llm';
+    /** Whether the score meets the configured threshold. null if score is null. */
+    passed: boolean | null;
+    /** The configured threshold used for pass/fail. */
+    threshold: number;
+    /** Human-readable explanation of how the score was determined. */
+    explanation: string;
+    /** Specific findings supporting the score. */
+    signals: EvalSignal[];
+    /** Number of LLM judge calls made for this metric. */
+    llmCalls: number;
+    /** Wall-clock duration in milliseconds. */
+    durationMs: number;
+}
+/** Cost tracking for an evaluation run. */
+export interface CostTracker {
+    llmCalls: number;
+    estimatedInputTokens: number;
+    estimatedOutputTokens: number;
+    parseFailures: number;
+}
+/** The result of evaluating a single EvalSample. */
+export interface EvalResult {
+    sample: EvalSample;
+    metrics: Record<MetricId, MetricResult>;
+    /**
+     * Weighted average of all computed metric scores.
+     * null if no metrics were successfully computed.
+     */
+    compositeScore: number | null;
+    /** true if all computed metrics passed their thresholds and composite score passes compositeThreshold. */
+    passed: boolean;
+    cost: CostTracker;
+    durationMs: number;
+    evaluatedAt: string;
+}
+/** Per-metric aggregate statistics across a batch. */
+export interface MetricAggregate {
+    metricId: MetricId;
+    mean: number;
+    median: number;
+    min: number;
+    max: number;
+    stdDev: number;
+    passRate: number;
+    nullRate: number;
+}
+/** Regression comparison for a single metric against a baseline. */
+export interface MetricRegression {
+    metricId: MetricId;
+    baselineMean: number;
+    currentMean: number;
+    delta: number;
+    regressed: boolean;
+}
+/** The result of evaluating a batch of EvalSamples. */
+export interface BatchEvalResult {
+    results: EvalResult[];
+    aggregates: Record<MetricId, MetricAggregate>;
+    compositeAggregate: {
+        mean: number;
+        median: number;
+        min: number;
+        max: number;
+        stdDev: number;
+        passRate: number;
+    };
+    passed: boolean;
+    totalCost: CostTracker;
+    totalDurationMs: number;
+    regressions?: MetricRegression[];
+    evaluatedAt: string;
+}
+export type JudgeFn = (prompt: string) => Promise<string>;
+export type PromptOverrides = Partial<Record<MetricId, string>>;
+export interface MetricThresholds {
+    faithfulness?: number;
+    answerRelevance?: number;
+    contextPrecision?: number;
+    contextRecall?: number;
+    contextRelevance?: number;
+    answerCorrectness?: number;
+    hallucinationRate?: number;
+}
+export interface HeuristicOptions {
+    claimSupportThreshold?: number;
+    chunkRelevanceThreshold?: number;
+    sentenceCoverageThreshold?: number;
+    ngramWeight?: number;
+    tfidfWeight?: number;
+    ngramSizes?: number[];
+    ngramWeights?: number[];
+}
+export interface EvaluateOptions {
+    mode?: EvaluationMode;
+    judge?: JudgeFn;
+    thresholds?: MetricThresholds;
+    metricModes?: Partial<Record<MetricId, 'heuristic' | 'llm'>>;
+    compositeThreshold?: number;
+    compositeWeights?: Partial<Record<MetricId, number>>;
+    heuristic?: HeuristicOptions;
+    promptOverrides?: PromptOverrides;
+}
+export interface BatchEvaluateOptions extends EvaluateOptions {
+    concurrency?: number;
+    onProgress?: (completed: number, total: number) => void;
+    baselineResult?: BatchEvalResult;
+    regressionThreshold?: number;
+}
+export interface EvaluatorConfig extends EvaluateOptions {
+    metrics?: MetricId[];
+}
+/** A pre-configured evaluator instance. */
+export interface Evaluator {
+    evaluate(sample: EvalSample, metrics?: MetricId[], options?: EvaluateOptions): Promise<EvalResult>;
+    evaluateBatch(samples: EvalSample[], metrics?: MetricId[], options?: BatchEvaluateOptions): Promise<BatchEvalResult>;
+    readonly config: EvaluatorConfig;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAEA,gFAAgF;AAChF,MAAM,WAAW,UAAU;IACzB,oCAAoC;IACpC,QAAQ,EAAE,MAAM,CAAC;IAEjB,kDAAkD;IAClD,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,QAAQ,EAAE,MAAM,EAAE,CAAC;IAEnB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,qEAAqE;IACrE,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAID,MAAM,MAAM,QAAQ,GAChB,cAAc,GACd,iBAAiB,GACjB,kBAAkB,GAClB,eAAe,GACf,kBAAkB,GAClB,mBAAmB,GACnB,mBAAmB,CAAC;AAExB,MAAM,MAAM,cAAc,GAAG,WAAW,GAAG,KAAK,GAAG,QAAQ,CAAC;AAE5D,6DAA6D;AAC7D,MAAM,WAAW,UAAU;IACzB,wDAAwD;IACxD,EAAE,EAAE,MAAM,CAAC;IAEX,yCAAyC;IACzC,QAAQ,EAAE,QAAQ,CAAC;IAEnB,oCAAoC;IACpC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,CAAC;IAE1C,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,0DAA0D;IAC1D,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,4DAA4D;AAC5D,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,QAAQ,CAAC;IAEnB;;;OAGG;IACH,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAErB,2EAA2E;IAC3E,IAAI,EAAE,WAAW,GAAG,KAAK,CAAC;IAE1B,+EAA+E;IAC/E,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC;IAEvB,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAC;IAElB,kEAAkE;IAClE,WAAW,EAAE,MAAM,CAAC;IAEpB,8CAA8C;IAC9C,OAAO,EAAE,UAAU,EAAE,CAAC;IAEtB,sDAAsD;IACtD,QAAQ,EAAE,MAAM,CAAC;IAEjB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACpB;AAID,2CAA2C;AAC3C,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAExC;;;OAGG;IACH,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAE9B,0GAA0G;IAC1G,MAAM,EAAE,OAAO,CAAC;IAEhB,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,oEAAoE;AACpE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,uDAAuD;AACvD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC9C,kBAAkB,EAAE;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,WAAW,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC;CACrB;AAID,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAE1D,MAAM,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAEhE,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,yBAAyB,CAAC,EAAE,MAAM,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,WAAW,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC;IAC7D,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,SAAS,CAAC,EAAE,gBAAgB,CAAC;IAC7B,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACxD,cAAc,CAAC,EAAE,eAAe,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAgB,SAAQ,eAAe;IACtD,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;CACtB;AAED,2CAA2C;AAC3C,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACnG,aAAa,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IACrH,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;CAClC"}

package/dist/types.js

@@ -0,0 +1,4 @@
+"use strict";
+// ── Input Types ──────────────────────────────────────────────────────
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":";AAAA,wEAAwE"}

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "rag-eval-node-ts",
+  "version": "0.3.0",
+  "description": "Lightweight RAG evaluation metrics for CI/CD pipelines",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "rag-eval": "dist/cli/cli.js"
+  },
+  "exports": {
+    ".": "./dist/index.js",
+    "./adapters/openai": "./dist/adapters/openai.js",
+    "./adapters/anthropic": "./dist/adapters/anthropic.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "lint": "eslint src/",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "openai": "^4.0.0",
+    "@anthropic-ai/sdk": "^0.20.0"
+  },
+  "peerDependenciesMeta": {
+    "openai": {
+      "optional": true
+    },
+    "@anthropic-ai/sdk": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "@typescript-eslint/eslint-plugin": "^8.57.1",
+    "@typescript-eslint/parser": "^8.57.1",
+    "eslint": "^10.1.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.0"
+  }
+}