rag-eval-node-ts 0.3.0

package/dist/__tests__/heuristic/token-f1.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=token-f1.test.d.ts.map

package/dist/__tests__/heuristic/token-f1.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-f1.test.d.ts","sourceRoot":"","sources":["../../../src/__tests__/heuristic/token-f1.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/heuristic/token-f1.test.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const token_f1_1 = require("../../heuristic/token-f1");
+(0, vitest_1.describe)('tokenF1', () => {
+    (0, vitest_1.it)('returns 1.0 for a perfect match', () => {
+        const text = 'the cat sat on the mat';
+        (0, vitest_1.expect)((0, token_f1_1.tokenF1)(text, text)).toBeCloseTo(1.0);
+    });
+    (0, vitest_1.it)('returns 0.0 for completely disjoint texts', () => {
+        (0, vitest_1.expect)((0, token_f1_1.tokenF1)('alpha beta gamma', 'delta epsilon zeta')).toBe(0.0);
+    });
+    (0, vitest_1.it)('returns 0.0 when reference is empty', () => {
+        (0, vitest_1.expect)((0, token_f1_1.tokenF1)('', 'hello world')).toBe(0.0);
+    });
+    (0, vitest_1.it)('returns 0.0 when hypothesis is empty', () => {
+        (0, vitest_1.expect)((0, token_f1_1.tokenF1)('hello world', '')).toBe(0.0);
+    });
+    (0, vitest_1.it)('returns partial score for partial overlap', () => {
+        // reference: [a, b, c], hypothesis: [a, b, d]
+        // common: a, b (2 tokens)
+        // precision = 2/3, recall = 2/3
+        // F1 = 2*(2/3)*(2/3) / (2/3 + 2/3) = 2/3
+        const score = (0, token_f1_1.tokenF1)('a b c', 'a b d');
+        (0, vitest_1.expect)(score).toBeCloseTo(2 / 3, 2);
+    });
+    (0, vitest_1.it)('handles duplicate tokens correctly using multiset intersection', () => {
+        // reference: [a, a, b], hypothesis: [a, b, c]
+        // common: a(min(2,1)=1), b(min(1,1)=1) = 2
+        // precision = 2/3, recall = 2/3
+        const score = (0, token_f1_1.tokenF1)('a a b', 'a b c');
+        (0, vitest_1.expect)(score).toBeCloseTo(2 / 3, 2);
+    });
+    (0, vitest_1.it)('returns a score between 0 and 1', () => {
+        const score = (0, token_f1_1.tokenF1)('RAG is a technique for grounded generation', 'RAG combines retrieval with generation');
+        (0, vitest_1.expect)(score).toBeGreaterThan(0);
+        (0, vitest_1.expect)(score).toBeLessThanOrEqual(1);
+    });
+});
+//# sourceMappingURL=token-f1.test.js.map

package/dist/__tests__/heuristic/token-f1.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-f1.test.js","sourceRoot":"","sources":["../../../src/__tests__/heuristic/token-f1.test.ts"],"names":[],"mappings":";;AAAA,mCAA8C;AAC9C,uDAAmD;AAEnD,IAAA,iBAAQ,EAAC,SAAS,EAAE,GAAG,EAAE;IACvB,IAAA,WAAE,EAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,IAAI,GAAG,wBAAwB,CAAC;QACtC,IAAA,eAAM,EAAC,IAAA,kBAAO,EAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,IAAA,eAAM,EAAC,IAAA,kBAAO,EAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,IAAA,eAAM,EAAC,IAAA,kBAAO,EAAC,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,IAAA,eAAM,EAAC,IAAA,kBAAO,EAAC,aAAa,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,8CAA8C;QAC9C,0BAA0B;QAC1B,gCAAgC;QAChC,yCAAyC;QACzC,MAAM,KAAK,GAAG,IAAA,kBAAO,EAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxC,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,8CAA8C;QAC9C,2CAA2C;QAC3C,gCAAgC;QAChC,MAAM,KAAK,GAAG,IAAA,kBAAO,EAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxC,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,KAAK,GAAG,IAAA,kBAAO,EAAC,4CAA4C,EAAE,wCAAwC,CAAC,CAAC;QAC9G,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACjC,IAAA,eAAM,EAAC,KAAK,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/__tests__/metrics/faithfulness.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=faithfulness.test.d.ts.map

package/dist/__tests__/metrics/faithfulness.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"faithfulness.test.d.ts","sourceRoot":"","sources":["../../../src/__tests__/metrics/faithfulness.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/metrics/faithfulness.test.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const faithfulness_1 = require("../../metrics/faithfulness");
+(0, vitest_1.describe)('scoreFaithfulness', () => {
+    (0, vitest_1.it)('returns a high score when the answer closely matches the context', async () => {
+        const sample = {
+            question: 'What is RAG?',
+            answer: 'RAG combines retrieval with generation to produce grounded answers.',
+            contexts: [
+                'Retrieval-augmented generation combines retrieval with generation to produce grounded answers.',
+            ],
+        };
+        const result = await (0, faithfulness_1.scoreFaithfulness)(sample);
+        (0, vitest_1.expect)(result.metricId).toBe('faithfulness');
+        (0, vitest_1.expect)(result.score).not.toBeNull();
+        (0, vitest_1.expect)(result.score).toBeGreaterThan(0.5);
+        (0, vitest_1.expect)(result.mode).toBe('heuristic');
+        (0, vitest_1.expect)(result.llmCalls).toBe(0);
+        (0, vitest_1.expect)(result.durationMs).toBeGreaterThanOrEqual(0);
+    });
+    (0, vitest_1.it)('returns a low score when the answer is not in the context', async () => {
+        const sample = {
+            question: 'What is the capital of France?',
+            answer: 'The president signed an executive order today regarding trade policy.',
+            contexts: [
+                'Paris is the capital of France and a major European city.',
+            ],
+        };
+        const result = await (0, faithfulness_1.scoreFaithfulness)(sample);
+        (0, vitest_1.expect)(result.score).toBeLessThan(0.5);
+    });
+    (0, vitest_1.it)('emits WARNING signals for unsupported sentences', async () => {
+        const sample = {
+            question: 'What is the sky made of?',
+            answer: 'The sky is filled with nitrogen oxygen and argon. Additionally dinosaurs went extinct 66 million years ago.',
+            contexts: ['The atmosphere contains nitrogen oxygen and trace gases.'],
+        };
+        const result = await (0, faithfulness_1.scoreFaithfulness)(sample);
+        const warnings = result.signals.filter(s => s.severity === 'warning');
+        (0, vitest_1.expect)(warnings.length).toBeGreaterThan(0);
+    });
+    (0, vitest_1.it)('returns score 0 when there are no contexts', async () => {
+        const sample = {
+            question: 'Q?',
+            answer: 'Some answer.',
+            contexts: [],
+        };
+        const result = await (0, faithfulness_1.scoreFaithfulness)(sample);
+        (0, vitest_1.expect)(result.score).toBe(0);
+        (0, vitest_1.expect)(result.passed).toBe(false);
+    });
+    (0, vitest_1.it)('passes threshold check when score >= 0.7', async () => {
+        const sample = {
+            question: 'What is machine learning?',
+            answer: 'Machine learning is a field of artificial intelligence that trains models on data.',
+            contexts: [
+                'Machine learning is a field of artificial intelligence. It trains models on data to make predictions.',
+            ],
+        };
+        const result = await (0, faithfulness_1.scoreFaithfulness)(sample);
+        (0, vitest_1.expect)(result.threshold).toBe(0.7);
+        (0, vitest_1.expect)(typeof result.passed).toBe('boolean');
+    });
+});
+//# sourceMappingURL=faithfulness.test.js.map

package/dist/__tests__/metrics/faithfulness.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"faithfulness.test.js","sourceRoot":"","sources":["../../../src/__tests__/metrics/faithfulness.test.ts"],"names":[],"mappings":";;AAAA,mCAA8C;AAC9C,6DAA+D;AAG/D,IAAA,iBAAQ,EAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,IAAA,WAAE,EAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QAChF,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,cAAc;YACxB,MAAM,EAAE,qEAAqE;YAC7E,QAAQ,EAAE;gBACR,gGAAgG;aACjG;SACF,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,IAAA,gCAAiB,EAAC,MAAM,CAAC,CAAC;QAC/C,IAAA,eAAM,EAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC7C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QACpC,IAAA,eAAM,EAAC,MAAM,CAAC,KAAe,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QACpD,IAAA,eAAM,EAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtC,IAAA,eAAM,EAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChC,IAAA,eAAM,EAAC,MAAM,CAAC,UAAU,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,gCAAgC;YAC1C,MAAM,EAAE,uEAAuE;YAC/E,QAAQ,EAAE;gBACR,2DAA2D;aAC5D;SACF,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,IAAA,gCAAiB,EAAC,MAAM,CAAC,CAAC;QAC/C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAe,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,0BAA0B;YACpC,MAAM,EAAE,6GAA6G;YACrH,QAAQ,EAAE,CAAC,0DAA0D,CAAC;SACvE,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,IAAA,gCAAiB,EAAC,MAAM,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;QACtE,IAAA,eAAM,EAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,cAAc;YACtB,QAAQ,EAAE,EAAE;SACb,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,IAAA,gCAAiB,EAAC,MAAM,CAAC,CAAC;QAC/C,IAAA,eAAM,EAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAA,eAAM,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC,CAAC,CAAC;IAEH,IAAA,WAAE,EAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,MAAM,GAAe;YACzB,QAAQ,EAAE,2BAA2B;YACrC,MAAM,EAAE,oFAAoF;YAC5F,QAAQ,EAAE;gBACR,uGAAuG;aACxG;SACF,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,IAAA,gCAAiB,EAAC,MAAM,CAAC,CAAC;QAC/C,IAAA,eAAM,EAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAA,eAAM,EAAC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/__tests__/types.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.test.d.ts.map

package/dist/__tests__/types.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/types.test.ts"],"names":[],"mappings":""}