ragas-js 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 pankajpatel19
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,66 @@
+# ragas-js
+
+LLM output evaluation metrics for Node.js/TypeScript — a JS-native alternative to [RAGAS](https://github.com/explodinggradients/ragas), for teams building RAG/LLM apps in Node instead of Python.
+
+## Why
+
+RAGAS is a popular framework for evaluating LLM/RAG outputs (hallucination checks, relevance scoring, etc.), but it's Python-only. Most Node/React/MERN teams building RAG apps have no equivalent way to evaluate their LLM outputs without spinning up a separate Python service.
+
+`ragas-js` brings the same idea — using an LLM as a judge to score another LLM's output — natively to Node/TypeScript.
+
+## Install
+
+\`\`\`bash
+npm install ragas-js
+\`\`\`
+
+## Usage
+
+\`\`\`ts
+import { faithfulness } from "ragas-js";
+
+const result = await faithfulness(
+{
+question: "What is the capital of France?",
+answer: "Paris is the capital of France.",
+context: ["Paris is the capital and most populous city of France."],
+},
+{
+provider: "openai",
+apiKey: process.env.OPENAI_API_KEY!,
+}
+);
+
+console.log(result);
+// { score: 1, statements: [{ statement: "...", supported: true }] }
+\`\`\`
+
+## What it does
+
+`faithfulness` measures how much of an answer is actually supported by the given context. It works in two LLM calls:
+
+1. Breaks the answer down into atomic factual statements
+2. Checks each statement against the context and marks it supported or not
+
+The score is the fraction of statements that were supported (0 to 1).
+
+## Current scope (v0.1)
+
+This is an early release — scope is intentionally narrow:
+
+- ✅ One metric: `faithfulness`
+- ✅ One LLM provider: OpenAI
+- ❌ No other metrics yet (answerRelevancy, contextPrecision, contextRecall)
+- ❌ No other LLM providers yet (Anthropic, Gemini)
+- ❌ No CLI, no dashboard, no persistence
+
+## Roadmap
+
+- [ ] `answerRelevancy` metric
+- [ ] `contextPrecision` / `contextRecall`
+- [ ] Support for Anthropic and Gemini
+- [ ] CLI for running evals against a JSON/CSV dataset
+
+## License
+
+MIT — see [LICENSE](./LICENSE).

package/dist/index.cjs

@@ -0,0 +1,93 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  faithfulness: () => faithfulness
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/metrics/faithfulness.ts
+var import_zod = require("zod");
+
+// src/llmClient.ts
+var import_client = require("openai/client");
+async function LLMCall(config, prompt) {
+  const client = new import_client.OpenAI({
+    apiKey: config.apiKey
+  });
+  const res = await client.chat.completions.create({
+    messages: [
+      {
+        role: "user",
+        content: prompt
+      }
+    ],
+    model: config.model,
+    temperature: config.temperature,
+    max_tokens: config.maxTokens
+  });
+  return res.choices[0]?.message?.content?.toString() ?? Promise.reject("No response from LLM");
+}
+
+// src/metrics/faithfulness.ts
+var statementSchema = import_zod.z.object({
+  statements: import_zod.z.array(import_zod.z.string())
+});
+var verdictSchema = import_zod.z.object({
+  verdicts: import_zod.z.array(
+    import_zod.z.object({
+      statement: import_zod.z.string(),
+      supported: import_zod.z.boolean()
+    })
+  )
+});
+async function faithfulness(input, llm) {
+  const extractionPrompt = `Given a question and an answer, break the answer into short, atomic factual statements. Return ONLY valid JSON: {"statements": ["...", "..."]}.
+
+Question: ${input.question}
+Answer: ${input.answer}`;
+  const rawStatements = await LLMCall(llm, extractionPrompt);
+  const parsed = statementSchema.safeParse(
+    JSON.parse(cleanJSON(rawStatements))
+  );
+  const verdictPrompt = `Given a context and a list of statements, judge for each statement whether it is directly supported by the context. Return ONLY valid JSON: {"verdicts": [{"statement": "...", "supported": true}]}.
+
+Context:
+${input.context.join("\n")}
+
+Statements:
+${parsed.success ? parsed.data.statements.map((s, i) => `${i + 1}. ${s}`).join("\n") : ""}`;
+  const verdict = await LLMCall(llm, verdictPrompt);
+  const parsedVerdict = verdictSchema.safeParse(JSON.parse(cleanJSON(verdict)));
+  const supportedCount = parsedVerdict.success ? parsedVerdict.data.verdicts.filter((v) => v.supported).length : 0;
+  const score = parsedVerdict.success && parsedVerdict.data.verdicts.length > 0 ? supportedCount / parsedVerdict.data.verdicts.length : 0;
+  return {
+    score,
+    statements: parsedVerdict.success ? parsedVerdict.data.verdicts : []
+  };
+}
+function cleanJSON(raw) {
+  return raw.replace(/```json|```/g, "").trim();
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  faithfulness
+});

package/dist/index.d.cts

@@ -0,0 +1,21 @@
+interface LLMconfig {
+    provider: "openai";
+    model: string;
+    apiKey: string;
+    temperature: number;
+    maxTokens: number;
+}
+
+declare function faithfulness(input: {
+    question: string;
+    answer: string;
+    context: string[];
+}, llm: LLMconfig): Promise<{
+    score: number;
+    statements: {
+        statement: string;
+        supported: boolean;
+    }[];
+}>;
+
+export { type LLMconfig, faithfulness };

package/dist/index.d.ts

@@ -0,0 +1,21 @@
+interface LLMconfig {
+    provider: "openai";
+    model: string;
+    apiKey: string;
+    temperature: number;
+    maxTokens: number;
+}
+
+declare function faithfulness(input: {
+    question: string;
+    answer: string;
+    context: string[];
+}, llm: LLMconfig): Promise<{
+    score: number;
+    statements: {
+        statement: string;
+        supported: boolean;
+    }[];
+}>;
+
+export { type LLMconfig, faithfulness };

package/dist/index.js

@@ -0,0 +1,66 @@
+// src/metrics/faithfulness.ts
+import { z } from "zod";
+
+// src/llmClient.ts
+import { OpenAI } from "openai/client";
+async function LLMCall(config, prompt) {
+  const client = new OpenAI({
+    apiKey: config.apiKey
+  });
+  const res = await client.chat.completions.create({
+    messages: [
+      {
+        role: "user",
+        content: prompt
+      }
+    ],
+    model: config.model,
+    temperature: config.temperature,
+    max_tokens: config.maxTokens
+  });
+  return res.choices[0]?.message?.content?.toString() ?? Promise.reject("No response from LLM");
+}
+
+// src/metrics/faithfulness.ts
+var statementSchema = z.object({
+  statements: z.array(z.string())
+});
+var verdictSchema = z.object({
+  verdicts: z.array(
+    z.object({
+      statement: z.string(),
+      supported: z.boolean()
+    })
+  )
+});
+async function faithfulness(input, llm) {
+  const extractionPrompt = `Given a question and an answer, break the answer into short, atomic factual statements. Return ONLY valid JSON: {"statements": ["...", "..."]}.
+
+Question: ${input.question}
+Answer: ${input.answer}`;
+  const rawStatements = await LLMCall(llm, extractionPrompt);
+  const parsed = statementSchema.safeParse(
+    JSON.parse(cleanJSON(rawStatements))
+  );
+  const verdictPrompt = `Given a context and a list of statements, judge for each statement whether it is directly supported by the context. Return ONLY valid JSON: {"verdicts": [{"statement": "...", "supported": true}]}.
+
+Context:
+${input.context.join("\n")}
+
+Statements:
+${parsed.success ? parsed.data.statements.map((s, i) => `${i + 1}. ${s}`).join("\n") : ""}`;
+  const verdict = await LLMCall(llm, verdictPrompt);
+  const parsedVerdict = verdictSchema.safeParse(JSON.parse(cleanJSON(verdict)));
+  const supportedCount = parsedVerdict.success ? parsedVerdict.data.verdicts.filter((v) => v.supported).length : 0;
+  const score = parsedVerdict.success && parsedVerdict.data.verdicts.length > 0 ? supportedCount / parsedVerdict.data.verdicts.length : 0;
+  return {
+    score,
+    statements: parsedVerdict.success ? parsedVerdict.data.verdicts : []
+  };
+}
+function cleanJSON(raw) {
+  return raw.replace(/```json|```/g, "").trim();
+}
+export {
+  faithfulness
+};

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "ragas-js",
+  "version": "0.1.0",
+  "description": "LLM output evaluation metrics for Node.js/TypeScript",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm,cjs --dts",
+    "test": "vitest run"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pankajpatel19/ragas-js.git"
+  },
+  "keywords": [
+    "LLM",
+    "evaluation",
+    "metrics",
+    "Node.js",
+    "TypeScript",
+    "ragas",
+    "AI"
+  ],
+  "author": "Pankaj Patel <pankajpatelinter@gmail.com>",
+  "license": "ISC",
+  "type": "module",
+  "engines": {
+    "node": ">=18"
+  },
+  "files": [
+    "dist"
+  ],
+  "bugs": {
+    "url": "https://github.com/pankajpatel19/ragas-js/issues"
+  },
+  "homepage": "https://github.com/pankajpatel19/ragas-js#readme",
+  "devDependencies": {
+    "@types/node": "^22.20.2",
+    "tsup": "^8.5.1",
+    "typescript": "5.9.3",
+    "vitest": "^5.0.0"
+  },
+  "dependencies": {
+    "openai": "^7.15.0",
+    "user": "^0.0.0",
+    "zod": "^4.6.4"
+  }
+}