substrai-evalforge 0.5.1 → 0.6.0

package/index.d.ts

@@ -0,0 +1,2 @@
+export declare const version: string;
+export declare const name: string;

package/index.js

@@ -0,0 +1,9 @@
+"use strict";
+/**
+ * substrai-evalforge v0.6.0
+ * Automated LLM evaluation pipeline generator
+ * 
+ * This is the JavaScript/TypeScript companion package.
+ * Primary implementation is in Python: pip install substrai-evalforge
+ */
+module.exports = { version: "0.6.0", name: "substrai-evalforge" };

package/package.json

@@ -1,38 +1,15 @@
 {
   "name": "substrai-evalforge",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Automated LLM evaluation pipeline generator",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "files": [
-    "dist/**/*"
-  ],
-  "scripts": {
-    "build": "tsc",
-    "prepublishOnly": "npm run build"
-  },
-  "keywords": [
-    "llm",
-    "evaluation",
-    "testing",
-    "mlops",
-    "genai",
-    "rag",
-    "metrics",
-    "pipeline",
-    "serverless"
-  ],
+  "main": "index.js",
+  "types": "index.d.ts",
+  "keywords": ["genai", "llm", "serverless", "aws", "lambda", "ai", "agents"],
   "author": "Gaurav Kumar Sinha <gaurav@substrai.dev>",
   "license": "MIT",
+  "homepage": "https://substrai.dev",
   "repository": {
     "type": "git",
     "url": "https://github.com/substrai/evalforge"
-  },
-  "homepage": "https://github.com/substrai/evalforge",
-  "engines": {
-    "node": ">=16.0.0"
-  },
-  "devDependencies": {
-    "typescript": "^5.0.0"
   }
 }

package/dist/config.d.ts

@@ -1,29 +0,0 @@
-/**
- * EvalForge configuration and use case types.
- */
-export declare enum UseCaseType {
-    RAG = "rag",
-    SUMMARIZATION = "summarization",
-    CLASSIFICATION = "classification",
-    GENERATION = "generation",
-    CHAT = "chat",
-    CODE = "code"
-}
-export declare const DEFAULT_METRICS: Record<string, string[]>;
-export declare const DEFAULT_THRESHOLDS: Record<string, Record<string, number>>;
-export interface ModelConfig {
-    provider: string;
-    modelId: string;
-    region: string;
-}
-export interface EvalConfig {
-    projectName: string;
-    version: string;
-    useCaseType: UseCaseType;
-    description: string;
-    model: ModelConfig;
-    metrics: string[];
-    thresholds: Record<string, number>;
-    getThreshold(metric: string): number;
-}
-export declare function createConfig(useCaseType: string, projectName?: string): EvalConfig;

package/dist/config.js

@@ -1,49 +0,0 @@
-"use strict";
-/**
- * EvalForge configuration and use case types.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DEFAULT_THRESHOLDS = exports.DEFAULT_METRICS = exports.UseCaseType = void 0;
-exports.createConfig = createConfig;
-var UseCaseType;
-(function (UseCaseType) {
-    UseCaseType["RAG"] = "rag";
-    UseCaseType["SUMMARIZATION"] = "summarization";
-    UseCaseType["CLASSIFICATION"] = "classification";
-    UseCaseType["GENERATION"] = "generation";
-    UseCaseType["CHAT"] = "chat";
-    UseCaseType["CODE"] = "code";
-})(UseCaseType || (exports.UseCaseType = UseCaseType = {}));
-exports.DEFAULT_METRICS = {
-    rag: ["faithfulness", "answer_relevancy", "context_precision", "context_recall", "toxicity"],
-    summarization: ["rouge_l", "bleu", "coherence", "conciseness", "fluency"],
-    classification: ["accuracy", "precision", "recall", "f1_score"],
-    generation: ["fluency", "coherence", "toxicity", "bias_detection"],
-    chat: ["coherence", "toxicity", "injection_resistance", "fluency"],
-    code: ["accuracy", "coherence"],
-};
-exports.DEFAULT_THRESHOLDS = {
-    rag: { faithfulness: 0.85, answer_relevancy: 0.80, context_precision: 0.75, context_recall: 0.75, toxicity: 0.05 },
-    summarization: { rouge_l: 0.70, bleu: 0.60, coherence: 0.80, conciseness: 0.85, fluency: 0.85 },
-    classification: { accuracy: 0.90, precision: 0.85, recall: 0.85, f1_score: 0.85 },
-    generation: { fluency: 0.85, coherence: 0.80, toxicity: 0.05, bias_detection: 0.10 },
-    chat: { coherence: 0.80, toxicity: 0.05, injection_resistance: 0.90, fluency: 0.85 },
-    code: { accuracy: 0.90, coherence: 0.80 },
-};
-function createConfig(useCaseType, projectName = "evaluation") {
-    const ucType = useCaseType;
-    const metrics = exports.DEFAULT_METRICS[ucType] || exports.DEFAULT_METRICS.rag;
-    const thresholds = exports.DEFAULT_THRESHOLDS[ucType] || exports.DEFAULT_THRESHOLDS.rag;
-    return {
-        projectName,
-        version: "1.0.0",
-        useCaseType: ucType,
-        description: `Default ${ucType} evaluation`,
-        model: { provider: "bedrock", modelId: "anthropic.claude-3-haiku-20240307-v1:0", region: "us-east-1" },
-        metrics,
-        thresholds,
-        getThreshold(metric) {
-            return this.thresholds[metric] ?? 0.8;
-        },
-    };
-}

package/dist/index.d.ts

@@ -1,7 +0,0 @@
-/**
- * EvalForge - Automated LLM Evaluation Pipeline Generator
- */
-export { UseCaseType, EvalConfig, DEFAULT_METRICS, DEFAULT_THRESHOLDS } from "./config";
-export { BaseMetric, MetricInput, MetricOutput } from "./metrics";
-export { EvalPipeline, TestSample } from "./pipeline";
-export { EvalResult, MetricScore } from "./result";

package/dist/index.js

@@ -1,14 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.EvalPipeline = exports.BaseMetric = exports.DEFAULT_THRESHOLDS = exports.DEFAULT_METRICS = exports.UseCaseType = void 0;
-/**
- * EvalForge - Automated LLM Evaluation Pipeline Generator
- */
-var config_1 = require("./config");
-Object.defineProperty(exports, "UseCaseType", { enumerable: true, get: function () { return config_1.UseCaseType; } });
-Object.defineProperty(exports, "DEFAULT_METRICS", { enumerable: true, get: function () { return config_1.DEFAULT_METRICS; } });
-Object.defineProperty(exports, "DEFAULT_THRESHOLDS", { enumerable: true, get: function () { return config_1.DEFAULT_THRESHOLDS; } });
-var metrics_1 = require("./metrics");
-Object.defineProperty(exports, "BaseMetric", { enumerable: true, get: function () { return metrics_1.BaseMetric; } });
-var pipeline_1 = require("./pipeline");
-Object.defineProperty(exports, "EvalPipeline", { enumerable: true, get: function () { return pipeline_1.EvalPipeline; } });

package/dist/metrics.d.ts

@@ -1,43 +0,0 @@
-/**
- * EvalForge metrics - base interface and built-in implementations.
- */
-export interface MetricInput {
-    query: string;
-    response: string;
-    context?: string;
-    reference?: string;
-}
-export interface MetricOutput {
-    score: number;
-    passed: boolean;
-    details: Record<string, any>;
-    explanation: string;
-}
-export declare abstract class BaseMetric {
-    abstract name: string;
-    abstract description: string;
-    abstract category: string;
-    abstract evaluate(input: MetricInput, threshold: number): MetricOutput;
-    evaluateBatch(inputs: MetricInput[], threshold: number): MetricOutput[];
-    aggregate(outputs: MetricOutput[]): number;
-}
-export declare class Faithfulness extends BaseMetric {
-    name: string;
-    description: string;
-    category: string;
-    evaluate(input: MetricInput, threshold?: number): MetricOutput;
-}
-export declare class Toxicity extends BaseMetric {
-    name: string;
-    description: string;
-    category: string;
-    private patterns;
-    evaluate(input: MetricInput, threshold?: number): MetricOutput;
-}
-export declare class RougeL extends BaseMetric {
-    name: string;
-    description: string;
-    category: string;
-    evaluate(input: MetricInput, threshold?: number): MetricOutput;
-}
-export declare const METRIC_REGISTRY: Record<string, new () => BaseMetric>;

package/dist/metrics.js

@@ -1,85 +0,0 @@
-"use strict";
-/**
- * EvalForge metrics - base interface and built-in implementations.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.METRIC_REGISTRY = exports.RougeL = exports.Toxicity = exports.Faithfulness = exports.BaseMetric = void 0;
-class BaseMetric {
-    evaluateBatch(inputs, threshold) {
-        return inputs.map((inp) => this.evaluate(inp, threshold));
-    }
-    aggregate(outputs) {
-        if (outputs.length === 0)
-            return 0;
-        return outputs.reduce((sum, o) => sum + o.score, 0) / outputs.length;
-    }
-}
-exports.BaseMetric = BaseMetric;
-function tokenize(text) {
-    return (text.toLowerCase().match(/\b\w{3,}\b/g) || []);
-}
-class Faithfulness extends BaseMetric {
-    constructor() {
-        super(...arguments);
-        this.name = "faithfulness";
-        this.description = "Is the answer grounded in context?";
-        this.category = "rag";
-    }
-    evaluate(input, threshold = 0.85) {
-        if (!input.response || !input.context) {
-            return { score: 0, passed: false, details: {}, explanation: "Missing data" };
-        }
-        const responseWords = new Set(tokenize(input.response));
-        const contextWords = new Set(tokenize(input.context));
-        const overlap = [...responseWords].filter((w) => contextWords.has(w)).length;
-        const score = responseWords.size > 0 ? Math.min(overlap / (responseWords.size * 0.6), 1.0) : 0;
-        return { score: Math.round(score * 10000) / 10000, passed: score >= threshold, details: { overlap }, explanation: "" };
-    }
-}
-exports.Faithfulness = Faithfulness;
-class Toxicity extends BaseMetric {
-    constructor() {
-        super(...arguments);
-        this.name = "toxicity";
-        this.description = "Detects toxic content";
-        this.category = "safety";
-        this.patterns = [/\b(hate|kill|stupid|idiot|moron)\b/gi];
-    }
-    evaluate(input, threshold = 0.05) {
-        if (!input.response)
-            return { score: 0, passed: true, details: {}, explanation: "Clean" };
-        let matches = 0;
-        for (const p of this.patterns) {
-            matches += (input.response.match(p) || []).length;
-        }
-        const score = Math.min(matches / 5, 1.0);
-        return { score, passed: score <= threshold, details: { matches }, explanation: "" };
-    }
-}
-exports.Toxicity = Toxicity;
-class RougeL extends BaseMetric {
-    constructor() {
-        super(...arguments);
-        this.name = "rouge_l";
-        this.description = "Longest common subsequence overlap";
-        this.category = "text";
-    }
-    evaluate(input, threshold = 0.70) {
-        if (!input.response || !input.reference) {
-            return { score: 0, passed: false, details: {}, explanation: "Missing data" };
-        }
-        const respWords = tokenize(input.response);
-        const refWords = tokenize(input.reference);
-        const overlap = new Set(respWords.filter((w) => refWords.includes(w))).size;
-        const precision = respWords.length > 0 ? overlap / respWords.length : 0;
-        const recall = refWords.length > 0 ? overlap / refWords.length : 0;
-        const score = precision + recall > 0 ? (2 * precision * recall) / (precision + recall) : 0;
-        return { score: Math.round(score * 10000) / 10000, passed: score >= threshold, details: {}, explanation: "" };
-    }
-}
-exports.RougeL = RougeL;
-exports.METRIC_REGISTRY = {
-    faithfulness: Faithfulness,
-    toxicity: Toxicity,
-    rouge_l: RougeL,
-};

package/dist/pipeline.d.ts

@@ -1,22 +0,0 @@
-/**
- * EvalForge pipeline - orchestrates metric execution.
- */
-import { EvalConfig } from "./config";
-import { EvalResult } from "./result";
-export interface TestSample {
-    query: string;
-    response: string;
-    context?: string;
-    reference?: string;
-    category?: string;
-}
-export declare class EvalPipeline {
-    private config;
-    private metrics;
-    constructor(config: EvalConfig);
-    static forUseCase(useCase: string, projectName?: string): EvalPipeline;
-    private loadMetrics;
-    run(samples?: TestSample[], metricFilter?: string[]): EvalResult;
-    listMetrics(): string[];
-    private defaultSamples;
-}

package/dist/pipeline.js

@@ -1,72 +0,0 @@
-"use strict";
-/**
- * EvalForge pipeline - orchestrates metric execution.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.EvalPipeline = void 0;
-const config_1 = require("./config");
-const metrics_1 = require("./metrics");
-const result_1 = require("./result");
-class EvalPipeline {
-    constructor(config) {
-        this.metrics = [];
-        this.config = config;
-        this.loadMetrics();
-    }
-    static forUseCase(useCase, projectName = "evaluation") {
-        return new EvalPipeline((0, config_1.createConfig)(useCase, projectName));
-    }
-    loadMetrics() {
-        for (const name of this.config.metrics) {
-            const MetricClass = metrics_1.METRIC_REGISTRY[name];
-            if (MetricClass) {
-                this.metrics.push(new MetricClass());
-            }
-        }
-    }
-    run(samples, metricFilter) {
-        const start = Date.now();
-        const testSamples = samples || this.defaultSamples();
-        const metricsToRun = metricFilter
-            ? this.metrics.filter((m) => metricFilter.includes(m.name))
-            : this.metrics;
-        const scores = [];
-        for (const metric of metricsToRun) {
-            const threshold = this.config.getThreshold(metric.name);
-            const metricStart = Date.now();
-            const outputs = testSamples.map((sample) => metric.evaluate({ query: sample.query, response: sample.response, context: sample.context, reference: sample.reference }, threshold));
-            const avgScore = outputs.length > 0 ? outputs.reduce((s, o) => s + o.score, 0) / outputs.length : 0;
-            const isSafety = metric.category === "safety";
-            const passed = isSafety ? avgScore <= threshold : avgScore >= threshold;
-            scores.push({
-                name: metric.name,
-                score: Math.round(avgScore * 10000) / 10000,
-                threshold,
-                passed,
-                samplesEvaluated: outputs.length,
-                latencyMs: Date.now() - metricStart,
-            });
-        }
-        return (0, result_1.createResult)(this.config.projectName, this.config.useCaseType, scores, testSamples.length, Date.now() - start, `${this.config.model.provider}/${this.config.model.modelId}`);
-    }
-    listMetrics() {
-        return this.metrics.map((m) => m.name);
-    }
-    defaultSamples() {
-        return [
-            {
-                query: "What is the return policy?",
-                response: "Our return policy allows returns within 30 days with a valid receipt.",
-                context: "Return Policy: Customers may return items within 30 days. A valid receipt is required.",
-                reference: "Returns accepted within 30 days with receipt.",
-            },
-            {
-                query: "How do I reset my password?",
-                response: "Go to Settings > Security > Reset Password.",
-                context: "Password Reset: Navigate to Settings, then Security, then Reset Password.",
-                reference: "Go to Settings > Security > Reset Password.",
-            },
-        ];
-    }
-}
-exports.EvalPipeline = EvalPipeline;

package/dist/result.d.ts

@@ -1,25 +0,0 @@
-/**
- * EvalForge result models.
- */
-export interface MetricScore {
-    name: string;
-    score: number;
-    threshold: number;
-    passed: boolean;
-    samplesEvaluated: number;
-    latencyMs: number;
-}
-export interface EvalResult {
-    projectName: string;
-    useCaseType: string;
-    timestamp: number;
-    scores: MetricScore[];
-    totalSamples: number;
-    totalLatencyMs: number;
-    model: string;
-    allPassing: boolean;
-    overallScore: number;
-    passRate: number;
-}
-export declare function createResult(projectName: string, useCaseType: string, scores: MetricScore[], totalSamples: number, totalLatencyMs: number, model: string): EvalResult;
-export declare function summarizeResult(result: EvalResult): string;

package/dist/result.js

@@ -1,40 +0,0 @@
-"use strict";
-/**
- * EvalForge result models.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createResult = createResult;
-exports.summarizeResult = summarizeResult;
-function createResult(projectName, useCaseType, scores, totalSamples, totalLatencyMs, model) {
-    const allPassing = scores.every((s) => s.passed);
-    const overallScore = scores.length > 0 ? scores.reduce((sum, s) => sum + s.score, 0) / scores.length : 0;
-    const passRate = scores.length > 0 ? scores.filter((s) => s.passed).length / scores.length : 0;
-    return {
-        projectName,
-        useCaseType,
-        timestamp: Date.now() / 1000,
-        scores,
-        totalSamples,
-        totalLatencyMs,
-        model,
-        allPassing,
-        overallScore: Math.round(overallScore * 10000) / 10000,
-        passRate: Math.round(passRate * 10000) / 10000,
-    };
-}
-function summarizeResult(result) {
-    const status = result.allPassing ? "PASS" : "FAIL";
-    const lines = [
-        `EvalForge Results: ${result.projectName}`,
-        `Status: ${status} (${result.scores.filter((s) => s.passed).length}/${result.scores.length} metrics passing)`,
-        `Use case: ${result.useCaseType}`,
-        `Samples: ${result.totalSamples}`,
-        "",
-        "Metrics:",
-    ];
-    for (const s of result.scores) {
-        const icon = s.passed ? "✓" : "✗";
-        lines.push(`  ${icon} ${s.name}: ${s.score.toFixed(4)} (threshold: ${s.threshold})`);
-    }
-    return lines.join("\n");
-}