praisonai 1.5.3 → 1.5.4

package/dist/cli/commands/eval.d.ts

@@ -7,6 +7,8 @@ export interface EvalOptions {
     iterations?: number;
     warmup?: number;
     'expected-tools'?: string;
+    criteria?: string;
+    threshold?: number;
     model?: string;
     verbose?: boolean;
     profile?: string;

package/dist/cli/commands/eval.js

@@ -45,12 +45,12 @@ const cli_spec_1 = require("../spec/cli-spec");
 const errors_1 = require("../output/errors");
 async function execute(args, options) {
     const subcommand = args[0];
-    if (!subcommand || !['accuracy', 'performance', 'reliability'].includes(subcommand)) {
+    if (!subcommand || !['accuracy', 'performance', 'reliability', 'judge'].includes(subcommand)) {
         if (options.json || options.output === 'json') {
-            (0, json_1.printError)(errors_1.ERROR_CODES.INVALID_ARGS, 'Please specify a subcommand: accuracy, performance, or reliability');
+            (0, json_1.printError)(errors_1.ERROR_CODES.INVALID_ARGS, 'Please specify a subcommand: accuracy, performance, reliability, or judge');
         }
         else {
-            await pretty.error('Please specify a subcommand: accuracy, performance, or reliability');
+            await pretty.error('Please specify a subcommand: accuracy, performance, reliability, or judge');
         }
         process.exit(cli_spec_1.EXIT_CODES.INVALID_ARGUMENTS);
     }
@@ -72,6 +72,9 @@ async function execute(args, options) {
             case 'reliability':
                 await runReliabilityEval(options, config, outputFormat);
                 break;
+            case 'judge':
+                await runJudgeEval(options, config, outputFormat);
+                break;
         }
     }
     catch (error) {
@@ -245,3 +248,55 @@ function calculateSimilarity(str1, str2) {
     const union = new Set([...words1, ...words2]);
     return intersection.length / union.size;
 }
+/**
+ * Run LLM-as-Judge evaluation
+ */
+async function runJudgeEval(options, config, outputFormat) {
+    // Lazy import to avoid performance impact
+    const { Judge } = await Promise.resolve().then(() => __importStar(require('../../eval/judge')));
+    const output = options.input;
+    if (!output) {
+        throw new Error('--input is required for judge evaluation (the output to judge)');
+    }
+    const startTime = Date.now();
+    const threshold = options.threshold ?? 7.0;
+    const judge = new Judge({
+        model: config.model,
+        threshold,
+        criteria: options.criteria,
+    });
+    const result = await judge.run({
+        output,
+        expected: options.expected,
+        criteria: options.criteria,
+    });
+    const duration = Date.now() - startTime;
+    if (outputFormat === 'json') {
+        (0, json_1.outputJson)((0, json_1.formatSuccess)({
+            type: 'judge',
+            threshold,
+            result: {
+                score: result.score,
+                passed: result.passed,
+                reasoning: result.reasoning,
+                suggestions: result.suggestions,
+            }
+        }, { duration_ms: duration, model: config.model }));
+    }
+    else {
+        await pretty.heading('LLM-as-Judge Evaluation Results');
+        await pretty.keyValue({
+            'Score': `${result.score.toFixed(1)}/10`,
+            'Status': result.passed ? '✅ PASSED' : '❌ FAILED',
+            'Threshold': threshold,
+            'Reasoning': result.reasoning,
+            'Duration': `${duration}ms`
+        });
+        if (result.suggestions.length > 0) {
+            await pretty.heading('Suggestions');
+            for (const suggestion of result.suggestions) {
+                console.log(`  • ${suggestion}`);
+            }
+        }
+    }
+}

package/dist/eval/index.d.ts

@@ -61,3 +61,4 @@ export declare class EvalSuite {
 }
 export { Evaluator, createEvaluator, createDefaultEvaluator, relevanceCriterion, lengthCriterion, containsKeywordsCriterion, noHarmfulContentCriterion, type EvalCriteria, type EvalResult as BaseEvalResult, type EvalSummary, type EvaluatorConfig, } from './base';
 export { EvalResults, createEvalResults, type TestResult, type AggregatedResults, type TrendPoint, } from './results';
+export { Judge, AccuracyJudge, CriteriaJudge, RecipeJudge, addJudge, getJudge, listJudges, removeJudge, addOptimizationRule, getOptimizationRule, listOptimizationRules, removeOptimizationRule, parseJudgeResponse, type JudgeConfig, type JudgeCriteriaConfig, type JudgeResult, type JudgeRunOptions, type JudgeOptions, type JudgeProtocol, } from './judge';

package/dist/eval/index.js

@@ -3,7 +3,7 @@
  * Evaluation Framework - Accuracy, Performance, and Reliability evaluation
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createEvalResults = exports.EvalResults = exports.noHarmfulContentCriterion = exports.containsKeywordsCriterion = exports.lengthCriterion = exports.relevanceCriterion = exports.createDefaultEvaluator = exports.createEvaluator = exports.Evaluator = exports.EvalSuite = void 0;
+exports.parseJudgeResponse = exports.removeOptimizationRule = exports.listOptimizationRules = exports.getOptimizationRule = exports.addOptimizationRule = exports.removeJudge = exports.listJudges = exports.getJudge = exports.addJudge = exports.RecipeJudge = exports.CriteriaJudge = exports.AccuracyJudge = exports.Judge = exports.createEvalResults = exports.EvalResults = exports.noHarmfulContentCriterion = exports.containsKeywordsCriterion = exports.lengthCriterion = exports.relevanceCriterion = exports.createDefaultEvaluator = exports.createEvaluator = exports.Evaluator = exports.EvalSuite = void 0;
 exports.accuracyEval = accuracyEval;
 exports.performanceEval = performanceEval;
 exports.reliabilityEval = reliabilityEval;
@@ -168,3 +168,18 @@ Object.defineProperty(exports, "noHarmfulContentCriterion", { enumerable: true,
 var results_1 = require("./results");
 Object.defineProperty(exports, "EvalResults", { enumerable: true, get: function () { return results_1.EvalResults; } });
 Object.defineProperty(exports, "createEvalResults", { enumerable: true, get: function () { return results_1.createEvalResults; } });
+// Re-export Judge (LLM-as-Judge)
+var judge_1 = require("./judge");
+Object.defineProperty(exports, "Judge", { enumerable: true, get: function () { return judge_1.Judge; } });
+Object.defineProperty(exports, "AccuracyJudge", { enumerable: true, get: function () { return judge_1.AccuracyJudge; } });
+Object.defineProperty(exports, "CriteriaJudge", { enumerable: true, get: function () { return judge_1.CriteriaJudge; } });
+Object.defineProperty(exports, "RecipeJudge", { enumerable: true, get: function () { return judge_1.RecipeJudge; } });
+Object.defineProperty(exports, "addJudge", { enumerable: true, get: function () { return judge_1.addJudge; } });
+Object.defineProperty(exports, "getJudge", { enumerable: true, get: function () { return judge_1.getJudge; } });
+Object.defineProperty(exports, "listJudges", { enumerable: true, get: function () { return judge_1.listJudges; } });
+Object.defineProperty(exports, "removeJudge", { enumerable: true, get: function () { return judge_1.removeJudge; } });
+Object.defineProperty(exports, "addOptimizationRule", { enumerable: true, get: function () { return judge_1.addOptimizationRule; } });
+Object.defineProperty(exports, "getOptimizationRule", { enumerable: true, get: function () { return judge_1.getOptimizationRule; } });
+Object.defineProperty(exports, "listOptimizationRules", { enumerable: true, get: function () { return judge_1.listOptimizationRules; } });
+Object.defineProperty(exports, "removeOptimizationRule", { enumerable: true, get: function () { return judge_1.removeOptimizationRule; } });
+Object.defineProperty(exports, "parseJudgeResponse", { enumerable: true, get: function () { return judge_1.parseJudgeResponse; } });

package/dist/eval/judge.d.ts

@@ -0,0 +1,275 @@
+/**
+ * Unified Judge class for LLM-as-judge evaluation.
+ *
+ * Provides a simple, unified API for evaluating agent outputs using LLM-as-judge.
+ * Follows PraisonAI naming conventions and engineering principles.
+ *
+ * DRY: Reuses existing provider infrastructure.
+ * Protocol-driven: Implements JudgeProtocol for extensibility.
+ * Zero performance impact: Lazy imports for LLM providers.
+ *
+ * @example
+ * ```typescript
+ * import { Judge } from 'praisonai';
+ * const result = await Judge.run({ output: "4", expected: "4" });
+ * console.log(`Score: ${result.score}/10`);
+ * ```
+ */
+/**
+ * Configuration for Judge instances.
+ */
+export interface JudgeConfig {
+    /** LLM model to use for judging (default: gpt-4o-mini) */
+    model?: string;
+    /** Temperature for LLM calls (default: 0.1 for consistency) */
+    temperature?: number;
+    /** Maximum tokens for LLM response */
+    maxTokens?: number;
+    /** Score threshold for passing (default: 7.0) */
+    threshold?: number;
+    /** Optional custom criteria for evaluation */
+    criteria?: string;
+}
+/**
+ * Dynamic criteria configuration for domain-agnostic judging.
+ *
+ * Enables judges to evaluate ANY domain, not just agent outputs:
+ * - Water flow optimization
+ * - Data pipeline efficiency
+ * - Manufacturing quality
+ * - Recipe/workflow optimization
+ * - Any custom domain
+ */
+export interface JudgeCriteriaConfig {
+    /** Name of the criteria configuration */
+    name: string;
+    /** Description of what is being evaluated */
+    description: string;
+    /** Custom prompt template with {output} placeholder */
+    promptTemplate: string;
+    /** List of dimensions to score (e.g., ["efficiency", "safety"]) */
+    scoringDimensions: string[];
+    /** Score threshold for passing (default: 7.0) */
+    threshold?: number;
+}
+/**
+ * Result from a Judge evaluation.
+ *
+ * This is the unified result type for all LLM-as-judge evaluations.
+ */
+export interface JudgeResult {
+    /** Quality score (1-10) */
+    score: number;
+    /** Whether the evaluation passed (score >= threshold) */
+    passed: boolean;
+    /** Explanation for the score */
+    reasoning: string;
+    /** The output that was judged */
+    output: string;
+    /** Optional expected output */
+    expected?: string;
+    /** Optional criteria used for evaluation */
+    criteria?: string;
+    /** List of improvement suggestions */
+    suggestions: string[];
+    /** When judging occurred */
+    timestamp: number;
+    /** Additional metadata */
+    metadata?: Record<string, any>;
+}
+/**
+ * Options for Judge.run() method
+ */
+export interface JudgeRunOptions {
+    /** The output to judge (required if no agent) */
+    output?: string;
+    /** Optional expected output for accuracy evaluation */
+    expected?: string;
+    /** Optional criteria for criteria evaluation */
+    criteria?: string;
+    /** Optional input context */
+    input?: string;
+    /** Optional Agent to run and judge */
+    agent?: any;
+    /** Optional Agents to run and judge */
+    agents?: any;
+    /** Whether to print result summary */
+    printSummary?: boolean;
+}
+/**
+ * Constructor options for Judge class
+ */
+export interface JudgeOptions {
+    /** LLM model to use */
+    model?: string;
+    /** Temperature for LLM calls */
+    temperature?: number;
+    /** Maximum tokens for response */
+    maxTokens?: number;
+    /** Score threshold for passing */
+    threshold?: number;
+    /** Custom criteria for evaluation */
+    criteria?: string;
+    /** Full JudgeConfig object */
+    config?: JudgeConfig;
+    /** Domain-agnostic criteria config */
+    criteriaConfig?: JudgeCriteriaConfig;
+    /** Session ID for trace isolation */
+    sessionId?: string;
+}
+/**
+ * Protocol interface for Judge implementations
+ */
+export interface JudgeProtocol {
+    run(options: JudgeRunOptions): Promise<JudgeResult>;
+    runAsync(options: JudgeRunOptions): Promise<JudgeResult>;
+}
+/**
+ * Parse LLM response into JudgeResult.
+ *
+ * @param responseText - Raw LLM response
+ * @param output - Original output
+ * @param expected - Original expected output
+ * @param criteria - Original criteria
+ * @param threshold - Score threshold for passing
+ * @returns JudgeResult with score, passed, reasoning, suggestions
+ */
+export declare function parseJudgeResponse(responseText: string, output: string, expected: string | null, criteria: string | null, threshold: number): JudgeResult;
+/**
+ * Unified LLM-as-judge for evaluating agent outputs.
+ *
+ * Provides a simple API for:
+ * - Accuracy evaluation (comparing output to expected)
+ * - Criteria evaluation (evaluating against custom criteria)
+ * - Custom evaluation (subclass for domain-specific judges)
+ *
+ * @example
+ * ```typescript
+ * // Simple accuracy check
+ * const result = await new Judge().run({ output: "4", expected: "4" });
+ *
+ * // Custom criteria
+ * const result = await new Judge({ criteria: "Response is helpful" }).run({ output: "Hello!" });
+ *
+ * // With agent
+ * const result = await new Judge().run({ agent: myAgent, input: "2+2", expected: "4" });
+ * ```
+ */
+export declare class Judge implements JudgeProtocol {
+    readonly model: string;
+    readonly temperature: number;
+    readonly maxTokens: number;
+    readonly threshold: number;
+    readonly criteria: string | null;
+    readonly criteriaConfig: JudgeCriteriaConfig | null;
+    readonly sessionId: string | null;
+    constructor(options?: JudgeOptions);
+    /**
+     * Build the appropriate prompt based on evaluation type.
+     */
+    protected buildPrompt(output: string, expected: string | null, criteria: string | null, input: string): string;
+    /**
+     * Get LLM provider lazily.
+     */
+    protected getProvider(): Promise<any>;
+    /**
+     * Get output from an Agent.
+     */
+    protected getAgentOutput(agent: any, input: string): Promise<string>;
+    /**
+     * Get output from Agents (multi-agent).
+     */
+    protected getAgentsOutput(agents: any, input: string): Promise<string>;
+    /**
+     * Judge an output.
+     *
+     * @param options - Evaluation options
+     * @returns JudgeResult with score, passed, reasoning, suggestions
+     */
+    run(options: JudgeRunOptions): Promise<JudgeResult>;
+    /**
+     * Judge an output asynchronously (alias for run).
+     */
+    runAsync(options: JudgeRunOptions): Promise<JudgeResult>;
+    /**
+     * Print a summary of the judge result.
+     */
+    printSummary(result: JudgeResult): void;
+}
+/**
+ * Judge for accuracy evaluation (comparing output to expected).
+ */
+export declare class AccuracyJudge extends Judge {
+}
+/**
+ * Judge for criteria-based evaluation.
+ */
+export declare class CriteriaJudge extends Judge {
+}
+/**
+ * Judge for evaluating recipe/workflow execution traces.
+ */
+export declare class RecipeJudge extends Judge {
+    readonly mode: string;
+    constructor(options?: JudgeOptions & {
+        mode?: string;
+    });
+    protected buildPrompt(output: string, expected: string | null, criteria: string | null, input: string): string;
+}
+type JudgeConstructor = new (options?: JudgeOptions) => Judge;
+/**
+ * Register a custom judge type.
+ *
+ * @param name - Name for the judge type
+ * @param judgeClass - Judge class to register
+ */
+export declare function addJudge(name: string, judgeClass: JudgeConstructor): void;
+/**
+ * Get a registered judge type by name.
+ *
+ * @param name - Name of the judge type
+ * @returns Judge class or undefined if not found
+ */
+export declare function getJudge(name: string): JudgeConstructor | undefined;
+/**
+ * List all registered judge types.
+ *
+ * @returns List of judge type names
+ */
+export declare function listJudges(): string[];
+/**
+ * Remove a registered judge type.
+ *
+ * @param name - Name of the judge type to remove
+ * @returns True if removed, false if not found
+ */
+export declare function removeJudge(name: string): boolean;
+type OptimizationRuleConstructor = new (...args: any[]) => any;
+/**
+ * Register a custom optimization rule.
+ *
+ * @param name - Name for the rule
+ * @param ruleClass - Rule class implementing OptimizationRuleProtocol
+ */
+export declare function addOptimizationRule(name: string, ruleClass: OptimizationRuleConstructor): void;
+/**
+ * Get a registered optimization rule by name.
+ *
+ * @param name - Name of the rule
+ * @returns Rule class or undefined if not found
+ */
+export declare function getOptimizationRule(name: string): OptimizationRuleConstructor | undefined;
+/**
+ * List all registered optimization rules.
+ *
+ * @returns List of rule names
+ */
+export declare function listOptimizationRules(): string[];
+/**
+ * Remove a registered optimization rule.
+ *
+ * @param name - Name of the rule to remove
+ * @returns True if removed, false if not found
+ */
+export declare function removeOptimizationRule(name: string): boolean;
+export default Judge;

package/dist/eval/judge.js

@@ -0,0 +1,528 @@
+"use strict";
+/**
+ * Unified Judge class for LLM-as-judge evaluation.
+ *
+ * Provides a simple, unified API for evaluating agent outputs using LLM-as-judge.
+ * Follows PraisonAI naming conventions and engineering principles.
+ *
+ * DRY: Reuses existing provider infrastructure.
+ * Protocol-driven: Implements JudgeProtocol for extensibility.
+ * Zero performance impact: Lazy imports for LLM providers.
+ *
+ * @example
+ * ```typescript
+ * import { Judge } from 'praisonai';
+ * const result = await Judge.run({ output: "4", expected: "4" });
+ * console.log(`Score: ${result.score}/10`);
+ * ```
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecipeJudge = exports.CriteriaJudge = exports.AccuracyJudge = exports.Judge = void 0;
+exports.parseJudgeResponse = parseJudgeResponse;
+exports.addJudge = addJudge;
+exports.getJudge = getJudge;
+exports.listJudges = listJudges;
+exports.removeJudge = removeJudge;
+exports.addOptimizationRule = addOptimizationRule;
+exports.getOptimizationRule = getOptimizationRule;
+exports.listOptimizationRules = listOptimizationRules;
+exports.removeOptimizationRule = removeOptimizationRule;
+// ============================================================================
+// Prompt Templates
+// ============================================================================
+/** Default prompt for accuracy evaluation */
+const ACCURACY_PROMPT = `You are an expert evaluator. Compare the actual output against the expected output.
+
+INPUT: {input}
+
+EXPECTED OUTPUT:
+{expected}
+
+ACTUAL OUTPUT:
+{output}
+
+Scoring Guidelines:
+- 10: Perfect match in meaning and completeness
+- 8-9: Very close, minor differences that don't affect correctness
+- 6-7: Mostly correct but missing some details or has minor errors
+- 4-5: Partially correct but significant issues
+- 2-3: Mostly incorrect but shows some understanding
+- 1: Completely wrong or irrelevant
+
+Respond in this EXACT format:
+SCORE: [number 1-10]
+REASONING: [brief explanation]
+SUGGESTIONS:
+- [improvement suggestion 1]
+- [improvement suggestion 2]
+`;
+/** Default prompt for criteria evaluation */
+const CRITERIA_PROMPT = `You are an expert evaluator. Evaluate the output against the given criteria.
+
+CRITERIA: {criteria}
+
+OUTPUT TO EVALUATE:
+{output}
+
+Score the output from 1-10 based on how well it meets the criteria.
+- 10: Perfectly meets all criteria
+- 8-9: Meets criteria very well with minor issues
+- 6-7: Meets most criteria but has some gaps
+- 4-5: Partially meets criteria
+- 2-3: Barely meets criteria
+- 1: Does not meet criteria at all
+
+Respond in this EXACT format:
+SCORE: [number 1-10]
+REASONING: [brief explanation]
+SUGGESTIONS:
+- [improvement suggestion 1]
+- [improvement suggestion 2]
+`;
+/** Default prompt for general quality evaluation */
+const GENERAL_PROMPT = `You are an expert evaluator for AI agent outputs. Your task is to grade the quality of an agent's response.
+
+INPUT (what the agent was asked):
+{input}
+
+AGENT OUTPUT (what the agent responded):
+{output}
+
+GRADING CRITERIA:
+- Accuracy: Is the response factually correct?
+- Completeness: Does it fully address the input?
+- Clarity: Is it well-written and easy to understand?
+- Relevance: Does it stay on topic?
+
+SCORING GUIDELINES:
+- 10: Perfect - Excellent in all criteria
+- 8-9: Very Good - Minor improvements possible
+- 6-7: Good - Some issues but mostly correct
+- 4-5: Fair - Significant issues
+- 2-3: Poor - Major problems
+- 1: Very Poor - Completely wrong or irrelevant
+
+Respond in this EXACT format:
+SCORE: [number 1-10]
+REASONING: [brief explanation of the score]
+SUGGESTIONS:
+- [first suggestion for improvement]
+- [second suggestion if applicable]
+`;
+/** Recipe evaluation prompt */
+const RECIPE_PROMPT = `You are an expert evaluator for AI agent workflow recipes.
+
+RECIPE OUTPUT TO EVALUATE:
+{output}
+
+EXPECTED BEHAVIOR:
+{expected}
+
+EVALUATION CRITERIA:
+{criteria}
+
+Evaluate the recipe execution on:
+1. Task completion (1-10): Did agents complete their assigned tasks?
+2. Context flow (1-10): Was context properly passed between agents?
+3. Output quality (1-10): Is the final output useful and accurate?
+
+Respond in this EXACT format:
+SCORE: [average of above, 1-10]
+REASONING: [brief explanation]
+SUGGESTIONS:
+- [improvement suggestion 1]
+- [improvement suggestion 2]
+`;
+// ============================================================================
+// Response Parsing
+// ============================================================================
+/**
+ * Parse LLM response into JudgeResult.
+ *
+ * @param responseText - Raw LLM response
+ * @param output - Original output
+ * @param expected - Original expected output
+ * @param criteria - Original criteria
+ * @param threshold - Score threshold for passing
+ * @returns JudgeResult with score, passed, reasoning, suggestions
+ */
+function parseJudgeResponse(responseText, output, expected, criteria, threshold) {
+    let score = 5.0;
+    let reasoning = 'Unable to parse response';
+    const suggestions = [];
+    const lines = responseText.trim().split('\n');
+    let inSuggestions = false;
+    for (const rawLine of lines) {
+        const line = rawLine.trim();
+        if (line.startsWith('SCORE:')) {
+            try {
+                const scoreStr = line.replace('SCORE:', '').trim();
+                score = parseFloat(scoreStr);
+                // Clamp to valid range
+                score = Math.max(1.0, Math.min(10.0, score));
+            }
+            catch {
+                // Keep default
+            }
+        }
+        else if (line.startsWith('REASONING:')) {
+            reasoning = line.replace('REASONING:', '').trim();
+        }
+        else if (line.startsWith('SUGGESTIONS:')) {
+            inSuggestions = true;
+            const rest = line.replace('SUGGESTIONS:', '').trim();
+            if (rest.toLowerCase() !== 'none' && rest) {
+                suggestions.push(rest);
+            }
+        }
+        else if (inSuggestions && line.startsWith('-')) {
+            const suggestion = line.replace(/^-\s*/, '').trim();
+            if (suggestion && suggestion.toLowerCase() !== 'none') {
+                suggestions.push(suggestion);
+            }
+        }
+    }
+    return {
+        score,
+        passed: score >= threshold,
+        reasoning,
+        output,
+        expected: expected ?? undefined,
+        criteria: criteria ?? undefined,
+        suggestions,
+        timestamp: Date.now(),
+    };
+}
+// ============================================================================
+// Judge Class
+// ============================================================================
+/**
+ * Unified LLM-as-judge for evaluating agent outputs.
+ *
+ * Provides a simple API for:
+ * - Accuracy evaluation (comparing output to expected)
+ * - Criteria evaluation (evaluating against custom criteria)
+ * - Custom evaluation (subclass for domain-specific judges)
+ *
+ * @example
+ * ```typescript
+ * // Simple accuracy check
+ * const result = await new Judge().run({ output: "4", expected: "4" });
+ *
+ * // Custom criteria
+ * const result = await new Judge({ criteria: "Response is helpful" }).run({ output: "Hello!" });
+ *
+ * // With agent
+ * const result = await new Judge().run({ agent: myAgent, input: "2+2", expected: "4" });
+ * ```
+ */
+class Judge {
+    constructor(options = {}) {
+        // Use config if provided, otherwise use individual params
+        const config = options.config;
+        this.model = options.model ?? config?.model ?? process.env.OPENAI_MODEL_NAME ?? 'gpt-4o-mini';
+        this.temperature = options.temperature ?? config?.temperature ?? 0.1;
+        this.maxTokens = options.maxTokens ?? config?.maxTokens ?? 500;
+        this.threshold = options.threshold ?? config?.threshold ?? options.criteriaConfig?.threshold ?? 7.0;
+        this.criteria = options.criteria ?? config?.criteria ?? options.criteriaConfig?.description ?? null;
+        this.criteriaConfig = options.criteriaConfig ?? null;
+        this.sessionId = options.sessionId ?? null;
+    }
+    /**
+     * Build the appropriate prompt based on evaluation type.
+     */
+    buildPrompt(output, expected, criteria, input) {
+        // Use criteriaConfig custom prompt template if available
+        if (this.criteriaConfig?.promptTemplate) {
+            return this.criteriaConfig.promptTemplate
+                .replace('{output}', output)
+                .replace('{input}', input || 'Not provided')
+                .replace('{input_text}', input || 'Not provided')
+                .replace('{expected}', expected || 'Not specified');
+        }
+        // Use instance criteria if not provided
+        const effectiveCriteria = criteria ?? this.criteria;
+        if (expected !== null) {
+            // Accuracy mode
+            return ACCURACY_PROMPT
+                .replace('{input}', input || 'Not provided')
+                .replace('{expected}', expected)
+                .replace('{output}', output);
+        }
+        else if (effectiveCriteria) {
+            // Criteria mode
+            return CRITERIA_PROMPT
+                .replace('{criteria}', effectiveCriteria)
+                .replace('{output}', output);
+        }
+        else {
+            // Default: general quality evaluation
+            return GENERAL_PROMPT
+                .replace('{input}', input || 'Not provided')
+                .replace('{output}', output);
+        }
+    }
+    /**
+     * Get LLM provider lazily.
+     */
+    async getProvider() {
+        // Lazy import to avoid performance impact
+        const { createProvider } = await Promise.resolve().then(() => __importStar(require('../llm/providers')));
+        return createProvider(this.model);
+    }
+    /**
+     * Get output from an Agent.
+     */
+    async getAgentOutput(agent, input) {
+        if (typeof agent.chat === 'function') {
+            return String(await agent.chat(input));
+        }
+        else if (typeof agent.start === 'function') {
+            const result = await agent.start(input);
+            if (result && typeof result === 'object' && 'raw' in result) {
+                return String(result.raw);
+            }
+            return String(result);
+        }
+        throw new Error("Agent must have 'chat' or 'start' method");
+    }
+    /**
+     * Get output from Agents (multi-agent).
+     */
+    async getAgentsOutput(agents, input) {
+        if (typeof agents.start === 'function') {
+            const result = await agents.start(input);
+            if (result && typeof result === 'object' && 'raw' in result) {
+                return String(result.raw);
+            }
+            return String(result);
+        }
+        throw new Error("Agents must have 'start' method");
+    }
+    /**
+     * Judge an output.
+     *
+     * @param options - Evaluation options
+     * @returns JudgeResult with score, passed, reasoning, suggestions
+     */
+    async run(options) {
+        let output = options.output ?? '';
+        // Get output from agent if provided
+        if (options.agent) {
+            output = await this.getAgentOutput(options.agent, options.input ?? '');
+        }
+        else if (options.agents) {
+            output = await this.getAgentsOutput(options.agents, options.input ?? '');
+        }
+        if (!output) {
+            return {
+                score: 0,
+                passed: false,
+                reasoning: 'No output provided to judge',
+                output: '',
+                expected: options.expected,
+                criteria: options.criteria ?? this.criteria ?? undefined,
+                suggestions: [],
+                timestamp: Date.now(),
+            };
+        }
+        try {
+            const provider = await this.getProvider();
+            const prompt = this.buildPrompt(output, options.expected ?? null, options.criteria ?? null, options.input ?? '');
+            const response = await provider.generateText({
+                messages: [{ role: 'user', content: prompt }],
+                temperature: this.temperature,
+                maxTokens: this.maxTokens,
+            });
+            const responseText = response.text ?? '';
+            const result = parseJudgeResponse(responseText, output, options.expected ?? null, options.criteria ?? this.criteria, this.threshold);
+            if (options.printSummary) {
+                this.printSummary(result);
+            }
+            return result;
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            return {
+                score: 0,
+                passed: false,
+                reasoning: `Evaluation error: ${errorMessage}`,
+                output,
+                expected: options.expected,
+                criteria: options.criteria ?? this.criteria ?? undefined,
+                suggestions: [],
+                timestamp: Date.now(),
+            };
+        }
+    }
+    /**
+     * Judge an output asynchronously (alias for run).
+     */
+    async runAsync(options) {
+        return this.run(options);
+    }
+    /**
+     * Print a summary of the judge result.
+     */
+    printSummary(result) {
+        const status = result.passed ? '✅ PASSED' : '❌ FAILED';
+        console.log(`\n=== Judge Result ===`);
+        console.log(`Score: ${result.score.toFixed(1)}/10`);
+        console.log(`Status: ${status}`);
+        console.log(`Reasoning: ${result.reasoning}`);
+        if (result.suggestions.length > 0) {
+            console.log(`Suggestions:`);
+            result.suggestions.forEach(s => console.log(`  - ${s}`));
+        }
+    }
+}
+exports.Judge = Judge;
+// ============================================================================
+// Built-in Judge Types
+// ============================================================================
+/**
+ * Judge for accuracy evaluation (comparing output to expected).
+ */
+class AccuracyJudge extends Judge {
+}
+exports.AccuracyJudge = AccuracyJudge;
+/**
+ * Judge for criteria-based evaluation.
+ */
+class CriteriaJudge extends Judge {
+}
+exports.CriteriaJudge = CriteriaJudge;
+/**
+ * Judge for evaluating recipe/workflow execution traces.
+ */
+class RecipeJudge extends Judge {
+    constructor(options = {}) {
+        const criteria = `Recipe execution quality in ${options.mode ?? 'context'} mode`;
+        super({
+            ...options,
+            criteria,
+            maxTokens: options.maxTokens ?? 800,
+        });
+        this.mode = options.mode ?? 'context';
+    }
+    buildPrompt(output, expected, criteria, input) {
+        return RECIPE_PROMPT
+            .replace('{output}', output)
+            .replace('{expected}', expected ?? 'Complete workflow execution')
+            .replace('{criteria}', criteria ?? this.criteria ?? `Recipe quality in ${this.mode} mode`);
+    }
+}
+exports.RecipeJudge = RecipeJudge;
+const JUDGE_REGISTRY = new Map([
+    ['accuracy', AccuracyJudge],
+    ['criteria', CriteriaJudge],
+    ['recipe', RecipeJudge],
+]);
+/**
+ * Register a custom judge type.
+ *
+ * @param name - Name for the judge type
+ * @param judgeClass - Judge class to register
+ */
+function addJudge(name, judgeClass) {
+    JUDGE_REGISTRY.set(name.toLowerCase(), judgeClass);
+}
+/**
+ * Get a registered judge type by name.
+ *
+ * @param name - Name of the judge type
+ * @returns Judge class or undefined if not found
+ */
+function getJudge(name) {
+    return JUDGE_REGISTRY.get(name.toLowerCase());
+}
+/**
+ * List all registered judge types.
+ *
+ * @returns List of judge type names
+ */
+function listJudges() {
+    return Array.from(JUDGE_REGISTRY.keys());
+}
+/**
+ * Remove a registered judge type.
+ *
+ * @param name - Name of the judge type to remove
+ * @returns True if removed, false if not found
+ */
+function removeJudge(name) {
+    return JUDGE_REGISTRY.delete(name.toLowerCase());
+}
+const OPTIMIZATION_RULE_REGISTRY = new Map();
+/**
+ * Register a custom optimization rule.
+ *
+ * @param name - Name for the rule
+ * @param ruleClass - Rule class implementing OptimizationRuleProtocol
+ */
+function addOptimizationRule(name, ruleClass) {
+    OPTIMIZATION_RULE_REGISTRY.set(name.toLowerCase(), ruleClass);
+}
+/**
+ * Get a registered optimization rule by name.
+ *
+ * @param name - Name of the rule
+ * @returns Rule class or undefined if not found
+ */
+function getOptimizationRule(name) {
+    return OPTIMIZATION_RULE_REGISTRY.get(name.toLowerCase());
+}
+/**
+ * List all registered optimization rules.
+ *
+ * @returns List of rule names
+ */
+function listOptimizationRules() {
+    return Array.from(OPTIMIZATION_RULE_REGISTRY.keys());
+}
+/**
+ * Remove a registered optimization rule.
+ *
+ * @param name - Name of the rule to remove
+ * @returns True if removed, false if not found
+ */
+function removeOptimizationRule(name) {
+    return OPTIMIZATION_RULE_REGISTRY.delete(name.toLowerCase());
+}
+// ============================================================================
+// Exports
+// ============================================================================
+exports.default = Judge;

package/dist/index.d.ts

@@ -61,7 +61,7 @@ export * from './guardrails';
 export { Handoff, handoff, handoffFilters, type HandoffConfig, type HandoffContext, type HandoffResult } from './agent/handoff';
 export { RouterAgent, createRouter, routeConditions, type RouterConfig, type RouteConfig, type RouteContext } from './agent/router';
 export { ContextAgent, createContextAgent, type ContextAgentConfig, type ContextMessage } from './agent/context';
-export { accuracyEval, performanceEval, reliabilityEval, EvalSuite, Evaluator, createEvaluator, createDefaultEvaluator, EvalResults, createEvalResults, relevanceCriterion, lengthCriterion, containsKeywordsCriterion, noHarmfulContentCriterion, type EvalResult, type PerformanceResult, type AccuracyEvalConfig, type PerformanceEvalConfig, type ReliabilityEvalConfig, type EvalCriteria, type EvaluatorConfig, type TestResult, type AggregatedResults } from './eval';
+export { accuracyEval, performanceEval, reliabilityEval, EvalSuite, Evaluator, createEvaluator, createDefaultEvaluator, EvalResults, createEvalResults, relevanceCriterion, lengthCriterion, containsKeywordsCriterion, noHarmfulContentCriterion, Judge, AccuracyJudge, CriteriaJudge, RecipeJudge, addJudge, getJudge, listJudges, removeJudge, addOptimizationRule, getOptimizationRule, listOptimizationRules, removeOptimizationRule, parseJudgeResponse, type EvalResult, type PerformanceResult, type AccuracyEvalConfig, type PerformanceEvalConfig, type ReliabilityEvalConfig, type EvalCriteria, type EvaluatorConfig, type TestResult, type AggregatedResults, type JudgeConfig, type JudgeCriteriaConfig, type JudgeResult, type JudgeRunOptions, type JudgeOptions, } from './eval';
 export { SkillManager, createSkillManager, parseSkillFile, type Skill, type SkillMetadata, type SkillDiscoveryOptions } from './skills';
 export { parseArgs, executeCommand, CLI_SPEC_VERSION } from './cli';
 export { Memory, createMemory } from './memory/memory';

package/dist/index.js

@@ -54,14 +54,14 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.tavilySearch = exports.codeExecution = exports.composeMiddleware = exports.createValidationMiddleware = exports.createTracingMiddleware = exports.createRetryMiddleware = exports.createRateLimitMiddleware = exports.createRedactionMiddleware = exports.createTimeoutMiddleware = exports.createLoggingMiddleware = exports.MissingEnvVarError = exports.MissingDependencyError = exports.ToolsRegistry = exports.resetToolsRegistry = exports.createToolsRegistry = exports.getToolsRegistry = exports.registerBuiltinTools = exports.tools = exports.createDelegator = exports.createSubagentTools = exports.createSubagentTool = exports.SubagentTool = exports.getTool = exports.registerTool = exports.getRegistry = exports.ToolRegistry = exports.tool = exports.FunctionTool = exports.createTool = exports.validateTool = exports.ToolValidationError = exports.BaseTool = exports.setDefaultDbAdapter = exports.getDefaultDbAdapter = exports.createDbAdapter = exports.db = exports.WorkflowStep = exports.repeatPattern = exports.Repeat = exports.loopPattern = exports.Loop = exports.repeat = exports.loop = exports.route = exports.parallel = exports.Workflow = exports.Router = exports.PraisonAIAgents = exports.Agents = exports.Agent = void 0;
 exports.EvalSuite = exports.reliabilityEval = exports.performanceEval = exports.accuracyEval = exports.createContextAgent = exports.ContextAgent = exports.routeConditions = exports.createRouter = exports.RouterAgent = exports.handoffFilters = exports.handoff = exports.Handoff = exports.createRateLimitPolicy = exports.createApiKeyPolicy = exports.createMCPSecurity = exports.MCPSecurity = exports.createMCPSession = exports.createMCPServer = exports.MCPServer = exports.getMCPTools = exports.createMCPClient = exports.MCPClient = exports.registerLocalTool = exports.registerNpmTool = exports.createCustomTool = exports.registerCustomTool = exports.codeMode = exports.airweaveSearch = exports.bedrockBrowserFill = exports.bedrockBrowserClick = exports.bedrockBrowserNavigate = exports.bedrockCodeInterpreter = exports.valyuCompanyResearch = exports.valyuEconomicsSearch = exports.valyuSecSearch = exports.valyuPatentSearch = exports.valyuBioSearch = exports.valyuPaperSearch = exports.valyuFinanceSearch = exports.valyuWebSearch = exports.superagentVerify = exports.superagentRedact = exports.superagentGuard = exports.firecrawlCrawl = exports.firecrawlScrape = exports.parallelSearch = exports.perplexitySearch = exports.exaSearch = exports.tavilyCrawl = exports.tavilyExtract = void 0;
-exports.createWorkflowHooks = exports.WorkflowHooksExecutor = exports.DisplayTypes = exports.clearAllCallbacks = exports.getRegisteredDisplayTypes = exports.hasApprovalCallback = exports.requestApproval = exports.executeCallback = exports.executeSyncCallback = exports.clearApprovalCallback = exports.registerApprovalCallback = exports.unregisterDisplayCallback = exports.registerDisplayCallback = exports.createValidationOperationHooks = exports.createLoggingOperationHooks = exports.createHooksManager = exports.HooksManager = exports.createDocsManager = exports.DocsManager = exports.createSafetyRules = exports.createRulesManager = exports.RulesManager = exports.createEncryptionHooks = exports.createValidationHooks = exports.createLoggingHooks = exports.createMemoryHooks = exports.MemoryHooks = exports.DEFAULT_POLICIES = exports.createLLMSummarizer = exports.createAutoMemory = exports.AutoMemory = exports.createFileMemory = exports.FileMemory = exports.createMemory = exports.Memory = exports.CLI_SPEC_VERSION = exports.executeCommand = exports.parseArgs = exports.parseSkillFile = exports.createSkillManager = exports.SkillManager = exports.noHarmfulContentCriterion = exports.containsKeywordsCriterion = exports.lengthCriterion = exports.relevanceCriterion = exports.createEvalResults = exports.EvalResults = exports.createDefaultEvaluator = exports.createEvaluator = exports.Evaluator = void 0;
-exports.AgentEvents = exports.AgentEventBus = exports.EventEmitterPubSub = exports.PubSub = exports.createFileCache = exports.createMemoryCache = exports.FileCache = exports.MemoryCache = exports.BaseCache = exports.createTaskAgent = exports.createPlanningAgent = exports.createPlanStorage = exports.createTodoList = exports.createPlan = exports.TaskAgent = exports.PlanningAgent = exports.PlanStorage = exports.TodoItem = exports.TodoList = exports.PlanStep = exports.Plan = exports.createLLMGuardrail = exports.LLMGuardrail = exports.createPromptExpanderAgent = exports.PromptExpanderAgent = exports.createQueryRewriterAgent = exports.QueryRewriterAgent = exports.createDeepResearchAgent = exports.DeepResearchAgent = exports.createAudioAgent = exports.AudioAgent = exports.createImageAgent = exports.ImageAgent = exports.createAutoAgents = exports.AutoAgents = exports.createHTTPSink = exports.createConsoleSink = exports.createTelemetryIntegration = exports.TelemetryIntegration = exports.createPerformanceMonitor = exports.PerformanceMonitor = exports.createAgentTelemetry = exports.cleanupTelemetry = exports.disableTelemetry = exports.enableTelemetry = exports.getTelemetry = exports.AgentTelemetry = exports.TelemetryCollector = exports.createTimingWorkflowHooks = exports.createLoggingWorkflowHooks = void 0;
-exports.GraphRAG = exports.GraphStore = exports.createLLMReranker = exports.createCrossEncoderReranker = exports.createCohereReranker = exports.LLMReranker = exports.CrossEncoderReranker = exports.CohereReranker = exports.BaseReranker = exports.createElevenLabsVoice = exports.createOpenAIVoice = exports.ElevenLabsVoiceProvider = exports.OpenAIVoiceProvider = exports.BaseVoiceProvider = exports.createLangfuseObservability = exports.createMemoryObservability = exports.createConsoleObservability = exports.LangfuseObservabilityProvider = exports.MemoryObservabilityProvider = exports.ConsoleObservabilityProvider = exports.BaseObservabilityProvider = exports.createChromaStore = exports.ChromaVectorStore = exports.createQdrantStore = exports.QdrantVectorStore = exports.createWeaviateStore = exports.WeaviateVectorStore = exports.createPineconeStore = exports.PineconeVectorStore = exports.createMemoryVectorStore = exports.MemoryVectorStore = exports.BaseVectorStore = exports.createPostgresSessionStorage = exports.createMemoryPostgres = exports.createNeonPostgres = exports.PostgresSessionStorage = exports.MemoryPostgresAdapter = exports.NeonPostgresAdapter = exports.createMemoryRedis = exports.createUpstashRedis = exports.MemoryRedisAdapter = exports.UpstashRedisAdapter = exports.createSQLiteAdapter = exports.SQLiteAdapter = exports.validateWorkflowDefinition = exports.loadWorkflowFromFile = exports.createWorkflowFromYAML = exports.parseYAMLWorkflow = exports.createPubSub = exports.createEventBus = void 0;
-exports.InteractiveTUI = exports.MODEL_PRICING = exports.formatCost = exports.estimateTokens = exports.createCostTracker = exports.CostTracker = exports.isSlashCommand = exports.executeSlashCommand = exports.parseSlashCommand = exports.registerCommand = exports.createSlashCommandHandler = exports.SlashCommandHandler = exports.ADAPTERS = exports.COMMUNITY_PROVIDERS = exports.PROVIDER_ALIASES = exports.AISDK_PROVIDERS = exports.trace = exports.resetObservabilityAdapter = exports.getObservabilityAdapter = exports.setObservabilityAdapter = exports.clearAdapterCache = exports.createObservabilityAdapter = exports.createConsoleAdapter = exports.ConsoleObservabilityAdapter = exports.createMemoryAdapter = exports.MemoryObservabilityAdapter = exports.noopAdapter = exports.NoopObservabilityAdapter = exports.hasObservabilityToolEnvVar = exports.listObservabilityTools = exports.getObservabilityToolInfo = exports.OBSERVABILITY_TOOLS = exports.registerBuiltinProviders = exports.createProviderRegistry = exports.getDefaultRegistry = exports.listProviders = exports.hasProvider = exports.unregisterProvider = exports.registerProvider = exports.ProviderRegistry = exports.BaseProvider = exports.GoogleProvider = exports.AnthropicProvider = exports.OpenAIProvider = exports.getAvailableProviders = exports.isProviderAvailable = exports.parseModelString = exports.getDefaultProvider = exports.createProvider = exports.createGraphRAG = void 0;
-exports.createN8NIntegration = exports.N8NIntegration = exports.externalAgentAsTool = exports.createExternalAgent = exports.getExternalAgentRegistry = exports.GenericExternalAgent = exports.AiderAgent = exports.CodexCliAgent = exports.GeminiCliAgent = exports.ClaudeCodeAgent = exports.BaseExternalAgent = exports.renderWorkflow = exports.createFlowDisplay = exports.FlowDisplay = exports.createFileCheckpointStorage = exports.FileCheckpointStorage = exports.MemoryCheckpointStorage = exports.createCheckpointManager = exports.CheckpointManager = exports.createFileJobStorage = exports.FileJobStorage = exports.MemoryJobStorage = exports.createJobQueue = exports.JobQueue = exports.cronExpressions = exports.createScheduler = exports.Scheduler = exports.MODE_POLICIES = exports.cliApprovalPrompt = exports.createAutonomyManager = exports.AutonomyManager = exports.DEFAULT_BLOCKED_PATHS = exports.DEFAULT_BLOCKED_COMMANDS = exports.CommandValidator = exports.sandboxExec = exports.createSandboxExecutor = exports.SandboxExecutor = exports.createDiffViewer = exports.DiffViewer = exports.createGitManager = exports.GitManager = exports.DEFAULT_IGNORE_PATTERNS = exports.getRepoTree = exports.createRepoMap = exports.RepoMap = exports.createHistoryManager = exports.HistoryManager = exports.createStatusDisplay = exports.StatusDisplay = exports.createInteractiveTUI = void 0;
-exports.createAgentLoop = exports.createPagesHandler = exports.createRouteHandler = exports.createNestHandler = exports.createFastifyHandler = exports.createHonoHandler = exports.createExpressHandler = exports.createHttpHandler = exports.mcpToolsToAITools = exports.closeAllMCPClients = exports.closeMCPClient = exports.getMCPClient = exports.createMCP = exports.isDataUrl = exports.isUrl = exports.uint8ArrayToBase64 = exports.base64ToUint8Array = exports.toMessageContent = exports.createMultimodalMessage = exports.createTextPart = exports.createPdfPart = exports.createFilePart = exports.createImagePart = exports.getAICacheStats = exports.clearAICache = exports.applyMiddleware = exports.wrapModel = exports.createAILoggingMiddleware = exports.createCachingMiddleware = exports.resolveModelAlias = exports.hasModelAlias = exports.listModelAliases = exports.MODEL_ALIASES = exports.parseModel = exports.getModel = exports.createModel = exports.functionToTool = exports.createToolSet = exports.defineTool = exports.aiEmbedMany = exports.aiEmbed = exports.aiGenerateImage = exports.aiStreamObject = exports.aiGenerateObject = exports.aiStreamText = exports.aiGenerateText = exports.getQuickContext = exports.createFastContext = exports.FastContext = exports.triggerN8NWebhook = void 0;
-exports.createSlackBot = exports.createTelemetrySettings = exports.clearEvents = exports.getEvents = exports.recordEvent = exports.createTelemetryMiddleware = exports.withSpan = exports.createAISpan = exports.getTracer = exports.initOpenTelemetry = exports.isTelemetryEnabled = exports.disableAITelemetry = exports.enableAITelemetry = exports.getTelemetrySettings = exports.configureTelemetry = exports.autoEnableDevTools = exports.createDevToolsMiddleware = exports.getDevToolsUrl = exports.getDevToolsState = exports.isDevToolsEnabled = exports.disableDevTools = exports.enableDevTools = exports.TRANSCRIPTION_MODELS = exports.SPEECH_MODELS = exports.transcribe = exports.generateSpeech = exports.createDangerousPatternChecker = exports.isDangerous = exports.DANGEROUS_PATTERNS = exports.ToolApprovalTimeoutError = exports.ToolApprovalDeniedError = exports.withApproval = exports.setApprovalManager = exports.getApprovalManager = exports.ApprovalManager = exports.pipeUIMessageStreamToResponse = exports.toUIMessageStreamResponse = exports.createApprovalResponse = exports.getToolsNeedingApproval = exports.hasPendingApprovals = exports.createSystemMessage = exports.createTextMessage = exports.safeValidateUIMessages = exports.validateUIMessages = exports.convertToUIMessages = exports.convertToModelMessages = exports.stopWhen = exports.stopWhenNoToolCalls = exports.stopAfterSteps = exports.AgentLoop = void 0;
-exports.createCLIApprovalPrompt = exports.createComputerUseAgent = exports.ComputerUseClient = exports.createComputerUse = exports.createPostgresTool = exports.NLPostgresClient = exports.createNLPostgres = exports.parseSlackMessage = exports.verifySlackSignature = exports.SlackBot = void 0;
+exports.createValidationOperationHooks = exports.createLoggingOperationHooks = exports.createHooksManager = exports.HooksManager = exports.createDocsManager = exports.DocsManager = exports.createSafetyRules = exports.createRulesManager = exports.RulesManager = exports.createEncryptionHooks = exports.createValidationHooks = exports.createLoggingHooks = exports.createMemoryHooks = exports.MemoryHooks = exports.DEFAULT_POLICIES = exports.createLLMSummarizer = exports.createAutoMemory = exports.AutoMemory = exports.createFileMemory = exports.FileMemory = exports.createMemory = exports.Memory = exports.CLI_SPEC_VERSION = exports.executeCommand = exports.parseArgs = exports.parseSkillFile = exports.createSkillManager = exports.SkillManager = exports.parseJudgeResponse = exports.removeOptimizationRule = exports.listOptimizationRules = exports.getOptimizationRule = exports.addOptimizationRule = exports.removeJudge = exports.listJudges = exports.getJudge = exports.addJudge = exports.RecipeJudge = exports.CriteriaJudge = exports.AccuracyJudge = exports.Judge = exports.noHarmfulContentCriterion = exports.containsKeywordsCriterion = exports.lengthCriterion = exports.relevanceCriterion = exports.createEvalResults = exports.EvalResults = exports.createDefaultEvaluator = exports.createEvaluator = exports.Evaluator = void 0;
+exports.createPlan = exports.TaskAgent = exports.PlanningAgent = exports.PlanStorage = exports.TodoItem = exports.TodoList = exports.PlanStep = exports.Plan = exports.createLLMGuardrail = exports.LLMGuardrail = exports.createPromptExpanderAgent = exports.PromptExpanderAgent = exports.createQueryRewriterAgent = exports.QueryRewriterAgent = exports.createDeepResearchAgent = exports.DeepResearchAgent = exports.createAudioAgent = exports.AudioAgent = exports.createImageAgent = exports.ImageAgent = exports.createAutoAgents = exports.AutoAgents = exports.createHTTPSink = exports.createConsoleSink = exports.createTelemetryIntegration = exports.TelemetryIntegration = exports.createPerformanceMonitor = exports.PerformanceMonitor = exports.createAgentTelemetry = exports.cleanupTelemetry = exports.disableTelemetry = exports.enableTelemetry = exports.getTelemetry = exports.AgentTelemetry = exports.TelemetryCollector = exports.createTimingWorkflowHooks = exports.createLoggingWorkflowHooks = exports.createWorkflowHooks = exports.WorkflowHooksExecutor = exports.DisplayTypes = exports.clearAllCallbacks = exports.getRegisteredDisplayTypes = exports.hasApprovalCallback = exports.requestApproval = exports.executeCallback = exports.executeSyncCallback = exports.clearApprovalCallback = exports.registerApprovalCallback = exports.unregisterDisplayCallback = exports.registerDisplayCallback = void 0;
+exports.BaseVoiceProvider = exports.createLangfuseObservability = exports.createMemoryObservability = exports.createConsoleObservability = exports.LangfuseObservabilityProvider = exports.MemoryObservabilityProvider = exports.ConsoleObservabilityProvider = exports.BaseObservabilityProvider = exports.createChromaStore = exports.ChromaVectorStore = exports.createQdrantStore = exports.QdrantVectorStore = exports.createWeaviateStore = exports.WeaviateVectorStore = exports.createPineconeStore = exports.PineconeVectorStore = exports.createMemoryVectorStore = exports.MemoryVectorStore = exports.BaseVectorStore = exports.createPostgresSessionStorage = exports.createMemoryPostgres = exports.createNeonPostgres = exports.PostgresSessionStorage = exports.MemoryPostgresAdapter = exports.NeonPostgresAdapter = exports.createMemoryRedis = exports.createUpstashRedis = exports.MemoryRedisAdapter = exports.UpstashRedisAdapter = exports.createSQLiteAdapter = exports.SQLiteAdapter = exports.validateWorkflowDefinition = exports.loadWorkflowFromFile = exports.createWorkflowFromYAML = exports.parseYAMLWorkflow = exports.createPubSub = exports.createEventBus = exports.AgentEvents = exports.AgentEventBus = exports.EventEmitterPubSub = exports.PubSub = exports.createFileCache = exports.createMemoryCache = exports.FileCache = exports.MemoryCache = exports.BaseCache = exports.createTaskAgent = exports.createPlanningAgent = exports.createPlanStorage = exports.createTodoList = void 0;
+exports.COMMUNITY_PROVIDERS = exports.PROVIDER_ALIASES = exports.AISDK_PROVIDERS = exports.trace = exports.resetObservabilityAdapter = exports.getObservabilityAdapter = exports.setObservabilityAdapter = exports.clearAdapterCache = exports.createObservabilityAdapter = exports.createConsoleAdapter = exports.ConsoleObservabilityAdapter = exports.createMemoryAdapter = exports.MemoryObservabilityAdapter = exports.noopAdapter = exports.NoopObservabilityAdapter = exports.hasObservabilityToolEnvVar = exports.listObservabilityTools = exports.getObservabilityToolInfo = exports.OBSERVABILITY_TOOLS = exports.registerBuiltinProviders = exports.createProviderRegistry = exports.getDefaultRegistry = exports.listProviders = exports.hasProvider = exports.unregisterProvider = exports.registerProvider = exports.ProviderRegistry = exports.BaseProvider = exports.GoogleProvider = exports.AnthropicProvider = exports.OpenAIProvider = exports.getAvailableProviders = exports.isProviderAvailable = exports.parseModelString = exports.getDefaultProvider = exports.createProvider = exports.createGraphRAG = exports.GraphRAG = exports.GraphStore = exports.createLLMReranker = exports.createCrossEncoderReranker = exports.createCohereReranker = exports.LLMReranker = exports.CrossEncoderReranker = exports.CohereReranker = exports.BaseReranker = exports.createElevenLabsVoice = exports.createOpenAIVoice = exports.ElevenLabsVoiceProvider = exports.OpenAIVoiceProvider = void 0;
+exports.FlowDisplay = exports.createFileCheckpointStorage = exports.FileCheckpointStorage = exports.MemoryCheckpointStorage = exports.createCheckpointManager = exports.CheckpointManager = exports.createFileJobStorage = exports.FileJobStorage = exports.MemoryJobStorage = exports.createJobQueue = exports.JobQueue = exports.cronExpressions = exports.createScheduler = exports.Scheduler = exports.MODE_POLICIES = exports.cliApprovalPrompt = exports.createAutonomyManager = exports.AutonomyManager = exports.DEFAULT_BLOCKED_PATHS = exports.DEFAULT_BLOCKED_COMMANDS = exports.CommandValidator = exports.sandboxExec = exports.createSandboxExecutor = exports.SandboxExecutor = exports.createDiffViewer = exports.DiffViewer = exports.createGitManager = exports.GitManager = exports.DEFAULT_IGNORE_PATTERNS = exports.getRepoTree = exports.createRepoMap = exports.RepoMap = exports.createHistoryManager = exports.HistoryManager = exports.createStatusDisplay = exports.StatusDisplay = exports.createInteractiveTUI = exports.InteractiveTUI = exports.MODEL_PRICING = exports.formatCost = exports.estimateTokens = exports.createCostTracker = exports.CostTracker = exports.isSlashCommand = exports.executeSlashCommand = exports.parseSlashCommand = exports.registerCommand = exports.createSlashCommandHandler = exports.SlashCommandHandler = exports.ADAPTERS = void 0;
+exports.isDataUrl = exports.isUrl = exports.uint8ArrayToBase64 = exports.base64ToUint8Array = exports.toMessageContent = exports.createMultimodalMessage = exports.createTextPart = exports.createPdfPart = exports.createFilePart = exports.createImagePart = exports.getAICacheStats = exports.clearAICache = exports.applyMiddleware = exports.wrapModel = exports.createAILoggingMiddleware = exports.createCachingMiddleware = exports.resolveModelAlias = exports.hasModelAlias = exports.listModelAliases = exports.MODEL_ALIASES = exports.parseModel = exports.getModel = exports.createModel = exports.functionToTool = exports.createToolSet = exports.defineTool = exports.aiEmbedMany = exports.aiEmbed = exports.aiGenerateImage = exports.aiStreamObject = exports.aiGenerateObject = exports.aiStreamText = exports.aiGenerateText = exports.getQuickContext = exports.createFastContext = exports.FastContext = exports.triggerN8NWebhook = exports.createN8NIntegration = exports.N8NIntegration = exports.externalAgentAsTool = exports.createExternalAgent = exports.getExternalAgentRegistry = exports.GenericExternalAgent = exports.AiderAgent = exports.CodexCliAgent = exports.GeminiCliAgent = exports.ClaudeCodeAgent = exports.BaseExternalAgent = exports.renderWorkflow = exports.createFlowDisplay = void 0;
+exports.getTelemetrySettings = exports.configureTelemetry = exports.autoEnableDevTools = exports.createDevToolsMiddleware = exports.getDevToolsUrl = exports.getDevToolsState = exports.isDevToolsEnabled = exports.disableDevTools = exports.enableDevTools = exports.TRANSCRIPTION_MODELS = exports.SPEECH_MODELS = exports.transcribe = exports.generateSpeech = exports.createDangerousPatternChecker = exports.isDangerous = exports.DANGEROUS_PATTERNS = exports.ToolApprovalTimeoutError = exports.ToolApprovalDeniedError = exports.withApproval = exports.setApprovalManager = exports.getApprovalManager = exports.ApprovalManager = exports.pipeUIMessageStreamToResponse = exports.toUIMessageStreamResponse = exports.createApprovalResponse = exports.getToolsNeedingApproval = exports.hasPendingApprovals = exports.createSystemMessage = exports.createTextMessage = exports.safeValidateUIMessages = exports.validateUIMessages = exports.convertToUIMessages = exports.convertToModelMessages = exports.stopWhen = exports.stopWhenNoToolCalls = exports.stopAfterSteps = exports.AgentLoop = exports.createAgentLoop = exports.createPagesHandler = exports.createRouteHandler = exports.createNestHandler = exports.createFastifyHandler = exports.createHonoHandler = exports.createExpressHandler = exports.createHttpHandler = exports.mcpToolsToAITools = exports.closeAllMCPClients = exports.closeMCPClient = exports.getMCPClient = exports.createMCP = void 0;
+exports.createCLIApprovalPrompt = exports.createComputerUseAgent = exports.ComputerUseClient = exports.createComputerUse = exports.createPostgresTool = exports.NLPostgresClient = exports.createNLPostgres = exports.parseSlackMessage = exports.verifySlackSignature = exports.SlackBot = exports.createSlackBot = exports.createTelemetrySettings = exports.clearEvents = exports.getEvents = exports.recordEvent = exports.createTelemetryMiddleware = exports.withSpan = exports.createAISpan = exports.getTracer = exports.initOpenTelemetry = exports.isTelemetryEnabled = exports.disableAITelemetry = exports.enableAITelemetry = void 0;
 // ============================================================================
 // CORE API - The main classes users should use
 // ============================================================================
@@ -230,6 +230,20 @@ Object.defineProperty(exports, "relevanceCriterion", { enumerable: true, get: fu
 Object.defineProperty(exports, "lengthCriterion", { enumerable: true, get: function () { return eval_1.lengthCriterion; } });
 Object.defineProperty(exports, "containsKeywordsCriterion", { enumerable: true, get: function () { return eval_1.containsKeywordsCriterion; } });
 Object.defineProperty(exports, "noHarmfulContentCriterion", { enumerable: true, get: function () { return eval_1.noHarmfulContentCriterion; } });
+// LLM-as-Judge
+Object.defineProperty(exports, "Judge", { enumerable: true, get: function () { return eval_1.Judge; } });
+Object.defineProperty(exports, "AccuracyJudge", { enumerable: true, get: function () { return eval_1.AccuracyJudge; } });
+Object.defineProperty(exports, "CriteriaJudge", { enumerable: true, get: function () { return eval_1.CriteriaJudge; } });
+Object.defineProperty(exports, "RecipeJudge", { enumerable: true, get: function () { return eval_1.RecipeJudge; } });
+Object.defineProperty(exports, "addJudge", { enumerable: true, get: function () { return eval_1.addJudge; } });
+Object.defineProperty(exports, "getJudge", { enumerable: true, get: function () { return eval_1.getJudge; } });
+Object.defineProperty(exports, "listJudges", { enumerable: true, get: function () { return eval_1.listJudges; } });
+Object.defineProperty(exports, "removeJudge", { enumerable: true, get: function () { return eval_1.removeJudge; } });
+Object.defineProperty(exports, "addOptimizationRule", { enumerable: true, get: function () { return eval_1.addOptimizationRule; } });
+Object.defineProperty(exports, "getOptimizationRule", { enumerable: true, get: function () { return eval_1.getOptimizationRule; } });
+Object.defineProperty(exports, "listOptimizationRules", { enumerable: true, get: function () { return eval_1.listOptimizationRules; } });
+Object.defineProperty(exports, "removeOptimizationRule", { enumerable: true, get: function () { return eval_1.removeOptimizationRule; } });
+Object.defineProperty(exports, "parseJudgeResponse", { enumerable: true, get: function () { return eval_1.parseJudgeResponse; } });
 // Note: Observability exports are at the bottom of this file with the full 14+ integrations
 // Export skills
 var skills_1 = require("./skills");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "praisonai",
-  "version": "1.5.3",
+  "version": "1.5.4",
   "description": "PraisonAI TypeScript AI Agents Framework - Node.js, npm, and Javascript AI Agents Framework",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",