praisonai 1.5.3 → 1.6.0

package/dist/eval/judge.d.ts

@@ -0,0 +1,275 @@
+/**
+ * Unified Judge class for LLM-as-judge evaluation.
+ *
+ * Provides a simple, unified API for evaluating agent outputs using LLM-as-judge.
+ * Follows PraisonAI naming conventions and engineering principles.
+ *
+ * DRY: Reuses existing provider infrastructure.
+ * Protocol-driven: Implements JudgeProtocol for extensibility.
+ * Zero performance impact: Lazy imports for LLM providers.
+ *
+ * @example
+ * ```typescript
+ * import { Judge } from 'praisonai';
+ * const result = await Judge.run({ output: "4", expected: "4" });
+ * console.log(`Score: ${result.score}/10`);
+ * ```
+ */
+/**
+ * Configuration for Judge instances.
+ */
+export interface JudgeConfig {
+    /** LLM model to use for judging (default: gpt-4o-mini) */
+    model?: string;
+    /** Temperature for LLM calls (default: 0.1 for consistency) */
+    temperature?: number;
+    /** Maximum tokens for LLM response */
+    maxTokens?: number;
+    /** Score threshold for passing (default: 7.0) */
+    threshold?: number;
+    /** Optional custom criteria for evaluation */
+    criteria?: string;
+}
+/**
+ * Dynamic criteria configuration for domain-agnostic judging.
+ *
+ * Enables judges to evaluate ANY domain, not just agent outputs:
+ * - Water flow optimization
+ * - Data pipeline efficiency
+ * - Manufacturing quality
+ * - Recipe/workflow optimization
+ * - Any custom domain
+ */
+export interface JudgeCriteriaConfig {
+    /** Name of the criteria configuration */
+    name: string;
+    /** Description of what is being evaluated */
+    description: string;
+    /** Custom prompt template with {output} placeholder */
+    promptTemplate: string;
+    /** List of dimensions to score (e.g., ["efficiency", "safety"]) */
+    scoringDimensions: string[];
+    /** Score threshold for passing (default: 7.0) */
+    threshold?: number;
+}
+/**
+ * Result from a Judge evaluation.
+ *
+ * This is the unified result type for all LLM-as-judge evaluations.
+ */
+export interface JudgeResult {
+    /** Quality score (1-10) */
+    score: number;
+    /** Whether the evaluation passed (score >= threshold) */
+    passed: boolean;
+    /** Explanation for the score */
+    reasoning: string;
+    /** The output that was judged */
+    output: string;
+    /** Optional expected output */
+    expected?: string;
+    /** Optional criteria used for evaluation */
+    criteria?: string;
+    /** List of improvement suggestions */
+    suggestions: string[];
+    /** When judging occurred */
+    timestamp: number;
+    /** Additional metadata */
+    metadata?: Record<string, any>;
+}
+/**
+ * Options for Judge.run() method
+ */
+export interface JudgeRunOptions {
+    /** The output to judge (required if no agent) */
+    output?: string;
+    /** Optional expected output for accuracy evaluation */
+    expected?: string;
+    /** Optional criteria for criteria evaluation */
+    criteria?: string;
+    /** Optional input context */
+    input?: string;
+    /** Optional Agent to run and judge */
+    agent?: any;
+    /** Optional Agents to run and judge */
+    agents?: any;
+    /** Whether to print result summary */
+    printSummary?: boolean;
+}
+/**
+ * Constructor options for Judge class
+ */
+export interface JudgeOptions {
+    /** LLM model to use */
+    model?: string;
+    /** Temperature for LLM calls */
+    temperature?: number;
+    /** Maximum tokens for response */
+    maxTokens?: number;
+    /** Score threshold for passing */
+    threshold?: number;
+    /** Custom criteria for evaluation */
+    criteria?: string;
+    /** Full JudgeConfig object */
+    config?: JudgeConfig;
+    /** Domain-agnostic criteria config */
+    criteriaConfig?: JudgeCriteriaConfig;
+    /** Session ID for trace isolation */
+    sessionId?: string;
+}
+/**
+ * Protocol interface for Judge implementations
+ */
+export interface JudgeProtocol {
+    run(options: JudgeRunOptions): Promise<JudgeResult>;
+    runAsync(options: JudgeRunOptions): Promise<JudgeResult>;
+}
+/**
+ * Parse LLM response into JudgeResult.
+ *
+ * @param responseText - Raw LLM response
+ * @param output - Original output
+ * @param expected - Original expected output
+ * @param criteria - Original criteria
+ * @param threshold - Score threshold for passing
+ * @returns JudgeResult with score, passed, reasoning, suggestions
+ */
+export declare function parseJudgeResponse(responseText: string, output: string, expected: string | null, criteria: string | null, threshold: number): JudgeResult;
+/**
+ * Unified LLM-as-judge for evaluating agent outputs.
+ *
+ * Provides a simple API for:
+ * - Accuracy evaluation (comparing output to expected)
+ * - Criteria evaluation (evaluating against custom criteria)
+ * - Custom evaluation (subclass for domain-specific judges)
+ *
+ * @example
+ * ```typescript
+ * // Simple accuracy check
+ * const result = await new Judge().run({ output: "4", expected: "4" });
+ *
+ * // Custom criteria
+ * const result = await new Judge({ criteria: "Response is helpful" }).run({ output: "Hello!" });
+ *
+ * // With agent
+ * const result = await new Judge().run({ agent: myAgent, input: "2+2", expected: "4" });
+ * ```
+ */
+export declare class Judge implements JudgeProtocol {
+    readonly model: string;
+    readonly temperature: number;
+    readonly maxTokens: number;
+    readonly threshold: number;
+    readonly criteria: string | null;
+    readonly criteriaConfig: JudgeCriteriaConfig | null;
+    readonly sessionId: string | null;
+    constructor(options?: JudgeOptions);
+    /**
+     * Build the appropriate prompt based on evaluation type.
+     */
+    protected buildPrompt(output: string, expected: string | null, criteria: string | null, input: string): string;
+    /**
+     * Get LLM provider lazily.
+     */
+    protected getProvider(): Promise<any>;
+    /**
+     * Get output from an Agent.
+     */
+    protected getAgentOutput(agent: any, input: string): Promise<string>;
+    /**
+     * Get output from Agents (multi-agent).
+     */
+    protected getAgentsOutput(agents: any, input: string): Promise<string>;
+    /**
+     * Judge an output.
+     *
+     * @param options - Evaluation options
+     * @returns JudgeResult with score, passed, reasoning, suggestions
+     */
+    run(options: JudgeRunOptions): Promise<JudgeResult>;
+    /**
+     * Judge an output asynchronously (alias for run).
+     */
+    runAsync(options: JudgeRunOptions): Promise<JudgeResult>;
+    /**
+     * Print a summary of the judge result.
+     */
+    printSummary(result: JudgeResult): void;
+}
+/**
+ * Judge for accuracy evaluation (comparing output to expected).
+ */
+export declare class AccuracyJudge extends Judge {
+}
+/**
+ * Judge for criteria-based evaluation.
+ */
+export declare class CriteriaJudge extends Judge {
+}
+/**
+ * Judge for evaluating recipe/workflow execution traces.
+ */
+export declare class RecipeJudge extends Judge {
+    readonly mode: string;
+    constructor(options?: JudgeOptions & {
+        mode?: string;
+    });
+    protected buildPrompt(output: string, expected: string | null, criteria: string | null, input: string): string;
+}
+type JudgeConstructor = new (options?: JudgeOptions) => Judge;
+/**
+ * Register a custom judge type.
+ *
+ * @param name - Name for the judge type
+ * @param judgeClass - Judge class to register
+ */
+export declare function addJudge(name: string, judgeClass: JudgeConstructor): void;
+/**
+ * Get a registered judge type by name.
+ *
+ * @param name - Name of the judge type
+ * @returns Judge class or undefined if not found
+ */
+export declare function getJudge(name: string): JudgeConstructor | undefined;
+/**
+ * List all registered judge types.
+ *
+ * @returns List of judge type names
+ */
+export declare function listJudges(): string[];
+/**
+ * Remove a registered judge type.
+ *
+ * @param name - Name of the judge type to remove
+ * @returns True if removed, false if not found
+ */
+export declare function removeJudge(name: string): boolean;
+type OptimizationRuleConstructor = new (...args: any[]) => any;
+/**
+ * Register a custom optimization rule.
+ *
+ * @param name - Name for the rule
+ * @param ruleClass - Rule class implementing OptimizationRuleProtocol
+ */
+export declare function addOptimizationRule(name: string, ruleClass: OptimizationRuleConstructor): void;
+/**
+ * Get a registered optimization rule by name.
+ *
+ * @param name - Name of the rule
+ * @returns Rule class or undefined if not found
+ */
+export declare function getOptimizationRule(name: string): OptimizationRuleConstructor | undefined;
+/**
+ * List all registered optimization rules.
+ *
+ * @returns List of rule names
+ */
+export declare function listOptimizationRules(): string[];
+/**
+ * Remove a registered optimization rule.
+ *
+ * @param name - Name of the rule to remove
+ * @returns True if removed, false if not found
+ */
+export declare function removeOptimizationRule(name: string): boolean;
+export default Judge;