ppef 1.1.0 → 1.2.0

package/dist/types/evaluator.d.ts

@@ -0,0 +1,449 @@
+/**
+ * Evaluator Type Definitions
+ *
+ * Core abstractions for the extensible evaluation system.
+ * All evaluators implement the Evaluator interface, enabling
+ * a unified API for claims, robustness, metrics, and custom evaluations.
+ */
+import type { AggregatedResult } from "./aggregate.js";
+import type { EvaluationResult } from "./result.js";
+/**
+ * Built-in evaluation types.
+ * Custom types can be registered at runtime.
+ */
+export type EvaluationType = "claims" | "robustness" | "metrics" | "exploratory" | "custom";
+/**
+ * Base configuration for any evaluator.
+ */
+export interface EvaluatorConfig {
+    /** Human-readable name for this evaluator */
+    name?: string;
+    /** Optional description */
+    description?: string;
+    /** Additional evaluator-specific options */
+    options?: Record<string, unknown>;
+}
+/**
+ * Result of validating evaluator configuration.
+ */
+export interface ValidationResult {
+    /** Whether the configuration is valid */
+    valid: boolean;
+    /** Error messages if invalid */
+    errors?: string[];
+    /** Warnings for non-fatal issues */
+    warnings?: string[];
+}
+/**
+ * Abstract evaluator interface for registry storage and collections.
+ *
+ * This non-generic interface uses `unknown` to accommodate any evaluator type.
+ * It enables evaluators to be stored in collections without type parameter conflicts.
+ *
+ * For type-safe operations, use the `Evaluator<TConfig, TInput, TOutput>` type instead.
+ */
+export interface IEvaluator {
+    /** Unique type identifier for this evaluator */
+    readonly type: EvaluationType;
+    /**
+     * Validate configuration before evaluation.
+     *
+     * @param config - Configuration to validate (unknown for compatibility)
+     * @returns Validation result
+     */
+    validateConfig(config: unknown): ValidationResult;
+    /**
+     * Perform evaluation.
+     *
+     * @param config - Evaluator configuration (unknown for compatibility)
+     * @param input - Input data for evaluation (unknown for compatibility)
+     * @returns Evaluation output
+     */
+    evaluate(config: unknown, input: unknown): EvaluationOutput<unknown>;
+    /**
+     * Create a summary of evaluation results.
+     *
+     * @param output - Evaluation output to summarize
+     * @returns Summary statistics
+     */
+    summarize(output: EvaluationOutput<unknown>): EvaluationSummary;
+}
+/**
+ * Generic evaluator type for type-safe evaluator implementations.
+ *
+ * This intersection type combines `IEvaluator` with type-specific method signatures.
+ * Classes that implement both interfaces (via structural typing) automatically satisfy
+ * this type - no code duplication needed.
+ *
+ * @template TConfig - Configuration type for this evaluator
+ * @template TInput - Input type for evaluation
+ * @template TOutput - Output type for evaluation results
+ *
+ * @example
+ * ```ts
+ * class MyEvaluator implements Evaluator<MyConfig, MyInput, MyOutput>, IEvaluator {
+ *   readonly type = "custom" as const;
+ *
+ *   // Generic type-safe methods (for Evaluator<>)
+ *   validateConfig(config: MyConfig): ValidationResult { ... }
+ *   evaluate(config: MyConfig, input: MyInput): EvaluationOutput<MyOutput> { ... }
+ *   summarize(output: EvaluationOutput<MyOutput>): EvaluationSummary { ... }
+ *
+ *   // Unknown methods are automatically satisfied by structural typing
+ *   // No need to duplicate - TypeScript treats MyConfig as assignable to unknown
+ * }
+ * ```
+ */
+export type Evaluator<TConfig extends EvaluatorConfig, TInput, TOutput> = IEvaluator & {
+    validateConfig(config: TConfig): ValidationResult;
+    evaluate(config: TConfig, input: TInput): EvaluationOutput<TOutput>;
+    summarize(output: EvaluationOutput<TOutput>): EvaluationSummary;
+};
+/**
+ * Generic evaluation output wrapper.
+ *
+ * All evaluators produce output in this format, providing
+ * a consistent shape for the renderer layer.
+ *
+ * @template T - Type-specific data
+ */
+export interface EvaluationOutput<T> {
+    /** Evaluator type that produced this output */
+    type: EvaluationType;
+    /** Schema version for this output format */
+    version: string;
+    /** Generation timestamp */
+    timestamp: string;
+    /** Type-specific evaluation data */
+    data: T;
+    /** Optional metadata */
+    metadata?: {
+        /** Input file or source */
+        inputSource?: string;
+        /** Configuration used */
+        config?: EvaluatorConfig;
+        /** Additional metadata */
+        [key: string]: unknown;
+    };
+}
+/**
+ * Generic evaluation summary.
+ */
+export interface EvaluationSummary {
+    /** Total number of items evaluated */
+    total: number;
+    /** Number of items that passed criteria */
+    passed?: number;
+    /** Number of items that failed criteria */
+    failed?: number;
+    /** Number of items with inconclusive results */
+    inconclusive?: number;
+    /** Pass rate (passed / total) */
+    passRate?: number;
+    /** Additional summary metrics */
+    additional?: Record<string, number | string>;
+}
+/**
+ * Evaluation context for evaluators that work with aggregates.
+ */
+export interface EvaluationContext {
+    /** Aggregated results */
+    aggregates: AggregatedResult[];
+    /** Optional raw results for more detailed analysis */
+    rawResults?: EvaluationResult[];
+    /** Metadata about the evaluation */
+    metadata?: {
+        /** Source of the aggregates */
+        source?: string;
+        /** Configuration hash for reproducibility */
+        configHash?: string;
+        /** Timestamp when aggregates were generated */
+        aggregatesTimestamp?: string;
+        /** Additional metadata */
+        [key: string]: unknown;
+    };
+}
+import type { ClaimEvaluationSummary, EvaluationClaim } from "./claims.js";
+/**
+ * Configuration for the claims evaluator.
+ */
+export interface ClaimsEvaluatorConfig extends EvaluatorConfig {
+    /** Claims to evaluate */
+    claims: EvaluationClaim[];
+    /** Optional significance level override */
+    significanceLevel?: number;
+    /** Optional minimum effect size override */
+    minEffectSize?: number;
+}
+/**
+ * Data type for claims evaluator output.
+ */
+export type ClaimsEvaluatorData = ClaimEvaluationSummary;
+import type { RobustnessAnalysisOutput } from "./perturbation.js";
+/**
+ * Configuration for the robustness evaluator.
+ */
+export interface RobustnessEvaluatorConfig extends EvaluatorConfig {
+    /** Metrics to analyze */
+    metrics: string[];
+    /** Perturbations applied */
+    perturbations: string[];
+    /** Intensity levels tested (if applicable) */
+    intensityLevels?: number[];
+    /** Number of runs per perturbation level */
+    runsPerLevel?: number;
+}
+/**
+ * Data type for robustness evaluator output.
+ */
+export type RobustnessEvaluatorData = RobustnessAnalysisOutput;
+/**
+ * Criterion type for metrics evaluation.
+ */
+export type MetricsCriterionType = "threshold" | "baseline" | "target-range";
+/**
+ * A single metrics evaluation criterion.
+ */
+export interface MetricsCriterion {
+    /** Unique identifier */
+    criterionId: string;
+    /** Human-readable description */
+    description: string;
+    /** Type of criterion */
+    type: MetricsCriterionType;
+    /** Metric to evaluate */
+    metric: string;
+    /** SUT to evaluate (or "*" for all SUTs) */
+    sut: string;
+    /** For threshold: operator and value */
+    threshold?: {
+        operator: "gt" | "gte" | "lt" | "lte" | "eq";
+        value: number;
+    };
+    /** For baseline: baseline SUT and comparison */
+    baseline?: {
+        sut: string;
+        operator: "gt" | "gte" | "lt" | "lte" | "eq";
+    };
+    /** For target-range: min and max values */
+    targetRange?: {
+        min?: number;
+        max?: number;
+        minInclusive?: boolean;
+        maxInclusive?: boolean;
+    };
+    /** Optional scope constraints */
+    scopeConstraints?: {
+        caseClass?: string | string[];
+    };
+    /** Tags for filtering */
+    tags?: readonly string[];
+}
+/**
+ * Result of evaluating a single metrics criterion.
+ */
+export interface MetricsCriterionResult {
+    /** The criterion being evaluated */
+    criterion: MetricsCriterion;
+    /** Pass/fail status */
+    status: "pass" | "fail" | "inconclusive";
+    /** Actual value(s) observed */
+    observed: {
+        sut: string;
+        value: number;
+    }[];
+    /** Expected value or range */
+    expected: {
+        type: MetricsCriterionType;
+        threshold?: number;
+        baselineValue?: number;
+        targetRange?: {
+            min?: number;
+            max?: number;
+        };
+    };
+    /** Reason for inconclusive status (if applicable) */
+    inconclusiveReason?: string;
+}
+/**
+ * Summary of metrics evaluation.
+ */
+export interface MetricsEvaluationSummary {
+    /** Schema version */
+    version: string;
+    /** Generation timestamp */
+    timestamp: string;
+    /** Individual criterion results */
+    results: MetricsCriterionResult[];
+    /** Summary statistics */
+    summary: {
+        /** Total criteria evaluated */
+        total: number;
+        /** Criteria passed */
+        passed: number;
+        /** Criteria failed */
+        failed: number;
+        /** Criteria inconclusive */
+        inconclusive: number;
+        /** Overall pass rate */
+        passRate: number;
+        /** Pass rate by SUT */
+        passRateBySut: Record<string, number>;
+    };
+}
+/**
+ * Configuration for the metrics evaluator.
+ */
+export interface MetricsEvaluatorConfig extends EvaluatorConfig {
+    /** Criteria to evaluate */
+    criteria: MetricsCriterion[];
+}
+/**
+ * Data type for metrics evaluator output.
+ */
+export type MetricsEvaluatorData = MetricsEvaluationSummary;
+/**
+ * Configuration for a custom evaluator.
+ * Users can extend this with their own properties.
+ */
+export interface CustomEvaluatorConfig extends EvaluatorConfig {
+    /** Custom evaluator type name */
+    customType: string;
+    /** Additional custom properties */
+    [key: string]: unknown;
+}
+/**
+ * Data type for custom evaluator output.
+ * Users can define their own output structure.
+ */
+export type CustomEvaluatorData = Record<string, unknown>;
+/**
+ * Metric direction for ranking interpretation.
+ */
+export type MetricDirection = "higher-better" | "lower-better";
+/**
+ * Ranking of a SUT for a specific metric.
+ */
+export interface SutMetricRanking {
+    /** SUT identifier */
+    sut: string;
+    /** Mean value for this metric */
+    mean: number;
+    /** Median value for this metric */
+    median: number;
+    /** Standard deviation */
+    std?: number;
+    /** Rank (1 = best based on metric direction) */
+    rank: number;
+    /** Number of observations */
+    n: number;
+}
+/**
+ * Pairwise comparison between two SUTs.
+ */
+export interface PairwiseComparison {
+    /** First SUT identifier */
+    sutA: string;
+    /** Second SUT identifier */
+    sutB: string;
+    /** Metric being compared */
+    metric: string;
+    /** Difference (sutA - sutB) */
+    delta: number;
+    /** Ratio (sutA / sutB) */
+    ratio: number;
+    /** p-value from statistical test */
+    pValue?: number;
+    /** Effect size (Cohen's d or similar) */
+    effectSize?: number;
+    /** Whether the difference is statistically significant */
+    significant: boolean;
+}
+/**
+ * Effect of a case class on SUT performance.
+ */
+export interface CaseClassEffect {
+    /** Case class identifier */
+    caseClass: string;
+    /** SUT identifier */
+    sut: string;
+    /** Metric being analyzed */
+    metric: string;
+    /** Deviation from the SUT's overall mean for this metric */
+    deviationFromMean: number;
+    /** Percentage deviation from mean */
+    percentageDeviation?: number;
+    /** Whether the effect is statistically significant */
+    significant: boolean;
+}
+/**
+ * Correlation between two metrics.
+ */
+export interface MetricCorrelation {
+    /** First metric */
+    metricA: string;
+    /** Second metric */
+    metricB: string;
+    /** Pearson correlation coefficient */
+    pearsonR: number;
+    /** Spearman rank correlation coefficient */
+    spearmanRho?: number;
+    /** Human-readable interpretation */
+    interpretation: string;
+}
+/**
+ * Configuration for the exploratory evaluator.
+ */
+export interface ExploratoryEvaluatorConfig extends EvaluatorConfig {
+    /** Metrics to analyze (if not specified, all available metrics are used) */
+    metrics?: string[];
+    /** SUTs to include (if not specified, all available SUTs are used) */
+    suts?: string[];
+    /** Metric directions for ranking interpretation */
+    metricDirections?: Record<string, MetricDirection>;
+    /** Significance level for statistical tests (default: 0.05) */
+    significanceLevel?: number;
+    /** Minimum effect size to consider meaningful */
+    minEffectSize?: number;
+    /** Whether to compute metric correlations */
+    computeCorrelations?: boolean;
+    /** Whether to analyze case-class effects */
+    analyzeCaseClassEffects?: boolean;
+}
+/**
+ * Summary of exploratory evaluation results.
+ */
+export interface ExploratoryEvaluationSummary {
+    /** Schema version */
+    version: string;
+    /** Generation timestamp */
+    timestamp: string;
+    /** SUT rankings per metric */
+    rankings: Record<string, SutMetricRanking[]>;
+    /** Pairwise comparisons between SUTs */
+    pairwiseComparisons: PairwiseComparison[];
+    /** Case-class effects (if analyzed) */
+    caseClassEffects?: CaseClassEffect[];
+    /** Metric correlations (if computed) */
+    metricCorrelations?: MetricCorrelation[];
+    /** Summary statistics */
+    summary: {
+        /** Number of SUTs analyzed */
+        sutsAnalyzed: number;
+        /** Number of metrics analyzed */
+        metricsAnalyzed: number;
+        /** Number of pairwise comparisons */
+        pairwiseComparisonsCount: number;
+        /** Number of significant differences found */
+        significantDifferences: number;
+        /** Number of case classes analyzed */
+        caseClassesAnalyzed?: number;
+        /** Best SUT per metric */
+        bestSutPerMetric: Record<string, string>;
+    };
+}
+/**
+ * Data type for exploratory evaluator output.
+ */
+export type ExploratoryEvaluatorData = ExploratoryEvaluationSummary;
+//# sourceMappingURL=evaluator.d.ts.map

package/dist/types/evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/types/evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpD;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,YAAY,GAAG,SAAS,GAAG,aAAa,GAAG,QAAQ,CAAC;AAE5F;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,2BAA2B;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,yCAAyC;IACzC,KAAK,EAAE,OAAO,CAAC;IAEf,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAElB,oCAAoC;IACpC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,UAAU;IAC1B,gDAAgD;IAChD,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAE9B;;;;;OAKG;IACH,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAAC;IAElD;;;;;;OAMG;IACH,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAErE;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC;CAChE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,MAAM,SAAS,CAAC,OAAO,SAAS,eAAe,EAAE,MAAM,EAAE,OAAO,IAAI,UAAU,GAAG;IACtF,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,gBAAgB,CAAC;IAClD,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACpE,SAAS,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,CAAC,GAAG,iBAAiB,CAAC;CAChE,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB,CAAC,CAAC;IAClC,+CAA+C;IAC/C,IAAI,EAAE,cAAc,CAAC;IAErB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAElB,oCAAoC;IACpC,IAAI,EAAE,CAAC,CAAC;IAER,wBAAwB;IACxB,QAAQ,CAAC,EAAE;QACV,2BAA2B;QAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB,yBAAyB;QACzB,MAAM,CAAC,EAAE,eAAe,CAAC;QAEzB,0BAA0B;QAC1B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IAEd,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,iCAAiC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,iCAAiC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CAC7C;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,yBAAyB;IACzB,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAE/B,sDAAsD;IACtD,UAAU,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAEhC,oCAAoC;IACpC,QAAQ,CAAC,EAAE;QACV,+BAA+B;QAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;QAEhB,6CAA6C;QAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB,+CAA+C;QAC/C,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAE7B,0BAA0B;QAC1B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;KACvB,CAAC;CACF;AAMD,OAAO,KAAK,EAAE,sBAAsB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE3E;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC7D,yBAAyB;IACzB,MAAM,EAAE,eAAe,EAAE,CAAC;IAE1B,2CAA2C;IAC3C,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,4CAA4C;IAC5C,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,sBAAsB,CAAC;AAMzD,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,eAAe;IACjE,yBAAyB;IACzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAElB,4BAA4B;IAC5B,aAAa,EAAE,MAAM,EAAE,CAAC;IAExB,8CAA8C;IAC9C,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B,4CAA4C;IAC5C,YAAY,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;AAM/D;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,GAAG,UAAU,GAAG,cAAc,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IAEpB,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAC;IAEpB,wBAAwB;IACxB,IAAI,EAAE,oBAAoB,CAAC;IAE3B,yBAAyB;IACzB,MAAM,EAAE,MAAM,CAAC;IAEf,4CAA4C;IAC5C,GAAG,EAAE,MAAM,CAAC;IAEZ,wCAAwC;IACxC,SAAS,CAAC,EAAE;QACX,QAAQ,EAAE,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC;QAC7C,KAAK,EAAE,MAAM,CAAC;KACd,CAAC;IAEF,gDAAgD;IAChD,QAAQ,CAAC,EAAE;QACV,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,EAAE,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC;KAC7C,CAAC;IAEF,2CAA2C;IAC3C,WAAW,CAAC,EAAE;QACb,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,YAAY,CAAC,EAAE,OAAO,CAAC;KACvB,CAAC;IAEF,iCAAiC;IACjC,gBAAgB,CAAC,EAAE;QAClB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;KAC9B,CAAC;IAEF,yBAAyB;IACzB,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,oCAAoC;IACpC,SAAS,EAAE,gBAAgB,CAAC;IAE5B,uBAAuB;IACvB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,cAAc,CAAC;IAEzC,+BAA+B;IAC/B,QAAQ,EAAE;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACd,EAAE,CAAC;IAEJ,8BAA8B;IAC9B,QAAQ,EAAE;QACT,IAAI,EAAE,oBAAoB,CAAC;QAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE;YAAE,GAAG,CAAC,EAAE,MAAM,CAAC;YAAC,GAAG,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAC7C,CAAC;IAEF,qDAAqD;IACrD,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC,qBAAqB;IACrB,OAAO,EAAE,MAAM,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAElB,mCAAmC;IACnC,OAAO,EAAE,sBAAsB,EAAE,CAAC;IAElC,yBAAyB;IACzB,OAAO,EAAE;QACR,+BAA+B;QAC/B,KAAK,EAAE,MAAM,CAAC;QAEd,sBAAsB;QACtB,MAAM,EAAE,MAAM,CAAC;QAEf,sBAAsB;QACtB,MAAM,EAAE,MAAM,CAAC;QAEf,4BAA4B;QAC5B,YAAY,EAAE,MAAM,CAAC;QAErB,wBAAwB;QACxB,QAAQ,EAAE,MAAM,CAAC;QAEjB,uBAAuB;QACvB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACtC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,WAAW,sBAAuB,SAAQ,eAAe;IAC9D,2BAA2B;IAC3B,QAAQ,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,wBAAwB,CAAC;AAM5D;;;GAGG;AACH,MAAM,WAAW,qBAAsB,SAAQ,eAAe;IAC7D,iCAAiC;IACjC,UAAU,EAAE,MAAM,CAAC;IAEnB,mCAAmC;IACnC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAM1D;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,eAAe,GAAG,cAAc,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,qBAAqB;IACrB,GAAG,EAAE,MAAM,CAAC;IAEZ,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IAEb,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;IAEf,yBAAyB;IACzB,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,gDAAgD;IAChD,IAAI,EAAE,MAAM,CAAC;IAEb,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;CACV;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,2BAA2B;IAC3B,IAAI,EAAE,MAAM,CAAC;IAEb,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IAEb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IAEf,+BAA+B;IAC/B,KAAK,EAAE,MAAM,CAAC;IAEd,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IAEd,oCAAoC;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,yCAAyC;IACzC,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,0DAA0D;IAC1D,WAAW,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAC;IAElB,qBAAqB;IACrB,GAAG,EAAE,MAAM,CAAC;IAEZ,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IAEf,4DAA4D;IAC5D,iBAAiB,EAAE,MAAM,CAAC;IAE1B,qCAAqC;IACrC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B,sDAAsD;IACtD,WAAW,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,mBAAmB;IACnB,OAAO,EAAE,MAAM,CAAC;IAEhB,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAEhB,sCAAsC;IACtC,QAAQ,EAAE,MAAM,CAAC;IAEjB,4CAA4C;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,oCAAoC;IACpC,cAAc,EAAE,MAAM,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,0BAA2B,SAAQ,eAAe;IAClE,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAEnB,sEAAsE;IACtE,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAEhB,mDAAmD;IACnD,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IAEnD,+DAA+D;IAC/D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,iDAAiD;IACjD,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,6CAA6C;IAC7C,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B,4CAA4C;IAC5C,uBAAuB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED;;GAEG;AACH,MAAM,WAAW,4BAA4B;IAC5C,qBAAqB;IACrB,OAAO,EAAE,MAAM,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAElB,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAE7C,wCAAwC;IACxC,mBAAmB,EAAE,kBAAkB,EAAE,CAAC;IAE1C,uCAAuC;IACvC,gBAAgB,CAAC,EAAE,eAAe,EAAE,CAAC;IAErC,wCAAwC;IACxC,kBAAkB,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAEzC,yBAAyB;IACzB,OAAO,EAAE;QACR,8BAA8B;QAC9B,YAAY,EAAE,MAAM,CAAC;QAErB,iCAAiC;QACjC,eAAe,EAAE,MAAM,CAAC;QAExB,qCAAqC;QACrC,wBAAwB,EAAE,MAAM,CAAC;QAEjC,8CAA8C;QAC9C,sBAAsB,EAAE,MAAM,CAAC;QAE/B,sCAAsC;QACtC,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAE7B,0BAA0B;QAC1B,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACzC,CAAC;CACF;AAED;;GAEG;AACH,MAAM,MAAM,wBAAwB,GAAG,4BAA4B,CAAC"}

package/dist/types/evaluator.js

@@ -0,0 +1,9 @@
+/**
+ * Evaluator Type Definitions
+ *
+ * Core abstractions for the extensible evaluation system.
+ * All evaluators implement the Evaluator interface, enabling
+ * a unified API for claims, robustness, metrics, and custom evaluations.
+ */
+export {};
+//# sourceMappingURL=evaluator.js.map

package/dist/types/evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../../src/types/evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG"}

package/dist/types/result.d.ts

@@ -88,6 +88,8 @@ export interface ResultMetrics {
     numeric: Record<string, number>;
     /** Additional metrics (overflow) */
     extra?: Record<string, number>;
+    /** Allow arbitrary number properties for scenario-specific metrics */
+    [key: string]: number | Record<string, number> | undefined;
 }
 /**
  * Provenance information for reproducibility.

package/dist/types/result.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../../src/types/result.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAExC;;GAEG;AACH,MAAM,MAAM,WAAW,GACpB,WAAW,GACX,mBAAmB,GACnB,sBAAsB,GACtB,WAAW,GACX,iBAAiB,GACjB,SAAS,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IAEd,qBAAqB;IACrB,GAAG,EAAE,MAAM,CAAC;IAEZ,yCAAyC;IACzC,OAAO,EAAE,OAAO,CAAC;IAEjB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IAEf,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAEnC,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,6CAA6C;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,wDAAwD;IACxD,cAAc,EAAE,OAAO,CAAC;IAExB,0CAA0C;IAC1C,cAAc,EAAE,OAAO,CAAC;IAExB,2CAA2C;IAC3C,KAAK,EAAE,OAAO,CAAC;IAEf,0DAA0D;IAC1D,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;IAEhC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,mCAAmC;IACnC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IAEf,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IAEd,mCAAmC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC;IAElD,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAEnC,sBAAsB;IACtB,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IAEvB,wCAAwC;IACxC,SAAS,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAEhC,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhC,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,4BAA4B;IAC5B,OAAO,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,wDAAwD;IACxD,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,uDAAuD;IACvD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B,2CAA2C;IAC3C,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IAExB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,iDAAiD;IACjD,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAChC,+BAA+B;IAC/B,GAAG,EAAE,UAAU,CAAC;IAEhB,6BAA6B;IAC7B,WAAW,EAAE,iBAAiB,CAAC;IAE/B,qCAAqC;IACrC,OAAO,EAAE,aAAa,CAAC;IAEvB,sBAAsB;IACtB,OAAO,EAAE,aAAa,CAAC;IAEvB,qCAAqC;IACrC,UAAU,EAAE,UAAU,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,qBAAqB;IACrB,OAAO,EAAE,MAAM,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAElB,gCAAgC;IAChC,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAE5B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC"}
+{"version":3,"file":"result.d.ts","sourceRoot":"","sources":["../../src/types/result.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC9D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAExC;;GAEG;AACH,MAAM,MAAM,WAAW,GACpB,WAAW,GACX,mBAAmB,GACnB,sBAAsB,GACtB,WAAW,GACX,iBAAiB,GACjB,SAAS,CAAC;AAEb;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IAEd,qBAAqB;IACrB,GAAG,EAAE,MAAM,CAAC;IAEZ,yCAAyC;IACzC,OAAO,EAAE,OAAO,CAAC;IAEjB,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IAEf,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,2CAA2C;IAC3C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAEnC,gCAAgC;IAChC,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,6CAA6C;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,wDAAwD;IACxD,cAAc,EAAE,OAAO,CAAC;IAExB,0CAA0C;IAC1C,cAAc,EAAE,OAAO,CAAC;IAExB,2CAA2C;IAC3C,KAAK,EAAE,OAAO,CAAC;IAEf,0DAA0D;IAC1D,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;IAEhC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,WAAW,CAAC;IAE1B,mCAAmC;IACnC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,sBAAsB;IACtB,MAAM,EAAE,MAAM,CAAC;IAEf,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IAEd,mCAAmC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC;IAElD,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAEnC,sBAAsB;IACtB,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IAEvB,wCAAwC;IACxC,SAAS,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAEhC,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,8BAA8B;IAC9B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhC,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/B,sEAAsE;IACtE,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CAC3D;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,4BAA4B;IAC5B,OAAO,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,sBAAsB;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,wDAAwD;IACxD,KAAK,CAAC,EAAE,OAAO,CAAC;IAEhB,uDAAuD;IACvD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B,2CAA2C;IAC3C,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IAExB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gDAAgD;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,iDAAiD;IACjD,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB,yCAAyC;IACzC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAChC,+BAA+B;IAC/B,GAAG,EAAE,UAAU,CAAC;IAEhB,6BAA6B;IAC7B,WAAW,EAAE,iBAAiB,CAAC;IAE/B,qCAAqC;IACrC,OAAO,EAAE,aAAa,CAAC;IAEvB,sBAAsB;IACtB,OAAO,EAAE,aAAa,CAAC;IAEvB,qCAAqC;IACrC,UAAU,EAAE,UAAU,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,qBAAqB;IACrB,OAAO,EAAE,MAAM,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAElB,gCAAgC;IAChC,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAE5B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ppef",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Portable Programmatic Evaluation Framework - Claim-driven, deterministic evaluation for experiments",
   "type": "module",
   "main": "dist/index.cjs",

package/dist/claims/__tests__/evaluator.unit.test.d.ts

@@ -1,12 +0,0 @@
-/**
- * Unit tests for Claims Evaluator
- *
- * Tests the evaluateClaim function and related functionality including:
- * - Satisfied claims with various directions
- * - Violated claims
- * - Inconclusive results from missing data
- * - Evidence computation (delta, ratio, pValue, effectSize)
- * - Scope filtering with caseClass constraints
- */
-export {};
-//# sourceMappingURL=evaluator.unit.test.d.ts.map

package/dist/claims/__tests__/evaluator.unit.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"evaluator.unit.test.d.ts","sourceRoot":"","sources":["../../../src/claims/__tests__/evaluator.unit.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG"}