ppef 1.0.1 → 1.2.0

package/dist/evaluators/claims-evaluator.js

@@ -0,0 +1,289 @@
+/**
+ * Claims Evaluator
+ *
+ * Evaluates explicit hypotheses (claims) against aggregated results.
+ * Refactored from src/claims/evaluator.ts into a class-based design
+ * that implements the Evaluator interface.
+ */
+/**
+ * Claims evaluator - evaluates hypotheses against aggregated results.
+ */
+export class ClaimsEvaluator {
+    /** Type identifier */
+    type = "claims";
+    /** Schema version */
+    static VERSION = "1.0.0";
+    /**
+     * Validate claims evaluator configuration.
+     *
+     * @param config - Configuration to validate
+     * @returns Validation result
+     */
+    validateConfig(config) {
+        const errors = [];
+        const warnings = [];
+        // Check claims array
+        if (!Array.isArray(config.claims)) {
+            errors.push("claims must be an array");
+            return { valid: false, errors, warnings };
+        }
+        if (config.claims.length === 0) {
+            warnings.push("No claims provided - evaluation will produce empty results");
+        }
+        // Validate each claim
+        for (let i = 0; i < config.claims.length; i++) {
+            const claim = config.claims[i];
+            const claimErrors = this.validateClaim(claim, i);
+            errors.push(...claimErrors);
+        }
+        return {
+            valid: errors.length === 0,
+            errors: errors.length > 0 ? errors : undefined,
+            warnings: warnings.length > 0 ? warnings : undefined,
+        };
+    }
+    /**
+     * Validate a single claim.
+     *
+     * @param claim - Claim to validate
+     * @param index - Index in claims array (for error messages)
+     * @returns Array of error messages
+     */
+    validateClaim(claim, index) {
+        const errors = [];
+        const prefix = `Claim[${index}]`;
+        if (!claim.claimId || typeof claim.claimId !== "string") {
+            errors.push(`${prefix}: claimId is required`);
+        }
+        if (!claim.description || typeof claim.description !== "string") {
+            errors.push(`${prefix}: description is required`);
+        }
+        if (!claim.sut || typeof claim.sut !== "string") {
+            errors.push(`${prefix}: sut is required`);
+        }
+        if (!claim.baseline || typeof claim.baseline !== "string") {
+            errors.push(`${prefix}: baseline is required`);
+        }
+        if (!claim.metric || typeof claim.metric !== "string") {
+            errors.push(`${prefix}: metric is required`);
+        }
+        if (!["greater", "less", "equal"].includes(claim.direction)) {
+            errors.push(`${prefix}: direction must be 'greater', 'less', or 'equal'`);
+        }
+        if (claim.threshold !== undefined && typeof claim.threshold !== "number") {
+            errors.push(`${prefix}: threshold must be a number`);
+        }
+        if (!["global", "caseClass", "parameterRange", "localStructure"].includes(claim.scope)) {
+            errors.push(`${prefix}: scope must be a valid ValidityScope`);
+        }
+        return errors;
+    }
+    /**
+     * Evaluate claims against aggregated results.
+     *
+     * @param config - Claims evaluator configuration
+     * @param input - Evaluation context with aggregates
+     * @returns Evaluation output
+     */
+    evaluate(config, input) {
+        const { aggregates } = input;
+        // Evaluate all claims
+        const evaluations = config.claims.map((claim) => this.evaluateClaim(claim, aggregates));
+        // Create summary
+        const summary = this.createClaimSummary(evaluations);
+        return {
+            type: "claims",
+            version: ClaimsEvaluator.VERSION,
+            timestamp: new Date().toISOString(),
+            data: summary,
+            metadata: {
+                inputSource: input.metadata?.source,
+                config,
+            },
+        };
+    }
+    /**
+     * Evaluate a single claim against aggregated results.
+     *
+     * @param claim - The claim to evaluate
+     * @param aggregates - Aggregated results from the pipeline
+     * @returns Claim evaluation with status and evidence
+     */
+    evaluateClaim(claim, aggregates) {
+        // Filter aggregates by scope constraints
+        const filteredAggregates = this.filterByScope(aggregates, claim);
+        // Find primary and baseline aggregates
+        const primaryAgg = filteredAggregates.find((a) => a.sut === claim.sut);
+        const baselineAgg = filteredAggregates.find((a) => a.sut === claim.baseline);
+        // Handle missing data
+        if (!primaryAgg || !baselineAgg) {
+            return this.createInconclusiveResult(claim, primaryAgg ? undefined : "Primary SUT not found", baselineAgg ? undefined : "Baseline SUT not found");
+        }
+        // Get metric values
+        const primaryMetric = claim.metric;
+        const baselineMetric = claim.metric;
+        const primaryStats = primaryAgg.metrics[primaryMetric];
+        const baselineStats = baselineAgg.metrics[baselineMetric];
+        if (!(primaryMetric in primaryAgg.metrics) || !(baselineMetric in baselineAgg.metrics)) {
+            return this.createInconclusiveResult(claim, "Metric not found in primary results", "Metric not found in baseline results");
+        }
+        // Compute evidence
+        const primaryValue = primaryStats.mean;
+        const baselineValue = baselineStats.mean;
+        const delta = primaryValue - baselineValue;
+        const ratio = baselineValue === 0 ? Infinity : primaryValue / baselineValue;
+        // Get statistical significance if available
+        const comparison = primaryAgg.comparisons?.[claim.baseline];
+        const pValue = comparison?.pValue;
+        const effectSize = comparison?.effectSize;
+        const evidence = {
+            primaryValue,
+            baselineValue,
+            delta,
+            ratio,
+            pValue,
+            effectSize,
+            n: primaryStats.n + baselineStats.n,
+        };
+        // Determine claim status
+        const status = this.determineClaimStatus(claim, evidence);
+        return {
+            claim,
+            status,
+            evidence,
+        };
+    }
+    /**
+     * Filter aggregates by claim scope constraints.
+     *
+     * @param aggregates - All aggregates
+     * @param claim - Claim with scope constraints
+     * @returns Filtered aggregates
+     */
+    filterByScope(aggregates, claim) {
+        if (!claim.scopeConstraints) {
+            return aggregates;
+        }
+        return aggregates.filter((agg) => {
+            for (const [key, value] of Object.entries(claim.scopeConstraints ?? {})) {
+                if (key === "caseClass") {
+                    const allowedClasses = Array.isArray(value) ? value : [value];
+                    if (!allowedClasses.includes(agg.caseClass)) {
+                        return false;
+                    }
+                }
+                // Add more scope constraint checks as needed
+            }
+            return true;
+        });
+    }
+    /**
+     * Create an inconclusive result with reasons.
+     *
+     * @param claim - The claim being evaluated
+     * @param reasons - Reasons for inconclusive status
+     * @returns Inconclusive claim evaluation
+     */
+    createInconclusiveResult(claim, ...reasons) {
+        const validReasons = reasons.filter((r) => r !== undefined);
+        return {
+            claim,
+            status: "inconclusive",
+            evidence: {
+                primaryValue: Number.NaN,
+                baselineValue: Number.NaN,
+                delta: Number.NaN,
+                ratio: Number.NaN,
+            },
+            inconclusiveReason: validReasons.join("; "),
+        };
+    }
+    /**
+     * Determine claim status based on evidence.
+     *
+     * @param claim - The claim being evaluated
+     * @param evidence - Computed evidence
+     * @returns Claim status
+     */
+    determineClaimStatus(claim, evidence) {
+        // Check for missing data
+        if (Number.isNaN(evidence.primaryValue) || Number.isNaN(evidence.baselineValue)) {
+            return "inconclusive";
+        }
+        // Check statistical significance if required
+        const significanceLevel = claim.significanceLevel ?? 0.05;
+        if (evidence.pValue !== undefined && evidence.pValue > significanceLevel) {
+            return "inconclusive";
+        }
+        // Check minimum effect size if required
+        if (claim.minEffectSize !== undefined &&
+            evidence.effectSize !== undefined &&
+            Math.abs(evidence.effectSize) < claim.minEffectSize) {
+            return "inconclusive";
+        }
+        // Evaluate direction
+        switch (claim.direction) {
+            case "greater": {
+                if (claim.threshold !== undefined) {
+                    return evidence.delta >= claim.threshold ? "satisfied" : "violated";
+                }
+                return evidence.delta > 0 ? "satisfied" : "violated";
+            }
+            case "less": {
+                if (claim.threshold !== undefined) {
+                    return evidence.delta <= -claim.threshold ? "satisfied" : "violated";
+                }
+                return evidence.delta < 0 ? "satisfied" : "violated";
+            }
+            case "equal": {
+                const epsilon = claim.threshold ?? 0.001;
+                return Math.abs(evidence.delta) <= epsilon ? "satisfied" : "violated";
+            }
+        }
+    }
+    /**
+     * Create a claim evaluation summary.
+     *
+     * @param evaluations - Completed claim evaluations
+     * @returns Summary with counts and rates
+     */
+    createClaimSummary(evaluations) {
+        const satisfied = evaluations.filter((e) => e.status === "satisfied").length;
+        const violated = evaluations.filter((e) => e.status === "violated").length;
+        const inconclusive = evaluations.filter((e) => e.status === "inconclusive").length;
+        const definitive = satisfied + violated;
+        const satisfactionRate = definitive > 0 ? satisfied / definitive : 0;
+        return {
+            version: "1.0.0",
+            timestamp: new Date().toISOString(),
+            evaluations,
+            summary: {
+                total: evaluations.length,
+                satisfied,
+                violated,
+                inconclusive,
+                satisfactionRate,
+            },
+        };
+    }
+    /**
+     * Summarize evaluation output.
+     *
+     * @param output - Evaluation output to summarize
+     * @returns Summary statistics
+     */
+    summarize(output) {
+        const { summary } = output.data;
+        return {
+            total: summary.total,
+            passed: summary.satisfied,
+            failed: summary.violated,
+            inconclusive: summary.inconclusive,
+            passRate: summary.satisfactionRate,
+            additional: {
+                satisfactionRate: summary.satisfactionRate,
+            },
+        };
+    }
+}
+//# sourceMappingURL=claims-evaluator.js.map

package/dist/evaluators/claims-evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claims-evaluator.js","sourceRoot":"","sources":["../../src/evaluators/claims-evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAsBH;;GAEG;AACH,MAAM,OAAO,eAAe;IAG3B,sBAAsB;IACb,IAAI,GAAG,QAAiB,CAAC;IAElC,qBAAqB;IACb,MAAM,CAAU,OAAO,GAAG,OAAO,CAAC;IAE1C;;;;;OAKG;IACH,cAAc,CAAC,MAA6B;QAC3C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;YACvC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAC3C,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,QAAQ,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC7E,CAAC;QAED,sBAAsB;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7B,CAAC;QAED,OAAO;YACN,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC9C,QAAQ,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;SACpD,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CAAC,KAAsB,EAAE,KAAa;QAC1D,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,SAAS,KAAK,GAAG,CAAC;QAEjC,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,uBAAuB,CAAC,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,2BAA2B,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,mBAAmB,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,wBAAwB,CAAC,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,sBAAsB,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,mDAAmD,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YAC1E,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,8BAA8B,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACxF,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;QAC/D,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACH,QAAQ,CACP,MAA6B,EAC7B,KAAwB;QAExB,MAAM,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;QAE7B,sBAAsB;QACtB,MAAM,WAAW,GAAsB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAClE,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,CACrC,CAAC;QAEF,iBAAiB;QACjB,MAAM,OAAO,GAAG,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,CAAC;QAErD,OAAO;YACN,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,eAAe,CAAC,OAAO;YAChC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE,OAAO;YACb,QAAQ,EAAE;gBACT,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM;gBACnC,MAAM;aACN;SACD,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CAAC,KAAsB,EAAE,UAA8B;QAC3E,yCAAyC;QACzC,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEjE,uCAAuC;QACvC,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,CAAC,CAAC;QACvE,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,QAAQ,CAAC,CAAC;QAE7E,sBAAsB;QACtB,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,OAAO,IAAI,CAAC,wBAAwB,CACnC,KAAK,EACL,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,uBAAuB,EAChD,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,wBAAwB,CAClD,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC;QACnC,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC;QACpC,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACvD,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAE1D,IAAI,CAAC,CAAC,aAAa,IAAI,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;YACxF,OAAO,IAAI,CAAC,wBAAwB,CACnC,KAAK,EACL,qCAAqC,EACrC,sCAAsC,CACtC,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC;QACvC,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC;QACzC,MAAM,KAAK,GAAG,YAAY,GAAG,aAAa,CAAC;QAC3C,MAAM,KAAK,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,YAAY,GAAG,aAAa,CAAC;QAE5E,4CAA4C;QAC5C,MAAM,UAAU,GAAG,UAAU,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,UAAU,EAAE,MAAM,CAAC;QAClC,MAAM,UAAU,GAAG,UAAU,EAAE,UAAU,CAAC;QAE1C,MAAM,QAAQ,GAAkB;YAC/B,YAAY;YACZ,aAAa;YACb,KAAK;YACL,KAAK;YACL,MAAM;YACN,UAAU;YACV,CAAC,EAAE,YAAY,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC;SACnC,CAAC;QAEF,yBAAyB;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAE1D,OAAO;YACN,KAAK;YACL,MAAM;YACN,QAAQ;SACR,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,aAAa,CACpB,UAA8B,EAC9B,KAAsB;QAEtB,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;YAC7B,OAAO,UAAU,CAAC;QACnB,CAAC;QAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,IAAI,EAAE,CAAC,EAAE,CAAC;gBACzE,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;oBACzB,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAC9D,IAAI,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAsB,CAAC,EAAE,CAAC;wBAC1D,OAAO,KAAK,CAAC;oBACd,CAAC;gBACF,CAAC;gBACD,6CAA6C;YAC9C,CAAC;YACD,OAAO,IAAI,CAAC;QACb,CAAC,CAAC,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,wBAAwB,CAC/B,KAAsB,EACtB,GAAG,OAA+B;QAElC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;QAEzE,OAAO;YACN,KAAK;YACL,MAAM,EAAE,cAAc;YACtB,QAAQ,EAAE;gBACT,YAAY,EAAE,MAAM,CAAC,GAAG;gBACxB,aAAa,EAAE,MAAM,CAAC,GAAG;gBACzB,KAAK,EAAE,MAAM,CAAC,GAAG;gBACjB,KAAK,EAAE,MAAM,CAAC,GAAG;aACjB;YACD,kBAAkB,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3C,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,oBAAoB,CAAC,KAAsB,EAAE,QAAuB;QAC3E,yBAAyB;QACzB,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;YACjF,OAAO,cAAc,CAAC;QACvB,CAAC;QAED,6CAA6C;QAC7C,MAAM,iBAAiB,GAAG,KAAK,CAAC,iBAAiB,IAAI,IAAI,CAAC;QAC1D,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;YAC1E,OAAO,cAAc,CAAC;QACvB,CAAC;QAED,wCAAwC;QACxC,IACC,KAAK,CAAC,aAAa,KAAK,SAAS;YACjC,QAAQ,CAAC,UAAU,KAAK,SAAS;YACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC,aAAa,EAClD,CAAC;YACF,OAAO,cAAc,CAAC;QACvB,CAAC;QAED,qBAAqB;QACrB,QAAQ,KAAK,CAAC,SAAS,EAAE,CAAC;YACzB,KAAK,SAAS,CAAC,CAAC,CAAC;gBAChB,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBACnC,OAAO,QAAQ,CAAC,KAAK,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;gBACrE,CAAC;gBACD,OAAO,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YACtD,CAAC;YAED,KAAK,MAAM,CAAC,CAAC,CAAC;gBACb,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBACnC,OAAO,QAAQ,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;gBACtE,CAAC;gBACD,OAAO,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YACtD,CAAC;YAED,KAAK,OAAO,CAAC,CAAC,CAAC;gBACd,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC;gBACzC,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YACvE,CAAC;QACF,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACK,kBAAkB,CAAC,WAA8B;QACxD,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,MAAM,CAAC;QAC7E,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,MAAM,CAAC;QAC3E,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM,CAAC;QAEnF,MAAM,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;QACxC,MAAM,gBAAgB,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAErE,OAAO;YACN,OAAO,EAAE,OAAO;YAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,WAAW;YACX,OAAO,EAAE;gBACR,KAAK,EAAE,WAAW,CAAC,MAAM;gBACzB,SAAS;gBACT,QAAQ;gBACR,YAAY;gBACZ,gBAAgB;aAChB;SACD,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,MAA6C;QACtD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;QAEhC,OAAO;YACN,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,SAAS;YACzB,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,QAAQ,EAAE,OAAO,CAAC,gBAAgB;YAClC,UAAU,EAAE;gBACX,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;aAC1C;SACD,CAAC;IACH,CAAC"}

package/dist/evaluators/exploratory-evaluator.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Exploratory Evaluator
+ *
+ * Hypothesis-free analysis for discovering patterns in evaluation data.
+ * Unlike ClaimsEvaluator which tests predefined hypotheses, this evaluator
+ * performs exploratory analysis including:
+ * - Ranking all SUTs by any metric (not just primary vs baseline)
+ * - Finding significant pairwise differences (N-way comparisons)
+ * - Discovering case-class effects
+ * - Computing metric correlations
+ */
+import type { EvaluationContext, EvaluationOutput, EvaluationSummary, Evaluator, ExploratoryEvaluatorConfig, ExploratoryEvaluatorData, IEvaluator, ValidationResult } from "../types/evaluator.js";
+/**
+ * Exploratory evaluator - hypothesis-free comparative analysis.
+ */
+export declare class ExploratoryEvaluator implements Evaluator<ExploratoryEvaluatorConfig, EvaluationContext, ExploratoryEvaluatorData>, IEvaluator {
+    /** Type identifier */
+    readonly type: "exploratory";
+    /** Schema version */
+    private static readonly VERSION;
+    /** Default significance level */
+    private static readonly DEFAULT_SIGNIFICANCE;
+    /**
+     * Validate exploratory evaluator configuration.
+     *
+     * @param config - Configuration to validate
+     * @returns Validation result
+     */
+    validateConfig(config: ExploratoryEvaluatorConfig): ValidationResult;
+    /**
+     * Perform exploratory evaluation.
+     *
+     * @param config - Exploratory evaluator configuration
+     * @param input - Evaluation context with aggregates
+     * @returns Evaluation output
+     */
+    evaluate(config: ExploratoryEvaluatorConfig, input: EvaluationContext): EvaluationOutput<ExploratoryEvaluatorData>;
+    /**
+     * Summarize evaluation output.
+     *
+     * @param output - Evaluation output to summarize
+     * @returns Summary statistics
+     */
+    summarize(output: EvaluationOutput<ExploratoryEvaluatorData>): EvaluationSummary;
+    /**
+     * Determine which SUTs to analyze.
+     * @param aggregates
+     * @param configSuts
+     */
+    private determineSuts;
+    /**
+     * Determine which metrics to analyze.
+     * @param aggregates
+     * @param configMetrics
+     */
+    private determineMetrics;
+    /**
+     * Compute rankings for a single metric.
+     * @param aggregates
+     * @param metric
+     * @param direction
+     */
+    private computeRankings;
+    /**
+     * Compute all pairwise comparisons.
+     * @param aggregates
+     * @param suts
+     * @param metrics
+     * @param significanceLevel
+     * @param minEffectSize
+     */
+    private computePairwiseComparisons;
+    /**
+     * Compare a single pair of SUTs for a metric.
+     * @param aggregates
+     * @param sutA
+     * @param sutB
+     * @param metric
+     * @param significanceLevel
+     * @param minEffectSize
+     */
+    private compareSutPair;
+    /**
+     * Analyze case-class effects on SUT performance.
+     * @param aggregates
+     * @param metrics
+     * @param significanceLevel
+     */
+    private analyzeCaseClassEffects;
+    /**
+     * Compute correlations between metrics.
+     * @param aggregates
+     * @param metrics
+     */
+    private computeMetricCorrelations;
+    /**
+     * Compute Pearson and Spearman correlation between two metrics.
+     * @param aggregates
+     * @param metricA
+     * @param metricB
+     */
+    private computeCorrelation;
+    /**
+     * Compute Pearson correlation coefficient.
+     * @param x
+     * @param y
+     */
+    private pearsonCorrelation;
+    /**
+     * Compute Spearman rank correlation coefficient.
+     * @param x
+     * @param y
+     */
+    private spearmanCorrelation;
+    /**
+     * Compute ranks for an array of values (handling ties).
+     * @param values
+     */
+    private computeRanks;
+    /**
+     * Interpret correlation coefficient.
+     * @param r
+     */
+    private interpretCorrelation;
+    /**
+     * Compute variance of an array.
+     * @param values
+     */
+    private variance;
+    /**
+     * Standard normal CDF approximation.
+     * @param z
+     */
+    private normalCdf;
+}
+//# sourceMappingURL=exploratory-evaluator.d.ts.map

package/dist/evaluators/exploratory-evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"exploratory-evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluators/exploratory-evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAEX,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,SAAS,EACT,0BAA0B,EAC1B,wBAAwB,EAExB,UAAU,EAKV,gBAAgB,EAChB,MAAM,uBAAuB,CAAC;AAE/B;;GAEG;AACH,qBAAa,oBACZ,YACC,SAAS,CAAC,0BAA0B,EAAE,iBAAiB,EAAE,wBAAwB,CAAC,EAClF,UAAU;IAEX,sBAAsB;IACtB,QAAQ,CAAC,IAAI,EAAG,aAAa,CAAU;IAEvC,qBAAqB;IACrB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAW;IAE1C,iCAAiC;IACjC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAEpD;;;;;OAKG;IACH,cAAc,CAAC,MAAM,EAAE,0BAA0B,GAAG,gBAAgB;IA2CpE;;;;;;OAMG;IACH,QAAQ,CACP,MAAM,EAAE,0BAA0B,EAClC,KAAK,EAAE,iBAAiB,GACtB,gBAAgB,CAAC,wBAAwB,CAAC;IAsF7C;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,gBAAgB,CAAC,wBAAwB,CAAC,GAAG,iBAAiB;IAchF;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAQrB;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAuEvB;;;;;;;OAOG;IACH,OAAO,CAAC,0BAA0B;IAmClC;;;;;;;;OAQG;IACH,OAAO,CAAC,cAAc;IAuFtB;;;;;OAKG;IACH,OAAO,CAAC,uBAAuB;IA8D/B;;;;OAIG;IACH,OAAO,CAAC,yBAAyB;IAqBjC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IA2C1B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAqB1B;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IAM3B;;;OAGG;IACH,OAAO,CAAC,YAAY;IA0BpB;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAS5B;;;OAGG;IACH,OAAO,CAAC,QAAQ;IAMhB;;;OAGG;IACH,OAAO,CAAC,SAAS;CAiBjB"}