ppef 1.0.0 → 1.0.1

package/dist/statistical/mann-whitney-u.js

@@ -0,0 +1,127 @@
+/**
+ * Mann-Whitney U Test
+ *
+ * Statistical test for comparing two independent samples.
+ * Tests whether two populations have the same distribution.
+ *
+ * H0: Both populations have the same distribution
+ * H1: Populations have different distributions
+ *
+ * Returns p-value (smaller = more significant difference)
+ */
+/**
+ * Standard normal cumulative distribution function.
+ * Uses the Abramowitz and Stegun approximation.
+ *
+ * @param z - Z-score
+ * @returns Cumulative probability from -infinity to z
+ */
+export const normalCDF = (z) => {
+    const sign = z < 0 ? -1 : 1;
+    z = Math.abs(z) / Math.sqrt(2);
+    const a1 = 0.254_829_592;
+    const a2 = -0.284_496_736;
+    const a3 = 1.421_413_741;
+    const a4 = -1.453_152_027;
+    const a5 = 1.061_405_429;
+    const p = 0.327_591_1;
+    const t = 1 / (1 + p * z);
+    const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-z * z);
+    return 0.5 * (1 + sign * y);
+};
+/**
+ * Mann-Whitney U test for comparing two independent samples.
+ *
+ * Non-parametric test that does not assume normal distribution.
+ * Tests whether two populations have the same distribution.
+ *
+ * @param sampleA - First sample array
+ * @param sampleB - Second sample array
+ * @returns Object containing U statistic, p-value, and significance flag
+ */
+export const mannWhitneyUTest = (sampleA, sampleB) => {
+    // Rank all values combined
+    const combined = [...sampleA, ...sampleB];
+    const sorted = [...combined].sort((a, b) => a - b);
+    // Assign ranks (handle ties)
+    const ranks = new Map();
+    for (const [index, value] of sorted.entries()) {
+        if (!ranks.has(value)) {
+            ranks.set(value, []);
+        }
+        const positions = ranks.get(value);
+        if (positions) {
+            positions.push(index + 1);
+        }
+    }
+    // Average rank for tied values
+    const avgRanks = new Map();
+    for (const [value, positions] of ranks) {
+        avgRanks.set(value, positions.reduce((a, b) => a + b, 0) / positions.length);
+    }
+    // Sum ranks for each sample
+    const rankSumA = sampleA.reduce((sum, value) => sum + (avgRanks.get(value) ?? 0), 0);
+    const rankSumB = sampleB.reduce((sum, value) => sum + (avgRanks.get(value) ?? 0), 0);
+    // Calculate U statistics
+    const n1 = sampleA.length;
+    const n2 = sampleB.length;
+    const u1 = rankSumA - (n1 * (n1 + 1)) / 2;
+    const u2 = rankSumB - (n2 * (n2 + 1)) / 2;
+    const u = Math.min(u1, u2);
+    // Calculate z-score for large samples
+    const meanU = (n1 * n2) / 2;
+    const stdU = Math.sqrt((n1 * n2 * (n1 + n2 + 1)) / 12);
+    const z = stdU > 0 ? (u - meanU) / stdU : 0;
+    // Two-tailed p-value from z-score (approximation)
+    const pValue = 2 * (1 - normalCDF(Math.abs(z)));
+    return {
+        u,
+        pValue,
+        significant: pValue < 0.05, // 95% confidence level
+    };
+};
+/**
+ * Calculate Cohen's d effect size.
+ *
+ * Measures the standardized difference between two means.
+ *
+ * Interpretation:
+ * - 0.2: Small effect
+ * - 0.5: Medium effect
+ * - 0.8: Large effect
+ *
+ * @param sampleA - First sample array
+ * @param sampleB - Second sample array
+ * @returns Cohen's d effect size
+ */
+export const cohensD = (sampleA, sampleB) => {
+    const n1 = sampleA.length;
+    const n2 = sampleB.length;
+    const mean1 = sampleA.reduce((a, b) => a + b, 0) / n1;
+    const mean2 = sampleB.reduce((a, b) => a + b, 0) / n2;
+    const variable1 = sampleA.reduce((sum, value) => sum + (value - mean1) ** 2, 0) / (n1 - 1);
+    const variable2 = sampleB.reduce((sum, value) => sum + (value - mean2) ** 2, 0) / (n2 - 1);
+    const pooledStd = Math.sqrt(((n1 - 1) * variable1 + (n2 - 1) * variable2) / (n1 + n2 - 2));
+    return pooledStd > 0 ? Math.abs(mean1 - mean2) / pooledStd : 0;
+};
+/**
+ * Calculate confidence interval for a mean.
+ *
+ * Uses t-distribution approximation (1.96 for 95% CI with large samples).
+ *
+ * @param values - Sample values
+ * @returns Object with lower and upper bounds
+ */
+export const confidenceInterval = (values) => {
+    const n = values.length;
+    const mean = values.reduce((a, b) => a + b, 0) / n;
+    const std = Math.sqrt(values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (n - 1));
+    const se = std / Math.sqrt(n);
+    const t = 1.96; // Approximation for large samples (95% CI)
+    const margin = t * se;
+    return {
+        lower: mean - margin,
+        upper: mean + margin,
+    };
+};
+//# sourceMappingURL=mann-whitney-u.js.map

package/dist/statistical/mann-whitney-u.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mann-whitney-u.js","sourceRoot":"","sources":["../../src/statistical/mann-whitney-u.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,CAAS,EAAU,EAAE;IAC9C,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,aAAa,CAAC;IACzB,MAAM,EAAE,GAAG,CAAC,aAAa,CAAC;IAC1B,MAAM,EAAE,GAAG,aAAa,CAAC;IACzB,MAAM,EAAE,GAAG,CAAC,aAAa,CAAC;IAC1B,MAAM,EAAE,GAAG,aAAa,CAAC;IACzB,MAAM,CAAC,GAAG,WAAW,CAAC;IAEtB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpF,OAAO,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;AAC7B,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC/B,OAAiB,EACjB,OAAiB,EAKhB,EAAE;IACH,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEnD,6BAA6B;IAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACvB,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACtB,CAAC;QACD,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,SAAS,EAAE,CAAC;YACf,SAAS,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC3B,CAAC;IACF,CAAC;IAED,+BAA+B;IAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,KAAK,EAAE,CAAC;QACxC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;IAED,4BAA4B;IAC5B,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAErF,yBAAyB;IACzB,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAE,GAAG,QAAQ,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,QAAQ,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAE3B,sCAAsC;IACtC,MAAM,KAAK,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5C,kDAAkD;IAClD,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhD,OAAO;QACN,CAAC;QACD,MAAM;QACN,WAAW,EAAE,MAAM,GAAG,IAAI,EAAE,uBAAuB;KACnD,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,OAAiB,EAAE,OAAiB,EAAU,EAAE;IACvE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;IAEtD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC3F,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAE3F,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,SAAS,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAE3F,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,MAAgB,EAAoC,EAAE;IACxF,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,2CAA2C;IAE3D,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC;IACtB,OAAO;QACN,KAAK,EAAE,IAAI,GAAG,MAAM;QACpB,KAAK,EAAE,IAAI,GAAG,MAAM;KACpB,CAAC;AACH,CAAC,CAAC"}

package/dist/types/aggregate.d.ts

@@ -0,0 +1,124 @@
+/**
+ * Aggregated Result Type Definitions
+ *
+ * Aggregated results summarise multiple evaluation runs with statistical
+ * measures. This is the intermediate format between raw results and
+ * final rendered output (LaTeX tables, etc.).
+ */
+import type { Primitive } from "./case.js";
+import type { SutRole } from "./sut.js";
+/**
+ * Summary statistics for a numeric metric.
+ */
+export interface SummaryStats {
+    /** Number of observations */
+    n: number;
+    /** Arithmetic mean */
+    mean: number;
+    /** Median (50th percentile) */
+    median: number;
+    /** Minimum value */
+    min: number;
+    /** Maximum value */
+    max: number;
+    /** Standard deviation (sample) */
+    std?: number;
+    /** 95% confidence interval [lower, upper] */
+    confidence95?: [number, number];
+    /** Sum of all values */
+    sum?: number;
+    /** 25th percentile */
+    p25?: number;
+    /** 75th percentile */
+    p75?: number;
+}
+/**
+ * Comparison metrics between primary and baseline SUTs.
+ */
+export interface ComparisonMetrics {
+    /** Absolute deltas (primary - baseline) */
+    deltas: Record<string, number>;
+    /** Ratios (primary / baseline) */
+    ratios: Record<string, number>;
+    /** Win rate (% of cases where primary beats baseline) */
+    betterRate?: number;
+    /** Mann-Whitney U statistic */
+    uStatistic?: number;
+    /** Statistical significance (p-value) */
+    pValue?: number;
+    /** Effect size (Cohen's d) */
+    effectSize?: number;
+}
+/**
+ * Coverage information for the aggregation.
+ */
+export interface CoverageMetrics {
+    /** Fraction of cases covered */
+    caseCoverage: number;
+    /** Metric availability (metric name -> coverage fraction) */
+    metricCoverage: Record<string, number>;
+    /** Missing case IDs */
+    missingCases?: string[];
+}
+/**
+ * Aggregated result for a SUT (optionally grouped by case class).
+ */
+export interface AggregatedResult {
+    /** SUT identifier */
+    sut: string;
+    /** SUT role */
+    sutRole: SutRole;
+    /** Case class (if grouped) */
+    caseClass?: string;
+    /** Grouping information */
+    group: {
+        /** Number of runs in this aggregate */
+        runCount: number;
+        /** Number of unique cases */
+        caseCount: number;
+        /** Hash of configuration (for homogeneity check) */
+        configHash?: string;
+    };
+    /** Correctness summary */
+    correctness: {
+        /** Fraction of runs that produced valid output */
+        validRate: number;
+        /** Fraction of runs that produced any output */
+        producedOutputRate: number;
+        /** Fraction of runs matching expected (if oracle available) */
+        matchesExpectedRate?: number;
+        /** Breakdown of failure types */
+        failureBreakdown?: Record<string, number>;
+    };
+    /** Aggregated metrics (metric name -> summary stats) */
+    metrics: Record<string, SummaryStats>;
+    /** Comparisons with baselines (baseline SUT id -> comparison) */
+    comparisons?: Record<string, ComparisonMetrics>;
+    /** Coverage information */
+    coverage?: CoverageMetrics;
+    /** Additional metadata */
+    metadata?: Record<string, Primitive>;
+}
+/**
+ * Complete aggregation output.
+ */
+export interface AggregationOutput {
+    /** Schema version */
+    version: string;
+    /** Generation timestamp */
+    timestamp: string;
+    /** Aggregated results */
+    aggregates: AggregatedResult[];
+    /** Global metadata */
+    metadata?: {
+        /** Total runs processed */
+        totalRuns: number;
+        /** Total unique cases */
+        totalCases: number;
+        /** SUTs included */
+        sutsIncluded: string[];
+        /** Case classes included */
+        caseClassesIncluded?: string[];
+    };
+}
+//# sourceMappingURL=aggregate.d.ts.map

package/dist/types/aggregate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aggregate.d.ts","sourceRoot":"","sources":["../../src/types/aggregate.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAExC;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,6BAA6B;IAC7B,CAAC,EAAE,MAAM,CAAC;IAEV,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IAEb,+BAA+B;IAC/B,MAAM,EAAE,MAAM,CAAC;IAEf,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IAEZ,oBAAoB;IACpB,GAAG,EAAE,MAAM,CAAC;IAEZ,kCAAkC;IAClC,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,6CAA6C;IAC7C,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhC,wBAAwB;IACxB,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,sBAAsB;IACtB,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb,sBAAsB;IACtB,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/B,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/B,yDAAyD;IACzD,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,+BAA+B;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,8BAA8B;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,gCAAgC;IAChC,YAAY,EAAE,MAAM,CAAC;IAErB,6DAA6D;IAC7D,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEvC,uBAAuB;IACvB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAChC,qBAAqB;IACrB,GAAG,EAAE,MAAM,CAAC;IAEZ,eAAe;IACf,OAAO,EAAE,OAAO,CAAC;IAEjB,8BAA8B;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,2BAA2B;IAC3B,KAAK,EAAE;QACN,uCAAuC;QACvC,QAAQ,EAAE,MAAM,CAAC;QAEjB,6BAA6B;QAC7B,SAAS,EAAE,MAAM,CAAC;QAElB,oDAAoD;QACpD,UAAU,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,0BAA0B;IAC1B,WAAW,EAAE;QACZ,kDAAkD;QAClD,SAAS,EAAE,MAAM,CAAC;QAElB,gDAAgD;QAChD,kBAAkB,EAAE,MAAM,CAAC;QAE3B,+DAA+D;QAC/D,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAE7B,iCAAiC;QACjC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC1C,CAAC;IAEF,wDAAwD;IACxD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEtC,iEAAiE;IACjE,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IAEhD,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,eAAe,CAAC;IAE3B,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,qBAAqB;IACrB,OAAO,EAAE,MAAM,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAElB,yBAAyB;IACzB,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAE/B,sBAAsB;IACtB,QAAQ,CAAC,EAAE;QACV,2BAA2B;QAC3B,SAAS,EAAE,MAAM,CAAC;QAElB,yBAAyB;QACzB,UAAU,EAAE,MAAM,CAAC;QAEnB,oBAAoB;QACpB,YAAY,EAAE,MAAM,EAAE,CAAC;QAEvB,4BAA4B;QAC5B,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;KAC/B,CAAC;CACF"}

package/dist/types/aggregate.js

@@ -0,0 +1,9 @@
+/**
+ * Aggregated Result Type Definitions
+ *
+ * Aggregated results summarise multiple evaluation runs with statistical
+ * measures. This is the intermediate format between raw results and
+ * final rendered output (LaTeX tables, etc.).
+ */
+export {};
+//# sourceMappingURL=aggregate.js.map

package/dist/types/aggregate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"aggregate.js","sourceRoot":"","sources":["../../src/types/aggregate.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG"}

package/dist/types/case.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Evaluation Case Type Definitions
+ *
+ * A case represents a single evaluation scenario with:
+ * - Deterministic inputs (graph, seeds, etc.)
+ * - Expected behavior or ground truth (if applicable)
+ * - Grouping metadata for aggregation
+ */
+/**
+ * Primitive types allowed in case summaries.
+ */
+export type Primitive = string | number | boolean | null;
+/**
+ * Reference to an external artefact (graph file, path set, etc.).
+ */
+export interface ArtefactReference {
+    /** Type of artefact */
+    type: "graph" | "path-set" | "subgraph" | "embedding" | "other";
+    /** URI or path to artefact */
+    uri: string;
+    /** Content hash for integrity verification */
+    hash?: string;
+    /** Optional metadata */
+    metadata?: Record<string, Primitive>;
+}
+/**
+ * Input specification for an evaluation case.
+ */
+export interface CaseInputs {
+    /** Scalar summary values (e.g., { nodes: 100, seeds: ["a", "b"] }) */
+    summary?: Record<string, Primitive | Primitive[]>;
+    /** References to external artefacts */
+    artefacts?: ArtefactReference[];
+}
+/**
+ * A single evaluation case.
+ *
+ * The caseId should be a deterministic hash of the canonical inputs
+ * to ensure reproducibility across runs.
+ */
+export interface EvaluationCase {
+    /** Deterministic ID (SHA-256 of canonical inputs) */
+    caseId: string;
+    /** Human-readable name */
+    name?: string;
+    /** Grouping label for aggregation (e.g., "scale-free", "bidirectional") */
+    caseClass?: string;
+    /** Input specification */
+    inputs: CaseInputs;
+    /** Optional expected output for oracle-based evaluation */
+    expectedOutput?: {
+        /** Expected summary values */
+        summary?: Record<string, Primitive | Primitive[]>;
+        /** Expected labels */
+        labels?: Record<string, Primitive>;
+        /** Expected ranking (for ranking tasks) */
+        ranking?: {
+            itemId: string;
+            score: number;
+        }[];
+    };
+    /** Version of this case definition */
+    version?: string;
+    /** Tags for filtering */
+    tags?: readonly string[];
+}
+/**
+ * Complete case definition with universal input factories.
+ *
+ * The framework doesn't need to know what "expander" or "seeds" mean.
+ * It only needs:
+ * 1. getInput() - Load whatever resource the algorithm needs (graph, dataset, API client, etc.)
+ * 2. getInputs() - Get algorithm-specific inputs from the case
+ *
+ * @template TInput - The resource type (e.g., Graph, Dataset, API client)
+ * @template TInputs - The algorithm inputs type
+ */
+export interface CaseDefinition<TInput = unknown, TInputs = unknown> {
+    /** The case specification */
+    case: EvaluationCase;
+    /**
+     * Load the primary resource needed by the algorithm.
+     * This is called once per case and cached.
+     *
+     * Examples:
+     * - Expansion: Load a benchmark graph
+     * - Ranking: Load a graph with source/target metadata
+     * - ML: Load training dataset
+     *
+     * @returns Promise resolving to the resource
+     */
+    getInput(): Promise<TInput>;
+    /**
+     * Get algorithm-specific inputs for this case.
+     *
+     * Examples:
+     * - Expansion: { seeds: ["node1", "node2"] }
+     * - Ranking: { source: "node1", target: "node2", maxPaths: 10 }
+     * - Classification: { labels: ["cat", "dog"], threshold: 0.5 }
+     *
+     * @returns Algorithm inputs
+     */
+    getInputs(): TInputs;
+}
+//# sourceMappingURL=case.d.ts.map

package/dist/types/case.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"case.d.ts","sourceRoot":"","sources":["../../src/types/case.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,uBAAuB;IACvB,IAAI,EAAE,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,OAAO,CAAC;IAEhE,8BAA8B;IAC9B,GAAG,EAAE,MAAM,CAAC;IAEZ,8CAA8C;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,wBAAwB;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,sEAAsE;IACtE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC;IAElD,uCAAuC;IACvC,SAAS,CAAC,EAAE,iBAAiB,EAAE,CAAC;CAChC;AAED;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC9B,qDAAqD;IACrD,MAAM,EAAE,MAAM,CAAC;IAEf,0BAA0B;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,2EAA2E;IAC3E,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,0BAA0B;IAC1B,MAAM,EAAE,UAAU,CAAC;IAEnB,2DAA2D;IAC3D,cAAc,CAAC,EAAE;QAChB,8BAA8B;QAC9B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC;QAElD,sBAAsB;QACtB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAEnC,2CAA2C;QAC3C,OAAO,CAAC,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;KAC9C,CAAC;IAEF,sCAAsC;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,yBAAyB;IACzB,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,cAAc,CAAC,MAAM,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO;IAClE,6BAA6B;IAC7B,IAAI,EAAE,cAAc,CAAC;IAErB;;;;;;;;;;OAUG;IACH,QAAQ,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAE5B;;;;;;;;;OASG;IACH,SAAS,IAAI,OAAO,CAAC;CACrB"}

package/dist/types/case.js

@@ -0,0 +1,10 @@
+/**
+ * Evaluation Case Type Definitions
+ *
+ * A case represents a single evaluation scenario with:
+ * - Deterministic inputs (graph, seeds, etc.)
+ * - Expected behavior or ground truth (if applicable)
+ * - Grouping metadata for aggregation
+ */
+export {};
+//# sourceMappingURL=case.js.map

package/dist/types/case.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"case.js","sourceRoot":"","sources":["../../src/types/case.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}

package/dist/types/claims.d.ts

@@ -0,0 +1,122 @@
+/**
+ * Evaluation Claims Type Definitions
+ *
+ * Claims represent explicit hypotheses to be tested. Each claim specifies:
+ * - Which SUTs are being compared
+ * - Which metric is being evaluated
+ * - The expected relationship (greater, less, equal)
+ * - The scope of validity
+ *
+ * This enables claim-driven evaluation where experiments are designed
+ * to test specific hypotheses rather than collect arbitrary metrics.
+ */
+import type { Primitive } from "./case.js";
+/**
+ * Scope of claim validity.
+ *
+ * - `global`: Claim should hold across all cases
+ * - `caseClass`: Claim holds within specific case classes
+ * - `parameterRange`: Claim holds for specific parameter ranges
+ * - `localStructure`: Claim depends on local graph structure
+ */
+export type ValidityScope = "global" | "caseClass" | "parameterRange" | "localStructure";
+/**
+ * Direction of comparison.
+ */
+export type ComparisonDirection = "greater" | "less" | "equal";
+/**
+ * An evaluation claim (hypothesis).
+ */
+export interface EvaluationClaim {
+    /** Unique identifier for this claim */
+    claimId: string;
+    /** Human-readable description */
+    description: string;
+    /** Primary SUT being evaluated */
+    sut: string;
+    /** Baseline SUT for comparison */
+    baseline: string;
+    /** Metric being compared */
+    metric: string;
+    /** Expected direction of difference */
+    direction: ComparisonDirection;
+    /** Optional threshold for the difference */
+    threshold?: number;
+    /** Scope of validity */
+    scope: ValidityScope;
+    /** Scope constraints (e.g., { caseClass: "scale-free" }) */
+    scopeConstraints?: Record<string, Primitive | Primitive[]>;
+    /** Required significance level (default: 0.05) */
+    significanceLevel?: number;
+    /** Minimum effect size (Cohen's d) */
+    minEffectSize?: number;
+    /** Tags for filtering */
+    tags?: readonly string[];
+    /** Citation/reference for the claim */
+    citation?: string;
+}
+/**
+ * Status of a claim evaluation.
+ */
+export type ClaimStatus = "satisfied" | "violated" | "inconclusive";
+/**
+ * Evidence supporting a claim evaluation.
+ */
+export interface ClaimEvidence {
+    /** Primary SUT metric value */
+    primaryValue: number;
+    /** Baseline SUT metric value */
+    baselineValue: number;
+    /** Absolute delta (primary - baseline) */
+    delta: number;
+    /** Ratio (primary / baseline) */
+    ratio: number;
+    /** P-value from statistical test */
+    pValue?: number;
+    /** Effect size (Cohen's d) */
+    effectSize?: number;
+    /** Number of observations */
+    n?: number;
+    /** 95% confidence interval for delta */
+    deltaCI95?: [number, number];
+}
+/**
+ * Result of evaluating a single claim.
+ */
+export interface ClaimEvaluation {
+    /** The claim being evaluated */
+    claim: EvaluationClaim;
+    /** Evaluation status */
+    status: ClaimStatus;
+    /** Supporting evidence */
+    evidence: ClaimEvidence;
+    /** Reason for inconclusive status (if applicable) */
+    inconclusiveReason?: string;
+    /** Additional notes */
+    notes?: string[];
+}
+/**
+ * Summary of all claim evaluations.
+ */
+export interface ClaimEvaluationSummary {
+    /** Schema version */
+    version: string;
+    /** Generation timestamp */
+    timestamp: string;
+    /** Individual claim evaluations */
+    evaluations: ClaimEvaluation[];
+    /** Summary statistics */
+    summary: {
+        /** Total claims evaluated */
+        total: number;
+        /** Claims satisfied */
+        satisfied: number;
+        /** Claims violated */
+        violated: number;
+        /** Claims inconclusive */
+        inconclusive: number;
+        /** Satisfaction rate (satisfied / (satisfied + violated)) */
+        satisfactionRate: number;
+    };
+}
+//# sourceMappingURL=claims.d.ts.map

package/dist/types/claims.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claims.d.ts","sourceRoot":"","sources":["../../src/types/claims.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE3C;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,WAAW,GAAG,gBAAgB,GAAG,gBAAgB,CAAC;AAEzF;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,SAAS,GAAG,MAAM,GAAG,OAAO,CAAC;AAE/D;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,uCAAuC;IACvC,OAAO,EAAE,MAAM,CAAC;IAEhB,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAC;IAEpB,kCAAkC;IAClC,GAAG,EAAE,MAAM,CAAC;IAEZ,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IAEjB,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IAEf,uCAAuC;IACvC,SAAS,EAAE,mBAAmB,CAAC;IAE/B,4CAA4C;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,wBAAwB;IACxB,KAAK,EAAE,aAAa,CAAC;IAErB,4DAA4D;IAC5D,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,EAAE,CAAC,CAAC;IAE3D,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,sCAAsC;IACtC,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,yBAAyB;IACzB,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAEzB,uCAAuC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,cAAc,CAAC;AAEpE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAC;IAErB,gCAAgC;IAChC,aAAa,EAAE,MAAM,CAAC;IAEtB,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IAEd,iCAAiC;IACjC,KAAK,EAAE,MAAM,CAAC;IAEd,oCAAoC;IACpC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,8BAA8B;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,6BAA6B;IAC7B,CAAC,CAAC,EAAE,MAAM,CAAC;IAEX,wCAAwC;IACxC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,gCAAgC;IAChC,KAAK,EAAE,eAAe,CAAC;IAEvB,wBAAwB;IACxB,MAAM,EAAE,WAAW,CAAC;IAEpB,0BAA0B;IAC1B,QAAQ,EAAE,aAAa,CAAC;IAExB,qDAAqD;IACrD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAE5B,uBAAuB;IACvB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,qBAAqB;IACrB,OAAO,EAAE,MAAM,CAAC;IAEhB,2BAA2B;IAC3B,SAAS,EAAE,MAAM,CAAC;IAElB,mCAAmC;IACnC,WAAW,EAAE,eAAe,EAAE,CAAC;IAE/B,yBAAyB;IACzB,OAAO,EAAE;QACR,6BAA6B;QAC7B,KAAK,EAAE,MAAM,CAAC;QAEd,uBAAuB;QACvB,SAAS,EAAE,MAAM,CAAC;QAElB,sBAAsB;QACtB,QAAQ,EAAE,MAAM,CAAC;QAEjB,0BAA0B;QAC1B,YAAY,EAAE,MAAM,CAAC;QAErB,6DAA6D;QAC7D,gBAAgB,EAAE,MAAM,CAAC;KACzB,CAAC;CACF"}

package/dist/types/claims.js

@@ -0,0 +1,14 @@
+/**
+ * Evaluation Claims Type Definitions
+ *
+ * Claims represent explicit hypotheses to be tested. Each claim specifies:
+ * - Which SUTs are being compared
+ * - Which metric is being evaluated
+ * - The expected relationship (greater, less, equal)
+ * - The scope of validity
+ *
+ * This enables claim-driven evaluation where experiments are designed
+ * to test specific hypotheses rather than collect arbitrary metrics.
+ */
+export {};
+//# sourceMappingURL=claims.js.map

package/dist/types/claims.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claims.js","sourceRoot":"","sources":["../../src/types/claims.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}

package/dist/types/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Framework Type Definitions
+ *
+ * Re-exports all canonical types for the evaluation framework.
+ */
+export type { SutDefinition, SutFactory, SutRegistration, SutRole } from "./sut.js";
+export type { ArtefactReference, CaseDefinition, CaseInputs, EvaluationCase, Primitive, } from "./case.js";
+export type { CorrectnessResult, EvaluationResult, FailureType, Provenance, RankedItem, ResultBatch, ResultMetrics, ResultOutputs, RunContext, } from "./result.js";
+export type { AggregatedResult, AggregationOutput, ComparisonMetrics, CoverageMetrics, SummaryStats, } from "./aggregate.js";
+export type { ClaimEvaluation, ClaimEvaluationSummary, ClaimEvidence, ClaimStatus, ComparisonDirection, EvaluationClaim, ValidityScope, } from "./claims.js";
+export type { Perturbation, PerturbationConfig, PerturbationType, RobustnessAnalysisOutput, RobustnessAnalysisResult, RobustnessMetrics, } from "./perturbation.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/types/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAGpF,YAAY,EACX,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,cAAc,EACd,SAAS,GACT,MAAM,WAAW,CAAC;AAGnB,YAAY,EACX,iBAAiB,EACjB,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,EACX,aAAa,EACb,aAAa,EACb,UAAU,GACV,MAAM,aAAa,CAAC;AAGrB,YAAY,EACX,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,YAAY,GACZ,MAAM,gBAAgB,CAAC;AAGxB,YAAY,EACX,eAAe,EACf,sBAAsB,EACtB,aAAa,EACb,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,aAAa,GACb,MAAM,aAAa,CAAC;AAGrB,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,gBAAgB,EAChB,wBAAwB,EACxB,wBAAwB,EACxB,iBAAiB,GACjB,MAAM,mBAAmB,CAAC"}

package/dist/types/index.js

@@ -0,0 +1,7 @@
+/**
+ * Framework Type Definitions
+ *
+ * Re-exports all canonical types for the evaluation framework.
+ */
+export {};
+//# sourceMappingURL=index.js.map

package/dist/types/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}