ppef 1.1.0 → 1.2.1

package/dist/robustness/analyzer.js

@@ -1,191 +0,0 @@
-/**
- * Robustness Analyzer
- *
- * Analyzes algorithm robustness under perturbations.
- * Computes variance, stability, and degradation metrics.
- */
-import { computeSummaryStats } from "../aggregation/aggregators.js";
-/**
- * Analyze robustness of a SUT under perturbation.
- *
- * @param baseResults - Results without perturbation
- * @param perturbedResults - Results with perturbation, keyed by perturbation name
- * @param metric - Metric to analyze
- * @returns Robustness metrics
- */
-export const analyzeRobustnessForMetric = (baseResults, perturbedResults, metric) => {
-    // Extract metric values
-    const baseValues = baseResults
-        .map((r) => r.metrics.numeric[metric])
-        .filter((v) => !Number.isNaN(v));
-    const perturbedValues = perturbedResults
-        .map((r) => r.metrics.numeric[metric])
-        .filter((v) => !Number.isNaN(v));
-    if (baseValues.length === 0 || perturbedValues.length === 0) {
-        return {
-            varianceUnderPerturbation: Number.NaN,
-            stdUnderPerturbation: Number.NaN,
-            coefficientOfVariation: Number.NaN,
-        };
-    }
-    // Compute statistics for perturbed results
-    // Note: baseStats could be used for relative comparison in future
-    const perturbedStats = computeSummaryStats(perturbedValues);
-    // Variance under perturbation
-    const varianceUnderPerturbation = perturbedStats.std === undefined ? Number.NaN : perturbedStats.std ** 2;
-    // Standard deviation
-    const stdUnderPerturbation = perturbedStats.std ?? Number.NaN;
-    // Coefficient of variation (relative variance)
-    const coefficientOfVariation = perturbedStats.mean !== 0 && perturbedStats.std !== undefined
-        ? perturbedStats.std / Math.abs(perturbedStats.mean)
-        : Number.NaN;
-    return {
-        varianceUnderPerturbation,
-        stdUnderPerturbation,
-        coefficientOfVariation,
-    };
-};
-/**
- * Analyze robustness across multiple perturbation levels.
- *
- * @param results - All results including perturbed ones
- * @param metric - Metric to analyze
- * @param intensityLevels - Perturbation intensity levels
- * @returns Robustness metrics with degradation curve
- */
-export const analyzeRobustnessWithCurve = (results, metric, intensityLevels) => {
-    // Group results by perturbation intensity
-    const byIntensity = new Map();
-    // Base results (no perturbation)
-    const baseResults = results.filter((r) => r.run.config?.perturbationIntensity === undefined);
-    byIntensity.set(0, baseResults);
-    // Perturbed results
-    for (const level of intensityLevels) {
-        const levelResults = results.filter((r) => r.run.config?.perturbationIntensity === level);
-        if (levelResults.length > 0) {
-            byIntensity.set(level, levelResults);
-        }
-    }
-    // Build degradation curve
-    const degradationCurve = [];
-    for (const [level, levelResults] of byIntensity) {
-        const values = levelResults
-            .map((r) => r.metrics.numeric[metric])
-            .filter((v) => !Number.isNaN(v));
-        if (values.length > 0) {
-            const stats = computeSummaryStats(values);
-            degradationCurve.push({
-                perturbationLevel: level,
-                metricValue: stats.mean,
-                stdDev: stats.std,
-            });
-        }
-    }
-    // Sort by level
-    degradationCurve.sort((a, b) => a.perturbationLevel - b.perturbationLevel);
-    // Find breakpoint (significant degradation)
-    let breakpoint;
-    if (degradationCurve.length >= 2) {
-        const baseValue = degradationCurve[0].metricValue;
-        for (let index = 1; index < degradationCurve.length; index++) {
-            const relativeChange = Math.abs((degradationCurve[index].metricValue - baseValue) / baseValue);
-            if (relativeChange > 0.1) {
-                // 10% degradation threshold
-                breakpoint = degradationCurve[index].perturbationLevel;
-                break;
-            }
-        }
-    }
-    // Compute overall variance from all perturbed results
-    const allPerturbedValues = results
-        .filter((r) => r.run.config?.perturbationIntensity !== undefined)
-        .map((r) => r.metrics.numeric[metric])
-        .filter((v) => !Number.isNaN(v));
-    const overallStats = computeSummaryStats(allPerturbedValues);
-    return {
-        varianceUnderPerturbation: overallStats.std === undefined ? Number.NaN : overallStats.std ** 2,
-        stdUnderPerturbation: overallStats.std ?? Number.NaN,
-        coefficientOfVariation: overallStats.mean !== 0 && overallStats.std !== undefined
-            ? overallStats.std / Math.abs(overallStats.mean)
-            : Number.NaN,
-        degradationCurve,
-        breakpoint,
-    };
-};
-/**
- * Compare robustness between two SUTs.
- *
- * @param sutAResults - Results for SUT A (including perturbed)
- * @param sutBResults - Results for SUT B (including perturbed)
- * @param metric - Metric to compare
- * @returns Object with comparison metrics
- */
-export const compareRobustness = (sutAResults, sutBResults, metric) => {
-    const sutAPerturbed = sutAResults.filter((r) => r.run.config?.perturbationIntensity !== undefined);
-    const sutBPerturbed = sutBResults.filter((r) => r.run.config?.perturbationIntensity !== undefined);
-    const sutABase = sutAResults.filter((r) => r.run.config?.perturbationIntensity === undefined);
-    const sutBBase = sutBResults.filter((r) => r.run.config?.perturbationIntensity === undefined);
-    const sutARobustness = analyzeRobustnessForMetric(sutABase, sutAPerturbed, metric);
-    const sutBRobustness = analyzeRobustnessForMetric(sutBBase, sutBPerturbed, metric);
-    const relativeRobustness = sutBRobustness.varianceUnderPerturbation === 0
-        ? Infinity
-        : sutARobustness.varianceUnderPerturbation / sutBRobustness.varianceUnderPerturbation;
-    return {
-        sutAVariance: sutARobustness.varianceUnderPerturbation,
-        sutBVariance: sutBRobustness.varianceUnderPerturbation,
-        relativeRobustness,
-    };
-};
-/**
- * Create a full robustness analysis output.
- *
- * @param results - All evaluation results (base and perturbed)
- * @param options - Analysis options
- * @returns Complete robustness analysis output
- */
-export const createRobustnessAnalysis = (results, options) => {
-    const analysisResults = [];
-    // Group results by SUT
-    const bySut = new Map();
-    for (const result of results) {
-        const existing = bySut.get(result.run.sut) ?? [];
-        existing.push(result);
-        bySut.set(result.run.sut, existing);
-    }
-    // Analyze each SUT for each metric and perturbation
-    for (const [sut, sutResults] of bySut) {
-        for (const metric of options.metrics) {
-            for (const perturbation of options.perturbations) {
-                // Filter to this perturbation
-                const baseResults = sutResults.filter((r) => !r.run.config?.perturbation);
-                const perturbedResults = sutResults.filter((r) => r.run.config?.perturbation === perturbation);
-                const robustness = analyzeRobustnessForMetric(baseResults, perturbedResults, metric);
-                // Get baseline value
-                const baseValues = baseResults.map((r) => r.metrics.numeric[metric]);
-                const baselineValue = baseValues.length > 0
-                    ? baseValues.reduce((a, b) => a + b, 0) / baseValues.length
-                    : Number.NaN;
-                analysisResults.push({
-                    sut,
-                    perturbation,
-                    metric,
-                    robustness,
-                    baselineValue,
-                    runCount: perturbedResults.length,
-                });
-            }
-        }
-    }
-    return {
-        version: "1.0.0",
-        timestamp: new Date().toISOString(),
-        results: analysisResults,
-        config: {
-            perturbations: options.perturbations,
-            metrics: options.metrics,
-            intensityLevels: options.intensityLevels,
-            runsPerLevel: options.runsPerLevel ?? 1,
-        },
-    };
-};
-//# sourceMappingURL=analyzer.js.map

package/dist/robustness/analyzer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"analyzer.js","sourceRoot":"","sources":["../../src/robustness/analyzer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAyBpE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CACzC,WAA+B,EAC/B,gBAAoC,EACpC,MAAc,EACM,EAAE;IACtB,wBAAwB;IACxB,MAAM,UAAU,GAAG,WAAW;SAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAElC,MAAM,eAAe,GAAG,gBAAgB;SACtC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAElC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7D,OAAO;YACN,yBAAyB,EAAE,MAAM,CAAC,GAAG;YACrC,oBAAoB,EAAE,MAAM,CAAC,GAAG;YAChC,sBAAsB,EAAE,MAAM,CAAC,GAAG;SAClC,CAAC;IACH,CAAC;IAED,2CAA2C;IAC3C,kEAAkE;IAClE,MAAM,cAAc,GAAG,mBAAmB,CAAC,eAAe,CAAC,CAAC;IAE5D,8BAA8B;IAC9B,MAAM,yBAAyB,GAC9B,cAAc,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,CAAC;IAEzE,qBAAqB;IACrB,MAAM,oBAAoB,GAAG,cAAc,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC;IAE9D,+CAA+C;IAC/C,MAAM,sBAAsB,GAC3B,cAAc,CAAC,IAAI,KAAK,CAAC,IAAI,cAAc,CAAC,GAAG,KAAK,SAAS;QAC5D,CAAC,CAAC,cAAc,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC;QACpD,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IAEf,OAAO;QACN,yBAAyB;QACzB,oBAAoB;QACpB,sBAAsB;KACtB,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CACzC,OAA2B,EAC3B,MAAc,EACd,eAAyB,EACL,EAAE;IACtB,0CAA0C;IAC1C,MAAM,WAAW,GAAG,IAAI,GAAG,EAA8B,CAAC;IAE1D,iCAAiC;IACjC,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,KAAK,SAAS,CAAC,CAAC;IAC7F,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IAEhC,oBAAoB;IACpB,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,KAAK,KAAK,CAAC,CAAC;QAC1F,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;QACtC,CAAC;IACF,CAAC;IAED,0BAA0B;IAC1B,MAAM,gBAAgB,GAIhB,EAAE,CAAC;IAET,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,WAAW,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,YAAY;aACzB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;aACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC1C,gBAAgB,CAAC,IAAI,CAAC;gBACrB,iBAAiB,EAAE,KAAK;gBACxB,WAAW,EAAE,KAAK,CAAC,IAAI;gBACvB,MAAM,EAAE,KAAK,CAAC,GAAG;aACjB,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,gBAAgB;IAChB,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAE3E,4CAA4C;IAC5C,IAAI,UAA8B,CAAC;IACnC,IAAI,gBAAgB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;QAClD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,gBAAgB,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9D,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC9B,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,SAAS,CAC7D,CAAC;YACF,IAAI,cAAc,GAAG,GAAG,EAAE,CAAC;gBAC1B,4BAA4B;gBAC5B,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,iBAAiB,CAAC;gBACvD,MAAM;YACP,CAAC;QACF,CAAC;IACF,CAAC;IAED,sDAAsD;IACtD,MAAM,kBAAkB,GAAG,OAAO;SAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,KAAK,SAAS,CAAC;SAChE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAElC,MAAM,YAAY,GAAG,mBAAmB,CAAC,kBAAkB,CAAC,CAAC;IAE7D,OAAO;QACN,yBAAyB,EAAE,YAAY,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC;QAC9F,oBAAoB,EAAE,YAAY,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG;QACpD,sBAAsB,EACrB,YAAY,CAAC,IAAI,KAAK,CAAC,IAAI,YAAY,CAAC,GAAG,KAAK,SAAS;YACxD,CAAC,CAAC,YAAY,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC;YAChD,CAAC,CAAC,MAAM,CAAC,GAAG;QACd,gBAAgB;QAChB,UAAU;KACV,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAChC,WAA+B,EAC/B,WAA+B,EAC/B,MAAc,EAKb,EAAE;IACH,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,KAAK,SAAS,CACxD,CAAC;IACF,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,KAAK,SAAS,CACxD,CAAC;IAEF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,KAAK,SAAS,CAAC,CAAC;IAC9F,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,qBAAqB,KAAK,SAAS,CAAC,CAAC;IAE9F,MAAM,cAAc,GAAG,0BAA0B,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IACnF,MAAM,cAAc,GAAG,0BAA0B,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IAEnF,MAAM,kBAAkB,GACvB,cAAc,CAAC,yBAAyB,KAAK,CAAC;QAC7C,CAAC,CAAC,QAAQ;QACV,CAAC,CAAC,cAAc,CAAC,yBAAyB,GAAG,cAAc,CAAC,yBAAyB,CAAC;IAExF,OAAO;QACN,YAAY,EAAE,cAAc,CAAC,yBAAyB;QACtD,YAAY,EAAE,cAAc,CAAC,yBAAyB;QACtD,kBAAkB;KAClB,CAAC;AACH,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACvC,OAA2B,EAC3B,OAAkC,EACP,EAAE;IAC7B,MAAM,eAAe,GAA+B,EAAE,CAAC;IAEvD,uBAAuB;IACvB,MAAM,KAAK,GAAG,IAAI,GAAG,EAA8B,CAAC;IACpD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACjD,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACrC,CAAC;IAED,oDAAoD;IACpD,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,KAAK,EAAE,CAAC;QACvC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACtC,KAAK,MAAM,YAAY,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;gBAClD,8BAA8B;gBAC9B,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;gBAC1E,MAAM,gBAAgB,GAAG,UAAU,CAAC,MAAM,CACzC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,EAAE,YAAY,KAAK,YAAY,CAClD,CAAC;gBAEF,MAAM,UAAU,GAAG,0BAA0B,CAAC,WAAW,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;gBAErF,qBAAqB;gBACrB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;gBACrE,MAAM,aAAa,GAClB,UAAU,CAAC,MAAM,GAAG,CAAC;oBACpB,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;oBAC3D,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gBAEf,eAAe,CAAC,IAAI,CAAC;oBACpB,GAAG;oBACH,YAAY;oBACZ,MAAM;oBACN,UAAU;oBACV,aAAa;oBACb,QAAQ,EAAE,gBAAgB,CAAC,MAAM;iBACjC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO;QACN,OAAO,EAAE,OAAO;QAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO,EAAE,eAAe;QACxB,MAAM,EAAE;YACP,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,eAAe,EAAE,OAAO,CAAC,eAAe;YACxC,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,CAAC;SACvC;KACD,CAAC;AACH,CAAC,CAAC"}