ppef 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -125
- package/bin/ppef.mjs +20 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.d.ts +8 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.d.ts.map +1 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.js +308 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.js.map +1 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.js +405 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.js +424 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/registry.unit.test.d.ts +7 -0
- package/dist/__tests__/evaluators/registry.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/registry.unit.test.js +173 -0
- package/dist/__tests__/evaluators/registry.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.js +260 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/framework-pipeline.integration.test.js +49 -20
- package/dist/__tests__/framework-pipeline.integration.test.js.map +1 -1
- package/dist/__tests__/index-exports.unit.test.d.ts +8 -0
- package/dist/__tests__/index-exports.unit.test.d.ts.map +1 -0
- package/dist/__tests__/index-exports.unit.test.js +124 -0
- package/dist/__tests__/index-exports.unit.test.js.map +1 -0
- package/dist/__tests__/registry-executor.integration.test.js +12 -9
- package/dist/__tests__/registry-executor.integration.test.js.map +1 -1
- package/dist/aggregation/__tests__/aggregators.unit.test.d.ts +7 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.d.ts.map +1 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.js +350 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.js.map +1 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.d.ts +7 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.d.ts.map +1 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.js +213 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.js.map +1 -0
- package/dist/aggregation/aggregators.d.ts +9 -0
- package/dist/aggregation/aggregators.d.ts.map +1 -1
- package/dist/aggregation/aggregators.js +1 -1
- package/dist/aggregation/aggregators.js.map +1 -1
- package/dist/aggregation/index.d.ts +1 -1
- package/dist/aggregation/index.d.ts.map +1 -1
- package/dist/aggregation/index.js +1 -1
- package/dist/aggregation/index.js.map +1 -1
- package/dist/aggregation/pipeline.d.ts.map +1 -1
- package/dist/aggregation/pipeline.js +40 -3
- package/dist/aggregation/pipeline.js.map +1 -1
- package/dist/claims/index.d.ts +6 -3
- package/dist/claims/index.d.ts.map +1 -1
- package/dist/claims/index.js +6 -3
- package/dist/claims/index.js.map +1 -1
- package/dist/cli/__tests__/aggregate.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.js +399 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/binary-sut.integration.test.d.ts +8 -0
- package/dist/cli/__tests__/binary-sut.integration.test.d.ts.map +1 -0
- package/dist/cli/__tests__/binary-sut.integration.test.js +165 -0
- package/dist/cli/__tests__/binary-sut.integration.test.js.map +1 -0
- package/dist/cli/__tests__/commands.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/commands.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/commands.unit.test.js +217 -0
- package/dist/cli/__tests__/commands.unit.test.js.map +1 -0
- package/dist/cli/__tests__/config-loader.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/config-loader.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/config-loader.unit.test.js +611 -0
- package/dist/cli/__tests__/config-loader.unit.test.js.map +1 -0
- package/dist/cli/__tests__/index.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/index.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/index.unit.test.js +65 -0
- package/dist/cli/__tests__/index.unit.test.js.map +1 -0
- package/dist/cli/__tests__/logger.unit.test.d.ts +11 -0
- package/dist/cli/__tests__/logger.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/logger.unit.test.js +180 -0
- package/dist/cli/__tests__/logger.unit.test.js.map +1 -0
- package/dist/cli/__tests__/module-loader.unit.test.d.ts +11 -0
- package/dist/cli/__tests__/module-loader.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/module-loader.unit.test.js +262 -0
- package/dist/cli/__tests__/module-loader.unit.test.js.map +1 -0
- package/dist/cli/__tests__/output-writer.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/output-writer.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/output-writer.unit.test.js +216 -0
- package/dist/cli/__tests__/output-writer.unit.test.js.map +1 -0
- package/dist/cli/__tests__/plan.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/plan.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/plan.command.unit.test.js +289 -0
- package/dist/cli/__tests__/plan.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/run.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/run.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/run.command.unit.test.js +422 -0
- package/dist/cli/__tests__/run.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/validate.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/validate.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/validate.command.unit.test.js +226 -0
- package/dist/cli/__tests__/validate.command.unit.test.js.map +1 -0
- package/dist/cli/command-deps.d.ts +137 -0
- package/dist/cli/command-deps.d.ts.map +1 -0
- package/dist/cli/command-deps.js +7 -0
- package/dist/cli/command-deps.js.map +1 -0
- package/dist/cli/commands/aggregate.d.ts +35 -0
- package/dist/cli/commands/aggregate.d.ts.map +1 -0
- package/dist/cli/commands/aggregate.js +124 -0
- package/dist/cli/commands/aggregate.js.map +1 -0
- package/dist/cli/commands/evaluate.d.ts +41 -0
- package/dist/cli/commands/evaluate.d.ts.map +1 -0
- package/dist/cli/commands/evaluate.js +287 -0
- package/dist/cli/commands/evaluate.js.map +1 -0
- package/dist/cli/commands/plan.d.ts +36 -0
- package/dist/cli/commands/plan.d.ts.map +1 -0
- package/dist/cli/commands/plan.js +109 -0
- package/dist/cli/commands/plan.js.map +1 -0
- package/dist/cli/commands/run.d.ts +33 -0
- package/dist/cli/commands/run.d.ts.map +1 -0
- package/dist/cli/commands/run.js +277 -0
- package/dist/cli/commands/run.js.map +1 -0
- package/dist/cli/commands/validate.d.ts +27 -0
- package/dist/cli/commands/validate.d.ts.map +1 -0
- package/dist/cli/commands/validate.js +88 -0
- package/dist/cli/commands/validate.js.map +1 -0
- package/dist/cli/config-loader.d.ts +30 -0
- package/dist/cli/config-loader.d.ts.map +1 -0
- package/dist/cli/config-loader.js +181 -0
- package/dist/cli/config-loader.js.map +1 -0
- package/dist/cli/index.d.ts +27 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +60 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/logger.d.ts +75 -0
- package/dist/cli/logger.d.ts.map +1 -0
- package/dist/cli/logger.js +131 -0
- package/dist/cli/logger.js.map +1 -0
- package/dist/cli/module-loader.d.ts +68 -0
- package/dist/cli/module-loader.d.ts.map +1 -0
- package/dist/cli/module-loader.js +134 -0
- package/dist/cli/module-loader.js.map +1 -0
- package/dist/cli/output-writer.d.ts +51 -0
- package/dist/cli/output-writer.d.ts.map +1 -0
- package/dist/cli/output-writer.js +65 -0
- package/dist/cli/output-writer.js.map +1 -0
- package/dist/cli/types.d.ts +193 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +7 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/collector/__tests__/result-collector.unit.test.d.ts +7 -0
- package/dist/collector/__tests__/result-collector.unit.test.d.ts.map +1 -0
- package/dist/collector/__tests__/result-collector.unit.test.js +1021 -0
- package/dist/collector/__tests__/result-collector.unit.test.js.map +1 -0
- package/dist/collector/__tests__/schema.unit.test.d.ts +7 -0
- package/dist/collector/__tests__/schema.unit.test.d.ts.map +1 -0
- package/dist/collector/__tests__/schema.unit.test.js +360 -0
- package/dist/collector/__tests__/schema.unit.test.js.map +1 -0
- package/dist/evaluators/claims-evaluator.d.ts +87 -0
- package/dist/evaluators/claims-evaluator.d.ts.map +1 -0
- package/dist/evaluators/claims-evaluator.js +289 -0
- package/dist/evaluators/claims-evaluator.js.map +1 -0
- package/dist/evaluators/exploratory-evaluator.d.ts +136 -0
- package/dist/evaluators/exploratory-evaluator.d.ts.map +1 -0
- package/dist/evaluators/exploratory-evaluator.js +545 -0
- package/dist/evaluators/exploratory-evaluator.js.map +1 -0
- package/dist/evaluators/index.d.ts +13 -0
- package/dist/evaluators/index.d.ts.map +1 -0
- package/dist/evaluators/index.js +14 -0
- package/dist/evaluators/index.js.map +1 -0
- package/dist/evaluators/metrics-evaluator.d.ts +114 -0
- package/dist/evaluators/metrics-evaluator.d.ts.map +1 -0
- package/dist/evaluators/metrics-evaluator.js +433 -0
- package/dist/evaluators/metrics-evaluator.js.map +1 -0
- package/dist/evaluators/registry.d.ts +106 -0
- package/dist/evaluators/registry.d.ts.map +1 -0
- package/dist/evaluators/registry.js +148 -0
- package/dist/evaluators/registry.js.map +1 -0
- package/dist/evaluators/robustness-evaluator.d.ts +57 -0
- package/dist/evaluators/robustness-evaluator.d.ts.map +1 -0
- package/dist/evaluators/robustness-evaluator.js +186 -0
- package/dist/evaluators/robustness-evaluator.js.map +1 -0
- package/dist/executor/__tests__/binary-sut.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/binary-sut.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/binary-sut.unit.test.js +313 -0
- package/dist/executor/__tests__/binary-sut.unit.test.js.map +1 -0
- package/dist/executor/__tests__/checkpoint-manager.unit.test.js +83 -1
- package/dist/executor/__tests__/checkpoint-manager.unit.test.js.map +1 -1
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.d.ts +3 -6
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.d.ts.map +1 -1
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.js +428 -159
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.js.map +1 -1
- package/dist/executor/__tests__/checkpoint-storage.unit.test.js +148 -1
- package/dist/executor/__tests__/checkpoint-storage.unit.test.js.map +1 -1
- package/dist/executor/__tests__/executor.unit.test.js +123 -8
- package/dist/executor/__tests__/executor.unit.test.js.map +1 -1
- package/dist/executor/__tests__/memory-monitor.unit.test.d.ts +7 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.js +285 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.js.map +1 -0
- package/dist/executor/__tests__/parallel-executor.unit.test.d.ts +2 -1
- package/dist/executor/__tests__/parallel-executor.unit.test.d.ts.map +1 -1
- package/dist/executor/__tests__/parallel-executor.unit.test.js +426 -156
- package/dist/executor/__tests__/parallel-executor.unit.test.js.map +1 -1
- package/dist/executor/__tests__/resource-calculator.unit.test.d.ts +10 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.js +104 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.js.map +1 -0
- package/dist/executor/__tests__/run-id.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/run-id.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/run-id.unit.test.js +156 -0
- package/dist/executor/__tests__/run-id.unit.test.js.map +1 -0
- package/dist/executor/__tests__/worker-entry.integration.test.d.ts +24 -0
- package/dist/executor/__tests__/worker-entry.integration.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-entry.integration.test.js +82 -0
- package/dist/executor/__tests__/worker-entry.integration.test.js.map +1 -0
- package/dist/executor/__tests__/worker-entry.unit.test.d.ts +7 -0
- package/dist/executor/__tests__/worker-entry.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-entry.unit.test.js +364 -0
- package/dist/executor/__tests__/worker-entry.unit.test.js.map +1 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.js +276 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.js.map +1 -0
- package/dist/executor/binary-sut.d.ts +105 -0
- package/dist/executor/binary-sut.d.ts.map +1 -0
- package/dist/executor/binary-sut.js +174 -0
- package/dist/executor/binary-sut.js.map +1 -0
- package/dist/executor/checkpoint-storage.d.ts.map +1 -1
- package/dist/executor/checkpoint-storage.js +6 -4
- package/dist/executor/checkpoint-storage.js.map +1 -1
- package/dist/executor/executor.d.ts +28 -0
- package/dist/executor/executor.d.ts.map +1 -1
- package/dist/executor/executor.js +85 -24
- package/dist/executor/executor.js.map +1 -1
- package/dist/executor/index.d.ts +4 -0
- package/dist/executor/index.d.ts.map +1 -1
- package/dist/executor/index.js +4 -0
- package/dist/executor/index.js.map +1 -1
- package/dist/executor/parallel-executor.d.ts +186 -0
- package/dist/executor/parallel-executor.d.ts.map +1 -1
- package/dist/executor/parallel-executor.js +218 -83
- package/dist/executor/parallel-executor.js.map +1 -1
- package/dist/executor/resource-calculator.d.ts +49 -0
- package/dist/executor/resource-calculator.d.ts.map +1 -0
- package/dist/executor/resource-calculator.js +129 -0
- package/dist/executor/resource-calculator.js.map +1 -0
- package/dist/executor/run-id.d.ts.map +1 -1
- package/dist/executor/run-id.js +8 -1
- package/dist/executor/run-id.js.map +1 -1
- package/dist/executor/worker-entry.d.ts +2 -0
- package/dist/executor/worker-entry.d.ts.map +1 -1
- package/dist/executor/worker-entry.js +46 -55
- package/dist/executor/worker-entry.js.map +1 -1
- package/dist/executor/worker-executor.d.ts +257 -0
- package/dist/executor/worker-executor.d.ts.map +1 -0
- package/dist/executor/worker-executor.js +308 -0
- package/dist/executor/worker-executor.js.map +1 -0
- package/dist/executor/worker-threads-executor.d.ts +245 -0
- package/dist/executor/worker-threads-executor.d.ts.map +1 -0
- package/dist/executor/worker-threads-executor.js +332 -0
- package/dist/executor/worker-threads-executor.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/renderers/latex-renderer.d.ts +60 -0
- package/dist/renderers/latex-renderer.d.ts.map +1 -1
- package/dist/renderers/latex-renderer.js +299 -0
- package/dist/renderers/latex-renderer.js.map +1 -1
- package/dist/renderers/types.d.ts +9 -0
- package/dist/renderers/types.d.ts.map +1 -1
- package/dist/renderers/types.js.map +1 -1
- package/dist/robustness/__tests__/perturbations.unit.test.d.ts +11 -0
- package/dist/robustness/__tests__/perturbations.unit.test.d.ts.map +1 -0
- package/dist/robustness/__tests__/perturbations.unit.test.js +284 -0
- package/dist/robustness/__tests__/perturbations.unit.test.js.map +1 -0
- package/dist/robustness/index.d.ts +5 -2
- package/dist/robustness/index.d.ts.map +1 -1
- package/dist/robustness/index.js +4 -2
- package/dist/robustness/index.js.map +1 -1
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.d.ts +7 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.d.ts.map +1 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.js +185 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.js.map +1 -0
- package/dist/types/evaluator.d.ts +449 -0
- package/dist/types/evaluator.d.ts.map +1 -0
- package/dist/types/evaluator.js +9 -0
- package/dist/types/evaluator.js.map +1 -0
- package/dist/types/result.d.ts +2 -0
- package/dist/types/result.d.ts.map +1 -1
- package/package.json +8 -1
- package/dist/claims/evaluator.d.ts +0 -33
- package/dist/claims/evaluator.d.ts.map +0 -1
- package/dist/claims/evaluator.js +0 -174
- package/dist/claims/evaluator.js.map +0 -1
- package/dist/robustness/analyzer.d.ts +0 -61
- package/dist/robustness/analyzer.d.ts.map +0 -1
- package/dist/robustness/analyzer.js +0 -191
- package/dist/robustness/analyzer.js.map +0 -1
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Metrics Evaluator
|
|
3
|
+
*
|
|
4
|
+
* Evaluates metrics against threshold, baseline, and target-range criteria.
|
|
5
|
+
* This evaluator enables evaluation without claims by checking if metrics
|
|
6
|
+
* meet specified criteria (thresholds, baselines, or target ranges).
|
|
7
|
+
*/
|
|
8
|
+
import type { IEvaluator, MetricsEvaluatorConfig, MetricsEvaluatorData, EvaluationContext, Evaluator, EvaluationOutput, EvaluationSummary, ValidationResult } from "../types/evaluator.js";
|
|
9
|
+
/**
|
|
10
|
+
* Metrics evaluator - evaluates metrics against criteria.
|
|
11
|
+
*/
|
|
12
|
+
export declare class MetricsEvaluator implements Evaluator<MetricsEvaluatorConfig, EvaluationContext, MetricsEvaluatorData>, IEvaluator {
|
|
13
|
+
/** Type identifier */
|
|
14
|
+
readonly type: "metrics";
|
|
15
|
+
/** Schema version */
|
|
16
|
+
private static readonly VERSION;
|
|
17
|
+
/**
|
|
18
|
+
* Validate metrics evaluator configuration.
|
|
19
|
+
*
|
|
20
|
+
* @param config - Configuration to validate
|
|
21
|
+
* @returns Validation result
|
|
22
|
+
*/
|
|
23
|
+
validateConfig(config: MetricsEvaluatorConfig): ValidationResult;
|
|
24
|
+
/**
|
|
25
|
+
* Validate a single criterion.
|
|
26
|
+
*
|
|
27
|
+
* @param criterion - Criterion to validate
|
|
28
|
+
* @param index - Index in criteria array (for error messages)
|
|
29
|
+
* @returns Array of error messages
|
|
30
|
+
*/
|
|
31
|
+
private validateCriterion;
|
|
32
|
+
/**
|
|
33
|
+
* Evaluate metrics against criteria.
|
|
34
|
+
*
|
|
35
|
+
* @param config - Metrics evaluator configuration
|
|
36
|
+
* @param input - Evaluation context with aggregates
|
|
37
|
+
* @returns Evaluation output
|
|
38
|
+
*/
|
|
39
|
+
evaluate(config: MetricsEvaluatorConfig, input: EvaluationContext): EvaluationOutput<MetricsEvaluatorData>;
|
|
40
|
+
/**
|
|
41
|
+
* Evaluate a single criterion.
|
|
42
|
+
*
|
|
43
|
+
* @param criterion - Criterion to evaluate
|
|
44
|
+
* @param aggregates - Aggregated results
|
|
45
|
+
* @returns Criterion result
|
|
46
|
+
*/
|
|
47
|
+
private evaluateCriterion;
|
|
48
|
+
/**
|
|
49
|
+
* Filter aggregates by SUT.
|
|
50
|
+
*
|
|
51
|
+
* @param aggregates - All aggregates
|
|
52
|
+
* @param sut - SUT identifier or "*" for all
|
|
53
|
+
* @returns Filtered aggregates
|
|
54
|
+
*/
|
|
55
|
+
private filterBySut;
|
|
56
|
+
/**
|
|
57
|
+
* Evaluate criterion by type.
|
|
58
|
+
*
|
|
59
|
+
* @param criterion - Criterion to evaluate
|
|
60
|
+
* @param observed - Observed values
|
|
61
|
+
* @param aggregates - All aggregates (needed for baseline to find baseline SUT)
|
|
62
|
+
* @returns Criterion result
|
|
63
|
+
*/
|
|
64
|
+
private evaluateByType;
|
|
65
|
+
/**
|
|
66
|
+
* Evaluate threshold criterion.
|
|
67
|
+
*
|
|
68
|
+
* @param criterion - Threshold criterion
|
|
69
|
+
* @param observed - Observed values
|
|
70
|
+
* @returns Criterion result
|
|
71
|
+
*/
|
|
72
|
+
private evaluateThreshold;
|
|
73
|
+
/**
|
|
74
|
+
* Evaluate baseline criterion.
|
|
75
|
+
*
|
|
76
|
+
* @param criterion - Baseline criterion
|
|
77
|
+
* @param observed - Observed values
|
|
78
|
+
* @param aggregates - Filtered aggregates
|
|
79
|
+
* @returns Criterion result
|
|
80
|
+
*/
|
|
81
|
+
private evaluateBaseline;
|
|
82
|
+
/**
|
|
83
|
+
* Evaluate target-range criterion.
|
|
84
|
+
*
|
|
85
|
+
* @param criterion - Target-range criterion
|
|
86
|
+
* @param observed - Observed values
|
|
87
|
+
* @returns Criterion result
|
|
88
|
+
*/
|
|
89
|
+
private evaluateTargetRange;
|
|
90
|
+
/**
|
|
91
|
+
* Compare value against threshold using operator.
|
|
92
|
+
*
|
|
93
|
+
* @param value - Value to check
|
|
94
|
+
* @param operator - Comparison operator
|
|
95
|
+
* @param threshold - Threshold value
|
|
96
|
+
* @returns Whether the comparison passes
|
|
97
|
+
*/
|
|
98
|
+
private compareOperator;
|
|
99
|
+
/**
|
|
100
|
+
* Compute summary statistics.
|
|
101
|
+
*
|
|
102
|
+
* @param results - All criterion results
|
|
103
|
+
* @returns Summary statistics
|
|
104
|
+
*/
|
|
105
|
+
private computeSummary;
|
|
106
|
+
/**
|
|
107
|
+
* Summarize evaluation output.
|
|
108
|
+
*
|
|
109
|
+
* @param output - Evaluation output to summarize
|
|
110
|
+
* @returns Summary statistics
|
|
111
|
+
*/
|
|
112
|
+
summarize(output: EvaluationOutput<MetricsEvaluatorData>): EvaluationSummary;
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=metrics-evaluator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics-evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluators/metrics-evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,KAAK,EACX,UAAU,EAIV,sBAAsB,EACtB,oBAAoB,EACpB,iBAAiB,EACjB,SAAS,EACT,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,MAAM,uBAAuB,CAAC;AAE/B;;GAEG;AACH,qBAAa,gBACZ,YAAW,SAAS,CAAC,sBAAsB,EAAE,iBAAiB,EAAE,oBAAoB,CAAC,EAAE,UAAU;IAEjG,sBAAsB;IACtB,QAAQ,CAAC,IAAI,EAAG,SAAS,CAAU;IAEnC,qBAAqB;IACrB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAW;IAE1C;;;;;OAKG;IACH,cAAc,CAAC,MAAM,EAAE,sBAAsB,GAAG,gBAAgB;IAyBhE;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;IAkEzB;;;;;;OAMG;IACH,QAAQ,CACP,MAAM,EAAE,sBAAsB,EAC9B,KAAK,EAAE,iBAAiB,GACtB,gBAAgB,CAAC,oBAAoB,CAAC;IA4BzC;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;IAkDzB;;;;;;OAMG;IACH,OAAO,CAAC,WAAW;IAOnB;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAiBtB;;;;;;OAMG;IACH,OAAO,CAAC,iBAAiB;IA8BzB;;;;;;;OAOG;IACH,OAAO,CAAC,gBAAgB;IAsExB;;;;;;OAMG;IACH,OAAO,CAAC,mBAAmB;IAqC3B;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe;IAiBvB;;;;;OAKG;IACH,OAAO,CAAC,cAAc;IAkCtB;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,gBAAgB,CAAC,oBAAoB,CAAC,GAAG,iBAAiB;CAc5E"}
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Metrics Evaluator
|
|
3
|
+
*
|
|
4
|
+
* Evaluates metrics against threshold, baseline, and target-range criteria.
|
|
5
|
+
* This evaluator enables evaluation without claims by checking if metrics
|
|
6
|
+
* meet specified criteria (thresholds, baselines, or target ranges).
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Metrics evaluator - evaluates metrics against criteria.
|
|
10
|
+
*/
|
|
11
|
+
export class MetricsEvaluator {
|
|
12
|
+
/** Type identifier */
|
|
13
|
+
type = "metrics";
|
|
14
|
+
/** Schema version */
|
|
15
|
+
static VERSION = "1.0.0";
|
|
16
|
+
/**
|
|
17
|
+
* Validate metrics evaluator configuration.
|
|
18
|
+
*
|
|
19
|
+
* @param config - Configuration to validate
|
|
20
|
+
* @returns Validation result
|
|
21
|
+
*/
|
|
22
|
+
validateConfig(config) {
|
|
23
|
+
const errors = [];
|
|
24
|
+
const warnings = [];
|
|
25
|
+
// Check criteria array
|
|
26
|
+
if (!Array.isArray(config.criteria)) {
|
|
27
|
+
errors.push("criteria must be an array");
|
|
28
|
+
}
|
|
29
|
+
else if (config.criteria.length === 0) {
|
|
30
|
+
warnings.push("No criteria provided - evaluation will produce empty results");
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
// Validate each criterion
|
|
34
|
+
for (let i = 0; i < config.criteria.length; i++) {
|
|
35
|
+
const criterion = config.criteria[i];
|
|
36
|
+
const criterionErrors = this.validateCriterion(criterion, i);
|
|
37
|
+
errors.push(...criterionErrors);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
valid: errors.length === 0,
|
|
42
|
+
errors: errors.length > 0 ? errors : undefined,
|
|
43
|
+
warnings: warnings.length > 0 ? warnings : undefined,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Validate a single criterion.
|
|
48
|
+
*
|
|
49
|
+
* @param criterion - Criterion to validate
|
|
50
|
+
* @param index - Index in criteria array (for error messages)
|
|
51
|
+
* @returns Array of error messages
|
|
52
|
+
*/
|
|
53
|
+
validateCriterion(criterion, index) {
|
|
54
|
+
const errors = [];
|
|
55
|
+
const prefix = `Criterion[${index}]`;
|
|
56
|
+
if (!criterion.criterionId || typeof criterion.criterionId !== "string") {
|
|
57
|
+
errors.push(`${prefix}: criterionId is required`);
|
|
58
|
+
}
|
|
59
|
+
if (!criterion.description || typeof criterion.description !== "string") {
|
|
60
|
+
errors.push(`${prefix}: description is required`);
|
|
61
|
+
}
|
|
62
|
+
if (!criterion.metric || typeof criterion.metric !== "string") {
|
|
63
|
+
errors.push(`${prefix}: metric is required`);
|
|
64
|
+
}
|
|
65
|
+
if (!criterion.sut || typeof criterion.sut !== "string") {
|
|
66
|
+
errors.push(`${prefix}: sut is required`);
|
|
67
|
+
}
|
|
68
|
+
// Validate type-specific fields
|
|
69
|
+
switch (criterion.type) {
|
|
70
|
+
case "threshold":
|
|
71
|
+
if (!criterion.threshold) {
|
|
72
|
+
errors.push(`${prefix}: threshold required for threshold type`);
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
if (!["gt", "gte", "lt", "lte", "eq"].includes(criterion.threshold.operator)) {
|
|
76
|
+
errors.push(`${prefix}: threshold.operator must be valid`);
|
|
77
|
+
}
|
|
78
|
+
if (typeof criterion.threshold.value !== "number") {
|
|
79
|
+
errors.push(`${prefix}: threshold.value must be a number`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
break;
|
|
83
|
+
case "baseline":
|
|
84
|
+
if (!criterion.baseline) {
|
|
85
|
+
errors.push(`${prefix}: baseline required for baseline type`);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
if (!criterion.baseline.sut || typeof criterion.baseline.sut !== "string") {
|
|
89
|
+
errors.push(`${prefix}: baseline.sut is required`);
|
|
90
|
+
}
|
|
91
|
+
if (!["gt", "gte", "lt", "lte", "eq"].includes(criterion.baseline.operator)) {
|
|
92
|
+
errors.push(`${prefix}: baseline.operator must be valid`);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
break;
|
|
96
|
+
case "target-range":
|
|
97
|
+
if (!criterion.targetRange) {
|
|
98
|
+
errors.push(`${prefix}: targetRange required for target-range type`);
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
if (criterion.targetRange.min === undefined && criterion.targetRange.max === undefined) {
|
|
102
|
+
errors.push(`${prefix}: targetRange must have min or max`);
|
|
103
|
+
}
|
|
104
|
+
if (criterion.targetRange.min !== undefined &&
|
|
105
|
+
criterion.targetRange.max !== undefined &&
|
|
106
|
+
criterion.targetRange.min > criterion.targetRange.max) {
|
|
107
|
+
errors.push(`${prefix}: targetRange.min must be <= targetRange.max`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
return errors;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Evaluate metrics against criteria.
|
|
116
|
+
*
|
|
117
|
+
* @param config - Metrics evaluator configuration
|
|
118
|
+
* @param input - Evaluation context with aggregates
|
|
119
|
+
* @returns Evaluation output
|
|
120
|
+
*/
|
|
121
|
+
evaluate(config, input) {
|
|
122
|
+
const { aggregates } = input;
|
|
123
|
+
// Evaluate all criteria
|
|
124
|
+
const results = config.criteria.map((criterion) => this.evaluateCriterion(criterion, aggregates));
|
|
125
|
+
// Compute summary
|
|
126
|
+
const summary = this.computeSummary(results);
|
|
127
|
+
return {
|
|
128
|
+
type: "metrics",
|
|
129
|
+
version: MetricsEvaluator.VERSION,
|
|
130
|
+
timestamp: new Date().toISOString(),
|
|
131
|
+
data: {
|
|
132
|
+
version: "1.0.0",
|
|
133
|
+
timestamp: new Date().toISOString(),
|
|
134
|
+
results,
|
|
135
|
+
summary,
|
|
136
|
+
},
|
|
137
|
+
metadata: {
|
|
138
|
+
inputSource: input.metadata?.source,
|
|
139
|
+
config,
|
|
140
|
+
},
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
/**
|
|
144
|
+
* Evaluate a single criterion.
|
|
145
|
+
*
|
|
146
|
+
* @param criterion - Criterion to evaluate
|
|
147
|
+
* @param aggregates - Aggregated results
|
|
148
|
+
* @returns Criterion result
|
|
149
|
+
*/
|
|
150
|
+
evaluateCriterion(criterion, aggregates) {
|
|
151
|
+
// Filter aggregates by SUT (or get all if "*")
|
|
152
|
+
const relevantAggregates = this.filterBySut(aggregates, criterion.sut);
|
|
153
|
+
if (relevantAggregates.length === 0) {
|
|
154
|
+
return {
|
|
155
|
+
criterion,
|
|
156
|
+
status: "inconclusive",
|
|
157
|
+
observed: [],
|
|
158
|
+
expected: {
|
|
159
|
+
type: criterion.type,
|
|
160
|
+
},
|
|
161
|
+
inconclusiveReason: `No aggregates found for SUT: ${criterion.sut}`,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
// Collect observed values
|
|
165
|
+
const observed = [];
|
|
166
|
+
for (const agg of relevantAggregates) {
|
|
167
|
+
if (criterion.metric in agg.metrics) {
|
|
168
|
+
const stats = agg.metrics[criterion.metric];
|
|
169
|
+
if (typeof stats.mean === "number") {
|
|
170
|
+
observed.push({
|
|
171
|
+
sut: agg.sut,
|
|
172
|
+
value: stats.mean,
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
if (observed.length === 0) {
|
|
178
|
+
return {
|
|
179
|
+
criterion,
|
|
180
|
+
status: "inconclusive",
|
|
181
|
+
observed: [],
|
|
182
|
+
expected: {
|
|
183
|
+
type: criterion.type,
|
|
184
|
+
},
|
|
185
|
+
inconclusiveReason: `Metric ${criterion.metric} not found in aggregates`,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
// Evaluate based on criterion type
|
|
189
|
+
// Note: Pass original aggregates for baseline evaluation (needs all SUTs)
|
|
190
|
+
return this.evaluateByType(criterion, observed, aggregates);
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Filter aggregates by SUT.
|
|
194
|
+
*
|
|
195
|
+
* @param aggregates - All aggregates
|
|
196
|
+
* @param sut - SUT identifier or "*" for all
|
|
197
|
+
* @returns Filtered aggregates
|
|
198
|
+
*/
|
|
199
|
+
filterBySut(aggregates, sut) {
|
|
200
|
+
if (sut === "*") {
|
|
201
|
+
return aggregates;
|
|
202
|
+
}
|
|
203
|
+
return aggregates.filter((agg) => agg.sut === sut);
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Evaluate criterion by type.
|
|
207
|
+
*
|
|
208
|
+
* @param criterion - Criterion to evaluate
|
|
209
|
+
* @param observed - Observed values
|
|
210
|
+
* @param aggregates - All aggregates (needed for baseline to find baseline SUT)
|
|
211
|
+
* @returns Criterion result
|
|
212
|
+
*/
|
|
213
|
+
evaluateByType(criterion, observed, aggregates) {
|
|
214
|
+
switch (criterion.type) {
|
|
215
|
+
case "threshold":
|
|
216
|
+
return this.evaluateThreshold(criterion, observed);
|
|
217
|
+
case "baseline":
|
|
218
|
+
return this.evaluateBaseline(criterion, observed, aggregates);
|
|
219
|
+
case "target-range":
|
|
220
|
+
return this.evaluateTargetRange(criterion, observed);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Evaluate threshold criterion.
|
|
225
|
+
*
|
|
226
|
+
* @param criterion - Threshold criterion
|
|
227
|
+
* @param observed - Observed values
|
|
228
|
+
* @returns Criterion result
|
|
229
|
+
*/
|
|
230
|
+
evaluateThreshold(criterion, observed) {
|
|
231
|
+
if (!criterion.threshold) {
|
|
232
|
+
throw new Error("Threshold criterion must have threshold property");
|
|
233
|
+
}
|
|
234
|
+
const { operator, value } = criterion.threshold;
|
|
235
|
+
// For single SUT, use that result; for multiple, require all to pass
|
|
236
|
+
const status = criterion.sut === "*"
|
|
237
|
+
? observed.every((obs) => this.compareOperator(obs.value, operator, value))
|
|
238
|
+
? "pass"
|
|
239
|
+
: "fail"
|
|
240
|
+
: observed.some((obs) => this.compareOperator(obs.value, operator, value))
|
|
241
|
+
? "pass"
|
|
242
|
+
: "fail";
|
|
243
|
+
return {
|
|
244
|
+
criterion,
|
|
245
|
+
status,
|
|
246
|
+
observed,
|
|
247
|
+
expected: {
|
|
248
|
+
type: "threshold",
|
|
249
|
+
threshold: value,
|
|
250
|
+
},
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Evaluate baseline criterion.
|
|
255
|
+
*
|
|
256
|
+
* @param criterion - Baseline criterion
|
|
257
|
+
* @param observed - Observed values
|
|
258
|
+
* @param aggregates - Filtered aggregates
|
|
259
|
+
* @returns Criterion result
|
|
260
|
+
*/
|
|
261
|
+
evaluateBaseline(criterion, observed, aggregates) {
|
|
262
|
+
if (!criterion.baseline) {
|
|
263
|
+
throw new Error("Baseline criterion must have baseline property");
|
|
264
|
+
}
|
|
265
|
+
const { sut: baselineSut, operator } = criterion.baseline;
|
|
266
|
+
// Find baseline aggregate
|
|
267
|
+
const baselineAgg = aggregates.find((agg) => agg.sut === baselineSut);
|
|
268
|
+
if (!baselineAgg) {
|
|
269
|
+
return {
|
|
270
|
+
criterion,
|
|
271
|
+
status: "inconclusive",
|
|
272
|
+
observed,
|
|
273
|
+
expected: {
|
|
274
|
+
type: "baseline",
|
|
275
|
+
},
|
|
276
|
+
inconclusiveReason: `Baseline SUT not found: ${baselineSut}`,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
if (!(criterion.metric in baselineAgg.metrics)) {
|
|
280
|
+
return {
|
|
281
|
+
criterion,
|
|
282
|
+
status: "inconclusive",
|
|
283
|
+
observed,
|
|
284
|
+
expected: {
|
|
285
|
+
type: "baseline",
|
|
286
|
+
},
|
|
287
|
+
inconclusiveReason: `Baseline metric not found: ${criterion.metric}`,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
const baselineStats = baselineAgg.metrics[criterion.metric];
|
|
291
|
+
if (typeof baselineStats.mean !== "number") {
|
|
292
|
+
return {
|
|
293
|
+
criterion,
|
|
294
|
+
status: "inconclusive",
|
|
295
|
+
observed,
|
|
296
|
+
expected: {
|
|
297
|
+
type: "baseline",
|
|
298
|
+
},
|
|
299
|
+
inconclusiveReason: `Baseline metric not found: ${criterion.metric}`,
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
const baselineValue = baselineStats.mean;
|
|
303
|
+
// Check primary observed values against baseline
|
|
304
|
+
const primaryObserved = observed.filter((obs) => obs.sut !== baselineSut);
|
|
305
|
+
const status = primaryObserved.some((obs) => this.compareOperator(obs.value, operator, baselineValue))
|
|
306
|
+
? "pass"
|
|
307
|
+
: "fail";
|
|
308
|
+
return {
|
|
309
|
+
criterion,
|
|
310
|
+
status,
|
|
311
|
+
observed,
|
|
312
|
+
expected: {
|
|
313
|
+
type: "baseline",
|
|
314
|
+
baselineValue,
|
|
315
|
+
},
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Evaluate target-range criterion.
|
|
320
|
+
*
|
|
321
|
+
* @param criterion - Target-range criterion
|
|
322
|
+
* @param observed - Observed values
|
|
323
|
+
* @returns Criterion result
|
|
324
|
+
*/
|
|
325
|
+
evaluateTargetRange(criterion, observed) {
|
|
326
|
+
if (!criterion.targetRange) {
|
|
327
|
+
throw new Error("Target-range criterion must have targetRange property");
|
|
328
|
+
}
|
|
329
|
+
const { min, max, minInclusive = true, maxInclusive = true } = criterion.targetRange;
|
|
330
|
+
// Check each observed value
|
|
331
|
+
const inRange = (value) => {
|
|
332
|
+
const aboveMin = min === undefined || (minInclusive ? value >= min : value > min);
|
|
333
|
+
const belowMax = max === undefined || (maxInclusive ? value <= max : value < max);
|
|
334
|
+
return aboveMin && belowMax;
|
|
335
|
+
};
|
|
336
|
+
// For single SUT, use that result; for multiple, require all to pass
|
|
337
|
+
const status = criterion.sut === "*"
|
|
338
|
+
? observed.every((obs) => inRange(obs.value))
|
|
339
|
+
? "pass"
|
|
340
|
+
: "fail"
|
|
341
|
+
: observed.some((obs) => inRange(obs.value))
|
|
342
|
+
? "pass"
|
|
343
|
+
: "fail";
|
|
344
|
+
return {
|
|
345
|
+
criterion,
|
|
346
|
+
status,
|
|
347
|
+
observed,
|
|
348
|
+
expected: {
|
|
349
|
+
type: "target-range",
|
|
350
|
+
targetRange: { min, max },
|
|
351
|
+
},
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Compare value against threshold using operator.
|
|
356
|
+
*
|
|
357
|
+
* @param value - Value to check
|
|
358
|
+
* @param operator - Comparison operator
|
|
359
|
+
* @param threshold - Threshold value
|
|
360
|
+
* @returns Whether the comparison passes
|
|
361
|
+
*/
|
|
362
|
+
compareOperator(value, operator, threshold) {
|
|
363
|
+
switch (operator) {
|
|
364
|
+
case "gt":
|
|
365
|
+
return value > threshold;
|
|
366
|
+
case "gte":
|
|
367
|
+
return value >= threshold;
|
|
368
|
+
case "lt":
|
|
369
|
+
return value < threshold;
|
|
370
|
+
case "lte":
|
|
371
|
+
return value <= threshold;
|
|
372
|
+
case "eq":
|
|
373
|
+
return Math.abs(value - threshold) < 0.0001; // Epsilon for floating point
|
|
374
|
+
default:
|
|
375
|
+
return false;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Compute summary statistics.
|
|
380
|
+
*
|
|
381
|
+
* @param results - All criterion results
|
|
382
|
+
* @returns Summary statistics
|
|
383
|
+
*/
|
|
384
|
+
computeSummary(results) {
|
|
385
|
+
const total = results.length;
|
|
386
|
+
const passed = results.filter((r) => r.status === "pass").length;
|
|
387
|
+
const failed = results.filter((r) => r.status === "fail").length;
|
|
388
|
+
const inconclusive = results.filter((r) => r.status === "inconclusive").length;
|
|
389
|
+
const passRate = total > 0 ? passed / total : 0;
|
|
390
|
+
// Compute pass rate by SUT
|
|
391
|
+
const passRateBySut = {};
|
|
392
|
+
const sutResults = new Map();
|
|
393
|
+
for (const result of results) {
|
|
394
|
+
for (const obs of result.observed) {
|
|
395
|
+
const existing = sutResults.get(obs.sut) ?? [];
|
|
396
|
+
existing.push(result);
|
|
397
|
+
sutResults.set(obs.sut, existing);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
for (const [sut, sutResultList] of sutResults) {
|
|
401
|
+
const sutPassed = sutResultList.filter((r) => r.status === "pass").length;
|
|
402
|
+
passRateBySut[sut] = sutPassed / sutResultList.length;
|
|
403
|
+
}
|
|
404
|
+
return {
|
|
405
|
+
total,
|
|
406
|
+
passed,
|
|
407
|
+
failed,
|
|
408
|
+
inconclusive,
|
|
409
|
+
passRate,
|
|
410
|
+
passRateBySut,
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Summarize evaluation output.
|
|
415
|
+
*
|
|
416
|
+
* @param output - Evaluation output to summarize
|
|
417
|
+
* @returns Summary statistics
|
|
418
|
+
*/
|
|
419
|
+
summarize(output) {
|
|
420
|
+
const { summary } = output.data;
|
|
421
|
+
return {
|
|
422
|
+
total: summary.total,
|
|
423
|
+
passed: summary.passed,
|
|
424
|
+
failed: summary.failed,
|
|
425
|
+
inconclusive: summary.inconclusive,
|
|
426
|
+
passRate: summary.passRate,
|
|
427
|
+
additional: {
|
|
428
|
+
passRateBySut: JSON.stringify(summary.passRateBySut),
|
|
429
|
+
},
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
//# sourceMappingURL=metrics-evaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics-evaluator.js","sourceRoot":"","sources":["../../src/evaluators/metrics-evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAiBH;;GAEG;AACH,MAAM,OAAO,gBAAgB;IAG5B,sBAAsB;IACb,IAAI,GAAG,SAAkB,CAAC;IAEnC,qBAAqB;IACb,MAAM,CAAU,OAAO,GAAG,OAAO,CAAC;IAE1C;;;;;OAKG;IACH,cAAc,CAAC,MAA8B;QAC5C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAE9B,uBAAuB;QACvB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAC1C,CAAC;aAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;QAC/E,CAAC;aAAM,CAAC;YACP,0BAA0B;YAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACrC,MAAM,eAAe,GAAG,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBAC7D,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;YACjC,CAAC;QACF,CAAC;QAED,OAAO;YACN,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;YAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YAC9C,QAAQ,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;SACpD,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,iBAAiB,CAAC,SAA2B,EAAE,KAAa;QACnE,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,aAAa,KAAK,GAAG,CAAC;QAErC,IAAI,CAAC,SAAS,CAAC,WAAW,IAAI,OAAO,SAAS,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,2BAA2B,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,WAAW,IAAI,OAAO,SAAS,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YACzE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,2BAA2B,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,MAAM,IAAI,OAAO,SAAS,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/D,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,sBAAsB,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,OAAO,SAAS,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,mBAAmB,CAAC,CAAC;QAC3C,CAAC;QAED,gCAAgC;QAChC,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;YACxB,KAAK,WAAW;gBACf,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;oBAC1B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,yCAAyC,CAAC,CAAC;gBACjE,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC9E,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,oCAAoC,CAAC,CAAC;oBAC5D,CAAC;oBACD,IAAI,OAAO,SAAS,CAAC,SAAS,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;wBACnD,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,oCAAoC,CAAC,CAAC;oBAC5D,CAAC;gBACF,CAAC;gBACD,MAAM;YAEP,KAAK,UAAU;gBACd,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;oBACzB,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,uCAAuC,CAAC,CAAC;gBAC/D,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,IAAI,OAAO,SAAS,CAAC,QAAQ,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;wBAC3E,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,4BAA4B,CAAC,CAAC;oBACpD,CAAC;oBACD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAC7E,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,mCAAmC,CAAC,CAAC;oBAC3D,CAAC;gBACF,CAAC;gBACD,MAAM;YAEP,KAAK,cAAc;gBAClB,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;oBAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,8CAA8C,CAAC,CAAC;gBACtE,CAAC;qBAAM,CAAC;oBACP,IAAI,SAAS,CAAC,WAAW,CAAC,GAAG,KAAK,SAAS,IAAI,SAAS,CAAC,WAAW,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;wBACxF,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,oCAAoC,CAAC,CAAC;oBAC5D,CAAC;oBACD,IACC,SAAS,CAAC,WAAW,CAAC,GAAG,KAAK,SAAS;wBACvC,SAAS,CAAC,WAAW,CAAC,GAAG,KAAK,SAAS;wBACvC,SAAS,CAAC,WAAW,CAAC,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,EACpD,CAAC;wBACF,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,8CAA8C,CAAC,CAAC;oBACtE,CAAC;gBACF,CAAC;gBACD,MAAM;QACR,CAAC;QAED,OAAO,MAAM,CAAC;IACf,CAAC;IAED;;;;;;OAMG;IACH,QAAQ,CACP,MAA8B,EAC9B,KAAwB;QAExB,MAAM,EAAE,UAAU,EAAE,GAAG,KAAK,CAAC;QAE7B,wBAAwB;QACxB,MAAM,OAAO,GAA6B,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAC3E,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,UAAU,CAAC,CAC7C,CAAC;QAEF,kBAAkB;QAClB,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAE7C,OAAO;YACN,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,gBAAgB,CAAC,OAAO;YACjC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,IAAI,EAAE;gBACL,OAAO,EAAE,OAAO;gBAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,OAAO;gBACP,OAAO;aACP;YACD,QAAQ,EAAE;gBACT,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM;gBACnC,MAAM;aACN;SACD,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,iBAAiB,CACxB,SAA2B,EAC3B,UAA8B;QAE9B,+CAA+C;QAC/C,MAAM,kBAAkB,GAAG,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC;QAEvE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO;gBACN,SAAS;gBACT,MAAM,EAAE,cAAc;gBACtB,QAAQ,EAAE,EAAE;gBACZ,QAAQ,EAAE;oBACT,IAAI,EAAE,SAAS,CAAC,IAAI;iBACpB;gBACD,kBAAkB,EAAE,gCAAgC,SAAS,CAAC,GAAG,EAAE;aACnE,CAAC;QACH,CAAC;QAED,0BAA0B;QAC1B,MAAM,QAAQ,GAAqC,EAAE,CAAC;QACtD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACtC,IAAI,SAAS,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC5C,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACpC,QAAQ,CAAC,IAAI,CAAC;wBACb,GAAG,EAAE,GAAG,CAAC,GAAG;wBACZ,KAAK,EAAE,KAAK,CAAC,IAAI;qBACjB,CAAC,CAAC;gBACJ,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO;gBACN,SAAS;gBACT,MAAM,EAAE,cAAc;gBACtB,QAAQ,EAAE,EAAE;gBACZ,QAAQ,EAAE;oBACT,IAAI,EAAE,SAAS,CAAC,IAAI;iBACpB;gBACD,kBAAkB,EAAE,UAAU,SAAS,CAAC,MAAM,0BAA0B;aACxE,CAAC;QACH,CAAC;QAED,mCAAmC;QACnC,0EAA0E;QAC1E,OAAO,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC7D,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,UAA8B,EAAE,GAAW;QAC9D,IAAI,GAAG,KAAK,GAAG,EAAE,CAAC;YACjB,OAAO,UAAU,CAAC;QACnB,CAAC;QACD,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;;OAOG;IACK,cAAc,CACrB,SAA2B,EAC3B,QAA0C,EAC1C,UAA8B;QAE9B,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;YACxB,KAAK,WAAW;gBACf,OAAO,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAEpD,KAAK,UAAU;gBACd,OAAO,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;YAE/D,KAAK,cAAc;gBAClB,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACvD,CAAC;IACF,CAAC;IAED;;;;;;OAMG;IACK,iBAAiB,CACxB,SAA2B,EAC3B,QAA0C;QAE1C,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACrE,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,SAAS,CAAC;QAEhD,qEAAqE;QACrE,MAAM,MAAM,GACX,SAAS,CAAC,GAAG,KAAK,GAAG;YACpB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAC1E,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,MAAM;YACT,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACzE,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,MAAM,CAAC;QAEZ,OAAO;YACN,SAAS;YACT,MAAM;YACN,QAAQ;YACR,QAAQ,EAAE;gBACT,IAAI,EAAE,WAAW;gBACjB,SAAS,EAAE,KAAK;aAChB;SACD,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,gBAAgB,CACvB,SAA2B,EAC3B,QAA0C,EAC1C,UAA8B;QAE9B,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,QAAQ,CAAC;QAE1D,0BAA0B;QAC1B,MAAM,WAAW,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,WAAW,CAAC,CAAC;QACtE,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,OAAO;gBACN,SAAS;gBACT,MAAM,EAAE,cAAc;gBACtB,QAAQ;gBACR,QAAQ,EAAE;oBACT,IAAI,EAAE,UAAU;iBAChB;gBACD,kBAAkB,EAAE,2BAA2B,WAAW,EAAE;aAC5D,CAAC;QACH,CAAC;QAED,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;YAChD,OAAO;gBACN,SAAS;gBACT,MAAM,EAAE,cAAc;gBACtB,QAAQ;gBACR,QAAQ,EAAE;oBACT,IAAI,EAAE,UAAU;iBAChB;gBACD,kBAAkB,EAAE,8BAA8B,SAAS,CAAC,MAAM,EAAE;aACpE,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAI,OAAO,aAAa,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC5C,OAAO;gBACN,SAAS;gBACT,MAAM,EAAE,cAAc;gBACtB,QAAQ;gBACR,QAAQ,EAAE;oBACT,IAAI,EAAE,UAAU;iBAChB;gBACD,kBAAkB,EAAE,8BAA8B,SAAS,CAAC,MAAM,EAAE;aACpE,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC;QAEzC,iDAAiD;QACjD,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,WAAW,CAAC,CAAC;QAC1E,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAC3C,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,aAAa,CAAC,CACxD;YACA,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,MAAM,CAAC;QAEV,OAAO;YACN,SAAS;YACT,MAAM;YACN,QAAQ;YACR,QAAQ,EAAE;gBACT,IAAI,EAAE,UAAU;gBAChB,aAAa;aACb;SACD,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,mBAAmB,CAC1B,SAA2B,EAC3B,QAA0C;QAE1C,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,YAAY,GAAG,IAAI,EAAE,YAAY,GAAG,IAAI,EAAE,GAAG,SAAS,CAAC,WAAW,CAAC;QAErF,4BAA4B;QAC5B,MAAM,OAAO,GAAG,CAAC,KAAa,EAAW,EAAE;YAC1C,MAAM,QAAQ,GAAG,GAAG,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;YAClF,MAAM,QAAQ,GAAG,GAAG,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;YAClF,OAAO,QAAQ,IAAI,QAAQ,CAAC;QAC7B,CAAC,CAAC;QAEF,qEAAqE;QACrE,MAAM,MAAM,GACX,SAAS,CAAC,GAAG,KAAK,GAAG;YACpB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC5C,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,MAAM;YACT,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC3C,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,MAAM,CAAC;QAEZ,OAAO;YACN,SAAS;YACT,MAAM;YACN,QAAQ;YACR,QAAQ,EAAE;gBACT,IAAI,EAAE,cAAc;gBACpB,WAAW,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE;aACzB;SACD,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,eAAe,CAAC,KAAa,EAAE,QAAgB,EAAE,SAAiB;QACzE,QAAQ,QAAQ,EAAE,CAAC;YAClB,KAAK,IAAI;gBACR,OAAO,KAAK,GAAG,SAAS,CAAC;YAC1B,KAAK,KAAK;gBACT,OAAO,KAAK,IAAI,SAAS,CAAC;YAC3B,KAAK,IAAI;gBACR,OAAO,KAAK,GAAG,SAAS,CAAC;YAC1B,KAAK,KAAK;gBACT,OAAO,KAAK,IAAI,SAAS,CAAC;YAC3B,KAAK,IAAI;gBACR,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC,6BAA6B;YAC3E;gBACC,OAAO,KAAK,CAAC;QACf,CAAC;IACF,CAAC;IAED;;;;;OAKG;IACK,cAAc,CAAC,OAAiC;QACvD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;QACjE,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM,CAAC;QAC/E,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAEhD,2BAA2B;QAC3B,MAAM,aAAa,GAA2B,EAAE,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoC,CAAC;QAE/D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACnC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;gBAC/C,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtB,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YACnC,CAAC;QACF,CAAC;QAED,KAAK,MAAM,CAAC,GAAG,EAAE,aAAa,CAAC,IAAI,UAAU,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,MAAM,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC;QACvD,CAAC;QAED,OAAO;YACN,KAAK;YACL,MAAM;YACN,MAAM;YACN,YAAY;YACZ,QAAQ;YACR,aAAa;SACb,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,MAA8C;QACvD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;QAEhC,OAAO;YACN,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,UAAU,EAAE;gBACX,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,aAAa,CAAC;aACpD;SACD,CAAC;IACH,CAAC"}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluator Registry
|
|
3
|
+
*
|
|
4
|
+
* Central registry for all evaluator types. Enables the plugin
|
|
5
|
+
* architecture by allowing custom evaluators to be registered
|
|
6
|
+
* alongside built-in evaluators.
|
|
7
|
+
*/
|
|
8
|
+
import type { IEvaluator, Evaluator, EvaluationType, EvaluatorConfig } from "../types/evaluator.js";
|
|
9
|
+
/**
|
|
10
|
+
* Registry of all available evaluators.
|
|
11
|
+
*
|
|
12
|
+
* Evaluators are stored by their type identifier. The registry
|
|
13
|
+
* provides methods to register new evaluators, retrieve evaluators
|
|
14
|
+
* by type, and list all registered types.
|
|
15
|
+
*/
|
|
16
|
+
export declare const EvaluatorRegistry: {
|
|
17
|
+
/**
|
|
18
|
+
* Register an evaluator.
|
|
19
|
+
*
|
|
20
|
+
* If an evaluator with the same type already exists, it will
|
|
21
|
+
* be replaced with the new evaluator.
|
|
22
|
+
*
|
|
23
|
+
* @param evaluator - Evaluator to register
|
|
24
|
+
*/
|
|
25
|
+
readonly register: <TConfig extends EvaluatorConfig, TInput, TOutput>(evaluator: Evaluator<TConfig, TInput, TOutput>) => void;
|
|
26
|
+
/**
|
|
27
|
+
* Get an evaluator by type (returns base IEvaluator).
|
|
28
|
+
*
|
|
29
|
+
* For type-safe retrieval, use the getAs() method instead.
|
|
30
|
+
*
|
|
31
|
+
* @param type - Evaluation type identifier
|
|
32
|
+
* @returns Evaluator instance or undefined if not found
|
|
33
|
+
*/
|
|
34
|
+
readonly get: (type: EvaluationType) => IEvaluator | undefined;
|
|
35
|
+
/**
|
|
36
|
+
* Get an evaluator by type with runtime type checking.
|
|
37
|
+
*
|
|
38
|
+
* Uses instanceof to verify the evaluator is of the expected class.
|
|
39
|
+
* Returns undefined if the evaluator doesn't exist or isn't of the expected type.
|
|
40
|
+
*
|
|
41
|
+
* @param type - Evaluation type identifier
|
|
42
|
+
* @param clazz - Constructor of the expected evaluator class
|
|
43
|
+
* @returns Typed evaluator instance or undefined
|
|
44
|
+
*
|
|
45
|
+
* @example
|
|
46
|
+
* ```ts
|
|
47
|
+
* const claimsEvaluator = EvaluatorRegistry.getAs("claims", ClaimsEvaluator);
|
|
48
|
+
* if (claimsEvaluator) {
|
|
49
|
+
* // Full type safety - claimsEvaluator is ClaimsEvaluator
|
|
50
|
+
* const validation = claimsEvaluator.validateConfig(config);
|
|
51
|
+
* }
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
readonly getAs: <T extends IEvaluator>(type: EvaluationType, clazz: new () => T) => T | undefined;
|
|
55
|
+
/**
|
|
56
|
+
* Get an evaluator by type, throwing if not found.
|
|
57
|
+
*
|
|
58
|
+
* @param type - Evaluation type identifier
|
|
59
|
+
* @param clazz - Constructor of the expected evaluator class
|
|
60
|
+
* @returns Evaluator instance
|
|
61
|
+
* @throws Error if evaluator type not found or wrong type
|
|
62
|
+
*/
|
|
63
|
+
readonly getOrThrow: <T extends IEvaluator>(type: EvaluationType, clazz: new () => T) => T;
|
|
64
|
+
/**
|
|
65
|
+
* List all registered evaluator types.
|
|
66
|
+
*
|
|
67
|
+
* @returns Array of registered evaluation type identifiers
|
|
68
|
+
*/
|
|
69
|
+
readonly types: () => EvaluationType[];
|
|
70
|
+
/**
|
|
71
|
+
* Check if an evaluator type is registered.
|
|
72
|
+
*
|
|
73
|
+
* @param type - Evaluation type identifier
|
|
74
|
+
* @returns True if evaluator is registered
|
|
75
|
+
*/
|
|
76
|
+
readonly has: (type: EvaluationType) => boolean;
|
|
77
|
+
/**
|
|
78
|
+
* Unregister an evaluator by type.
|
|
79
|
+
*
|
|
80
|
+
* This is primarily useful for testing.
|
|
81
|
+
*
|
|
82
|
+
* @param type - Evaluation type identifier
|
|
83
|
+
* @returns True if evaluator was unregistered
|
|
84
|
+
*/
|
|
85
|
+
readonly unregister: (type: EvaluationType) => boolean;
|
|
86
|
+
/**
|
|
87
|
+
* Clear all registered evaluators.
|
|
88
|
+
*
|
|
89
|
+
* This is primarily useful for testing.
|
|
90
|
+
*/
|
|
91
|
+
readonly clear: () => void;
|
|
92
|
+
/**
|
|
93
|
+
* Get the number of registered evaluators.
|
|
94
|
+
*
|
|
95
|
+
* @returns Count of registered evaluators
|
|
96
|
+
*/
|
|
97
|
+
readonly size: number;
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Auto-register built-in evaluators on module load.
|
|
101
|
+
*
|
|
102
|
+
* This ensures that the standard evaluators (claims, robustness, metrics, exploratory)
|
|
103
|
+
* are always available without requiring explicit registration.
|
|
104
|
+
*/
|
|
105
|
+
export declare function registerBuiltInEvaluators(): void;
|
|
106
|
+
//# sourceMappingURL=registry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/evaluators/registry.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,SAAS,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAKpG;;;;;;GAMG;AACH,eAAO,MAAM,iBAAiB;IAC7B;;;;;;;OAOG;wBACM,OAAO,SAAS,eAAe,EAAE,MAAM,EAAE,OAAO,aAC7C,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,KAC5C,IAAI;IAQP;;;;;;;OAOG;yBACO,cAAc,KAAG,UAAU,GAAG,SAAS;IAIjD;;;;;;;;;;;;;;;;;;OAkBG;qBACG,CAAC,SAAS,UAAU,QAAQ,cAAc,SAAS,UAAU,CAAC,KAAG,CAAC,GAAG,SAAS;IAKpF;;;;;;;OAOG;0BACQ,CAAC,SAAS,UAAU,QAAQ,cAAc,SAAS,UAAU,CAAC,KAAG,CAAC;IAQ7E;;;;OAIG;0BACM,cAAc,EAAE;IAIzB;;;;;OAKG;yBACO,cAAc,KAAG,OAAO;IAIlC;;;;;;;OAOG;gCACc,cAAc,KAAG,OAAO;IAIzC;;;;OAIG;0BACM,IAAI;IAIb;;;;OAIG;mBACS,MAAM;CAGT,CAAC;AAWX;;;;;GAKG;AACH,wBAAgB,yBAAyB,IAAI,IAAI,CAKhD"}
|