ppef 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -125
- package/bin/ppef.mjs +20 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.d.ts +8 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.d.ts.map +1 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.js +308 -0
- package/dist/__tests__/cli/evaluate-command.integration.test.js.map +1 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.js +405 -0
- package/dist/__tests__/evaluators/claims-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.js +424 -0
- package/dist/__tests__/evaluators/metrics-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/registry.unit.test.d.ts +7 -0
- package/dist/__tests__/evaluators/registry.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/registry.unit.test.js +173 -0
- package/dist/__tests__/evaluators/registry.unit.test.js.map +1 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.d.ts +8 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.d.ts.map +1 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.js +260 -0
- package/dist/__tests__/evaluators/robustness-evaluator.unit.test.js.map +1 -0
- package/dist/__tests__/framework-pipeline.integration.test.js +49 -20
- package/dist/__tests__/framework-pipeline.integration.test.js.map +1 -1
- package/dist/__tests__/index-exports.unit.test.d.ts +8 -0
- package/dist/__tests__/index-exports.unit.test.d.ts.map +1 -0
- package/dist/__tests__/index-exports.unit.test.js +124 -0
- package/dist/__tests__/index-exports.unit.test.js.map +1 -0
- package/dist/__tests__/registry-executor.integration.test.js +12 -9
- package/dist/__tests__/registry-executor.integration.test.js.map +1 -1
- package/dist/aggregation/__tests__/aggregators.unit.test.d.ts +7 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.d.ts.map +1 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.js +350 -0
- package/dist/aggregation/__tests__/aggregators.unit.test.js.map +1 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.d.ts +7 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.d.ts.map +1 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.js +213 -0
- package/dist/aggregation/__tests__/pipeline.unit.test.js.map +1 -0
- package/dist/aggregation/aggregators.d.ts +9 -0
- package/dist/aggregation/aggregators.d.ts.map +1 -1
- package/dist/aggregation/aggregators.js +1 -1
- package/dist/aggregation/aggregators.js.map +1 -1
- package/dist/aggregation/index.d.ts +1 -1
- package/dist/aggregation/index.d.ts.map +1 -1
- package/dist/aggregation/index.js +1 -1
- package/dist/aggregation/index.js.map +1 -1
- package/dist/aggregation/pipeline.d.ts.map +1 -1
- package/dist/aggregation/pipeline.js +40 -3
- package/dist/aggregation/pipeline.js.map +1 -1
- package/dist/claims/index.d.ts +6 -3
- package/dist/claims/index.d.ts.map +1 -1
- package/dist/claims/index.js +6 -3
- package/dist/claims/index.js.map +1 -1
- package/dist/cli/__tests__/aggregate.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.js +399 -0
- package/dist/cli/__tests__/aggregate.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/binary-sut.integration.test.d.ts +8 -0
- package/dist/cli/__tests__/binary-sut.integration.test.d.ts.map +1 -0
- package/dist/cli/__tests__/binary-sut.integration.test.js +165 -0
- package/dist/cli/__tests__/binary-sut.integration.test.js.map +1 -0
- package/dist/cli/__tests__/commands.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/commands.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/commands.unit.test.js +217 -0
- package/dist/cli/__tests__/commands.unit.test.js.map +1 -0
- package/dist/cli/__tests__/config-loader.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/config-loader.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/config-loader.unit.test.js +611 -0
- package/dist/cli/__tests__/config-loader.unit.test.js.map +1 -0
- package/dist/cli/__tests__/index.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/index.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/index.unit.test.js +65 -0
- package/dist/cli/__tests__/index.unit.test.js.map +1 -0
- package/dist/cli/__tests__/logger.unit.test.d.ts +11 -0
- package/dist/cli/__tests__/logger.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/logger.unit.test.js +180 -0
- package/dist/cli/__tests__/logger.unit.test.js.map +1 -0
- package/dist/cli/__tests__/module-loader.unit.test.d.ts +11 -0
- package/dist/cli/__tests__/module-loader.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/module-loader.unit.test.js +262 -0
- package/dist/cli/__tests__/module-loader.unit.test.js.map +1 -0
- package/dist/cli/__tests__/output-writer.unit.test.d.ts +10 -0
- package/dist/cli/__tests__/output-writer.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/output-writer.unit.test.js +216 -0
- package/dist/cli/__tests__/output-writer.unit.test.js.map +1 -0
- package/dist/cli/__tests__/plan.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/plan.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/plan.command.unit.test.js +289 -0
- package/dist/cli/__tests__/plan.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/run.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/run.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/run.command.unit.test.js +422 -0
- package/dist/cli/__tests__/run.command.unit.test.js.map +1 -0
- package/dist/cli/__tests__/validate.command.unit.test.d.ts +7 -0
- package/dist/cli/__tests__/validate.command.unit.test.d.ts.map +1 -0
- package/dist/cli/__tests__/validate.command.unit.test.js +226 -0
- package/dist/cli/__tests__/validate.command.unit.test.js.map +1 -0
- package/dist/cli/command-deps.d.ts +137 -0
- package/dist/cli/command-deps.d.ts.map +1 -0
- package/dist/cli/command-deps.js +7 -0
- package/dist/cli/command-deps.js.map +1 -0
- package/dist/cli/commands/aggregate.d.ts +35 -0
- package/dist/cli/commands/aggregate.d.ts.map +1 -0
- package/dist/cli/commands/aggregate.js +124 -0
- package/dist/cli/commands/aggregate.js.map +1 -0
- package/dist/cli/commands/evaluate.d.ts +41 -0
- package/dist/cli/commands/evaluate.d.ts.map +1 -0
- package/dist/cli/commands/evaluate.js +287 -0
- package/dist/cli/commands/evaluate.js.map +1 -0
- package/dist/cli/commands/plan.d.ts +36 -0
- package/dist/cli/commands/plan.d.ts.map +1 -0
- package/dist/cli/commands/plan.js +109 -0
- package/dist/cli/commands/plan.js.map +1 -0
- package/dist/cli/commands/run.d.ts +33 -0
- package/dist/cli/commands/run.d.ts.map +1 -0
- package/dist/cli/commands/run.js +277 -0
- package/dist/cli/commands/run.js.map +1 -0
- package/dist/cli/commands/validate.d.ts +27 -0
- package/dist/cli/commands/validate.d.ts.map +1 -0
- package/dist/cli/commands/validate.js +88 -0
- package/dist/cli/commands/validate.js.map +1 -0
- package/dist/cli/config-loader.d.ts +30 -0
- package/dist/cli/config-loader.d.ts.map +1 -0
- package/dist/cli/config-loader.js +181 -0
- package/dist/cli/config-loader.js.map +1 -0
- package/dist/cli/index.d.ts +27 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +60 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/logger.d.ts +75 -0
- package/dist/cli/logger.d.ts.map +1 -0
- package/dist/cli/logger.js +131 -0
- package/dist/cli/logger.js.map +1 -0
- package/dist/cli/module-loader.d.ts +68 -0
- package/dist/cli/module-loader.d.ts.map +1 -0
- package/dist/cli/module-loader.js +134 -0
- package/dist/cli/module-loader.js.map +1 -0
- package/dist/cli/output-writer.d.ts +51 -0
- package/dist/cli/output-writer.d.ts.map +1 -0
- package/dist/cli/output-writer.js +65 -0
- package/dist/cli/output-writer.js.map +1 -0
- package/dist/cli/types.d.ts +193 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +7 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/collector/__tests__/result-collector.unit.test.d.ts +7 -0
- package/dist/collector/__tests__/result-collector.unit.test.d.ts.map +1 -0
- package/dist/collector/__tests__/result-collector.unit.test.js +1021 -0
- package/dist/collector/__tests__/result-collector.unit.test.js.map +1 -0
- package/dist/collector/__tests__/schema.unit.test.d.ts +7 -0
- package/dist/collector/__tests__/schema.unit.test.d.ts.map +1 -0
- package/dist/collector/__tests__/schema.unit.test.js +360 -0
- package/dist/collector/__tests__/schema.unit.test.js.map +1 -0
- package/dist/evaluators/claims-evaluator.d.ts +87 -0
- package/dist/evaluators/claims-evaluator.d.ts.map +1 -0
- package/dist/evaluators/claims-evaluator.js +289 -0
- package/dist/evaluators/claims-evaluator.js.map +1 -0
- package/dist/evaluators/exploratory-evaluator.d.ts +136 -0
- package/dist/evaluators/exploratory-evaluator.d.ts.map +1 -0
- package/dist/evaluators/exploratory-evaluator.js +545 -0
- package/dist/evaluators/exploratory-evaluator.js.map +1 -0
- package/dist/evaluators/index.d.ts +13 -0
- package/dist/evaluators/index.d.ts.map +1 -0
- package/dist/evaluators/index.js +14 -0
- package/dist/evaluators/index.js.map +1 -0
- package/dist/evaluators/metrics-evaluator.d.ts +114 -0
- package/dist/evaluators/metrics-evaluator.d.ts.map +1 -0
- package/dist/evaluators/metrics-evaluator.js +433 -0
- package/dist/evaluators/metrics-evaluator.js.map +1 -0
- package/dist/evaluators/registry.d.ts +106 -0
- package/dist/evaluators/registry.d.ts.map +1 -0
- package/dist/evaluators/registry.js +148 -0
- package/dist/evaluators/registry.js.map +1 -0
- package/dist/evaluators/robustness-evaluator.d.ts +57 -0
- package/dist/evaluators/robustness-evaluator.d.ts.map +1 -0
- package/dist/evaluators/robustness-evaluator.js +186 -0
- package/dist/evaluators/robustness-evaluator.js.map +1 -0
- package/dist/executor/__tests__/binary-sut.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/binary-sut.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/binary-sut.unit.test.js +313 -0
- package/dist/executor/__tests__/binary-sut.unit.test.js.map +1 -0
- package/dist/executor/__tests__/checkpoint-manager.unit.test.js +83 -1
- package/dist/executor/__tests__/checkpoint-manager.unit.test.js.map +1 -1
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.d.ts +3 -6
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.d.ts.map +1 -1
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.js +428 -159
- package/dist/executor/__tests__/checkpoint-merge-bug.diagnostic.test.js.map +1 -1
- package/dist/executor/__tests__/checkpoint-storage.unit.test.js +148 -1
- package/dist/executor/__tests__/checkpoint-storage.unit.test.js.map +1 -1
- package/dist/executor/__tests__/executor.unit.test.js +123 -8
- package/dist/executor/__tests__/executor.unit.test.js.map +1 -1
- package/dist/executor/__tests__/memory-monitor.unit.test.d.ts +7 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.js +285 -0
- package/dist/executor/__tests__/memory-monitor.unit.test.js.map +1 -0
- package/dist/executor/__tests__/parallel-executor.unit.test.d.ts +2 -1
- package/dist/executor/__tests__/parallel-executor.unit.test.d.ts.map +1 -1
- package/dist/executor/__tests__/parallel-executor.unit.test.js +426 -156
- package/dist/executor/__tests__/parallel-executor.unit.test.js.map +1 -1
- package/dist/executor/__tests__/resource-calculator.unit.test.d.ts +10 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.js +104 -0
- package/dist/executor/__tests__/resource-calculator.unit.test.js.map +1 -0
- package/dist/executor/__tests__/run-id.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/run-id.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/run-id.unit.test.js +156 -0
- package/dist/executor/__tests__/run-id.unit.test.js.map +1 -0
- package/dist/executor/__tests__/worker-entry.integration.test.d.ts +24 -0
- package/dist/executor/__tests__/worker-entry.integration.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-entry.integration.test.js +82 -0
- package/dist/executor/__tests__/worker-entry.integration.test.js.map +1 -0
- package/dist/executor/__tests__/worker-entry.unit.test.d.ts +7 -0
- package/dist/executor/__tests__/worker-entry.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-entry.unit.test.js +364 -0
- package/dist/executor/__tests__/worker-entry.unit.test.js.map +1 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.d.ts +8 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.d.ts.map +1 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.js +276 -0
- package/dist/executor/__tests__/worker-threads-executor.unit.test.js.map +1 -0
- package/dist/executor/binary-sut.d.ts +105 -0
- package/dist/executor/binary-sut.d.ts.map +1 -0
- package/dist/executor/binary-sut.js +174 -0
- package/dist/executor/binary-sut.js.map +1 -0
- package/dist/executor/checkpoint-storage.d.ts.map +1 -1
- package/dist/executor/checkpoint-storage.js +6 -4
- package/dist/executor/checkpoint-storage.js.map +1 -1
- package/dist/executor/executor.d.ts +28 -0
- package/dist/executor/executor.d.ts.map +1 -1
- package/dist/executor/executor.js +85 -24
- package/dist/executor/executor.js.map +1 -1
- package/dist/executor/index.d.ts +4 -0
- package/dist/executor/index.d.ts.map +1 -1
- package/dist/executor/index.js +4 -0
- package/dist/executor/index.js.map +1 -1
- package/dist/executor/parallel-executor.d.ts +186 -0
- package/dist/executor/parallel-executor.d.ts.map +1 -1
- package/dist/executor/parallel-executor.js +218 -83
- package/dist/executor/parallel-executor.js.map +1 -1
- package/dist/executor/resource-calculator.d.ts +49 -0
- package/dist/executor/resource-calculator.d.ts.map +1 -0
- package/dist/executor/resource-calculator.js +129 -0
- package/dist/executor/resource-calculator.js.map +1 -0
- package/dist/executor/run-id.d.ts.map +1 -1
- package/dist/executor/run-id.js +8 -1
- package/dist/executor/run-id.js.map +1 -1
- package/dist/executor/worker-entry.d.ts +2 -0
- package/dist/executor/worker-entry.d.ts.map +1 -1
- package/dist/executor/worker-entry.js +46 -55
- package/dist/executor/worker-entry.js.map +1 -1
- package/dist/executor/worker-executor.d.ts +257 -0
- package/dist/executor/worker-executor.d.ts.map +1 -0
- package/dist/executor/worker-executor.js +308 -0
- package/dist/executor/worker-executor.js.map +1 -0
- package/dist/executor/worker-threads-executor.d.ts +245 -0
- package/dist/executor/worker-threads-executor.d.ts.map +1 -0
- package/dist/executor/worker-threads-executor.js +332 -0
- package/dist/executor/worker-threads-executor.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/renderers/latex-renderer.d.ts +60 -0
- package/dist/renderers/latex-renderer.d.ts.map +1 -1
- package/dist/renderers/latex-renderer.js +299 -0
- package/dist/renderers/latex-renderer.js.map +1 -1
- package/dist/renderers/types.d.ts +9 -0
- package/dist/renderers/types.d.ts.map +1 -1
- package/dist/renderers/types.js.map +1 -1
- package/dist/robustness/__tests__/perturbations.unit.test.d.ts +11 -0
- package/dist/robustness/__tests__/perturbations.unit.test.d.ts.map +1 -0
- package/dist/robustness/__tests__/perturbations.unit.test.js +284 -0
- package/dist/robustness/__tests__/perturbations.unit.test.js.map +1 -0
- package/dist/robustness/index.d.ts +5 -2
- package/dist/robustness/index.d.ts.map +1 -1
- package/dist/robustness/index.js +4 -2
- package/dist/robustness/index.js.map +1 -1
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.d.ts +7 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.d.ts.map +1 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.js +185 -0
- package/dist/statistical/__tests__/mann-whitney-u.unit.test.js.map +1 -0
- package/dist/types/evaluator.d.ts +449 -0
- package/dist/types/evaluator.d.ts.map +1 -0
- package/dist/types/evaluator.js +9 -0
- package/dist/types/evaluator.js.map +1 -0
- package/dist/types/result.d.ts +2 -0
- package/dist/types/result.d.ts.map +1 -1
- package/package.json +8 -1
- package/dist/claims/evaluator.d.ts +0 -33
- package/dist/claims/evaluator.d.ts.map +0 -1
- package/dist/claims/evaluator.js +0 -174
- package/dist/claims/evaluator.js.map +0 -1
- package/dist/robustness/analyzer.d.ts +0 -61
- package/dist/robustness/analyzer.d.ts.map +0 -1
- package/dist/robustness/analyzer.js +0 -191
- package/dist/robustness/analyzer.js.map +0 -1
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluate Command
|
|
3
|
+
*
|
|
4
|
+
* Evaluates results using the extensible evaluator system.
|
|
5
|
+
* Supports claims, robustness, metrics, and custom evaluators.
|
|
6
|
+
*/
|
|
7
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
8
|
+
import { EvaluatorRegistry, ClaimsEvaluator, RobustnessEvaluator, MetricsEvaluator, } from "../../evaluators/index.js";
|
|
9
|
+
import { LaTeXRenderer } from "../../renderers/latex-renderer.js";
|
|
10
|
+
/**
|
|
11
|
+
* Execute evaluate command with injected dependencies.
|
|
12
|
+
*
|
|
13
|
+
* This function is exported for testing.
|
|
14
|
+
*
|
|
15
|
+
* @param aggregatesFile - Path to aggregates JSON file
|
|
16
|
+
* @param options - Command options
|
|
17
|
+
* @param dependencies - Injected dependencies
|
|
18
|
+
*/
|
|
19
|
+
export async function executeEvaluate(aggregatesFile, options, dependencies) {
|
|
20
|
+
const { logger, fileSystem, processExit } = dependencies;
|
|
21
|
+
try {
|
|
22
|
+
logger.header("Evaluating Results");
|
|
23
|
+
// Load aggregates
|
|
24
|
+
logger.info(`Reading aggregates from: ${aggregatesFile}`);
|
|
25
|
+
const content = await fileSystem.readFile(aggregatesFile, "utf-8");
|
|
26
|
+
const data = JSON.parse(content);
|
|
27
|
+
// Determine if file has aggregates or raw results
|
|
28
|
+
const hasAggregates = data.aggregates && Array.isArray(data.aggregates);
|
|
29
|
+
const hasResults = data.results && Array.isArray(data.results);
|
|
30
|
+
let aggregates;
|
|
31
|
+
let rawResults;
|
|
32
|
+
if (hasAggregates) {
|
|
33
|
+
aggregates = data.aggregates;
|
|
34
|
+
logger.info(`Found ${aggregates.length} aggregated results`);
|
|
35
|
+
}
|
|
36
|
+
else if (hasResults) {
|
|
37
|
+
// Need to aggregate first
|
|
38
|
+
logger.info(`Found ${data.results?.length ?? 0} raw results - need to aggregate first`);
|
|
39
|
+
logger.error("Please run 'ppef aggregate' on the results file first, or use an aggregates file");
|
|
40
|
+
processExit(1);
|
|
41
|
+
return; // Type narrowing
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
throw new Error("Invalid file: must contain 'aggregates' or 'results' array");
|
|
45
|
+
}
|
|
46
|
+
// Load evaluator config
|
|
47
|
+
let evaluatorConfig;
|
|
48
|
+
if (options.config) {
|
|
49
|
+
logger.info(`Loading evaluator config from: ${options.config}`);
|
|
50
|
+
const configContent = await fileSystem.readFile(options.config, "utf-8");
|
|
51
|
+
evaluatorConfig = JSON.parse(configContent);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
logger.warn("No evaluator config provided - using default empty config");
|
|
55
|
+
evaluatorConfig = {};
|
|
56
|
+
}
|
|
57
|
+
// Get evaluator from registry using type-safe retrieval
|
|
58
|
+
logger.info(`Evaluator type: ${options.type}`);
|
|
59
|
+
// Type-safe evaluator retrieval and execution
|
|
60
|
+
let output;
|
|
61
|
+
let summary;
|
|
62
|
+
switch (options.type) {
|
|
63
|
+
case "claims": {
|
|
64
|
+
const evaluator = EvaluatorRegistry.getAs("claims", ClaimsEvaluator);
|
|
65
|
+
if (!evaluator) {
|
|
66
|
+
logger.error(`Evaluator not found for type: claims`);
|
|
67
|
+
processExit(1);
|
|
68
|
+
return; // Type narrowing
|
|
69
|
+
}
|
|
70
|
+
// Validate config
|
|
71
|
+
logger.subheader("Validating evaluator configuration...");
|
|
72
|
+
const validation = evaluator.validateConfig(evaluatorConfig);
|
|
73
|
+
if (!validation.valid) {
|
|
74
|
+
logger.error("Evaluator configuration validation failed:");
|
|
75
|
+
for (const error of validation.errors ?? []) {
|
|
76
|
+
logger.error(` - ${error}`);
|
|
77
|
+
}
|
|
78
|
+
processExit(1);
|
|
79
|
+
return; // Type narrowing
|
|
80
|
+
}
|
|
81
|
+
if (validation.warnings && validation.warnings.length > 0) {
|
|
82
|
+
logger.warn("Configuration warnings:");
|
|
83
|
+
for (const warning of validation.warnings) {
|
|
84
|
+
logger.warn(` - ${warning}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
logger.info("Configuration valid");
|
|
88
|
+
// Prepare evaluation context
|
|
89
|
+
const context = {
|
|
90
|
+
aggregates,
|
|
91
|
+
rawResults,
|
|
92
|
+
metadata: {
|
|
93
|
+
source: aggregatesFile,
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
// Run evaluation
|
|
97
|
+
logger.subheader("Running evaluation...");
|
|
98
|
+
const evalOutput = evaluator.evaluate(evaluatorConfig, context);
|
|
99
|
+
output = evalOutput;
|
|
100
|
+
summary = evaluator.summarize(evalOutput);
|
|
101
|
+
break;
|
|
102
|
+
}
|
|
103
|
+
case "robustness": {
|
|
104
|
+
const evaluator = EvaluatorRegistry.getAs("robustness", RobustnessEvaluator);
|
|
105
|
+
if (!evaluator) {
|
|
106
|
+
logger.error(`Evaluator not found for type: robustness`);
|
|
107
|
+
processExit(1);
|
|
108
|
+
return; // Type narrowing
|
|
109
|
+
}
|
|
110
|
+
// Validate config
|
|
111
|
+
logger.subheader("Validating evaluator configuration...");
|
|
112
|
+
const validation = evaluator.validateConfig(evaluatorConfig);
|
|
113
|
+
if (!validation.valid) {
|
|
114
|
+
logger.error("Evaluator configuration validation failed:");
|
|
115
|
+
for (const error of validation.errors ?? []) {
|
|
116
|
+
logger.error(` - ${error}`);
|
|
117
|
+
}
|
|
118
|
+
processExit(1);
|
|
119
|
+
return; // Type narrowing
|
|
120
|
+
}
|
|
121
|
+
if (validation.warnings && validation.warnings.length > 0) {
|
|
122
|
+
logger.warn("Configuration warnings:");
|
|
123
|
+
for (const warning of validation.warnings) {
|
|
124
|
+
logger.warn(` - ${warning}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
logger.info("Configuration valid");
|
|
128
|
+
// Robustness evaluator requires raw results, not aggregates
|
|
129
|
+
if (!rawResults || rawResults.length === 0) {
|
|
130
|
+
logger.error("Robustness evaluation requires raw results, but none found");
|
|
131
|
+
processExit(1);
|
|
132
|
+
return; // Type narrowing
|
|
133
|
+
}
|
|
134
|
+
// Run evaluation
|
|
135
|
+
logger.subheader("Running evaluation...");
|
|
136
|
+
const evalOutput = evaluator.evaluate(evaluatorConfig, rawResults);
|
|
137
|
+
output = evalOutput;
|
|
138
|
+
summary = evaluator.summarize(evalOutput);
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
case "metrics": {
|
|
142
|
+
const evaluator = EvaluatorRegistry.getAs("metrics", MetricsEvaluator);
|
|
143
|
+
if (!evaluator) {
|
|
144
|
+
logger.error(`Evaluator not found for type: metrics`);
|
|
145
|
+
processExit(1);
|
|
146
|
+
return; // Type narrowing
|
|
147
|
+
}
|
|
148
|
+
// Validate config
|
|
149
|
+
logger.subheader("Validating evaluator configuration...");
|
|
150
|
+
const validation = evaluator.validateConfig(evaluatorConfig);
|
|
151
|
+
if (!validation.valid) {
|
|
152
|
+
logger.error("Evaluator configuration validation failed:");
|
|
153
|
+
for (const error of validation.errors ?? []) {
|
|
154
|
+
logger.error(` - ${error}`);
|
|
155
|
+
}
|
|
156
|
+
processExit(1);
|
|
157
|
+
return; // Type narrowing
|
|
158
|
+
}
|
|
159
|
+
if (validation.warnings && validation.warnings.length > 0) {
|
|
160
|
+
logger.warn("Configuration warnings:");
|
|
161
|
+
for (const warning of validation.warnings) {
|
|
162
|
+
logger.warn(` - ${warning}`);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
logger.info("Configuration valid");
|
|
166
|
+
// Prepare evaluation context
|
|
167
|
+
const context = {
|
|
168
|
+
aggregates,
|
|
169
|
+
rawResults,
|
|
170
|
+
metadata: {
|
|
171
|
+
source: aggregatesFile,
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
// Run evaluation
|
|
175
|
+
logger.subheader("Running evaluation...");
|
|
176
|
+
const evalOutput = evaluator.evaluate(evaluatorConfig, context);
|
|
177
|
+
output = evalOutput;
|
|
178
|
+
summary = evaluator.summarize(evalOutput);
|
|
179
|
+
break;
|
|
180
|
+
}
|
|
181
|
+
default:
|
|
182
|
+
logger.error(`Unsupported evaluator type: ${options.type}`);
|
|
183
|
+
processExit(1);
|
|
184
|
+
return; // Type narrowing
|
|
185
|
+
}
|
|
186
|
+
logger.info(`Evaluation complete: ${output.type}`);
|
|
187
|
+
// Display summary
|
|
188
|
+
if (options.verbose) {
|
|
189
|
+
logger.subheader("Evaluation Summary");
|
|
190
|
+
logger.info(`Total items: ${summary.total}`);
|
|
191
|
+
if (summary.passed !== undefined) {
|
|
192
|
+
logger.info(`Passed: ${summary.passed}`);
|
|
193
|
+
}
|
|
194
|
+
if (summary.failed !== undefined) {
|
|
195
|
+
logger.info(`Failed: ${summary.failed}`);
|
|
196
|
+
}
|
|
197
|
+
if (summary.inconclusive !== undefined) {
|
|
198
|
+
logger.info(`Inconclusive: ${summary.inconclusive}`);
|
|
199
|
+
}
|
|
200
|
+
if (summary.passRate !== undefined) {
|
|
201
|
+
logger.info(`Pass rate: ${(summary.passRate * 100).toFixed(1)}%`);
|
|
202
|
+
}
|
|
203
|
+
if (summary.additional) {
|
|
204
|
+
logger.info(`Additional:`);
|
|
205
|
+
for (const [key, value] of Object.entries(summary.additional)) {
|
|
206
|
+
logger.info(` ${key}: ${value}`);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
// Determine output path
|
|
211
|
+
const format = options.format ?? "json-pretty";
|
|
212
|
+
const defaultOutputName = aggregatesFile.replace(/-aggregates/, `-evaluation-${options.type}`);
|
|
213
|
+
const outputPath = options.output ?? defaultOutputName;
|
|
214
|
+
// Format and write output
|
|
215
|
+
logger.subheader("Writing output...");
|
|
216
|
+
let outputContent;
|
|
217
|
+
let outputFilename;
|
|
218
|
+
if (format === "latex") {
|
|
219
|
+
// Use LaTeX renderer
|
|
220
|
+
const renderer = new LaTeXRenderer();
|
|
221
|
+
const rendered = renderer.renderEvaluation(output);
|
|
222
|
+
outputContent = rendered.content;
|
|
223
|
+
outputFilename = rendered.filename;
|
|
224
|
+
if (options.output) {
|
|
225
|
+
outputFilename = options.output;
|
|
226
|
+
}
|
|
227
|
+
await fileSystem.writeFile(outputFilename, outputContent, "utf-8");
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
// JSON output
|
|
231
|
+
outputContent =
|
|
232
|
+
format === "json-pretty" ? JSON.stringify(output, null, 2) : JSON.stringify(output);
|
|
233
|
+
outputFilename = outputPath.endsWith(".json") ? outputPath : `${outputPath}.json`;
|
|
234
|
+
await fileSystem.writeFile(outputFilename, outputContent, "utf-8");
|
|
235
|
+
}
|
|
236
|
+
logger.info(`Output written to: ${outputFilename}`);
|
|
237
|
+
logger.info("");
|
|
238
|
+
logger.info("Evaluation completed successfully!");
|
|
239
|
+
}
|
|
240
|
+
catch (error) {
|
|
241
|
+
if (error instanceof Error) {
|
|
242
|
+
logger.error(error.message);
|
|
243
|
+
if (options.verbose && error.stack) {
|
|
244
|
+
logger.debug(error.stack);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
logger.error(String(error));
|
|
249
|
+
}
|
|
250
|
+
processExit(1);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Register the evaluate command.
|
|
255
|
+
*
|
|
256
|
+
* @param program - Commander program instance
|
|
257
|
+
*/
|
|
258
|
+
export function registerEvaluateCommand(program) {
|
|
259
|
+
program
|
|
260
|
+
.command("evaluate")
|
|
261
|
+
.description("Evaluate results using the extensible evaluator system")
|
|
262
|
+
.argument("<aggregates-file>", "Path to aggregates JSON file")
|
|
263
|
+
.requiredOption("-t, --type <type>", "Evaluation type (claims|robustness|metrics|custom)")
|
|
264
|
+
.option("-c, --config <path>", "Evaluator configuration JSON file")
|
|
265
|
+
.option("-o, --output <path>", "Output file path")
|
|
266
|
+
.option("-f, --format <format>", "Output format (json|json-pretty|latex|markdown)", "json-pretty")
|
|
267
|
+
.option("-v, --verbose", "Verbose output with summary statistics")
|
|
268
|
+
.action(async (aggregatesFile, options) => {
|
|
269
|
+
const { createLogger } = await import("../logger.js");
|
|
270
|
+
const logger = createLogger();
|
|
271
|
+
await executeEvaluate(aggregatesFile, {
|
|
272
|
+
type: options.type,
|
|
273
|
+
config: options.config,
|
|
274
|
+
output: options.output,
|
|
275
|
+
format: options.format,
|
|
276
|
+
verbose: options.verbose,
|
|
277
|
+
}, {
|
|
278
|
+
logger,
|
|
279
|
+
fileSystem: {
|
|
280
|
+
readFile: (path, encoding) => readFile(path, encoding),
|
|
281
|
+
writeFile: (path, data, encoding) => writeFile(path, data, encoding),
|
|
282
|
+
},
|
|
283
|
+
processExit: (code) => process.exit(code),
|
|
284
|
+
});
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
//# sourceMappingURL=evaluate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../../src/cli/commands/evaluate.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAIvD,OAAO,EACN,iBAAiB,EACjB,eAAe,EACf,mBAAmB,EACnB,gBAAgB,GAChB,MAAM,2BAA2B,CAAC;AAanC,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAC;AAOlE;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,cAAsB,EACtB,OAMC,EACD,YAIC;IAED,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;IAEzD,IAAI,CAAC;QACJ,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAEpC,kBAAkB;QAClB,MAAM,CAAC,IAAI,CAAC,4BAA4B,cAAc,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAoD,CAAC;QAEpF,kDAAkD;QAClD,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE/D,IAAI,UAA8B,CAAC;QACnC,IAAI,UAA0C,CAAC;QAE/C,IAAI,aAAa,EAAE,CAAC;YACnB,UAAU,GAAG,IAAI,CAAC,UAAgC,CAAC;YACnD,MAAM,CAAC,IAAI,CAAC,SAAS,UAAU,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,UAAU,EAAE,CAAC;YACvB,0BAA0B;YAC1B,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxF,MAAM,CAAC,KAAK,CACX,kFAAkF,CAClF,CAAC;YACF,WAAW,CAAC,CAAC,CAAC,CAAC;YACf,OAAO,CAAC,iBAAiB;QAC1B,CAAC;aAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;QAC/E,CAAC;QAED,wBAAwB;QACxB,IAAI,eAAgC,CAAC;QACrC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,kCAAkC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAChE,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACzE,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAoB,CAAC;QAChE,CAAC;aAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC;YACzE,eAAe,GAAG,EAAE,CAAC;QACtB,CAAC;QAED,wDAAwD;QACxD,MAAM,CAAC,IAAI,CAAC,mBAAmB,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAE/C,8CAA8C;QAC9C,IAAI,MAAiC,CAAC;QACtC,IAAI,OAOH,CAAC;QAEF,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;YACtB,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACf,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;gBACrE,IAAI,CAAC,SAAS,EAAE,CAAC;oBAChB,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;oBACrD,WAAW,CAAC,CAAC,CAAC,CAAC;oBACf,OAAO,CAAC,iBAAiB;gBAC1B,CAAC;gBAED,kBAAkB;gBAClB,MAAM,CAAC,SAAS,CAAC,uCAAuC,CAAC,CAAC;gBAC1D,MAAM,UAAU,GAAG,SAAS,CAAC,cAAc,CAAC,eAAwC,CAAC,CAAC;gBACtF,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACvB,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;oBAC3D,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBAC7C,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;oBAC9B,CAAC;oBACD,WAAW,CAAC,CAAC,CAAC,CAAC;oBACf,OAAO,CAAC,iBAAiB;gBAC1B,CAAC;gBACD,IAAI,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3D,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBACvC,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;wBAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;oBAC/B,CAAC;gBACF,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAEnC,6BAA6B;gBAC7B,MAAM,OAAO,GAAsB;oBAClC,UAAU;oBACV,UAAU;oBACV,QAAQ,EAAE;wBACT,MAAM,EAAE,cAAc;qBACtB;iBACD,CAAC;gBAEF,iBAAiB;gBACjB,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;gBAC1C,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,eAAwC,EAAE,OAAO,CAAC,CAAC;gBACzF,MAAM,GAAG,UAAU,CAAC;gBACpB,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACP,CAAC;YACD,KAAK,YAAY,CAAC,CAAC,CAAC;gBACnB,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;gBAC7E,IAAI,CAAC,SAAS,EAAE,CAAC;oBAChB,MAAM,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;oBACzD,WAAW,CAAC,CAAC,CAAC,CAAC;oBACf,OAAO,CAAC,iBAAiB;gBAC1B,CAAC;gBAED,kBAAkB;gBAClB,MAAM,CAAC,SAAS,CAAC,uCAAuC,CAAC,CAAC;gBAC1D,MAAM,UAAU,GAAG,SAAS,CAAC,cAAc,CAAC,eAA4C,CAAC,CAAC;gBAC1F,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACvB,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;oBAC3D,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBAC7C,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;oBAC9B,CAAC;oBACD,WAAW,CAAC,CAAC,CAAC,CAAC;oBACf,OAAO,CAAC,iBAAiB;gBAC1B,CAAC;gBACD,IAAI,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3D,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBACvC,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;wBAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;oBAC/B,CAAC;gBACF,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAEnC,4DAA4D;gBAC5D,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC5C,MAAM,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAC;oBAC3E,WAAW,CAAC,CAAC,CAAC,CAAC;oBACf,OAAO,CAAC,iBAAiB;gBAC1B,CAAC;gBAED,iBAAiB;gBACjB,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;gBAC1C,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CACpC,eAA4C,EAC5C,UAAU,CACV,CAAC;gBACF,MAAM,GAAG,UAAU,CAAC;gBACpB,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACP,CAAC;YACD,KAAK,SAAS,CAAC,CAAC,CAAC;gBAChB,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;gBACvE,IAAI,CAAC,SAAS,EAAE,CAAC;oBAChB,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;oBACtD,WAAW,CAAC,CAAC,CAAC,CAAC;oBACf,OAAO,CAAC,iBAAiB;gBAC1B,CAAC;gBAED,kBAAkB;gBAClB,MAAM,CAAC,SAAS,CAAC,uCAAuC,CAAC,CAAC;gBAC1D,MAAM,UAAU,GAAG,SAAS,CAAC,cAAc,CAAC,eAAyC,CAAC,CAAC;gBACvF,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;oBACvB,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;oBAC3D,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;wBAC7C,MAAM,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;oBAC9B,CAAC;oBACD,WAAW,CAAC,CAAC,CAAC,CAAC;oBACf,OAAO,CAAC,iBAAiB;gBAC1B,CAAC;gBACD,IAAI,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3D,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;oBACvC,KAAK,MAAM,OAAO,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;wBAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,OAAO,EAAE,CAAC,CAAC;oBAC/B,CAAC;gBACF,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;gBAEnC,6BAA6B;gBAC7B,MAAM,OAAO,GAAsB;oBAClC,UAAU;oBACV,UAAU;oBACV,QAAQ,EAAE;wBACT,MAAM,EAAE,cAAc;qBACtB;iBACD,CAAC;gBAEF,iBAAiB;gBACjB,MAAM,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;gBAC1C,MAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,eAAyC,EAAE,OAAO,CAAC,CAAC;gBAC1F,MAAM,GAAG,UAAU,CAAC;gBACpB,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACP,CAAC;YACD;gBACC,MAAM,CAAC,KAAK,CAAC,+BAA+B,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC5D,WAAW,CAAC,CAAC,CAAC,CAAC;gBACf,OAAO,CAAC,iBAAiB;QAC3B,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,wBAAwB,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAEnD,kBAAkB;QAClB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;YAC7C,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACnE,CAAC;YACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC/D,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC;gBACnC,CAAC;YACF,CAAC;QACF,CAAC;QAED,wBAAwB;QACxB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAC;QAC/C,MAAM,iBAAiB,GAAG,cAAc,CAAC,OAAO,CAAC,aAAa,EAAE,eAAe,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC/F,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,iBAAiB,CAAC;QAEvD,0BAA0B;QAC1B,MAAM,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QACtC,IAAI,aAAqB,CAAC;QAC1B,IAAI,cAAsB,CAAC;QAE3B,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;YACxB,qBAAqB;YACrB,MAAM,QAAQ,GAAG,IAAI,aAAa,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;YACnD,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC;YACjC,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAC;YACnC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpB,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;YACjC,CAAC;YACD,MAAM,UAAU,CAAC,SAAS,CAAC,cAAc,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;QACpE,CAAC;aAAM,CAAC;YACP,cAAc;YACd,aAAa;gBACZ,MAAM,KAAK,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YACrF,cAAc,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,OAAO,CAAC;YAClF,MAAM,UAAU,CAAC,SAAS,CAAC,cAAc,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,sBAAsB,cAAc,EAAE,CAAC,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC5B,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5B,IAAI,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBACpC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,CAAC;QACD,WAAW,CAAC,CAAC,CAAC,CAAC;IAChB,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAgB;IACvD,OAAO;SACL,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CAAC,wDAAwD,CAAC;SACrE,QAAQ,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SAC7D,cAAc,CAAC,mBAAmB,EAAE,oDAAoD,CAAC;SACzF,MAAM,CAAC,qBAAqB,EAAE,mCAAmC,CAAC;SAClE,MAAM,CAAC,qBAAqB,EAAE,kBAAkB,CAAC;SACjD,MAAM,CACN,uBAAuB,EACvB,iDAAiD,EACjD,aAAa,CACb;SACA,MAAM,CAAC,eAAe,EAAE,wCAAwC,CAAC;SACjE,MAAM,CAAC,KAAK,EAAE,cAAsB,EAAE,OAAgC,EAAE,EAAE;QAC1E,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAE9B,MAAM,eAAe,CACpB,cAAc,EACd;YACC,IAAI,EAAE,OAAO,CAAC,IAAsB;YACpC,MAAM,EAAE,OAAO,CAAC,MAA4B;YAC5C,MAAM,EAAE,OAAO,CAAC,MAA4B;YAC5C,MAAM,EAAE,OAAO,CAAC,MAAkC;YAClD,OAAO,EAAE,OAAO,CAAC,OAA8B;SAC/C,EACD;YACC,MAAM;YACN,UAAU,EAAE;gBACX,QAAQ,EAAE,CAAC,IAAY,EAAE,QAAgB,EAAE,EAAE,CAC5C,QAAQ,CAAC,IAAI,EAAE,QAA0B,CAAC;gBAC3C,SAAS,EAAE,CAAC,IAAY,EAAE,IAAY,EAAE,QAAgB,EAAE,EAAE,CAC3D,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,QAA0B,CAAC;aAClD;YACD,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;SACjD,CACD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plan Command
|
|
3
|
+
*
|
|
4
|
+
* Shows execution plan without running experiments.
|
|
5
|
+
*/
|
|
6
|
+
import type { Command } from "commander";
|
|
7
|
+
import type { ICaseDefinition, ICommandLogger, IConfigLoader, IModuleLoader, ISutFactory } from "../command-deps.js";
|
|
8
|
+
/**
|
|
9
|
+
* Execute plan command with injected dependencies.
|
|
10
|
+
*
|
|
11
|
+
* This function is exported for testing.
|
|
12
|
+
*
|
|
13
|
+
* @param configFile - Path to config file
|
|
14
|
+
* @param dependencies - Injected dependencies
|
|
15
|
+
*/
|
|
16
|
+
export declare function executePlan(configFile: string, dependencies: {
|
|
17
|
+
logger: ICommandLogger;
|
|
18
|
+
configLoader: IConfigLoader;
|
|
19
|
+
moduleLoader: IModuleLoader;
|
|
20
|
+
executor: {
|
|
21
|
+
plan: (suts: ISutFactory[], cases: ICaseDefinition[]) => {
|
|
22
|
+
sutId: string;
|
|
23
|
+
caseId: string;
|
|
24
|
+
repetition: number;
|
|
25
|
+
seed: number;
|
|
26
|
+
}[];
|
|
27
|
+
};
|
|
28
|
+
processExit: (code: number) => never;
|
|
29
|
+
}): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Register the plan command.
|
|
32
|
+
*
|
|
33
|
+
* @param program - Commander program instance
|
|
34
|
+
*/
|
|
35
|
+
export declare function registerPlanCommand(program: Command): void;
|
|
36
|
+
//# sourceMappingURL=plan.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plan.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/plan.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGzC,OAAO,KAAK,EACX,eAAe,EACf,cAAc,EACd,aAAa,EACb,aAAa,EACb,WAAW,EACX,MAAM,oBAAoB,CAAC;AAM5B;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAChC,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE;IACb,MAAM,EAAE,cAAc,CAAC;IACvB,YAAY,EAAE,aAAa,CAAC;IAC5B,YAAY,EAAE,aAAa,CAAC;IAC5B,QAAQ,EAAE;QACT,IAAI,EAAE,CACL,IAAI,EAAE,WAAW,EAAE,EACnB,KAAK,EAAE,eAAe,EAAE,KACpB;YACJ,KAAK,EAAE,MAAM,CAAC;YACd,MAAM,EAAE,MAAM,CAAC;YACf,UAAU,EAAE,MAAM,CAAC;YACnB,IAAI,EAAE,MAAM,CAAC;SACb,EAAE,CAAC;KACJ,CAAC;IACF,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;CACrC,GACC,OAAO,CAAC,IAAI,CAAC,CAqFf;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAwB1D"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plan Command
|
|
3
|
+
*
|
|
4
|
+
* Shows execution plan without running experiments.
|
|
5
|
+
*/
|
|
6
|
+
import { Executor } from "../../executor/index.js";
|
|
7
|
+
import { loadAndValidateConfig } from "../config-loader.js";
|
|
8
|
+
import { loadCaseDefinition, loadSutFactory } from "../module-loader.js";
|
|
9
|
+
// Note: createLogger is imported in the action handler to avoid circular dependency with logger.ts
|
|
10
|
+
/**
|
|
11
|
+
* Execute plan command with injected dependencies.
|
|
12
|
+
*
|
|
13
|
+
* This function is exported for testing.
|
|
14
|
+
*
|
|
15
|
+
* @param configFile - Path to config file
|
|
16
|
+
* @param dependencies - Injected dependencies
|
|
17
|
+
*/
|
|
18
|
+
export async function executePlan(configFile, dependencies) {
|
|
19
|
+
const { logger, configLoader, moduleLoader, executor, processExit } = dependencies;
|
|
20
|
+
try {
|
|
21
|
+
logger.header("Execution Plan");
|
|
22
|
+
const loaded = await configLoader.loadAndValidateConfig(configFile);
|
|
23
|
+
const { config, baseDir } = loaded;
|
|
24
|
+
// Load SUTs
|
|
25
|
+
logger.subheader("Loading SUTs...");
|
|
26
|
+
const sutDefinitions = await Promise.all(config.suts.map((sutConfig) => moduleLoader.loadSutFactory(sutConfig.module, sutConfig.exportName, baseDir, {
|
|
27
|
+
id: sutConfig.id,
|
|
28
|
+
name: sutConfig.registration.name,
|
|
29
|
+
version: sutConfig.registration.version,
|
|
30
|
+
role: sutConfig.registration.role,
|
|
31
|
+
config: sutConfig.config ?? {},
|
|
32
|
+
tags: sutConfig.registration.tags ?? [],
|
|
33
|
+
description: sutConfig.registration.description,
|
|
34
|
+
}, sutConfig.config)));
|
|
35
|
+
// Load cases
|
|
36
|
+
logger.subheader("Loading cases...");
|
|
37
|
+
const caseDefinitions = await Promise.all(config.cases.map((caseConfig) => moduleLoader.loadCaseDefinition(caseConfig.module, caseConfig.exportName, baseDir)));
|
|
38
|
+
// Plan runs
|
|
39
|
+
logger.subheader("Planning runs...");
|
|
40
|
+
const plannedRuns = executor.plan(sutDefinitions, caseDefinitions);
|
|
41
|
+
// Group and display plan
|
|
42
|
+
logger.info("");
|
|
43
|
+
logger.info(`Total planned runs: ${plannedRuns.length}`);
|
|
44
|
+
logger.info("");
|
|
45
|
+
// Group by SUT
|
|
46
|
+
const bySut = new Map();
|
|
47
|
+
for (const run of plannedRuns) {
|
|
48
|
+
const existing = bySut.get(run.sutId) ?? [];
|
|
49
|
+
existing.push(run);
|
|
50
|
+
bySut.set(run.sutId, existing);
|
|
51
|
+
}
|
|
52
|
+
for (const [sutId, runs] of bySut) {
|
|
53
|
+
const sutDef = sutDefinitions.find((s) => s.id === sutId);
|
|
54
|
+
logger.subheader(sutDef?.id ?? sutId);
|
|
55
|
+
// Group by case class
|
|
56
|
+
const byCaseClass = new Map();
|
|
57
|
+
for (const run of runs) {
|
|
58
|
+
const caseDef = caseDefinitions.find((c) => c.case.caseId === run.caseId);
|
|
59
|
+
const caseClass = caseDef?.case.caseClass ?? "uncategorized";
|
|
60
|
+
const existing = byCaseClass.get(caseClass) ?? [];
|
|
61
|
+
existing.push(run);
|
|
62
|
+
byCaseClass.set(caseClass, existing);
|
|
63
|
+
}
|
|
64
|
+
for (const [caseClass, classRuns] of byCaseClass) {
|
|
65
|
+
logger.info(` ${caseClass}: ${classRuns.length} runs`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
logger.info("");
|
|
69
|
+
logger.info("Execution plan validated successfully!");
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
if (error instanceof Error) {
|
|
73
|
+
logger.error(error.message);
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
logger.error(String(error));
|
|
77
|
+
}
|
|
78
|
+
processExit(1);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Register the plan command.
|
|
83
|
+
*
|
|
84
|
+
* @param program - Commander program instance
|
|
85
|
+
*/
|
|
86
|
+
export function registerPlanCommand(program) {
|
|
87
|
+
program
|
|
88
|
+
.command("plan")
|
|
89
|
+
.description("Show execution plan without running experiments")
|
|
90
|
+
.argument("<config-file>", "Path to experiment configuration JSON file")
|
|
91
|
+
.action(async (configFile) => {
|
|
92
|
+
const { createLogger } = await import("../logger.js");
|
|
93
|
+
const { loadMetricsExtractor } = await import("../module-loader.js");
|
|
94
|
+
const logger = createLogger();
|
|
95
|
+
const executor = new Executor({});
|
|
96
|
+
await executePlan(configFile, {
|
|
97
|
+
logger,
|
|
98
|
+
configLoader: { loadAndValidateConfig },
|
|
99
|
+
moduleLoader: {
|
|
100
|
+
loadSutFactory,
|
|
101
|
+
loadCaseDefinition,
|
|
102
|
+
loadMetricsExtractor,
|
|
103
|
+
},
|
|
104
|
+
executor: executor,
|
|
105
|
+
processExit: (code) => process.exit(code),
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=plan.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"plan.js","sourceRoot":"","sources":["../../../src/cli/commands/plan.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAQnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAEzE,mGAAmG;AAEnG;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAChC,UAAkB,EAClB,YAgBC;IAED,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,YAAY,CAAC;IAEnF,IAAI,CAAC;QACJ,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAEhC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,qBAAqB,CAAC,UAAU,CAAC,CAAC;QACpE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC;QAEnC,YAAY;QACZ,MAAM,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QACpC,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAC7B,YAAY,CAAC,cAAc,CAC1B,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,UAAU,EACpB,OAAO,EACP;YACC,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,YAAY,CAAC,IAAI;YACjC,OAAO,EAAE,SAAS,CAAC,YAAY,CAAC,OAAO;YACvC,IAAI,EAAE,SAAS,CAAC,YAAY,CAAC,IAAI;YACjC,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,EAAE;YAC9B,IAAI,EAAE,SAAS,CAAC,YAAY,CAAC,IAAI,IAAI,EAAE;YACvC,WAAW,EAAE,SAAS,CAAC,YAAY,CAAC,WAAW;SAC/C,EACD,SAAS,CAAC,MAAM,CAChB,CACD,CACD,CAAC;QAEF,aAAa;QACb,MAAM,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QACrC,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CACxC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAC/B,YAAY,CAAC,kBAAkB,CAAC,UAAU,CAAC,MAAM,EAAE,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,CAClF,CACD,CAAC;QAEF,YAAY;QACZ,MAAM,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;QAEnE,yBAAyB;QACzB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,uBAAuB,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEhB,eAAe;QACf,MAAM,KAAK,GAAG,IAAI,GAAG,EAA8B,CAAC;QACpD,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChC,CAAC;QAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,CAAC;YAC1D,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,KAAK,CAAC,CAAC;YAEtC,sBAAsB;YACtB,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;YACnD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC1E,MAAM,SAAS,GAAG,OAAO,EAAE,IAAI,CAAC,SAAS,IAAI,eAAe,CAAC;gBAC7D,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBAClD,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACtC,CAAC;YAED,KAAK,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,WAAW,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,KAAK,SAAS,CAAC,MAAM,OAAO,CAAC,CAAC;YACzD,CAAC;QACF,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC5B,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;aAAM,CAAC;YACP,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,CAAC;QACD,WAAW,CAAC,CAAC,CAAC,CAAC;IAChB,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAgB;IACnD,OAAO;SACL,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,iDAAiD,CAAC;SAC9D,QAAQ,CAAC,eAAe,EAAE,4CAA4C,CAAC;SACvE,MAAM,CAAC,KAAK,EAAE,UAAkB,EAAE,EAAE;QACpC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;QACtD,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAErE,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAC;QAElC,MAAM,WAAW,CAAC,UAAU,EAAE;YAC7B,MAAM;YACN,YAAY,EAAE,EAAE,qBAAqB,EAAE;YACvC,YAAY,EAAE;gBACb,cAAc;gBACd,kBAAkB;gBAClB,oBAAoB;aACX;YACV,QAAQ,EAAE,QAAiB;YAC3B,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;SACjD,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run Command
|
|
3
|
+
*
|
|
4
|
+
* Executes experiments based on JSON configuration.
|
|
5
|
+
*/
|
|
6
|
+
import type { Command } from "commander";
|
|
7
|
+
import type { IAggregator, ICommandLogger, IConfigLoader, IExecutor, IModuleLoader, IOutputWriter } from "../command-deps.js";
|
|
8
|
+
import type { CliOptions } from "../types.js";
|
|
9
|
+
/**
|
|
10
|
+
* Execute run command with injected dependencies.
|
|
11
|
+
*
|
|
12
|
+
* This function is exported for testing.
|
|
13
|
+
*
|
|
14
|
+
* @param configFile - Path to config file
|
|
15
|
+
* @param options - Command options
|
|
16
|
+
* @param dependencies - Injected dependencies
|
|
17
|
+
*/
|
|
18
|
+
export declare function executeRun(configFile: string, options: CliOptions, dependencies: {
|
|
19
|
+
logger: ICommandLogger;
|
|
20
|
+
configLoader: IConfigLoader;
|
|
21
|
+
moduleLoader: IModuleLoader;
|
|
22
|
+
createExecutor: (config: unknown) => IExecutor;
|
|
23
|
+
aggregator: IAggregator;
|
|
24
|
+
outputWriter: IOutputWriter;
|
|
25
|
+
processExit: (code: number) => never;
|
|
26
|
+
}): Promise<void>;
|
|
27
|
+
/**
|
|
28
|
+
* Register the run command.
|
|
29
|
+
*
|
|
30
|
+
* @param program - Commander program instance
|
|
31
|
+
*/
|
|
32
|
+
export declare function registerRunCommand(program: Command): void;
|
|
33
|
+
//# sourceMappingURL=run.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/run.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMzC,OAAO,KAAK,EACX,WAAW,EACX,cAAc,EACd,aAAa,EACb,SAAS,EACT,aAAa,EACb,aAAa,EACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAyF9C;;;;;;;;GAQG;AACH,wBAAsB,UAAU,CAC/B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,UAAU,EACnB,YAAY,EAAE;IACb,MAAM,EAAE,cAAc,CAAC;IACvB,YAAY,EAAE,aAAa,CAAC;IAC5B,YAAY,EAAE,aAAa,CAAC;IAC5B,cAAc,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,SAAS,CAAC;IAC/C,UAAU,EAAE,WAAW,CAAC;IACxB,YAAY,EAAE,aAAa,CAAC;IAC5B,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC;CACrC,GACC,OAAO,CAAC,IAAI,CAAC,CAkNf;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAkCzD"}
|